Struct Token

Represents the data of a token in a sequence.

Implements

IEquatable<Token>

Inherited Members

object.Equals(object, object)

object.GetType()

object.ReferenceEquals(object, object)

Namespace: Unity.InferenceEngine.Tokenization

Assembly: Unity.InferenceEngine.Tokenization.dll

Syntax

public readonly struct Token : IEquatable<Token>

Constructors

Token(int, string, bool, int, bool, Range)

Initializes a new instance of the Token type.

Declaration

public Token(int id, string value = null, bool attention = false, int typeId = 0, bool special = false, Range offsets = default)

Parameters

Type	Name	Description
int	id	ID of the token.
string	value	Value of the token.
bool	attention	Attention of the token.
int	typeId	Identifies the subsequence this token belongs to, in a sequence of token computed from a pair of inputs.
bool	special	Whether the token is special.
Range	offsets	The portion of the original input represented by this Token.

Fields

Attention

Attention of the token.

Declaration

public readonly bool Attention

Field Value

Type	Description
bool

Id

ID of the token.

Declaration

public readonly int Id

Field Value

Type	Description
int

Offsets

The portion of the original input represented by this Token.

Declaration

public readonly Range Offsets

Field Value

Type	Description
Range

Special

Tells whether the token is special.

Declaration

public readonly bool Special

Field Value

Type	Description
bool

TypeId

Identifies the subsequence this token belongs to, in a sequence of token computed from a pair of input.

Declaration

public readonly int TypeId

Field Value

Type	Description
int

Value

Value of the token.

Declaration

public readonly string Value

Field Value

Type	Description
string

Methods

Deconstruct(out int, out string, out int, out int, out int, out Range)

Deconstructs a token.

Declaration

public void Deconstruct(out int id, out string value, out int attention, out int typeId, out int special, out Range offsets)

Parameters

Type	Name	Description
int	id	ID of the token.
string	value	Value of the token.
int	attention	Attention of the token.
int	typeId	Identifies the subsequence this token belongs to, in a sequence of token computed from a pair of inputs.
int	special	Tells whether the token is special.
Range	offsets	The portion of the original input represented by this Token.

Equals(object)

Declaration

public override bool Equals(object obj)

Parameters

Type	Name	Description
object	obj

Returns

Type	Description
bool

Overrides

ValueType.Equals(object)

Equals(Token)

Declaration

public bool Equals(Token other)

Parameters

Type	Name	Description
Token	other

Returns

Type	Description
bool

GetHashCode()

Declaration

public override int GetHashCode()

Returns

Type	Description
int

Overrides

ValueType.GetHashCode()

SetAttention(bool)

Creates a new token with a new Attention value.

Declaration

public Token SetAttention(bool attention)

Parameters

Type	Name	Description
bool	attention	The attention of the new token.

Returns

Type	Description
Token	The new token, with a attention set to attention.

SetId(int)

Creates a new token with a new Id value.

Declaration

public Token SetId(int id)

Parameters

Type	Name	Description
int	id	The Id of the new token.

Returns

Type	Description
Token	The new token, with a id set to id.

SetOffsets(Range)

Creates a new token with a new Offsets value.

Declaration

public Token SetOffsets(Range offsets)

Parameters

Type	Name	Description
Range	offsets	The range of the new token.

Returns

Type	Description
Token	The new token, with a offsets set to offsets.

SetSpecial(bool)

Creates a new token with a new Special value.

Declaration

public Token SetSpecial(bool special)

Parameters

Type	Name	Description
bool	special	The special state of the new token.

Returns

Type	Description
Token	The new token, with a special set to special.

SetTypeId(int)

Creates a new token with a new TypeId value.

Declaration

public Token SetTypeId(int typeId)

Parameters

Type	Name	Description
int	typeId	The typeId of the new token.

Returns

Type	Description
Token	The new token, with a typeid set to typeId.

SetValue(string)

Creates a new token with a new Value value.

Declaration

public Token SetValue(string value)

Parameters

Type	Name	Description
string	value	The value of the new token.

Returns

Type	Description
Token	The new token, with a value set to value.

ToString()

Declaration

public override string ToString()

Returns

Type	Description
string

Overrides

ValueType.ToString()

Implements

IEquatable<T>