Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -745,8 +745,7 @@ public BestPathNode()

private void Encode(ReadOnlySpan<byte> normalized, Span<BestPathNode> bestPathEndsAt)
{
Debug.Assert(normalized.Length > 0);

// normalized can be empty when normalization removes every character, e.g. whitespace only input.
int size = normalized.Length;
float unkScore = _minScore - UnkPenalty;

Expand Down
20 changes: 9 additions & 11 deletions src/Microsoft.ML.Tokenizers/Normalizer/SentencePieceNormalizer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -317,8 +317,8 @@ private void InsertDummyPrefixAtEnd(Span<char> span, ref int bufferIndex)
}

// Returns the longest consumed prefix of |input| that can be normalized.
// if we return normalizedPrefix == default, means no normalization and the original input span should be used.
private int NormalizePrefix(ReadOnlySpan<byte> input, out Memory<byte> normalizedPrefix)
// normalizedPrefix is what should be written for that prefix. It can be empty when the charsmap deletes the prefix.
private int NormalizePrefix(ReadOnlySpan<byte> input, out ReadOnlySpan<byte> normalizedPrefix)
{
Debug.Assert(!input.IsEmpty);

Expand Down Expand Up @@ -354,12 +354,12 @@ private int NormalizePrefix(ReadOnlySpan<byte> input, out Memory<byte> normalize
// Found a malformed utf8.
// The rune is set to be 0xFFFD (REPLACEMENT CHARACTER), which is a valid Unicode of three bytes in utf8, but here we only consume one byte.
result = 1;
normalizedPrefix = new Memory<byte>(_replacementBytes, 0, 3);
normalizedPrefix = _replacementBytes.AsSpan(0, 3);
}
else
{
result = length;
normalizedPrefix = default;
normalizedPrefix = input.Slice(0, length);
}
}
else
Expand All @@ -374,7 +374,7 @@ private int NormalizePrefix(ReadOnlySpan<byte> input, out Memory<byte> normalize
{
normalizedLength++;
}
normalizedPrefix = new Memory<byte>(_normalized, longestValue, normalizedLength - longestValue);
normalizedPrefix = _normalized.AsSpan(longestValue, normalizedLength - longestValue);
}

return result;
Expand Down Expand Up @@ -427,17 +427,16 @@ private int NormalizeCore(ReadOnlySpan<byte> input, ref Span<byte> normalized, r
{
while (!input.IsEmpty)
{
int p = NormalizePrefix(input, out Memory<byte> normalizedPrefix);
int p = NormalizePrefix(input, out ReadOnlySpan<byte> normalizedPrefix);

Debug.Assert(p > 0);

if (p != 1)
if (p != 1 || normalizedPrefix.IsEmpty)
{
break;
}

ReadOnlySpan<byte> normalizedByte = normalizedPrefix.Length == 0 ? input.Slice(0, p) : normalizedPrefix.Span;
if (normalizedByte[0] != (byte)' ')
if (normalizedPrefix[0] != (byte)' ')
{
break;
}
Expand Down Expand Up @@ -466,8 +465,7 @@ private int NormalizeCore(ReadOnlySpan<byte> input, ref Span<byte> normalized, r

while (!input.IsEmpty)
{
int p = NormalizePrefix(input, out Memory<byte> normalizedPrefix);
ReadOnlySpan<byte> sp = normalizedPrefix.Length == 0 ? input.Slice(0, p) : normalizedPrefix.Span;
int p = NormalizePrefix(input, out ReadOnlySpan<byte> sp);

// Removes heading spaces in sentence piece, if the previous sentence piece ends with whitespace.
while (isPrevSpace && sp.Length > 0 && sp[0] == (byte)' ')
Expand Down
68 changes: 63 additions & 5 deletions test/Microsoft.ML.Tokenizers.Tests/UnigramTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,22 @@ public static IEnumerable<object[]> UnigramTestData()

yield return new object[]
{
"\u001f", // string start with control character
"▁\u001f",
"\u001f", // control character that the charsmap deletes
"",
new int[] { 5, 0 },
new string[] { "▁", "\u001f"},
new Range[] { new Range(0, 1), new Range(1, 2) }
"",
new int[0],
new string[0],
new Range[0]
};

yield return new object[]
{
" ", // whitespace only
"",
"",
new int[0],
new string[0],
new Range[0]
};

yield return new object[]
Expand Down Expand Up @@ -666,6 +676,54 @@ public void CreateFromTokenizerJsonNormalizationTest()
}
}

public static IEnumerable<object[]> CharsMapDeletedCodePoints()
{
// nmt_nfkc maps these code points to an empty string, so SentencePiece and Hugging Face delete them.
int[] codePoints = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14,
0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x7F, 0x8F, 0x9F];
foreach (int codePoint in codePoints)
{
yield return new object[] { codePoint, false };
yield return new object[] { codePoint, true };
}
}

[Theory]
[MemberData(nameof(CharsMapDeletedCodePoints))]
public void CharsMapEmptyReplacementDeletesInputTest(int codePoint, bool fromTokenizerJson)
{
SentencePieceTokenizer tokenizer = fromTokenizerJson ? CreateFromParaphraseTokenizerJson() : _unigramTokenizer;
string c = ((char)codePoint).ToString();

Assert.Equal(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, $"a{c}b"));
Assert.Equal(EncodeIds(tokenizer, "hello world"), EncodeIds(tokenizer, $"hello{c} world"));
Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"{c}hello"));
Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"{c} hello"));
Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"hello{c}"));
Assert.Equal(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, $"a{c}{c}b"));
Assert.DoesNotContain(tokenizer.UnknownId, EncodeIds(tokenizer, $"a{c}b{c}{c}c"));
}

[Fact]
public void CharsMapNonEmptyReplacementStillAppliedTest()
{
foreach (SentencePieceTokenizer tokenizer in new[] { _unigramTokenizer, CreateFromParaphraseTokenizerJson() })
{
// Tab is mapped to a space by the charsmap, not deleted.
Assert.Equal(EncodeIds(tokenizer, "a b"), EncodeIds(tokenizer, "a\tb"));
Assert.NotEqual(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, "a\tb"));
}
}

private static SentencePieceTokenizer CreateFromParaphraseTokenizerJson()
{
using Stream jsonStream = File.OpenRead(Path.Combine("Paraphrase-multilingual-MiniLM-L12-v2", "tokenizer.json"));
return SentencePieceTokenizer.CreateFromTokenizerJson(jsonStream, addBeginningOfSentence: false, addEndOfSentence: false);
}

private static IReadOnlyList<int> EncodeIds(SentencePieceTokenizer tokenizer, string text) =>
tokenizer.EncodeToIds(text, addBeginningOfSentence: false, addEndOfSentence: false);

[Fact]
public void CreateFromVocabNoSpecialTokensTest()
{
Expand Down
Loading