From d621b119b8757f0106fcafee7f604e5bcd7844e8 Mon Sep 17 00:00:00 2001 From: Lior Shmueli <48066416+ShmueLior@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:22:13 +0200 Subject: [PATCH] Apply SentencePiece charsmap entries that map to an empty string NormalizePrefix used an empty Memory to mean "no rule matched", so a rule whose replacement is empty (nmt_nfkc deletes U+0001-U+0008, U+000B, U+000E-U+001F, U+007F, U+008F and U+009F) fell back to the original bytes and the character was encoded as . Return the bytes to write directly so an empty replacement deletes the input, like SentencePiece and Hugging Face do. Input that normalizes to nothing (whitespace only, or only deleted characters) no longer trips the Debug.Assert in the Unigram Encode. Fixes #7908 --- .../Model/SentencePieceUnigramModel.cs | 3 +- .../Normalizer/SentencePieceNormalizer.cs | 20 +++--- .../UnigramTests.cs | 68 +++++++++++++++++-- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.ML.Tokenizers/Model/SentencePieceUnigramModel.cs b/src/Microsoft.ML.Tokenizers/Model/SentencePieceUnigramModel.cs index 0057005f4d..d442be91a2 100644 --- a/src/Microsoft.ML.Tokenizers/Model/SentencePieceUnigramModel.cs +++ b/src/Microsoft.ML.Tokenizers/Model/SentencePieceUnigramModel.cs @@ -745,8 +745,7 @@ public BestPathNode() private void Encode(ReadOnlySpan normalized, Span bestPathEndsAt) { - Debug.Assert(normalized.Length > 0); - + // normalized can be empty when normalization removes every character, e.g. whitespace only input. int size = normalized.Length; float unkScore = _minScore - UnkPenalty; diff --git a/src/Microsoft.ML.Tokenizers/Normalizer/SentencePieceNormalizer.cs b/src/Microsoft.ML.Tokenizers/Normalizer/SentencePieceNormalizer.cs index c1b9c210d5..3cbb4d3183 100644 --- a/src/Microsoft.ML.Tokenizers/Normalizer/SentencePieceNormalizer.cs +++ b/src/Microsoft.ML.Tokenizers/Normalizer/SentencePieceNormalizer.cs @@ -317,8 +317,8 @@ private void InsertDummyPrefixAtEnd(Span span, ref int bufferIndex) } // Returns the longest consumed prefix of |input| that can be normalized. - // if we return normalizedPrefix == default, means no normalization and the original input span should be used. - private int NormalizePrefix(ReadOnlySpan input, out Memory normalizedPrefix) + // normalizedPrefix is what should be written for that prefix. It can be empty when the charsmap deletes the prefix. + private int NormalizePrefix(ReadOnlySpan input, out ReadOnlySpan normalizedPrefix) { Debug.Assert(!input.IsEmpty); @@ -354,12 +354,12 @@ private int NormalizePrefix(ReadOnlySpan input, out Memory normalize // Found a malformed utf8. // The rune is set to be 0xFFFD (REPLACEMENT CHARACTER), which is a valid Unicode of three bytes in utf8, but here we only consume one byte. result = 1; - normalizedPrefix = new Memory(_replacementBytes, 0, 3); + normalizedPrefix = _replacementBytes.AsSpan(0, 3); } else { result = length; - normalizedPrefix = default; + normalizedPrefix = input.Slice(0, length); } } else @@ -374,7 +374,7 @@ private int NormalizePrefix(ReadOnlySpan input, out Memory normalize { normalizedLength++; } - normalizedPrefix = new Memory(_normalized, longestValue, normalizedLength - longestValue); + normalizedPrefix = _normalized.AsSpan(longestValue, normalizedLength - longestValue); } return result; @@ -427,17 +427,16 @@ private int NormalizeCore(ReadOnlySpan input, ref Span normalized, r { while (!input.IsEmpty) { - int p = NormalizePrefix(input, out Memory normalizedPrefix); + int p = NormalizePrefix(input, out ReadOnlySpan normalizedPrefix); Debug.Assert(p > 0); - if (p != 1) + if (p != 1 || normalizedPrefix.IsEmpty) { break; } - ReadOnlySpan normalizedByte = normalizedPrefix.Length == 0 ? input.Slice(0, p) : normalizedPrefix.Span; - if (normalizedByte[0] != (byte)' ') + if (normalizedPrefix[0] != (byte)' ') { break; } @@ -466,8 +465,7 @@ private int NormalizeCore(ReadOnlySpan input, ref Span normalized, r while (!input.IsEmpty) { - int p = NormalizePrefix(input, out Memory normalizedPrefix); - ReadOnlySpan sp = normalizedPrefix.Length == 0 ? input.Slice(0, p) : normalizedPrefix.Span; + int p = NormalizePrefix(input, out ReadOnlySpan sp); // Removes heading spaces in sentence piece, if the previous sentence piece ends with whitespace. while (isPrevSpace && sp.Length > 0 && sp[0] == (byte)' ') diff --git a/test/Microsoft.ML.Tokenizers.Tests/UnigramTests.cs b/test/Microsoft.ML.Tokenizers.Tests/UnigramTests.cs index e8315aee06..607dfcdc11 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/UnigramTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/UnigramTests.cs @@ -107,12 +107,22 @@ public static IEnumerable UnigramTestData() yield return new object[] { - "\u001f", // string start with control character - "▁\u001f", + "\u001f", // control character that the charsmap deletes "", - new int[] { 5, 0 }, - new string[] { "▁", "\u001f"}, - new Range[] { new Range(0, 1), new Range(1, 2) } + "", + new int[0], + new string[0], + new Range[0] + }; + + yield return new object[] + { + " ", // whitespace only + "", + "", + new int[0], + new string[0], + new Range[0] }; yield return new object[] @@ -666,6 +676,54 @@ public void CreateFromTokenizerJsonNormalizationTest() } } + public static IEnumerable CharsMapDeletedCodePoints() + { + // nmt_nfkc maps these code points to an empty string, so SentencePiece and Hugging Face delete them. + int[] codePoints = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x7F, 0x8F, 0x9F]; + foreach (int codePoint in codePoints) + { + yield return new object[] { codePoint, false }; + yield return new object[] { codePoint, true }; + } + } + + [Theory] + [MemberData(nameof(CharsMapDeletedCodePoints))] + public void CharsMapEmptyReplacementDeletesInputTest(int codePoint, bool fromTokenizerJson) + { + SentencePieceTokenizer tokenizer = fromTokenizerJson ? CreateFromParaphraseTokenizerJson() : _unigramTokenizer; + string c = ((char)codePoint).ToString(); + + Assert.Equal(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, $"a{c}b")); + Assert.Equal(EncodeIds(tokenizer, "hello world"), EncodeIds(tokenizer, $"hello{c} world")); + Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"{c}hello")); + Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"{c} hello")); + Assert.Equal(EncodeIds(tokenizer, "hello"), EncodeIds(tokenizer, $"hello{c}")); + Assert.Equal(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, $"a{c}{c}b")); + Assert.DoesNotContain(tokenizer.UnknownId, EncodeIds(tokenizer, $"a{c}b{c}{c}c")); + } + + [Fact] + public void CharsMapNonEmptyReplacementStillAppliedTest() + { + foreach (SentencePieceTokenizer tokenizer in new[] { _unigramTokenizer, CreateFromParaphraseTokenizerJson() }) + { + // Tab is mapped to a space by the charsmap, not deleted. + Assert.Equal(EncodeIds(tokenizer, "a b"), EncodeIds(tokenizer, "a\tb")); + Assert.NotEqual(EncodeIds(tokenizer, "ab"), EncodeIds(tokenizer, "a\tb")); + } + } + + private static SentencePieceTokenizer CreateFromParaphraseTokenizerJson() + { + using Stream jsonStream = File.OpenRead(Path.Combine("Paraphrase-multilingual-MiniLM-L12-v2", "tokenizer.json")); + return SentencePieceTokenizer.CreateFromTokenizerJson(jsonStream, addBeginningOfSentence: false, addEndOfSentence: false); + } + + private static IReadOnlyList EncodeIds(SentencePieceTokenizer tokenizer, string text) => + tokenizer.EncodeToIds(text, addBeginningOfSentence: false, addEndOfSentence: false); + [Fact] public void CreateFromVocabNoSpecialTokensTest() {