From 4f3c3023f5d63a8ed81d686d0ad02a72eb20f493 Mon Sep 17 00:00:00 2001 From: Gyanu Date: Mon, 24 Aug 2026 02:57:56 +0530 Subject: [PATCH] don't split German gender-inclusive colon forms Kund:innen and similar were broken on the letter-colon-letter infix. Keep <>= splitting; drop only the colon from that class. Co-authored-by: Cursor --- spacy/lang/de/punctuation.py | 2 +- spacy/tests/lang/de/test_prefix_suffix_infix.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/spacy/lang/de/punctuation.py b/spacy/lang/de/punctuation.py index 862207649a3..5f4c0cd1326 100644 --- a/spacy/lang/de/punctuation.py +++ b/spacy/lang/de/punctuation.py @@ -41,7 +41,7 @@ + [ r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), - r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[<>=](?=[{a}])".format(a=ALPHA), r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), r"(?<=[0-9{a}])\/(?=[0-9{a}])".format(a=ALPHA), r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), diff --git a/spacy/tests/lang/de/test_prefix_suffix_infix.py b/spacy/tests/lang/de/test_prefix_suffix_infix.py index 82bd8ed69ed..43efe24b18c 100644 --- a/spacy/tests/lang/de/test_prefix_suffix_infix.py +++ b/spacy/tests/lang/de/test_prefix_suffix_infix.py @@ -106,3 +106,18 @@ def test_de_tokenizer_splits_double_hyphen_infix(de_tokenizer): assert tokens[6].text == "--" assert tokens[7].text == "sind" assert tokens[8].text == "kompliziert" + + +@pytest.mark.parametrize( + "text", ["Kund:innen", "Mitarbeiter:innen", "eine:n", "Ärzt:in"] +) +def test_de_tokenizer_keeps_gender_colon(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].text == text + + +@pytest.mark.parametrize("text", ["xb", "n=m"]) +def test_de_tokenizer_still_splits_comparison_infix(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3