fix: issue with tokens that match prefixes (#106)

adbar · Jun 15, 2023 · 8c06597 · 8c06597
1 parent fdfff3d
commit 8c06597
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/simplemma/strategies/prefix_decomposition.py b/simplemma/strategies/prefix_decomposition.py
@@ -85,6 +85,9 @@ def get_lemma(self, token: str, lang: str) -> Optional[str]:
             return None
         prefix = prefix_match[1]
 
+        if prefix == token:
+            return None
+
         subword = self._dictionary_lookup.get_lemma(token[len(prefix) :], lang)
         if subword is None:
             return None

diff --git a/tests/strategies/test_strategies.py b/tests/strategies/test_strategies.py
@@ -4,6 +4,7 @@
     DefaultStrategy,
     GreedyDictionaryLookupStrategy,
     AffixDecompositionStrategy,
+    PrefixDecompositionStrategy,
 )
 
 
@@ -51,3 +52,5 @@ def test_search() -> None:
     )
 
     assert AffixDecompositionStrategy(greedy=True).get_lemma("ccc", "de") is None
+
+    assert PrefixDecompositionStrategy().get_lemma("auf", "de") is None