Fix edge case that input str is removed by hugging face tokenizers fix …

…#1651 (comment)
hankcs · May 21, 2021 · f1c1c71 · f1c1c71
1 parent 40e59ed
commit f1c1c71
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 6 deletions.
diff --git a/hanlp/transform/transformer_tokenizer.py b/hanlp/transform/transformer_tokenizer.py
@@ -195,7 +195,8 @@ def __init__(self,
             cls_token_at_end = xlnet
             pad_on_left = xlnet
         if isinstance(tokenizer, str):
-            tokenizer = AutoTokenizer_.from_pretrained(tokenizer, use_fast=use_fast, do_basic_tokenize=do_basic_tokenize)
+            tokenizer = AutoTokenizer_.from_pretrained(tokenizer, use_fast=use_fast,
+                                                       do_basic_tokenize=do_basic_tokenize)
         if use_fast:
             # Dirty fix upstream bug: https://github.com/hankcs/HanLP/issues/1602
             if hasattr(tokenizer, '_tokenizer') and hasattr(tokenizer._tokenizer, 'no_truncation'):
@@ -277,6 +278,13 @@ def tokenize_str(input_str, add_special_tokens=True):
                     if add_special_tokens:
                         subtoken_offsets = subtoken_offsets[1 if self.has_cls else 0:-1]
 
+                    # Edge case that the input_str is swallowed in whole
+                    if not subtoken_offsets and not input_str.isspace():
+                        __index = 1 if add_special_tokens and self.has_cls else 0
+                        input_tokens.insert(__index, input_str)
+                        input_ids.insert(__index, tokenizer.unk_token_id)
+                        subtoken_offsets.append((0, len(input_str)))
+
                     if not self.has_cls:
                         input_tokens = [self.cls_token] + input_tokens
                         input_ids = [self.cls_token_id] + input_ids

diff --git a/hanlp/version.py b/hanlp/version.py
@@ -2,5 +2,5 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-alpha.43'
+__version__ = '2.1.0-alpha.44'
 """HanLP version"""
diff --git a/tests/test_mtl.py b/tests/test_mtl.py
@@ -11,10 +11,6 @@ def tokenize(mtl, text):
 
 
 class TestMultiTaskLearning(unittest.TestCase):
-
-    def setUp(self) -> None:
-        super().setUp()
-
     def test_mtl_single_sent(self):
         doc: Document = mtl('商品和服务')
         self.assertSequenceEqual(doc['tok/fine'], ["商品", "和", "服务"])
@@ -47,6 +43,9 @@ def test_emoji(self):
         self.assertSequenceEqual(mtl('( ͡° ͜ʖ ͡ °)你好', tasks='tok/fine')['tok/fine'],
                                  ["( ͡° ͜ʖ ͡ °)", "你", "好"])
 
+    def test_unicode_removed_by_hf(self):
+        self.assertSequenceEqual(mtl('͡', tasks='tok/fine')['tok/fine'], ['͡'])
+
     def test_space(self):
         doc: Document = mtl('商品 和服务')
         self.assertSequenceEqual(doc['tok/fine'], ["商品", "和", "服务"])