fixed proper division into multiwords

combo/data/tokenizers/lambo_tokenizer.py

@@ -84,6 +84,15 @@ class LamboTokenizer(Tokenizer):
                     _reset_idx()
                     sentence_tokens = []
                     for token in sentence.tokens:
+                        if len(token.subwords) > 0 and split_subwords:
+                            # @TODO this is a very dirty fix for Lambo model's shortcomings
+                            # I noticed that for longer words with multiwords it tends to remove the last letter in the last multiword
+                            # so this is a quick workaround to fix it
+
+                            # check if subwords in token.subwords are consistent with token.text
+                            if "".join(token.subwords) != token.text:
+                                fixed_subwords = fix_subwords(token)
+                                token.subwords = fixed_subwords
                         sentence_tokens.extend(_sentence_tokens(token, split_subwords))
                     tokens.append(sentence_tokens)
         else:
@@ -130,6 +139,20 @@ class LamboTokenizer(Tokenizer):
 
                         # check if subwords in token.subwords are consistent with token.text
                         if "".join(token.subwords) != token.text:
+                            fixed_subwords = fix_subwords(token)
+                            token.subwords = fixed_subwords
+                        # sentence_tokens.extend(_sentence_tokens(token, split_subwords))
+                    # else:
+                    sentence_tokens.extend(_sentence_tokens(token, split_subwords))
+                if not turns:
+                    sentences.append(sentence_tokens)
+            if turns:
+                sentences.append(sentence_tokens)
+
+        return sentences
+
+
+def fix_subwords(token: Token):
     fixed_subwords = []
     text_it = 0
     for i, subword in enumerate(token.subwords):
@@ -141,13 +164,4 @@ class LamboTokenizer(Tokenizer):
         else:
             fixed_subwords.append(token.text[text_it:text_it + len(subword)])
             text_it += len(subword)
-                            token.subwords = fixed_subwords
-                        # sentence_tokens.extend(_sentence_tokens(token, split_subwords))
+    return fixed_subwords
\ No newline at end of file
-                    # else:
-                    sentence_tokens.extend(_sentence_tokens(token, split_subwords))
-                if not turns:
-                    sentences.append(sentence_tokens)
-            if turns:
-                sentences.append(sentence_tokens)
-
-        return sentences