Trim to long tokens.

poldeepner2/utils/sequences.py

@@ -112,7 +112,8 @@ class FeatureGenerator:
             labels = ["O"] * len(tokens)
         for word, label_1 in zip(tokens, labels):
             subtokens = self.encode_method(word.strip())
-            # Temporal hack to shorten token to max 14 subtokens
+            if len(subtokens) > 6:
+                logging.warning(f"Token {word} was truncated to 6 subtokens: {subtokens}")
                 subtokens = subtokens[:6]
             if len(subtokens) == 0:
                 replacement = "x" * len(word.strip())