Add multiword support to lambo_tokenizer.py

combo/data/tokenizers/lambo_tokenizer.py

@@ -43,9 +43,16 @@ class LamboTokenizer(Tokenizer):
 
         document = self.__tokenizer.segment(text)
         sentences = []
+        sentence_tokens = []
 
         for turn in document.turns:
             for sentence in turn.sentences:
-                sentences.append([t.text for t in sentence.tokens])
+                sentence_tokens = []
+                for token in sentence.tokens:
+                    if len(token.subwords) > 0:
+                        sentence_tokens.extend([s for s in token.subwords])
+                    else:
+                        sentence_tokens.append(token.text)
+                sentences.append(sentence_tokens)
 
         return sentences