Initial structure for subwords.

src/lambo/data/token.py

@@ -19,7 +19,7 @@ class Token:
         self.end = end
         self.text = text
         self.is_multi_word = is_multi_word
-        self.words = []
+        self.subwords = []
         
-    def addWord(self,word):
-        self.words.append(word)
+    def addSubword(self, subword):
+        self.subwords.append(subword)

src/lambo/utils/ud_reader.py

@@ -162,7 +162,7 @@ def read_document(file_path, random_separators):
             if len(numbers) == 1:
                 if word_range[0] <= numbers[0] <= word_range[1]:
                     # Individual word within multi-word token
-                    lastToken.addWord(form)
+                    lastToken.addSubword(form)
                 else:
                     # Individual word not covered
                     token = Token(current_offset, current_offset + len(form), form, False)