Reduced window size.

src/lambo/evaluation/evaluate.py

-from lambo.evaluation.conll18_ud_eval import load_conllu, evaluate
+from lambo.evaluation.conll18_ud_eval import load_conllu, evaluate, UDError
 from lambo.utils.printer import print_document_to_conll
 
 
@@ -19,8 +19,13 @@ def evaluate_segmenter(segmenter, test_text, gold_path, tmp_path):
         with open(gold_path) as fGold:
             pred = load_conllu(fPred)
             gold = load_conllu(fGold)
-            conll_result = evaluate(gold, pred)
-            for category in ['Tokens', 'Words', 'Sentences']:
-                result[category] = {'F1': conll_result[category].f1, 'precision': conll_result[category].precision,
-                                    'recall': conll_result[category].recall}
+            try:
+                conll_result = evaluate(gold, pred)
+                for category in ['Tokens', 'Words', 'Sentences']:
+                    result[category] = {'F1': conll_result[category].f1, 'precision': conll_result[category].precision,
+                                        'recall': conll_result[category].recall}
+            except UDError as e:
+                for category in ['Tokens', 'Words', 'Sentences']:
+                    result[category] = {'F1': 0.0, 'precision': 0.0,
+                                        'recall': 0.0}
     return result

src/lambo/learning/train.py

@@ -121,7 +121,7 @@ def train_new_and_save(model_name, treebank_path, save_path, epochs=10, device='
     BATCH_SIZE = 32
     print("Initiating the model.")
     
-    MAX_LEN = 1024
+    MAX_LEN = 256
     dict, train_dataloader, test_dataloader = prepare_dataloaders_withdict([train_doc, dev_doc], [test_doc],
                                                                            MAX_LEN,
                                                                            BATCH_SIZE)
@@ -168,7 +168,7 @@ def train_pretrained_and_save(language, treebank_path, save_path, pretrained_pat
     train_doc, dev_doc, test_doc = read_treebank(treebank_path, True)
     
     print("Initiating the model.")
-    MAX_LEN = 1024
+    MAX_LEN = 256
     model = LamboNetwork(MAX_LEN, dict, len(utf_category_dictionary), pretrained=pretrained_model)
     
     print("Preparing data")