From d7388391d9a4e308bff2cc7ddc4ba7a9bc4c2e7f Mon Sep 17 00:00:00 2001
From: Maja Jablonska <majajjablonska@gmail.com>
Date: Thu, 16 Nov 2023 20:15:19 +1100
Subject: [PATCH] Change default tokenizer to LAMBO
---
combo/predict.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/combo/predict.py b/combo/predict.py
index 1eba4b4..6743d8f 100644
--- a/combo/predict.py
+++ b/combo/predict.py
@@ -30,7 +30,7 @@ class COMBO(PredictorModule):
def __init__(self,
model: Model,
dataset_reader: DatasetReader,
- tokenizer: data.Tokenizer = tokenizers.WhitespaceTokenizer(),
+ tokenizer: data.Tokenizer = tokenizers.LamboTokenizer(),
batch_size: int = 1024,
line_to_conllu: bool = True) -> None:
super().__init__(model, dataset_reader)
--
GitLab