diff --git a/combo/data/tokenizers/lambo_tokenizer.py b/combo/data/tokenizers/lambo_tokenizer.py index 635ab5d7961e478fd48c1684d18026159ec9e016..fafec8e4433dcbd7dc8cbf8091b4746c1e86e242 100644 --- a/combo/data/tokenizers/lambo_tokenizer.py +++ b/combo/data/tokenizers/lambo_tokenizer.py @@ -71,9 +71,9 @@ class LamboTokenizer(Tokenizer): if split_level.upper() == "TURN": for turn in document.turns: + _reset_idx() sentence_tokens = [] for sentence in turn.sentences: - _reset_idx() for token in sentence.tokens: sentence_tokens.extend(_sentence_tokens(token, split_multiwords)) tokens.append(sentence_tokens) @@ -96,8 +96,8 @@ class LamboTokenizer(Tokenizer): tokens.append(sentence_tokens) else: for turn in document.turns: + _reset_idx() for sentence in turn.sentences: - _reset_idx() for token in sentence.tokens: tokens.extend(_sentence_tokens(token, split_multiwords)) tokens = [tokens] diff --git a/pyproject.toml b/pyproject.toml index c17c0dfde2563b7b666e9cfe07bbaae50009ff05..5952c1b86387eadb99eda623892b00606a71e8cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] [project] name = "combo" -version = "3.2.1" +version = "3.2.2" authors = [ {name = "Maja Jablonska", email = "maja.jablonska@ipipan.waw.pl"} ]