Skip to content
Snippets Groups Projects
Commit 11e898eb authored by Maja Jablonska's avatar Maja Jablonska
Browse files

Fix sentence IDs in turns

parent b7d60405
Branches
Tags
No related merge requests found
Pipeline #16841 passed with stage
in 24 seconds
......@@ -71,9 +71,9 @@ class LamboTokenizer(Tokenizer):
if split_level.upper() == "TURN":
for turn in document.turns:
_reset_idx()
sentence_tokens = []
for sentence in turn.sentences:
_reset_idx()
for token in sentence.tokens:
sentence_tokens.extend(_sentence_tokens(token, split_multiwords))
tokens.append(sentence_tokens)
......@@ -96,8 +96,8 @@ class LamboTokenizer(Tokenizer):
tokens.append(sentence_tokens)
else:
for turn in document.turns:
_reset_idx()
for sentence in turn.sentences:
_reset_idx()
for token in sentence.tokens:
tokens.extend(_sentence_tokens(token, split_multiwords))
tokens = [tokens]
......
......@@ -3,7 +3,7 @@ requires = ["setuptools"]
[project]
name = "combo"
version = "3.2.1"
version = "3.2.2"
authors = [
{name = "Maja Jablonska", email = "maja.jablonska@ipipan.waw.pl"}
]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment