Skip to content
Snippets Groups Projects
Commit 3e276e98 authored by Marek Maziarz's avatar Marek Maziarz
Browse files

formatting2

parent 55ac972e
No related merge requests found
Pipeline #19510 failed with stages
in 27 seconds
......@@ -33,12 +33,10 @@ class WinerWorker(nlp_ws.NLPWorker):
self._model._tokenizer.model_max_length = 512
@staticmethod
def pack_sentences_to_max_tokens(
plain_inputs: list,
tokenized_inputs: list,
sent_starts: list,
max_tokens: int = 510
):
def pack_sentences_to_max_tokens(plain_inputs: list,
tokenized_inputs: list,
sent_starts: list,
max_tokens: int = 510):
"""Pack sentences without exceeding a maximum token count.
This method takes in plain text sentences, their tokenized
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment