diff --git a/combo/main.py b/combo/main.py index 49a0fc74e10e8693d70871e7a6ee6e2cb409803f..b8130746a4a4829643e9d4a70c70d133ddb6ede6 100755 --- a/combo/main.py +++ b/combo/main.py @@ -85,8 +85,8 @@ flags.DEFINE_string(name="config_path", default="", help="Config file path.") flags.DEFINE_list(name="datasets_for_vocabulary", default=["train"], help="") -flags.DEFINE_boolean(name="turns", default=False, - help="Segment into sentences on sentence break or on turn break.") +flags.DEFINE_enum(name="split_level", default="sentence", enum_values=["none", "turn", "sentence"], + help="Don\'t segment, or segment into sentences on sentence break or on turn break.") flags.DEFINE_boolean(name="split_multiwords", default=False, help="Split subwords (e.g. don\'t = do, n\'t) into separate tokens.") flags.DEFINE_boolean(name="transformer_encoder", default=False, help="Use transformer encoder.") @@ -424,12 +424,13 @@ def run(_): prefix=prefix) dataset_reader = default_ud_dataset_reader(FLAGS.pretrained_transformer_name, tokenizer=LamboTokenizer(tokenizer_language, - default_split_level="TURNS" if FLAGS.turns else "SENTENCES", + default_split_level=FLAGS.split_level, default_split_multiwords=FLAGS.split_multiwords) ) predictor = COMBO(model, dataset_reader) + if FLAGS.input_file == '-': print("Interactive mode.") sentence = input("Sentence: ") @@ -465,9 +466,9 @@ def run(_): else: tokenizer = LamboTokenizer(tokenizer_language) with open(FLAGS.input_file, "r", encoding='utf-8') as file: - input_sentences = tokenizer.segment(file.read(), - turns=FLAGS.turns, - split_multiwords=FLAGS.split_multiwords) + input_sentences = tokenizer.tokenize(file.read(), + split_level=FLAGS.split_level.upper(), + split_multiwords=FLAGS.split_multiwords) predictions = predictor.predict(input_sentences) with open(FLAGS.output_file, "w") as file: for prediction in tqdm(predictions): diff --git a/pyproject.toml b/pyproject.toml index 457b9d802cb12098ccbf53748b62cf8e8c9fe660..da51df8c03270e10b7287719b9248b746e57e61a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] [project] name = "combo" -version = "3.1.5" +version = "3.2.0" authors = [ {name = "Maja Jablonska", email = "maja.jablonska@ipipan.waw.pl"} ]