diff --git a/combo/data/dataset_readers/universal_dependencies_dataset_reader.py b/combo/data/dataset_readers/universal_dependencies_dataset_reader.py index 22960a217e272c81995d4624a93f05ff0035da61..8ddfa78757d5296d6311940d5333a442b38f6d1e 100644 --- a/combo/data/dataset_readers/universal_dependencies_dataset_reader.py +++ b/combo/data/dataset_readers/universal_dependencies_dataset_reader.py @@ -130,7 +130,6 @@ class UniversalDependenciesDatasetReader(DatasetReader, ABC): for annotation in conllu.parse_incr(f, fields=self.fields, field_parsers=self.field_parsers): yield self.text_to_instance([Token.from_conllu_token(t) for t in annotation if isinstance(t.get("id"), int)]) - def text_to_instance(self, tree: List[Token]) -> Instance: fields_: Dict[str, Field] = {} @@ -193,8 +192,10 @@ class UniversalDependenciesDatasetReader(DatasetReader, ABC): # TODO: co robic gdy nie ma xpostag? if all([t is None for t in target_values]): continue - fields_[target_name] = SequenceLabelField(target_values, text_field, - label_namespace=target_name + "_labels") + else: + target_values = [t or '_' for t in target_values] + fields_[target_name] = SequenceLabelField(target_values, text_field, + label_namespace=target_name + "_labels") # Restore feats fields to string representation # parser.serialize_field doesn't handle key without value diff --git a/combo/main.py b/combo/main.py index b8130746a4a4829643e9d4a70c70d133ddb6ede6..1fefe6c53dc44e4f5a727b6acb4ca7de73cb8924 100755 --- a/combo/main.py +++ b/combo/main.py @@ -87,7 +87,7 @@ flags.DEFINE_list(name="datasets_for_vocabulary", default=["train"], help="") flags.DEFINE_enum(name="split_level", default="sentence", enum_values=["none", "turn", "sentence"], help="Don\'t segment, or segment into sentences on sentence break or on turn break.") -flags.DEFINE_boolean(name="split_multiwords", default=False, +flags.DEFINE_boolean(name="split_multiwords", default=True, help="Split subwords (e.g. don\'t = do, n\'t) into separate tokens.") flags.DEFINE_boolean(name="transformer_encoder", default=False, help="Use transformer encoder.") @@ -162,7 +162,7 @@ def get_defaults(dataset_reader: Optional[DatasetReader], # Dataset reader is required to read training data and/or for training (and validation) data loader dataset_reader = default_ud_dataset_reader(FLAGS.pretrained_transformer_name, tokenizer=LamboTokenizer(FLAGS.tokenizer_language, - default_split_level="TURNS" if FLAGS.turns else "SENTENCES", + default_split_level=FLAGS.split_level.upper(), default_split_multiwords=FLAGS.split_multiwords) ) @@ -459,8 +459,8 @@ def run(_): with open(FLAGS.output_file, "w") as file: for tree in tqdm(test_trees): prediction = predictor.predict_instance(tree) - file.writelines(api.serialize_token_list(api.sentence2conllu(prediction, - keep_semrel=dataset_reader.use_sem))) + file.writelines(api.sentence2conllu(prediction, + keep_semrel=dataset_reader.use_sem).serialize()) predictions.append(prediction) else: @@ -472,8 +472,8 @@ def run(_): predictions = predictor.predict(input_sentences) with open(FLAGS.output_file, "w") as file: for prediction in tqdm(predictions): - file.writelines(api.serialize_token_list(api.sentence2conllu(prediction, - keep_semrel=dataset_reader.use_sem))) + file.writelines(api.sentence2conllu(prediction, + keep_semrel=dataset_reader.use_sem).serialize()) if FLAGS.save_matrices: logger.info("Saving matrices", prefix=prefix) diff --git a/pyproject.toml b/pyproject.toml index da51df8c03270e10b7287719b9248b746e57e61a..c17c0dfde2563b7b666e9cfe07bbaae50009ff05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] [project] name = "combo" -version = "3.2.0" +version = "3.2.1" authors = [ {name = "Maja Jablonska", email = "maja.jablonska@ipipan.waw.pl"} ]