diff --git a/combo/data/api.py b/combo/data/api.py index b36b6165dc7464d11bbc80228a85659c2f2324af..ab00da569b52d1973072a8b76d13540303f5d98d 100644 --- a/combo/data/api.py +++ b/combo/data/api.py @@ -86,7 +86,7 @@ def serialize_field(field: Any) -> str: return "{}".format(field) def serialize_token_list(tokenlist: conllu.models.TokenList) -> str: - KEYS_ORDER = ['idx', 'text', 'lemma', 'upostag', 'xpostag', 'entity_type', 'feats', 'head', 'deprel', 'deps', 'misc'] + KEYS_ORDER = ['idx', 'text', 'lemma', 'upostag', 'xpostag', 'feats', 'head', 'deprel', 'deps'] lines = [] if tokenlist.metadata: @@ -99,6 +99,18 @@ def serialize_token_list(tokenlist: conllu.models.TokenList) -> str: for token_data in tokenlist: line = '\t'.join(serialize_field(token_data[k]) for k in KEYS_ORDER) + serialized_misc = serialize_field(token_data['misc']) + serialized_entity_type = serialize_field(token_data['entity_type']) + if serialized_misc == '_' and serialized_entity_type == '_': + serialized_last_column = '_' + elif serialized_misc == '_': + serialized_last_column = serialized_entity_type + elif serialized_entity_type == '_': + serialized_last_column = serialized_misc + else: + serialized_last_column = serialized_entity_type + ' | ' + serialized_misc + + line += '\t' + serialized_last_column lines.append(line) return '\n'.join(lines) + "\n\n"