Release 1.0.4

scripts/evaluate_iwpt21.py

+import pathlib
+
+from absl import app
+from absl import flags
+
+from scripts import utils
+
+CODE2LANG = {
+    "ar": "Arabic",
+    "bg": "Bulgarian",
+    "cs": "Czech",
+    "nl": "Dutch",
+    "en": "English",
+    "et": "Estonian",
+    "fi": "Finnish",
+    "fr": "French",
+    "it": "Italian",
+    "lv": "Latvian",
+    "lt": "Lithuanian",
+    "pl": "Polish",
+    "ru": "Russian",
+    "sk": "Slovak",
+    "sv": "Swedish",
+    "ta": "Tamil",
+    "uk": "Ukrainian",
+}
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string(name="data_dir", default="",
+                    help="Path to IWPT'21 data directory.")
+flags.DEFINE_string(name="models_dir", default="/tmp/",
+                    help="Model serialization dir.")
+flags.DEFINE_integer(name="cuda_device", default=-1,
+                     help="Cuda device id (-1 for cpu).")
+flags.DEFINE_string(name="evaluate_script_path", default="iwpt21_xud_eval.py",
+                    help="Path to 'iwpt21_xud_eval.py' eval script.")
+flags.DEFINE_boolean(name="expect_prefix", default=True,
+                     help="Whether to expect allennlp prefix.")
+
+
+def run(_):
+    models_dir = pathlib.Path(FLAGS.models_dir)
+    for model_dir in models_dir.iterdir():
+        if model_dir.name not in CODE2LANG:
+            print("Skipping unknown directory: ", model_dir.name)
+            continue
+
+        treebank_name = f"UD_{CODE2LANG[model_dir.name]}-IWPT"
+
+        if FLAGS.expect_prefix:
+            model_dir = list(model_dir.iterdir())
+            assert len(model_dir) == 1, f"There is incorrect count of models {model_dir}"
+            model_dir = model_dir[0]
+
+        treebank_dir = pathlib.Path(FLAGS.data_dir) / treebank_name
+        files = list(treebank_dir.iterdir())
+
+        test_file = [f for f in files if "dev" in f.name and ".conllu" in f.name]
+        assert len(test_file) == 1, f"Couldn't find test file."
+        test_file = test_file[0]
+
+        if not (model_dir / "results.txt").exists():
+            output_pred = model_dir / 'predictions.conllu'
+            command = f"""combo --mode predict --model_path {model_dir / 'model.tar.gz'}
+            --input_file {test_file}
+            --output_file {output_pred}
+            --cuda_device {FLAGS.cuda_device}
+            --silent
+            """
+            utils.execute_command(command)
+
+            output_collapsed = utils.path_to_str(output_pred).replace('.conllu', '.collapsed.conllu')
+            utils.collapse_nodes(pathlib.Path(FLAGS.data_dir), output_pred, output_collapsed)
+
+            command = f"""python {FLAGS.evaluate_script_path} -v 
+            {test_file}
+            {output_collapsed} 
+            """
+            utils.execute_command(command, output_file=model_dir / "results.txt")
+
+
+def main():
+    app.run(run)
+
+
+if __name__ == "__main__":
+    main()