Add result read script

scripts/mg_results.json

+{"attack_xai_local": {"ag_news": {"success_prc": 8.67, "time": 396}}, "attack_textfooler": {"ag_news": {"success_prc": 1.89, "time": 333}}, "attack_xai_reverse_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "imdb": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_reverse": {"ag_news": {"success_prc": 1.86, "time": 119}, "imdb": {"success_prc": 1.56, "time": 33}, "enron_spam": {"success_prc": 0.33, "time": 37}, "multi_emo": {"success_prc": 0.1, "time": 3}, "wiki_pl": {"success_prc": 0.08, "time": 11}}, "attack_xai_reverse_char_mixin": {"ag_news": {"success_prc": 3.32, "time": 145}, "enron_spam": {"success_prc": 0.61, "time": 90}, "multi_emo": {"success_prc": 0.82, "time": 66}, "wiki_pl": {"success_prc": 0.41, "time": 202}}, "attack_xai": {"ag_news": {"success_prc": 2.27, "time": 128}, "imdb": {"success_prc": 4.28, "time": 35}, "enron_spam": {"success_prc": 0.72, "time": 36}, "multi_emo": {"success_prc": 4.98, "time": 20}, "wiki_pl": {"success_prc": 2.94, "time": 62}}, "attack_xai_discard": {"ag_news": {"success_prc": 3.08, "time": 37}, "imdb": {"success_prc": 12.22, "time": 12}, "enron_spam": {"success_prc": 1.33, "time": 5}, "poleval": {"success_prc": 8.89, "time": 12}, "multi_emo": {"success_prc": 4.01, "time": 4}, "wiki_pl": {"success_prc": 10.8, "time": 17}}, "attack_xai_char_mixin": {"ag_news": {"success_prc": 3.63, "time": 43}, "imdb": {"success_prc": 12.83, "time": 74}, "enron_spam": {"success_prc": 1.78, "time": 49}, "multi_emo": {"success_prc": 4.12, "time": 32}, "wiki_pl": {"success_prc": 12.08, "time": 122}}, "attack_xai_directed": {"ag_news": {"success_prc": 3.01, "time": 123}, "imdb": {"success_prc": 2.56, "time": 83}, "enron_spam": {"success_prc": 0.39, "time": 90}, "multi_emo": {"success_prc": 100.0, "time": 20}, "wiki_pl": {"success_prc": 0.83, "time": 80}}, "attack_xai_insert": {"imdb": {"success_prc": 100.0, "time": 1340}, "wiki_pl": {"success_prc": 100.0, "time": 3067}}, "attack_xai_char_discard": {"ag_news": {"success_prc": 4.39, "time": 49}, "imdb": {"success_prc": 11.89, "time": 17}, "enron_spam": {"success_prc": 1.5, "time": 9}, "poleval": {"success_prc": 10.56, "time": 69}, "multi_emo": {"success_prc": 5.25, "time": 7}, "wiki_pl": {"success_prc": 9.45, "time": 20}}, "attack_xai_directed_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "imdb": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "multi_emo": {"success_prc": 0.0, "time": 1}, "wiki_pl": {"success_prc": 0.0, "time": 7}}, "attack_xai_directed_char_mixin": {"ag_news": {"success_prc": 5.92, "time": 106}, "imdb": {"success_prc": 3.06, "time": 83}, "enron_spam": {"success_prc": 0.78, "time": 99}, "multi_emo": {"success_prc": 4.22, "time": 55}, "wiki_pl": {"success_prc": 2.67, "time": 165}}, "attack_xai_char_substitute": {"ag_news": {"success_prc": 3.01, "time": 47}, "imdb": {"success_prc": 13.5, "time": 88}, "enron_spam": {"success_prc": 1.39, "time": 56}, "multi_emo": {"success_prc": 3.81, "time": 33}, "wiki_pl": {"success_prc": 11.25, "time": 133}}}
\ No newline at end of file

scripts/pw_results.json

+{"attack_textfooler": {"ag_news": {"success_prc": 1.89, "time": 333}}, "attack_xai_reverse": {"wiki_pl": {"success_prc": 0.08, "time": 11}, "ag_news": {"success_prc": 1.86, "time": 119}, "enron_spam": {"success_prc": 0.33, "time": 37}, "imdb": {"success_prc": 1.56, "time": 33}, "multi_emo": {"success_prc": 0.1, "time": 3}}, "attack_xai_char_mixin": {"wiki_pl": {"success_prc": 12.08, "time": 122}, "ag_news": {"success_prc": 3.63, "time": 43}, "enron_spam": {"success_prc": 1.78, "time": 49}, "imdb": {"success_prc": 12.83, "time": 74}, "multi_emo": {"success_prc": 4.12, "time": 32}}, "attack_xai_char_substitute": {"wiki_pl": {"success_prc": 11.25, "time": 133}, "ag_news": {"success_prc": 3.01, "time": 47}, "enron_spam": {"success_prc": 1.39, "time": 56}, "imdb": {"success_prc": 13.5, "time": 88}, "multi_emo": {"success_prc": 3.81, "time": 33}}, "attack_xai_reverse_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "imdb": {"success_prc": 0.0, "time": 2}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_discard": {"wiki_pl": {"success_prc": 10.8, "time": 17}, "ag_news": {"success_prc": 3.08, "time": 37}, "enron_spam": {"success_prc": 1.33, "time": 5}, "imdb": {"success_prc": 12.22, "time": 12}, "multi_emo": {"success_prc": 4.01, "time": 4}, "poleval": {"success_prc": 8.89, "time": 12}}, "attack_xai_char_discard": {"wiki_pl": {"success_prc": 9.45, "time": 20}, "ag_news": {"success_prc": 4.39, "time": 49}, "enron_spam": {"success_prc": 1.5, "time": 9}, "imdb": {"success_prc": 11.89, "time": 17}, "multi_emo": {"success_prc": 5.25, "time": 7}, "poleval": {"success_prc": 10.56, "time": 69}}, "attack_xai": {"wiki_pl": {"success_prc": 2.94, "time": 62}, "ag_news": {"success_prc": 2.27, "time": 128}, "enron_spam": {"success_prc": 0.72, "time": 36}, "imdb": {"success_prc": 4.28, "time": 35}, "multi_emo": {"success_prc": 4.98, "time": 20}}, "attack_xai_local": {"ag_news": {"success_prc": 8.67, "time": 396}}, "attack_xai_directed": {"wiki_pl": {"success_prc": 0.83, "time": 80}, "ag_news": {"success_prc": 3.01, "time": 123}, "imdb": {"success_prc": 2.56, "time": 83}, "multi_emo": {"success_prc": 100.0, "time": 20}}, "attack_xai_directed_discard": {"wiki_pl": {"success_prc": 0.0, "time": 7}, "ag_news": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "imdb": {"success_prc": 0.0, "time": 2}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_insert": {"wiki_pl": {"success_prc": 100.0, "time": 3067}, "imdb": {"success_prc": 100.0, "time": 1340}}, "attack_xai_directed_char_mixin": {"wiki_pl": {"success_prc": 2.67, "time": 165}, "ag_news": {"success_prc": 5.92, "time": 106}, "imdb": {"success_prc": 3.06, "time": 83}, "multi_emo": {"success_prc": 4.22, "time": 55}}, "attack_xai_reverse_char_mixin": {"wiki_pl": {"success_prc": 0.41, "time": 202}, "ag_news": {"success_prc": 3.32, "time": 145}, "enron_spam": {"success_prc": 0.61, "time": 90}, "imdb": {"success_prc": 2.5, "time": 113}, "multi_emo": {"success_prc": 0.82, "time": 66}}}
\ No newline at end of file

scripts/read_results.py

+import os
+import json
+import click
+from collections import defaultdict
+
+
+@click.command()
+@click.option(
+    "--results-dir",
+    help="Path to the directory containing the results.",
+)
+@click.option(
+    "--result-name",
+    help="Name of the file to save the results.",
+)
+def main(results_dir: str, result_name: str):
+    results = defaultdict(dict)
+    for attack in os.listdir(results_dir):
+        if attack == ".gitignore":
+            continue
+        for dataset in os.listdir(f"{results_dir}/{attack}"):
+            # print(attack, dataset)
+            if not os.path.exists(f"{results_dir}/{attack}/{dataset}/metrics.json"):
+                continue
+            with open(f"{results_dir}/{attack}/{dataset}/metrics.json", "r") as f:
+                res = json.load(f)
+            if "samples" not in res:
+                continue
+            results[attack][dataset] = {"success_prc": round((res["samples_succ"] / res["samples"])*100, 2), "time": round(res["time"] / 60)}
+    print(results)
+    with open(f"{result_name}.json", "w") as f:
+        json.dump(results, f)
+
+main()