From 355a63f437d0d153e7744f0b29cf4593099c9e0d Mon Sep 17 00:00:00 2001 From: pwalkow <pawel.walkowiak@pwr.edu.pl> Date: Wed, 26 Feb 2025 12:21:06 +0100 Subject: [PATCH] Add result read script --- scripts/mg_results.json | 1 + scripts/pw_results.json | 1 + scripts/read_results.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 scripts/mg_results.json create mode 100644 scripts/pw_results.json create mode 100644 scripts/read_results.py diff --git a/scripts/mg_results.json b/scripts/mg_results.json new file mode 100644 index 0000000..1c39eb3 --- /dev/null +++ b/scripts/mg_results.json @@ -0,0 +1 @@ +{"attack_xai_local": {"ag_news": {"success_prc": 8.67, "time": 396}}, "attack_textfooler": {"ag_news": {"success_prc": 1.89, "time": 333}}, "attack_xai_reverse_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "imdb": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_reverse": {"ag_news": {"success_prc": 1.86, "time": 119}, "imdb": {"success_prc": 1.56, "time": 33}, "enron_spam": {"success_prc": 0.33, "time": 37}, "multi_emo": {"success_prc": 0.1, "time": 3}, "wiki_pl": {"success_prc": 0.08, "time": 11}}, "attack_xai_reverse_char_mixin": {"ag_news": {"success_prc": 3.32, "time": 145}, "enron_spam": {"success_prc": 0.61, "time": 90}, "multi_emo": {"success_prc": 0.82, "time": 66}, "wiki_pl": {"success_prc": 0.41, "time": 202}}, "attack_xai": {"ag_news": {"success_prc": 2.27, "time": 128}, "imdb": {"success_prc": 4.28, "time": 35}, "enron_spam": {"success_prc": 0.72, "time": 36}, "multi_emo": {"success_prc": 4.98, "time": 20}, "wiki_pl": {"success_prc": 2.94, "time": 62}}, "attack_xai_discard": {"ag_news": {"success_prc": 3.08, "time": 37}, "imdb": {"success_prc": 12.22, "time": 12}, "enron_spam": {"success_prc": 1.33, "time": 5}, "poleval": {"success_prc": 8.89, "time": 12}, "multi_emo": {"success_prc": 4.01, "time": 4}, "wiki_pl": {"success_prc": 10.8, "time": 17}}, "attack_xai_char_mixin": {"ag_news": {"success_prc": 3.63, "time": 43}, "imdb": {"success_prc": 12.83, "time": 74}, "enron_spam": {"success_prc": 1.78, "time": 49}, "multi_emo": {"success_prc": 4.12, "time": 32}, "wiki_pl": {"success_prc": 12.08, "time": 122}}, "attack_xai_directed": {"ag_news": {"success_prc": 3.01, "time": 123}, "imdb": {"success_prc": 2.56, "time": 83}, "enron_spam": {"success_prc": 0.39, "time": 90}, "multi_emo": {"success_prc": 100.0, "time": 20}, "wiki_pl": {"success_prc": 0.83, "time": 80}}, "attack_xai_insert": {"imdb": {"success_prc": 100.0, "time": 1340}, "wiki_pl": {"success_prc": 100.0, "time": 3067}}, "attack_xai_char_discard": {"ag_news": {"success_prc": 4.39, "time": 49}, "imdb": {"success_prc": 11.89, "time": 17}, "enron_spam": {"success_prc": 1.5, "time": 9}, "poleval": {"success_prc": 10.56, "time": 69}, "multi_emo": {"success_prc": 5.25, "time": 7}, "wiki_pl": {"success_prc": 9.45, "time": 20}}, "attack_xai_directed_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "imdb": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "multi_emo": {"success_prc": 0.0, "time": 1}, "wiki_pl": {"success_prc": 0.0, "time": 7}}, "attack_xai_directed_char_mixin": {"ag_news": {"success_prc": 5.92, "time": 106}, "imdb": {"success_prc": 3.06, "time": 83}, "enron_spam": {"success_prc": 0.78, "time": 99}, "multi_emo": {"success_prc": 4.22, "time": 55}, "wiki_pl": {"success_prc": 2.67, "time": 165}}, "attack_xai_char_substitute": {"ag_news": {"success_prc": 3.01, "time": 47}, "imdb": {"success_prc": 13.5, "time": 88}, "enron_spam": {"success_prc": 1.39, "time": 56}, "multi_emo": {"success_prc": 3.81, "time": 33}, "wiki_pl": {"success_prc": 11.25, "time": 133}}} \ No newline at end of file diff --git a/scripts/pw_results.json b/scripts/pw_results.json new file mode 100644 index 0000000..1f0a409 --- /dev/null +++ b/scripts/pw_results.json @@ -0,0 +1 @@ +{"attack_textfooler": {"ag_news": {"success_prc": 1.89, "time": 333}}, "attack_xai_reverse": {"wiki_pl": {"success_prc": 0.08, "time": 11}, "ag_news": {"success_prc": 1.86, "time": 119}, "enron_spam": {"success_prc": 0.33, "time": 37}, "imdb": {"success_prc": 1.56, "time": 33}, "multi_emo": {"success_prc": 0.1, "time": 3}}, "attack_xai_char_mixin": {"wiki_pl": {"success_prc": 12.08, "time": 122}, "ag_news": {"success_prc": 3.63, "time": 43}, "enron_spam": {"success_prc": 1.78, "time": 49}, "imdb": {"success_prc": 12.83, "time": 74}, "multi_emo": {"success_prc": 4.12, "time": 32}}, "attack_xai_char_substitute": {"wiki_pl": {"success_prc": 11.25, "time": 133}, "ag_news": {"success_prc": 3.01, "time": 47}, "enron_spam": {"success_prc": 1.39, "time": 56}, "imdb": {"success_prc": 13.5, "time": 88}, "multi_emo": {"success_prc": 3.81, "time": 33}}, "attack_xai_reverse_discard": {"ag_news": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "imdb": {"success_prc": 0.0, "time": 2}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_discard": {"wiki_pl": {"success_prc": 10.8, "time": 17}, "ag_news": {"success_prc": 3.08, "time": 37}, "enron_spam": {"success_prc": 1.33, "time": 5}, "imdb": {"success_prc": 12.22, "time": 12}, "multi_emo": {"success_prc": 4.01, "time": 4}, "poleval": {"success_prc": 8.89, "time": 12}}, "attack_xai_char_discard": {"wiki_pl": {"success_prc": 9.45, "time": 20}, "ag_news": {"success_prc": 4.39, "time": 49}, "enron_spam": {"success_prc": 1.5, "time": 9}, "imdb": {"success_prc": 11.89, "time": 17}, "multi_emo": {"success_prc": 5.25, "time": 7}, "poleval": {"success_prc": 10.56, "time": 69}}, "attack_xai": {"wiki_pl": {"success_prc": 2.94, "time": 62}, "ag_news": {"success_prc": 2.27, "time": 128}, "enron_spam": {"success_prc": 0.72, "time": 36}, "imdb": {"success_prc": 4.28, "time": 35}, "multi_emo": {"success_prc": 4.98, "time": 20}}, "attack_xai_local": {"ag_news": {"success_prc": 8.67, "time": 396}}, "attack_xai_directed": {"wiki_pl": {"success_prc": 0.83, "time": 80}, "ag_news": {"success_prc": 3.01, "time": 123}, "imdb": {"success_prc": 2.56, "time": 83}, "multi_emo": {"success_prc": 100.0, "time": 20}}, "attack_xai_directed_discard": {"wiki_pl": {"success_prc": 0.0, "time": 7}, "ag_news": {"success_prc": 0.0, "time": 2}, "enron_spam": {"success_prc": 0.0, "time": 1}, "imdb": {"success_prc": 0.0, "time": 2}, "multi_emo": {"success_prc": 0.0, "time": 1}}, "attack_xai_insert": {"wiki_pl": {"success_prc": 100.0, "time": 3067}, "imdb": {"success_prc": 100.0, "time": 1340}}, "attack_xai_directed_char_mixin": {"wiki_pl": {"success_prc": 2.67, "time": 165}, "ag_news": {"success_prc": 5.92, "time": 106}, "imdb": {"success_prc": 3.06, "time": 83}, "multi_emo": {"success_prc": 4.22, "time": 55}}, "attack_xai_reverse_char_mixin": {"wiki_pl": {"success_prc": 0.41, "time": 202}, "ag_news": {"success_prc": 3.32, "time": 145}, "enron_spam": {"success_prc": 0.61, "time": 90}, "imdb": {"success_prc": 2.5, "time": 113}, "multi_emo": {"success_prc": 0.82, "time": 66}}} \ No newline at end of file diff --git a/scripts/read_results.py b/scripts/read_results.py new file mode 100644 index 0000000..2e20546 --- /dev/null +++ b/scripts/read_results.py @@ -0,0 +1,34 @@ +import os +import json +import click +from collections import defaultdict + + +@click.command() +@click.option( + "--results-dir", + help="Path to the directory containing the results.", +) +@click.option( + "--result-name", + help="Name of the file to save the results.", +) +def main(results_dir: str, result_name: str): + results = defaultdict(dict) + for attack in os.listdir(results_dir): + if attack == ".gitignore": + continue + for dataset in os.listdir(f"{results_dir}/{attack}"): + # print(attack, dataset) + if not os.path.exists(f"{results_dir}/{attack}/{dataset}/metrics.json"): + continue + with open(f"{results_dir}/{attack}/{dataset}/metrics.json", "r") as f: + res = json.load(f) + if "samples" not in res: + continue + results[attack][dataset] = {"success_prc": round((res["samples_succ"] / res["samples"])*100, 2), "time": round(res["time"] / 60)} + print(results) + with open(f"{result_name}.json", "w") as f: + json.dump(results, f) + +main() -- GitLab