Add metric calculator

experiment/sentence_wer_processor/wikineural_multilingual_ner_transformers_wer_processor_base.py

@@ -15,7 +15,8 @@ class WikineuralMultilingualNerTransformersWerProcessorBase(SentenceWerProcessor
         super().__init__(gold_transcript_property_name, asr_property_name, alignment_property_name, wer_property_name,
                          task_name, require_update)
         self._tokenizer = AutoTokenizer.from_pretrained("Babelscape/wikineural-multilingual-ner")
-        self._model = AutoModelForTokenClassification.from_pretrained("Babelscape/wikineural-multilingual-ner")
+        self._model = AutoModelForTokenClassification.from_pretrained("Babelscape/wikineural-multilingual-ner").to(
+            'cuda')
         self._pipeline = pipeline("ner", model=self._model, tokenizer=self._tokenizer)
 
     def process_text(self, text: str) -> List[str]:

new_experiment/add_to_queue_pipeline.py

@@ -9,13 +9,13 @@ from pika.adapters.blocking_connection import BlockingChannel
 from new_experiment.new_dependency_provider import get_experiment_repository
 
 COMMANDS = [
-    'run_word_wer_classic_pipeline',
-    'run_word_wer_embedding_pipeline',
+    # 'run_word_wer_classic_pipeline',
+    # 'run_word_wer_embedding_pipeline',
     # 'run_spacy_dep_tag_wer_pipeline',
     # 'run_spacy_ner_wer_pipeline',
     # 'run_spacy_pos_wer_pipeline',
     'run_wikineural_ner_pipeline',
-    'run_flair_upos_multi_pipeline'
+    # 'run_flair_upos_multi_pipeline'
 ]
 LANGUAGES = ['nl', 'fr', 'de', 'it', 'pl', 'es', 'en']
 WHISPER_ASR_MODEL = ['tiny', 'base', 'small', 'medium', 'large-v2']
@@ -140,7 +140,8 @@ def main():
     # add_facebook_hf_wav2vec2_asr(channel)
     # add_facebook_hf_wav2vec2_pipeline(channel)
     # add_nvidia(channel)
-    add_calculate_metrics(channel)
+    add_all(channel)
+    # add_calculate_metrics(channel)
     connection.close()
 
 

results/flair_pos.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.5128568490657601,0.3514757370625669,0.1832678281966678,0.12380286640773615,0.09569981498750402,0.1925248398755547,-1.0
+nl_minds14,0.7325008850026735,0.5548460791328241,0.34604214610991785,0.26785786041003734,0.24476764324104927,0.3193020982890474,-1.0
+nl_voxpopuli,0.472828904291453,0.36430846802521094,0.24143408419509113,0.1930472932566233,0.21055632766856316,0.3042888916261802,-1.0
+fr_google_fleurs,0.44236076819496845,0.32195340523034965,0.2400160279613234,0.18813171319374955,0.174074903748747,0.23336209963520754,0.15913885762717164
+fr_minds14,0.6287744479697867,0.5277812093346057,0.47212377128654737,0.4177637279866131,0.4518296332152451,0.4568353437027064,0.35393428386125453
+fr_voxpopuli,0.3654709924571846,0.3040973363438772,0.25186708911210887,0.22609854479598038,0.23500553093227616,0.2592280488045928,0.15094970653382792
+de_google_fleurs,0.3465858252025954,0.22720269047810474,0.1584525357659024,0.12139888740461777,0.10754966877762259,0.12320433710931088,0.05026491118243037
+de_minds14,0.4464452908680333,0.3467418590608336,0.26502089340235035,0.22944924406050748,0.2264768331647229,0.23856032298373403,0.14752449445646817
+de_voxpopuli,0.3666391841537238,0.2700862688834597,0.21548706581493196,0.18120426998436107,0.22184769210918578,0.23426828044441572,0.06718088969000191
+it_google_fleurs,0.31301046710771296,0.2101309200725639,0.14404509170545263,0.1175670696552392,0.10663974059484682,0.16595363388381548,0.10410330272693828
+it_minds14,0.6333335169134818,0.476970266730833,0.3375840040788103,0.27510273161589305,0.2651017698725975,0.31050806971811973,0.16809665406604052
+it_voxpopuli,0.4391051583926259,0.36357704902505245,0.3107327236487131,0.2789680162753996,0.28310320136782174,-1.0,0.19856464726278672
+pl_google_fleurs,0.5205242857401786,0.35892871895052686,0.19040656818768317,0.1237062070730666,0.09898117720565483,0.24288976088763142,-1.0
+pl_minds14,0.8613662825527818,0.6667379597696379,0.439213987359154,0.370197736647061,0.36117236927972074,0.5428314194103238,-1.0
+pl_voxpopuli,0.4049810084672203,0.30111262121925136,0.19470192171934297,0.1566441794464554,0.15160063128496318,0.24006996555622762,-1.0
+es_google_fleurs,0.20488404962390622,0.13501753104385247,0.08628135144862038,0.07860762524537634,0.06794040280825796,0.1053269094464826,0.050018774899258385
+es_minds14,0.7524251141671842,0.6012400142586597,0.5113200371183868,0.5054829500298336,0.4972488981953535,0.5357580304997958,0.4017298638816005
+es_voxpopuli,0.23301268671843306,0.1797365233957996,0.20248450673984855,0.1679185939703077,0.20838092519332788,0.14800145346923332,0.05496272509071882
+en_google_fleurs,0.1982091941383234,0.157780242046229,0.12536003616851438,0.11113800666343415,0.11201208028193228,0.11621138297642805,0.09232162004079439
+en_minds14,0.46349949039412097,0.44622243280584867,0.44234639955865274,0.44417454893635283,0.43804794126384305,0.4344448996709846,0.37118845185669014
+en_voxpopuli,0.3259762910109974,0.29415370504278343,0.30645321426593813,0.45109143631825516,0.41453453892967496,0.12075433149375665,0.05542773021817536

results/spacy_dep.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.5829162976334935,0.4273641113304075,0.2791896493055206,0.22940184464809937,0.21237269117521373,0.16095681315376875,-1.0
+nl_minds14,0.8889885205573127,0.7021069927272965,0.5118650285197052,0.44008063150172844,0.415820999328982,0.2985834609126623,-1.0
+nl_voxpopuli,0.45195033154297287,0.35022827625599706,0.23306134090292266,0.18846103141181214,0.20866449581994992,0.34065579813718216,-1.0
+fr_google_fleurs,0.46841469264510127,0.33892688325578885,0.2601571030310364,0.20724071198786528,0.194586722917292,0.14156045133124184,0.07366709679830713
+fr_minds14,0.7007351451733806,0.6193822910777104,0.5674873937741992,0.5135737987476685,0.5528256321306745,0.3366555675171398,0.2367702135610348
+fr_voxpopuli,0.3106610782756462,0.23559577960697295,0.1809428791053797,0.15328834247216147,0.15986658579185167,0.2452287053793374,0.16460689520334282
+de_google_fleurs,0.44963987493523083,0.3440010626732555,0.2820876100612969,0.2756337301506401,0.2640933300137521,0.09420580235965058,0.0531476412259453
+de_minds14,0.6088134055595739,0.5295985249493463,0.4722053815851853,0.4430939184548618,0.44165621205819816,0.22897986936024708,0.15785527689808235
+de_voxpopuli,0.34765280867619563,0.24806021482791354,0.1980005177427861,0.16823732334253969,0.2050588932147993,0.3137039560211158,0.20363319805144914
+it_google_fleurs,0.3646995933863305,0.26909235240948964,0.2183608431778368,0.18963172017789925,0.18910775317046497,0.11521172222871585,0.057875459119117664
+it_minds14,0.735663369022031,0.5977235234184719,0.5003766882433877,0.43834400033260107,0.4177850602671256,0.28553099994914055,0.1532501214995452
+it_voxpopuli,0.40173796423628366,0.3322573693586659,0.27898827733716525,0.24546845703021283,0.24763773067695904,-1.0,0.23610618021049726
+pl_google_fleurs,0.5942845990666885,0.45256986232454416,0.318702058386024,0.27647479112157713,0.2611938730970087,0.18499361398064174,-1.0
+pl_minds14,0.9889926331393841,0.8534314758404862,0.6536927984936637,0.5858840447828013,0.5974675143718239,0.45493938339160295,-1.0
+pl_voxpopuli,0.37454419404599276,0.2772897613159953,0.19868503784732797,0.16452352931904018,0.16188708297048007,0.3097521141364381,-1.0
+es_google_fleurs,0.2844989806365395,0.22474762440702908,0.1873652067557157,0.18956091761632282,0.18402784458098992,0.0964759185969363,0.05140105136500811
+es_minds14,0.8809918173153016,0.7476774342699631,0.695293666624561,0.6907494148178319,0.6978843552573668,0.5088175271372467,0.38421491370135774
+es_voxpopuli,0.25246263478156405,0.2062251126721369,0.22970613522835304,0.1958460313621695,0.23158673181879283,0.2303512349057837,0.17398730563244433
+en_google_fleurs,0.2958525061205758,0.25092809319221465,0.2244832027403743,0.21885456382493407,0.21847863064643,0.3674137915251593,0.07890360958153639
+en_minds14,0.6343509110337681,0.6239619768303191,0.6269418057909657,0.6265880246712795,0.6209530884473694,0.5845465290621714,0.3292824253771989
+en_voxpopuli,0.34583622058218405,0.3194929524686721,0.31906001467432227,0.4664103727871316,0.40894939997043284,0.3771002587092677,0.16088310901140507

results/spacy_ner.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.3161237339690155,0.2308450255094158,0.18693561212834653,0.17014951030768935,0.16505738887496518,0.0827806307478379,-1.0
+nl_minds14,0.46308435347863025,0.40999338358425746,0.3609344150783841,0.3316129360565079,0.32417176172896284,0.14215531790567,-1.0
+nl_voxpopuli,0.2151580626754643,0.17871595119222772,0.1329598779202076,0.118041915441359,0.1399584583800468,0.20040336938440825,-1.0
+fr_google_fleurs,0.2642906766344454,0.19343641908607892,0.1773022469629892,0.14746388396649565,0.14127591644135662,0.08316978479064052,0.053154878860036295
+fr_minds14,0.46686002221658374,0.46882168507822686,0.4717541865579766,0.4448537740862202,0.4850901698675142,0.22035752854821256,0.18911074492221522
+fr_voxpopuli,0.16138582163759913,0.13114429522233054,0.1130971579228055,0.09911409932520975,0.111775838751783,0.16956384642535782,0.12795776601232278
+de_google_fleurs,0.31617544087132954,0.2574542875017518,0.23416257854767267,0.23975037963438592,0.23671509461637588,0.08342347905666013,0.05167292408313949
+de_minds14,0.4356808798069878,0.42571228078952067,0.4128959437923648,0.39861678563521713,0.39876192320292564,0.18393274494045384,0.14698806531557074
+de_voxpopuli,0.20024522164378158,0.15550239624527337,0.1332514993013951,0.11694941242968462,0.1563708547096921,0.24249804263648722,0.16885416043010723
+it_google_fleurs,0.20630133500713563,0.17252739152106852,0.1611948763222098,0.15665489015868286,0.16067651052906687,0.06718097066660256,0.0390397063511667
+it_minds14,0.48749338164762673,0.44887355443677834,0.4326793465371167,0.41603518258754857,0.39270546565924874,0.19880858599312193,0.14623512926612414
+it_voxpopuli,0.1603647373763574,0.13946071669971405,0.1389659591588509,0.1231302376302245,0.1306905502885798,-1.0,0.1539599945400542
+pl_google_fleurs,0.3349363683333008,0.27302466756654503,0.2276621043895336,0.21096202104527975,0.2090269574201934,0.08815678425787603,-1.0
+pl_minds14,0.6571939310109077,0.5915879062047097,0.48734396553025583,0.47401281595774647,0.48789062029558666,0.23769231904699262,-1.0
+pl_voxpopuli,0.2035480092276859,0.15852551918767724,0.12627983986306934,0.1107841452028173,0.11777970227820521,0.18436790401581038,-1.0
+es_google_fleurs,0.18760672292298558,0.15987294628377563,0.14710378540957528,0.1552095936878963,0.15465747514794018,0.05783031439128734,0.03890324050525823
+es_minds14,0.7212948475851672,0.6703632188688116,0.6662778849859748,0.6730575815341928,0.6803410601549468,0.4119272209287014,0.3428951114977295
+es_voxpopuli,0.1338049156609253,0.11622247150670188,0.11988222690959874,0.1066097834941947,0.12203639823791315,0.1482249545010369,0.1284557160936599
+en_google_fleurs,0.21784267766405332,0.18880961699310436,0.18640702089368935,0.18365555974451866,0.1845679586504467,0.18052342550593456,0.07142097990250293
+en_minds14,0.5620680295445741,0.5669985470890051,0.5803690352687086,0.5839454444685104,0.5780792354057481,0.3253041818139508,0.2930826770701386
+en_voxpopuli,0.22497977677418438,0.20395903946584412,0.21027849596311884,0.3226881294358427,0.2808771065826151,0.18270755590358512,0.12441606059366528

results/spacy_pos.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.5829162976334935,0.4273641113304075,0.2791896493055206,0.22940184464809937,0.21237269117521373,0.16095681315376875,-1.0
+nl_minds14,0.8889885205573127,0.7021069927272965,0.5118650285197052,0.44008063150172844,0.415820999328982,0.2985834609126623,-1.0
+nl_voxpopuli,0.45195033154297287,0.35022827625599706,0.23306134090292266,0.18846103141181214,0.20866449581994992,0.34065579813718216,-1.0
+fr_google_fleurs,0.46841469264510127,0.33892688325578885,0.2601571030310364,0.20724071198786528,0.194586722917292,0.14156045133124184,0.07366709679830713
+fr_minds14,0.7007351451733806,0.6193822910777104,0.5674873937741992,0.5135737987476685,0.5528256321306745,0.3366555675171398,0.2367702135610348
+fr_voxpopuli,0.3106610782756462,0.23559577960697295,0.1809428791053797,0.15328834247216147,0.15986658579185167,0.2452287053793374,0.16460689520334282
+de_google_fleurs,0.44963987493523083,0.3440010626732555,0.2820876100612969,0.2756337301506401,0.2640933300137521,0.09420580235965058,0.0531476412259453
+de_minds14,0.6088134055595739,0.5295985249493463,0.4722053815851853,0.4430939184548618,0.44165621205819816,0.22897986936024708,0.15785527689808235
+de_voxpopuli,0.34765280867619563,0.24806021482791354,0.1980005177427861,0.16823732334253969,0.2050588932147993,0.3137039560211158,0.20363319805144914
+it_google_fleurs,0.3646995933863305,0.26909235240948964,0.2183608431778368,0.18963172017789925,0.18910775317046497,0.11521172222871585,0.057875459119117664
+it_minds14,0.735663369022031,0.5977235234184719,0.5003766882433877,0.43834400033260107,0.4177850602671256,0.28553099994914055,0.1532501214995452
+it_voxpopuli,0.40173796423628366,0.3322573693586659,0.27898827733716525,0.24546845703021283,0.24763773067695904,-1.0,0.23610618021049726
+pl_google_fleurs,0.5942845990666885,0.45256986232454416,0.318702058386024,0.27647479112157713,0.2611938730970087,0.18499361398064174,-1.0
+pl_minds14,0.9889926331393841,0.8534314758404862,0.6536927984936637,0.5858840447828013,0.5974675143718239,0.45493938339160295,-1.0
+pl_voxpopuli,0.37454419404599276,0.2772897613159953,0.19868503784732797,0.16452352931904018,0.16188708297048007,0.3097521141364381,-1.0
+es_google_fleurs,0.2844989806365395,0.22474762440702908,0.1873652067557157,0.18956091761632282,0.18402784458098992,0.0964759185969363,0.05140105136500811
+es_minds14,0.8809918173153016,0.7476774342699631,0.695293666624561,0.6907494148178319,0.6978843552573668,0.5088175271372467,0.38421491370135774
+es_voxpopuli,0.25246263478156405,0.2062251126721369,0.22970613522835304,0.1958460313621695,0.23158673181879283,0.2303512349057837,0.17398730563244433
+en_google_fleurs,0.2958525061205758,0.25092809319221465,0.2244832027403743,0.21885456382493407,0.21847863064643,0.3674137915251593,0.07890360958153639
+en_minds14,0.6343509110337681,0.6239619768303191,0.6269418057909657,0.6265880246712795,0.6209530884473694,0.5845465290621714,0.3292824253771989
+en_voxpopuli,0.34583622058218405,0.3194929524686721,0.31906001467432227,0.4664103727871316,0.40894939997043284,0.3771002587092677,0.16088310901140507

results/word_wer_classic_metrics.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.7080198204642507,0.5356923964488436,0.3653458135697763,0.2960998986403285,0.2619505012755527,0.27375214375877915,-1.0
+nl_minds14,0.8974471707133489,0.7144984360077963,0.50343619989782,0.4190827144681917,0.38912482703086787,0.4654939351976398,-1.0
+nl_voxpopuli,0.6457154301129577,0.526938708264302,0.3969399281955793,0.3450343929127979,0.3580225634169237,0.38083549898550806,-1.0
+fr_google_fleurs,0.6001853344125808,0.47080826994674113,0.378477900901456,0.32423598286149874,0.3095695135840552,0.3051829660733236,0.2064330075620991
+fr_minds14,0.8059773405214599,0.7007727121475422,0.6426194543340974,0.5833233362170459,0.6164113630272453,0.5648853985109629,0.4411537294676458
+fr_voxpopuli,0.5106227790884983,0.4403400911159554,0.3829611991773308,0.3596333492051914,0.36581128121534917,0.32335142948445655,0.1870737774419929
+de_google_fleurs,0.651989294142829,0.5517658935306804,0.5069439259606154,0.4784758754564045,0.46904496694540054,0.18239511264966746,0.07216197976121866
+de_minds14,0.6598899262849806,0.5544373523550716,0.47451338993567643,0.42927436689483134,0.42513354947808735,0.43736940909649014,0.35784769213950446
+de_voxpopuli,0.6458977370886917,0.5588758831968453,0.5189755629114159,0.48819380947830204,0.5255812986634112,0.29220314648715856,0.0882559571483075
+it_google_fleurs,0.46529846289240334,0.35587745001762633,0.28749147399148783,0.2543836855330587,0.2516966404869229,0.2186893660599154,0.14056400793022397
+it_minds14,0.7794287222482653,0.6215460371656099,0.5026695496691157,0.4378051504507911,0.42278141507671585,0.42994045491186755,0.27600186172091723
+it_voxpopuli,0.5627286028101925,0.4778544360274492,0.4203868725984355,0.3889042212106447,0.39396401624957433,-1.0,0.23307579422634225
+pl_google_fleurs,0.7008528827207772,0.5530734276379661,0.3841422570886412,0.31820311135163293,0.2982473717417661,0.3358699844317801,-1.0
+pl_minds14,1.0233241532364357,0.8606255448727004,0.6337659043847286,0.572826392381843,0.563293258991981,0.6975840975849457,-1.0
+pl_voxpopuli,0.5884637431843275,0.48926462960487194,0.38088255070573357,0.34562276283708676,0.3498963579139405,0.32422926846361416,-1.0
+es_google_fleurs,0.3336575242844486,0.2613519391737562,0.2139499353452219,0.20635148694336083,0.202078344515759,0.14552169455131528,0.06768590515119453
+es_minds14,0.8846887474293932,0.7406039496007797,0.6648308444297483,0.6560896378714521,0.6503278595728569,0.6024941955004669,0.43657035151443874
+es_voxpopuli,0.3471121389767832,0.2941921098134998,0.33350006838758134,0.2954720913873091,0.35327286422753756,0.19124174178100423,0.06736284277573418
+en_google_fleurs,0.34815171222194247,0.3072073990382695,0.27885655459176045,0.2689168673418541,0.2702083876552681,1.0314848724340477,0.11496640935215942
+en_minds14,0.5883754265520795,0.5718451496167216,0.5663812472768062,0.5675376064430723,0.5626511316300407,1.2032515146101705,0.46729650951263696
+en_voxpopuli,0.47561226703247045,0.45158568654948933,0.4531320996866833,0.594545897613299,0.5497549580225748,1.02051443577821,0.0679191991879864

results/word_wer_embedding_metrics.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.5128568490657601,0.3514757370625669,0.1832678281966678,0.12380286640773615,0.09569981498750402,0.1925248398755547,-1.0
+nl_minds14,0.7325008850026735,0.5548460791328241,0.34604214610991785,0.26785786041003734,0.24476764324104927,0.3193020982890474,-1.0
+nl_voxpopuli,0.472828904291453,0.36430846802521094,0.24143408419509113,0.1930472932566233,0.21055632766856316,0.3042888916261802,-1.0
+fr_google_fleurs,0.44236076819496845,0.32195340523034965,0.2400160279613234,0.18813171319374955,0.174074903748747,0.23336209963520754,0.15913885762717164
+fr_minds14,0.6287744479697867,0.5277812093346057,0.47212377128654737,0.4177637279866131,0.4518296332152451,0.4568353437027064,0.35393428386125453
+fr_voxpopuli,0.3654709924571846,0.3040973363438772,0.25186708911210887,0.22609854479598038,0.23500553093227616,0.2592280488045928,0.15094970653382792
+de_google_fleurs,0.3465858252025954,0.22720269047810474,0.1584525357659024,0.12139888740461777,0.10754966877762259,0.12320433710931088,0.05026491118243037
+de_minds14,0.4464452908680333,0.3467418590608336,0.26502089340235035,0.22944924406050748,0.2264768331647229,0.23856032298373403,0.14752449445646817
+de_voxpopuli,0.3666391841537238,0.2700862688834597,0.21548706581493196,0.18120426998436107,0.22184769210918578,0.23426828044441572,0.06718088969000191
+it_google_fleurs,0.31301046710771296,0.2101309200725639,0.14404509170545263,0.1175670696552392,0.10663974059484682,0.16595363388381548,0.10410330272693828
+it_minds14,0.6333335169134818,0.476970266730833,0.3375840040788103,0.27510273161589305,0.2651017698725975,0.31050806971811973,0.16809665406604052
+it_voxpopuli,0.4391051583926259,0.36357704902505245,0.3107327236487131,0.2789680162753996,0.28310320136782174,-1.0,0.19856464726278672
+pl_google_fleurs,0.5205242857401786,0.35892871895052686,0.19040656818768317,0.1237062070730666,0.09898117720565483,0.24288976088763142,-1.0
+pl_minds14,0.8613662825527818,0.6667379597696379,0.439213987359154,0.370197736647061,0.36117236927972074,0.5428314194103238,-1.0
+pl_voxpopuli,0.4049810084672203,0.30111262121925136,0.19470192171934297,0.1566441794464554,0.15160063128496318,0.24006996555622762,-1.0
+es_google_fleurs,0.20488404962390622,0.13501753104385247,0.08628135144862038,0.07860762524537634,0.06794040280825796,0.1053269094464826,0.050018774899258385
+es_minds14,0.7524251141671842,0.6012400142586597,0.5113200371183868,0.5054829500298336,0.4972488981953535,0.5357580304997958,0.4017298638816005
+es_voxpopuli,0.23301268671843306,0.1797365233957996,0.20248450673984855,0.1679185939703077,0.20838092519332788,0.14800145346923332,0.05496272509071882
+en_google_fleurs,0.1982091941383234,0.157780242046229,0.12536003616851438,0.11113800666343415,0.11201208028193228,0.11621138297642805,0.09232162004079439
+en_minds14,0.46349949039412097,0.44622243280584867,0.44234639955865274,0.44417454893635283,0.43804794126384305,0.4344448996709846,0.37118845185669014
+en_voxpopuli,0.3259762910109974,0.29415370504278343,0.30645321426593813,0.45109143631825516,0.41453453892967496,0.12075433149375665,0.05542773021817536

results/word_wer_soft_metrics.csv

+,whisper_tiny,whisper_base,whisper_small,whisper_medium,whisper_large-v2,facebook_wav2vec2,nvidia_stt
+nl_google_fleurs,0.4870197773495576,0.3328258844694986,0.17381466518820615,0.1183119738171876,0.09216375573740883,0.1861375820998239,-1.0
+nl_minds14,0.696386974399987,0.5288071903683318,0.32315271625044095,0.25185544667973697,0.23476634010226508,0.30664787001613525,-1.0
+nl_voxpopuli,0.44076479195013063,0.34922639815948614,0.2333979846624523,0.18769358589303506,0.203840330351997,0.2954500928703489,-1.0
+fr_google_fleurs,0.4220048065232591,0.3080314104634551,0.23095946828769665,0.1815195000573236,0.16757513398216545,0.2257447559178647,0.1545878867171351
+fr_minds14,0.598663964012827,0.49963220840828954,0.44775733449249705,0.39565391818067924,0.42932694896827234,0.44122406972784367,0.3426367863418972
+fr_voxpopuli,0.34990621244280495,0.2916534573257225,0.2423138137693979,0.21819346515610386,0.2266814091685934,0.2510041541656641,0.14778602513574868
+de_google_fleurs,0.3289276182048546,0.21351452721873002,0.15105999241218784,0.11687072170975221,0.10482724827639953,0.11899880350628178,0.048662647503197085
+de_minds14,0.425754495517847,0.33131654192372123,0.2556198103235964,0.2226020966429888,0.2201038814448124,0.23253258303984767,0.14330619933043048
+de_voxpopuli,0.34922401740114545,0.25990993072367846,0.20832803676002826,0.1764781974388757,0.21569201857798787,0.22857231889635857,0.06566121190641055
+it_google_fleurs,0.29787653549942666,0.20127565575711978,0.1394354151626103,0.11457866977153304,0.1039253971542114,0.1614138467965735,0.10128480152029833
+it_minds14,0.6037426096027818,0.45530553276032937,0.3235265689815623,0.26479676336813834,0.25538260396739326,0.29921632065003106,0.16275287708746614
+it_voxpopuli,0.4180957511927371,0.3456874701080864,0.29807895486520664,0.2668877439740146,0.2706693390058772,-1.0,0.19369164204641623
+pl_google_fleurs,0.49329526249816336,0.336318804366444,0.18304646482118392,0.11945294809160295,0.09662476868469452,0.2328511075002199,-1.0
+pl_minds14,0.822963838473573,0.6333993938805039,0.42006661959397856,0.3537095925776635,0.3428915721620752,0.5196837284129723,-1.0
+pl_voxpopuli,0.38592337016831707,0.28833619541828037,0.1884132278261526,0.15232135201108876,0.14746298777686856,0.23240958556609695,-1.0
+es_google_fleurs,0.19605539451549214,0.1301094985121495,0.08411390243345176,0.07730197238799653,0.06729451433617974,0.10232387999096239,0.04899742742377105
+es_minds14,0.7230857500930252,0.5816243222205137,0.4970367287735149,0.493568473478763,0.48816956547576756,0.5222085521204625,0.3973154256602858
+es_voxpopuli,0.2225049491883292,0.17276447729433134,0.19574601070950448,0.16249494079910376,0.20146803547620717,0.1435783702689086,0.05372137843313578
+en_google_fleurs,0.1913289247939793,0.15169298081003088,0.12113403773904852,0.10757846654400742,0.10860943867024678,0.11146553913321706,0.08860939275147313
+en_minds14,0.4497833240938392,0.4338391626203246,0.42778839014619985,0.4310425208627183,0.4249693796644962,0.42498380965557586,0.3636424377438345
+en_voxpopuli,0.31458127393554913,0.2868021899495161,0.2978194832900427,0.4396803103879028,0.4025547250014984,0.11829558915390843,0.054175600448712995