{
"cells": [
{
"cell_type": "markdown",
"source": [
"# Run experiment\n"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"source": [
"To run any experiment steps in `import_any_dataset.ipynb` and `use_any_asr.ipynb` must be done.\n",
"Experiment is managed by `ExperimentRepository`. Below is example of simple experiment."
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# imports\n",
"from experiment.sentence_wer_processor.flair_upos_multi_transformers_wer_processor_base import \\\n",
" FlairUposMultiTransformersWerProcessorBase\n",
"from experiment.sentence_wer_processor.wikineural_multilingual_ner_transformers_wer_processor_base import \\\n",
" WikineuralMultilingualNerTransformersWerProcessorBase\n",
"from experiment.sentence_wer_processor.spacy_pos_sentence_dep_tag_processor import SpacyDepTagSentenceWerProcessor\n",
"from experiment.sentence_wer_processor.spacy_ner_sentence_wer_processor import SpacyNerSentenceWerProcessor\n",
"from experiment.sentence_wer_processor.spacy_pos_sentence_wer_processor import SpacyPosSentenceWerProcessor\n",
"from sziszapangma.core.transformer.fasttext_embedding_transformer import FasttextEmbeddingTransformer\n",
"from sziszapangma.integration.task.embedding_wer_metrics_task import EmbeddingWerMetricsTask\n",
"from sziszapangma.integration.task.classic_wer_metric_task import ClassicWerMetricTask\n",
"from experiment.hf_asr.wav2vec2_hf import Wav2Vec2AsrProcessor\n",
"from experiment.utils.property_helper import PropertyHelper\n",
"from sziszapangma.integration.task.asr_task import AsrTask\n",
"from sziszapangma.integration.experiment_manager import ExperimentManager\n",
"from sziszapangma.integration.audio_repository.local_audio_record_repository import LocalAudioRecordRepository\n",
"from experiment.utils.loaded_dataset_helper import LoadedDatasetHelper\n",
"from pathlib import Path\n",
"from sziszapangma.integration.repository.multi_files_experiment_repository import MultiFilesExperimentRepository"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# globals\n",
"DATASET_NAME = 'dataset_name'\n",
"ASR_NAME = 'asr_name'\n",
"REPOSITORY_ROOT_PATH = Path.home() / 'asr-benchmarks-repository'\n",
"AUDIO_ROOT_PATH = Path.home() / '.cache/asr-benchmarks'\n",
"FASTTEXT_LANGUAGE_CODE = 'pl'\n",
"WIKINEURAL = \"wikineural\"\n",
"SPACY_MODEL_NAME = 'pl_core_news_lg'\n",
"FLAIR_UPOS_MULTI = 'flair_upos_multi'"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# data providers\n",
"repository = MultiFilesExperimentRepository(REPOSITORY_ROOT_PATH, DATASET_NAME)\n",
"record_provider = LoadedDatasetHelper(\n",
" repository, LocalAudioRecordRepository(AUDIO_ROOT_PATH, DATASET_NAME), DATASET_NAME\n",
")"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# experiment definition\n",
"experiment_processor = ExperimentManager(\n",
" record_id_iterator=record_provider,\n",
" processing_tasks=[\n",
" AsrTask(\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" task_name=f\"AsrTask___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" asr_processor=Wav2Vec2AsrProcessor(\"facebook/wav2vec2-large-xlsr-53-polish\"),\n",
" record_path_provider=record_provider,\n",
" ),\n",
" ClassicWerMetricTask(\n",
" task_name=f\"ClassicWerMetricTask___{DATASET_NAME}___{ASR_NAME}\",\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_words(),\n",
" metrics_property_name=PropertyHelper.word_wer_classic_metrics(ASR_NAME),\n",
" require_update=False,\n",
" alignment_property_name=PropertyHelper.word_wer_classic_alignment(ASR_NAME),\n",
" ),\n",
" EmbeddingWerMetricsTask(\n",
" task_name=f\"EmbeddingWerMetricsTask___{DATASET_NAME}___{ASR_NAME}\",\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_words(),\n",
" require_update=False,\n",
" embedding_transformer=FasttextEmbeddingTransformer(FASTTEXT_LANGUAGE_CODE),\n",
" embeddings_alignment_property_name=PropertyHelper.word_wer_embeddings_alignment(ASR_NAME),\n",
" embeddings_metrics_property_name=PropertyHelper.word_wer_embeddings_metrics(ASR_NAME),\n",
" soft_alignment_property_name=PropertyHelper.word_wer_soft_alignment(ASR_NAME),\n",
" soft_metrics_property_name=PropertyHelper.word_wer_soft_metrics(ASR_NAME),\n",
" ),\n",
" SpacyPosSentenceWerProcessor(\n",
" model_name=SPACY_MODEL_NAME,\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_raw(),\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" alignment_property_name=PropertyHelper.pos_alignment(ASR_NAME, SPACY_MODEL_NAME),\n",
" wer_property_name=PropertyHelper.pos_metrics(ASR_NAME, SPACY_MODEL_NAME),\n",
" task_name=f\"SpacyPosSentenceWerProcessor___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" ),\n",
" SpacyNerSentenceWerProcessor(\n",
" model_name=SPACY_MODEL_NAME,\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_raw(),\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" alignment_property_name=PropertyHelper.ner_alignment(\n",
" ASR_NAME, SPACY_MODEL_NAME\n",
" ),\n",
" wer_property_name=PropertyHelper.ner_metrics(ASR_NAME, SPACY_MODEL_NAME),\n",
" task_name=f\"SpacyNerSentenceWerProcessor___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" ),\n",
" SpacyDepTagSentenceWerProcessor(\n",
" model_name=SPACY_MODEL_NAME,\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_raw(),\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" alignment_property_name=PropertyHelper.dep_tag_alignment(\n",
" ASR_NAME, SPACY_MODEL_NAME\n",
" ),\n",
" wer_property_name=PropertyHelper.dep_tag_metrics(ASR_NAME, SPACY_MODEL_NAME),\n",
" task_name=f\"SpacyDepTagSentenceWerProcessor___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" ),\n",
" WikineuralMultilingualNerTransformersWerProcessorBase(\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_raw(),\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" alignment_property_name=PropertyHelper.ner_alignment(\n",
" ASR_NAME, WIKINEURAL\n",
" ),\n",
" wer_property_name=PropertyHelper.ner_metrics(ASR_NAME, WIKINEURAL),\n",
" task_name=f\"WikineuralMultilingualNerTransformersWerProcessorBase___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" ),\n",
" FlairUposMultiTransformersWerProcessorBase(\n",
" gold_transcript_property_name=PropertyHelper.get_gold_transcript_raw(),\n",
" asr_property_name=PropertyHelper.asr_result(ASR_NAME),\n",
" alignment_property_name=PropertyHelper.pos_alignment(\n",
" ASR_NAME, FLAIR_UPOS_MULTI\n",
" ),\n",
" wer_property_name=PropertyHelper.pos_metrics(\n",
" ASR_NAME, FLAIR_UPOS_MULTI\n",
" ),\n",
" task_name=f\"FlairUposMultiTransformersWerProcessorBase___{DATASET_NAME}___{ASR_NAME}\",\n",
" require_update=False,\n",
" )\n",
" ],\n",
" experiment_repository=repository,\n",
")"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# run experiment\n",
"experiment_processor.process()\n"
],
"metadata": {
"collapsed": false
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}