dvc.yaml

stages:

    luna_import_to_common_format:
        cmd: PYTHONPATH=. python experiment/luna/import_dataset/import_luna.py
        deps:
            - experiment/luna/import_dataset/import_luna.py
            - experiment_data/dataset/LUNA.PL
        outs:
            - experiment_data/dataset_relation_manager_data/luna

    luna_gold_transcript_processing:
        cmd: |
            PYTHONPATH=. python experiment/luna/pipeline/luna_gold_transcript_processing.py
        deps:
            - experiment/luna/pipeline/luna_gold_transcript_processing.py
            - experiment_data/dataset_relation_manager_data/luna
            - experiment_data/dataset/LUNA.PL
        outs:
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript_spacy

    luna_techmo_processing:
        cmd: |
            PYTHONPATH=. python experiment/luna/pipeline/luna_techmo_processing.py
        deps:
            - experiment/luna/pipeline/luna_techmo_processing.py
            - experiment_data/dataset/LUNA.PL
            - experiment_data/cached_asr/luna_techmo
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_luna/techmo_polish_asr
            - experiment_data/pipeline/asr_benchmark_luna/word_techmo_metrics_wer
            - experiment_data/pipeline/asr_benchmark_luna/word_techmo_alignment_wer
            - experiment_data/pipeline/asr_benchmark_luna/techmo_spacy
            - experiment_data/pipeline/asr_benchmark_luna/pos_techmo_alignment_wer
            - experiment_data/pipeline/asr_benchmark_luna/pos_techmo_metrics_wer
            - experiment_data/pipeline/asr_benchmark_luna/word_techmo_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_luna/word_techmo_alignment_wer_embeddings

    luna_ajn_processing:
        cmd: |
            PYTHONPATH=. python experiment/luna/pipeline/luna_ajn_asr_processing.py
        deps:
            - experiment/luna/pipeline/luna_ajn_asr_processing.py
            - experiment_data/dataset/LUNA.PL
            - experiment_data/cached_asr/luna_ajn_polish_asr
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_luna/ajn_polish_asr
            - experiment_data/pipeline/asr_benchmark_luna/word_ajn_metrics_wer
            - experiment_data/pipeline/asr_benchmark_luna/word_ajn_alignment_wer
            - experiment_data/pipeline/asr_benchmark_luna/ajn_spacy
            - experiment_data/pipeline/asr_benchmark_luna/pos_ajn_alignment_wer
            - experiment_data/pipeline/asr_benchmark_luna/pos_ajn_metrics_wer
            - experiment_data/pipeline/asr_benchmark_luna/word_ajn_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_luna/word_ajn_alignment_wer_embeddings

    luna_tag_spacy_processing:
        cmd: |
            PYTHONPATH=. python -u experiment/luna/pipeline/spacy_tag_processing.py
        deps:
            - experiment/luna/pipeline/spacy_tag_processing.py
            - experiment_data/dataset/LUNA.PL
            - experiment_data/pipeline/asr_benchmark_luna/ajn_spacy
            - experiment_data/pipeline/asr_benchmark_luna/gold_transcript_spacy
            - experiment_data/pipeline/asr_benchmark_luna/techmo_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_luna/tag_spacy_techmo_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_luna/tag_spacy_techmo_alignment_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_luna/tag_spacy_ajn_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_luna/tag_spacy_ajn_alignment_wer_embeddings


    voicelab_import_to_common_format:
        cmd: PYTHONPATH=. python -u experiment/voicelab/import_data.py
        deps:
            - experiment/voicelab/import_data.py
            - experiment_data/dataset/voicelab_cbiz_testset_20220322
        outs:
            - experiment_data/dataset_relation_manager_data/voicelab_cbiz_testset_20220322

    voicelab_gold_transcript_processing:
        cmd: |
            PYTHONPATH=. python -u experiment/voicelab/voicelab_pipeline_gold_transcript.py
        deps:
            - experiment/voicelab/voicelab_pipeline_gold_transcript.py
            - experiment_data/dataset_relation_manager_data/voicelab_cbiz_testset_20220322
            - experiment_data/dataset/voicelab_cbiz_testset_20220322
        outs:
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript_spacy

    voicelab_techmo_processing:
        cmd: |
            PYTHONPATH=. python -u experiment/voicelab/voicelab_pipeline_techmo.py
        deps:
            - experiment/voicelab/voicelab_pipeline_techmo.py
            - experiment_data/dataset/voicelab_cbiz_testset_20220322
            - experiment_data/cached_asr/voicelab_cbiz_testset_20220322_techmo
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/techmo_polish_asr
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_techmo_metrics_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_techmo_alignment_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/techmo_spacy
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/pos_techmo_alignment_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/pos_techmo_metrics_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_techmo_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_techmo_alignment_wer_embeddings

    voicelab_ajn_processing:
        cmd: |
            PYTHONPATH=. python -u experiment/voicelab/voicelab_pipeline_ajn_asr.py
        deps:
            - experiment/voicelab/voicelab_pipeline_ajn_asr.py
            - experiment_data/cached_asr/voicelab_cbiz_testset_20220322_ajn
            - experiment_data/dataset/voicelab_cbiz_testset_20220322
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/ajn_polish_asr
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_ajn_metrics_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_ajn_alignment_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/ajn_spacy
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/pos_ajn_alignment_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/pos_ajn_metrics_wer
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_ajn_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/word_ajn_alignment_wer_embeddings

    voicelab_tag_spacy_processing:
        cmd: |
            PYTHONPATH=. python -u experiment/voicelab/spacy_tag_processing.py
        deps:
            - experiment/voicelab/spacy_tag_processing.py
            - experiment_data/dataset/voicelab_cbiz_testset_20220322
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/ajn_spacy
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/gold_transcript_spacy
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/techmo_spacy
        outs:
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/tag_spacy_techmo_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/tag_spacy_techmo_alignment_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/tag_spacy_ajn_metrics_wer_embeddings
            - experiment_data/pipeline/asr_benchmark_voicelab_cbiz_testset_20220322/tag_spacy_ajn_alignment_wer_embeddings

# concurrent features, multiprocessing