pos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9febd313-2446-49bb-8508-997c0b2bc0ca",
   "metadata": {},
   "source": [
    "# Imports and configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1929f9bb-5060-4530-811b-823d69a5b00f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from experiment.luna.luna_record_provider import LunaRecordProvider\n",
    "from sziszapangma.integration.experiment_manager import ExperimentManager\n",
    "from sziszapangma.integration.repository.multi_files_experiment_repository import MultiFilesExperimentRepository\n",
    "from sziszapangma.integration.path_filter import ExtensionPathFilter\n",
    "from pymongo import MongoClient\n",
    "from spacy.tokens.doc import Doc\n",
    "import pandas as pd\n",
    "from experiment.voicelab.voicelab_telco_record_provider import VoicelabTelcoRecordProvider\n",
    "from sziszapangma.integration.repository.experiment_repository import ExperimentRepository\n",
    "from dataclasses import dataclass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "321a93d9-0c5d-4d42-ba8f-4b704a05d78c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b0a963f-ef3b-4f0f-bebc-68a0ea3e4f6c",
   "metadata": {},
   "source": [
    "# Load datasets and other pipeline objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbe2a7bf-bb6d-42ee-b5ce-48e6ec7fcd94",
   "metadata": {},
   "outputs": [],
   "source": [
    "VOICELAB_DATASET_DIRECTORY = 'experiment_data/dataset/voicelab_cbiz_testset_20220322'\n",
    "LUNA_DATASET_DIRECTORY = 'experiment_data/dataset/LUNA.PL'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4bb1a1c0-8784-4e0d-9426-13495718e087",
   "metadata": {},
   "outputs": [],
   "source": [
    "## repository collections\n",
    "GOLD_TRANSCRIPT = 'gold_transcript'\n",
    "GOLD_TRANSCRIPT_SPACY = 'gold_transcript_spacy'\n",
    "\n",
    "TECHMO_POLISH_ASR = 'techmo_polish_asr'\n",
    "WORD_TECHMO_MERTICS_WER = 'word_techmo_metrics_wer'\n",
    "WORD_TECHMO_ALIGNMENT_WER = 'word_techmo_alignment_wer'\n",
    "TECHMO_SPACY = 'techmo_spacy'\n",
    "POS_TECHMO_ALIGNMENT_WER = 'pos_techmo_alignment_wer'\n",
    "POS_TECHMO_METRICS_WER = 'pos_techmo_metrics_wer'\n",
    "\n",
    "AJN_POLISH_ASR = 'ajn_polish_asr'\n",
    "WORD_AJN_MERTICS_WER = 'word_ajn_metrics_wer'\n",
    "WORD_AJN_ALIGNMENT_WER = 'word_ajn_alignment_wer'\n",
    "AJN_SPACY = 'ajn_spacy'\n",
    "POS_AJN_ALIGNMENT_WER = 'pos_ajn_metrics_wer'\n",
    "POS_AJN_METRICS_WER = 'pos_ajn_alignment_wer'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d4265253-755a-4160-97f7-72604fdf41d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class CollectionsConfig:\n",
    "    config_name: str\n",
    "    gold_transcript: str\n",
    "    gold_transcript_spacy: str\n",
    "    asr: str\n",
    "    word_asr_metric_wer: str\n",
    "    word_asr_alignment_wer: str\n",
    "    asr_spacy: str\n",
    "    pos_asr_metric_wer: str\n",
    "    pos_asr_alignment_wer: str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "950b0bb8-e5ae-46e0-97a2-a832b7c8a70f",
   "metadata": {},
   "outputs": [],
   "source": [
    "techmo_connections_config = CollectionsConfig(\n",
    "    config_name='TECHMO ASR',\n",
    "    gold_transcript=GOLD_TRANSCRIPT,\n",
    "    gold_transcript_spacy=GOLD_TRANSCRIPT_SPACY,\n",
    "    asr=TECHMO_POLISH_ASR,\n",
    "    word_asr_metric_wer=WORD_TECHMO_MERTICS_WER,\n",
    "    word_asr_alignment_wer=WORD_TECHMO_ALIGNMENT_WER,\n",
    "    asr_spacy=TECHMO_SPACY,\n",
    "    pos_asr_metric_wer=POS_TECHMO_METRICS_WER,\n",
    "    pos_asr_alignment_wer=POS_TECHMO_ALIGNMENT_WER\n",
    ")\n",
    "ajn_connections_config = CollectionsConfig(\n",
    "    config_name='AJN ASR',\n",
    "    gold_transcript=GOLD_TRANSCRIPT,\n",
    "    gold_transcript_spacy=GOLD_TRANSCRIPT_SPACY,\n",
    "    asr=AJN_POLISH_ASR,\n",
    "    word_asr_metric_wer=WORD_AJN_MERTICS_WER,\n",
    "    word_asr_alignment_wer=WORD_AJN_ALIGNMENT_WER,\n",
    "    asr_spacy=AJN_SPACY,\n",
    "    pos_asr_metric_wer=POS_AJN_ALIGNMENT_WER,\n",
    "    pos_asr_alignment_wer=POS_AJN_METRICS_WER\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4dec626b-02e4-4c78-a238-04ef2f090ea5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voicelab examples count 800\n",
      "luna examples count 500\n"
     ]
    }
   ],
   "source": [
    "voicelab_experiment_repository = MultiFilesExperimentRepository(\n",
    "    'experiment_data/pipeline',\n",
    "    'asr_benchmark_voicelab_cbiz_testset_20220322'\n",
    ")\n",
    "luna_experiment_repository = MultiFilesExperimentRepository(\n",
    "    'experiment_data/pipeline',\n",
    "    'asr_benchmark_luna'\n",
    ")\n",
    "print(f'voicelab examples count {len(voicelab_experiment_repository.get_all_record_ids())}')\n",
    "print(f'luna examples count {len(luna_experiment_repository.get_all_record_ids())}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "98c6ff1d-4fbd-4b68-9e23-ecea33852b12",
   "metadata": {},
   "outputs": [],
   "source": [
    "voicelab_record_provider = VoicelabTelcoRecordProvider(ExtensionPathFilter(\n",
    "    root_directory=VOICELAB_DATASET_DIRECTORY,\n",
    "    extension='wav',\n",
    "), relation_manager_root_path='experiment_data/dataset_relation_manager_data/voicelab_cbiz_testset_20220322')\n",
    "\n",
    "luna_record_provider = LunaRecordProvider(ExtensionPathFilter(\n",
    "    root_directory=f'{LUNA_DATASET_DIRECTORY}/LUNA.PL',\n",
    "    extension='wav',\n",
    "), relation_manager_root_path='experiment_data/dataset_relation_manager_data/luna')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "74bf17ba-8572-4758-8116-794708b6ea08",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gold_transcript_for(record_id: str, experiment_repository: ExperimentRepository, \n",
    "                            collections_config: CollectionsConfig) -> str:\n",
    "    gold_trnascript_property = experiment_repository.get_property_for_key(\n",
    "        record_id, collections_config.gold_transcript)\n",
    "    return ' '.join([it['word'] for it in gold_trnascript_property])\n",
    "\n",
    "def get_asr_transcript_for(record_id: str, experiment_repository: ExperimentRepository,\n",
    "                           collections_config: CollectionsConfig) -> str:\n",
    "    gold_trnascript_property = experiment_repository.get_property_for_key(record_id, collections_config.asr)\n",
    "    return gold_trnascript_property['full_text']\n",
    "\n",
    "def get_word_alignment_df(record_id: str, experiment_repository: ExperimentRepository, \n",
    "                        collections_config: CollectionsConfig) -> pd.DataFrame:\n",
    "    word_alignment_wer = experiment_repository.get_property_for_key(record_id, \n",
    "                                                                    collections_config.word_asr_alignment_wer)\n",
    "    arr = [\n",
    "        {\n",
    "            'step_type': it['step_type'],\n",
    "            'reference_word_text': it['step_words']['reference_word']['text'] \n",
    "            if 'reference_word' in it['step_words'] else '',\n",
    "            'hypothesis_word_text': it['step_words']['hypothesis_word']['text']\n",
    "            if 'hypothesis_word' in it['step_words'] else '',\n",
    "        }\n",
    "        for it in word_alignment_wer\n",
    "    ]\n",
    "    return pd.DataFrame(arr)\n",
    "\n",
    "\n",
    "def get_pos_alignment_df(record_id: str, experiment_repository: ExperimentRepository, \n",
    "                        collections_config: CollectionsConfig) -> pd.DataFrame:\n",
    "    pos_alignment_wer = experiment_repository.get_property_for_key(record_id, \n",
    "                                                                   collections_config.pos_asr_alignment_wer)\n",
    "    gold_transcript_spacy = experiment_repository.get_property_for_key(record_id,\n",
    "                                                                       collections_config.gold_transcript_spacy)\n",
    "    gold_trnascript_spacy_word_dict = {it['id']: it['word'] for it in gold_transcript_spacy}\n",
    "    asr_spacy = experiment_repository.get_property_for_key(record_id, collections_config.asr_spacy)\n",
    "    asr_spacy_word_dict = {it['id']: it['word'] for it in asr_spacy}\n",
    "    arr = [\n",
    "        {\n",
    "            'step_type': it['step_type'],\n",
    "            'reference_word_pos': it['step_words']['reference_word']['text'] \n",
    "            if 'reference_word' in it['step_words'] else '',\n",
    "            'reference_word_text': gold_trnascript_spacy_word_dict[it['step_words']['reference_word']['id']] \n",
    "            if 'reference_word' in it['step_words'] else '',\n",
    "            'hypothesis_word_pos': it['step_words']['hypothesis_word']['text']\n",
    "            if 'hypothesis_word' in it['step_words'] else '',\n",
    "            'hypothesis_word_text': asr_spacy_word_dict[it['step_words']['hypothesis_word']['id']] \n",
    "            if 'hypothesis_word' in it['step_words'] else ''\n",
    "        }\n",
    "        for it in pos_alignment_wer\n",
    "    ]\n",
    "    return pd.DataFrame(arr)\n",
    "    \n",
    "    \n",
    "def show_report_for(record_id: str, experiment_repository: ExperimentRepository,\n",
    "                    collections_config: CollectionsConfig):\n",
    "    print('record_id:', record_id)\n",
    "    print('properties_confiog:', collections_config.config_name)\n",
    "    print()\n",
    "    print(f'gold transcript: {get_gold_transcript_for(record_id, experiment_repository, collections_config)}')    \n",
    "    print()\n",
    "    print(f'asr transcript: {get_asr_transcript_for(record_id, experiment_repository, collections_config)}')\n",
    "    print()\n",
    "    print(f\"word wer {experiment_repository.get_property_for_key(record_id, collections_config.word_asr_metric_wer)}\")\n",
    "    print()\n",
    "    print(f\"pos wer {experiment_repository.get_property_for_key(record_id, collections_config.pos_asr_metric_wer)}\")\n",
    "    print()\n",
    "    display(get_word_alignment_df(record_id, experiment_repository, collections_config))\n",
    "    print()\n",
    "    display(get_pos_alignment_df(record_id, experiment_repository, collections_config))\n",
    "    print('--------------------------------------------------------------')\n",
    "    print('--------------------------------------------------------------')\n",
    "    print('--------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "327bbc4b-7bcb-4a28-8d4a-22660ecaf6c2",
   "metadata": {},
   "source": [
    "# Report for LUNA and TECHMO ASR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3395e091-3488-4b49-aede-af15a0055a8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "record_id: LUNA.PL__KIEDY__DOBRAJAKOSC__F__2_2007-03-27_10_24_45__2_2007-03-27_10_24_45\n",
      "properties_confiog: TECHMO ASR\n",
      "\n",
      "gold transcript: xxx dzień dobry dzień dobry ja chciałam zapytać się o tramwaj trzydzieści dwa czy miał zmianę może rozkładu ? a już sprawdzę chwileczkę dziękuję trzydzieści dwa tak ma zmieniony a mam taką prośbę mógłby pan podać mi tramwaje niskopokładowe w jakich godzinach teraz jeżdżą a skąd ? z Czynszowej z pętli z Pragi wszystkie w ciągu dnia ? to znaczy nie może od godziny dwunastej pan mi podyktuje na dwunasta trzydzieści trzy tak czternasta zero trzy tak i jeszcze piętnasta dwadzieścia trzy piętnasta dwadzieścia trzy dziękuję ślicznie do widzenia proszę bardzo do widzenia\n",
      "\n",
      "asr transcript: dzień dobry dzień dobry ja chciałam zapytać się o tramwaj 32 czy miał zmianę może rozkładu a już sprawdzę chwileczkę 32 tak ma zmieniony a mam taką prośbę mógłby pan podać mi tylko pogodowe w jakich godzinach teraz je archont czynszowej z pętli z pragi wszystkie w ciągu dnia to znaczy nie może od godziny dwunastej pan mi dwunasta 33 tak czternasta 0 3 tak i jeszcze piętnasta 23 piętnasta 23 dziękuję do widzenia\n",
      "\n",
      "word wer {'classic_wer': 0.34408602150537637}\n",
      "\n",
      "pos wer {'classic_wer': 0.3118279569892473}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step_type</th>\n",
       "      <th>reference_word_text</th>\n",
       "      <th>hypothesis_word_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>xxx</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dzień</td>\n",
       "      <td>dzień</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dobry</td>\n",
       "      <td>dobry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dzień</td>\n",
       "      <td>dzień</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dobry</td>\n",
       "      <td>dobry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>ja</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>chciałam</td>\n",
       "      <td>chciałam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>zapytać</td>\n",
       "      <td>zapytać</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>się</td>\n",
       "      <td>się</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>o</td>\n",
       "      <td>o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>tramwaj</td>\n",
       "      <td>tramwaj</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>trzydzieści</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>dwa</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>czy</td>\n",
       "      <td>czy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>miał</td>\n",
       "      <td>miał</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>zmianę</td>\n",
       "      <td>zmianę</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>może</td>\n",
       "      <td>może</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>rozkładu</td>\n",
       "      <td>rozkładu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>?</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>już</td>\n",
       "      <td>już</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>sprawdzę</td>\n",
       "      <td>sprawdzę</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>chwileczkę</td>\n",
       "      <td>chwileczkę</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>dziękuję</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>trzydzieści</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>dwa</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>tak</td>\n",
       "      <td>tak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>ma</td>\n",
       "      <td>ma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>zmieniony</td>\n",
       "      <td>zmieniony</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>mam</td>\n",
       "      <td>mam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>taką</td>\n",
       "      <td>taką</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>prośbę</td>\n",
       "      <td>prośbę</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>mógłby</td>\n",
       "      <td>mógłby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>pan</td>\n",
       "      <td>pan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>podać</td>\n",
       "      <td>podać</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>mi</td>\n",
       "      <td>mi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>tramwaje</td>\n",
       "      <td>tylko</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>niskopokładowe</td>\n",
       "      <td>pogodowe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>jakich</td>\n",
       "      <td>jakich</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>godzinach</td>\n",
       "      <td>godzinach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>teraz</td>\n",
       "      <td>teraz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>jeżdżą</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>a</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>skąd</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>?</td>\n",
       "      <td>je</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>z</td>\n",
       "      <td>archont</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>Czynszowej</td>\n",
       "      <td>czynszowej</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>z</td>\n",
       "      <td>z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>pętli</td>\n",
       "      <td>pętli</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>z</td>\n",
       "      <td>z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>Pragi</td>\n",
       "      <td>pragi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>wszystkie</td>\n",
       "      <td>wszystkie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>ciągu</td>\n",
       "      <td>ciągu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dnia</td>\n",
       "      <td>dnia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>?</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>to</td>\n",
       "      <td>to</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>znaczy</td>\n",
       "      <td>znaczy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>nie</td>\n",
       "      <td>nie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>może</td>\n",
       "      <td>może</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>od</td>\n",
       "      <td>od</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>godziny</td>\n",
       "      <td>godziny</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dwunastej</td>\n",
       "      <td>dwunastej</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>pan</td>\n",
       "      <td>pan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>mi</td>\n",
       "      <td>mi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>podyktuje</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>na</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dwunasta</td>\n",
       "      <td>dwunasta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>trzydzieści</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>trzy</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>tak</td>\n",
       "      <td>tak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>czternasta</td>\n",
       "      <td>czternasta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>zero</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>trzy</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>tak</td>\n",
       "      <td>tak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>i</td>\n",
       "      <td>i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>jeszcze</td>\n",
       "      <td>jeszcze</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>piętnasta</td>\n",
       "      <td>piętnasta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>dwadzieścia</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>trzy</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>piętnasta</td>\n",
       "      <td>piętnasta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>dwadzieścia</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>SUBSTITUTION</td>\n",
       "      <td>trzy</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>dziękuję</td>\n",
       "      <td>dziękuję</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>ślicznie</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>do</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>widzenia</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>proszę</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>DELETION</td>\n",
       "      <td>bardzo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>do</td>\n",
       "      <td>do</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>CORRECT</td>\n",
       "      <td>widzenia</td>\n",
       "      <td>widzenia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       step_type reference_word_text hypothesis_word_text\n",
       "0       DELETION                 xxx                     \n",
       "1        CORRECT               dzień                dzień\n",
       "2        CORRECT               dobry                dobry\n",
       "3        CORRECT               dzień                dzień\n",
       "4        CORRECT               dobry                dobry\n",
       "5        CORRECT                  ja                   ja\n",
       "6        CORRECT            chciałam             chciałam\n",
       "7        CORRECT             zapytać              zapytać\n",
       "8        CORRECT                 się                  się\n",
       "9        CORRECT                   o                    o\n",
       "10       CORRECT             tramwaj              tramwaj\n",
       "11      DELETION         trzydzieści                     \n",
       "12  SUBSTITUTION                 dwa                   32\n",
       "13       CORRECT                 czy                  czy\n",
       "14       CORRECT                miał                 miał\n",
       "15       CORRECT              zmianę               zmianę\n",
       "16       CORRECT                może                 może\n",
       "17       CORRECT            rozkładu             rozkładu\n",
       "18      DELETION                   ?                     \n",
       "19       CORRECT                   a                    a\n",
       "20       CORRECT                 już                  już\n",
       "21       CORRECT            sprawdzę             sprawdzę\n",
       "22       CORRECT          chwileczkę           chwileczkę\n",
       "23      DELETION            dziękuję                     \n",
       "24      DELETION         trzydzieści                     \n",
       "25  SUBSTITUTION                 dwa                   32\n",
       "26       CORRECT                 tak                  tak\n",
       "27       CORRECT                  ma                   ma\n",
       "28       CORRECT           zmieniony            zmieniony\n",
       "29       CORRECT                   a                    a\n",
       "30       CORRECT                 mam                  mam\n",
       "31       CORRECT                taką                 taką\n",
       "32       CORRECT              prośbę               prośbę\n",
       "33       CORRECT              mógłby               mógłby\n",
       "34       CORRECT                 pan                  pan\n",
       "35       CORRECT               podać                podać\n",
       "36       CORRECT                  mi                   mi\n",
       "37  SUBSTITUTION            tramwaje                tylko\n",
       "38  SUBSTITUTION      niskopokładowe             pogodowe\n",
       "39       CORRECT                   w                    w\n",
       "40       CORRECT              jakich               jakich\n",
       "41       CORRECT           godzinach            godzinach\n",
       "42       CORRECT               teraz                teraz\n",
       "43      DELETION              jeżdżą                     \n",
       "44      DELETION                   a                     \n",
       "45      DELETION                skąd                     \n",
       "46  SUBSTITUTION                   ?                   je\n",
       "47  SUBSTITUTION                   z              archont\n",
       "48  SUBSTITUTION          Czynszowej           czynszowej\n",
       "49       CORRECT                   z                    z\n",
       "50       CORRECT               pętli                pętli\n",
       "51       CORRECT                   z                    z\n",
       "52  SUBSTITUTION               Pragi                pragi\n",
       "53       CORRECT           wszystkie            wszystkie\n",
       "54       CORRECT                   w                    w\n",
       "55       CORRECT               ciągu                ciągu\n",
       "56       CORRECT                dnia                 dnia\n",
       "57      DELETION                   ?                     \n",
       "58       CORRECT                  to                   to\n",
       "59       CORRECT              znaczy               znaczy\n",
       "60       CORRECT                 nie                  nie\n",
       "61       CORRECT                może                 może\n",
       "62       CORRECT                  od                   od\n",
       "63       CORRECT             godziny              godziny\n",
       "64       CORRECT           dwunastej            dwunastej\n",
       "65       CORRECT                 pan                  pan\n",
       "66       CORRECT                  mi                   mi\n",
       "67      DELETION           podyktuje                     \n",
       "68      DELETION                  na                     \n",
       "69       CORRECT            dwunasta             dwunasta\n",
       "70      DELETION         trzydzieści                     \n",
       "71  SUBSTITUTION                trzy                   33\n",
       "72       CORRECT                 tak                  tak\n",
       "73       CORRECT          czternasta           czternasta\n",
       "74  SUBSTITUTION                zero                    0\n",
       "75  SUBSTITUTION                trzy                    3\n",
       "76       CORRECT                 tak                  tak\n",
       "77       CORRECT                   i                    i\n",
       "78       CORRECT             jeszcze              jeszcze\n",
       "79       CORRECT           piętnasta            piętnasta\n",
       "80      DELETION         dwadzieścia                     \n",
       "81  SUBSTITUTION                trzy                   23\n",
       "82       CORRECT           piętnasta            piętnasta\n",
       "83      DELETION         dwadzieścia                     \n",
       "84  SUBSTITUTION                trzy                   23\n",
       "85       CORRECT            dziękuję             dziękuję\n",
       "86      DELETION            ślicznie                     \n",
       "87      DELETION                  do                     \n",
       "88      DELETION            widzenia                     \n",
       "89      DELETION              proszę                     \n",
       "90      DELETION              bardzo                     \n",
       "91       CORRECT                  do                   do\n",
       "92       CORRECT            widzenia             widzenia"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",