diff --git a/README.md b/README.md index 1dfaba45d3ca829e7bce3877eec73ac4e91abbf5..6e1e306c160a2dc75fb4c26745f81ddc268c649f 100644 --- a/README.md +++ b/README.md @@ -38,3 +38,4 @@ We encourage you to use the [beginner's tutorial](https://colab.research.google. - [**Pre-trained models**](docs/models.md) - [**Training**](docs/training.md) - [**Prediction**](docs/prediction.md) +- [**Model performance**](docs/performance.md) diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000000000000000000000000000000000000..31b0628c3193251bff2b6908299b20c67fcbecbd --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,55 @@ +# Model performance + +The models were trained with COMBO boosted with FastText embeddings and BERT embeddings (english-bert-base, polish-herbert-base, and polish-herbert-large). + +|Treebank |Model name |UPOS |XPOS |UFeats|AllTags|Lemmas|UAS |LAS |CLAS |MLAS |BLEX |Language model |LICENSE | +|-------------------------|--------------------|-----|------|------|-------|------|-----|-----|-----|-----|-----|---------------|----------------------------------------------------------------------------------------| +|UD_English-EWT |[english-bert-base](http://mozart.ipipan.waw.pl/~mklimaszewski/models/english-bert-base.tar.gz) |96.53|96.41 |97.34 |95.04 |97.72 |92.17|89.75|87.25|82.14|84.65|bert-case-cased|https://github.com/UniversalDependencies/UD_English-EWT/blob/r2.7/LICENSE.txt | +|UD_Polish_PDB |[polish-herbert-base](http://mozart.ipipan.waw.pl/~mklimaszewski/models/english-bert-base.tar.gz) |98.93|96.42 |96.79 |95.73 |98.08 |95.53|93.81|92.30|87.44|89.86|herbert-base |https://github.com/UniversalDependencies/UD_Polish-PDB/blob/r2.7/LICENSE.txt | +|UD_Polish_PDB |[polish-herbert-large](http://mozart.ipipan.waw.pl/~mklimaszewski/models/polish-herbert-large.tar.gz)|99.04|96.65 |96.97 |95.95 |98.13 |95.48|93.84|92.31|87.84|90.04|herbert-large |https://github.com/UniversalDependencies/UD_Polish-PDB/blob/r2.7/LICENSE.txt | +|UD_Afrikaans-AfriBooms |[afrikaans-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/afrikaans-ud27.tar.gz) |97.13|91.79 |96.54 |91.32 |96.78 |87.43|84.30|78.08|72.56|74.06|fastText |https://github.com/UniversalDependencies/UD_Afrikaans-AfriBooms/blob/r2.7/LICENSE.txt | +|UD_Armenian-ArmTDP |[armenian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/armenian-ud27.tar.gz) |94.23|100.00|88.04 |85.99 |93.88 |82.77|76.04|70.08|58.26|65.62|fastText |https://github.com/UniversalDependencies/UD_Armenian-ArmTDP/blob/r2.7/LICENSE.txt | +|UD_Basque-BDT |[basque-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/basque-ud27.tar.gz) |94.28|100.00|90.44 |87.52 |95.47 |84.64|80.44|78.82|67.33|74.95|fastText |https://github.com/UniversalDependencies/UD_Basque-BDT/blob/r2.7/LICENSE.txt | +|UD_Belarusian-HSE |[belarusian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/belarusian-ud27.tar.gz) |98.13|96.33 |91.06 |89.24 |92.88 |83.91|79.84|75.87|67.39|68.81|fastText |https://github.com/UniversalDependencies/UD_Belarusian-HSE/blob/r2.7/LICENSE.txt | +|UD_Bulgarian-BTB |[bulgarian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/bulgarian-ud27.tar.gz) |98.84|95.24 |97.33 |94.73 |97.44 |93.36|90.05|86.55|82.58|83.22|fastText |https://github.com/UniversalDependencies/UD_Bulgarian-BTB/blob/r2.7/LICENSE.txt | +|UD_Catalan-AnCora |[catalan-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/catalan-ud27.tar.gz) |98.68|98.60 |98.23 |97.63 |99.13 |92.92|90.62|86.35|83.81|85.67|fastText |https://github.com/UniversalDependencies/UD_Catalan-AnCora/blob/r2.7/LICENSE.txt | +|UD_Chinese-GSD |[chinese-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/chinese-ud27.tar.gz) |94.77|94.55 |99.18 |93.56 |98.93 |82.80|78.75|77.24|71.42|76.03|fastText |https://github.com/UniversalDependencies/UD_Chinese-GSD/blob/r2.7/LICENSE.txt | +|UD_Chinese-GSDSimp |[chinese-simp-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/chinese-simp-ud27.tar.gz) |94.86|94.70 |99.16 |93.70 |98.98 |83.47|79.50|77.93|72.54|76.70|fastText |https://github.com/UniversalDependencies/UD_Chinese-GSDSimp/blob/r2.7/LICENSE.txt | +|UD_Croatian-SET |[croatian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/croatian-ud27.tar.gz) |97.48|94.09 |94.67 |93.06 |96.85 |89.45|85.46|82.34|75.32|78.86|fastText |https://github.com/UniversalDependencies/UD_Croatian-SET/blob/r2.7/LICENSE.txt | +|UD_Czech-PDT |[czech-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/czech-ud27.tar.gz) |99.01|96.60 |96.77 |95.78 |98.64 |93.51|91.67|90.18|86.24|88.74|fastText |https://github.com/UniversalDependencies/UD_Czech-PDT/blob/r2.7/LICENSE.txt | +|UD_Danish-DDT |[danish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/danish-ud27.tar.gz) |97.08|100.00|96.20 |95.15 |96.89 |85.41|82.37|78.83|72.63|75.76|fastText |https://github.com/UniversalDependencies/UD_Danish-DDT/blob/r2.7/LICENSE.txt | +|UD_Dutch-Alpino |[dutch-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/dutch-ud27.tar.gz) |96.22|94.13 |95.97 |93.28 |95.78 |91.48|87.91|82.48|75.92|77.34|fastText |https://github.com/UniversalDependencies/UD_Dutch-Alpino/blob/r2.7/LICENSE.txt | +|UD_English-EWT |[english-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/english-ud27.tar.gz) |95.76|95.39 |96.59 |93.89 |97.63 |88.85|86.00|82.63|77.00|80.20|fastText |https://github.com/UniversalDependencies/UD_English-EWT/blob/r2.7/LICENSE.txt | +|UD_Estonian-EWT |[estonian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/estonian-ud27.tar.gz) |90.61|93.62 |87.91 |83.79 |81.63 |74.98|68.33|63.71|53.27|49.69|fastText |https://github.com/UniversalDependencies/UD_Estonian-EWT/blob/r2.7/LICENSE.txt | +|UD_Finnish-TDT |[finnish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/finnish-ud27.tar.gz) |95.89|97.98 |93.60 |92.24 |88.53 |89.43|86.46|84.32|77.06|72.07|fastText |https://github.com/UniversalDependencies/UD_Finnish-TDT/blob/r2.7/LICENSE.txt | +|UD_French-GSD |[french-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/french-ud27.tar.gz) |97.88|100.00|97.97 |97.07 |98.35 |92.70|90.67|86.19|82.54|84.06|fastText |https://github.com/UniversalDependencies/UD_French-GSD/blob/r2.7/LICENSE.txt | +|UD_Galician-CTG |[galician-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/galician-ud27.tar.gz) |97.22|95.62 |99.81 |95.17 |98.31 |85.44|82.41|76.72|69.65|75.10|fastText |https://github.com/UniversalDependencies/UD_Galician-CTG/blob/r2.7/LICENSE.txt | +|UD_German-GSD |[german-gsd-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/german-gsd-ud27.tar.gz) |94.26|96.62 |90.16 |84.68 |96.46 |86.78|82.01|76.78|59.23|72.22|fastText |https://github.com/UniversalDependencies/UD_German-GSD/blob/r2.7/LICENSE.txt | +|UD_German-HDT |[german-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/german-ud27.tar.gz) |98.39|98.35 |93.52 |93.07 |93.33 |96.90|95.86|93.60|83.53|83.60|fastText |https://github.com/UniversalDependencies/UD_German-HDT/blob/r2.7/LICENSE.txt | +|UD_Greek-GDT |[greek-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/greek-ud27.tar.gz) |97.34|97.37 |94.31 |93.45 |95.73 |91.19|88.78|84.30|76.26|78.58|fastText |https://github.com/UniversalDependencies/UD_Greek-GDT/blob/r2.7/LICENSE.txt | +|UD_Hebrew-HTB |[hebrew-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/hebrew-ud27.tar.gz) |96.94|96.91 |95.69 |94.72 |96.18 |89.21|86.07|80.27|73.70|75.56|fastText |https://github.com/UniversalDependencies/UD_Hebrew-HTB/blob/r2.7/LICENSE.txt | +|UD_Hindi-HDTB |[hindi-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/hindi-ud27.tar.gz) |97.38|96.93 |93.99 |91.81 |98.46 |94.59|91.39|87.59|77.73|86.12|fastText |https://github.com/UniversalDependencies/UD_Hindi-HDTB/blob/r2.7/LICENSE.txt | +|UD_Hungarian-Szeged |[hungarian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/hungarian-ud27.tar.gz) |92.96|100.00|88.90 |86.37 |92.30 |80.91|75.29|73.44|59.11|66.32|fastText |https://github.com/UniversalDependencies/UD_Hungarian-Szeged/blob/r2.7/LICENSE.txt | +|UD_Icelandic-IcePaHC |[icelandic-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/icelandic-ud27.tar.gz) |96.50|92.07 |90.44 |85.24 |95.54 |87.16|83.02|77.58|64.71|73.51|fastText |https://github.com/UniversalDependencies/UD_Icelandic-IcePaHC/blob/r2.7/LICENSE.txt | +|UD_Indonesian-GSD |[indonesian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/indonesian-ud27.tar.gz) |93.46|91.83 |95.51 |86.40 |98.93 |84.79|77.97|75.74|66.72|74.60|fastText |https://github.com/UniversalDependencies/UD_Indonesian-GSD/blob/r2.7/LICENSE.txt | +|UD_Irish-IDT |[irish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/irish-ud27.tar.gz) |93.81|93.46 |73.25 |69.84 |90.77 |84.01|76.89|70.53|40.29|61.57|fastText |https://github.com/UniversalDependencies/UD_Irish-IDT/blob/r2.7/LICENSE.txt | +|UD_Italian-ISDT |[italian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/italian-ud27.tar.gz) |98.11|97.96 |97.78 |97.02 |98.29 |93.13|91.02|86.76|83.25|84.65|fastText |https://github.com/UniversalDependencies/UD_Italian-ISDT/blob/r2.7/LICENSE.txt | +|UD_Maltese-MUDT |[maltese-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/maltese-ud27.tar.gz) |92.74|90.77 |100.00|90.01 |100.00|81.01|74.37|65.18|58.71|65.18|fastText |https://github.com/UniversalDependencies/UD_Maltese-MUDT/blob/r2.7/LICENSE.txt | +|UD_Norwegian-Bokmaal |[norwegian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/norwegian-ud27.tar.gz) |97.50|100.00|95.98 |95.19 |98.04 |91.95|89.80|87.04|81.27|84.71|fastText |https://github.com/UniversalDependencies/UD_Norwegian-Bokmaal/blob/r2.7/LICENSE.txt | +|UD_Persian-PerDT |[persian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/persian-ud27.tar.gz) |97.20|97.21 |97.72 |95.14 |98.84 |93.41|90.95|88.91|84.97|87.57|fastText |https://github.com/UniversalDependencies/UD_Persian-PerDT/blob/r2.7/LICENSE.txt | +|UD_Polish-PDB |[polish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/polish-ud27.tar.gz) |98.20|94.26 |94.50 |93.23 |97.48 |92.83|90.44|88.07|81.29|85.54|fastText |https://github.com/UniversalDependencies/UD_Polish-PDB/blob/r2.7/LICENSE.txt | +|UD_Portuguese-GSD |[portuguese-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/portuguese-ud27.tar.gz) |98.06|98.08 |99.86 |97.97 |99.28 |92.72|91.15|86.93|84.76|86.15|fastText |https://github.com/UniversalDependencies/UD_Portuguese-GSD/blob/r2.7/LICENSE.txt | +|UD_Romanian-RRT |[romanian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/romanian-ud27.tar.gz) |97.66|96.83 |96.92 |96.33 |98.32 |90.55|85.90|81.50|77.14|80.02|fastText |https://github.com/UniversalDependencies/UD_Romanian-RRT/blob/r2.7/LICENSE.txt | +|UD_Russian-Taiga |[russian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/russian-ud27.tar.gz) |92.98|97.41 |88.42 |83.46 |92.42 |78.28|72.72|69.31|56.22|62.93|fastText |https://github.com/UniversalDependencies/UD_Russian-Taiga/blob/r2.7/LICENSE.txt | +|UD_Scottish_Gaelic-ARCOSG|[scottish_gaelic-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/scottish_gaelic-ud27.tar.gz)|94.11|87.39 |89.66 |85.41 |94.10 |84.35|78.60|72.35|62.43|67.05|fastText |https://github.com/UniversalDependencies/UD_Scottish_Gaelic-ARCOSG/blob/r2.7/LICENSE.txt| +|UD_Serbian-SET |[serbian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/serbian-ud27.tar.gz) |97.94|93.51 |93.72 |92.63 |96.71 |90.61|87.38|84.70|77.06|81.33|fastText |https://github.com/UniversalDependencies/UD_Serbian-SET/blob/r2.7/LICENSE.txt | +|UD_Slovak-SNK |[slovak-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/slovak-ud27.tar.gz) |94.94|85.37 |90.24 |84.00 |95.30 |89.98|86.90|84.72|72.26|80.10|fastText |https://github.com/UniversalDependencies/UD_Slovak-SNK/blob/r2.7/LICENSE.txt | +|UD_Slovenian-SSJ |[slovenian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/slovenian-ud27.tar.gz) |98.12|94.85 |95.18 |94.00 |97.79 |91.90|90.03|87.33|81.47|85.06|fastText |https://github.com/UniversalDependencies/UD_Slovenian-SSJ/blob/r2.7/LICENSE.txt | +|UD_Spanish-AnCora |[spanish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/spanish-ud27.tar.gz) |98.83|98.72 |98.56 |97.88 |99.34 |92.38|90.12|86.13|83.81|85.55|fastText |https://github.com/UniversalDependencies/UD_Spanish-AnCora/blob/r2.7/LICENSE.txt | +|UD_Swedish-LinES |[swedish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/swedish-ud27.tar.gz) |96.28|93.23 |89.17 |85.43 |96.86 |87.38|83.05|79.42|65.18|76.39|fastText |https://github.com/UniversalDependencies/UD_Swedish-LinES/blob/r2.7/LICENSE.txt | +|UD_Telugu-MTG |[telugu-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/telugu-ud27.tar.gz) |91.82|100.00|100.00|91.82 |100.00|91.40|81.83|77.36|72.78|77.36|fastText |https://github.com/UniversalDependencies/UD_Telugu-MTG/blob/r2.7/LICENSE.txt | +|UD_Turkish-BOUN |[turkish-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/turkish-ud27.tar.gz) |90.35|91.01 |92.83 |84.29 |95.25 |78.12|70.91|68.32|55.74|64.25|fastText |https://github.com/UniversalDependencies/UD_Turkish-BOUN/blob/r2.7/LICENSE.txt | +|UD_Ukrainian-IU |[ukrainian-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/ukrainian-ud27.tar.gz) |96.24|91.09 |91.15 |89.47 |96.58 |86.11|82.76|78.86|69.72|75.78|fastText |https://github.com/UniversalDependencies/UD_Ukrainian-IU/blob/r2.7/LICENSE.txt | +|UD_Urdu-UDTB |[urdu-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/urdu-ud27.tar.gz) |93.51|91.46 |82.80 |78.06 |96.27 |86.76|80.65|74.35|54.93|70.82|fastText |https://github.com/UniversalDependencies/UD_Urdu-UDTB/blob/r2.7/LICENSE.txt | +|UD_Uyghur-UDT |[uyghur-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/uyghur-ud27.tar.gz) |86.47|89.22 |85.15 |73.81 |94.37 |76.10|63.67|56.90|40.30|52.76|fastText |https://github.com/UniversalDependencies/UD_Uyghur-UDT/blob/r2.7/LICENSE.txt | +|UD_Vietnamese-VTB |[vietnamese-ud27](http://mozart.ipipan.waw.pl/~mklimaszewski/models/vietnamese-ud27.tar.gz) |86.64|82.09 |99.35 |81.51 |99.87 |66.19|56.65|53.44|47.44|53.35|fastText |https://github.com/UniversalDependencies/UD_Vietnamese-VTB/blob/r2.7/LICENSE.txt |