sudo apt-get install python3-pip python3-dev python-virtualenv
sudo pip install -U pip
virtualenv -p python3.6 venv
source venv/bin/activate
pip install -U pip
pip install -r requirements.txt

python -m spacy download pl_core_news_sm

python -m pip install pl_core_news_sm-2.3.0.tar.gz

docker run -d -p 9003:9003 -it djstrong/krnnt:1.0.0

mkdir models/roberta_base_fairseq -p
wget https://github.com/sdadas/polish-roberta/releases/download/models/roberta_base_fairseq.zip
unzip roberta_base_fairseq.zip -d models/roberta_base_fairseq
rm roberta_base_fairseq.zip

mkdir models/roberta_large_fairseq -p
wget https://github.com/sdadas/polish-roberta/releases/download/models/roberta_large_fairseq.zip
unzip roberta_large_fairseq.zip -d models/roberta_large_fairseq
rm roberta_large_fairseq.zip

wget "https://minio.clarin-pl.eu/public/models/poldeepner2/nkjp_base_sq.zip" -O models/nkjp_base_sq.zip
unzip models/nkjp_base_sq.zip -d models 

wget http://mozart.ipipan.waw.pl/~axw/poleval2018/POLEVAL-NER_GOLD.json -O data/POLEVAL-NER_GOLD.json

time python process_poleval_pretokenized.py \
  --input data/poleval2018ner-data/index.list \
  --output poleval2018-predictions-pretokenized.json \
  --pretrained_path models/roberta_base_fairseq \
  --model models/nkjp_base \
  --max_seq_length 256 \
  --device cuda:0

python poleval_ner_test.py \
  --goldfile data/POLEVAL-NER_GOLD.json \
  --userfile poleval2018-predictions-pretokenized.json

time python process_poleval.py \
  --input data/POLEVAL-NER_GOLD.json \
  --output poleval2018-predictions-spacy.json \
  --pretrained_path models/roberta_base_fairseq \
  --model models/nkjp_base \
  --max_seq_length 256 \
  --tokenization spacy-ext \
  --device cuda:0

python poleval_ner_test.py \
  --goldfile data/POLEVAL-NER_GOLD.json \
  --userfile poleval2018-predictions-spacy.json

time python process_poleval.py \
  --input data/POLEVAL-NER_GOLD.json \
  --output poleval2018-predictions-spacy-sq.json \
  --pretrained_path models/roberta_base_fairseq \
  --model models/nkjp_base_sq \
  --max_seq_length 256 \
  --tokenization spacy-ext \
  --squeeze \
  --device cuda:0

python poleval_ner_test.py \
  --goldfile data/POLEVAL-NER_GOLD.json \
  --userfile poleval2018-predictions-spacy-sq.json

python poleval_ner_test_v2.py \
  --goldfile data/POLEVAL-NER_GOLD.json \
  --userfile poleval2018-predictions-spacy-sq.json \
  --categories-main

python sample.py

--------------------
Marek Nowak z Politechniki Wrocławskiej mieszka przy ul. Sądeckiej.
0:11     nam_liv_person       Marek Nowak
14:39    nam_org_organization Politechniki Wrocławskiej
57:66    nam_fac_road         Sądeckiej
--------------------
#PoselAdamNowak Co Pan myśli na temat fuzji Orlenu i Lotosu?
6:15     nam_liv_person       AdamNowak
44:50    nam_org_group_team   Orlenu
53:59    nam_org_group_team   Lotosu

python server.py \
   --pretrained_path models/roberta_base_fairseq \
   --model models/nkjp_base_sq/ \
   --tokenization spacy-ext \
   --device cuda:0 \
   --squeeze

curl -XPOST localhost:8000/predict -d "Marek Nowak z Politechniki Wrocławskiej mieszka przy ul. Sądeckiej."

{
  "entities": [
    {
      "begin": 0, 
      "end": 11, 
      "label": "persName", 
      "text": "Marek Nowak"
    }, 
    {
      "begin": 0, 
      "end": 5, 
      "label": "persName_forename", 
      "text": "Marek"
    }, 
    {
      "begin": 6, 
      "end": 11, 
      "label": "persName_surname", 
      "text": "Nowak"
    }, 
    {
      "begin": 14, 
      "end": 39, 
      "label": "orgName", 
      "text": "Politechniki Wroc\u0142awskiej"
    }, 
    {
      "begin": 27, 
      "end": 39, 
      "label": "placeName_settlement", 
      "text": "Wroc\u0142awskiej"
    }, 
    {
      "begin": 53, 
      "end": 67, 
      "label": "geogName", 
      "text": "ul. S\u0105deckiej."
    }
  ], 
  "text": "Marek Nowak z Politechniki Wroc\u0142awskiej mieszka przy ul. S\u0105deckiej."
}

time python main.py  \
      --data_dir=data/kpwr_n82/  \
      --task_name=ner \
      --output_dir=models/kpwr_n82_base/   \
      --max_seq_length=128   \
      --num_train_epochs 30  \
      --do_eval \
      --warmup_proportion=0.0 \
      --pretrained_path models/roberta_base_fairseq \
      --learning_rate 6e-5 \
      --gradient_accumulation_steps 4 \
      --do_train \
      --eval_on test \
      --train_batch_size 32 \
      --dropout 0.3

time python main.py  \
      --data_dir=data/cen_n82/  \
      --task_name=ner \
      --output_dir=models/cen_n82_large/   \
      --max_seq_length=512   \
      --num_train_epochs 30  \
      --do_eval \
      --warmup_proportion=0.0 \
      --pretrained_path models/roberta_large_fairseq \
      --learning_rate 6e-5 \
      --gradient_accumulation_steps 4 \
      --do_train \
      --eval_on test \
      --train_batch_size 32 \
      --dropout 0.3

time python main.py  \
      --data_dir=data/nkjp-nested-full-aug/  \
      --task_name=ner \
      --output_dir=models/nkjp_base_sq/   \
      --max_seq_length=256   \
      --num_train_epochs 10  \
      --do_eval \
      --warmup_proportion=0.0 \
      --pretrained_path models/roberta_base_fairseq \
      --learning_rate 6e-5 \
      --gradient_accumulation_steps 4 \
      --do_train \
      --eval_on test \
      --train_batch_size 32 \
      --dropout 0.3 \
      --squeeze

time python main.py  \
      --data_dir=data/nkjp-nested-full-aug/  \
      --task_name=ner \
      --output_dir=models/nkjp_base_sq/   \
      --max_seq_length=256   \
      --num_train_epochs 10  \
      --do_eval \
      --warmup_proportion=0.0 \
      --pretrained_path models/roberta_base_fairseq \
      --learning_rate 6e-5 \
      --gradient_accumulation_steps 4 \
      --do_train \
      --eval_on test \
      --train_batch_size 32 \
      --dropout 0.3 \
      --model=Herbert \
      --squeeze