Merge branch 'gitlab-registry' into 'master'

Gitlab registry

See merge request !7

.gitlab-ci.yml

@@ -20,13 +20,12 @@ build_image:
   image: 'docker:18.09.7'
   only:
     - master
+    - gitlab-registry
   services:
     - 'docker:18.09.7-dind'
   before_script:
     - ''
   script:
-    - docker build -t clarinpl/plwordnet .
-    - echo $DOCKER_PASSWORD > pass.txt
-    - cat pass.txt | docker login --username $DOCKER_USERNAME --password-stdin
-    - rm pass.txt
+    - docker build -t $CI_REGISTRY_IMAGE .
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker push $CI_REGISTRY_IMAGE
\ No newline at end of file
-    - docker push clarinpl/plwordnet

Dockerfile

@@ -4,11 +4,7 @@ WORKDIR /home/worker
 COPY ./src ./src
 COPY ./main.py .
 COPY ./requirements.txt .
-COPY ./entrypoint.sh /
+COPY ./entrypoint.py .
 COPY ./info.json .
-
 RUN python3.6 -m pip install -r requirements.txt
-RUN ["chmod", "+x", "/entrypoint.sh"]
-
-#CMD ["python3.6", "main.py", "service"]
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["python3.6", "./entrypoint.py"]

entrypoint.py

+#!/usr/bin/python3
+from subprocess import run
+
+
+s3_endpoint = "https://s3.clarin-pl.eu"
+s3_location = "s3://workers/plwordnet"
+local_models_location = "/home/worker/model"
+
+main_cmd = \
+    'aws --no-sign-request --endpoint-url "{endpoint}" s3 sync --delete'.format(
+        endpoint=s3_endpoint
+    )
+location_params = \
+    ' "{loc}" "{local_loc}"'.format(
+        loc=s3_location,
+        local_loc=local_models_location
+    )
+cmd = main_cmd + location_params
+
+run(cmd, shell=True)
+run(["python3.6", "main.py", "service"])

entrypoint.sh

-#!/bin/sh
-cd /home/worker
-if [ ! -d "model" ]; then
-  mkdir model
-fi
-if [ -z "$(ls -A model)" ]; then #if is empty download
-echo "Downloading model"
-wget http://minio.clarin-pl.eu/public/models/plwn_dump_27-03-2018.sqlite -O ./model/plwn_dump_27-03-2018.sqlite
-fi
-python3.6 main.py service
\ No newline at end of file

requirements.txt

 lex-ws
 pika==0.12
 plwn_api
+awscli
\ No newline at end of file

src/plwordnet_worker.py

 """Implementation of plwordnet_worker."""
-
 import json
 import logging
+import time
 import traceback
 
 import lex_ws
@@ -10,7 +10,6 @@ import plwn
 import sqlite3
 print(sqlite3.sqlite_version)
 my_logger = logging.getLogger(__name__)
-_log = logging.getLogger(__name__)
 
 
 def syn2str(synset):
@@ -48,9 +47,12 @@ class PLWordnetWorker(lex_ws.LexWorker):
     def init(self):
         """Initializes PLWordnet worker."""
         my_logger.info("Loading models...")
+        start_time = time.time()
         model_path = self.config['tool']["model"]
         self.wn = plwn.load(model_path, "sqlite3")
-        my_logger.info("Loading finished.")
+        exec_time = time.time() - start_time
+        print("Loading took: " + str(exec_time))
+        my_logger.info("Loading finished. Loading took: " + str(exec_time))
         return
 
     def process(self, input):

tox.ini

@@ -41,4 +41,4 @@ max-line-length = 80
 # D411 Missing blank line before section
 ignore = D104,D203,D213,D214,D215,D401,D405,D406,D407,D408,D409,D410,D411
 match-dir = ^(?!\.tox|venv).*
-match = ^(?!setup).*\.py
+match = ^(?!setup|entrypoint).*\.py