Merge branch 'dev-python3.6' into 'master'

Python 3.6 port, CI, ready for deployment on Rancher

See merge request !2

.gitlab-ci.yml

+image: 'clarinpl/python:3.6'
+cache:
+  paths:
+    - .tox
+stages:
+  - check_style
+  - build
+before_script:
+  - pip install tox==2.9.1
+pep8:
+  stage: check_style
+  script:
+    - tox -v -e pep8
+docstyle:
+  stage: check_style
+  script:
+    - tox -v -e docstyle
+build_image:
+  stage: build
+  image: 'docker:18.09.7'
+  only:
+    - master
+  services:
+    - 'docker:18.09.7-dind'
+  before_script:
+    - ''
+  script:
+    - docker build -t clarinpl/mwe .
+    - echo $DOCKER_PASSWORD > pass.txt
+    - cat pass.txt | docker login --username $DOCKER_USERNAME --password-stdin
+    - rm pass.txt
+    - docker push clarinpl/mwe

Dockerfile

-FROM clarinpl/builder AS builder
-FROM clarinpl/python:2.7
+FROM clarinpl/python:3.6
 
 RUN apt-get update && apt-get install -y  \
-    libxml++2.6-dev \
-    libloki-dev \
-    libboost-all-dev \
-    libicu-dev \
-    libffi-dev \
-    libssl-dev \
-    libxml2-utils    
-    
-COPY --from=builder /install/corpus2 /
-COPY --from=builder /install/wccl /
-COPY --from=builder /install/corpus2mwe /
-RUN ldconfig
+    corpus2-python3.6 \
+    wccl-python3.6 \
+    corpus2mwe-python3.6
 
 WORKDIR /home/worker
 COPY requirements.txt .
-COPY mwe_worker.py .
-RUN pip install -r requirements.txt
-CMD ["python", "mwe_worker.py"]
+COPY main.py .
+COPY ./src ./src
+RUN python3.6 -m pip install -r requirements.txt
+CMD ["python", "main.py", "service"]

config.ini

@@ -2,17 +2,19 @@
 tool = mwe
 
 root = /samba/requests/
-rabbit_host = rabbit.clarin.ws
-rabbit_user = clarin
-rabbit_password = clarin123
+rabbit_host = rabbitmq
+rabbit_user = test
+rabbit_password = test
+queue_prefix =nlp_
 
 [tool]
 workers_number = 5
+processed_lines = 1000
 
 [logging]
-port = 0
+port = 9912
 local_log_level = INFO
 
 [logging_levels]
 __main__ = INFO
-wsd_worker = INFO
+

docker-compose.yml

-version: '3'
-services:  
-  mwe:
-    container_name: clarin_mwe
-    build: ./
-    
-    working_dir: /home/worker
-    entrypoint:
-          - python2
-          - mwe_worker.py
-    volumes:
-        - /samba:/samba
-        - ./config.ini:/home/worker/config.ini        
-        - ./mwe_worker.py:/home/worker/mwe_worker.py        
-    restart: always    
\ No newline at end of file

main.py

+"""Implementation of mwe service."""
+import argparse
+
+import nlp_ws
+
+from src.worker import Worker
+
+
+def get_args():
+    """Gets command line arguments."""
+    parser = argparse.ArgumentParser(description="mwe implementation")
+
+    subparsers = parser.add_subparsers(dest="algorithm")
+    subparsers.required = True
+
+    subparsers.add_parser("service", help="Run as a service")
+    return parser.parse_args()
+
+
+def main():
+    """Runs the program."""
+    args = get_args()
+
+    generators = {
+        "service": lambda: nlp_ws.NLPService.main(Worker),
+    }
+
+    gen_fn = generators.get(args.algorithm, lambda: None)
+    gen_fn()
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

-corpus-ccl
-nlp-ws
\ No newline at end of file
+nlp-ws
+corpus-ccl
\ No newline at end of file

mwe_worker.py → src/worker.py

-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
+"""Implementation of nlp_worker."""
+import logging
+import subprocess
 
-import nlp_ws
 from corpus_ccl import cclutils as ccl
 import corpus2mwe as mwe
-import subprocess
-import logging
+import nlp_ws
+
 
 _log = logging.getLogger(__name__)
 
 
-class MWEWorker(nlp_ws.NLPWorker):
-    
+class Worker(nlp_ws.NLPWorker):
+    """Implements nlp_worker for mwe service."""
+
     @classmethod
     def static_init(cls, config):
-        _log.info( "Static init" )
-                
-        
+        """Static_init for Worker."""
+        _log.info("Static init")
+
     def init(self):
-        _log.info( "Worker started loading models %s","AS" )    
-        self.tagset = ccl.get_tagset('nkjp')
+        """Initialize Worker."""
+        _log.info("Worker started loading models %s", "AS")
+        self.tagset = ccl.get_tagset("nkjp")
         self.reader = None
-    
-    def process(self, inputFile, taskOptions, outputFile):    
-        _validate_xml(inputFile)
+
+    def process(self, input_file, task_options, output_file):
+        """Runs a single nlp_task."""
+        _validate_xml(input_file)
         if not self.reader:
-            self.reader = mwe.CclMWEReader(inputFile, self.tagset)
+            self.reader = mwe.CclMWEReader(input_file, self.tagset)
             self.reader.use_annotations(False)
         else:
-            self.reader.set_files(inputFile)
+            self.reader.set_files(input_file)
         mwe_doc = self.reader.read()
-        ccl.write_ccl(mwe_doc, str(outputFile))    
+        ccl.write_ccl(mwe_doc, str(output_file))
+
 
 class _InvalidXMLInRequest(Exception):
     pass
-            
-def _validate_xml(xmlfile):
-    """
-    This will do nothing if XML is valid and raise exception if it's not.
-    """
 
+
+def _validate_xml(xml_file):
+    """Will do nothing if XML is valid and raise exception if it's not."""
     lint_call = subprocess.Popen(
-        ('xmllint', '--nonet', '--noout', xmlfile),
-        stderr=subprocess.PIPE,
+        ("xmllint", "--nonet", "--noout", xml_file), stderr=subprocess.PIPE,
     )
-    xml_err = lint_call.communicate()[1]
+    lint_call.communicate()
 
     if lint_call.returncode != 0:
         raise _InvalidXMLInRequest("Wrong XML in input data")
 
-if __name__ == '__main__':
-    nlp_ws.NLPService.main(MWEWorker)
-    
+
+if __name__ == "__main__":
+    nlp_ws.NLPService.main(Worker)

tox.ini

+[tox]
+envlist = pep8,docstyle
+skipsdist = True
+
+[testenv:pep8]
+deps =
+    flake8
+basepython = python3
+commands =
+    flake8 {posargs}
+
+[testenv:docstyle]
+deps =
+    pydocstyle
+basepython = python3
+commands =
+    pydocstyle --verbose {posargs}
+
+[flake8]
+# W503 line break before binary operator
+# W504 skipped because it is overeager and unnecessary
+ignore = W503,W504
+show-source = True
+exclude = .git,.venv,.tox,dist,doc,*egg,build,venv
+import-order-style = pep8
+max-line-length = 80
+
+
+[pydocstyle]
+# D104 Missing docstring in public package
+# D203 1 blank line required before class docstring
+# D213 Multi-line docstring summary should start at the second line
+# D214 Section is over-indented
+# D215 Section underline is over-indented
+# D401 First line should be in imperative mood; try rephrasing
+# D405 Section name should be properly capitalized
+# D406 Section name should end with a newline
+# D407 Missing dashed underline after section
+# D408 Section underline should be in the line following the section’s name
+# D409 Section underline should match the length of its name
+# D410 Missing blank line after section
+# D411 Missing blank line before section
+ignore = D104,D203,D213,D214,D215,D401,D405,D406,D407,D408,D409,D410,D411
+match-dir = ^(?!\.tox|venv).*
+match = ^(?!setup).*\.py