Commit 301a9e1e authored by Szymon Ciombor's avatar Szymon Ciombor Committed by Mateusz Gniewkowski

Python 3.6 port, CI, ready for deployment on Rancher

parent 85e78dd2
image: 'clarinpl/python:3.6'
cache:
paths:
- .tox
stages:
- check_style
- build
before_script:
- pip install tox==2.9.1
pep8:
stage: check_style
script:
- tox -v -e pep8
docstyle:
stage: check_style
script:
- tox -v -e docstyle
build_image:
stage: build
image: 'docker:18.09.7'
only:
- master
services:
- 'docker:18.09.7-dind'
before_script:
- ''
script:
- docker build -t clarinpl/mwe .
- echo $DOCKER_PASSWORD > pass.txt
- cat pass.txt | docker login --username $DOCKER_USERNAME --password-stdin
- rm pass.txt
- docker push clarinpl/mwe
FROM clarinpl/builder AS builder
FROM clarinpl/python:2.7
FROM clarinpl/python:3.6
RUN apt-get update && apt-get install -y \
libxml++2.6-dev \
libloki-dev \
libboost-all-dev \
libicu-dev \
libffi-dev \
libssl-dev \
libxml2-utils
COPY --from=builder /install/corpus2 /
COPY --from=builder /install/wccl /
COPY --from=builder /install/corpus2mwe /
RUN ldconfig
corpus2-python3.6 \
wccl-python3.6 \
corpus2mwe-python3.6
WORKDIR /home/worker
COPY requirements.txt .
COPY mwe_worker.py .
RUN pip install -r requirements.txt
CMD ["python", "mwe_worker.py"]
COPY main.py .
COPY ./src ./src
RUN python3.6 -m pip install -r requirements.txt
CMD ["python", "main.py", "service"]
......@@ -2,17 +2,19 @@
tool = mwe
root = /samba/requests/
rabbit_host = rabbit.clarin.ws
rabbit_user = clarin
rabbit_password = clarin123
rabbit_host = rabbitmq
rabbit_user = test
rabbit_password = test
queue_prefix =nlp_
[tool]
workers_number = 5
processed_lines = 1000
[logging]
port = 0
port = 9912
local_log_level = INFO
[logging_levels]
__main__ = INFO
wsd_worker = INFO
version: '3'
services:
mwe:
container_name: clarin_mwe
build: ./
working_dir: /home/worker
entrypoint:
- python2
- mwe_worker.py
volumes:
- /samba:/samba
- ./config.ini:/home/worker/config.ini
- ./mwe_worker.py:/home/worker/mwe_worker.py
restart: always
\ No newline at end of file
"""Implementation of mwe service."""
import argparse
import nlp_ws
from src.worker import Worker
def get_args():
"""Gets command line arguments."""
parser = argparse.ArgumentParser(description="mwe implementation")
subparsers = parser.add_subparsers(dest="algorithm")
subparsers.required = True
subparsers.add_parser("service", help="Run as a service")
return parser.parse_args()
def main():
"""Runs the program."""
args = get_args()
generators = {
"service": lambda: nlp_ws.NLPService.main(Worker),
}
gen_fn = generators.get(args.algorithm, lambda: None)
gen_fn()
if __name__ == "__main__":
main()
corpus-ccl
nlp-ws
\ No newline at end of file
nlp-ws
corpus-ccl
\ No newline at end of file
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Implementation of nlp_worker."""
import logging
import subprocess
import nlp_ws
from corpus_ccl import cclutils as ccl
import corpus2mwe as mwe
import subprocess
import logging
import nlp_ws
_log = logging.getLogger(__name__)
class MWEWorker(nlp_ws.NLPWorker):
class Worker(nlp_ws.NLPWorker):
"""Implements nlp_worker for mwe service."""
@classmethod
def static_init(cls, config):
_log.info( "Static init" )
"""Static_init for Worker."""
_log.info("Static init")
def init(self):
_log.info( "Worker started loading models %s","AS" )
self.tagset = ccl.get_tagset('nkjp')
"""Initialize Worker."""
_log.info("Worker started loading models %s", "AS")
self.tagset = ccl.get_tagset("nkjp")
self.reader = None
def process(self, inputFile, taskOptions, outputFile):
_validate_xml(inputFile)
def process(self, input_file, task_options, output_file):
"""Runs a single nlp_task."""
_validate_xml(input_file)
if not self.reader:
self.reader = mwe.CclMWEReader(inputFile, self.tagset)
self.reader = mwe.CclMWEReader(input_file, self.tagset)
self.reader.use_annotations(False)
else:
self.reader.set_files(inputFile)
self.reader.set_files(input_file)
mwe_doc = self.reader.read()
ccl.write_ccl(mwe_doc, str(outputFile))
ccl.write_ccl(mwe_doc, str(output_file))
class _InvalidXMLInRequest(Exception):
pass
def _validate_xml(xmlfile):
"""
This will do nothing if XML is valid and raise exception if it's not.
"""
def _validate_xml(xml_file):
"""Will do nothing if XML is valid and raise exception if it's not."""
lint_call = subprocess.Popen(
('xmllint', '--nonet', '--noout', xmlfile),
stderr=subprocess.PIPE,
("xmllint", "--nonet", "--noout", xml_file), stderr=subprocess.PIPE,
)
xml_err = lint_call.communicate()[1]
lint_call.communicate()
if lint_call.returncode != 0:
raise _InvalidXMLInRequest("Wrong XML in input data")
if __name__ == '__main__':
nlp_ws.NLPService.main(MWEWorker)
if __name__ == "__main__":
nlp_ws.NLPService.main(Worker)
[tox]
envlist = pep8,docstyle
skipsdist = True
[testenv:pep8]
deps =
flake8
basepython = python3
commands =
flake8 {posargs}
[testenv:docstyle]
deps =
pydocstyle
basepython = python3
commands =
pydocstyle --verbose {posargs}
[flake8]
# W503 line break before binary operator
# W504 skipped because it is overeager and unnecessary
ignore = W503,W504
show-source = True
exclude = .git,.venv,.tox,dist,doc,*egg,build,venv
import-order-style = pep8
max-line-length = 80
[pydocstyle]
# D104 Missing docstring in public package
# D203 1 blank line required before class docstring
# D213 Multi-line docstring summary should start at the second line
# D214 Section is over-indented
# D215 Section underline is over-indented
# D401 First line should be in imperative mood; try rephrasing
# D405 Section name should be properly capitalized
# D406 Section name should end with a newline
# D407 Missing dashed underline after section
# D408 Section underline should be in the line following the section’s name
# D409 Section underline should match the length of its name
# D410 Missing blank line after section
# D411 Missing blank line before section
ignore = D104,D203,D213,D214,D215,D401,D405,D406,D407,D408,D409,D410,D411
match-dir = ^(?!\.tox|venv).*
match = ^(?!setup).*\.py
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment