nlpworkers
nlp_ws

Repository

"""Implementetion of dummy worker."""

import nlp_ws

class Worker(nlp_ws.NLPWorker):
    """Class that implements example worker."""

    def process(self, input_file: str, task_options: dict, output_file: str) -> None:
        """Implementation of example tasks that copies files."""
        shutil.copy(input_file, output_file)


if __name__ == '__main__':
    nlp_ws.NLPService.main(Worker)
l_subtask = SubTask(input_path, ["wcrft2", "liner2"])
l_subtask.run()
t_subtask.get_output_path()
[{'tab2text':{"files":"B", "texts":"E"}}]
["wcrft2"]
["wcrft2", "liner2"]
#!/usr/bin/python
"""Implementation of dummy worker."""
import shutil
import logging
import time
from nlp_ws import NLPWorker, NLPService, SubTask
import json

log = logging.getLogger(__name__)

SubTask.turn_on()


class DummyWorker(NLPWorker):
    """Class that implements example worker with subtasks."""

    def subtasks(self, input_path, task_options, output_path):
        """Worker task with subtasks.

        :param input_path: input path
        :type input_path: str
        :param task_options: task options
        :type task_options: dict
        :param output_path: output path
        :type output_path: str
        """
        l_subtask = SubTask(input_path, ["wcrft2", "liner2"])
        l_subtask.run(blocking=False)
        t_subtask = SubTask(input_path, ["textclass"])
        t_subtask.run(blocking=False)
        self.update_progress(0.1)
        time.sleep(0.1)
        log.info("Dummy waits for results")
        l_result = l_subtask.get_output_path()
        self.update_progress(0.3)
        t_result = t_subtask.get_output_path()
        self.update_progress(0.5)
        log.info("Result paths {} {}".format(l_result, t_result))
        with open(t_result, "rt") as f:
            text = json.loads(f.read())
            log.info(text)
        self.update_progress(1)

    def process(self, input_path, task_options, output_path):
        """Implementation of process method inherited from NLPworker.

        :param input_path: input path
        :type input_path: str
        :param task_options: task options
        :type task_options: dict
        :param output_path: output path
        :type output_path: str
        """
        log.info("Processing!!!")
        self.subtasks(input_path, task_options, output_path)


if __name__ == '__main__':
    NLPService.main(DummyWorker)
sentences = ["My name is Jonas E. Smith. Please turn to p. 55.", "My name is Jonas E. Smith. Please turn to p. 55."]
l_subtask = Sentence(sentences, 'fast_kgr10')
l_subtask.run()
t_subtask.get_results()
#!/usr/bin/python
"""Implementation of dummy worker."""
import shutil
import logging
import time
from nlp_ws import NLPWorker, NLPService, Sentence
import json

log = logging.getLogger(__name__)

Sentence.turn_on()


class DummyWorker(NLPWorker):
    """Class that implements example worker with subtasks."""

    def subtasks(self, input_path, task_options, output_path):
        """Worker task with subtasks.

        :param input_path: input path
        :type input_path: str
        :param task_options: task options
        :type task_options: dict
        :param output_path: output path
        :type output_path: str
        """
        sentences = ["My name is Jonas E. Smith. Please turn to p. 55.", "My name is Jonas E. Smith. Please turn to p. 55."]
        l_subtask = Sentence(sentences, 'fast_kgr10')
        l_subtask.run(blocking=False)
        self.update_progress(0.1)
        time.sleep(0.1)
        log.info("Dummy waits for results")
        l_result = l_subtask.get_results()
        self.update_progress(0.3)
        log.info("Result {}".format(l_result))
        self.update_progress(1)

    def process(self, input_path, task_options, output_path):
        """Implementation of process method inherited from NLPworker.

        :param input_path: input path
        :type input_path: str
        :param task_options: task options
        :type task_options: dict
        :param output_path: output path
        :type output_path: str
        """
        log.info("Processing!!!")
        self.subtasks(input_path, task_options, output_path)


if __name__ == '__main__':
    NLPService.main(DummyWorker)
   src/ - źródła aplikacji
   tests/ - testy
   main.py - skrypt wejściowy
   config.ini - plik konfuguracyjny (opisany niżej)
   Dockerfile - definicja budowanie obrazu Docker
   .gitlab-ci.yml - definicja potoku CI
   .gitignore - pliki mające być ignorowane przez system git
   tox.ini - konfiguracja narzędzia tox
   requirements.txt - wymagane biblioteki
[service]
tool = nazwa_uslugi

root = /samba/requests/
rabbit_host = addr
rabbit_user = test
rabbit_password = test

[tool]
workers_number = 1

[logging]
port = 9981
local_log_level = INFO


[logging levels]
__main__ == INFO
		CFG_S_SERV - odpowiada sekcji [service]
        CFG_S_TOOL - odpowiada sekcji [tool]
        CFG_S_LOG - odpowiada sekcji [logging]
        CFG_S_LOGLEVELS - odpowiada sekcji [logging_levels]
Dla sekcji service:
    CFG_S_SERV_OPT_ROOT
    CFG_S_SERV_OPT_TOOL
    CFG_S_SERV_OPT_QUEUE_PREFIX
    CFG_S_SERV_OPT_RABBIT_HOST
    CFG_S_SERV_OPT_RABBIT_USER
    CFG_S_SERV_OPT_RABBIT_PASSWORD
    CFG_S_SERV_OPT_IS_PRODUCTION

Dla sekcji tool:
    CFG_S_TOOL_OPT_WORKERS_NUMBER

Dla sekcji log:
    CFG_S_LOG_OPT_PORT
    CFG_S_LOG_OPT_LOGFILE_NAME
    CFG_S_LOG_OPT_LOGFILE_MAXSIZE
    CFG_S_LOG_OPT_LOGFILE_MAXBACKUPS
    CFG_S_LOG_OPT_LOG_FORMAT
    CFG_S_LOG_OPT_LOCAL_LOG_LEVEL
CFG_S_SERV_OPT_MY_AWESOME_VARIABLE
CFG_S_SERV_OPT_ROOT
CFG_S_SERV_OPT_TOOL
CFG_S_SERV_OPT_RABBIT_HOST
CFG_S_SERV_OPT_RABBIT_USER
CFG_S_SERV_OPT_RABBIT_PASSWORD
FROM clarinpl/cuda-python:3.6

WORKDIR /home/worker
COPY ./src ./src
COPY ./cmc_service.py .
COPY ./requirements.txt .
COPY ./models/CMC ./models/CMC
COPY ./entrypoint.sh .

RUN apt update && apt install -y g++ gdb

RUN git clone https://github.com/facebookresearch/fastText.git && \
    cd fastText && \
    python3.6 -m pip install . && \
    cd .. && \
    rm -rf fastText

RUN python3.6 -m pip install -r requirements.txt

RUN ["chmod", "+x", "./entrypoint.sh"]

CMD ["./entrypoint.sh"]

image: clarinpl/python:3.6

cache:
  paths:
    - .tox

stages:
  - check_style
  - build

before_script:
  - pip install tox==2.9.1

pep8:
  stage: check_style
  script:
    - tox -v -e pep8

docstyle:
  stage: check_style
  script:
    - tox -v -e docstyle

build_image:
  stage: build
  image: docker:18.09.7
  only:
    - master
  services:
    - docker:18.09.7-dind
  before_script:
    - ''
  script:
    - docker build -t clarinpl/websim .
    - echo $DOCKER_PASSWORD > pass.txt
    - cat pass.txt | docker login --username $DOCKER_USERNAME --password-stdin
    - rm pass.txt
    - docker push clarinpl/websim