Add MultifiWorker

parent c7f98d2c
Pipeline #1855 passed with stage
in 50 seconds
.idea
/idea
log.txt
__pycache__
image: clarinpl/python:3.8
cache:
paths:
- .tox
stages:
- check_style
- build
before_script:
- pip install tox==2.9.1
pep8:
stage: check_style
script:
- tox -v -e pep8
\ No newline at end of file
import nlp_ws
from src.Multifit_worker import MultifitWorker
if __name__ == '__main__':
nlp_ws.NLPService.main(MultifitWorker)
import json
import logging
import nlp_ws
from fastai.text import load_learner
from torch import to_np
log = logging.getLogger(__name__)
class MultifitWorker(nlp_ws.NLPWorker):
@classmethod
def static_init(self, config):
self.config = config
log.debug("static_init(%s)", config)
def init(self):
log.debug("init()")
self._classifier = MultifitClassifier()
def process(self, input_path, task_options, output_path):
task = task_options.get("type", None)
with open(input_path, "r") as f:
text = f.read()
lang = text.split('__label__')[1]
text = text.split('__label__')[0]
result = self._classifier.predict(text, lang=lang, task_options=task)
print(result)
with open(output_path, "w") as f:
json.dump(result, f, indent=4)
class MultifitClassifier(object):
def __init__(self):
self.labels_text = ["__label__meta_amb", "__label__meta_minus_m",
"__label__meta_plus_m", "___label__meta_zero"]
self.labels_sen = ["__label__z_amb", "__label__z_minus_m",
"__label__z_plus_m", "___label__z_zero"]
def predict(self, ccl, lang=None, task_options=None):
path = ""
if task_options == "sentence":
path = lang + "-sent-sen.pkl"
labels = self.label_sen
else:
path = lang + "-sent.pkl"
labels = self.labels_text
learner = load_learner("models", path)
results = learner.predict("xxbos " + str(ccl))
probabilities = [str(x) for x in to_np(results[2])]
result = zip(probabilities, labels)
return result
[tox]
envlist = pep8
skipsdist = True
[testenv:pep8]
deps =
flake8
basepython = python3.8
commands =
flake8 {posargs}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment