From cd9ff7d699a895b98bb37df3ee3d5d117cdfa8d9 Mon Sep 17 00:00:00 2001
From: Grzegorz Kostkowski <grzegorz.kostkowski@pwr.edu.pl>
Date: Fri, 22 May 2020 11:56:29 +0200
Subject: [PATCH] Workaround to avoid unexpected stop of execution
Unfortunatelly, there are some limitations related with using
together: multiprocessing and multithreading (forking, handled by
omp).
As a workaround, graph is not shared, each worker has own instance.
There is also possibility to change multiprocessing mode to 'spawn'
(and this case was checked) and it works with omp multithreading,
but it results in same bahavior: model is copied in memory (because
OS cannot postpone copying according copy-on-write strategy - which
is available in 'fork' mode).
---
src/worker.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/src/worker.py b/src/worker.py
index e8e1f7e..27cf34a 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -28,13 +28,14 @@ class Worker(nlp_ws.NLPWorker):
logging_lvl = config["logging_levels"]["wsd_worker"]
if logging_lvl:
logging.basicConfig(level=logging_lvl)
- _log.info("Worker started loading models")
cls.configtool = config['tool']
- cls.model = wosedon_plugin.WoSeDonPlugin(
- config['tool']['config_file'], config['tool']['model_dir'])
- _log.info("Using config: %s and model: %s",
- config['tool']['config_file'],
- config['tool']['model_dir'])
+ _log.info("Using config: %s", config['tool']['config_file'])
+
+ def init(self):
+ _log.info("Using model: %s", self.configtool['model_dir'])
+ _log.info("Worker started loading models")
+ self.model = wosedon_plugin.WoSeDonPlugin(
+ self.configtool['config_file'], self.configtool['model_dir'])
_log.info("Worker finished loading models ")
def process(self, inputFile, taskOptions, outputFile):
--
GitLab