From 50106399b3def042696a872a96753cfa02fa7665 Mon Sep 17 00:00:00 2001
From: Bartosz Walkowiak <bwalkow@e-science.pl>
Date: Tue, 28 Feb 2023 10:01:45 +0000
Subject: [PATCH] Develop
---
.gitlab-ci.yml | 2 +-
src/archive_extractor_worker.py | 32 ++++++++++++++++++++++++++++----
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 963cbc7..9d7aa93 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ build:
image: docker:18.09.7
only:
- master
- - Init
+ - develop
services:
- 'docker:18.09.7-dind'
script:
diff --git a/src/archive_extractor_worker.py b/src/archive_extractor_worker.py
index 8f77231..18a0028 100644
--- a/src/archive_extractor_worker.py
+++ b/src/archive_extractor_worker.py
@@ -236,7 +236,7 @@ class ArchiveExtractor(nlp_ws.NLPWorker):
def _safe_report(self, input_size_report: Dict, input_size_sum: int,
output_size_report: Dict, output_size_sum: int,
error_report_before: Dict, error_report_after: Dict,
- files_to_convert: List, dir_path: str):
+ files_to_convert: List, output_path: str, dir_path: str):
"""Save size and error reports to json file.
:param input_size_report: input size report
@@ -253,6 +253,8 @@ class ArchiveExtractor(nlp_ws.NLPWorker):
:type error_report_after: Dict
:param files_to_convert: converted files
:type files_to_convert: List
+ :param output_path: Path to output location.
+ :type output_path: str
:param dir_path: Path to directory where the
worker will store result files.
:type dir_path: str
@@ -267,10 +269,28 @@ class ArchiveExtractor(nlp_ws.NLPWorker):
self.INPUT_SIZES_SUM: input_size_sum,
self.OUTPUT_SIZES: output_size_report,
self.OUTPUT_SIZES_SUM: output_size_sum}
- filename = os.path.join(dir_path, 'report.json')
+ filename = os.path.join(output_path, 'report.json')
with open(filename, 'w') as outfile:
json.dump(data, outfile)
+ def _pack_result(self, output_path: str, dir_path: str):
+ """Pack cleaned files to zip, removes unpacked files.
+
+ :param output_path: Path to output location.
+ :type output_path: str
+ :param dir_path: Path to directory where the
+ worker store result files.
+ :type dir_path: str
+ """
+ zip_name = os.path.join(output_path, 'result.zip')
+ _log.info("Packing results to zip...")
+ with ZipFile(zip_name, 'w') as zip_obj:
+ for file_path, filename in self._file_generator(dir_path):
+ zip_obj.write(file_path, filename)
+
+ _log.info("Packing completed, removing unpacked files")
+ shutil.rmtree(dir_path)
+
def process(self, input_path, task_options, output_path):
"""Called for each request made to the worker.
@@ -291,7 +311,8 @@ class ArchiveExtractor(nlp_ws.NLPWorker):
if not is_zipfile(input_path):
raise Exception("Input is not a zip archive")
- dir_path = output_path
+ dir_path = os.path.join(output_path, 'result')
+ os.mkdir(dir_path)
self._unpack_zip(input_path, dir_path)
files_to_remove, files_to_convert, input_size_sum = [], [], 0
input_size_report, error_report_before = {}, {member.value: []
@@ -345,6 +366,9 @@ class ArchiveExtractor(nlp_ws.NLPWorker):
f" {output_size_sum} is too large,"
f" max size is {self.output_max_size}")
+ self._pack_result(output_path, dir_path)
+
self._safe_report(input_size_report, input_size_sum, output_size_report,
output_size_sum, error_report_before,
- error_report_after, files_to_convert, dir_path)
+ error_report_after, files_to_convert, output_path,
+ dir_path)
--
GitLab