From 89e51f99f6371093e60120c43f9c48ea0eadfd19 Mon Sep 17 00:00:00 2001
From: szymekc <szymekc98@gmail.com>
Date: Mon, 7 Feb 2022 12:16:24 +0100
Subject: [PATCH 1/4] added download as dict, filename specification, fixed
progress bar
---
lpmn_client/__init__.py | 4 +++-
lpmn_client/__main__.py | 10 ++++++--
lpmn_client/file_handler.py | 47 +++++++++++++++++++++++++++++++++----
lpmn_client/task.py | 2 +-
setup.py | 2 +-
5 files changed, 56 insertions(+), 9 deletions(-)
diff --git a/lpmn_client/__init__.py b/lpmn_client/__init__.py
index a3d9ab9..e2be96c 100644
--- a/lpmn_client/__init__.py
+++ b/lpmn_client/__init__.py
@@ -1,7 +1,8 @@
from .config import Config
from .file_handler import (
upload_file,
- download_file
+ download_file,
+ download_file_as_dict
)
from .task import Task
@@ -11,4 +12,5 @@ __all__ = [
"Task",
"upload_file",
"download_file",
+ "download_file_as_dict"
]
diff --git a/lpmn_client/__main__.py b/lpmn_client/__main__.py
index 7d98868..a1cdf6f 100644
--- a/lpmn_client/__main__.py
+++ b/lpmn_client/__main__.py
@@ -25,7 +25,13 @@ def get_args():
parser.add_argument(
"-o",
"--output",
- help="Path to a output file on local machine."
+ help="Path to a output directory on local machine."
+ )
+ parser.add_argument(
+ "-f",
+ "--filename",
+ help="Downloaded file name.",
+ default=None
)
parser.add_argument(
"lpmn",
@@ -40,7 +46,7 @@ def main():
file_id = upload_file(file_path=args.input)
t = Task(args.lpmn)
output_file_id = t.run(file_id=file_id, verbose=args.verbose, timeout=0)
- downloaded = download_file(file_id=output_file_id, output_path=args.output)
+ downloaded = download_file(file_id=output_file_id, output_path=args.output, filename=args.filename)
print(f"Result saved to {downloaded}")
diff --git a/lpmn_client/file_handler.py b/lpmn_client/file_handler.py
index 8db30b0..21b309d 100644
--- a/lpmn_client/file_handler.py
+++ b/lpmn_client/file_handler.py
@@ -60,15 +60,18 @@ def upload_file(file_path: str) -> FileID:
def download_file(
file_id: FileID,
- output_path: tp.Optional[str] = None
+ output_path: tp.Optional[str] = None,
+ filename: tp.Optional[str] = None
) -> str:
"""Downloads file by its FileID.
Args:
file_id (FileID): File id of a file to download
- path (Optional[str]): Path for downloaded file.
+ output_path (Optional[str]): Directory to save downloaded file in.
If not given the file will be saved in default location
(config.ini).
+ filename (Optional[str]): Output filename without .zip format.
+ If not specified, filename is random.
Returns:
str. A path to downloaded file.
@@ -76,8 +79,8 @@ def download_file(
response = requests.get(url=f"{Config.get('base_url')}/download{file_id}")
if response.status_code == 404 or response.status_code == 500:
raise FileNotFoundError("File not found")
-
- filename = file_id.split("/")[-1] + ".zip"
+ if not filename:
+ filename = f"{file_id.split('/')[-1]}.zip"
if not output_path:
output_path = Config.get("output_path")
os.makedirs(output_path, exist_ok=True)
@@ -86,3 +89,39 @@ def download_file(
with open(output_path, "wb") as f:
f.write(response.content)
return output_path
+
+
+def download_file_as_dict(
+ file_id: FileID,
+) -> tp.Dict[str, str]:
+ """Downloads file by its FileID.
+
+ Args:
+ file_id (FileID): File id of a file to download
+
+ Returns:
+ dict. Dictionary mapping filenames in output zipfile to file contents.
+
+ """
+ def try_decode(file):
+ try:
+ file.decode("utf-8")
+ return True
+ except UnicodeDecodeError:
+ return False
+
+ response = requests.get(url=f"{Config.get('base_url')}/download{file_id}")
+ if response.status_code == 404 or response.status_code == 500:
+ raise FileNotFoundError("File not found")
+
+ zip_buffer = io.BytesIO(initial_bytes=response.content)
+ with zipfile.ZipFile(
+ zip_buffer,
+ mode="r",
+ compression=zipfile.ZIP_DEFLATED,
+ ) as zip_file:
+ return {i.filename: zip_file.open(i).read().decode("utf-8")
+ for i in filter(
+ lambda x: try_decode(zip_file.open(x).read())
+ and x.filename[-1] != '/',
+ zip_file.infolist())}
diff --git a/lpmn_client/task.py b/lpmn_client/task.py
index 36be732..24dfa1e 100644
--- a/lpmn_client/task.py
+++ b/lpmn_client/task.py
@@ -103,7 +103,7 @@ class Task(object):
time.sleep(0.5)
progress = self.get_progress()
if pbar:
- pbar.update(progress * 100)
+ pbar.update(progress * 100 - pbar.n)
if pbar:
pbar.update(100)
diff --git a/setup.py b/setup.py
index 02a6da2..e31b207 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="lpmn_client",
- version="1.4.3",
+ version="1.4.4",
author="Szymon Ciombor, Mateusz Gniewkowski",
author_email="Szymon.Ciombor@pwr.edu.pl, mateusz.gniewkowski@pwr.edu.pl",
description="Client for writing LPMN queries.",
--
GitLab
From 4e5953aa7a3edcf3e7f445a4342b8633af26140f Mon Sep 17 00:00:00 2001
From: szymekc <szymekc98@gmail.com>
Date: Mon, 7 Feb 2022 12:18:07 +0100
Subject: [PATCH 2/4] pep8
---
lpmn_client/__main__.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lpmn_client/__main__.py b/lpmn_client/__main__.py
index a1cdf6f..c425b30 100644
--- a/lpmn_client/__main__.py
+++ b/lpmn_client/__main__.py
@@ -46,7 +46,9 @@ def main():
file_id = upload_file(file_path=args.input)
t = Task(args.lpmn)
output_file_id = t.run(file_id=file_id, verbose=args.verbose, timeout=0)
- downloaded = download_file(file_id=output_file_id, output_path=args.output, filename=args.filename)
+ downloaded = download_file(file_id=output_file_id,
+ output_path=args.output,
+ filename=args.filename)
print(f"Result saved to {downloaded}")
--
GitLab
From 4c4d6ab05d6e5cb7584fa3ae9e939ab26621b0c6 Mon Sep 17 00:00:00 2001
From: szymekc <szymekc98@gmail.com>
Date: Mon, 7 Feb 2022 12:43:29 +0100
Subject: [PATCH 3/4] is_dir() check
---
lpmn_client/file_handler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lpmn_client/file_handler.py b/lpmn_client/file_handler.py
index 21b309d..5fce077 100644
--- a/lpmn_client/file_handler.py
+++ b/lpmn_client/file_handler.py
@@ -123,5 +123,5 @@ def download_file_as_dict(
return {i.filename: zip_file.open(i).read().decode("utf-8")
for i in filter(
lambda x: try_decode(zip_file.open(x).read())
- and x.filename[-1] != '/',
+ and not x.is_dir(),
zip_file.infolist())}
--
GitLab
From 67f8351de99d16d4bf52aef7889de848e23f6b23 Mon Sep 17 00:00:00 2001
From: szymekc <szymekc98@gmail.com>
Date: Tue, 8 Feb 2022 14:20:56 +0100
Subject: [PATCH 4/4] added test_download_file_as_dict, fixed indentation and
pbar update
---
lpmn_client/__main__.py | 8 +++++---
lpmn_client/file_handler.py | 17 ++++++++++-------
lpmn_client/task.py | 2 +-
tests/lpmn_test.py | 11 ++++++++++-
4 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/lpmn_client/__main__.py b/lpmn_client/__main__.py
index c425b30..6bfee0b 100644
--- a/lpmn_client/__main__.py
+++ b/lpmn_client/__main__.py
@@ -46,9 +46,11 @@ def main():
file_id = upload_file(file_path=args.input)
t = Task(args.lpmn)
output_file_id = t.run(file_id=file_id, verbose=args.verbose, timeout=0)
- downloaded = download_file(file_id=output_file_id,
- output_path=args.output,
- filename=args.filename)
+ downloaded = download_file(
+ file_id=output_file_id,
+ output_path=args.output,
+ filename=args.filename
+ )
print(f"Result saved to {downloaded}")
diff --git a/lpmn_client/file_handler.py b/lpmn_client/file_handler.py
index 5fce077..96baaa7 100644
--- a/lpmn_client/file_handler.py
+++ b/lpmn_client/file_handler.py
@@ -103,9 +103,9 @@ def download_file_as_dict(
dict. Dictionary mapping filenames in output zipfile to file contents.
"""
- def try_decode(file):
+ def try_decode(bytes_data):
try:
- file.decode("utf-8")
+ bytes_data.decode("utf-8")
return True
except UnicodeDecodeError:
return False
@@ -120,8 +120,11 @@ def download_file_as_dict(
mode="r",
compression=zipfile.ZIP_DEFLATED,
) as zip_file:
- return {i.filename: zip_file.open(i).read().decode("utf-8")
- for i in filter(
- lambda x: try_decode(zip_file.open(x).read())
- and not x.is_dir(),
- zip_file.infolist())}
+ return {
+ i.filename: zip_file.open(i).read().decode("utf-8")
+ for i in filter(
+ lambda x:
+ try_decode(zip_file.read(x)) and not x.is_dir(),
+ zip_file.infolist()
+ )
+ }
diff --git a/lpmn_client/task.py b/lpmn_client/task.py
index 24dfa1e..1344380 100644
--- a/lpmn_client/task.py
+++ b/lpmn_client/task.py
@@ -106,7 +106,7 @@ class Task(object):
pbar.update(progress * 100 - pbar.n)
if pbar:
- pbar.update(100)
+ pbar.update(100 - pbar.n)
self.status = "done"
def run(
diff --git a/tests/lpmn_test.py b/tests/lpmn_test.py
index 050d857..763b4cf 100644
--- a/tests/lpmn_test.py
+++ b/tests/lpmn_test.py
@@ -3,7 +3,7 @@ import os
import pytest
import shutil
-from lpmn_client import download_file, upload_file
+from lpmn_client import download_file, download_file_as_dict, upload_file
from lpmn_client import Task
@@ -37,6 +37,15 @@ class TestLPMN(object):
path = download_file(file_id, output_path="./out_test")
assert os.path.isfile(path)
+ @pytest.mark.parametrize(
+ "file_id", ["uploaded_file", "uploaded_dir", "uploaded_zip"]
+ )
+ def test_download_file_as_dict(self, file_id, request):
+ """Test downloading of uploaded files as dicts."""
+ file_id = request.getfixturevalue(file_id)
+ data = download_file_as_dict(file_id)
+ assert len(data) > 0
+
@pytest.mark.parametrize(
"file_id", ["uploaded_file", "uploaded_dir", "uploaded_zip"]
)
--
GitLab