From 367eccd79312e6be3532d539d1ea87579def9997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20W=C4=85troba?= <marcin.watroba@pwr.edu.pl>
Date: Fri, 10 Feb 2023 07:03:45 +0100
Subject: [PATCH] Fix lint
---
pyproject.toml | 10 +++++++
.../fasttext_embedding_transformer.py | 6 ++---
sziszapangma/core/wer/wer_calculator.py | 3 ---
.../asr_processor/asr_processor.py | 2 +-
.../local_audio_record_repository.py | 4 ++-
.../minio_audio_record_repository.py | 8 ++++--
.../integration/experiment_manager.py | 1 +
.../repository/experiment_repository.py | 2 +-
.../fsspec_experiment_repository.py | 13 ++-------
.../repository/mongo_experiment_repository.py | 2 +-
.../multi_files_experiment_repository.py | 27 ++++++++++---------
sziszapangma/integration/task/asr_task.py | 1 -
.../task/embedding_wer_metrics_task.py | 8 ------
...s_multi_transformers_wer_processor_base.py | 5 ++--
.../task/sentence_wer_processor.py | 1 -
.../task/spacy_ner_sentence_wer_processor.py | 6 ++---
.../spacy_pos_sentence_dep_tag_processor.py | 6 ++---
.../task/spacy_pos_sentence_wer_processor.py | 6 ++---
...ual_ner_transformers_wer_processor_base.py | 8 +++---
19 files changed, 52 insertions(+), 67 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 5d89515..8b612fe 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,16 @@ isort = {extras = ["pyproject"], version = "^5.11.4"}
pyflakes = "^3.0.1"
poethepoet = "^0.16.5"
+[tool.poe.tasks]
+black = "black -v --check sziszapangma"
+isort = "isort --profile black sziszapangma -c"
+pyflakes = "pyflakes sziszapangma"
+mypy = "mypy sziszapangma"
+test = "pytest"
+check = ["black", "isort", "pyflakes"]
+all = ["check", "test"]
+
+
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
diff --git a/sziszapangma/core/transformer/fasttext_embedding_transformer.py b/sziszapangma/core/transformer/fasttext_embedding_transformer.py
index 3b82634..bf1e2d6 100644
--- a/sziszapangma/core/transformer/fasttext_embedding_transformer.py
+++ b/sziszapangma/core/transformer/fasttext_embedding_transformer.py
@@ -1,12 +1,10 @@
-import json
-from typing import Dict, List, Optional
+from typing import Dict, List
+import fasttext.util
import numpy as np
-import requests
from fasttext.FastText import _FastText
from sziszapangma.core.transformer.embedding_transformer import EmbeddingTransformer
-import fasttext.util
class FasttextEmbeddingTransformer(EmbeddingTransformer):
diff --git a/sziszapangma/core/wer/wer_calculator.py b/sziszapangma/core/wer/wer_calculator.py
index 8d05465..a018d79 100644
--- a/sziszapangma/core/wer/wer_calculator.py
+++ b/sziszapangma/core/wer/wer_calculator.py
@@ -1,11 +1,8 @@
from abc import ABC
from typing import List
-import numpy as np
-
from sziszapangma.core.alignment.alignment_step import AlignmentStep
from sziszapangma.core.alignment.alignment_util import AlignmentUtil
-from sziszapangma.core.alignment.step_type import StepType
from sziszapangma.core.wer.span import Span
diff --git a/sziszapangma/integration/asr_processor/asr_processor.py b/sziszapangma/integration/asr_processor/asr_processor.py
index 389beec..752727b 100644
--- a/sziszapangma/integration/asr_processor/asr_processor.py
+++ b/sziszapangma/integration/asr_processor/asr_processor.py
@@ -1,7 +1,7 @@
import json
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import Any, Dict, Optional, List
+from typing import Any, Dict, List, Optional
import requests
from pydub import AudioSegment
diff --git a/sziszapangma/integration/audio_repository/local_audio_record_repository.py b/sziszapangma/integration/audio_repository/local_audio_record_repository.py
index 2bdb5ec..6ed197a 100644
--- a/sziszapangma/integration/audio_repository/local_audio_record_repository.py
+++ b/sziszapangma/integration/audio_repository/local_audio_record_repository.py
@@ -1,7 +1,9 @@
import shutil
from pathlib import Path
-from sziszapangma.integration.audio_repository.audio_record_repository import AudioRecordRepository
+from sziszapangma.integration.audio_repository.audio_record_repository import (
+ AudioRecordRepository,
+)
class LocalAudioRecordRepository(AudioRecordRepository):
diff --git a/sziszapangma/integration/audio_repository/minio_audio_record_repository.py b/sziszapangma/integration/audio_repository/minio_audio_record_repository.py
index 39c5c86..6eb5f86 100644
--- a/sziszapangma/integration/audio_repository/minio_audio_record_repository.py
+++ b/sziszapangma/integration/audio_repository/minio_audio_record_repository.py
@@ -3,7 +3,9 @@ from pathlib import Path
from minio import Minio
from urllib3 import HTTPResponse
-from sziszapangma.integration.audio_repository.audio_record_repository import AudioRecordRepository
+from sziszapangma.integration.audio_repository.audio_record_repository import (
+ AudioRecordRepository,
+)
class MinioAudioRecordRepository(AudioRecordRepository):
@@ -29,7 +31,9 @@ class MinioAudioRecordRepository(AudioRecordRepository):
)
def load_file(self, record_id: str) -> Path:
- record_path = (Path.home() / f".cache/asr_benchmark/{self._dataset_name}/{record_id}.wav")
+ record_path = (
+ Path.home() / f".cache/asr_benchmark/{self._dataset_name}/{record_id}.wav"
+ )
if not record_path.exists():
record_response: HTTPResponse = self._minio.get_object(
self._bucket, self._get_record_path(self._dataset_name, record_id)
diff --git a/sziszapangma/integration/experiment_manager.py b/sziszapangma/integration/experiment_manager.py
index 0f8d987..8160547 100644
--- a/sziszapangma/integration/experiment_manager.py
+++ b/sziszapangma/integration/experiment_manager.py
@@ -3,6 +3,7 @@ from typing import List
from sziszapangma.integration.repository.experiment_repository import (
ExperimentRepository,
)
+
from .record_id_iterator import RecordIdIterator
from .task.processing_task import ProcessingTask
diff --git a/sziszapangma/integration/repository/experiment_repository.py b/sziszapangma/integration/repository/experiment_repository.py
index 73dd702..6c8f268 100644
--- a/sziszapangma/integration/repository/experiment_repository.py
+++ b/sziszapangma/integration/repository/experiment_repository.py
@@ -1,6 +1,6 @@
"""Repository to manage results of asr experiment processing."""
from abc import ABC, abstractmethod
-from typing import Any, Optional, Set, Dict
+from typing import Any, Dict, Optional, Set
class ExperimentRepository(ABC):
diff --git a/sziszapangma/integration/repository/fsspec_experiment_repository.py b/sziszapangma/integration/repository/fsspec_experiment_repository.py
index 9966e4f..a15954b 100644
--- a/sziszapangma/integration/repository/fsspec_experiment_repository.py
+++ b/sziszapangma/integration/repository/fsspec_experiment_repository.py
@@ -1,21 +1,12 @@
+import datetime
import json
from typing import Any, Optional, Set
-import datetime
-import fsspec
-import s3fs
+
from fsspec import AbstractFileSystem
-from fsspec.implementations.local import LocalFileSystem
-from pymongo import MongoClient
from sziszapangma.integration.repository.experiment_repository import (
ExperimentRepository,
)
-from sziszapangma.integration.repository.mongo_experiment_repository import (
- MongoExperimentRepository,
-)
-from sziszapangma.integration.repository.multi_files_experiment_repository import (
- MultiFilesExperimentRepository,
-)
class FsspecExperimentRepository(ExperimentRepository):
diff --git a/sziszapangma/integration/repository/mongo_experiment_repository.py b/sziszapangma/integration/repository/mongo_experiment_repository.py
index 8f1d39b..ca3631b 100644
--- a/sziszapangma/integration/repository/mongo_experiment_repository.py
+++ b/sziszapangma/integration/repository/mongo_experiment_repository.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional, Set, Dict
+from typing import Any, Dict, Optional, Set
from pymongo import MongoClient
from pymongo.database import Database
diff --git a/sziszapangma/integration/repository/multi_files_experiment_repository.py b/sziszapangma/integration/repository/multi_files_experiment_repository.py
index 3d08f19..f1c7814 100644
--- a/sziszapangma/integration/repository/multi_files_experiment_repository.py
+++ b/sziszapangma/integration/repository/multi_files_experiment_repository.py
@@ -1,7 +1,7 @@
import json
import os.path
from pathlib import Path
-from typing import Any, List, Optional, Set, Dict
+from typing import Any, Dict, List, Optional, Set
from sziszapangma.integration.repository.experiment_repository import (
ExperimentRepository,
@@ -48,10 +48,8 @@ class MultiFilesExperimentRepository(ExperimentRepository):
def get_all_record_ids(self) -> Set[str]:
ids: List[str] = []
for property_name in self.get_all_properties():
- path = (
- self._root_directory
- .joinpath(self._experiment_name)
- .joinpath(property_name)
+ path = self._root_directory.joinpath(self._experiment_name).joinpath(
+ property_name
)
property_ids = set(
[children.name.replace(".json", "") for children in path.iterdir()]
@@ -65,19 +63,22 @@ class MultiFilesExperimentRepository(ExperimentRepository):
def _get_file_path(self, property_name: str, record_id: str) -> Path:
return (
- self._root_directory
- .joinpath(self._experiment_name)
+ self._root_directory.joinpath(self._experiment_name)
.joinpath(property_name)
.joinpath(f"{record_id}.json")
)
def get_all_record_ids_for_property(self, property_name: str) -> Set[str]:
- property_directory_path = self._root_directory / self._experiment_name / property_name
- return set([
- it.name.replace('.json', '')
- for it in property_directory_path.iterdir()
- if it.name.endswith('.json')
- ])
+ property_directory_path = (
+ self._root_directory / self._experiment_name / property_name
+ )
+ return set(
+ [
+ it.name.replace(".json", "")
+ for it in property_directory_path.iterdir()
+ if it.name.endswith(".json")
+ ]
+ )
def get_all_values_from_property(self, property_name: str) -> Dict[str, Any]:
ids = self.get_all_record_ids_for_property(property_name)
diff --git a/sziszapangma/integration/task/asr_task.py b/sziszapangma/integration/task/asr_task.py
index c144aa1..54b4253 100644
--- a/sziszapangma/integration/task/asr_task.py
+++ b/sziszapangma/integration/task/asr_task.py
@@ -5,7 +5,6 @@ from sziszapangma.integration.repository.experiment_repository import (
)
from sziszapangma.integration.task.processing_task import ProcessingTask
from sziszapangma.model.model_creators import create_new_word
-from sziszapangma.model.relation_manager import RelationManager
class AsrTask(ProcessingTask):
diff --git a/sziszapangma/integration/task/embedding_wer_metrics_task.py b/sziszapangma/integration/task/embedding_wer_metrics_task.py
index 22db6c1..71a17b3 100644
--- a/sziszapangma/integration/task/embedding_wer_metrics_task.py
+++ b/sziszapangma/integration/task/embedding_wer_metrics_task.py
@@ -118,14 +118,6 @@ class EmbeddingWerMetricsTask(ProcessingTask):
soft_wer = self._wer_calculator.calculate_wer(soft_alignment)
embedding_wer = self._wer_calculator.calculate_wer(embedding_alignment)
- alignment_results = {
- "soft_alignment": [
- AlignmentStepMapper.to_json_dict(it) for it in soft_alignment
- ],
- "embedding_alignment": [
- AlignmentStepMapper.to_json_dict(it) for it in embedding_alignment
- ],
- }
wer_results = {"soft_wer": soft_wer, "embedding_wer": embedding_wer}
print(wer_results)
diff --git a/sziszapangma/integration/task/flair_upos_multi_transformers_wer_processor_base.py b/sziszapangma/integration/task/flair_upos_multi_transformers_wer_processor_base.py
index 0433440..891f510 100644
--- a/sziszapangma/integration/task/flair_upos_multi_transformers_wer_processor_base.py
+++ b/sziszapangma/integration/task/flair_upos_multi_transformers_wer_processor_base.py
@@ -1,10 +1,9 @@
from typing import List
+
from flair.data import Sentence
from flair.models import SequenceTagger
-from sziszapangma.integration.task.sentence_wer_processor import (
- SentenceWerProcessor,
-)
+from sziszapangma.integration.task.sentence_wer_processor import SentenceWerProcessor
class FlairUposMultiTransformersWerProcessorBase(SentenceWerProcessor):
diff --git a/sziszapangma/integration/task/sentence_wer_processor.py b/sziszapangma/integration/task/sentence_wer_processor.py
index c05736e..08a569f 100644
--- a/sziszapangma/integration/task/sentence_wer_processor.py
+++ b/sziszapangma/integration/task/sentence_wer_processor.py
@@ -12,7 +12,6 @@ from sziszapangma.integration.repository.experiment_repository import (
)
from sziszapangma.integration.task.processing_task import ProcessingTask
from sziszapangma.model.model import Word
-from sziszapangma.model.relation_manager import RelationManager
class SentenceWerProcessor(ProcessingTask):
diff --git a/sziszapangma/integration/task/spacy_ner_sentence_wer_processor.py b/sziszapangma/integration/task/spacy_ner_sentence_wer_processor.py
index ec3fc40..3b78057 100644
--- a/sziszapangma/integration/task/spacy_ner_sentence_wer_processor.py
+++ b/sziszapangma/integration/task/spacy_ner_sentence_wer_processor.py
@@ -1,10 +1,8 @@
-from typing import List, Any
+from typing import Any, List
import spacy
-from sziszapangma.integration.task.sentence_wer_processor import (
- SentenceWerProcessor,
-)
+from sziszapangma.integration.task.sentence_wer_processor import SentenceWerProcessor
class SpacyNerSentenceWerProcessor(SentenceWerProcessor):
diff --git a/sziszapangma/integration/task/spacy_pos_sentence_dep_tag_processor.py b/sziszapangma/integration/task/spacy_pos_sentence_dep_tag_processor.py
index 34d6d86..57940d6 100644
--- a/sziszapangma/integration/task/spacy_pos_sentence_dep_tag_processor.py
+++ b/sziszapangma/integration/task/spacy_pos_sentence_dep_tag_processor.py
@@ -1,10 +1,8 @@
-from typing import List, Any
+from typing import Any, List
import spacy
-from sziszapangma.integration.task.sentence_wer_processor import (
- SentenceWerProcessor,
-)
+from sziszapangma.integration.task.sentence_wer_processor import SentenceWerProcessor
class SpacyDepTagSentenceWerProcessor(SentenceWerProcessor):
diff --git a/sziszapangma/integration/task/spacy_pos_sentence_wer_processor.py b/sziszapangma/integration/task/spacy_pos_sentence_wer_processor.py
index 38d351c..17254b7 100644
--- a/sziszapangma/integration/task/spacy_pos_sentence_wer_processor.py
+++ b/sziszapangma/integration/task/spacy_pos_sentence_wer_processor.py
@@ -1,10 +1,8 @@
-from typing import List, Any
+from typing import Any, List
import spacy
-from sziszapangma.integration.task.sentence_wer_processor import (
- SentenceWerProcessor,
-)
+from sziszapangma.integration.task.sentence_wer_processor import SentenceWerProcessor
class SpacyPosSentenceWerProcessor(SentenceWerProcessor):
diff --git a/sziszapangma/integration/task/wikineural_multilingual_ner_transformers_wer_processor_base.py b/sziszapangma/integration/task/wikineural_multilingual_ner_transformers_wer_processor_base.py
index d7d71b1..6e009e0 100644
--- a/sziszapangma/integration/task/wikineural_multilingual_ner_transformers_wer_processor_base.py
+++ b/sziszapangma/integration/task/wikineural_multilingual_ner_transformers_wer_processor_base.py
@@ -1,16 +1,14 @@
from typing import List
from transformers import (
- AutoTokenizer,
AutoModelForTokenClassification,
- pipeline,
+ AutoTokenizer,
Pipeline,
PreTrainedTokenizer,
+ pipeline,
)
-from sziszapangma.integration.task.sentence_wer_processor import (
- SentenceWerProcessor,
-)
+from sziszapangma.integration.task.sentence_wer_processor import SentenceWerProcessor
class WikineuralMultilingualNerTransformersWerProcessorBase(SentenceWerProcessor):
--
GitLab