diff --git a/cli.py b/cli.py index 2b2dd65b3dbe84225514d9a988f88ff8b80ee1d1..1bc12e081ef7f486f72c1c48aa2dde31280667bc 100644 --- a/cli.py +++ b/cli.py @@ -1,6 +1,7 @@ -from src.worker import Worker import argparse +from src.worker import Worker + if __name__ == "__main__": parser = argparse.ArgumentParser(description="anonymizer") parser.add_argument("input_path", type=str, help="Path to the input file") diff --git a/main.py b/main.py index 399051287ac7f640eb677884ebad1cfc255aef05..42deb74f0338ff182595799a8fabb0b1f3a04bda 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,10 @@ """Implementation of anonymizer service.""" -import nlp_ws -from src.worker import Worker import logging + import nlp_ws +from src.worker import Worker + _log = logging.getLogger(__name__) diff --git a/print_config.py b/print_config.py index 2859f3c560b7db4bf2beb436e20bea2b30ef027d..a02740f1e505cdff7f6ad865b1d76b76ce6fde46 100644 --- a/print_config.py +++ b/print_config.py @@ -1,6 +1,7 @@ +import json + import hydra from omegaconf import OmegaConf -import json @hydra.main(config_path="config", config_name="config") diff --git a/scripts/cli.py b/scripts/cli.py index 6ede48f7e48e52cc53c56b21664b7143bb18f03b..ff8fbe5203d267ea028a89e2bd8500c42ef6f657 100644 --- a/scripts/cli.py +++ b/scripts/cli.py @@ -1,9 +1,8 @@ """Implementation of anonymizer service.""" import argparse - -from src.anonymizers.polish_anonymizer import PolishAnonymizer from src.anonymizers.english_anonymizer import EnglishAnonymizer +from src.anonymizers.polish_anonymizer import PolishAnonymizer from src.anonymizers.russian_anonymizer import RussianAnonymizer diff --git a/scripts/winer_input.py b/scripts/winer_input.py index fda1cc70d5db5b5b593e8d579a3261e85ed688cc..ec0726e189610d77db4e918bb5c63dfef96e5949 100644 --- a/scripts/winer_input.py +++ b/scripts/winer_input.py @@ -1,6 +1,7 @@ -from lpmn_client_biz import Connection, IOType, upload, download, Task, delete import json +from lpmn_client_biz import Connection, IOType, Task, delete, download, upload + lpmn = [ "morphodita", {"posconverter": {"input_fromat": "ccl", "output_fromat": "json"}}, diff --git a/src/detections/__init__.py b/src/detections/__init__.py index 554c9a5ead5a6afa9d79063aa9f4aab454480c67..9126ffb727d5685d795ba382a05aa416ac9e6efb 100644 --- a/src/detections/__init__.py +++ b/src/detections/__init__.py @@ -1,5 +1,5 @@ -from src.detections.detection import * from src.detections.date import * +from src.detections.detection import * from src.utils.subclasses import get_sublcasses DETECTION_CLASSES_MAP = { diff --git a/src/detections/date.py b/src/detections/date.py index b9e9888c5b8e941c3df57c1e2e0eb0d4d27db019..f5d01fd42db0e37c8b3ef87c6a364ad952c28719 100644 --- a/src/detections/date.py +++ b/src/detections/date.py @@ -1,5 +1,6 @@ +from typing import List, Optional, Tuple + from src.detections.detection import Detection -from typing import List, Tuple, Optional class DateDetection(Detection): diff --git a/src/detections/detection.py b/src/detections/detection.py index a2df4be5638b26db9bcf8c585df0039232d06da0..90704977cf905a7930a1485fa18fa5f1bfb00ea7 100644 --- a/src/detections/detection.py +++ b/src/detections/detection.py @@ -1,5 +1,5 @@ -from typing import Optional from dataclasses import dataclass +from typing import Optional @dataclass diff --git a/src/detectors/date/date.py b/src/detectors/date/date.py index 43544d9cc96ccd933ef429799e4f879daf781174..9e938ac10ebd4663c020f2a70302067aa7787f5e 100644 --- a/src/detectors/date/date.py +++ b/src/detectors/date/date.py @@ -1,9 +1,11 @@ -from typing import List, Dict, Any, Tuple +from typing import Any, Dict, List, Tuple + +from src.detections import DateDetection, Detection +from src.detectors.interface import Detector + from .en import detect_dates_en from .pl import detect_dates_pl from .ru import detect_dates_ru -from src.detections import Detection, DateDetection -from src.detectors.interface import Detector class DateDetector(Detector): diff --git a/src/detectors/date/en.py b/src/detectors/date/en.py index 90ed8430cacb82c114ea3d864e67241f37312020..22423da96e3ed1aca194d281ddec9681712ff29e 100644 --- a/src/detectors/date/en.py +++ b/src/detectors/date/en.py @@ -1,7 +1,8 @@ -import regex as re from typing import List, Tuple -from src.detections import DateDetection +import regex as re + +from src.detections import DateDetection from src.detectors.date.utils import parse_date_to_format EN_DATES_REGEX = re.compile( diff --git a/src/detectors/date/pl.py b/src/detectors/date/pl.py index fc539858c13641b1d9c9b7e792a76e348206b081..3000a750ba9d72f9d74768fd877fb97cae07b7ea 100644 --- a/src/detectors/date/pl.py +++ b/src/detectors/date/pl.py @@ -1,9 +1,10 @@ -import regex as re from typing import List, Tuple -from src.detections import DateDetection -from src.suppressors.order_based import OrderBasedSuppressor +import regex as re + +from src.detections import DateDetection from src.detectors.date.utils import parse_date_to_format +from src.suppressors.order_based import OrderBasedSuppressor PL_DATES_REGEX = re.compile( r"\b(?P<day_or_month_year>" diff --git a/src/detectors/date/ru.py b/src/detectors/date/ru.py index fbc18a0a688866f7360e652e5e9d2f6c3cb6d833..247d04004b645020776b61b42f2e85f758d4f994 100644 --- a/src/detectors/date/ru.py +++ b/src/detectors/date/ru.py @@ -1,7 +1,8 @@ -import regex as re from typing import List, Tuple -from src.detections import DateDetection +import regex as re + +from src.detections import DateDetection from src.detectors.date.utils import parse_date_to_format RU_DATES_REGEX = re.compile( diff --git a/src/detectors/date/utils.py b/src/detectors/date/utils.py index b239d3d3381fc452e8f1437c9e971a8c820c9b39..356c185d1e47ce1cb972c0f918f81f333b4963df 100644 --- a/src/detectors/date/utils.py +++ b/src/detectors/date/utils.py @@ -1,4 +1,5 @@ from typing import List, Tuple + from src.detections import DateDetection, Optional diff --git a/src/detectors/email/email.py b/src/detectors/email/email.py index f14d9fc3d18b05de3c6efe8a7c6ef8d1d3b4d16b..e63b3a40ef3f17e37a8812ab4dd3516ee754a386 100644 --- a/src/detectors/email/email.py +++ b/src/detectors/email/email.py @@ -1,5 +1,7 @@ +from typing import Any, Dict, List, Tuple + import regex as re -from typing import List, Dict, Any, Tuple + from src.detections import EmailDetection from src.detectors.interface import Detector diff --git a/src/detectors/interface.py b/src/detectors/interface.py index 6978b027242636d69e2599dfafa6e3b2d7c85dd1..ca682551284a04af5bdcfac8ce720b77c8a3dbaa 100644 --- a/src/detectors/interface.py +++ b/src/detectors/interface.py @@ -1,6 +1,7 @@ -from typing import List, Dict, Any, Tuple -from src.detections import Detection from abc import ABC, abstractmethod +from typing import Any, Dict, List, Tuple + +from src.detections import Detection class Detector(ABC): @@ -17,4 +18,3 @@ class Detector(ABC): Returns: List[Tuple[int, int, Detection]]: List of detections. """ - pass diff --git a/src/detectors/ner/ner.py b/src/detectors/ner/ner.py index f534da9c9514db5f82e685d2edb105680456964b..704a0abb0057095a47da833664eddde48d9ef712 100644 --- a/src/detectors/ner/ner.py +++ b/src/detectors/ner/ner.py @@ -1,8 +1,10 @@ -from typing import List, Dict, Any, Tuple +from typing import Dict, List, Tuple + +from src.annotations import (Annotation, MorphosyntacticAnnotation, + NerAnnotation) +from src.detections import (DETECTION_CLASSES_MAP, Detection, + MorphosyntacticInfoMixin) from src.detectors.interface import Detector -from src.detections import Detection, MorphosyntacticInfoMixin -from src.annotations import Annotation, NerAnnotation, MorphosyntacticAnnotation -from src.detections import DETECTION_CLASSES_MAP class NerDetector(Detector): diff --git a/src/detectors/number/number.py b/src/detectors/number/number.py index 1939bc61481e50feb478a92fbbad56b9c3abc123..b3fed57904b3e21eb7481fb8864ee73de24b699a 100644 --- a/src/detectors/number/number.py +++ b/src/detectors/number/number.py @@ -1,5 +1,7 @@ +from typing import Any, Dict, List, Tuple + import regex as re -from typing import List, Dict, Any, Tuple + from src.detections import NumberDetection from src.detectors.interface import Detector diff --git a/src/detectors/phone/phone.py b/src/detectors/phone/phone.py index 861a5e1fd35ed42125173d2ba5bc3e284ccf1531..9ba952094f74086055137710ae301b98c9953030 100644 --- a/src/detectors/phone/phone.py +++ b/src/detectors/phone/phone.py @@ -1,5 +1,7 @@ +from typing import Any, Dict, List, Tuple + import regex as re -from typing import List, Dict, Any, Tuple + from src.detections import PhoneNumberDetection from src.detectors.interface import Detector diff --git a/src/detectors/url/common.py b/src/detectors/url/common.py index cba0020f12cd3732c50c766f23360260b6a0cec3..d3e814dc283e1df611725fa70af14feea496de3e 100644 --- a/src/detectors/url/common.py +++ b/src/detectors/url/common.py @@ -1,5 +1,6 @@ +from typing import List + import regex as re -from typing import List, Tuple def generate_url_regex(exeptions: List[str]) -> str: diff --git a/src/detectors/url/url.py b/src/detectors/url/url.py index ac67b2d1744b3787b0814726064510c0259241b2..9104f050c98e4ce6df402897a5107827275c0638 100644 --- a/src/detectors/url/url.py +++ b/src/detectors/url/url.py @@ -1,10 +1,13 @@ +from typing import Any, Dict, List, Tuple + import regex as re -from typing import List, Dict, Any, Tuple -from .pl import URL_REGEX_PL -from .common import generate_url_regex + from src.detections import UrlDetection from src.detectors.interface import Detector +from .common import generate_url_regex +from .pl import URL_REGEX_PL + class UrlDetector(Detector): def __init__(self, language: str = "pl") -> None: diff --git a/src/detectors/user/user.py b/src/detectors/user/user.py index 3a891ab26b37fc8820bfbbe28975f9d4da47647c..c197352d9d4834308450433f99c1675ffa15e525 100644 --- a/src/detectors/user/user.py +++ b/src/detectors/user/user.py @@ -1,5 +1,7 @@ +from typing import Any, Dict, List, Tuple + import regex as re -from typing import List, Dict, Any, Tuple + from src.detections import UserDetection from src.detectors.interface import Detector diff --git a/src/dictionaries/morphosyntactic/__init__.py b/src/dictionaries/morphosyntactic/__init__.py index d0142845628e39c70f2797e4bcb78b2cdb325f18..22d33398d9452153349ae96274538e5532e6da96 100644 --- a/src/dictionaries/morphosyntactic/__init__.py +++ b/src/dictionaries/morphosyntactic/__init__.py @@ -1 +1,2 @@ -from src.dictionaries.morphosyntactic.interface import MorphosyntacticDictionary +from src.dictionaries.morphosyntactic.interface import \ + MorphosyntacticDictionary diff --git a/src/dictionaries/morphosyntactic/interface.py b/src/dictionaries/morphosyntactic/interface.py index 3ec644b11e239947d430fda6e9fbfc3982d1af82..8e22432a11f23f42c28c2f1aab1802d8ef46223e 100644 --- a/src/dictionaries/morphosyntactic/interface.py +++ b/src/dictionaries/morphosyntactic/interface.py @@ -1,6 +1,7 @@ -from src.detections import Detection -from typing import Optional, List, Type from abc import ABC, abstractmethod +from typing import List, Optional, Type + +from src.detections import Detection class MorphosyntacticDictionary(ABC): @@ -9,10 +10,8 @@ class MorphosyntacticDictionary(ABC): """ Returns a list of supported detection classess """ - pass def get_random_replacement(self, original_entry: Detection) -> Optional[str]: """ Returns a random replacement for the original entry """ - pass diff --git a/src/dictionaries/morphosyntactic/ner_file.py b/src/dictionaries/morphosyntactic/ner_file.py index e94cdb09991dd8bbf0e7eed04facfc223de92f71..e143403efe7330e9c384889ced04959924ad578f 100644 --- a/src/dictionaries/morphosyntactic/ner_file.py +++ b/src/dictionaries/morphosyntactic/ner_file.py @@ -1,8 +1,11 @@ -from typing import Dict, List, Optional, Type -from collections import defaultdict -from src.detections import Detection, MorphosyntacticInfoMixin, DETECTION_CLASSES_MAP -from src.dictionaries.morphosyntactic.interface import MorphosyntacticDictionary import random +from collections import defaultdict +from typing import List, Optional, Type + +from src.detections import (DETECTION_CLASSES_MAP, Detection, + MorphosyntacticInfoMixin) +from src.dictionaries.morphosyntactic.interface import \ + MorphosyntacticDictionary class NERFileMorphosyntacticDictionary(MorphosyntacticDictionary): diff --git a/src/dictionaries/morphosyntactic/ner_file_nkjp.py b/src/dictionaries/morphosyntactic/ner_file_nkjp.py index 6b8f49bdbd1731b479fa68ce5e06238eaaa90572..80eb16a2c5290ed13d6be4668395529b42cae3a2 100644 --- a/src/dictionaries/morphosyntactic/ner_file_nkjp.py +++ b/src/dictionaries/morphosyntactic/ner_file_nkjp.py @@ -1,8 +1,9 @@ -from typing import Optional -from src.detections import Detection, MorphosyntacticInfoMixin import random +from typing import Optional -from src.dictionaries.morphosyntactic.ner_file import NERFileMorphosyntacticDictionary +from src.detections import Detection, MorphosyntacticInfoMixin +from src.dictionaries.morphosyntactic.ner_file import \ + NERFileMorphosyntacticDictionary class NERFileNKJPMorphosyntacticDictionary(NERFileMorphosyntacticDictionary): diff --git a/src/input_parsers/ccl.py b/src/input_parsers/ccl.py index 8572a49a84083c800648d8b8e40d98aff44df280..44b5946d7980c9ed836d5b13b7aa42cfa7f679d3 100644 --- a/src/input_parsers/ccl.py +++ b/src/input_parsers/ccl.py @@ -1,9 +1,10 @@ -from typing import Dict, List, Tuple +from typing import List, Tuple + from lxml import etree -from collections import defaultdict +from src.annotations import (Annotation, MorphosyntacticAnnotation, + NerAnnotation) from src.input_parsers.interface import InputParser -from src.annotations import Annotation, MorphosyntacticAnnotation, NerAnnotation class CCLInputParser(InputParser): diff --git a/src/input_parsers/interface.py b/src/input_parsers/interface.py index 6309f918b7b5a1ecc7f2ab1a85852eb8b2a61e83..ce48fb9ec3ac18a4a844970c5f667e4c0cdf7c33 100644 --- a/src/input_parsers/interface.py +++ b/src/input_parsers/interface.py @@ -1,5 +1,5 @@ -from typing import Dict, List, Tuple, Any from abc import ABC, abstractmethod +from typing import Any, List, Tuple class InputParser(ABC): @@ -16,4 +16,3 @@ class InputParser(ABC): Returns: Tuple[str, Dict[str, List[Tuple[int, int, Any]]]]: Text and annotations. """ - pass diff --git a/src/input_parsers/wiktor_ner.py b/src/input_parsers/wiktor_ner.py index 14e195df987b0ba1dedcf8c3e0a19a46a99d62ed..cb7a75d63ad3a6d39951e7cc02e831b183648cf6 100644 --- a/src/input_parsers/wiktor_ner.py +++ b/src/input_parsers/wiktor_ner.py @@ -1,8 +1,9 @@ -from typing import List, Tuple import json +from typing import List, Tuple +from src.annotations import (Annotation, MorphosyntacticAnnotation, + NerAnnotation) from src.input_parsers.interface import InputParser -from src.annotations import Annotation, MorphosyntacticAnnotation, NerAnnotation class WiktorNERInputParser(InputParser): diff --git a/src/pipeline/default.py b/src/pipeline/default.py index cdf2db3463a4db3c319e238cde32835485b48c3b..0185154bad5399658060528da7f0057e92c42402 100644 --- a/src/pipeline/default.py +++ b/src/pipeline/default.py @@ -1,9 +1,10 @@ -from src.pipeline.interface import Pipeline from typing import Dict -from src.suppressors.interface import Suppressor + from src.detectors.interface import Detector -from src.replacers.interface import ReplacerInterface from src.input_parsers.interface import InputParser +from src.pipeline.interface import Pipeline +from src.replacers.interface import ReplacerInterface +from src.suppressors.interface import Suppressor class DefaultPipeline(Pipeline): diff --git a/src/pipeline/interface.py b/src/pipeline/interface.py index 9a183eb7ed756c7ea3173412ac02a73bc1c09186..4e92e24a18a215ff32518ceabdb10f98b3fe770c 100644 --- a/src/pipeline/interface.py +++ b/src/pipeline/interface.py @@ -12,4 +12,3 @@ class Pipeline(ABC): Returns: str: Anonymized text. """ - pass diff --git a/src/pipeline/sequential_jsonl.py b/src/pipeline/sequential_jsonl.py index 01effdb36a653011ebfc96dc79d597838e8e60ba..bfb464a79956ec05a1659d0319aaf23efaf04c21 100644 --- a/src/pipeline/sequential_jsonl.py +++ b/src/pipeline/sequential_jsonl.py @@ -1,10 +1,11 @@ -from src.pipeline.interface import Pipeline +import json from typing import Dict -from src.suppressors.interface import Suppressor + from src.detectors.interface import Detector -from src.replacers.interface import ReplacerInterface from src.input_parsers.interface import InputParser -import json +from src.pipeline.interface import Pipeline +from src.replacers.interface import ReplacerInterface +from src.suppressors.interface import Suppressor class SequentialJSONLPipeline(Pipeline): diff --git a/src/replacers/date_replacer.py b/src/replacers/date_replacer.py index 1ad698996f3a7212fadc94e75629f6c77c8bbf5a..011868f26ffb206f3a9e11aa7f7f6643083a0d19 100644 --- a/src/replacers/date_replacer.py +++ b/src/replacers/date_replacer.py @@ -1,11 +1,9 @@ +import random from typing import List, Tuple -from src.detections import ( - Detection, - DateDetection, -) -from src.string_replacements import replace_and_update + +from src.detections import DateDetection, Detection from src.replacers.interface import ReplacerInterface -import random +from src.string_replacements import replace_and_update # TODO: Add support for other languages months_map = { diff --git a/src/replacers/delete_replacer.py b/src/replacers/delete_replacer.py index b51413bc5ebd41d0c167f9b7bb9beea6a8cbd357..d8cd3e8afff3de6f9900ca9ed869a519862d3f3f 100644 --- a/src/replacers/delete_replacer.py +++ b/src/replacers/delete_replacer.py @@ -1,7 +1,8 @@ from typing import List, Tuple + from src.detections import Detection -from src.string_replacements import replace from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace class DeleteReplacer(ReplacerInterface): diff --git a/src/replacers/email_replacer.py b/src/replacers/email_replacer.py index cecba4d15af853536f878979e1aa31fd7864a1ff..f4f5f89d4a5413e8f14c9f352f3424c31dc9f889 100644 --- a/src/replacers/email_replacer.py +++ b/src/replacers/email_replacer.py @@ -1,12 +1,10 @@ -from typing import List, Tuple -from src.detections import ( - Detection, - EmailDetection, -) -from src.string_replacements import replace_and_update -from src.replacers.interface import ReplacerInterface import random import string +from typing import List, Tuple + +from src.detections import Detection, EmailDetection +from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace_and_update def random_char(char_num): diff --git a/src/replacers/interface.py b/src/replacers/interface.py index e2621c92aeb7ae8e1833fef621b1db8d22a6e4c0..dbf439633f10102bc017d07de619874bf135b6a6 100644 --- a/src/replacers/interface.py +++ b/src/replacers/interface.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from typing import List, Tuple + from src.detections import Detection @@ -19,4 +20,3 @@ class ReplacerInterface(ABC): replaced with anonimized version and list of detections that were not processed by this replacer. """ - pass diff --git a/src/replacers/ner_replacer.py b/src/replacers/ner_replacer.py index 56aee0d6ec2fca2ecadbf2de65806dfc50669fa5..98cf05523f4f6cae22b1485a79a42ed0a9d81239 100644 --- a/src/replacers/ner_replacer.py +++ b/src/replacers/ner_replacer.py @@ -1,10 +1,9 @@ from typing import List, Tuple -from src.detections import ( - Detection, -) -from src.string_replacements import replace_and_update -from src.replacers.interface import ReplacerInterface + +from src.detections import Detection from src.dictionaries.morphosyntactic import MorphosyntacticDictionary +from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace_and_update class NERReplacer(ReplacerInterface): diff --git a/src/replacers/number_replacer.py b/src/replacers/number_replacer.py index 14eaa25546c8aabed205be71c8e57b62da7c9b2d..6ae331a3852c3faae59e454ad86ca5de7208df33 100644 --- a/src/replacers/number_replacer.py +++ b/src/replacers/number_replacer.py @@ -1,12 +1,10 @@ -from typing import List, Tuple -from src.detections import ( - Detection, - NumberDetection, -) -from src.string_replacements import replace_and_update -from src.replacers.interface import ReplacerInterface import random import string +from typing import List, Tuple + +from src.detections import Detection, NumberDetection +from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace_and_update def randomize_digits_in_text(text: str) -> str: diff --git a/src/replacers/tag_replacer.py b/src/replacers/tag_replacer.py index 427a3d6946cb9c1f45007886470cd6e4106173d7..836cd14b9fd36d68e39ad60cc829d0b0ffc5cced 100644 --- a/src/replacers/tag_replacer.py +++ b/src/replacers/tag_replacer.py @@ -1,28 +1,15 @@ from typing import List, Tuple -from src.detections import ( - Detection, - NameDetection, - SurnameDetection, - StreetNameDetection, - LocationDetection, - OrganizationNameDetection, - ProperNameDetection, - CityDetection, - CountryDetection, - PhoneNumberDetection, - UrlDetection, - UserDetection, - EmailDetection, - DateDetection, - TINDetection, - TitleDetection, - HydronymDetection, - SerialNumberDetection, - KRSDetection, - NumberDetection, -) -from src.string_replacements import replace + +from src.detections import (CityDetection, CountryDetection, DateDetection, + Detection, EmailDetection, HydronymDetection, + KRSDetection, LocationDetection, NameDetection, + NumberDetection, OrganizationNameDetection, + PhoneNumberDetection, ProperNameDetection, + SerialNumberDetection, StreetNameDetection, + SurnameDetection, TINDetection, TitleDetection, + UrlDetection, UserDetection) from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace class TagReplacer(ReplacerInterface): diff --git a/src/replacers/user_replacer.py b/src/replacers/user_replacer.py index 97206d8a127ddacf697325234e3ab4ed43da450f..518e569fade8ff4427b2f382fdee0ce4faeff60d 100644 --- a/src/replacers/user_replacer.py +++ b/src/replacers/user_replacer.py @@ -1,12 +1,11 @@ from typing import List, Tuple -from src.detections import ( - Detection, - UserDetection, -) -from src.string_replacements import replace_and_update -from src.replacers.interface import ReplacerInterface + from random_username.generate import generate_username +from src.detections import Detection, UserDetection +from src.replacers.interface import ReplacerInterface +from src.string_replacements import replace_and_update + class UserReplacer(ReplacerInterface): def __init__(self): diff --git a/src/string_replacements.py b/src/string_replacements.py index 5b5f584a5e336a59000339f4be5c103d10084653..06182bd5713f3d0c774b44fa81c78c1c7da46ef1 100644 --- a/src/string_replacements.py +++ b/src/string_replacements.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Any, TypeVar +from typing import List, Tuple, TypeVar def replace(original_string: str, replacements: List[Tuple[int, int, str]]) -> str: diff --git a/src/suppressors/interface.py b/src/suppressors/interface.py index f6a3420f4f3b2df7eef15bada11c3e2fea905277..960449ce60304db4c47f9634b2f3ca247c1703eb 100644 --- a/src/suppressors/interface.py +++ b/src/suppressors/interface.py @@ -1,5 +1,5 @@ -from typing import List, Tuple, Any from abc import ABC, abstractmethod +from typing import Any, List, Tuple class Suppressor(ABC): diff --git a/src/suppressors/order_based.py b/src/suppressors/order_based.py index 1a6438f7e7b907711348cfb11aed3d63d5135040..c466c6918c77b00b9a792e706b2178d738f5d237 100644 --- a/src/suppressors/order_based.py +++ b/src/suppressors/order_based.py @@ -1,5 +1,7 @@ -from typing import List, Tuple, Dict, Any +from typing import Any, List, Tuple + from bitarray import bitarray + from src.suppressors.interface import Suppressor diff --git a/src/worker.py b/src/worker.py index ecbb0e843c3d02a6c05b8f8145e4bed1cbcacdb9..7c84fd0bf47ec8c7a741624afeeded6bc66237ad 100644 --- a/src/worker.py +++ b/src/worker.py @@ -1,5 +1,5 @@ """Implementation of nlp_worker.""" -from hydra import initialize, compose +from hydra import compose, initialize from hydra.utils import instantiate diff --git a/tests/integration/ccl_configuration/test_ccl_configuration.py b/tests/integration/ccl_configuration/test_ccl_configuration.py index 357bb32f3fe73db6b059c1a23076d24690c1e79c..463b89d4f123af442d95357096b5e9cae184a020 100644 --- a/tests/integration/ccl_configuration/test_ccl_configuration.py +++ b/tests/integration/ccl_configuration/test_ccl_configuration.py @@ -1,4 +1,4 @@ -from hydra import initialize, compose +from hydra import compose, initialize from hydra.utils import instantiate diff --git a/tests/integration/wiktorner_jsonl_configuration/test_wiktorner_jsonl_configuration.py b/tests/integration/wiktorner_jsonl_configuration/test_wiktorner_jsonl_configuration.py index 2b7261f7f6ba1985ead62575765ac8796e1b43fc..48133bfd7446515ec8774d4858a7102efa14c003 100644 --- a/tests/integration/wiktorner_jsonl_configuration/test_wiktorner_jsonl_configuration.py +++ b/tests/integration/wiktorner_jsonl_configuration/test_wiktorner_jsonl_configuration.py @@ -1,4 +1,4 @@ -from hydra import initialize, compose +from hydra import compose, initialize from hydra.utils import instantiate diff --git a/tests/integration/wiktorner_jsonl_txt_output_configuration/test_wiktorner_jsonl_txt_output_configuration.py b/tests/integration/wiktorner_jsonl_txt_output_configuration/test_wiktorner_jsonl_txt_output_configuration.py index 6974f224483edae9ca902c477e1d9e6b24171ca2..d8aded5ac20b403ecbb9145b90181f45a80173f4 100644 --- a/tests/integration/wiktorner_jsonl_txt_output_configuration/test_wiktorner_jsonl_txt_output_configuration.py +++ b/tests/integration/wiktorner_jsonl_txt_output_configuration/test_wiktorner_jsonl_txt_output_configuration.py @@ -1,4 +1,4 @@ -from hydra import initialize, compose +from hydra import compose, initialize from hydra.utils import instantiate diff --git a/tests/unit/detectors/ner/test_ner.py b/tests/unit/detectors/ner/test_ner.py index 33a4329b9bcd993bc558f79d04d772c0977ea219..f59e7227bac78107d43948702e45194f45bcd567 100644 --- a/tests/unit/detectors/ner/test_ner.py +++ b/tests/unit/detectors/ner/test_ner.py @@ -1,5 +1,5 @@ -from src.annotations import NerAnnotation, MorphosyntacticAnnotation -from src.detections import NameDetection, SurnameDetection, CityDetection +from src.annotations import MorphosyntacticAnnotation, NerAnnotation +from src.detections import CityDetection, NameDetection, SurnameDetection from src.detectors.ner import NerDetector diff --git a/tests/unit/detectors/url/test_url.py b/tests/unit/detectors/url/test_url.py index c47071e437db9c0f6a8459f4664fac140915237f..110d50323f74433ed5f9a6a2dfce36112c5df971 100644 --- a/tests/unit/detectors/url/test_url.py +++ b/tests/unit/detectors/url/test_url.py @@ -1,5 +1,5 @@ -from src.detectors.url import UrlDetector from src.detections import UrlDetection +from src.detectors.url import UrlDetector def test_detect_urls(): diff --git a/tests/unit/detectors/user/test_user.py b/tests/unit/detectors/user/test_user.py index 56220d6623a927bdaebb5f6d2d733326a0191722..f56218bbe32e53cf596ea4a5865759de45840942 100644 --- a/tests/unit/detectors/user/test_user.py +++ b/tests/unit/detectors/user/test_user.py @@ -1,5 +1,5 @@ -from src.detectors.user import UserDetector from src.detections import UserDetection +from src.detectors.user import UserDetector def test_detect_users(): diff --git a/tests/unit/dictionaries/morphosyntactic/test_ner_file.py b/tests/unit/dictionaries/morphosyntactic/test_ner_file.py index abe02cb18a83e321d6335680f6543c933a84a84c..f91286981a3cdc72f7d9121e447dd7e2b8209ac4 100644 --- a/tests/unit/dictionaries/morphosyntactic/test_ner_file.py +++ b/tests/unit/dictionaries/morphosyntactic/test_ner_file.py @@ -1,7 +1,9 @@ -from src.dictionaries.morphosyntactic.ner_file import NERFileMorphosyntacticDictionary -from src.detections import NameDetection, CityDetection, SurnameDetection from tempfile import NamedTemporaryFile +from src.detections import CityDetection, NameDetection, SurnameDetection +from src.dictionaries.morphosyntactic.ner_file import \ + NERFileMorphosyntacticDictionary + def test_ner_file_morphosyntactic_dictionary(): with NamedTemporaryFile() as file: diff --git a/tests/unit/dictionaries/morphosyntactic/test_ner_file_nkjp.py b/tests/unit/dictionaries/morphosyntactic/test_ner_file_nkjp.py index f886b552bb33f498c9e94303bfe6f792ee6c1ab8..c928e5bada57c27654f4b0ef22ae3dc7d08e4aeb 100644 --- a/tests/unit/dictionaries/morphosyntactic/test_ner_file_nkjp.py +++ b/tests/unit/dictionaries/morphosyntactic/test_ner_file_nkjp.py @@ -1,9 +1,9 @@ -from src.dictionaries.morphosyntactic.ner_file_nkjp import ( - NERFileNKJPMorphosyntacticDictionary, -) -from src.detections import NameDetection, CityDetection, SurnameDetection from tempfile import NamedTemporaryFile +from src.detections import CityDetection, NameDetection, SurnameDetection +from src.dictionaries.morphosyntactic.ner_file_nkjp import \ + NERFileNKJPMorphosyntacticDictionary + def test_ner_file_nkjp_morphosyntactic_dictionary(): with NamedTemporaryFile() as file: diff --git a/tests/unit/input_parsers/test_ccl.py b/tests/unit/input_parsers/test_ccl.py index 6032503eacc7803dfa2a50ea25d8cd203ed0fd47..9a41c7733cd1b3451f247da86b30dae845b00b54 100644 --- a/tests/unit/input_parsers/test_ccl.py +++ b/tests/unit/input_parsers/test_ccl.py @@ -1,5 +1,5 @@ +from src.annotations import MorphosyntacticAnnotation, NerAnnotation from src.input_parsers.ccl import CCLInputParser -from src.annotations import NerAnnotation, MorphosyntacticAnnotation example_ccl = """<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE chunkList SYSTEM "ccl.dtd"> diff --git a/tests/unit/input_parsers/test_wiktor_ner.py b/tests/unit/input_parsers/test_wiktor_ner.py index 04d8bf389fa4d62de51f57e9e995d879ae399de9..8b8fa73c2a97f6f38f4925980963c9cf2e0c2362 100644 --- a/tests/unit/input_parsers/test_wiktor_ner.py +++ b/tests/unit/input_parsers/test_wiktor_ner.py @@ -1,5 +1,5 @@ +from src.annotations import MorphosyntacticAnnotation, NerAnnotation from src.input_parsers.wiktor_ner import WiktorNERInputParser -from src.annotations import NerAnnotation, MorphosyntacticAnnotation example_json = """{ "filename": "test_filename", diff --git a/tests/unit/pipeline/test_default.py b/tests/unit/pipeline/test_default.py index 9131792659d08863627d692740fbda104513d825..c83189a6eb3075d075cf95060d4c541f251d2e0c 100644 --- a/tests/unit/pipeline/test_default.py +++ b/tests/unit/pipeline/test_default.py @@ -1,10 +1,11 @@ -from src.pipeline.default import DefaultPipeline +from tempfile import NamedTemporaryFile + from src.detections import NameDetection -from src.input_parsers.interface import InputParser from src.detectors.interface import Detector -from src.suppressors.interface import Suppressor +from src.input_parsers.interface import InputParser +from src.pipeline.default import DefaultPipeline from src.replacers.interface import ReplacerInterface -from tempfile import NamedTemporaryFile +from src.suppressors.interface import Suppressor class MockInputParser(InputParser): diff --git a/tests/unit/pipeline/test_sequential_jsonl.py b/tests/unit/pipeline/test_sequential_jsonl.py index ddbf45f73024398ddc65faaae94a3455c367d22f..aaaa63d9b5414c342b31715d7f33405a36b7e33f 100644 --- a/tests/unit/pipeline/test_sequential_jsonl.py +++ b/tests/unit/pipeline/test_sequential_jsonl.py @@ -1,10 +1,11 @@ -from src.pipeline.sequential_jsonl import SequentialJSONLPipeline +from tempfile import NamedTemporaryFile + from src.detections import NameDetection -from src.input_parsers.interface import InputParser from src.detectors.interface import Detector -from src.suppressors.interface import Suppressor +from src.input_parsers.interface import InputParser +from src.pipeline.sequential_jsonl import SequentialJSONLPipeline from src.replacers.interface import ReplacerInterface -from tempfile import NamedTemporaryFile +from src.suppressors.interface import Suppressor class MockInputParser(InputParser): diff --git a/tests/unit/replacers/test_date_replacer.py b/tests/unit/replacers/test_date_replacer.py index 08117016b1b1569946a559e2d99cb70606d1fe8e..2995438e7819d26b1c89cf045ce6966d44f24dc9 100644 --- a/tests/unit/replacers/test_date_replacer.py +++ b/tests/unit/replacers/test_date_replacer.py @@ -1,5 +1,6 @@ +from src.detections import (CityDetection, DateDetection, NameDetection, + SurnameDetection) from src.replacers.date_replacer import DateReplacer -from src.detections import NameDetection, SurnameDetection, DateDetection, CityDetection def test_date_replacer(): diff --git a/tests/unit/replacers/test_email_replacer.py b/tests/unit/replacers/test_email_replacer.py index e86b8c9266dee17e6aa4b3f65361e9711fd6f8f3..fdb25cc57798d25d89f541926c90bd0d9df96a3b 100644 --- a/tests/unit/replacers/test_email_replacer.py +++ b/tests/unit/replacers/test_email_replacer.py @@ -1,5 +1,6 @@ +from src.detections import (CityDetection, DateDetection, EmailDetection, + UserDetection) from src.replacers.email_replacer import EmailReplacer -from src.detections import DateDetection, CityDetection, UserDetection, EmailDetection def test_email_replacer(): diff --git a/tests/unit/replacers/test_ner_replacer.py b/tests/unit/replacers/test_ner_replacer.py index 8b7fc6561296b78ae80b99d3333930d08b167d09..6b262efbf648f40fef35cd4709a27a94ea561330 100644 --- a/tests/unit/replacers/test_ner_replacer.py +++ b/tests/unit/replacers/test_ner_replacer.py @@ -1,8 +1,11 @@ -from src.replacers.ner_replacer import NERReplacer -from src.detections import NameDetection, SurnameDetection, DateDetection, CityDetection -from src.dictionaries.morphosyntactic.ner_file import NERFileMorphosyntacticDictionary from tempfile import NamedTemporaryFile +from src.detections import (CityDetection, DateDetection, NameDetection, + SurnameDetection) +from src.dictionaries.morphosyntactic.ner_file import \ + NERFileMorphosyntacticDictionary +from src.replacers.ner_replacer import NERReplacer + def test_ner_replacer(): with NamedTemporaryFile() as file: diff --git a/tests/unit/replacers/test_tag_replacer.py b/tests/unit/replacers/test_tag_replacer.py index a13439479861c9a3a903b897f31a8df84a7943f3..35af41f5b460b53565d2e4f9da206655d518c99f 100644 --- a/tests/unit/replacers/test_tag_replacer.py +++ b/tests/unit/replacers/test_tag_replacer.py @@ -1,5 +1,6 @@ +from src.detections import (CityDetection, DateDetection, NameDetection, + SurnameDetection) from src.replacers.tag_replacer import TagReplacer -from src.detections import NameDetection, SurnameDetection, DateDetection, CityDetection def test_replace_with_tags(): diff --git a/tests/unit/replacers/test_user_replacer.py b/tests/unit/replacers/test_user_replacer.py index 8afaa51e808491a954077e13af3610c93540d6ae..3a49f2e5e33a7d65adb05871ec9013ab0611e4a9 100644 --- a/tests/unit/replacers/test_user_replacer.py +++ b/tests/unit/replacers/test_user_replacer.py @@ -1,5 +1,5 @@ +from src.detections import CityDetection, DateDetection, UserDetection from src.replacers.user_replacer import UserReplacer -from src.detections import DateDetection, CityDetection, UserDetection def test_user_replacer():