diff --git a/.dvc/config b/.dvc/config
index 158cbc50390f94de5ad59fabefb8311fa2435878..d852f2796014c3ad8f700b9d504eab0cedd28b3f 100644
--- a/.dvc/config
+++ b/.dvc/config
@@ -1,5 +1,6 @@
[core]
remote = clarin-biz-asr-benchmarks
+ autostage = true
['remote "clarin-biz-asr-benchmarks"']
url = s3://projects/clarin-biz-asr-benchmark/dvc
endpointurl = https://s3.clarin-pl.eu
diff --git a/.gitignore b/.gitignore
index c9151758700ff67a361783902bfe1957ab97e541..10f676d5d4d0d6704c3eb30567bc78bfdb3090b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -111,4 +111,3 @@ debug_run/
publish_to_theliver.sh
.idea
-/experiment_data
diff --git a/dvc.lock b/dvc.lock
new file mode 100644
index 0000000000000000000000000000000000000000..96eddc5e8f02ff730de5d0cb7711f5377442ab39
--- /dev/null
+++ b/dvc.lock
@@ -0,0 +1,17 @@
+schema: '2.0'
+stages:
+ import_luna_to_common_format:
+ cmd: PYTHONPATH=. python experiment/luna/import_dataset/import_luna.py
+ deps:
+ - path: experiment/luna/import_dataset/import_luna.py
+ md5: d938162187616f7e7390983ecb9e120b
+ size: 8269
+ - path: experiment_data/dataset/LUNA.PL
+ md5: d342155b1871e881797cf7da09d5dc3c.dir
+ size: 1578358645
+ nfiles: 4500
+ outs:
+ - path: experiment_data/dataset_relation_manager_data/luna
+ md5: c68722cc69375259a4d1a4b6a2bd4dc3.dir
+ size: 3016826
+ nfiles: 10
diff --git a/dvc.yaml b/dvc.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ef4354e84b0fbb2a92efd231e0e141f4990ad285
--- /dev/null
+++ b/dvc.yaml
@@ -0,0 +1,17 @@
+stages:
+
+ import_luna_to_common_format:
+ cmd: PYTHONPATH=. python experiment/luna/import_dataset/import_luna.py
+ deps:
+ - experiment/luna/import_dataset/import_luna.py
+ - experiment_data/dataset/LUNA.PL
+ outs:
+ - experiment_data/dataset_relation_manager_data/luna
+
+# luna_main_pipeline:
+# cmd: PYTHONPATH=. python experiment/luna/pipeline/luna_main.py
+# deps:
+# - experiment_data/dataset_relation_manager_data/luna
+# - experiment_data/dataset/LUNA.PL
+# outs:
+# - experiment_data/pipeline/luna
diff --git a/examples/import_dataset/luna/luna_record_provider.py b/examples/import_dataset/luna/luna_record_provider.py
deleted file mode 100644
index 56d30d38f0c7c4efc3619a9ea38a104a7c6e7c48..0000000000000000000000000000000000000000
--- a/examples/import_dataset/luna/luna_record_provider.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from typing import Dict, Set
-
-from sziszapangma.integration.path_filter import PathFilter
-from sziszapangma.integration.record_id_iterator import RecordIdIterator
-from sziszapangma.integration.record_path_provider import RecordPathProvider
-from sziszapangma.integration.relation_manager_provider import RelationManagerProvider
-from sziszapangma.model.relation_manager import RelationManager, FileRelationManager
-
-
-class LunaRecordProvider(RecordIdIterator, RecordPathProvider, RelationManagerProvider):
- _path_by_id: Dict[str, str]
-
- def __init__(self, path_filter: PathFilter):
- self._path_by_id = dict({
- self._get_id(it): it
- for it in path_filter.get_list_of_files()
- })
-
- def get_all_records(self) -> Set[str]:
- return set(self._path_by_id.keys())
-
- def get_path(self, record_id: str) -> str:
- return self._path_by_id[record_id]
-
- def get_item_file_path(self, record_id: str, file) -> str:
- return self._path_by_id[record_id]
-
- def get_relation_manager(self, record_id: str) -> RelationManager:
- basic_path = self.get_path(record_id)[:-4]
- return FileRelationManager(f'{basic_path}_ab_relations.csv', f'{basic_path}_ab_items.json')
-
- @staticmethod
- def _get_id(record_file_path: str) -> str:
- path = record_file_path.replace('.wav', '')
- return '/'.join(path.split('/')[-6:])
diff --git a/examples/import_dataset/luna/relation_manager_provider.py b/examples/import_dataset/luna/relation_manager_provider.py
deleted file mode 100644
index eb8241d19cbfcd32650f2fd366efbccf4c268655..0000000000000000000000000000000000000000
--- a/examples/import_dataset/luna/relation_manager_provider.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from examples.import_dataset.luna.luna_record_provider import LunaRecordProvider
-from sziszapangma.model.relation_manager import RelationManager, FileRelationManager
-
-
-class RelationManagerProvider:
- _luna_record_provider: LunaRecordProvider
-
- def __init__(self, luna_record_provider: LunaRecordProvider):
- self._luna_record_provider = luna_record_provider
-
- def get_relation_manager(self, record_id: str) -> RelationManager:
- basic_path = self._luna_record_provider.get_path(record_id)[:-4]
- return FileRelationManager(f'{basic_path}_relations.csv', f'{basic_path}_items.json')
diff --git a/examples/luna/load_datasource_luna.py b/examples/luna/load_datasource_luna.py
deleted file mode 100644
index 062a6c8e0ae0b903bb50955472b2b992a35b14c9..0000000000000000000000000000000000000000
--- a/examples/luna/load_datasource_luna.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from lhotse import AudioSource, SupervisionSegment
-
-
-def load_single_datasource(path: str) -> AudioSource:
- return AudioSource('file', [1], path)
diff --git a/examples/__init__.py b/experiment/__init__.py
similarity index 100%
rename from examples/__init__.py
rename to experiment/__init__.py
diff --git a/examples/luna/__init__.py b/experiment/luna/__init__.py
similarity index 100%
rename from examples/luna/__init__.py
rename to experiment/luna/__init__.py
diff --git a/examples/import_dataset/__init__.py b/experiment/luna/import_dataset/__init__.py
similarity index 100%
rename from examples/import_dataset/__init__.py
rename to experiment/luna/import_dataset/__init__.py
diff --git a/examples/import_dataset/luna/import_luna.py b/experiment/luna/import_dataset/import_luna.py
similarity index 87%
rename from examples/import_dataset/luna/import_luna.py
rename to experiment/luna/import_dataset/import_luna.py
index 30386326cc862b64363a6909da42a8048e79153d..673e1869f47c3a2b02705a499d930dd411fd3697 100644
--- a/examples/import_dataset/luna/import_luna.py
+++ b/experiment/luna/import_dataset/import_luna.py
@@ -1,8 +1,7 @@
-from pprint import pprint
from typing import Tuple, List
from xml.etree import ElementTree
-from examples.import_dataset.luna.luna_record_provider import LunaRecordProvider
+from experiment.luna.luna_record_provider import LunaRecordProvider
from sziszapangma.integration.path_filter import ExtensionPathFilter
from sziszapangma.model.model import Word, SingleAnnotation
from sziszapangma.model.model_creators import create_new_word, create_new_single_annotation, \
@@ -149,6 +148,7 @@ class LunaAdapter:
def import_record(self, record_id: str):
print(f'record {record_id}')
relation_manager = self._record_provider.get_relation_manager(record_id)
+ relation_manager.clear_all()
basic_path = self._record_provider.get_path(record_id)[:-4]
words_path = f"{basic_path}_words.xml"
@@ -157,27 +157,33 @@ class LunaAdapter:
turn_path = f"{basic_path}_turns.xml"
words, single_annotations = self.save_words(words_path, relation_manager)
- print('save_words')
+ # print('save_words')
self.read_concepts(words, concept_path, relation_manager)
- print('read_concepts')
+ # print('read_concepts')
self.read_chunks(words, chunks_path, relation_manager)
- print('read_chunks')
+ # print('read_chunks')
self.read_turns(words, turn_path, relation_manager)
- print('read_turns')
+ # print('read_turns')
relation_manager.commit()
+def main():
+ luna_directory = 'experiment_data/dataset/LUNA.PL'
+ luna_record_provider = LunaRecordProvider(
+ ExtensionPathFilter(
+ root_directory=luna_directory,
+ extension='wav'
+ ),
+ relation_manager_root_path='experiment_data/dataset_relation_manager_data/luna'
+ )
+ luna_adapter = LunaAdapter(luna_record_provider)
+ record_ids = list(luna_record_provider.get_all_records())
+ index = 0
+ for record_id in record_ids:
+ index += 1
+ print(f'{index}/{len(record_ids)}')
+ luna_adapter.import_record(record_id)
+
+
if __name__ == "__main__":
- luna_directory = '/Users/marcinwatroba/Desktop/LUNA/LUNA.PL'
- luna_record_provider = LunaRecordProvider(ExtensionPathFilter(
- root_directory=luna_directory,
- extension='wav'
- ))
- for it in list(luna_record_provider.get_all_records())[:1]:
- relation_manager = luna_record_provider.get_relation_manager(it)
- document = [itt for itt in relation_manager.get_all_items() if itt['type'] == 'Document'][0]
- document_words = [relation_manager.get_item_by_id(item_id)
- for item_id in document['word_ids']]
- all_relations = relation_manager.get_all_relations_for_item(document_words[0]['id'])
- ids = [it['second_id'] for it in all_relations if it['second_type'] in ['lemma', 'pos']]
- pprint([relation_manager.get_item_by_id(it) for it in ids])
+ main()
diff --git a/examples/luna/luna_record_provider.py b/experiment/luna/luna_record_provider.py
similarity index 68%
rename from examples/luna/luna_record_provider.py
rename to experiment/luna/luna_record_provider.py
index aa0f30a0fa1d516f61011d73f04fc1d8c0ca221d..34ac13c2a73b6041be22e904763b900e3f1f5b02 100644
--- a/examples/luna/luna_record_provider.py
+++ b/experiment/luna/luna_record_provider.py
@@ -1,3 +1,4 @@
+from pathlib import Path
from typing import Dict, Set
from sziszapangma.integration.path_filter import PathFilter
@@ -9,8 +10,10 @@ from sziszapangma.model.relation_manager import RelationManager, FileRelationMan
class LunaRecordProvider(RecordIdIterator, RecordPathProvider, RelationManagerProvider):
_path_by_id: Dict[str, str]
+ _relation_manager_root_path: str
- def __init__(self, path_filter: PathFilter):
+ def __init__(self, path_filter: PathFilter, relation_manager_root_path: str):
+ self._relation_manager_root_path = relation_manager_root_path
self._path_by_id = dict({
self._get_id(it): it
for it in path_filter.get_list_of_files()
@@ -22,12 +25,13 @@ class LunaRecordProvider(RecordIdIterator, RecordPathProvider, RelationManagerPr
def get_path(self, record_id: str) -> str:
return self._path_by_id[record_id]
- def get_item_file_path(self, record_id: str, file) -> str:
- return self._path_by_id[record_id]
-
def get_relation_manager(self, record_id: str) -> RelationManager:
- basic_path = self.get_path(record_id)[:-4]
- return FileRelationManager(f'{basic_path}_ab_relations.csv', f'{basic_path}_ab_items.json')
+ record_path = Path(self._relation_manager_root_path).joinpath(record_id)
+ record_path.mkdir(parents=True, exist_ok=True)
+ return FileRelationManager(
+ str(record_path.joinpath('ab_relations.csv')),
+ str(record_path.joinpath('ab_items.json'))
+ )
@staticmethod
def _get_id(record_file_path: str) -> str:
diff --git a/experiment/luna/move_asr.py b/experiment/luna/move_asr.py
new file mode 100644
index 0000000000000000000000000000000000000000..87910a00d85560a743d6f47909323b563b7114f7
--- /dev/null
+++ b/experiment/luna/move_asr.py
@@ -0,0 +1,21 @@
+import json
+from pprint import pprint
+
+from experiment.luna.luna_record_provider import LunaRecordProvider
+from sziszapangma.integration.path_filter import ExtensionPathFilter
+
+if __name__ == '__main__':
+ luna_directory = 'experiment_data/dataset/LUNA.PL'
+ luna_record_provider = LunaRecordProvider(
+ ExtensionPathFilter(
+ root_directory=luna_directory,
+ extension='wav'
+ ),
+ relation_manager_root_path='experiment_data/dataset_relation_manager_data/luna'
+ )
+ pprint(luna_record_provider.get_all_records())
+ for record_id in luna_record_provider.get_all_records():
+ path = f'experiment_data/cached_asr/LUNA_techmo_asr_cache/{record_id.replace("__", "/")}.wav.techmo.json'
+ raw = json.load(open(path, 'r'))
+ output_path = f'experiment_data/cached_asr/luna_techmo/{record_id}.json'
+ json.dump(raw, open(output_path, 'w'))
diff --git a/examples/import_dataset/luna/__init__.py b/experiment/luna/pipeline/__init__.py
similarity index 100%
rename from examples/import_dataset/luna/__init__.py
rename to experiment/luna/pipeline/__init__.py
diff --git a/examples/luna/luna_gold_transcript_processor.py b/experiment/luna/pipeline/luna_gold_transcript_processor.py
similarity index 90%
rename from examples/luna/luna_gold_transcript_processor.py
rename to experiment/luna/pipeline/luna_gold_transcript_processor.py
index b3d508a522a36b5f498a8156e9f0eec71e6d1fdf..4bfce0afaef4e8bbf618b9ccf5c1174cb0e43a0a 100644
--- a/examples/luna/luna_gold_transcript_processor.py
+++ b/experiment/luna/pipeline/luna_gold_transcript_processor.py
@@ -1,7 +1,7 @@
from typing import List, Dict
-# from examples.luna.luna_record_provider import LunaRecordProvider
-from examples.luna.luna_record_provider import LunaRecordProvider
+# from experiment.luna.luna_record_provider import LunaRecordProvider
+from experiment.luna.luna_record_provider import LunaRecordProvider
from sziszapangma.integration.gold_transcript_processor import GoldTranscriptProcessor
from sziszapangma.model.relation_manager import RelationManager
diff --git a/examples/luna/luna_main.py b/experiment/luna/pipeline/luna_main.py
similarity index 79%
rename from examples/luna/luna_main.py
rename to experiment/luna/pipeline/luna_main.py
index 98684c5f1b3097ac9e9179b716471a1dabcf505f..7b6b7193d161862af3b56957de023af27fd4555d 100644
--- a/examples/luna/luna_main.py
+++ b/experiment/luna/pipeline/luna_main.py
@@ -1,10 +1,11 @@
-from examples.luna.luna_gold_transcript_processor import LunaGoldTranscriptProcessor
-from examples.luna.luna_record_provider import LunaRecordProvider
-from examples.luna.pos_processing.asr_spacy_token_pos_processing_task import \
+from experiment.luna.pipeline.luna_gold_transcript_processor import LunaGoldTranscriptProcessor
+from experiment.luna.pipeline.pos_processing.asr_spacy_token_pos_processing_task import \
AsrSpacyTokenPosProcessingTask
-from examples.luna.pos_processing.gold_transcript_spacy_token_pos_processing_task import \
+from experiment.luna.pipeline.pos_processing.gold_transcript_spacy_token_pos_processing_task import \
GoldTranscriptSpacyTokenPosProcessingTask
-from examples.luna.pos_processing.spacy_pos_wer_processing_task import SpacyPosWerProcessingTask
+from experiment.luna.pipeline.pos_processing.spacy_pos_wer_processing_task import \
+ SpacyPosWerProcessingTask
+from experiment.luna.luna_record_provider import LunaRecordProvider
from sziszapangma.integration.asr_processor import AsrPathCacheClient
from sziszapangma.integration.experiment_manager import ExperimentManager
from sziszapangma.integration.path_filter import ExtensionPathFilter
@@ -15,7 +16,7 @@ from sziszapangma.integration.task.asr_task import AsrTask
from sziszapangma.integration.task.classic_wer_metric_task import ClassicWerMetricTask
from sziszapangma.integration.task.gold_transcript_task import GoldTranscriptTask
-LUNA_DIRECTORY = '/Users/marcinwatroba/Desktop/LUNA/LUNA.PL'
+LUNA_DIRECTORY = 'experiment_data/dataset/LUNA.PL'
GOLD_TRANSCRIPT = 'gold_transcript'
TECHMO_POLISH_ASR = 'techmo_polish_asr'
TECHMO_POLISH_CLASSIC_WER_METRIC = 'techmo_polish_classic_wer_metric'
@@ -27,10 +28,13 @@ POS_METRICS_WER = 'pos_metrics_wer'
def run_luna_experiment(experiment_repository: ExperimentRepository):
- record_provider = LunaRecordProvider(ExtensionPathFilter(
- root_directory=LUNA_DIRECTORY,
- extension='wav'
- ))
+ record_provider = LunaRecordProvider(
+ ExtensionPathFilter(
+ root_directory=LUNA_DIRECTORY,
+ extension='wav'
+ ),
+ relation_manager_root_path='experiment_data/dataset_relation_manager_data/luna'
+ )
experiment_processor = ExperimentManager(
record_id_iterator=record_provider,
processing_tasks=[
@@ -84,7 +88,8 @@ def run_luna_experiment(experiment_repository: ExperimentRepository):
def example_run():
- experiment_repository = MultiFilesExperimentRepository('experiment_data', 'asr_benchmark_luna')
+ experiment_repository = MultiFilesExperimentRepository(
+ 'experiment_data/pipeline', 'asr_benchmark_luna')
run_luna_experiment(experiment_repository)
diff --git a/examples/luna/pos_processing/__init__.py b/experiment/luna/pipeline/pos_processing/__init__.py
similarity index 100%
rename from examples/luna/pos_processing/__init__.py
rename to experiment/luna/pipeline/pos_processing/__init__.py
diff --git a/examples/luna/pos_processing/asr_spacy_token_pos_processing_task.py b/experiment/luna/pipeline/pos_processing/asr_spacy_token_pos_processing_task.py
similarity index 84%
rename from examples/luna/pos_processing/asr_spacy_token_pos_processing_task.py
rename to experiment/luna/pipeline/pos_processing/asr_spacy_token_pos_processing_task.py
index 1f6b11eda6eea1add22991b452b0c1a4a61bb93d..42a0e25175b9710f5e150c7382c8c550891b0540 100644
--- a/examples/luna/pos_processing/asr_spacy_token_pos_processing_task.py
+++ b/experiment/luna/pipeline/pos_processing/asr_spacy_token_pos_processing_task.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass
from typing import Any
-from examples.luna.pos_processing.spacy_token_pos_processing_task import SpacyTokenPosProcessingTask
+from experiment.luna.pipeline.pos_processing.spacy_token_pos_processing_task import SpacyTokenPosProcessingTask
@dataclass
diff --git a/examples/luna/pos_processing/gold_transcript_spacy_token_pos_processing_task.py b/experiment/luna/pipeline/pos_processing/gold_transcript_spacy_token_pos_processing_task.py
similarity index 85%
rename from examples/luna/pos_processing/gold_transcript_spacy_token_pos_processing_task.py
rename to experiment/luna/pipeline/pos_processing/gold_transcript_spacy_token_pos_processing_task.py
index 52861da05635ad2ac84cb55c016104e4093ce9ae..e142212aa4b5727e4e1d1e78d0206500fdc646fe 100644
--- a/examples/luna/pos_processing/gold_transcript_spacy_token_pos_processing_task.py
+++ b/experiment/luna/pipeline/pos_processing/gold_transcript_spacy_token_pos_processing_task.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass
from typing import Any
-from examples.luna.pos_processing.spacy_token_pos_processing_task import SpacyTokenPosProcessingTask
+from experiment.luna.pipeline.pos_processing.spacy_token_pos_processing_task import SpacyTokenPosProcessingTask
@dataclass
diff --git a/examples/luna/pos_processing/spacy_pos_wer_processing_task.py b/experiment/luna/pipeline/pos_processing/spacy_pos_wer_processing_task.py
similarity index 100%
rename from examples/luna/pos_processing/spacy_pos_wer_processing_task.py
rename to experiment/luna/pipeline/pos_processing/spacy_pos_wer_processing_task.py
diff --git a/examples/luna/pos_processing/spacy_token_pos_processing_task.py b/experiment/luna/pipeline/pos_processing/spacy_token_pos_processing_task.py
similarity index 100%
rename from examples/luna/pos_processing/spacy_token_pos_processing_task.py
rename to experiment/luna/pipeline/pos_processing/spacy_token_pos_processing_task.py
diff --git a/examples/luna/pos.ipynb b/experiment/luna/pos.ipynb
similarity index 100%
rename from examples/luna/pos.ipynb
rename to experiment/luna/pos.ipynb
diff --git a/experiment_data.dvc b/experiment_data.dvc
deleted file mode 100644
index 17694841b6d20e595e52196d97629dc52eba74ae..0000000000000000000000000000000000000000
--- a/experiment_data.dvc
+++ /dev/null
@@ -1,5 +0,0 @@
-outs:
-- md5: 599ed5f6253699162bcae7ec02a8cb10.dir
- size: 1654734492
- nfiles: 9101
- path: experiment_data
diff --git a/experiment_data/cached_asr/.gitignore b/experiment_data/cached_asr/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..eeada193c62a9d070b0bff4704c446c0a645a03a
--- /dev/null
+++ b/experiment_data/cached_asr/.gitignore
@@ -0,0 +1 @@
+/luna_techmo
diff --git a/experiment_data/cached_asr/luna_techmo.dvc b/experiment_data/cached_asr/luna_techmo.dvc
new file mode 100644
index 0000000000000000000000000000000000000000..120a4dfe5d294823e649fb01ab3d2a9a5cae0037
--- /dev/null
+++ b/experiment_data/cached_asr/luna_techmo.dvc
@@ -0,0 +1,5 @@
+outs:
+- md5: 033ea7b5434dded73bf869bfdd299462.dir
+ size: 4256479
+ nfiles: 500
+ path: luna_techmo
diff --git a/experiment_data/dataset/.gitignore b/experiment_data/dataset/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..beaf70115d311f2cddb1fe658108705439bbf737
--- /dev/null
+++ b/experiment_data/dataset/.gitignore
@@ -0,0 +1 @@
+/LUNA.PL
diff --git a/experiment_data/dataset/LUNA.PL.dvc b/experiment_data/dataset/LUNA.PL.dvc
new file mode 100644
index 0000000000000000000000000000000000000000..2fdb21a12b3f29baaba50bcda6851071c2abf747
--- /dev/null
+++ b/experiment_data/dataset/LUNA.PL.dvc
@@ -0,0 +1,5 @@
+outs:
+- md5: d342155b1871e881797cf7da09d5dc3c.dir
+ size: 1578358645
+ nfiles: 4500
+ path: LUNA.PL
diff --git a/experiment_data/dataset_relation_manager_data/.gitignore b/experiment_data/dataset_relation_manager_data/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..70bb48337a4a38ef11fefb66a7975abb75fc22c1
--- /dev/null
+++ b/experiment_data/dataset_relation_manager_data/.gitignore
@@ -0,0 +1 @@
+/luna
diff --git a/sziszapangma/integration/__pycache__/path_filter.cpython-38.pyc b/sziszapangma/integration/__pycache__/path_filter.cpython-38.pyc
index c41c7eb3674f5dc55a2723bb84bfaa528e31c5a4..b70f37888ee88dc389a9aca5fdb47356cc822f56 100644
Binary files a/sziszapangma/integration/__pycache__/path_filter.cpython-38.pyc and b/sziszapangma/integration/__pycache__/path_filter.cpython-38.pyc differ
diff --git a/sziszapangma/integration/path_filter.py b/sziszapangma/integration/path_filter.py
index a5e55f90c90db25a564e94a982edd39af8ec2161..3053335d5d8aec049b0b008af7414ff926b9bf16 100644
--- a/sziszapangma/integration/path_filter.py
+++ b/sziszapangma/integration/path_filter.py
@@ -34,6 +34,6 @@ class ExtensionPathFilter(PathFilter):
"""
Implementation of searching files with extension.
"""
- path_generator = Path(self._root_directory).glob(f"**/*.{self._extension}")
+ path_generator = Path(self._root_directory).glob(f"LUNA.PL/**/*.{self._extension}")
all_files = [str(it) for it in path_generator]
return all_files if self._files_limit is None else all_files[: self._files_limit]
diff --git a/sziszapangma/model/relation_manager.py b/sziszapangma/model/relation_manager.py
index 74ec58181bc2dc659018a476528a096c5231a490..64a742deb4f878525792fbb7a0ebb81cc138cacd 100644
--- a/sziszapangma/model/relation_manager.py
+++ b/sziszapangma/model/relation_manager.py
@@ -44,6 +44,10 @@ class RelationManager(ABC):
def get_all_items(self) -> List[UUIDable]:
pass
+ @abstractmethod
+ def clear_all(self) -> None:
+ pass
+
class FileRelationManager(RelationManager):
relations_csv_path: str
@@ -106,3 +110,8 @@ class FileRelationManager(RelationManager):
items_json = json.dumps(self.items_dict)
with open(self.items_json_path, "w") as f:
f.write(items_json)
+
+ def clear_all(self) -> None:
+ self.items_dict.clear()
+ self.relations_dataframe = self.relations_dataframe[0:0]
+