Commit aeda722a authored by Mateusz Gniewkowski's avatar Mateusz Gniewkowski

Merge branch 'developer' into 'master'

Developer

See merge request !4
parents 47844db7 548d445f
Pipeline #2935 passed with stages
in 17 minutes and 51 seconds
......@@ -28,7 +28,7 @@ class MewexWorker(NLPWorker):
)
files = args['input_files']
option = True
step = 1000
step = 10
counter = 0
if len(files) > step:
for counter, chunk in enumerate(
......@@ -56,12 +56,12 @@ class MewexWorker(NLPWorker):
def merge_data(self, path, counter):
"""Merge subfolder/mewex.csv to one mewex.csv."""
mewex = pd.read_csv(f'{path}/1/mewex.csv',
mewex = pd.read_csv(f'{path}/0/mewex.csv',
header=None,
skiprows=2,
delimiter='\t')
shutil.rmtree(f'{path}/1')
for filename in range(2, counter + 1):
shutil.rmtree(f'{path}/0')
for filename in range(1, counter + 1):
mewex = mewex.append(pd.read_csv(f'{path}/{filename}/mewex.csv',
header=None,
skiprows=2,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment