Skip to content
Snippets Groups Projects
Commit ab07051d authored by piotrmp's avatar piotrmp
Browse files

Bug fix.

parent 9a7523c4
No related branches found
No related tags found
1 merge request!1Migration to UD 2.11
Show whitespace changes
Inline Side-by-side
src/lambo/utils/oscar.py
+ 5
− 3
View file @ ab07051d
...@@ -4,6 +4,7 @@ Functions used to obtain multilingual corpora from `OSCAR <https://oscar-corpus. ...@@ -4,6 +4,7 @@ Functions used to obtain multilingual corpora from `OSCAR <https://oscar-corpus.
import json import json
import random import random
import urllib import urllib
import time
from urllib.error import HTTPError from urllib.error import HTTPError
from lambo.data.document import Document from lambo.data.document import Document
...@@ -46,10 +47,11 @@ def download_archive1_from_oscar(language, path, OSCAR_LOGIN, OSCAR_PASSWORD, re ...@@ -46,10 +47,11 @@ def download_archive1_from_oscar(language, path, OSCAR_LOGIN, OSCAR_PASSWORD, re
return return
except HTTPError as err: except HTTPError as err:
error = err error = err
if i == retry - 1: if i == retry - 1 or err.code<500:
raise error raise error
time = ((i + 1) * (i + 1) * (i + 1) * 15) secs = ((i + 1) * (i + 1) * (i + 1) * 15)
print("[Got " + str(error.code) + ", retrying after " + str(time) + " seconds...]") print("[Got " + str(error.code) + ", retrying after " + str(secs) + " seconds...]")
time.sleep(secs)
def read_jsonl_to_documents(fileobj, MAX_LEN=3000000): def read_jsonl_to_documents(fileobj, MAX_LEN=3000000):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment