Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
combo
Manage
Activity
Members
Labels
Plan
Issues
20
Issue boards
Milestones
Wiki
Redmine
Code
Merge requests
2
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Syntactic Tools
combo
Commits
a843ed90
Commit
a843ed90
authored
2 years ago
by
Martyna Wiącek
Browse files
Options
Downloads
Patches
Plain Diff
unescape ascii characters
parent
2e2d1673
Branches
Branches containing commit
No related merge requests found
Pipeline
#9714
failed with stage
in 1 minute and 42 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
combo/data/api.py
+3
-2
3 additions, 2 deletions
combo/data/api.py
combo/predict.py
+2
-1
2 additions, 1 deletion
combo/predict.py
with
5 additions
and
3 deletions
combo/data/api.py
+
3
−
2
View file @
a843ed90
import
collections
import
collections
import
dataclasses
import
dataclasses
import
html
import
json
import
json
import
os
import
os
import
string
import
string
...
@@ -55,12 +56,12 @@ class Sentence:
...
@@ -55,12 +56,12 @@ class Sentence:
# numpy.savez(hash + '_trimmed.npz', self.relation_distribution[1:, 1:])
# numpy.savez(hash + '_trimmed.npz', self.relation_distribution[1:, 1:])
return
json
.
dumps
({
return
json
.
dumps
({
"
tokens
"
:
[(
t
.
token
,
t
.
lemma
,
t
.
upostag
,
t
.
xpostag
,
t
.
feats
,
t
.
head
,
t
.
deprel
)
for
t
in
self
.
tokens
],
"
tokens
"
:
[(
str
(
html
.
unescape
(
t
.
token
)),
str
(
html
.
unescape
(
t
.
lemma
))
,
t
.
upostag
,
t
.
xpostag
,
t
.
feats
,
t
.
head
,
t
.
deprel
)
for
t
in
self
.
tokens
],
# "sentence_embedding": self.sentence_embedding,
# "sentence_embedding": self.sentence_embedding,
"
head
"
:
[
t
.
head
for
t
in
self
.
tokens
],
"
head
"
:
[
t
.
head
for
t
in
self
.
tokens
],
"
relation_distribution_hash
"
:
hash
,
"
relation_distribution_hash
"
:
hash
,
"
path_file
"
:
str
(
os
.
path
.
join
(
save_relation_distribution_path
,
hash
+
'
.npz
'
))
"
path_file
"
:
str
(
os
.
path
.
join
(
save_relation_distribution_path
,
hash
+
'
.npz
'
))
},
cls
=
NumpyArrayEncoder
)
},
cls
=
NumpyArrayEncoder
,
ensure_ascii
=
False
)
def
__len__
(
self
):
def
__len__
(
self
):
return
len
(
self
.
tokens
)
return
len
(
self
.
tokens
)
...
...
This diff is collapsed.
Click to expand it.
combo/predict.py
+
2
−
1
View file @
a843ed90
...
@@ -58,6 +58,7 @@ class COMBO(predictor.Predictor):
...
@@ -58,6 +58,7 @@ class COMBO(predictor.Predictor):
sys
.
exit
(
1
)
sys
.
exit
(
1
)
def
predict
(
self
,
sentence
:
Union
[
str
,
List
[
str
],
List
[
List
[
str
]],
List
[
data
.
Sentence
]]):
def
predict
(
self
,
sentence
:
Union
[
str
,
List
[
str
],
List
[
List
[
str
]],
List
[
data
.
Sentence
]]):
sentence
=
sentence
.
replace
(
"
.
"
,
"
.
"
)
if
isinstance
(
sentence
,
str
):
if
isinstance
(
sentence
,
str
):
if
isinstance
(
self
.
_tokenizer
,
lambo
.
LamboTokenizer
):
if
isinstance
(
self
.
_tokenizer
,
lambo
.
LamboTokenizer
):
segmented
=
self
.
_tokenizer
.
segment
(
sentence
)
segmented
=
self
.
_tokenizer
.
segment
(
sentence
)
...
@@ -247,7 +248,7 @@ class COMBO(predictor.Predictor):
...
@@ -247,7 +248,7 @@ class COMBO(predictor.Predictor):
@classmethod
@classmethod
def
with_lambo_tokenizer
(
cls
,
model
:
models
.
Model
,
def
with_lambo_tokenizer
(
cls
,
model
:
models
.
Model
,
dataset_reader
:
allen_data
.
DatasetReader
,
lambo_model_name
:
str
=
'
e
n
'
):
dataset_reader
:
allen_data
.
DatasetReader
,
lambo_model_name
:
str
=
'
d
e
'
):
return
cls
(
model
,
dataset_reader
,
lambo
.
LamboTokenizer
(
lambo_model_name
))
return
cls
(
model
,
dataset_reader
,
lambo
.
LamboTokenizer
(
lambo_model_name
))
@classmethod
@classmethod
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment