From 9adcdc9a4b949763627056cc0515a95eb5c0e290 Mon Sep 17 00:00:00 2001
From: dcz <dcz@ipipan.waw.pl>
Date: Thu, 2 Nov 2023 13:08:53 +0100
Subject: [PATCH] Update database script. Documentation. Tests. Efficiency
improvements.
---
entries/static/entries/js/free_lus_list.js | 2 +-
entries/views.py | 9 +-
freelus/tests/__init__.py | 0
freelus/tests/test.py | 92 +++++++++++++++++++
freelus/views.py | 52 ++++++++++-
.../unification/free_lu/FreeLuEdit.vue | 9 +-
importer/Phrase.py | 25 ++---
importer/PhraseAttributes.py | 22 ++++-
importer/WalentyPreprocessXML.py | 16 ++--
importer/WalentyXML.py | 12 ++-
shellvalier/settings-test.py | 76 +++++++++++++++
syntax/management/commands/update_tei.py | 70 ++++++++++++++
unifier/views.py | 83 ++++++++---------
update_db.sh | 13 +++
14 files changed, 399 insertions(+), 82 deletions(-)
create mode 100644 freelus/tests/__init__.py
create mode 100644 freelus/tests/test.py
create mode 100644 shellvalier/settings-test.py
create mode 100644 syntax/management/commands/update_tei.py
create mode 100755 update_db.sh
diff --git a/entries/static/entries/js/free_lus_list.js b/entries/static/entries/js/free_lus_list.js
index 49c0aa3..4bd80e5 100644
--- a/entries/static/entries/js/free_lus_list.js
+++ b/entries/static/entries/js/free_lus_list.js
@@ -1,7 +1,7 @@
function setup_free_lus_list(options) {
const can_see_assignees = has_permission("users.view_assignment");
- const ajaxURL = can_see_assignees ? '/' + lang + '/entries/get_entries/?with_lexical_units=false&without_frames=true' : '/' + lang + '/unifier/get_unified_frames/?exclude_status=N&restrict_to_user='+window.USER_USERNAME;
+ const ajaxURL = can_see_assignees ? '/' + lang + '/entries/get_entries/?with_lexical_units=false&without_frames=true&show_linked_entries_disabled=true' : '/' + lang + '/unifier/get_unified_frames/?exclude_status=N&show_linked_entries_disabled=true&restrict_to_user='+window.USER_USERNAME;
const datatable = setup_datatable({
element: options.table,
diff --git a/entries/views.py b/entries/views.py
index e569546..3c6c04c 100644
--- a/entries/views.py
+++ b/entries/views.py
@@ -437,6 +437,7 @@ def get_entries(request):
exclude_status = request.GET.get('exclude_status')
restrict_to_user = request.GET.get('restrict_to_user')
has_unified_frame = request.GET.get('has_unified_frame')
+ show_linked_entries_disabled = request.GET.get('show_linked_entries_disabled')
entries = get_filtered_objects(forms).filter(import_error=False)
# TODO restrictions for testing – remove!!!
@@ -445,7 +446,8 @@ def get_entries(request):
#entries = entries.filter(subentries__schema_hooks__alternation=2)
if without_frames:
- entries = entries.filter(Exists(LexicalUnit.objects.filter(base=OuterRef('name'), frames__isnull=True)))
+ entries = entries.filter(name__in=(LexicalUnit.objects.filter(frames__isnull=True).values("base")))
+ print('entries.query: ', entries.query)
total = entries.count()
if scroller_params['filter']:
@@ -458,7 +460,7 @@ def get_entries(request):
assert(not errors_dict)
linked_ids = set()
- if request.session['show_linked_entries'] and has_unified_frame != 'true':
+ if request.session['show_linked_entries'] and has_unified_frame != 'true' and not show_linked_entries_disabled:
entries_linked = Entry.objects.filter(pk__in=(
Entry.objects
.filter(subentries__schema_hooks__argument_connections__schema_connections__subentry__entry__in=entries)
@@ -863,7 +865,8 @@ def get_entry(request):
apply_filters = not simplejson.loads(request.POST['no_filters'])
filter_schemata = apply_filters and entry_form.cleaned_data['filter_schemata']
filter_frames = apply_filters and entry_form.cleaned_data['filter_frames']
- lexical_unit = LexicalUnit.objects.get(pk=lu_id) if (lu_id := request.POST.get("lexical_unit_id")) else None
+ lu_id = request.POST.get("lexical_unit_id")
+ lexical_unit = LexicalUnit.objects.get(pk=lu_id) if lu_id else None
if filter_schemata:
schema_forms = []
# e.g. entry has schema that satisfies X & entry has schema that satisfies Y
diff --git a/freelus/tests/__init__.py b/freelus/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/freelus/tests/test.py b/freelus/tests/test.py
new file mode 100644
index 0000000..9879bea
--- /dev/null
+++ b/freelus/tests/test.py
@@ -0,0 +1,92 @@
+import json
+
+from django.test import Client
+from django.test import RequestFactory, TestCase
+
+# initialize the APIClient app
+from connections.models import Entry, POS, Status, ExampleConnection
+from examples.models import Example, ExampleOpinion, ExampleSource
+from freelus.views import create_new_slowal_frame, add_argument_to_frame, attach_examples_to_frame
+from meanings.models import LexicalUnit, Synset
+from semantics.models import FrameOpinion, Frame, SemanticRole, ArgumentRole
+
+client = Client()
+
+
+class ApiTest(TestCase):
+
+ def tearDown(self) -> None:
+ pass
+
+ @classmethod
+ def tearDownClass(cls):
+ pass
+
+ def setUp(self):
+ self.factory = RequestFactory()
+ FrameOpinion.objects.create(key='unk', priority=1)
+ self.synset = Synset.objects.create(id=1, definition='definition')
+ self.lu = LexicalUnit.objects.create(base='test', sense=1, pos='A', synset=self.synset)
+ pos = POS.objects.create(tag='test_tag')
+ status = Status.objects.create(key='status_key', priority=1)
+ self.entry = Entry.objects.create(name='test', status=status, pos=pos)
+ self.semantic_role = SemanticRole.objects.create(role='test_role', priority=1)
+ self.argument_role = ArgumentRole.objects.create(role=self.semantic_role)
+ example_opinion = ExampleOpinion.objects.create(key='key', priority=1)
+ example_source = ExampleSource.objects.create(key='key', priority=1)
+ self.example = Example.objects.create(entry=self.entry, sentence='test sentence', opinion=example_opinion, source=example_source)
+
+ def test_create_new_slowal_frame(self):
+ request = self.factory.post("/pl/freelus/create_new_slowal_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'entry_id': self.entry.pk,
+ 'lu_ids': json.dumps([self.lu.pk], separators=(',', ':'))})
+ response = create_new_slowal_frame(request)
+ self.assertEqual(1, Frame.objects.all().count())
+ frame = Frame.objects.all()[0]
+ self.assertEqual(0, len(frame.arguments.all()))
+
+ def test_add_argument_to_frame(self):
+
+ request = self.factory.post("/pl/freelus/create_new_slowal_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'entry_id': self.entry.pk,
+ 'lu_ids': json.dumps([self.lu.pk], separators=(',', ':'))})
+ response = create_new_slowal_frame(request)
+
+ frame = Frame.objects.all()[0]
+ request = self.factory.post("/pl/freelus/add_argument_to_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'frame_id': frame.pk,
+ 'role_id': self.argument_role.pk,
+ 'role_type': 1})
+ response = add_argument_to_frame(request)
+ frame = Frame.objects.all()[0]
+ self.assertEqual(1, len(frame.arguments.all()))
+
+ def test_attach_examples_to_frame(self):
+
+ request = self.factory.post("/pl/freelus/create_new_slowal_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'entry_id': self.entry.pk,
+ 'lu_ids': json.dumps([self.lu.pk], separators=(',', ':'))})
+ response = create_new_slowal_frame(request)
+
+ frame = Frame.objects.all()[0]
+ request = self.factory.post("/pl/freelus/add_argument_to_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'frame_id': frame.pk,
+ 'role_id': self.argument_role.pk,
+ 'role_type': 1})
+ response = add_argument_to_frame(request)
+ argument = frame.arguments.all()[0]
+
+ request = self.factory.post("/pl/freelus/attach_examples_to_frame/",
+ HTTP_X_REQUESTED_WITH='XMLHttpRequest',
+ data={'frame_id': frame.pk,
+ 'argument_id': argument.pk,
+ 'example_ids': json.dumps([self.example.pk], separators=(',', ':'))})
+ response = attach_examples_to_frame(request)
+
+ examples = ExampleConnection.objects.filter(arguments=argument)
+ self.assertEqual(1, examples.count())
diff --git a/freelus/views.py b/freelus/views.py
index a32a18c..d492fe0 100644
--- a/freelus/views.py
+++ b/freelus/views.py
@@ -12,6 +12,12 @@ from semantics.models import Frame, FrameOpinion, Argument, ArgumentRole, Semant
@ajax_required
@transaction.atomic
def change_status(request):
+ """
+ Changing status of slowal frame in building process.
+ The request has to contain 'entry_id' and 'status'.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
entry_id = request.POST['entry_id']
status = request.POST['status']
@@ -27,6 +33,12 @@ def change_status(request):
@ajax_required
@transaction.atomic
def create_new_slowal_frame(request):
+ """
+ Create new empty slowal frame for lus without sementic layer.
+ The request has to contain 'entry_id' and 'lu_ids' list for witch slowal frame has to be created.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
entry_id = request.POST['entry_id']
lu_ids = json.loads(request.POST['lu_ids'])
@@ -51,6 +63,12 @@ def create_new_slowal_frame(request):
@ajax_required
@transaction.atomic
def add_argument_to_frame(request):
+ """
+ Creating a new argument for the specified slowal frame in building process.
+ The request has to contain 'frame_id' that represents slowal frame in building process.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
frame_id = request.POST['frame_id']
frame = Frame.objects.get(id=frame_id)
@@ -67,6 +85,13 @@ def add_argument_to_frame(request):
@ajax_required
@transaction.atomic
def change_role(request):
+ """
+ Changing role of a specified slowal frame argument.
+ The request has to contain 'frame_id' and 'argument_id' that represents slowal frame in building process.
+ As well as other params required for role change ('role_id', 'role_type', 'attribute_id', 'sub_attribute_id').
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
frame_id = request.POST['frame_id']
argument_id = request.POST['argument_id']
@@ -79,7 +104,6 @@ def change_role(request):
def change_role_base(frame_argument, request):
role_id = request.POST['role_id']
- role_type = request.POST['role_type']
attribute_id = request.POST.get('attribute_id', None)
sub_attribute_id = request.POST.get('sub_attribute_id', None)
@@ -93,12 +117,18 @@ def change_role_base(frame_argument, request):
argument_role.save()
frame_argument.role = argument_role
- frame_argument.role_type = RoleType.objects.get(type=role_type)
@ajax_required
@transaction.atomic
def attach_examples_to_frame(request):
+ """
+ Attaching selected examples to the slowal frame in building process.
+ The request has to contain 'frame_id' and 'argument_id' that represents slowal frame in building process.
+ List of example ids ('example_ids') is also required.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
frame_id = request.POST['frame_id']
argument_id = request.POST['argument_id']
@@ -122,6 +152,12 @@ def attach_examples_to_frame(request):
@ajax_required
@transaction.atomic
def attach_schema_to_argument(request):
+ """
+ Attaching selected schema to the specifier argument of slowal frame in building process.
+ The request has to contain 'frame_id', 'argument_id', schema_id and 'schema_position_id'.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
frame_id = request.POST['frame_id']
argument_id = request.POST['argument_id']
@@ -150,6 +186,12 @@ def attach_schema_to_argument(request):
@ajax_required
@transaction.atomic
def delete_schema_to_argument_connection(request):
+ """
+ Deleting a schema and frame argument connection.
+ The request has to contain 'argument_id', schema_id and 'schema_position_id'.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
argument_id = request.POST['argument_id']
schema_id = request.POST['schema_id']
@@ -169,6 +211,12 @@ def delete_schema_to_argument_connection(request):
@ajax_required
@transaction.atomic
def finish_frame_processing(request):
+ """
+ Changing frame status to created.
+ The request has to contain 'frame_id'.
+ :param request: http request
+ :return: Empty json response
+ """
if request.method == 'POST':
frame_id = request.POST['frame_id']
diff --git a/frontend/src/components/unification/free_lu/FreeLuEdit.vue b/frontend/src/components/unification/free_lu/FreeLuEdit.vue
index 9f563bb..dc58af9 100644
--- a/frontend/src/components/unification/free_lu/FreeLuEdit.vue
+++ b/frontend/src/components/unification/free_lu/FreeLuEdit.vue
@@ -293,12 +293,7 @@
let subAttributesHTML = role_sub_attributes.map(subAttribute => {
return `<label><input type="radio" name="sub_attribute" value="${subAttribute.id}" /> ${subAttribute.sub_attribute}</label><br />`;
}).join("");
- // const roleTypeHTML = ['required', 'typical'].map(type => {
- const roleTypeHTML = ['role', 'modifier'].map(type => {
- return `<label><input type="radio" name="role_type" value="${type}" /> ${this.roleStrMapping(type)}</label><br />`;
- }).join("");
return '<div class="row">' +
- '<div class="column"><div class="role_select_header">Type</div>' + roleTypeHTML + '</div>' +
'<div class="column"><div class="role_select_header">Role</div>' + rolesHTML + '</div>' +
'<div class="column"><div class="role_select_header">Atrybuty</div>' + attributesHTML + '</div>' +
'<div class="column"><div class="role_select_header">Podatrybuty</div>' + subAttributesHTML + '</div>' +
@@ -319,9 +314,8 @@
e.preventDefault();
const role_id = normalizeFormData(f.role)[0];
- const role_type = normalizeFormData(f.role_type)[0];
- if (role_id != null && role_type != null) {
+ if (role_id != null) {
const attribute_id = normalizeFormData(f.attribute)[0];
const sub_attribute_id = normalizeFormData(f.sub_attribute)[0];
@@ -329,7 +323,6 @@
const data = {
'frame_id': this.frame_in_progress.id,
'argument_id': selected_frame_argument_id,
- 'role_type': role_type,
'role_id': role_id,
'attribute_id': attribute_id,
'sub_attribute_id': sub_attribute_id
diff --git a/importer/Phrase.py b/importer/Phrase.py
index 6f112da..090fe28 100644
--- a/importer/Phrase.py
+++ b/importer/Phrase.py
@@ -208,10 +208,13 @@ class NonLexPhrase(Phrase):
phraseologic=False,
defaults={'priority': 2})
attributes = empty_attributes() if self._no_attributes else get_attributes(self, stored_positions)
- phrase, _ = PhraseType.objects.get_or_create(main_type=main_type,
- attributes=attributes,
- lexicalized_phrase=None,
- text_rep=str(self))
+
+ text_rep = str(self)
+ print('text_rep: ', text_rep, 'main_type: ', main_type, 'attributes: ', attributes)
+
+ phrase, _ = PhraseType.objects.get_or_create(text_rep=text_rep, defaults={'main_type': main_type,
+ 'attributes': attributes,
+ 'lexicalized_phrase': None})
# position is None for nested lex phrases
if position is not None:
@@ -263,13 +266,13 @@ class LexPhrase(Phrase):
lemma_operator, lemma_cooccur, lemmata, modification = None, None, None, None
attributes = empty_attributes() if self._no_attributes else get_lex_attributes(self, stored_positions)
- phrase, _ = PhraseType.objects.get_or_create(main_type=main_type,
- attributes=attributes,
- lexicalized_phrase=lex,
- lemma_operator=lemma_operator,
- lemma_cooccur=lemma_cooccur,
- modification=modification,
- text_rep=str(self))
+ phrase, _ = PhraseType.objects.get_or_create(text_rep=str(self),
+ defaults={'main_type': main_type,
+ 'attributes': attributes,
+ 'lexicalized_phrase': lex,
+ 'lemma_operator': lemma_operator,
+ 'lemma_cooccur': lemma_cooccur,
+ 'modification': modification})
if lemmata:
phrase.lemmata.set(lemmata)
diff --git a/importer/PhraseAttributes.py b/importer/PhraseAttributes.py
index 7a05884..2f53d0a 100644
--- a/importer/PhraseAttributes.py
+++ b/importer/PhraseAttributes.py
@@ -1,5 +1,6 @@
#! /usr/bin/python
# -*- coding: utf-8 -*-
+from django.db.models import Count
from syntax.models_phrase import *
from syntax.models import Position
@@ -341,11 +342,26 @@ def do_get_attributes(phrase, stored_positions, lex):
if attrs_key in stored_attributes:
attrs = stored_attributes[attrs_key]
else:
- attrs = cls.objects.create(**kwargs)
+ query = cls.objects.filter(**kwargs)
for key, values in m2m_attrs.items():
if values:
- getattr(attrs, key).set(values)
- stored_attributes[attrs_key] = attrs
+ query = query.annotate(count=Count(key)).filter(count=len(values))
+ for val in values:
+ query = query.filter(**{key: val})
+ # field_name = 'text_rep' if 'PhraseType' in key else 'name'
+ # query = query.filter(**{"{}__{}".format(key, field_name): val})
+ else:
+ query = query.filter(**{"{}__isnull".format(key): True})
+
+ if query.count() == 1:
+ stored_attributes[attrs_key] = query[0]
+ attrs = query[0]
+ else:
+ attrs = cls.objects.create(**kwargs)
+ for key, values in m2m_attrs.items():
+ if values:
+ getattr(attrs, key).set(values)
+ stored_attributes[attrs_key] = attrs
return attrs
return None
diff --git a/importer/WalentyPreprocessXML.py b/importer/WalentyPreprocessXML.py
index fb0b8a2..240d440 100644
--- a/importer/WalentyPreprocessXML.py
+++ b/importer/WalentyPreprocessXML.py
@@ -85,13 +85,15 @@ class WalentyPreprocessTeiHandler(handler.ContentHandler):
if lus:
lu = lus[0]
changed = False
- assert(lu.luid == meaning._luid)
- assert(lu.synset.id == meaning._sid)
- if lu.gloss != meaning._gloss:
- print(' updating gloss for :', lu, ' --- ', repr(lu.gloss), '->', repr(meaning._gloss))
- lu.gloss = meaning._gloss
- if changed:
- lu.save()
+ if lu.luid == meaning._luid:
+ print(' lu.luid != meaning._luid :', lu.luid, ' != ', meaning._luid, ', name: ', meaning._name, ', variant: ', meaning._variant)
+ assert(lu.luid == meaning._luid)
+ assert(lu.synset.id == meaning._sid)
+ if lu.gloss != meaning._gloss:
+ print(' updating gloss for :', lu, ' --- ', repr(lu.gloss), '->', repr(meaning._gloss))
+ lu.gloss = meaning._gloss
+ if changed:
+ lu.save()
else:
print(' new lu: {}-{}-{}-{}'.format(meaning._id, meaning._name, meaning._variant, pos))
meaning.save(pos)
diff --git a/importer/WalentyXML.py b/importer/WalentyXML.py
index 3c70a4e..d597931 100644
--- a/importer/WalentyXML.py
+++ b/importer/WalentyXML.py
@@ -4,6 +4,8 @@
import traceback
from xml.sax import handler
+
+import connections.models
from importer.Entry import Entry
examples_out_file = 'examples_ambig.txt'
@@ -75,10 +77,14 @@ class WalentyTeiHandler(handler.ContentHandler):
base = self._subtree._children[0]._children[0]._content
try:
entry = Entry(self._subtree, self._entry_meanings, self._meanings, self._frames, self._examples_in, self._examples_out, self._misconnected_out)
- if entry._status == '(S) sprawdzone' or entry._status == 'sprawdzone' or entry._status == '(S) gotowe':
- entry.store(self._meanings, self._stored_positions)
+ if not connections.models.Entry.objects.filter(id=int(entry._id)).exists():
+ print("Entry not exists in database: {}, status: {}".format(entry._base, entry._status))
+ if entry._status == '(S) sprawdzone' or entry._status == 'sprawdzone' or entry._status == '(S) gotowe':
+ entry.store(self._meanings, self._stored_positions)
+ else:
+ print("Odrzucono niegotowe: {}, status: {}".format(entry._base, entry._status))
else:
- print("Odrzucono niegotowe: {}, status: {}".format(entry._base, entry._status))
+ print("Entry exists in database: {}, status: {}".format(entry._base, entry._status))
except Exception as e:
#raise
traceback.print_exc()
diff --git a/shellvalier/settings-test.py b/shellvalier/settings-test.py
new file mode 100644
index 0000000..dd60eac
--- /dev/null
+++ b/shellvalier/settings-test.py
@@ -0,0 +1,76 @@
+
+from .environment import get_environment, boolean_mapper
+
+DEBUG = get_environment('DEBUG', mapper=boolean_mapper)
+
+# make tests faster
+SOUTH_TESTS_MIGRATE = False
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': 'test',
+ }}
+
+
+INSTALLED_APPS = [
+ 'django.contrib.admin',
+ 'django.contrib.auth',
+ 'django.contrib.contenttypes',
+ 'django.contrib.sessions',
+ 'django.contrib.messages',
+ 'django.contrib.staticfiles',
+ 'common.apps.CommonConfig',
+ 'connections.apps.ConnectionsConfig',
+ 'examples.apps.ExamplesConfig',
+ 'meanings.apps.MeaningsConfig',
+ 'semantics.apps.SemanticsConfig',
+ 'syntax.apps.SyntaxConfig',
+ 'entries.apps.EntriesConfig',
+ 'phrase_expansions.apps.PhraseExpansionsConfig',
+ 'dictionary_statistics.apps.DictionaryStatisticsConfig',
+ 'download.apps.DownloadConfig',
+ 'users.apps.UsersConfig',
+ 'crispy_forms',
+ 'django_extensions',
+ 'unifier.apps.UnifierConfig',
+ 'financial_settlement.apps.FinStatementConfig',
+ 'freelus.apps.FreeLusConfig',
+]
+
+SECRET_KEY = get_environment('SECRET_KEY')
+
+ROOT_URLCONF = 'shellvalier.urls'
+
+
+CRISPY_TEMPLATE_PACK = 'bootstrap4'
+CRISPY_FAIL_SILENTLY = not DEBUG
+
+MIDDLEWARE = [
+ 'django.middleware.security.SecurityMiddleware',
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+ 'django.middleware.locale.LocaleMiddleware',
+]
+
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [],
+ 'APP_DIRS': True,
+ 'OPTIONS': {
+ 'context_processors': [
+ 'django.template.context_processors.debug',
+ 'django.template.context_processors.request',
+ 'django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages',
+ ],
+ },
+ },
+]
+
+WSGI_APPLICATION = 'shellvalier.wsgi.application'
+
diff --git a/syntax/management/commands/update_tei.py b/syntax/management/commands/update_tei.py
new file mode 100644
index 0000000..2924542
--- /dev/null
+++ b/syntax/management/commands/update_tei.py
@@ -0,0 +1,70 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import logging
+import os
+from xml.sax import handler, make_parser
+
+from django.core.management.base import BaseCommand
+from django.db.models import Max
+from django.db import connection
+
+from common.models import ImportInProgress
+from importer.WalentyPreprocessXML import WalentyPreprocessTeiHandler
+from importer.WalentyXML import WalentyTeiHandler
+from semantics.models import Argument, Frame
+from shellvalier.settings import BASE_DIR
+
+
+class Command(BaseCommand):
+ args = 'none'
+ help = ''
+
+ def handle(self, **options):
+ update_tei()
+
+
+def update_tei():
+ logging.basicConfig(filename='import.log', level=logging.DEBUG)
+
+ xml_file_name = os.getenv('WALENTY_FILE_NAME', default='walenty_20210913_smaller.xml')
+
+ print("Loading walenty dict from: {}".format(xml_file_name))
+
+ xml_file = os.path.join(BASE_DIR, 'data', 'walenty', xml_file_name)
+
+ xml_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), xml_file)
+
+ parser = make_parser()
+ parser.setFeature(handler.feature_external_ges, False)
+
+ parser.setContentHandler(WalentyPreprocessTeiHandler())
+ parser.parse(xml_path)
+
+ entry_meanings = parser.getContentHandler().entry_meanings
+ meanings = parser.getContentHandler().meanings
+ frames = parser.getContentHandler().frames
+
+ parser.setContentHandler(WalentyTeiHandler(entry_meanings, meanings, frames))
+ parser.parse(xml_path)
+ ImportInProgress.objects.all().delete()
+
+ argument_max_id = Argument.objects.aggregate(Max('id'))
+ cursor = connection.cursor()
+ cursor.execute("Select setval('semantics_argument_id_seq', {} )".format(argument_max_id['id__max'] + 1))
+ cursor.close()
+
+ frame_max_id = Frame.objects.aggregate(Max('id'))
+ cursor = connection.cursor()
+ cursor.execute("Select setval('semantics_frame_id_seq', {} )".format(frame_max_id['id__max'] + 1))
+ cursor.close()
+
+ cursor = connection.cursor()
+ cursor.execute("create index if not exists meanings_lexicalunit_base_index on meanings_lexicalunit (base);")
+ cursor.close()
+
+ cursor = connection.cursor()
+ cursor.execute("create index if not exists connections_entry_name_index on connections_entry (name);")
+ cursor.close()
+
+
diff --git a/unifier/views.py b/unifier/views.py
index 307eb20..e11c3b0 100644
--- a/unifier/views.py
+++ b/unifier/views.py
@@ -1,15 +1,19 @@
+import io
import json
-import requests
+from xml.etree.ElementTree import Element, SubElement, tostring
+from xml.sax import handler, make_parser
+
import simplejson
+import urllib3
+from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db import transaction
+from django.db.models import Q
from django.http import JsonResponse, HttpResponse
from django.shortcuts import get_object_or_404
-from django.test import override_settings
from django.views.decorators.csrf import csrf_exempt
from common.decorators import ajax_required, ajax
-from connections.models import SchemaHook, ExampleConnection
from entries.polish_strings import EXAMPLE_SOURCE, EXAMPLE_OPINION
from entries.views import get_scroller_params, get_alternations, get_prefs_list, schema2dict, \
frame2dict, collect_forms, get_filtered_objects, get_local_schema_filter_form, get_local_frame_filter_form
@@ -18,24 +22,12 @@ from importer.unification.UnificationPreprocessXML import UnificationPreprocessH
from meanings.models import LexicalUnit
from semantics.choices import FrameStatus
from semantics.models import Frame, ArgumentRole, SemanticRole, RoleAttribute, RoleType, RoleSubAttribute, \
- PredefinedSelectionalPreference, Argument
+ PredefinedSelectionalPreference
from syntax.models import Schema
from unifier.models import UnifiedFrameArgument, UnifiedRelationalSelectionalPreference, UnifiedFrame, \
UnifiedFrame2SlowalFrameMapping, UnifiedFrameArgumentSlowalFrameMapping, HierarchyModel
from users.models import Assignment
from . import choices
-from xml.etree.ElementTree import Element, SubElement, tostring
-import io
-from xml.sax import handler, make_parser
-from django.db.models import Q, Prefetch
-
-from django.conf import settings
-import logging
-
-from django.views.decorators.csrf import csrf_exempt
-
-import urllib3
-
from .apps import synset_hierarchy_dict, SynsetHierarchy
from .choices import UnifiedFrameStatus
@@ -58,7 +50,8 @@ def save_synset_preference(request):
complement_id = request.POST['complement_id']
synset_preference_id = request.POST['synset_preference_id']
- check_sysnet_hierarchy_constraints_ret = save_synset_preference_base(frame_id, complement_id, synset_preference_id)
+ check_sysnet_hierarchy_constraints_ret = save_synset_preference_base(frame_id, complement_id,
+ synset_preference_id)
return JsonResponse(check_sysnet_hierarchy_constraints_ret)
@@ -66,10 +59,10 @@ def save_synset_preference(request):
def save_synset_preference_base(frame_id, complement_id, synset_preference_id):
-
unified_frame_argument = UnifiedFrameArgument.objects.get(unified_frame_id=int(frame_id), id=int(complement_id))
- check_sysnet_hierarchy_constraints_ret = check_sysnet_hierarchy_constraints([synset_preference_id], unified_frame_argument)
+ check_sysnet_hierarchy_constraints_ret = check_sysnet_hierarchy_constraints([synset_preference_id],
+ unified_frame_argument)
if check_sysnet_hierarchy_constraints_ret['succ'] is True:
unified_frame_argument.synsets.add(int(synset_preference_id))
@@ -94,8 +87,9 @@ def check_sysnet_hierarchy_constraints(synset_preference_ids, unified_frame_argu
"conflict_exists": "Wybrana preferencja selekcyjna konfliktuje z predefiniowanÄ… preferencjÄ… selekcyjnÄ… ALL."}
for synset_preference_id in synset_preference_ids:
- conflict_hyponym, conflict_hyperonym, conflict_exists = synset_hierarchy_constraint_check(int(synset_preference_id),
- set(synset_ids_list))
+ conflict_hyponym, conflict_hyperonym, conflict_exists = synset_hierarchy_constraint_check(
+ int(synset_preference_id),
+ set(synset_ids_list))
if conflict_hyponym is None and conflict_hyperonym is None and conflict_exists is None:
return {"succ": True}
else:
@@ -104,8 +98,8 @@ def check_sysnet_hierarchy_constraints(synset_preference_ids, unified_frame_argu
conflict_hyperonym_lu_str = conflict_lu_to_str(conflict_hyperonym)
ret = {"succ": False, "conflict_hyponym": conflict_hyponym_lu_str,
- "conflict_hyperonym": conflict_hyperonym_lu_str,
- "conflict_exists": "Wybrana preferencja selekcyjna nie mogła zostać zapisana ponieważ widnieje już na liście preferencji selekcyjnych: " + conflict_exists_lu_str if conflict_exists_lu_str is not None else None}
+ "conflict_hyperonym": conflict_hyperonym_lu_str,
+ "conflict_exists": "Wybrana preferencja selekcyjna nie mogła zostać zapisana ponieważ widnieje już na liście preferencji selekcyjnych: " + conflict_exists_lu_str if conflict_exists_lu_str is not None else None}
return ret
@@ -117,12 +111,6 @@ def conflict_lu_to_str(conflict_lu):
map(lambda s: str(s), conflict_hyponym_lu.all())) if conflict_lu is not None else None
-# class MyException(Exception):
-# def __init__(self, message, errors):
-# super().__init__(message)
-# self.errors = errors
-
-
@ajax_required
@transaction.atomic
def save_bunch_of_preferences(request):
@@ -132,7 +120,8 @@ def save_bunch_of_preferences(request):
predefined_preference_ids = json.loads(request.POST['predefined_preference_ids'])
synset_ids = json.loads(request.POST['synset_preference_ids'])
- check_sysnet_hierarchy_constraints_ret = save_bunch_of_preferences_base(frame_id, complement_id, predefined_preference_ids, synset_ids)
+ check_sysnet_hierarchy_constraints_ret = save_bunch_of_preferences_base(frame_id, complement_id,
+ predefined_preference_ids, synset_ids)
return JsonResponse(check_sysnet_hierarchy_constraints_ret)
return JsonResponse({})
@@ -141,7 +130,8 @@ def save_bunch_of_preferences(request):
@transaction.atomic
def save_bunch_of_preferences_base(frame_id, complement_id, predefined_preference_ids, synset_ids):
for predefined_preference_id in predefined_preference_ids:
- check_sysnet_hierarchy_constraints_ret = save_predefined_preference_base(frame_id, complement_id, predefined_preference_id)
+ check_sysnet_hierarchy_constraints_ret = save_predefined_preference_base(frame_id, complement_id,
+ predefined_preference_id)
if check_sysnet_hierarchy_constraints_ret['succ'] is False:
transaction.set_rollback(True)
return check_sysnet_hierarchy_constraints_ret
@@ -163,7 +153,8 @@ def save_predefined_preference(request):
complement_id = request.POST['complement_id']
predefined_preference_id = request.POST['predefined_preference_id']
- check_sysnet_hierarchy_constraints_ret = save_predefined_preference_base(frame_id, complement_id, predefined_preference_id)
+ check_sysnet_hierarchy_constraints_ret = save_predefined_preference_base(frame_id, complement_id,
+ predefined_preference_id)
return JsonResponse(check_sysnet_hierarchy_constraints_ret)
@@ -180,14 +171,15 @@ def save_predefined_preference_base(frame_id, complement_id, predefined_preferen
if len(unified_frame_argument.predefined.all()) > 0 or len(unified_frame_argument.synsets.all()) > 0 or \
len(unified_frame_argument.relations.all()) > 0:
return {"succ": False, "conflict_hyponym": None,
- "conflict_hyperonym": None,
- "conflict_exists": "Predefiniowana preferencja selekcyjna ALL może być dodana tylko, gdy nie na liście nie widnieją inne preferencje selekcyjne."}
+ "conflict_hyperonym": None,
+ "conflict_exists": "Predefiniowana preferencja selekcyjna ALL może być dodana tylko, gdy nie na liście nie widnieją inne preferencje selekcyjne."}
else:
check_sysnet_hierarchy_constraints_ret = {"succ": True}
else:
get_predefined_preference_synsets([pred_sel_pref], predefined_synset_ids)
- check_sysnet_hierarchy_constraints_ret = check_sysnet_hierarchy_constraints(predefined_synset_ids, unified_frame_argument)
+ check_sysnet_hierarchy_constraints_ret = check_sysnet_hierarchy_constraints(predefined_synset_ids,
+ unified_frame_argument)
if check_sysnet_hierarchy_constraints_ret['succ'] is True:
unified_frame_argument.predefined.add(int(predefined_preference_id))
@@ -208,8 +200,9 @@ def save_relational_selectional_preference(request):
unified_frame_argument = UnifiedFrameArgument.objects.get(unified_frame_id=int(frame_id),
id=int(complement_id_from))
- relational_selectional_preference, xx = UnifiedRelationalSelectionalPreference.objects.get_or_create(to_id=complement_id_to,
- relation_id=relation_id)
+ relational_selectional_preference, xx = UnifiedRelationalSelectionalPreference.objects.get_or_create(
+ to_id=complement_id_to,
+ relation_id=relation_id)
relational_selectional_preference.save()
unified_frame_argument.relations.add(relational_selectional_preference)
@@ -286,7 +279,7 @@ def get_unified_frames(request):
res_processed = []
for key, value in res.items():
if ((exclude_status is None or value['status'] != exclude_status) and
- (restrict_to_user is None or value['assignee_username'] == restrict_to_user)) or \
+ (restrict_to_user is None or value['assignee_username'] == restrict_to_user)) or \
value['status'] == choices.UnifiedFrameStatus.VERIFIED:
res_processed.append(value)
@@ -439,9 +432,12 @@ def get_unified_frame_json(unified_frame, request):
slowal_frames_db = slowal_frames_db.prefetch_related("arguments__role__attribute")
slowal_frames_db = slowal_frames_db.prefetch_related("arguments__role__sub_attribute")
slowal_frames_db = slowal_frames_db.prefetch_related("arguments__example_connections__example")
- slowal_frames_db = slowal_frames_db.prefetch_related("arguments__example_connections__example__example_connections__arguments")
- slowal_frames_db = slowal_frames_db.prefetch_related("arguments__example_connections__example__example_connections__lexical_unit")
- slowal_frames_db = slowal_frames_db.prefetch_related("arguments__example_connections__example__example_connections__schema_connections")
+ slowal_frames_db = slowal_frames_db.prefetch_related(
+ "arguments__example_connections__example__example_connections__arguments")
+ slowal_frames_db = slowal_frames_db.prefetch_related(
+ "arguments__example_connections__example__example_connections__lexical_unit")
+ slowal_frames_db = slowal_frames_db.prefetch_related(
+ "arguments__example_connections__example__example_connections__schema_connections")
slowal_frames = slowal_frames_db.all()
@@ -686,7 +682,8 @@ def save_new_role(request):
sub_attribute_id=sub_attribute_id).first()
if argument_role is None:
argument_role = ArgumentRole(role=SemanticRole.objects.get(pk=role_id),
- attribute=None if attribute_id is None else RoleAttribute.objects.get(pk=attribute_id),
+ attribute=None if attribute_id is None else RoleAttribute.objects.get(
+ pk=attribute_id),
sub_attribute=RoleSubAttribute.objects.get(pk=sub_attribute_id))
argument_role.save()
@@ -933,7 +930,6 @@ def remove_unified_frame_mappings_and_assigments(unified_frame):
@ajax(login_required=True, method='post')
@transaction.atomic
def delete_unified_frame(request, unified_frame_id):
-
unified_frame = UnifiedFrame.objects.get(id=unified_frame_id)
remove_unified_frame_mappings_and_assigments(unified_frame)
@@ -1130,7 +1126,6 @@ def attach_lu_to_unified_frame(request):
def synset_hierarchy_constraint_check(lu_id, pref_lu_ids):
-
conflict_exists = None
if lu_id in pref_lu_ids:
conflict_exists = lu_id
diff --git a/update_db.sh b/update_db.sh
new file mode 100755
index 0000000..37e23d7
--- /dev/null
+++ b/update_db.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+#python manage.py makemigrations
+#python manage.py migrate
+
+rm import.log || true
+
+time python manage.py update_tei
+
+head import.log
+wc import.log
--
GitLab