Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
P
PLWN_API
Manage
Activity
Members
Labels
Plan
Issues
10
Issue boards
Milestones
Wiki
Redmine
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Libraries
PLWN_API
Commits
df320c1c
Commit
df320c1c
authored
2 years ago
by
Grzegorz Kuboń
Browse files
Options
Downloads
Patches
Plain Diff
Added auto sqlite dump generation and upload.
parent
267496df
Branches
chore_add_auto_sqlite_dump_upload
No related merge requests found
Pipeline
#5674
failed with stages
in 1 hour, 1 minute, and 31 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
.gitlab-ci.yml
+66
-31
66 additions, 31 deletions
.gitlab-ci.yml
scripts/grant_privileges.sql
+4
-0
4 additions, 0 deletions
scripts/grant_privileges.sql
scripts/mysql_to_sqlite.py
+50
-0
50 additions, 0 deletions
scripts/mysql_to_sqlite.py
with
120 additions
and
31 deletions
.gitlab-ci.yml
+
66
−
31
View file @
df320c1c
image
:
clarinpl/python:3.6
before_script
:
-
pip install tox==2.9.1
### Set r-values below in gitlab UI under Variables section.
### MySQL credentials are here only for purpose of quick config.
### MySQL in services uses $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE.
### If you change database name or username, you have to change sql scripts.
variables
:
MINIO_HOSTNAME
:
$MINIO_HOST
MINIO_FILENAME
:
$MINIO_FILE
MINIO_ACCESS_KEY
:
$MINIO_ACCESS
MINIO_SECRET_KEY
:
$MINIO_SECRET
MYSQL_ROOT_PASSWORD
:
rootpass
# $ROOTPASS
MYSQL_USER_PASSWORD
:
userpass
# $USERPASS
MYSQL_DATABASE
:
wordnet_work
MYSQL_USER
:
wordnet
MYSQL_HOST
:
mysql
SQL_DUMP
:
http://ws.clarin-pl.eu/public/wordnet-work.LATEST.sql.gz
# PIPY_USER: fake
# PIPY_PASS: fake
before_script
:
-
pip install tox==2.9.1
cache
:
paths
:
-
.tox
paths
:
-
.tox
stages
:
-
check_style
-
tests
-
push_wheel
-
check_style
-
tests
-
push_wheel
-
upload_sqlite
pep8
:
stage
:
check_style
script
:
-
tox -v -e pep8
stage
:
check_style
script
:
-
tox -v -e pep8
docstyle
:
stage
:
check_style
script
:
-
tox -v -e docstyle
stage
:
check_style
script
:
-
tox -v -e docstyle
test
:
stage
:
tests
before_script
:
-
apt-get update
-
apt-get install -y language-pack-pl-base
-
dpkg-reconfigure locales
script
:
-
python setup.py test
stage
:
tests
before_script
:
-
apt-get update
-
apt-get install -y language-pack-pl-base
-
dpkg-reconfigure locales
script
:
-
python setup.py test
push_wheel
:
before_script
:
-
pip install twine
only
:
-
master
stage
:
push_wheel
when
:
on_success
script
:
-
python setup.py sdist bdist_wheel
-
python -m twine upload
--repository-url https://pypi.clarin-pl.eu/
-u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl
before_script
:
-
pip install twine
only
:
-
master
stage
:
push_wheel
when
:
on_success
script
:
-
python setup.py sdist bdist_wheel
-
python -m twine upload
--repository-url https://pypi.clarin-pl.eu/
-u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl
upload_sqlite
:
stage
:
upload_sqlite
services
:
-
mysql:latest
before_script
:
-
apt-get update && apt-get install -y atool gcc git curl libmcrypt-dev mysql-client libmysqlclient-dev
-
pip install --upgrade pip
-
pip install . pymysql plwn_comments sqlalchemy minio mysqlclient
-
echo "$MYSQL_HOST+mysqldb://$MYSQL_USER:$MYSQL_USER_PASSWORD@$MYSQL_HOST/$MYSQL_DATABASE?charset=utf8" >> connection.txt
-
wget $SQL_DUMP
script
:
-
atool -x wordnet-work.LATEST.sql.gz
-
mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < wordnet-work.LATEST.sql
-
mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/grant_privileges.sql
-
mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/clean_wndb.sql
-
python scripts/mysql_to_sqlite.py --s3_minio_host "$MINIO_HOSTNAME" --minio_access "$MINIO_ACCESS_KEY" --minio_secret "$MINIO_SECRET_KEY" --minio_filename "$MINIO_FILENAME"
This diff is collapsed.
Click to expand it.
scripts/grant_privileges.sql
0 → 100644
+
4
−
0
View file @
df320c1c
CREATE
USER
wordnet
IDENTIFIED
BY
'userpass'
;
GRANT
ALL
PRIVILEGES
ON
*
.
*
TO
wordnet
;
FLUSH
PRIVILEGES
;
This diff is collapsed.
Click to expand it.
scripts/mysql_to_sqlite.py
0 → 100644
+
50
−
0
View file @
df320c1c
#!/usr/bin/python
"""
Script converts plWordNet mysql database to sqlite and uploads to MinIo.
"""
from
minio
import
Minio
from
minio.error
import
S3Error
import
argparse
as
argp
import
logging
as
log
import
sys
def
main
():
ap
=
argp
.
ArgumentParser
(
description
=
__doc__
)
ap
.
add_argument
(
'
--s3_minio_host
'
,
help
=
'
in format s3.yourhost.com
'
)
ap
.
add_argument
(
'
--minio_access
'
,
help
=
'
minio access key
'
)
ap
.
add_argument
(
'
--minio_secret
'
,
help
=
'
minio secret key
'
)
ap
.
add_argument
(
'
--minio_filename
'
,
help
=
'
name of sqlite dump at minio
'
)
av
=
ap
.
parse_args
()
# Log every error and warning from the plwn module along with this script's
# output
plwn_log
=
log
.
getLogger
(
'
plwn
'
)
plwn_log_h
=
log
.
StreamHandler
(
sys
.
stdout
)
plwn_log_h
.
setFormatter
(
log
.
Formatter
(
'
!! Log from %(name)s: %(message)s
'
))
plwn_log
.
addHandler
(
plwn_log_h
)
plwn_log
.
setLevel
(
log
.
WARNING
)
_script
(
av
.
s3_minio_host
,
av
.
minio_access
,
av
.
minio_secret
,
av
.
minio_filename
)
def
_script
(
s3_minio_host
,
minio_access
,
minio_secret
,
minio_filename
):
import
plwn
plwn
.
read
(
"
connection.txt
"
,
"
database
"
,
"
plwn-new.db
"
,
"
sqlite3
"
)
client
=
Minio
(
s3_minio_host
,
access_key
=
minio_access
,
secret_key
=
minio_secret
,
)
client
.
fput_object
(
"
projects
"
,
minio_filename
,
"
plwn-new.db
"
,
)
if
__name__
==
'
__main__
'
:
try
:
main
()
except
S3Error
as
exc
:
print
(
"
error:
"
,
exc
)
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment