Skip to content
Snippets Groups Projects
Commit df320c1c authored by Grzegorz Kuboń's avatar Grzegorz Kuboń
Browse files

Added auto sqlite dump generation and upload.

parent 267496df
No related merge requests found
Pipeline #5674 failed with stages
in 1 hour, 1 minute, and 31 seconds
image: clarinpl/python:3.6
before_script:
- pip install tox==2.9.1
### Set r-values below in gitlab UI under Variables section.
### MySQL credentials are here only for purpose of quick config.
### MySQL in services uses $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE.
### If you change database name or username, you have to change sql scripts.
variables:
MINIO_HOSTNAME: $MINIO_HOST
MINIO_FILENAME: $MINIO_FILE
MINIO_ACCESS_KEY: $MINIO_ACCESS
MINIO_SECRET_KEY: $MINIO_SECRET
MYSQL_ROOT_PASSWORD: rootpass # $ROOTPASS
MYSQL_USER_PASSWORD: userpass # $USERPASS
MYSQL_DATABASE: wordnet_work
MYSQL_USER: wordnet
MYSQL_HOST: mysql
SQL_DUMP: http://ws.clarin-pl.eu/public/wordnet-work.LATEST.sql.gz
# PIPY_USER: fake
# PIPY_PASS: fake
before_script:
- pip install tox==2.9.1
cache:
paths:
- .tox
paths:
- .tox
stages:
- check_style
- tests
- push_wheel
- check_style
- tests
- push_wheel
- upload_sqlite
pep8:
stage: check_style
script:
- tox -v -e pep8
stage: check_style
script:
- tox -v -e pep8
docstyle:
stage: check_style
script:
- tox -v -e docstyle
stage: check_style
script:
- tox -v -e docstyle
test:
stage: tests
before_script:
- apt-get update
- apt-get install -y language-pack-pl-base
- dpkg-reconfigure locales
script:
- python setup.py test
stage: tests
before_script:
- apt-get update
- apt-get install -y language-pack-pl-base
- dpkg-reconfigure locales
script:
- python setup.py test
push_wheel:
before_script:
- pip install twine
only:
- master
stage: push_wheel
when: on_success
script:
- python setup.py sdist bdist_wheel
- python -m twine upload
--repository-url https://pypi.clarin-pl.eu/
-u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl
before_script:
- pip install twine
only:
- master
stage: push_wheel
when: on_success
script:
- python setup.py sdist bdist_wheel
- python -m twine upload
--repository-url https://pypi.clarin-pl.eu/
-u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl
upload_sqlite:
stage: upload_sqlite
services:
- mysql:latest
before_script:
- apt-get update && apt-get install -y atool gcc git curl libmcrypt-dev mysql-client libmysqlclient-dev
- pip install --upgrade pip
- pip install . pymysql plwn_comments sqlalchemy minio mysqlclient
- echo "$MYSQL_HOST+mysqldb://$MYSQL_USER:$MYSQL_USER_PASSWORD@$MYSQL_HOST/$MYSQL_DATABASE?charset=utf8" >> connection.txt
- wget $SQL_DUMP
script:
- atool -x wordnet-work.LATEST.sql.gz
- mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < wordnet-work.LATEST.sql
- mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/grant_privileges.sql
- mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/clean_wndb.sql
- python scripts/mysql_to_sqlite.py --s3_minio_host "$MINIO_HOSTNAME" --minio_access "$MINIO_ACCESS_KEY" --minio_secret "$MINIO_SECRET_KEY" --minio_filename "$MINIO_FILENAME"
CREATE USER wordnet IDENTIFIED BY 'userpass';
GRANT ALL PRIVILEGES ON * . * TO wordnet;
FLUSH PRIVILEGES;
#!/usr/bin/python
"""Script converts plWordNet mysql database to sqlite and uploads to MinIo."""
from minio import Minio
from minio.error import S3Error
import argparse as argp
import logging as log
import sys
def main():
ap = argp.ArgumentParser(description=__doc__)
ap.add_argument('--s3_minio_host',
help='in format s3.yourhost.com')
ap.add_argument('--minio_access', help='minio access key')
ap.add_argument('--minio_secret', help='minio secret key')
ap.add_argument('--minio_filename',
help='name of sqlite dump at minio')
av = ap.parse_args()
# Log every error and warning from the plwn module along with this script's
# output
plwn_log = log.getLogger('plwn')
plwn_log_h = log.StreamHandler(sys.stdout)
plwn_log_h.setFormatter(log.Formatter('!! Log from %(name)s: %(message)s'))
plwn_log.addHandler(plwn_log_h)
plwn_log.setLevel(log.WARNING)
_script(av.s3_minio_host, av.minio_access, av.minio_secret,
av.minio_filename)
def _script(s3_minio_host, minio_access, minio_secret, minio_filename):
import plwn
plwn.read("connection.txt", "database", "plwn-new.db", "sqlite3")
client = Minio(
s3_minio_host,
access_key=minio_access,
secret_key=minio_secret,
)
client.fput_object(
"projects", minio_filename, "plwn-new.db",
)
if __name__ == '__main__':
try:
main()
except S3Error as exc:
print("error: ", exc)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment