diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b80625cf855e9ac8c7c9a72d19b6675e47f355de..0fb4c87a006a531e25ecab23f14a22e3ad6ac528 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,46 +1,81 @@ image: clarinpl/python:3.6 -before_script: - - pip install tox==2.9.1 +### Set r-values below in gitlab UI under Variables section. +### MySQL credentials are here only for purpose of quick config. +### MySQL in services uses $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE. +### If you change database name or username, you have to change sql scripts. + +variables: + MINIO_HOSTNAME: $MINIO_HOST + MINIO_FILENAME: $MINIO_FILE + MINIO_ACCESS_KEY: $MINIO_ACCESS + MINIO_SECRET_KEY: $MINIO_SECRET + MYSQL_ROOT_PASSWORD: rootpass # $ROOTPASS + MYSQL_USER_PASSWORD: userpass # $USERPASS + MYSQL_DATABASE: wordnet_work + MYSQL_USER: wordnet + MYSQL_HOST: mysql + SQL_DUMP: http://ws.clarin-pl.eu/public/wordnet-work.LATEST.sql.gz +# PIPY_USER: fake +# PIPY_PASS: fake +before_script: + - pip install tox==2.9.1 cache: - paths: - - .tox + paths: + - .tox stages: - - check_style - - tests - - push_wheel + - check_style + - tests + - push_wheel + - upload_sqlite pep8: - stage: check_style - script: - - tox -v -e pep8 + stage: check_style + script: + - tox -v -e pep8 docstyle: - stage: check_style - script: - - tox -v -e docstyle + stage: check_style + script: + - tox -v -e docstyle test: - stage: tests - before_script: - - apt-get update - - apt-get install -y language-pack-pl-base - - dpkg-reconfigure locales - script: - - python setup.py test + stage: tests + before_script: + - apt-get update + - apt-get install -y language-pack-pl-base + - dpkg-reconfigure locales + script: + - python setup.py test push_wheel: - before_script: - - pip install twine - only: - - master - stage: push_wheel - when: on_success - script: - - python setup.py sdist bdist_wheel - - python -m twine upload - --repository-url https://pypi.clarin-pl.eu/ - -u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl + before_script: + - pip install twine + only: + - master + stage: push_wheel + when: on_success + script: + - python setup.py sdist bdist_wheel + - python -m twine upload + --repository-url https://pypi.clarin-pl.eu/ + -u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl +upload_sqlite: + stage: upload_sqlite + services: + - mysql:latest + before_script: + - apt-get update && apt-get install -y atool gcc git curl libmcrypt-dev mysql-client libmysqlclient-dev + - pip install --upgrade pip + - pip install . pymysql plwn_comments sqlalchemy minio mysqlclient + - echo "$MYSQL_HOST+mysqldb://$MYSQL_USER:$MYSQL_USER_PASSWORD@$MYSQL_HOST/$MYSQL_DATABASE?charset=utf8" >> connection.txt + - wget $SQL_DUMP + script: + - atool -x wordnet-work.LATEST.sql.gz + - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < wordnet-work.LATEST.sql + - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/grant_privileges.sql + - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/clean_wndb.sql + - python scripts/mysql_to_sqlite.py --s3_minio_host "$MINIO_HOSTNAME" --minio_access "$MINIO_ACCESS_KEY" --minio_secret "$MINIO_SECRET_KEY" --minio_filename "$MINIO_FILENAME" diff --git a/scripts/grant_privileges.sql b/scripts/grant_privileges.sql new file mode 100644 index 0000000000000000000000000000000000000000..4ad7ed3f744cba8ffaf236f59ff6cad463b2b588 --- /dev/null +++ b/scripts/grant_privileges.sql @@ -0,0 +1,4 @@ + +CREATE USER wordnet IDENTIFIED BY 'userpass'; +GRANT ALL PRIVILEGES ON * . * TO wordnet; +FLUSH PRIVILEGES; diff --git a/scripts/mysql_to_sqlite.py b/scripts/mysql_to_sqlite.py new file mode 100644 index 0000000000000000000000000000000000000000..5e0a5f60dad7834e19b40143cdbe7a6ef54082a6 --- /dev/null +++ b/scripts/mysql_to_sqlite.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +"""Script converts plWordNet mysql database to sqlite and uploads to MinIo.""" + +from minio import Minio +from minio.error import S3Error +import argparse as argp +import logging as log +import sys + + +def main(): + ap = argp.ArgumentParser(description=__doc__) + ap.add_argument('--s3_minio_host', + help='in format s3.yourhost.com') + ap.add_argument('--minio_access', help='minio access key') + ap.add_argument('--minio_secret', help='minio secret key') + ap.add_argument('--minio_filename', + help='name of sqlite dump at minio') + + av = ap.parse_args() + # Log every error and warning from the plwn module along with this script's + # output + plwn_log = log.getLogger('plwn') + plwn_log_h = log.StreamHandler(sys.stdout) + plwn_log_h.setFormatter(log.Formatter('!! Log from %(name)s: %(message)s')) + plwn_log.addHandler(plwn_log_h) + plwn_log.setLevel(log.WARNING) + + _script(av.s3_minio_host, av.minio_access, av.minio_secret, + av.minio_filename) + + +def _script(s3_minio_host, minio_access, minio_secret, minio_filename): + import plwn + plwn.read("connection.txt", "database", "plwn-new.db", "sqlite3") + client = Minio( + s3_minio_host, + access_key=minio_access, + secret_key=minio_secret, + ) + client.fput_object( + "projects", minio_filename, "plwn-new.db", + ) + + +if __name__ == '__main__': + try: + main() + except S3Error as exc: + print("error: ", exc)