Added auto sqlite dump generation and upload.

.gitlab-ci.yml

 image: clarinpl/python:3.6
 
+### Set r-values below in gitlab UI under Variables section.
+### MySQL credentials are here only for purpose of quick config.
+### MySQL in services uses $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE.
+### If you change database name or username, you have to change sql scripts.
+
+variables:
+    MINIO_HOSTNAME: $MINIO_HOST
+    MINIO_FILENAME: $MINIO_FILE
+    MINIO_ACCESS_KEY: $MINIO_ACCESS
+    MINIO_SECRET_KEY: $MINIO_SECRET
+    MYSQL_ROOT_PASSWORD: rootpass # $ROOTPASS
+    MYSQL_USER_PASSWORD: userpass # $USERPASS
+    MYSQL_DATABASE: wordnet_work
+    MYSQL_USER: wordnet
+    MYSQL_HOST: mysql
+    SQL_DUMP: http://ws.clarin-pl.eu/public/wordnet-work.LATEST.sql.gz
+#    PIPY_USER: fake
+#    PIPY_PASS: fake
+
 before_script:
     - pip install tox==2.9.1
-
 cache:
     paths:
         - .tox
@@ -11,6 +29,7 @@ stages:
     - check_style
     - tests
     - push_wheel
+    - upload_sqlite
 
 pep8:
     stage: check_style
@@ -44,3 +63,19 @@ push_wheel:
             --repository-url https://pypi.clarin-pl.eu/
             -u $PIPY_USER -p $PIPY_PASS dist/plwn_api*.whl
 
+upload_sqlite:
+    stage: upload_sqlite
+    services:
+        - mysql:latest
+    before_script:
+        - apt-get update && apt-get install -y atool gcc git curl libmcrypt-dev mysql-client libmysqlclient-dev
+        - pip install --upgrade pip
+        - pip install . pymysql plwn_comments sqlalchemy minio mysqlclient
+        - echo "$MYSQL_HOST+mysqldb://$MYSQL_USER:$MYSQL_USER_PASSWORD@$MYSQL_HOST/$MYSQL_DATABASE?charset=utf8" >> connection.txt
+        - wget $SQL_DUMP
+    script:
+        - atool -x wordnet-work.LATEST.sql.gz
+        - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < wordnet-work.LATEST.sql
+        - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/grant_privileges.sql
+        - mysql -u root -p"$MYSQL_ROOT_PASSWORD" -D "$MYSQL_DATABASE" < scripts/clean_wndb.sql
+        - python scripts/mysql_to_sqlite.py --s3_minio_host "$MINIO_HOSTNAME" --minio_access "$MINIO_ACCESS_KEY" --minio_secret "$MINIO_SECRET_KEY" --minio_filename "$MINIO_FILENAME"

scripts/grant_privileges.sql

+
+CREATE USER wordnet IDENTIFIED BY 'userpass';
+GRANT ALL PRIVILEGES ON * . * TO wordnet;
+FLUSH PRIVILEGES;

scripts/mysql_to_sqlite.py

+#!/usr/bin/python
+"""Script converts plWordNet mysql database to sqlite and uploads to MinIo."""
+
+from minio import Minio
+from minio.error import S3Error
+import argparse as argp
+import logging as log
+import sys
+
+
+def main():
+    ap = argp.ArgumentParser(description=__doc__)
+    ap.add_argument('--s3_minio_host',
+                    help='in format s3.yourhost.com')
+    ap.add_argument('--minio_access', help='minio access key')
+    ap.add_argument('--minio_secret', help='minio secret key')
+    ap.add_argument('--minio_filename',
+                    help='name of sqlite dump at minio')
+
+    av = ap.parse_args()
+    # Log every error and warning from the plwn module along with this script's
+    # output
+    plwn_log = log.getLogger('plwn')
+    plwn_log_h = log.StreamHandler(sys.stdout)
+    plwn_log_h.setFormatter(log.Formatter('!! Log from %(name)s: %(message)s'))
+    plwn_log.addHandler(plwn_log_h)
+    plwn_log.setLevel(log.WARNING)
+
+    _script(av.s3_minio_host, av.minio_access, av.minio_secret,
+            av.minio_filename)
+
+
+def _script(s3_minio_host, minio_access, minio_secret, minio_filename):
+    import plwn
+    plwn.read("connection.txt", "database", "plwn-new.db", "sqlite3")
+    client = Minio(
+        s3_minio_host,
+        access_key=minio_access,
+        secret_key=minio_secret,
+    )
+    client.fput_object(
+        "projects", minio_filename, "plwn-new.db",
+    )
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except S3Error as exc:
+        print("error: ", exc)