Merge pull request #641 from guzman-raphael/ext-migrate-test

Add test cases for external blob migration 0.11 to 0.12

Author: dimitri-yatsenko

.gitignore

@@ -18,3 +18,8 @@ build/
 ./tests/.coverage
 ./tests/dj-store/*
 *.log
+*.env
+LNX-docker-compose.yml
+notebooks/*
+test.dockerfile
+__main__.py

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 dist: xenial
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ENDPOINT="127.0.0.1:9000" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint" S3_BUCKET="datajoint-test"
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint"
 python:
   - "3.4"
   - "3.5"
@@ -16,8 +16,6 @@ before_install:
   - docker pull minio/minio
   - docker run -d -p 9000:9000 -e "MINIO_ACCESS_KEY=$S3_ACCESS_KEY" -e "MINIO_SECRET_KEY=$S3_SECRET_KEY" minio/minio server /data
   - sleep 120
-  - docker pull minio/mc
-  - docker run --network="host" --entrypoint=/bin/sh minio/mc -c "mc config host add dj-s3 http://$S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY;mc mb $S3_BUCKET;mc policy download $S3_BUCKET;exit 0;"
   - sudo apt-get install -y libblas-dev liblapack-dev libatlas-dev gfortran
   - sudo apt-get install -y graphviz graphviz-dev pkg-config
   - mysql --version 

datajoint/utils.py

@@ -95,3 +95,19 @@ def safe_copy(src, dest, overwrite=False):
         temp_file = dest + '.copying'
         shutil.copyfile(src, temp_file)
         os.rename(temp_file, dest)
+
+
+def parse_sql(filepath):
+    DELIMITER = ';'
+    statement = []
+    with open(filepath, 'rt') as f:
+        for line in f:
+            line = line.strip()
+            if not line.startswith('--') and len(line) > 1:
+                if line.startswith('DELIMITER'):
+                    DELIMITER = line.split()[1]
+                else:
+                    statement.append(line)
+                    if line.endswith(DELIMITER):
+                        yield ' '.join(statement)
+                        statement = []

tests/__init__.py

@@ -9,8 +9,15 @@
 from os import environ, remove
 import datajoint as dj
 from distutils.version import LooseVersion
+import os
+from pathlib import Path
+from minio import Minio
+import urllib3
+import certifi
+import shutil
+from datajoint.utils import parse_sql
 
-__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko'
+__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman'
 
 # turn on verbose logging
 logging.basicConfig(level=logging.DEBUG)
@@ -61,7 +68,7 @@
     conn_root.query(
         "GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';")
 else:
-    # grant permissions. For mysql5.6/5.7 this also automatically creates user 
+    # grant permissions. For mysql5.6/5.7 this also automatically creates user
     # if not exists
     conn_root.query("""
         GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%'
@@ -76,6 +83,25 @@
         REQUIRE SSL;
         """)
 
+# Initialize httpClient with relevant timeout.
+httpClient = urllib3.PoolManager(
+            timeout=30,
+            cert_reqs='CERT_REQUIRED',
+            ca_certs=certifi.where(),
+            retries=urllib3.Retry(
+                total=3,
+                backoff_factor=0.2,
+                status_forcelist=[500, 502, 503, 504]
+            )
+        )
+
+# Initialize minioClient with an endpoint and access/secret keys.
+minioClient = Minio(S3_CONN_INFO['endpoint'],
+                    access_key=S3_CONN_INFO['access_key'],
+                    secret_key=S3_CONN_INFO['secret_key'],
+                    secure=False,
+                    http_client=httpClient)
+
 
 def setup_package():
     """
@@ -84,6 +110,44 @@ def setup_package():
     """
     dj.config['safemode'] = False
 
+    # Add old MySQL
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data"
+    db_name = "djtest_blob_migrate"
+    db_file = "v0_11.sql"
+    conn_root.query("""
+        CREATE DATABASE {};
+        """.format(db_name))
+
+    statements = parse_sql('{}/{}'.format(source, db_file))
+    for s in statements:
+        conn_root.query(s)
+
+    # Add old S3
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data/s3"
+    bucket = "migrate-test"
+    region = "us-east-1"
+    minioClient.make_bucket(bucket, location=region)
+
+    pathlist = Path(source).glob('**/*')
+    for path in pathlist:
+        if os.path.isfile(str(path)) and ".sql" not in str(path):
+            minioClient.fput_object(
+                    bucket, os.path.relpath(
+                        str(path),
+                        '{}/{}'.format(source, bucket)
+                        ), str(path))
+
+    # Add S3
+    minioClient.make_bucket("datajoint-test", location=region)
+
+    # Add old File Content
+    shutil.copytree(
+            os.path.dirname(os.path.realpath(__file__)) +
+            "/external-legacy-data/file/temp",
+            os.path.expanduser('~/temp'))
+
 
 def teardown_package():
     """
@@ -99,3 +163,21 @@ def teardown_package():
         conn.query('DROP DATABASE `{}`'.format(db[0]))
     conn.query('SET FOREIGN_KEY_CHECKS=1')
     remove("dj_local_conf.json")
+
+    # Remove old S3
+    bucket = "migrate-test"
+    objs = list(minioClient.list_objects_v2(
+            bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove S3
+    bucket = "datajoint-test"
+    objs = list(minioClient.list_objects_v2(bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove old File Content
+    shutil.rmtree(os.path.expanduser('~/temp'))

tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local

@@ -0,0 +1,138 @@
+USE djtest_blob_migrate;
+-- MySQL dump 10.13  Distrib 5.7.26, for Linux (x86_64)
+--
+-- Host: localhost    Database: djtest_blob_migrate
+-- ------------------------------------------------------
+-- Server version	5.7.26
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `~external`
+--
+
+DROP TABLE IF EXISTS `~external`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `~external` (
+  `hash` char(51) NOT NULL COMMENT 'the hash of stored object + store name',
+  `size` bigint(20) unsigned NOT NULL COMMENT 'size of object in bytes',
+  `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'automatic timestamp',
+  PRIMARY KEY (`hash`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COMMENT='external storage tracking';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `~external`
+--
+
+LOCK TABLES `~external` WRITE;
+/*!40000 ALTER TABLE `~external` DISABLE KEYS */;
+INSERT INTO `~external` VALUES ('e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal',237,'2019-07-31 17:55:01'),('FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared',37,'2019-07-31 17:55:01'),('NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared',53,'2019-07-31 17:55:01'),('Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared',53,'2019-07-31 17:55:01'),('_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA',237,'2019-07-31 17:55:01'),('_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94',40,'2019-07-31 17:55:01'),('_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local',40,'2019-07-31 17:55:01');
+/*!40000 ALTER TABLE `~external` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `~log`
+--
+
+DROP TABLE IF EXISTS `~log`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `~log` (
+  `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `version` varchar(12) NOT NULL COMMENT 'datajoint version',
+  `user` varchar(255) NOT NULL COMMENT 'user@host',
+  `host` varchar(255) NOT NULL DEFAULT '' COMMENT 'system hostname',
+  `event` varchar(255) NOT NULL DEFAULT '' COMMENT 'custom message',
+  PRIMARY KEY (`timestamp`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COMMENT='event logging table for `djtest_blob_migrate`';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `~log`
+--
+
+LOCK TABLES `~log` WRITE;
+/*!40000 ALTER TABLE `~log` DISABLE KEYS */;
+INSERT INTO `~log` VALUES ('2019-07-31 17:54:49','0.11.1py','[email protected]','297df05ab17c','Declared `djtest_blob_migrate`.`~log`'),('2019-07-31 17:54:54','0.11.1py','[email protected]','297df05ab17c','Declared `djtest_blob_migrate`.`~external`'),('2019-07-31 17:54:55','0.11.1py','[email protected]','297df05ab17c','Declared `djtest_blob_migrate`.`b`');
+/*!40000 ALTER TABLE `~log` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `a`
+--
+
+DROP TABLE IF EXISTS `a`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `a` (
+  `id` int(11) NOT NULL,
+  `blob_external` char(51) NOT NULL COMMENT ':external:uses S3',
+  `blob_share` char(51) NOT NULL COMMENT ':external-shared:uses S3',
+  PRIMARY KEY (`id`),
+  KEY `blob_external` (`blob_external`),
+  KEY `blob_share` (`blob_share`),
+  CONSTRAINT `a_ibfk_1` FOREIGN KEY (`blob_external`) REFERENCES `~external` (`hash`),
+  CONSTRAINT `a_ibfk_2` FOREIGN KEY (`blob_share`) REFERENCES `~external` (`hash`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `a`
+--
+
+LOCK TABLES `a` WRITE;
+/*!40000 ALTER TABLE `a` DISABLE KEYS */;
+INSERT INTO `a` VALUES (0,'_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA','NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared'),(1,'_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94','FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared');
+/*!40000 ALTER TABLE `a` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `b`
+--
+
+DROP TABLE IF EXISTS `b`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `b` (
+  `id` int(11) NOT NULL,
+  `blob_local` char(51) NOT NULL COMMENT ':external-local:uses files',
+  `blob_share` char(51) NOT NULL COMMENT ':external-shared:uses S3',
+  PRIMARY KEY (`id`),
+  KEY `blob_local` (`blob_local`),
+  KEY `blob_share` (`blob_share`),
+  CONSTRAINT `b_ibfk_1` FOREIGN KEY (`blob_local`) REFERENCES `~external` (`hash`),
+  CONSTRAINT `b_ibfk_2` FOREIGN KEY (`blob_share`) REFERENCES `~external` (`hash`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `b`
+--
+
+LOCK TABLES `b` WRITE;
+/*!40000 ALTER TABLE `b` DISABLE KEYS */;
+INSERT INTO `b` VALUES (0,'e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal','Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared'),(1,'_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local','FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared');
+/*!40000 ALTER TABLE `b` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2019-07-31 18:16:40

tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal

@@ -11,7 +11,7 @@
 schema = dj.schema(PREFIX + '_extern', connection=dj.conn(**CONN_INFO))
 
 
-dj.config['stores'] = {
+stores_config = {
 
     'raw': dict(
         protocol='file',
@@ -40,6 +40,8 @@
         subfolding=(2, 4))
 }
 
+dj.config['stores'] = stores_config
+
 dj.config['cache'] = tempfile.mkdtemp()