Merge pull request #688 from guzman-raphael/str-adapted-types

Correct Attribute Adapter from casting to string + minor updates

Author: dimitri-yatsenko

.gitignore

@@ -21,4 +21,5 @@ build/
 *.env
 local-docker-compose.yml
 notebooks/*
-__main__.py
+__main__.py
+jupyter_custom.js

.travis.yml

@@ -13,14 +13,15 @@ services:
 main: &main
       stage: Alpine
       os: linux
+      dist: xenial # precise, trusty, xenial, bionic
       language: shell
       script:
         - docker-compose -f LNX-docker-compose.yml up --build --exit-code-from dj
 jobs:
   include:
     - <<: *main
       env:
-      - PY_VER: "3.8-rc"
+      - PY_VER: "3.8"
       - MYSQL_VER: "5.7"
     - <<: *main
       env:
@@ -36,7 +37,7 @@ jobs:
       - MYSQL_VER: "5.7"
     - <<: *main
       env:
-      - PY_VER: "3.8-rc"
+      - PY_VER: "3.8"
       - MYSQL_VER: "8.0"
     - <<: *main
       env:
@@ -52,7 +53,7 @@ jobs:
       - MYSQL_VER: "8.0"
     - <<: *main
       env:
-      - PY_VER: "3.8-rc"
+      - PY_VER: "3.8"
       - MYSQL_VER: "5.6"
     - <<: *main
       env:

CHANGELOG.md

@@ -1,6 +1,9 @@
 ## Release notes
 
-### 0.12.0 -- October 31, 2019
+### 0.12.1 -- Nov 2, 2019
+* Bugfix - AttributeAdapter converts into a string (#684)
+
+### 0.12.0 -- Oct 31, 2019
 * Dropped support for Python 3.4
 * Support secure connections with TLS (aka SSL) PR #620
 * Convert numpy array from python object to appropriate data type if all elements are of the same type (#587) PR #608
@@ -31,7 +34,7 @@
 ### 0.11.3 -- Jul 26, 2019
 * Fix incompatibility with pyparsing 2.4.1 (#629) PR #631
 
-### 0.11.2 -- July 25, 2019
+### 0.11.2 -- Jul 25, 2019
 * Fix #628 - incompatibility with pyparsing 2.4.1
 
 ### 0.11.1 -- Nov 15, 2018

datajoint/external.py

@@ -251,7 +251,7 @@ def download_filepath(self, filepath_hash):
                 checksum = uuid_from_file(local_filepath)
                 if checksum != contents_hash:  # this should never happen without outside interference
                     raise DataJointError("'{file}' downloaded but did not pass checksum'".format(file=local_filepath))
-            return local_filepath, contents_hash
+            return str(local_filepath), contents_hash
 
     # --- UTILITIES ---
 

datajoint/fetch.py

@@ -50,7 +50,7 @@ def _get(connection, attr, data, squeeze, download_path):
     adapt = attr.adapter.get if attr.adapter else lambda x: x
 
     if attr.is_filepath:
-        return str(adapt(extern.download_filepath(uuid.UUID(bytes=data))[0]))
+        return adapt(extern.download_filepath(uuid.UUID(bytes=data))[0])
 
     if attr.is_attachment:
         # Steps:
@@ -65,22 +65,22 @@ def _get(connection, attr, data, squeeze, download_path):
         if local_filepath.is_file():
             attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)
             if attachment_checksum == hash.uuid_from_file(local_filepath, init_string=attachment_name + '\0'):
-                return str(adapt(local_filepath))  # checksum passed, no need to download again
+                return adapt(str(local_filepath))  # checksum passed, no need to download again
             # generate the next available alias filename
             for n in itertools.count():
                 f = local_filepath.parent / (local_filepath.stem + '_%04x' % n + local_filepath.suffix)
                 if not f.is_file():
                     local_filepath = f
                     break
                 if attachment_checksum == hash.uuid_from_file(f, init_string=attachment_name + '\0'):
-                    return str(adapt(f))  # checksum passed, no need to download again
+                    return adapt(str(f))  # checksum passed, no need to download again
         # Save attachment
         if attr.is_external:
             extern.download_attachment(_uuid, attachment_name, local_filepath)
         else:
             # write from buffer
             safe_write(local_filepath, data.split(b"\0", 1)[1])
-        return str(adapt(local_filepath))  # download file from remote store
+        return adapt(str(local_filepath))  # download file from remote store
 
     return adapt(uuid.UUID(bytes=data) if attr.uuid else (
             blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze) 

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.12.0"
+__version__ = "0.12.1"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

docs-parts/admin/5-blob-config_lang1.rst

@@ -8,7 +8,7 @@
                   location = 'datajoint-projects/lab1',
                   access_key='1234567',
                   secret_key='foaf1234'),
-    'external-raw'] = dict( # 'raw' storage for this pipeline
+    'external-raw': dict( # 'raw' storage for this pipeline
                   protocol='file',
                   location='/net/djblobs/myschema')
   }

docs-parts/intro/Releases_lang1.rst

@@ -1,4 +1,8 @@
-0.12.0 -- October 31, 2019
+0.12.1 -- Nov 2, 2019
+-------------------------
+* Bugfix - AttributeAdapter converts into a string (#684)
+
+0.12.0 -- Oct 31, 2019
 -------------------------
 * Dropped support for Python 3.4
 * Support secure connections with TLS (aka SSL) PR #620

tests/schema_adapted.py

@@ -1,9 +1,19 @@
 import datajoint as dj
 import networkx as nx
+from pathlib import Path
+import tempfile
 from datajoint import errors
 
-from . import PREFIX, CONN_INFO
+from . import PREFIX, CONN_INFO, S3_CONN_INFO
 
+stores_config = {
+    'repo_s3': dict(
+        S3_CONN_INFO,
+        protocol='s3',
+        location='adapted/repo',
+        stage=tempfile.mkdtemp())
+}
+dj.config['stores'] = stores_config
 
 schema_name = PREFIX + '_test_custom_datatype'
 schema = dj.schema(schema_name, connection=dj.conn(**CONN_INFO))
@@ -40,5 +50,41 @@ class Connectivity(dj.Manual):
     conn_graph = null : <graph>
     """
 
+errors._switch_filepath_types(True)
 
+
+class Filepath2GraphAdapter(dj.AttributeAdapter):
+
+    attribute_type = 'filepath@repo_s3'
+
+    @staticmethod
+    def get(obj):
+        s = open(obj, "r").read()
+        return nx.spring_layout(
+            nx.lollipop_graph(4, 2), seed=int(s))
+
+    @staticmethod
+    def put(obj):
+        path = Path(
+            dj.config['stores']['repo_s3']['stage'], 'sample.txt')
+
+        f = open(path, "w")
+        f.write(str(obj*obj))
+        f.close()
+
+        return path
+
+
+file2graph = Filepath2GraphAdapter()
+
+
+@schema
+class Position(dj.Manual):
+    definition = """
+    pos_id : int
+    ---
+    seed_root: <file2graph>
+    """
+
+errors._switch_filepath_types(False)
 errors._switch_adapted_types(False)  # disable again

tests/test_adapted_attributes.py

@@ -3,7 +3,7 @@
 from itertools import zip_longest
 from nose.tools import assert_true, assert_equal
 from . import schema_adapted as adapted
-from .schema_adapted import graph
+from .schema_adapted import graph, file2graph
 
 
 def test_adapted_type():
@@ -20,6 +20,22 @@ def test_adapted_type():
     dj.errors._switch_adapted_types(False)
 
 
+def test_adapted_filepath_type():
+    # https://github.com/datajoint/datajoint-python/issues/684
+    dj.errors._switch_adapted_types(True)
+    dj.errors._switch_filepath_types(True)
+    c = adapted.Position()
+    Position.insert([{'pos_id': 0, 'seed_root': 3}])
+    result = (Position & 'pos_id=0').fetch1('seed_root')
+
+    assert_true(isinstance(result, dict))
+    assert_equal(0.3761992090175474, result[1][0])
+    assert_true(6 == len(result))
+
+    c.delete()
+    dj.errors._switch_filepath_types(False)
+    dj.errors._switch_adapted_types(False)
+
 # test spawned classes
 local_schema = dj.schema(adapted.schema_name)
 local_schema.spawn_missing_classes()

tests/test_filepath.py

@@ -10,6 +10,7 @@
 def setUp(self):
     dj.config['stores'] = stores_config
 
+
 def test_path_match(store="repo"):
     """ test file path matches and empty file"""
     ext = schema.external[store]
@@ -22,7 +23,7 @@ def test_path_match(store="repo"):
     open(str(managed_file), 'a').close()
 
     # put the file
-    uuid = ext.upload_filepath(managed_file)
+    uuid = ext.upload_filepath(str(managed_file))
 
     #remove
     managed_file.unlink()
@@ -35,12 +36,13 @@ def test_path_match(store="repo"):
 
     # # Download the file and check its contents.
     restored_path, checksum = ext.download_filepath(uuid)
-    assert_equal(restored_path, managed_file)
-    assert_equal(checksum, dj.hash.uuid_from_file(managed_file))
+    assert_equal(restored_path, str(managed_file))
+    assert_equal(checksum, dj.hash.uuid_from_file(str(managed_file)))
 
     # cleanup
     ext.delete(delete_external_files=True)
 
+
 def test_filepath(store="repo"):
     """ test file management """
     ext = schema.external[store]
@@ -56,8 +58,8 @@ def test_filepath(store="repo"):
         f.write(data)
 
     # put the same file twice to ensure storing once
-    uuid1 = ext.upload_filepath(managed_file)
-    uuid2 = ext.upload_filepath(managed_file)   # no duplication should arise if file is the same
+    uuid1 = ext.upload_filepath(str(managed_file))
+    uuid2 = ext.upload_filepath(str(managed_file))   # no duplication should arise if file is the same
     assert_equal(uuid1, uuid2)
 
     # remove to ensure downloading
@@ -67,8 +69,8 @@ def test_filepath(store="repo"):
     # Download the file and check its contents. Repeat causes no download from remote
     for _ in 1, 2:
         restored_path, checksum = ext.download_filepath(uuid1)
-        assert_equal(restored_path, managed_file)
-        assert_equal(checksum, dj.hash.uuid_from_file(managed_file))
+        assert_equal(restored_path, str(managed_file))
+        assert_equal(checksum, dj.hash.uuid_from_file(str(managed_file)))
 
     # verify same data
     with managed_file.open('rb') as f:
@@ -92,8 +94,8 @@ def test_duplicate_upload(store="repo"):
     managed_file.parent.mkdir(parents=True, exist_ok=True)
     with managed_file.open('wb') as f:
         f.write(os.urandom(300))
-    ext.upload_filepath(managed_file)
-    ext.upload_filepath(managed_file)  # this is fine because the file is the same
+    ext.upload_filepath(str(managed_file))
+    ext.upload_filepath(str(managed_file))  # this is fine because the file is the same
 
 
 def test_duplicate_upload_s3():
@@ -110,10 +112,10 @@ def test_duplicate_error(store="repo"):
     managed_file.parent.mkdir(parents=True, exist_ok=True)
     with managed_file.open('wb') as f:
         f.write(os.urandom(300))
-    ext.upload_filepath(managed_file)
+    ext.upload_filepath(str(managed_file))
     with managed_file.open('wb') as f:
         f.write(os.urandom(300))
-    ext.upload_filepath(managed_file)  # this should raise exception because the file has changed
+    ext.upload_filepath(str(managed_file))  # this should raise exception because the file has changed
 
 
 def test_duplicate_error_s3():
@@ -135,7 +137,7 @@ def test_filepath_class(table=Filepath(), store="repo"):
     assert_equal(data, contents)
 
     # upload file into shared repo
-    table.insert1((1, managed_file))
+    table.insert1((1, str(managed_file)))
 
     # remove file locally
     managed_file.unlink()
@@ -187,7 +189,7 @@ def test_filepath_cleanup(table=Filepath(), store="repo"):
         managed_file.parent.mkdir(parents=True, exist_ok=True)
         with managed_file.open('wb') as f:
             f.write(contents)  # same in all files
-        table.insert1((i, managed_file))
+        table.insert1((i, str(managed_file)))
     assert_equal(len(table), n)
 
     ext = schema.external[store]
@@ -235,7 +237,7 @@ def test_return_string(table=Filepath(), store="repo"):
     assert_equal(data, contents)
 
     # upload file into shared repo
-    table.insert1((138, managed_file))
+    table.insert1((138, str(managed_file)))
 
     # remove file locally
     managed_file.unlink()