Merge pull request #586 from girder/pooch-to-fetch

Use pooch to fetch external data.
girder · Apr 2, 2021 · ec3fb2a · ec3fb2a
2 parents 5a6c88f + 3e47e26
commit ec3fb2a
Show file tree

Hide file tree

Showing 41 changed files with 237 additions and 244 deletions.
diff --git a/girder/test_girder/girder_utilities.py b/girder/test_girder/girder_utilities.py
@@ -3,7 +3,8 @@
 from girder.models.folder import Folder
 from girder.models.upload import Upload
 
-from test.utilities import externaldata, JFIFHeader, JPEGHeader, PNGHeader  # noqa
+from test.datastore import datastore
+from test.utilities import JFIFHeader, JPEGHeader, PNGHeader  # noqa
 
 
 def namedFolder(user, folderName='Public'):
@@ -24,7 +25,7 @@ def uploadFile(filePath, user, assetstore, folderName='Public', name=None):
 
 
 def uploadExternalFile(hashPath, user, assetstore, folderName='Public', name=None):
-    imagePath = externaldata(hashPath)
+    imagePath = datastore.fetch(hashPath)
     return uploadFile(imagePath, user=user, assetstore=assetstore, folderName=folderName, name=name)
 
 

diff --git a/girder/test_girder/test_large_image.py b/girder/test_girder/test_large_image.py
@@ -115,7 +115,7 @@ def testSettings(server):
 @pytest.mark.usefixtures('unbindLargeImage')
 @pytest.mark.plugin('large_image')
 def testThumbnailFileJob(server, admin, user, fsAssetstore):
-    file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
+    file = utilities.uploadExternalFile('sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
 
     # We should report zero thumbnails
@@ -299,7 +299,7 @@ def testCaches(server, admin):
 @pytest.mark.usefixtures('unbindLargeImage')
 @pytest.mark.plugin('large_image')
 def testAssociateImageCaching(server, admin, user, fsAssetstore):
-    file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
+    file = utilities.uploadExternalFile('sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles/images/label' % itemId,
                           user=admin, isJson=False)
@@ -334,7 +334,7 @@ def testListSources(server):
 @pytest.mark.usefixtures('unbindLargeImage')
 @pytest.mark.plugin('large_image')
 def testGetLargeImagePath(server, admin, fsAssetstore):
-    file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
+    file = utilities.uploadExternalFile('sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     item = Item().load(itemId, user=admin)
     ts = ImageItem().tileSource(item)

diff --git a/girder/test_girder/test_tiles_rest.py b/girder/test_girder/test_tiles_rest.py
@@ -208,7 +208,7 @@ def _postTileViaHttp(server, admin, itemId, fileId, jobAction=None, data=None, c
 @pytest.mark.plugin('large_image')
 def testTilesFromPTIF(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     fileId = str(file['_id'])
     # We should already have tile information.  Ask to delete it so we can
@@ -327,7 +327,7 @@ def testTilesFromPTIF(server, admin, fsAssetstore):
 def testTilesFromTest(server, admin, fsAssetstore):
     publicFolder = utilities.namedFolder(admin, 'Public')
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     items = [{'itemId': str(file['itemId']), 'fileId': str(file['_id'])}]
     # We should already have tile information.  Ask to delete it so we can
     # do other tests
@@ -528,7 +528,7 @@ def testTilesWithUnicodeName(server, admin, fsAssetstore):
     # Unicode file names shouldn't cause problems when accessing ptifs.
     # This requires an appropriate version of the python libtiff module.
     name = '\u0441\u043b\u0430\u0439\u0434.ptif'
-    origpath = utilities.externaldata('data/sample_image.ptif.sha512')
+    origpath = utilities.datastore.fetch('sample_image.ptif')
     altpath = os.path.join(os.path.dirname(origpath), name)
     if os.path.exists(altpath):
         os.unlink(altpath)
@@ -568,7 +568,7 @@ def testTilesFromBadFiles(boundServer, admin, fsAssetstore, girderWorker):
 @pytest.mark.plugin('large_image')
 def testThumbnails(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     fileId = str(file['_id'])
     # We should already have tile information.  Ask to delete it so we can
@@ -667,7 +667,7 @@ def testThumbnails(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testContentDisposition(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
 
     params = {'encoding': 'PNG', 'width': 200}
@@ -704,7 +704,7 @@ def testContentDisposition(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testRegions(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     # Get metadata to use in our tests
     resp = server.request(path='/item/%s/tiles' % itemId, user=admin)
@@ -788,7 +788,7 @@ def testRegions(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testPixel(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
 
     # Test bad parameters
@@ -823,15 +823,15 @@ def testPixel(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testGetTileSource(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     # We should have access via getGirderTileSource
     source = getGirderTileSource(itemId, user=admin)
     image, mime = source.getThumbnail(encoding='PNG', height=200)
     assert image[:len(utilities.PNGHeader)] == utilities.PNGHeader
 
     # We can also use a file with getTileSource.  The user is ignored.
-    imagePath = utilities.externaldata('data/sample_image.ptif.sha512')
+    imagePath = utilities.datastore.fetch('sample_image.ptif')
     source = getTileSource(imagePath, user=admin, encoding='PNG')
     image, mime = source.getThumbnail(encoding='JPEG', width=200)
     assert image[:len(utilities.JPEGHeader)] == utilities.JPEGHeader
@@ -851,7 +851,7 @@ def testTilesLoadModelCache(server, admin, fsAssetstore):
     loadmodelcache.invalidateLoadModelCache()
     token = str(Token().createToken(admin)['_id'])
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     # Now the tile request should tell us about the file.  These are
     # specific to our test file
@@ -868,7 +868,7 @@ def testTilesLoadModelCache(server, admin, fsAssetstore):
 def testTilesModelLookupCache(server, user, admin, fsAssetstore):
     User().load = mock.Mock(wraps=User().load)
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     token = str(Token().createToken(user)['_id'])
     lastCount = User().load.call_count
@@ -887,7 +887,7 @@ def testTilesModelLookupCache(server, user, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testTilesDZIEndpoints(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles' % itemId, user=admin)
     assert utilities.respStatus(resp) == 200
@@ -1013,7 +1013,7 @@ def testTilesAfterCopyItem(boundServer, admin, fsAssetstore, girderWorker):
 @pytest.mark.plugin('large_image')
 def testTilesAutoSetOption(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore,
+        'sample_image.ptif', admin, fsAssetstore,
         name='sample_image.PTIF')
     itemId = str(file['itemId'])
     # We should already have tile information.
@@ -1022,15 +1022,15 @@ def testTilesAutoSetOption(server, admin, fsAssetstore):
     # Turn off auto-set and try again
     Setting().set(constants.PluginSettings.LARGE_IMAGE_AUTO_SET, 'false')
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles' % itemId, user=admin)
     assert utilities.respStatus(resp) == 400
     assert 'No large image file' in resp.json['message']
     # Turn it back on
     Setting().set(constants.PluginSettings.LARGE_IMAGE_AUTO_SET, 'true')
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles' % itemId, user=admin)
     assert utilities.respStatus(resp) == 200
@@ -1040,7 +1040,7 @@ def testTilesAutoSetOption(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testTilesAssociatedImages(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
 
     resp = server.request(path='/item/%s/tiles/images' % itemId, user=admin)
@@ -1079,7 +1079,7 @@ def testTilesAssociatedImages(server, admin, fsAssetstore):
 
     # Test with an image that doesn't have associated images
     file = utilities.uploadExternalFile(
-        'data/sample_Easy1.png.sha512', admin, fsAssetstore)
+        'sample_Easy1.png', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles' % itemId, method='POST', user=admin)
     assert utilities.respStatus(resp) == 200
@@ -1098,7 +1098,7 @@ def testTilesAssociatedImages(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testTilesWithFrameNumbers(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample.ome.tif.sha512', admin, fsAssetstore)
+        'sample.ome.tif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     # Test that we can get frames via either tiles/zxy or tiles/fzxy and
     # that the frames are different
@@ -1129,7 +1129,7 @@ def testTilesWithFrameNumbers(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testTilesHistogram(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(
         path='/item/%s/tiles/histogram' % itemId,
@@ -1144,7 +1144,7 @@ def testTilesHistogram(server, admin, fsAssetstore):
 @pytest.mark.plugin('large_image')
 def testTilesInternalMetadata(server, admin, fsAssetstore):
     file = utilities.uploadExternalFile(
-        'data/sample_image.ptif.sha512', admin, fsAssetstore)
+        'sample_image.ptif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     resp = server.request(path='/item/%s/tiles/internal_metadata' % itemId)
     assert resp.json['tilesource'] == 'tiff'
@@ -1173,7 +1173,7 @@ def testTilesFromMultipleDotName(boundServer, admin, fsAssetstore, girderWorker)
 @pytest.mark.plugin('large_image')
 def testTilesForcedConversion(boundServer, admin, fsAssetstore, girderWorker):
     file = utilities.uploadExternalFile(
-        'data/landcover_sample_1000.tif.sha512', admin, fsAssetstore)
+        'landcover_sample_1000.tif', admin, fsAssetstore)
     itemId = str(file['itemId'])
     fileId = str(file['_id'])
     # We should already have tile information.  Ask to delete it so we can

diff --git a/girder_annotation/test_annotation/girder_utilities.py b/girder_annotation/test_annotation/girder_utilities.py
@@ -3,7 +3,7 @@
 from girder.models.folder import Folder
 from girder.models.upload import Upload
 
-from test.utilities import externaldata
+from test.datastore import datastore
 
 
 def namedFolder(user, folderName='Public'):
@@ -24,7 +24,7 @@ def uploadFile(filePath, user, assetstore, folderName='Public', name=None):
 
 
 def uploadExternalFile(hashPath, user, assetstore, folderName='Public', name=None):
-    imagePath = externaldata(hashPath)
+    imagePath = datastore.fetch(hashPath)
     return uploadFile(imagePath, user=user, assetstore=assetstore, folderName=folderName, name=name)
 
 

diff --git a/girder_annotation/test_annotation/test_annotations_rest.py b/girder_annotation/test_annotation/test_annotations_rest.py
@@ -264,7 +264,7 @@ def create_annotation(item, user):
 
         def upload(name, user=user, private=False):
             file = utilities.uploadExternalFile(
-                'data/sample_image.ptif.sha512', admin, fsAssetstore, name=name)
+                'sample_image.ptif', admin, fsAssetstore, name=name)
             item = Item().load(file['itemId'], level=AccessType.READ, user=admin)
 
             create_annotation(item, user)

diff --git a/test/data/04091217_ruc.nc.sha512 b/test/data/04091217_ruc.nc.sha512
diff --git a/test/data/DDX58_AXL_EGFR_well2_XY01.ome.tif.sha512 b/test/data/DDX58_AXL_EGFR_well2_XY01.ome.tif.sha512
diff --git a/test/data/G10-3_pelvis_crop-powers-of-3.tif.sha512 b/test/data/G10-3_pelvis_crop-powers-of-3.tif.sha512
diff --git a/test/data/HENormalN801.czi.sha512 b/test/data/HENormalN801.czi.sha512
diff --git a/test/data/ITGA3Hi_export_crop2.nd2.sha512 b/test/data/ITGA3Hi_export_crop2.nd2.sha512
diff --git a/test/data/JK-kidney_B-gal_H3_4C_1-500sec.jp2.sha512 b/test/data/JK-kidney_B-gal_H3_4C_1-500sec.jp2.sha512
diff --git a/test/data/JK-kidney_H3_4C_1-500sec.tif.sha512 b/test/data/JK-kidney_H3_4C_1-500sec.tif.sha512
diff --git a/test/data/TCGA-AA-A02O-11A-01-BS1.8b76f05c-4a8b-44ba-b581-6b8b4f437367.svs.sha512 b/test/data/TCGA-AA-A02O-11A-01-BS1.8b76f05c-4a8b-44ba-b581-6b8b4f437367.svs.sha512
diff --git a/test/data/d042-353.crop.small.float32.tif.sha512 b/test/data/d042-353.crop.small.float32.tif.sha512
diff --git a/test/data/huron.image2_jpeg2k.tif.sha512 b/test/data/huron.image2_jpeg2k.tif.sha512
diff --git a/test/data/landcover_sample_1000.tif.sha512 b/test/data/landcover_sample_1000.tif.sha512
diff --git a/test/data/sample.ome.tif.sha512 b/test/data/sample.ome.tif.sha512
diff --git a/test/data/sample.subifd.ome.tif.sha512 b/test/data/sample.subifd.ome.tif.sha512
diff --git a/test/data/sample_Easy1.jpeg.sha512 b/test/data/sample_Easy1.jpeg.sha512
diff --git a/test/data/sample_Easy1.png.sha512 b/test/data/sample_Easy1.png.sha512
diff --git a/test/data/sample_image.jp2.sha512 b/test/data/sample_image.jp2.sha512
diff --git a/test/data/sample_image.ptif.sha512 b/test/data/sample_image.ptif.sha512
diff --git a/...sample_jp2k_33003_TCGA-CV-7242-11A-01-TS1.1838afb1-9eee-4a70-9ae3-50e3ab45e242.svs.sha512 b/...sample_jp2k_33003_TCGA-CV-7242-11A-01-TS1.1838afb1-9eee-4a70-9ae3-50e3ab45e242.svs.sha512
diff --git a/test/data/sample_leica.scn.sha512 b/test/data/sample_leica.scn.sha512
diff --git a/.../sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512 b/.../sample_svs_image.TCGA-DU-6399-01A-01-TS1.e8eb65de-d63e-42db-af6f-14fefbbdf7bd.svs.sha512