From eaccbc48c122d034c25771aa04a6afc4259fd2c0 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Mon, 16 Aug 2021 08:33:40 -0400 Subject: [PATCH] Better associated image cache control. Before, the per-image cache count was for all images (thumbnails, thumbnails of associated images, etc). Now it is by type. Further, a specific imageKey can be queried or deleted independently of the others (e.g., just macro). --- CHANGELOG.md | 5 ++++ .../girder_large_image/models/image_item.py | 20 +++++++++++---- .../rest/large_image_resource.py | 25 ++++++++++++++----- girder/test_girder/test_large_image.py | 13 ++++++++++ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cdc3d256..7310fcc8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## Unreleased + +### Improvements +- More control over associated image caching + ## Version 1.7.0 ### Features diff --git a/girder/girder_large_image/models/image_item.py b/girder/girder_large_image/models/image_item.py index 0429c54f9..9aa772c06 100644 --- a/girder/girder_large_image/models/image_item.py +++ b/girder/girder_large_image/models/image_item.py @@ -326,7 +326,7 @@ def _getAndCacheImageOrData( 'attachedToType': 'item', 'attachedToId': item['_id'], 'isLargeImageThumbnail' if not pickleCache else 'isLargeImageData': True, - 'thumbnailKey': key + 'thumbnailKey': key, }) if existing: if checkAndCreate and checkAndCreate != 'nosave': @@ -354,7 +354,8 @@ def _getAndCacheImageOrData( constants.PluginSettings.LARGE_IMAGE_MAX_THUMBNAIL_FILES)) saveFile = maxThumbnailFiles > 0 # Make sure we don't exceed the desired number of thumbnails - self.removeThumbnailFiles(item, maxThumbnailFiles - 1) + self.removeThumbnailFiles( + item, maxThumbnailFiles - 1, imageKey=keydict.get('imageKey') or 'none') if (saveFile and checkAndCreate != 'nosave' and ( pickleCache or isinstance(imageData, bytes))): dataStored = imageData if not pickleCache else pickle.dumps(imageData, protocol=4) @@ -376,7 +377,7 @@ def _getAndCacheImageOrData( File().save(datafile) return imageData, imageMime - def removeThumbnailFiles(self, item, keep=0, sort=None, **kwargs): + def removeThumbnailFiles(self, item, keep=0, sort=None, imageKey=None, **kwargs): """ Remove all large image thumbnails from an item. @@ -384,6 +385,8 @@ def removeThumbnailFiles(self, item, keep=0, sort=None, **kwargs): :param keep: keep this many entries. :param sort: the sort method used. The first (keep) records in this sort order are kept. + :param imageKey: None for the basic thumbnail, otherwise an associated + imageKey. :param kwargs: additional parameters to determine which files to remove. :returns: a tuple of (the number of files before removal, the number of @@ -400,6 +403,11 @@ def removeThumbnailFiles(self, item, keep=0, sort=None, **kwargs): 'attachedToId': item['_id'], key: True, } + if imageKey and key == 'isLargeImageThumbnail': + if imageKey == 'none': + query['thumbnailKey'] = {'not': {'$regex': '"imageKey":'}} + else: + query['thumbnailKey'] = {'$regex': '"imageKey":"%s"' % imageKey} query.update(kwargs) present = 0 removed = 0 @@ -446,8 +454,10 @@ def tileFrames(self, item, checkAndCreate='nosave', **kwargs): and framesAcross. :returns: regionData, regionMime: the image data and the mime type. """ + imageKey = 'tileFrames' return self._getAndCacheImageOrData( - item, 'tileFrames', checkAndCreate, dict(kwargs), **kwargs) + item, 'tileFrames', checkAndCreate, + dict(kwargs, imageKey=imageKey), **kwargs) def getPixel(self, item, **kwargs): """ @@ -477,7 +487,7 @@ def histogram(self, item, **kwargs): imageKey = 'histogram' result = self._getAndCacheImageOrData( item, 'histogram', False, dict(kwargs, imageKey=imageKey), - imageKey=imageKey, pickleCache=True, **kwargs)[0] + pickleCache=True, **kwargs)[0] return result def getBandInformation(self, item, statistics=True, **kwargs): diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 9ee82fb64..991546604 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -17,6 +17,7 @@ import concurrent.futures import datetime import json +import re import sys import time @@ -276,12 +277,15 @@ def countThumbnails(self, params): @describeRoute( Description('Count the number of cached associated image files for ' 'large_image items.') + .param('imageKey', 'If specific, only include images with the ' + 'specified key', required=False) ) @access.admin def countAssociatedImages(self, params): - return self._countCachedImages(None, associatedImages=True) + return self._countCachedImages( + None, associatedImages=True, imageKey=params.get('imageKey')) - def _countCachedImages(self, spec, associatedImages=False): + def _countCachedImages(self, spec, associatedImages=False, imageKey=None): if spec is not None: try: spec = json.loads(spec) @@ -299,7 +303,10 @@ def _countCachedImages(self, spec, associatedImages=False): if entry is not None: query['thumbnailKey'] = entry elif associatedImages: - query['thumbnailKey'] = {'$regex': '"imageKey":'} + if imageKey and re.match(r'^[0-9A-Za-z].*$', imageKey): + query['thumbnailKey'] = {'$regex': '"imageKey":"%s"' % imageKey} + else: + query['thumbnailKey'] = {'$regex': '"imageKey":'} count += File().find(query).count() return count @@ -362,12 +369,15 @@ def deleteThumbnails(self, params): @describeRoute( Description('Delete cached associated image files from large_image items.') + .param('imageKey', 'If specific, only include images with the ' + 'specified key', required=False) ) @access.admin def deleteAssociatedImages(self, params): - return self._deleteCachedImages(None, associatedImages=True) + return self._deleteCachedImages( + None, associatedImages=True, imageKey=params.get('imageKey')) - def _deleteCachedImages(self, spec, associatedImages=False): + def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): if spec is not None: try: spec = json.loads(spec) @@ -385,7 +395,10 @@ def _deleteCachedImages(self, spec, associatedImages=False): if entry is not None: query['thumbnailKey'] = entry elif associatedImages: - query['thumbnailKey'] = {'$regex': '"imageKey":'} + if imageKey and re.match(r'^[0-9A-Za-z].*$', imageKey): + query['thumbnailKey'] = {'$regex': '"imageKey":"%s"' % imageKey} + else: + query['thumbnailKey'] = {'$regex': '"imageKey":'} for file in File().find(query): File().remove(file) removed += 1 diff --git a/girder/test_girder/test_large_image.py b/girder/test_girder/test_large_image.py index df5703b41..296896589 100644 --- a/girder/test_girder/test_large_image.py +++ b/girder/test_girder/test_large_image.py @@ -308,10 +308,23 @@ def testAssociateImageCaching(server, admin, user, fsAssetstore): resp = server.request(path='/large_image/associated_images', user=admin) assert utilities.respStatus(resp) == 200 assert resp.json == 1 + resp = server.request(path='/large_image/associated_images', user=admin, params={ + 'imageKey': 'label'}) + assert utilities.respStatus(resp) == 200 + assert resp.json == 1 + resp = server.request(path='/large_image/associated_images', user=admin, params={ + 'imageKey': 'macro'}) + assert utilities.respStatus(resp) == 200 + assert resp.json == 0 # Test DELETE associated_images resp = server.request( method='DELETE', path='/large_image/associated_images', user=user) assert utilities.respStatus(resp) == 403 + resp = server.request( + method='DELETE', path='/large_image/associated_images', user=admin, params={ + 'imageKey': 'macro'}) + assert utilities.respStatus(resp) == 200 + assert resp.json == 0 resp = server.request( method='DELETE', path='/large_image/associated_images', user=admin) assert utilities.respStatus(resp) == 200