From 5b17116af7ffa356356698393edf6e8dd2f832e2 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Wed, 29 Jan 2025 12:58:06 -0500 Subject: [PATCH] Improve how we use vips to read lower tile levels --- CHANGELOG.md | 6 ++ docs/format_examples_datastore.py | 12 ++- .../vips/large_image_source_vips/__init__.py | 83 ++++++++++++++++--- 3 files changed, 87 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e72c7fc71..cfa9a957c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## 1.31.1 + +### Improvements + +- Improve how we use vips to read lower tile levels ([#1794](../../pull/1794)) + ## 1.31.0 ### Features diff --git a/docs/format_examples_datastore.py b/docs/format_examples_datastore.py index 2ee00ff06..ab5126200 100644 --- a/docs/format_examples_datastore.py +++ b/docs/format_examples_datastore.py @@ -100,7 +100,8 @@ examples=[ dict( filename='sample_image.nd2', - # originally from 'https://downloads.openmicroscopy.org/images/ND2/aryeh/MeOh_high_fluo_003.nd2', + # originally from + # 'https://downloads.openmicroscopy.org/images/ND2/aryeh/MeOh_high_fluo_003.nd2', url='https://data.kitware.com/api/v1/file/hashsum/sha512/4e76e490c915b10f646cb516f85a4d36d52aa7eff94715b90222644180e26fef6768493887c05adf182cf0351ba0bce659204041c4698a0f6b08423586788f4d/download', hash='8e23bb594cd18314f9c18e70d736088ae46f8bc696ab7dc047784be416d7a706', ), @@ -127,7 +128,8 @@ examples=[ dict( filename='US-MONO2-8-8x-execho.dcm', - # originally from 'https://downloads.openmicroscopy.org/images/DICOM/samples/US-MONO2-8-8x-execho.dcm', + # originally from + # 'https://downloads.openmicroscopy.org/images/DICOM/samples/US-MONO2-8-8x-execho.dcm', url='https://data.kitware.com/api/v1/file/hashsum/sha512/5332044f887d82c7f3693c6ca180f07accf5f00c2b7b1a3a29ef9ae737d5f1975478b5e2d5846c391987b8051416068f57a7062e848323c700412236b35679db/download', hash='7d3f54806d0315c6cfc8b7371649a242b5ef8f31e0d20221971dd8087f2ff1ea', ), @@ -141,7 +143,8 @@ examples=[ dict( filename='20191025 Test FRET 585. 423, 426.lif', - # originally from 'https://downloads.openmicroscopy.org/images/Leica-LIF/imagesc-30856/20191025%20Test%20FRET%20585.%20423,%20426.lif', + # originally from + # 'https://downloads.openmicroscopy.org/images/Leica-LIF/imagesc-30856/20191025%20Test%20FRET%20585.%20423,%20426.lif', url='https://data.kitware.com/api/v1/file/hashsum/sha512/d25de002d8a81dfcaf6b062b9f429ca85bb81423bc09d3aa33d9d51e9392cc4ace2b8521475e373ceecaf958effd0fade163e7173c467aab66c957da14482ed7/download', hash='8d4ee62868b9616b832c2eb28e7d62ec050fb032e0bc11ea0a392f5c84390c71', ), @@ -155,7 +158,8 @@ examples=[ dict( filename='Animated_PNG_example_bouncing_beach_ball.png', - # originally from 'https://upload.wikimedia.org/wikipedia/commons/1/14/Animated_PNG_example_bouncing_beach_ball.png', + # originally from + # 'https://upload.wikimedia.org/wikipedia/commons/1/14/Animated_PNG_example_bouncing_beach_ball.png', url='https://data.kitware.com/api/v1/file/hashsum/sha512/465ebdc2e81b2576dfc96b34e82db7f968e6d4f32f0fa80ef4bb0e44ed216230e6be1a2e4b11ae301a2905cc582dd24cbd2c360d9567ff7b1dac2c871f6d1e37/download', hash='3b28e2462f1b31d0d15d795e6e58baf397899c3f864be7034bf47939b5bbbc3b', ), diff --git a/sources/vips/large_image_source_vips/__init__.py b/sources/vips/large_image_source_vips/__init__.py index e7d491706..7d2cb8c28 100644 --- a/sources/vips/large_image_source_vips/__init__.py +++ b/sources/vips/large_image_source_vips/__init__.py @@ -16,7 +16,7 @@ dtypeToGValue) from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError from large_image.tilesource import FileTileSource -from large_image.tilesource.utilities import _imageToNumpy, _newFromFileLock +from large_image.tilesource.utilities import _imageToNumpy, _newFromFileLock, nearPowerOfTwo logging.getLogger('pyvips').setLevel(logging.ERROR) @@ -98,6 +98,7 @@ def __init__(self, path, **kwargs): # noqa if 'n-pages' in self._image.get_fields(): pages = self._image.get('n-pages') self._frames = [0] + self._lowres = {} for page in range(1, pages): subInputPath = self._largeImagePath + f'[page={page}{self._suffix}]' with _newFromFileLock: @@ -113,8 +114,13 @@ def __init__(self, path, **kwargs): # noqa self._frames.append(page) continue if subImage.width * subImage.height < self.sizeX * self.sizeY: + if (nearPowerOfTwo(self.sizeX, subImage.width) and + nearPowerOfTwo(self.sizeY, subImage.height)): + level = int(round(math.log(self.sizeX / subImage.width) / math.log(2))) + self._lowres.setdefault(len(self._frames) - 1, {})[level] = page continue self._frames = [page] + self._lowres = {} self.sizeX = subImage.width self.sizeY = subImage.height try: @@ -122,12 +128,41 @@ def __init__(self, path, **kwargs): # noqa except Exception: pass self._image = subImage + self._checkLowerLevels() self.levels = int(max(1, math.ceil(math.log( float(max(self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2)) + 1)) - if len(self._frames) > 1: + if len(self._frames) > 1 or self._lowres is not None: self._recentFrames = cachetools.LRUCache(maxsize=6) self._frameLock = threading.RLock() + def _checkLowerLevels(self): + if (len(self._lowres) != len(self._frames) or + min(len(v) for v in self._lowres.values()) != + max(len(v) for v in self._lowres.values()) or + min(len(v) for v in self._lowres.values()) == 0): + self._lowres = None + if len(self._frames) == 1 and 'openslide.level-count' in self._image.get_fields(): + self._lowres = [{}] + for oslevel in range(1, int(self._image.get('openslide.level-count'))): + with _newFromFileLock: + try: + subImage = pyvips.Image.new_from_file( + self._largeImagePath, level=oslevel) + except Exception: + continue + if subImage.width * subImage.height < self.sizeX * self.sizeY: + if (nearPowerOfTwo(self.sizeX, subImage.width) and + nearPowerOfTwo(self.sizeY, subImage.height)): + level = int(round(math.log( + self.sizeX / subImage.width) / math.log(2))) + self._lowres[0][level] = (self._frames[0], oslevel) + if not len(self._lowres[0]): + self._lowres = None + else: + self._lowres = list(self._lowres.values()) + if self._lowres is not None: + self._populatedLevels = len(self._lowres[0]) + 1 + def _initNew(self, **kwargs): """ Initialize the tile class for creating a new image. @@ -139,6 +174,7 @@ def _initNew(self, **kwargs): self.sizeX = self.sizeY = self.levels = 0 self.tileWidth = self.tileHeight = self._tileSize self._frames = [0] + self._lowres = None self._cacheValue = str(uuid.uuid4()) self._output = None self._editable = True @@ -194,25 +230,38 @@ def getMetadata(self): self._addMetadataFrameInformation(result) return result - def _getFrameImage(self, frame=0): + def _getFrameImage(self, frame=0, lowres=None): """ Get the vips image associated with a specific frame. :param frame: the 0-based frame to get. + :param lowres: the lower resolution part of the frame :returns: a vips image. """ if self._image is None and self._output: self._outputToImage() img = self._image - if frame > 0: + if frame > 0 or lowres: with self._frameLock: - if frame not in self._recentFrames: - subpath = self._largeImagePath + f'[page={self._frames[frame]}{self._suffix}]' + key = (frame, lowres) + frameval = self._frames[frame] + params = {} + if lowres is not None: + if isinstance(self._lowres[frame][lowres], tuple): + frameval = self._lowres[frame][lowres][0] + params = {'level': self._lowres[frame][lowres][1]} + else: + frameval = self._lowres[frame][lowres] + if key not in self._recentFrames: + if frameval: + subpath = self._largeImagePath + f'[page={frameval}{self._suffix}]' + else: + subpath = self._largeImagePath with _newFromFileLock: - img = pyvips.Image.new_from_file(subpath) - self._recentFrames[frame] = img + img = pyvips.Image.new_from_file(subpath, **params) + self._recentFrames[key] = img else: - img = self._recentFrames[frame] + img = self._recentFrames[key] return img def getNativeMagnification(self): @@ -231,8 +280,22 @@ def getNativeMagnification(self): def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): frame = self._getFrame(**kwargs) self._xyzInRange(x, y, z, frame, len(self._frames)) - img = self._getFrameImage(frame) x0, y0, x1, y1, step = self._xyzToCorners(x, y, z) + lowres = None + if self._lowres and step > 1: + use = 0 + for ll in self._lowres[frame].keys(): + if 2 ** ll <= step: + use = max(use, ll) + if use: + lowres = use + div = 2 ** use + x0 //= div + y0 //= div + x1 //= div + y1 //= div + step //= div + img = self._getFrameImage(frame, lowres) tileimg = img.crop(min(x0, img.width), min(y0, img.height), min(x1, img.width) - min(x0, img.width), min(y1, img.height) - min(y0, img.height))