diff --git a/sources/ometiff/large_image_source_ometiff/__init__.py b/sources/ometiff/large_image_source_ometiff/__init__.py index 02a7d9609..bb78f2d5e 100644 --- a/sources/ometiff/large_image_source_ometiff/__init__.py +++ b/sources/ometiff/large_image_source_ometiff/__init__.py @@ -16,6 +16,7 @@ # limitations under the License. ############################################################################## +import copy import math import numpy import PIL.Image @@ -101,30 +102,7 @@ def __init__(self, path, **kwargs): raise TileSourceException('Not an OME Tiff') self._omeinfo = info['OME'] self._checkForOMEZLoop(largeImagePath) - if isinstance(self._omeinfo['Image'], dict): - self._omeinfo['Image'] = [self._omeinfo['Image']] - for img in self._omeinfo['Image']: - if isinstance(img['Pixels'].get('TiffData'), dict): - img['Pixels']['TiffData'] = [img['Pixels']['TiffData']] - if isinstance(img['Pixels'].get('Plane'), dict): - img['Pixels']['Plane'] = [img['Pixels']['Plane']] - try: - self._omebase = self._omeinfo['Image'][0]['Pixels'] - if ((not len(self._omebase['TiffData']) or ( - len(self._omebase['TiffData']) == 1 and - self._omebase['TiffData'][0] == {})) and - len(self._omebase['Plane'])): - self._omebase['TiffData'] = self._omebase['Plane'] - if len({entry.get('UUID', {}).get('FileName', '') - for entry in self._omebase['TiffData']}) > 1: - raise TileSourceException('OME Tiff references multiple files') - if (len(self._omebase['TiffData']) != int(self._omebase['SizeC']) * - int(self._omebase['SizeT']) * int(self._omebase['SizeZ']) or - len(self._omebase['TiffData']) != len( - self._omebase.get('Plane', self._omebase['TiffData']))): - raise TileSourceException('OME Tiff contains frames that contain multiple planes') - except (KeyError, ValueError, IndexError): - raise TileSourceException('OME Tiff does not contain an expected record') + self._parseOMEInfo() omeimages = [ entry['Pixels'] for entry in self._omeinfo['Image'] if len(entry['Pixels']['TiffData']) == len(self._omebase['TiffData'])] @@ -206,6 +184,39 @@ def _checkForOMEZLoop(self, largeImagePath): info['Image']['Pixels']['PlanesFromZloop'] = 'true' info['Image']['Pixels']['SizeZ'] = str(zloop) + def _parseOMEInfo(self): + if isinstance(self._omeinfo['Image'], dict): + self._omeinfo['Image'] = [self._omeinfo['Image']] + for img in self._omeinfo['Image']: + if isinstance(img['Pixels'].get('TiffData'), dict): + img['Pixels']['TiffData'] = [img['Pixels']['TiffData']] + if isinstance(img['Pixels'].get('Plane'), dict): + img['Pixels']['Plane'] = [img['Pixels']['Plane']] + try: + self._omebase = self._omeinfo['Image'][0]['Pixels'] + if ((not len(self._omebase['TiffData']) or + len(self._omebase['TiffData']) == 1) and + len(self._omebase['Plane'])): + if not len(self._omebase['TiffData']) or self._omebase['TiffData'][0] == {}: + self._omebase['TiffData'] = self._omebase['Plane'] + elif (int(self._omebase['TiffData'][0].get('PlaneCount', 0)) == + len(self._omebase['Plane'])): + planes = copy.deepcopy(self._omebase['Plane']) + for idx, plane in enumerate(planes): + plane['IFD'] = plane.get( + 'IFD', int(self._omebase['TiffData'][0].get('IFD', 0)) + idx) + self._omebase['TiffData'] = planes + if len({entry.get('UUID', {}).get('FileName', '') + for entry in self._omebase['TiffData']}) > 1: + raise TileSourceException('OME Tiff references multiple files') + if (len(self._omebase['TiffData']) != int(self._omebase['SizeC']) * + int(self._omebase['SizeT']) * int(self._omebase['SizeZ']) or + len(self._omebase['TiffData']) != len( + self._omebase.get('Plane', self._omebase['TiffData']))): + raise TileSourceException('OME Tiff contains frames that contain multiple planes') + except (KeyError, ValueError, IndexError): + raise TileSourceException('OME Tiff does not contain an expected record') + def getMetadata(self): """ Return a dictionary of metadata containing levels, sizeX, sizeY, diff --git a/sources/tiff/large_image_source_tiff/__init__.py b/sources/tiff/large_image_source_tiff/__init__.py index a4c7a855d..c47ef6cb1 100644 --- a/sources/tiff/large_image_source_tiff/__init__.py +++ b/sources/tiff/large_image_source_tiff/__init__.py @@ -65,6 +65,14 @@ class TiffFileTileSource(FileTileSource): 'image/x-ptif': SourcePriority.PREFERRED, } + # When getting tiles for otherwise empty directories (missing powers of + # two), we composite the tile from higher resolution levels. This can use + # excessive memory if there are too many missing levels. For instance, if + # there are six missing levels and the tile size is 1024 square RGBA, then + # 16 Gb are needed for the composited tile at a minimum. By setting + # _maxSkippedLevels, such large gaps are composited in stages. + _maxSkippedLevels = 3 + def __init__(self, path, **kwargs): """ Initialize the tile class. See the base class for other available @@ -272,9 +280,13 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, try: allowStyle = True if self._tiffDirectories[z] is None: - if sparseFallback: - raise IOTiffException('Missing z level %d' % z) - tile = self.getTileFromEmptyDirectory(x, y, z, **kwargs) + try: + tile = self.getTileFromEmptyDirectory(x, y, z, **kwargs) + except Exception: + if sparseFallback: + raise IOTiffException('Missing z level %d' % z) + else: + raise allowStyle = False format = TILE_FORMAT_PIL else: @@ -327,10 +339,14 @@ def getTileFromEmptyDirectory(self, x, y, z, **kwargs): :param z: original level. :returns: tile in PIL format. """ + basez = z scale = 1 while self._tiffDirectories[z] is None: scale *= 2 z += 1 + while z - basez > self._maxSkippedLevels: + z -= self._maxSkippedLevels + scale = int(scale / 2 ** self._maxSkippedLevels) tile = PIL.Image.new( 'RGBA', (self.tileWidth * scale, self.tileHeight * scale)) maxX = 2.0 ** (z + 1 - self.levels) * self.sizeX / self.tileWidth @@ -361,8 +377,11 @@ def getPreferredLevel(self, level): :returns level: a level with actual data that is no lower resolution. """ level = max(0, min(level, self.levels - 1)) + baselevel = level while self._tiffDirectories[level] is None and level < self.levels - 1: level += 1 + while level - baselevel > self._maxSkippedLevels: + level -= self._maxSkippedLevels return level def getAssociatedImagesList(self): diff --git a/tox.ini b/tox.ini index f636f4c7f..0c1edc087 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,8 @@ deps = pytest-cov>=2.6 pytest-girder>=3.0.4 pytest-xdist + celery!=4.4.4 +# celery 4.4.4 is broken; avoid it until a new version is released whitelist_externals = rm npx