Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce bioformats source memory usage #1038

Merged
merged 1 commit into from
Jan 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
### Improvements
- Speed up generating tiles for some multi source files ([#1035](../../pull/1035))
- Render item lists faster ([#1036](../../pull/1036))
- Reduce bioformats source memory usage ([#1038](../../pull/1038))

## 1.19.3

Expand Down
85 changes: 60 additions & 25 deletions sources/bioformats/large_image_source_bioformats/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ class BioformatsFileTileSource(FileTileSource, metaclass=LruCacheMetaclass):
_singleTileThreshold = 2048
_tileSize = 512
_associatedImageMaxSize = 8192
_maxSkippedLevels = 3

def __init__(self, path, **kwargs): # noqa
"""
Expand Down Expand Up @@ -482,6 +483,35 @@ def getInternalMetadata(self, **kwargs):
"""
return self._metadata

def _getTileFromEmptyLevel(self, x, y, z, **kwargs):
"""
Composite tiles from missing levels from larger levels in pieces to
avoid using too much memory.
"""
fac = int(2 ** self._maxSkippedLevels)
z += self._maxSkippedLevels
scale = 2 ** (self.levels - 1 - z)
result = None
for tx in range(fac - 1, -1, -1):
if x * fac + tx >= int(math.ceil(self.sizeX / self.tileWidth / scale)):
continue
for ty in range(fac - 1, -1, -1):
if y * fac + ty >= int(math.ceil(self.sizeY / self.tileHeight / scale)):
continue
tile = self.getTile(
x * fac + tx, y * fac + ty, z, pilImageAllowed=False,
numpyAllowed=True, **kwargs)
if result is None:
result = numpy.zeros((
ty * fac + tile.shape[0],
tx * fac + tile.shape[1],
tile.shape[2]), dtype=tile.dtype)
result[
ty * fac:ty * fac + tile.shape[0],
tx * fac:tx * fac + tile.shape[1],
::] = tile
return result[::scale, ::scale, ::]

@methodcache()
def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
self._xyzInRange(x, y, z)
Expand Down Expand Up @@ -514,31 +544,36 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
width = min(width, sizeXAtScale - offsetx)
height = min(height, sizeYAtScale - offsety)

with self._tileLock:
try:
javabridge.attach()
if width > 0 and height > 0:
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(offsetx, offsety, width, height))
else:
# We need the same dtype, so read 1x1 at 0x0
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(0, 0, 1, 1))
tile = numpy.zeros(tuple([0, 0] + list(tile.shape[2:])), dtype=tile.dtype)
format = TILE_FORMAT_NUMPY
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
raise TileSourceError('Failed to get Bioformat region (%s, %r).' % (es, (
fc, fz, ft, fseries, self.sizeX, self.sizeY, offsetx, offsety, width, height)))
finally:
if javabridge.get_env():
javabridge.detach()
if scale > 1:
tile = tile[::scale, ::scale]
if scale >= 2 ** self._maxSkippedLevels:
tile = self._getTileFromEmptyLevel(x, y, z, **kwargs)
format = TILE_FORMAT_NUMPY
else:
with self._tileLock:
try:
javabridge.attach()
if width > 0 and height > 0:
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(offsetx, offsety, width, height))
else:
# We need the same dtype, so read 1x1 at 0x0
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(0, 0, 1, 1))
tile = numpy.zeros(tuple([0, 0] + list(tile.shape[2:])), dtype=tile.dtype)
format = TILE_FORMAT_NUMPY
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
raise TileSourceError('Failed to get Bioformat region (%s, %r).' % (es, (
fc, fz, ft, fseries, self.sizeX, self.sizeY, offsetx,
offsety, width, height)))
finally:
if javabridge.get_env():
javabridge.detach()
if scale > 1:
tile = tile[::scale, ::scale]
if tile.shape[:2] != (finalHeight, finalWidth):
fillValue = 0
if tile.dtype == numpy.uint16:
Expand Down