From 97b9c6fc770780ce738c14323d6bdcda61d51058 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Fri, 10 Dec 2021 17:19:56 -0500 Subject: [PATCH] Allow named color palettes to be used by all sources. This also increase the ability to handle css and named colors. --- CHANGELOG.md | 3 + README.rst | 2 +- docs/tilesource_options.rst | 5 +- large_image/tilesource/base.py | 16 +++-- large_image/tilesource/utilities.py | 71 +++++++++++++++++++ requirements-dev.txt | 1 + setup.py | 2 + .../gdal/large_image_source_gdal/__init__.py | 11 ++- sources/gdal/setup.py | 1 - .../large_image_source_mapnik/__init__.py | 50 ++----------- test/test_source_base.py | 28 ++++++++ test/test_source_gdal.py | 2 +- test/test_source_mapnik.py | 4 +- 13 files changed, 133 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cbc6104a..70dbba324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Features +- Better palette support in styles ([#724](../../pull/724)) + ### Improvements - The openjpeg girder source handles uploaded files better ([#721](../../pull/721)) diff --git a/README.rst b/README.rst index f68a9fc33..449120d36 100644 --- a/README.rst +++ b/README.rst @@ -69,7 +69,7 @@ Modules Large Image consists of several Python modules designed to work together. These include: - ``large-image``: The core module. - You can specify extras_require of the name of any tile source included with this repository, ``sources`` for all of the tile sources in the repository, a specific source name (e.g., ``tiff``), ``memcached`` for using memcached for tile caching, ``converter`` to include the converter module, or ``all`` for all of the tile sources, memcached, and the converter module. + You can specify extras_require of the name of any tile source included with this repository, ``sources`` for all of the tile sources in the repository, a specific source name (e.g., ``tiff``), ``memcached`` for using memcached for tile caching, ``converter`` to include the converter module, ``colormap`` for using matplotlib for named color palettes used in styles, or ``all`` for all of the tile sources, memcached, the converter module, and better color map support. - ``large-image-converter``: A utility for using pyvips and other libraries to convert images into pyramidal tiff files that can be read efficiently by large_image. You can specify extras_require of ``jp2k`` to include modules to allow output to JPEG2000 compression, ``sources`` to include all sources, and ``stats`` to include modules to allow computing compression noise statistics. diff --git a/docs/tilesource_options.rst b/docs/tilesource_options.rst index 37b759e84..968c63ffd 100644 --- a/docs/tilesource_options.rst +++ b/docs/tilesource_options.rst @@ -45,7 +45,10 @@ A band definition is an object which can contain the following keys: - ``max``: the value to map to the last palette value. Defaults to 255. 'auto' to use 0 if the reported minimum and maximum of the band are between [0, 255] or use the reported maximum otherwise. 'min' or 'max' to always uses the reported minimum or maximum. -- ``palette``: a list of two or more color strings, where color strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA. The values between min and max are interpolated using a piecewise linear algorithm to map to the specified palette values. +- ``palette``: This is a list or two or more colors. The values between min and max are interpolated using a piecewise linear algorithm to map to the specified palette values. It can be specified in a variety of ways: + - a list of two or more color values, where the color values are css-style strings (e.g., of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA, or a css ``rgb``, ``rgba``, ``hsl``, or ``hsv`` string, or a css color name), or, if matplotlib is available, a matplotlib color name, or a list or tuple of RGB(A) values on a scale of [0-1]. + - a single string that is a color string as above. This is functionally a two-color palette with the first color as solid black (``#000``), and the second color the specified value + - a named color palette from the palettable library (e.g., ``matplotlib.Plasma_6``) or, if available, from the matplotlib library or one of its plugins (e.g., ``viridis``). - ``nodata``: the value to use for missing data. null or unset to not use a nodata value. diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index 1a7da6e95..0200fa101 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -22,7 +22,7 @@ from .utilities import (_encodeImage, _gdalParameters, # noqa: F401 _imageToNumpy, _imageToPIL, _letterboxImage, _vipsCast, _vipsParameters, dictToEtree, etreeToDict, - nearPowerOfTwo) + getPaletteColors, nearPowerOfTwo) class TileSource: @@ -87,7 +87,12 @@ def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0, maximum otherwise. 'min' or 'max' to always uses the reported minimum or maximum. :palette: a list of two or more color strings, where color - strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA. + strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA, or + any string parseable by the PIL modules, or, if it is + installed, byt matplotlib. Alternately, this can be a + single color, which implies ['#000', ], or the name + of a palettable paletter or, if available, a matplotlib + palette. :nodata: the value to use for missing data. null or unset to not use a nodata value. :composite: either 'lighten' or 'multiply'. Defaults to @@ -1063,10 +1068,9 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa composite = entry.get('composite', 'multiply') if band is None: band = image[:, :, bandidx] - palette = numpy.array([ - PIL.ImageColor.getcolor(clr, 'RGBA') for clr in entry.get( - 'palette', ['#000', '#FFF'] - if entry.get('band') != 'alpha' else ['#FFF0', '#FFFF'])]) + palette = getPaletteColors(entry.get( + 'palette', ['#000', '#FFF'] + if entry.get('band') != 'alpha' else ['#FFF0', '#FFFF'])) palettebase = numpy.linspace(0, 1, len(palette), endpoint=True) nodata = entry.get('nodata') min = self._getMinMax('min', entry.get('min', 'auto'), image.dtype, bandidx, frame) diff --git a/large_image/tilesource/utilities.py b/large_image/tilesource/utilities.py index 29a7c1888..55c0853d4 100644 --- a/large_image/tilesource/utilities.py +++ b/large_image/tilesource/utilities.py @@ -2,8 +2,10 @@ import math import xml.etree.ElementTree from collections import defaultdict +from operator import attrgetter import numpy +import palettable import PIL import PIL.Image import PIL.ImageColor @@ -11,6 +13,7 @@ from ..constants import (TILE_FORMAT_IMAGE, TILE_FORMAT_NUMPY, TILE_FORMAT_PIL, TileOutputMimeTypes, TileOutputPILFormat) +from ..exceptions import TileSourceError # Turn off decompression warning check PIL.Image.MAX_IMAGE_PIXELS = None @@ -398,3 +401,71 @@ def nearPowerOfTwo(val1, val2, tolerance=0.02): log2ratio = math.log(float(val1) / float(val2)) / math.log(2) # Compare the mantissa of the ratio's log2 value. return abs(log2ratio - round(log2ratio)) < tolerance + + +def _arrayToPalette(palette): + """ + Given an array of color strings, tuples, or lists, return a numpy array. + + :param palette: an array of color strings, tuples, or lists. + :returns: a numpy array of RGBA value on the scale of [0-255]. + """ + arr = [] + for clr in palette: + if isinstance(clr, (tuple, list)): + arr.append(numpy.array((list(clr) + [1, 1, 1])[:4]) * 255) + else: + try: + arr.append(PIL.ImageColor.getcolor(str(clr), 'RGBA')) + except ValueError: + try: + import matplotlib + + arr.append(PIL.ImageColor.getcolor(matplotlib.colors.to_hex(clr), 'RGBA')) + except (ImportError, ValueError): + raise TileSourceError('Value cannot be used as a color palette.') + return numpy.array(arr) + + +def getPaletteColors(value): + """ + Given a list or a name, return a list of colors in the form of a numpy + array of RGBA. If a list, each entry is a color name resolvable by either + PIL.ImageColor.getcolor, by matplotlib.colors, or a 3 or 4 element list or + tuple of RGB(A) values on a scale of 0-1. If this is NOT a list, then, if + it can be parsed as a color, it is treated as ['#000', ]. If that + cannot be parsed, then it is assumed to be a named palette in palettable + (such as viridis.Viridis_12) or a named palette in matplotlib (including + plugins). + + :param value: Either a list, a single color name, or a palette name. See + above. + :returns: a numpy array of RGBA value on the scale of [0-255]. + """ + palette = None + if isinstance(value, (tuple, list)): + palette = value + if palette is None: + try: + color = PIL.ImageColor.getcolor(str(value), 'RGBA') + palette = ['#000', color] + except ValueError: + pass + if palette is None: + try: + palette = attrgetter(str(value))(palettable).hex_colors + except AttributeError: + pass + if palette is None: + try: + import matplotlib + + palette = ( + ['#0000', matplotlib.colors.to_hex(value)] + if value in matplotlib.colors.get_named_colors_mapping() + else matplotlib.cm.get_cmap(value).colors) + except (ImportError, ValueError): + pass + if palette is None: + raise TileSourceError('Value cannot be used as a color palette.') + return _arrayToPalette(palette) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9e742321f..8b15b0d4d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,6 +25,7 @@ girder-jobs>=3.0.3 # Extras from main setup.py pylibmc>=1.5.1 +matplotlib # External dependencies pip>=9 diff --git a/setup.py b/setup.py index 7a3831008..49e9ab969 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ extraReqs = { 'memcached': ['pylibmc>=1.5.1'] if platform.system() != 'Windows' else [], 'converter': ['large-image-converter'], + 'colormaps': ['matplotlib'], } sources = { 'bioformats': ['large-image-source-bioformats'], @@ -70,6 +71,7 @@ def prerelease_local_scheme(version): ], install_requires=[ 'cachetools>=3.0.0', + 'palettable', # We don't pin the version of Pillow, as anything newer than 3.0 # probably works, though we'd rather have the latest. 8.3.0 won't # save jpeg compressed tiffs properly. diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index db1ce299d..8f5d024a4 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -20,10 +20,8 @@ import struct import tempfile import threading -from operator import attrgetter import numpy -import palettable import PIL.Image from osgeo import gdal, gdal_array, gdalconst, osr # noqa I001 # pyproj stopped supporting older pythons, so its database is aging; as such, @@ -39,6 +37,7 @@ TileInputUnits, TileOutputMimeTypes) from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError from large_image.tilesource import FileTileSource +from large_image.tilesource.utilities import getPaletteColors try: __version__ = get_distribution(__name__).version @@ -224,7 +223,7 @@ def _setDefaultStyle(self): bstyle['palette'] = [( '#%02X%02X%02X' if len(entry) == 3 else '#%02X%02X%02X%02X') % entry for entry in bandInfo['colortable']] - if not isinstance(bstyle['palette'], list): + else: bstyle['palette'] = self.getHexColors(bstyle['palette']) if bstyle.get('nodata') == 'auto': bandInfo = self.getOneBandInformation(bstyle.get('band', 0)) @@ -355,10 +354,8 @@ def getHexColors(palette): :returns: List of colors """ - try: - return attrgetter(palette)(palettable).hex_colors - except AttributeError: - raise TileSourceError('Palette is not a valid palettable path.') + palette = getPaletteColors(palette) + return ['#%02X%02X%02X%02X' % tuple(clr) for clr in palette] def getProj4String(self): """ diff --git a/sources/gdal/setup.py b/sources/gdal/setup.py index dce7eb227..b932a75e8 100644 --- a/sources/gdal/setup.py +++ b/sources/gdal/setup.py @@ -45,7 +45,6 @@ def prerelease_local_scheme(version): install_requires=[ 'large-image>=1.0.0', 'gdal', - 'palettable', 'pyproj>=2.2.0', ], extras_require={ diff --git a/sources/mapnik/large_image_source_mapnik/__init__.py b/sources/mapnik/large_image_source_mapnik/__init__.py index 267df7dd5..df40b0a2a 100644 --- a/sources/mapnik/large_image_source_mapnik/__init__.py +++ b/sources/mapnik/large_image_source_mapnik/__init__.py @@ -82,46 +82,16 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): and 'EPSG:3857' are all equivalent. :param style: if None, use the default style for the file. Otherwise, this is a string with a json-encoded dictionary. The style is - ignored if it does not contain 'band' or 'bands'. The style can - contain the following keys: - - band: either -1 for the default band(s), a 1-based value for - the band to use for styling, or a string that matches the - interpretation of the band ('red', 'green', 'blue', gray', - etc.). - min: the value of the band to map to the first palette value. - Defaults to 0. 'auto' to use 0 if the reported minimum and - maximum of the band are between [0, 255] or use the - reported minimum otherwise. 'min' or 'max' to always uses - the reported minimum or maximum. - max: the value of the band to map to the last palette value. - Defaults to 255. 'auto' to use 0 if the reported minimum - and maximum of the band are between [0, 255] or use the - reported maximum otherwise. 'min' or 'max' to always uses - the reported minimum or maximum. + ignored if it does not contain 'band' or 'bands'. In addition to + the base class parameters, the style can also contain the following + keys: + scheme: one of the mapnik.COLORIZER_xxx values. Case insensitive. Possible values are at least 'discrete', 'linear', and 'exact'. This defaults to 'linear'. - palette: either a list of two or more color strings, a string - with a dotted class name from the python palettable - package, or 'colortable' to use the band's color table - (palette). Color strings must be parsable by mapnik's - Color class. Many css strings work. If scheme is - unspecified and the band is one of 'red', 'green', 'blue', - gray', or 'alpha', this defaults to an appropriate band - pair. Otherwise, this defaults to the band's color table - (palette) if it has one and a black-to-white palette if it - does not. - nodata: the value to use for missing data. 'auto' to use the - band reported value, if any. null or unset to not use a - nodata value. composite: this is a string containing one of the mapnik CompositeOp properties. It defaults to 'lighten'. - Alternately, the style object can contain a single key of 'bands', - which has a value which is a list of style dictionaries as above, - excepting that each must have a band that is not -1. Bands are - composited in the order listed. :param unitsPerPixel: The size of a pixel at the 0 tile size. Ignored if the projection is None. For projections, None uses the default, which is the distance between (-180,0) and (180,0) in EPSG:4326 @@ -265,20 +235,12 @@ def _colorizerFromStyle(self, style): for value, color in enumerate(bandInfo['colortable']): colorizer.add_stop(value, mapnik.Color(*color)) else: - colors = style.get('palette', ['#000000', '#ffffff']) - if not isinstance(colors, list): - colors = self.getHexColors(colors) - else: - colors = [color if isinstance(color, bytes) else - color.encode('utf8') for color in colors] + colors = self.getHexColors(style.get('palette', ['#000000', '#ffffff'])) if len(colors) < 2: raise TileSourceError('A palette must have at least 2 colors.') values = self.interpolateMinMax(minimum, maximum, len(colors)) for value, color in sorted(zip(values, colors)): - try: - colorizer.add_stop(value, mapnik.Color(color)) - except RuntimeError: - raise TileSourceError('Mapnik failed to parse color %r.' % color) + colorizer.add_stop(value, mapnik.Color(color)) return colorizer diff --git a/test/test_source_base.py b/test/test_source_base.py index d1693ccc0..682a2c444 100644 --- a/test/test_source_base.py +++ b/test/test_source_base.py @@ -132,3 +132,31 @@ def testSourcesTilesAndMethods(source, filename): tsf = sourceClass(imagePath, frame=len(tileMetadata['frames']) - 1) tileMetadata = tsf.getMetadata() utilities.checkTilesZXY(tsf, tileMetadata) + + +@pytest.mark.parametrize('palette', [ + ['#000', '#FFF'], + ['#000', '#888', '#FFF'], + '#fff', + 'black', + 'rgba(128, 128, 128, 128)', + 'rgb(128, 128, 128)', + 'xkcd:blue', + 'viridis', + 'matplotlib.Plasma_6', + [(0.5, 0.5, 0.5), (0.1, 0.1, 0.1, 0.1), 'xkcd:blue'], +]) +def testGoodGetPaletteColors(palette): + large_image.tilesource.utilities.getPaletteColors(palette) + + +@pytest.mark.parametrize('palette', [ + 'notacolor', + [0.5, 0.5, 0.5], + ['notacolor', '#fff'], + 'notapalette', + 'matplotlib.Plasma_128', +]) +def testBadGetPaletteColors(palette): + with pytest.raises(large_image.exceptions.TileSourceError): + large_image.tilesource.utilities.getPaletteColors(palette) diff --git a/test/test_source_gdal.py b/test/test_source_gdal.py index ec298d5d5..fe8e0f51c 100644 --- a/test/test_source_gdal.py +++ b/test/test_source_gdal.py @@ -125,7 +125,7 @@ def _assertStyleResponse(imagePath, style, message): _assertStyleResponse(imagePath, { 'band': 1, 'palette': 'nonexistent.palette' - }, 'Palette is not a valid palettable path.') + }, 'Value cannot be used as a color palette.') _assertStyleResponse(imagePath, ['style'], 'Style is not a valid json object.') diff --git a/test/test_source_mapnik.py b/test/test_source_mapnik.py index 326c348d0..f163b3c2d 100644 --- a/test/test_source_mapnik.py +++ b/test/test_source_mapnik.py @@ -148,12 +148,12 @@ def _assertStyleResponse(imagePath, style, message): _assertStyleResponse(imagePath, { 'band': 1, 'palette': 'nonexistent.palette' - }, 'Palette is not a valid palettable path.') + }, 'Value cannot be used as a color palette.') _assertStyleResponse(imagePath, { 'band': 1, 'palette': ['notacolor', '#00ffff'] - }, 'Mapnik failed to parse color') + }, 'Value cannot be used as a color palette.') _assertStyleResponse(imagePath, { 'band': 1,