Allow named color palettes to be used by all sources.

This also increase the ability to handle css and named colors.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+### Features
+- Better palette support in styles ([#724](../../pull/724))
+
 ### Improvements
 - The openjpeg girder source handles uploaded files better ([#721](../../pull/721))
 

README.rst

@@ -69,7 +69,7 @@ Modules
 Large Image consists of several Python modules designed to work together.  These include:
 
 - ``large-image``: The core module.
-  You can specify extras_require of the name of any tile source included with this repository, ``sources`` for all of the tile sources in the repository, a specific source name (e.g., ``tiff``), ``memcached`` for using memcached for tile caching, ``converter`` to include the converter module, or ``all`` for all of the tile sources, memcached, and the converter module.
+  You can specify extras_require of the name of any tile source included with this repository, ``sources`` for all of the tile sources in the repository, a specific source name (e.g., ``tiff``), ``memcached`` for using memcached for tile caching, ``converter`` to include the converter module, ``colormap`` for using matplotlib for named color palettes used in styles, or ``all`` for all of the tile sources, memcached, the converter module, and better color map support.
 
 - ``large-image-converter``: A utility for using pyvips and other libraries to convert images into pyramidal tiff files that can be read efficiently by large_image.
   You can specify extras_require of ``jp2k`` to include modules to allow output to JPEG2000 compression, ``sources`` to include all sources, and ``stats`` to include modules to allow computing compression noise statistics.

docs/tilesource_options.rst

@@ -45,7 +45,10 @@ A band definition is an object which can contain the following keys:
 
 - ``max``: the value to map to the last palette value.  Defaults to 255.  'auto' to use 0 if the reported minimum and maximum of the band are between [0, 255] or use the reported maximum otherwise.  'min' or 'max' to always uses the reported minimum or maximum.
 
-- ``palette``: a list of two or more color strings, where color strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA.  The values between min and max are interpolated using a piecewise linear algorithm to map to the specified palette values.
+- ``palette``: This is a list or two or more colors. The values between min and max are interpolated using a piecewise linear algorithm to map to the specified palette values.  It can be specified in a variety of ways:
+  - a list of two or more color values, where the color values are css-style strings (e.g., of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA, or a css ``rgb``, ``rgba``, ``hsl``, or ``hsv`` string, or a css color name), or, if matplotlib is available, a matplotlib color name, or a list or tuple of RGB(A) values on a scale of [0-1].
+  - a single string that is a color string as above.  This is functionally a two-color palette with the first color as solid black (``#000``), and the second color the specified value
+  - a named color palette from the palettable library (e.g., ``matplotlib.Plasma_6``) or, if available, from the matplotlib library or one of its plugins (e.g., ``viridis``).
 
 - ``nodata``: the value to use for missing data.  null or unset to not use a nodata value.
 

large_image/tilesource/base.py

@@ -22,7 +22,7 @@
 from .utilities import (_encodeImage, _gdalParameters,  # noqa: F401
                         _imageToNumpy, _imageToPIL, _letterboxImage, _vipsCast,
                         _vipsParameters, dictToEtree, etreeToDict,
-                        nearPowerOfTwo)
+                        getPaletteColors, nearPowerOfTwo)
 
 
 class TileSource:
@@ -87,7 +87,12 @@ def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0,
                     maximum otherwise.  'min' or 'max' to always uses the
                     reported minimum or maximum.
                 :palette: a list of two or more color strings, where color
-                    strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA.
+                    strings are of the form #RRGGBB, #RRGGBBAA, #RGB, #RGBA, or
+                    any string parseable by the PIL modules, or, if it is
+                    installed, byt matplotlib.  Alternately, this can be a
+                    single color, which implies ['#000', <color>], or the name
+                    of a palettable paletter or, if available, a matplotlib
+                    palette.
                 :nodata: the value to use for missing data.  null or unset to
                     not use a nodata value.
                 :composite: either 'lighten' or 'multiply'.  Defaults to
@@ -1063,10 +1068,9 @@ def _applyStyle(self, image, style, x, y, z, frame=None):  # noqa
                 composite = entry.get('composite', 'multiply')
             if band is None:
                 band = image[:, :, bandidx]
-            palette = numpy.array([
-                PIL.ImageColor.getcolor(clr, 'RGBA') for clr in entry.get(
-                    'palette', ['#000', '#FFF']
-                    if entry.get('band') != 'alpha' else ['#FFF0', '#FFFF'])])
+            palette = getPaletteColors(entry.get(
+                'palette', ['#000', '#FFF']
+                if entry.get('band') != 'alpha' else ['#FFF0', '#FFFF']))
             palettebase = numpy.linspace(0, 1, len(palette), endpoint=True)
             nodata = entry.get('nodata')
             min = self._getMinMax('min', entry.get('min', 'auto'), image.dtype, bandidx, frame)

large_image/tilesource/utilities.py

@@ -2,15 +2,18 @@
 import math
 import xml.etree.ElementTree
 from collections import defaultdict
+from operator import attrgetter
 
 import numpy
+import palettable
 import PIL
 import PIL.Image
 import PIL.ImageColor
 import PIL.ImageDraw
 
 from ..constants import (TILE_FORMAT_IMAGE, TILE_FORMAT_NUMPY, TILE_FORMAT_PIL,
                          TileOutputMimeTypes, TileOutputPILFormat)
+from ..exceptions import TileSourceError
 
 # Turn off decompression warning check
 PIL.Image.MAX_IMAGE_PIXELS = None
@@ -398,3 +401,71 @@ def nearPowerOfTwo(val1, val2, tolerance=0.02):
     log2ratio = math.log(float(val1) / float(val2)) / math.log(2)
     # Compare the mantissa of the ratio's log2 value.
     return abs(log2ratio - round(log2ratio)) < tolerance
+
+
+def _arrayToPalette(palette):
+    """
+    Given an array of color strings, tuples, or lists, return a numpy array.
+
+    :param palette: an array of color strings, tuples, or lists.
+    :returns: a numpy array of RGBA value on the scale of [0-255].
+    """
+    arr = []
+    for clr in palette:
+        if isinstance(clr, (tuple, list)):
+            arr.append(numpy.array((list(clr) + [1, 1, 1])[:4]) * 255)
+        else:
+            try:
+                arr.append(PIL.ImageColor.getcolor(str(clr), 'RGBA'))
+            except ValueError:
+                try:
+                    import matplotlib
+
+                    arr.append(PIL.ImageColor.getcolor(matplotlib.colors.to_hex(clr), 'RGBA'))
+                except (ImportError, ValueError):
+                    raise TileSourceError('Value cannot be used as a color palette.')
+    return numpy.array(arr)
+
+
+def getPaletteColors(value):
+    """
+    Given a list or a name, return a list of colors in the form of a numpy
+    array of RGBA.  If a list, each entry is a color name resolvable by either
+    PIL.ImageColor.getcolor, by matplotlib.colors, or a 3 or 4 element list or
+    tuple of RGB(A) values on a scale of 0-1.  If this is NOT a list, then, if
+    it can be parsed as a color, it is treated as ['#000', <value>].  If that
+    cannot be parsed, then it is assumed to be a named palette in palettable
+    (such as viridis.Viridis_12) or a named palette in matplotlib (including
+    plugins).
+
+    :param value: Either a list, a single color name, or a palette name.  See
+        above.
+    :returns: a numpy array of RGBA value on the scale of [0-255].
+    """
+    palette = None
+    if isinstance(value, (tuple, list)):
+        palette = value
+    if palette is None:
+        try:
+            color = PIL.ImageColor.getcolor(str(value), 'RGBA')
+            palette = ['#000', color]
+        except ValueError:
+            pass
+    if palette is None:
+        try:
+            palette = attrgetter(str(value))(palettable).hex_colors
+        except AttributeError:
+            pass
+    if palette is None:
+        try:
+            import matplotlib
+
+            palette = (
+                ['#0000', matplotlib.colors.to_hex(value)]
+                if value in matplotlib.colors.get_named_colors_mapping()
+                else matplotlib.cm.get_cmap(value).colors)
+        except (ImportError, ValueError):
+            pass
+    if palette is None:
+        raise TileSourceError('Value cannot be used as a color palette.')
+    return _arrayToPalette(palette)

requirements-dev.txt

@@ -25,6 +25,7 @@ girder-jobs>=3.0.3
 
 # Extras from main setup.py
 pylibmc>=1.5.1
+matplotlib
 
 # External dependencies
 pip>=9

setup.py

@@ -12,6 +12,7 @@
 extraReqs = {
     'memcached': ['pylibmc>=1.5.1'] if platform.system() != 'Windows' else [],
     'converter': ['large-image-converter'],
+    'colormaps': ['matplotlib'],
 }
 sources = {
     'bioformats': ['large-image-source-bioformats'],
@@ -70,6 +71,7 @@ def prerelease_local_scheme(version):
     ],
     install_requires=[
         'cachetools>=3.0.0',
+        'palettable',
         # We don't pin the version of Pillow, as anything newer than 3.0
         # probably works, though we'd rather have the latest.  8.3.0 won't
         # save jpeg compressed tiffs properly.

sources/gdal/large_image_source_gdal/__init__.py

@@ -20,10 +20,8 @@
 import struct
 import tempfile
 import threading
-from operator import attrgetter
 
 import numpy
-import palettable
 import PIL.Image
 from osgeo import gdal, gdal_array, gdalconst, osr  # noqa I001
 # pyproj stopped supporting older pythons, so its database is aging; as such,
@@ -39,6 +37,7 @@
                                    TileInputUnits, TileOutputMimeTypes)
 from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError
 from large_image.tilesource import FileTileSource
+from large_image.tilesource.utilities import getPaletteColors
 
 try:
     __version__ = get_distribution(__name__).version
@@ -224,7 +223,7 @@ def _setDefaultStyle(self):
                         bstyle['palette'] = [(
                             '#%02X%02X%02X' if len(entry) == 3 else
                             '#%02X%02X%02X%02X') % entry for entry in bandInfo['colortable']]
-                    if not isinstance(bstyle['palette'], list):
+                    else:
                         bstyle['palette'] = self.getHexColors(bstyle['palette'])
                 if bstyle.get('nodata') == 'auto':
                     bandInfo = self.getOneBandInformation(bstyle.get('band', 0))
@@ -355,10 +354,8 @@ def getHexColors(palette):
 
         :returns: List of colors
         """
-        try:
-            return attrgetter(palette)(palettable).hex_colors
-        except AttributeError:
-            raise TileSourceError('Palette is not a valid palettable path.')
+        palette = getPaletteColors(palette)
+        return ['#%02X%02X%02X%02X' % tuple(clr) for clr in palette]
 
     def getProj4String(self):
         """

sources/gdal/setup.py

@@ -45,7 +45,6 @@ def prerelease_local_scheme(version):
     install_requires=[
         'large-image>=1.0.0',
         'gdal',
-        'palettable',
         'pyproj>=2.2.0',
     ],
     extras_require={

sources/mapnik/large_image_source_mapnik/__init__.py

@@ -82,46 +82,16 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
             and 'EPSG:3857' are all equivalent.
         :param style: if None, use the default style for the file.  Otherwise,
             this is a string with a json-encoded dictionary.  The style is
-            ignored if it does not contain 'band' or 'bands'.  The style can
-            contain the following keys:
-
-                band: either -1 for the default band(s), a 1-based value for
-                    the band to use for styling, or a string that matches the
-                    interpretation of the band ('red', 'green', 'blue', gray',
-                    etc.).
-                min: the value of the band to map to the first palette value.
-                    Defaults to 0.  'auto' to use 0 if the reported minimum and
-                    maximum of the band are between [0, 255] or use the
-                    reported minimum otherwise.  'min' or 'max' to always uses
-                    the reported minimum or maximum.
-                max: the value of the band to map to the last palette value.
-                    Defaults to 255.  'auto' to use 0 if the reported minimum
-                    and maximum of the band are between [0, 255] or use the
-                    reported maximum otherwise.  'min' or 'max' to always uses
-                    the reported minimum or maximum.
+            ignored if it does not contain 'band' or 'bands'.  In addition to
+            the base class parameters, the style can also contain the following
+            keys:
+
                 scheme: one of the mapnik.COLORIZER_xxx values.  Case
                     insensitive.  Possible values are at least 'discrete',
                     'linear', and 'exact'.  This defaults to 'linear'.
-                palette: either a list of two or more color strings, a string
-                    with a dotted class name from the python palettable
-                    package, or 'colortable' to use the band's color table
-                    (palette).  Color strings must be parsable by mapnik's
-                    Color class.  Many css strings work.  If scheme is
-                    unspecified and the band is one of 'red', 'green', 'blue',
-                    gray', or 'alpha', this defaults to an appropriate band
-                    pair.  Otherwise, this defaults to the band's color table
-                    (palette) if it has one and a black-to-white palette if it
-                    does not.
-                nodata: the value to use for missing data.  'auto' to use the
-                    band reported value, if any.  null or unset to not use a
-                    nodata value.
                 composite: this is a string containing one of the mapnik
                     CompositeOp properties.  It defaults to 'lighten'.
 
-            Alternately, the style object can contain a single key of 'bands',
-            which has a value which is a list of style dictionaries as above,
-            excepting that each must have a band that is not -1.  Bands are
-            composited in the order listed.
         :param unitsPerPixel: The size of a pixel at the 0 tile size.  Ignored
             if the projection is None.  For projections, None uses the default,
             which is the distance between (-180,0) and (180,0) in EPSG:4326
@@ -265,20 +235,12 @@ def _colorizerFromStyle(self, style):
             for value, color in enumerate(bandInfo['colortable']):
                 colorizer.add_stop(value, mapnik.Color(*color))
         else:
-            colors = style.get('palette', ['#000000', '#ffffff'])
-            if not isinstance(colors, list):
-                colors = self.getHexColors(colors)
-            else:
-                colors = [color if isinstance(color, bytes) else
-                          color.encode('utf8') for color in colors]
+            colors = self.getHexColors(style.get('palette', ['#000000', '#ffffff']))
             if len(colors) < 2:
                 raise TileSourceError('A palette must have at least 2 colors.')
             values = self.interpolateMinMax(minimum, maximum, len(colors))
             for value, color in sorted(zip(values, colors)):
-                try:
-                    colorizer.add_stop(value, mapnik.Color(color))
-                except RuntimeError:
-                    raise TileSourceError('Mapnik failed to parse color %r.' % color)
+                colorizer.add_stop(value, mapnik.Color(color))
 
         return colorizer
 

test/test_source_base.py

@@ -132,3 +132,31 @@ def testSourcesTilesAndMethods(source, filename):
         tsf = sourceClass(imagePath, frame=len(tileMetadata['frames']) - 1)
         tileMetadata = tsf.getMetadata()
         utilities.checkTilesZXY(tsf, tileMetadata)
+
+
+@pytest.mark.parametrize('palette', [
+    ['#000', '#FFF'],
+    ['#000', '#888', '#FFF'],
+    '#fff',
+    'black',
+    'rgba(128, 128, 128, 128)',
+    'rgb(128, 128, 128)',
+    'xkcd:blue',
+    'viridis',
+    'matplotlib.Plasma_6',
+    [(0.5, 0.5, 0.5), (0.1, 0.1, 0.1, 0.1), 'xkcd:blue'],
+])
+def testGoodGetPaletteColors(palette):
+    large_image.tilesource.utilities.getPaletteColors(palette)
+
+
+@pytest.mark.parametrize('palette', [
+    'notacolor',
+    [0.5, 0.5, 0.5],
+    ['notacolor', '#fff'],
+    'notapalette',
+    'matplotlib.Plasma_128',
+])
+def testBadGetPaletteColors(palette):
+    with pytest.raises(large_image.exceptions.TileSourceError):
+        large_image.tilesource.utilities.getPaletteColors(palette)

test/test_source_gdal.py

@@ -125,7 +125,7 @@ def _assertStyleResponse(imagePath, style, message):
     _assertStyleResponse(imagePath, {
         'band': 1,
         'palette': 'nonexistent.palette'
-    }, 'Palette is not a valid palettable path.')
+    }, 'Value cannot be used as a color palette.')
 
     _assertStyleResponse(imagePath, ['style'],
                          'Style is not a valid json object.')

test/test_source_mapnik.py

@@ -148,12 +148,12 @@ def _assertStyleResponse(imagePath, style, message):
     _assertStyleResponse(imagePath, {
         'band': 1,
         'palette': 'nonexistent.palette'
-    }, 'Palette is not a valid palettable path.')
+    }, 'Value cannot be used as a color palette.')
 
     _assertStyleResponse(imagePath, {
         'band': 1,
         'palette': ['notacolor', '#00ffff']
-    }, 'Mapnik failed to parse color')
+    }, 'Value cannot be used as a color palette.')
 
     _assertStyleResponse(imagePath, {
         'band': 1,