From a34cbed63aec84e0c1aa2f77e72a9a1c413352df Mon Sep 17 00:00:00 2001 From: David Manthey Date: Tue, 10 Aug 2021 16:02:10 -0400 Subject: [PATCH] Use tile-frames for better frame browsing. This requires a version of GeoJS newer than v1.2.0 to have any effect. --- CHANGELOG.md | 3 +- .../views/imageViewerWidget/geojs.js | 42 +++- .../views/imageViewerWidget/setFrameQuad.js | 203 ++++++++++++++++++ 3 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 girder/girder_large_image/web_client/views/imageViewerWidget/setFrameQuad.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b469bc1b..a0ea0154e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,12 @@ ### Features - Provide band information on all tile sources (#622, #623) - Add a tileFrames method to tile sources and appropriate endpoints to composite regions from multiple frames to a single output image (#629) -- The test tile source now support frames (#631) +- The test tile source now support frames (#631, #632, #634, #634) ### Improvements - Better handle TIFFs with missing levels and tiles (#624, #627) - Better report inefficient TIFFs (#626) +- Smoother cross-frame navigation (#635) ## Version 1.6.2 diff --git a/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js b/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js index 703620044..931a28bf2 100644 --- a/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js +++ b/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js @@ -7,6 +7,7 @@ import d3 from 'd3'; import { restRequest } from '@girder/core/rest'; import ImageViewerWidget from './base'; +import setFrameQuad from './setFrameQuad.js'; window.hammerjs = Hammer; window.d3 = d3; @@ -77,6 +78,16 @@ var GeojsImageViewerWidget = ImageViewerWidget.extend({ this.viewer = geo.map(params.map); params.layer.autoshareRenderer = false; this._layer = this.viewer.createLayer('osm', params.layer); + if (this.metadata.frames && this.metadata.frames.length > 1) { + setFrameQuad(this.metadata, this._layer, { + // allow more and larger textures is slower, balancing + // performance and appearance + // maxTextures: 16, + // maxTotalTexturePixels: 256 * 1024 * 1024, + baseUrl: this._getTileUrl('{z}', '{x}', '{y}').split('/tiles/')[0] + '/tiles' + }); + this._layer.setFrameQuad(0); + } } else { params = { keepLower: false, @@ -134,19 +145,40 @@ var GeojsImageViewerWidget = ImageViewerWidget.extend({ if (frame === 0) { return; } - // use two layers to get smooth transitions - this._layer2 = this.viewer.createLayer('osm', this._layer._options); - this._layer2.moveDown(); - this._baseurl = this._layer.url(); this._frame = 0; + this._baseurl = this._layer.url(); + let quadLoaded = ((this._layer.setFrameQuad || {}).status || {}).loaded; + if (!quadLoaded) { + // use two layers to get smooth transitions until we load + // background quads. + this._layer2 = this.viewer.createLayer('osm', this._layer._options); + this._layer2.moveDown(); + setFrameQuad((this._layer.setFrameQuad.status || {}).tileinfo, this._layer2, (this._layer.setFrameQuad.status || {}).options); + this._layer2.setFrameQuad(0); + } } this._nextframe = frame; if (frame !== this._frame && !this._updating) { - this._updating = true; this._frame = frame; this.trigger('g:imageFrameChanging', this, frame); + let quadLoaded = ((this._layer.setFrameQuad || {}).status || {}).loaded; + if (quadLoaded) { + if (this._layer2) { + this.viewer.deleteLayer(this._layer2); + delete this._layer2; + } + this._layer.url(this.getFrameAndUrl().url); + this._layer.setFrameQuad(frame); + this._layer.frame = frame; + this.trigger('g:imageFrameChanged', this, frame); + return; + } + + this._updating = true; this.viewer.onIdle(() => { this._layer2.url(this.getFrameAndUrl().url); + this._layer2.setFrameQuad(frame); + this._layer2.frame = frame; this.viewer.onIdle(() => { this._layer.moveDown(); var ltemp = this._layer; diff --git a/girder/girder_large_image/web_client/views/imageViewerWidget/setFrameQuad.js b/girder/girder_large_image/web_client/views/imageViewerWidget/setFrameQuad.js new file mode 100644 index 000000000..0b3686529 --- /dev/null +++ b/girder/girder_large_image/web_client/views/imageViewerWidget/setFrameQuad.js @@ -0,0 +1,203 @@ +/** + * Given metadata on a tile source, a GeoJS tileLayer, and a set of options, + * add a function to the layer `setFrameQuad()` that will, if possible, + * set the baseQuad to a cropped section of an image that contains excerpts of + * all frames. + * + * @param {object} tileinfo The metadata of the source image. This expects + * ``sizeX`` and ``sizeY`` to be the width and height of the image and + * ``frames`` to contain a list of the frames of the image or be undefined if + * there is only one frame. + * @param {geo.tileLayer} layer The GeoJS layer to add the function to. This + * is also used to get a maximal texture size if the layer is a webGL + * layer. + * @param {object} options Additional options for the function. This must + * minimally include ``baseUrl``. + * @param {string} options.baseUrl The reference to the tile endpoint, e.g., + * /api/v1/item//tiles. + * @param {string} [options.format='encoding=JPEG&jpegQuality=85&jpegSubsampling=1'] + * The compression and format for the texture. + * @param {string} [options.query] Additional query options to add to the + * tile_frames endpoint, e.g. 'style={"min":"min","max":"max"}'. Do not + * include framesAcross or frameList. You must specify 'cache=true' if + * that is desired. + * @param {number} [options.frameBase=0] Starting frame number used. + * @param {number} [options.frameStride=1] Only use ever ``frameStride`` frame + * of the image. + * @param {number} [options.maxTextureSize] Limit the maximum texture size to a + * square of this size. The size is also limited by the WebGL maximum + * size for webgl-based layers or 16384 for canvas-based layers. + * @param {number} [options.maxTextures=1] If more than one, allow multiple + * textures to increase the size of the individual frames. The number of + * textures will be capped by ``maxTotalTexturePixels`` as well as this + * number. + * @param {number} [options.maxTotalTexturePixels=1073741824] Limit the + * maximum texture size and maximum number of textures so that the combined + * set does not exceed this number of pixels. + * @param {number} [options.alignment=16] Individual frames are buffer to an + * alignment of this maxy pixels. If JPEG compression is used, this should + * be 8 for monochrome images or jpegs without subsampling, or 16 for jpegs + * with moderate subsampling to avoid compression artifacts from leaking + * between frames. + * @param {number} [options.adjustMinLevel=true] If truthy, adjust the tile + * layer's minLevel after the quads are loaded. + * @param {number} [options.maxFrameSize] If set, limit the maximum width and + * height of an individual frame to this value. + * @param {string} [options.crossOrigin] If specified, use this as the + * crossOrigin policy for images. + */ +function setFrameQuad(tileinfo, layer, options) { + layer.setFrameQuad = function () { }; + if (!tileinfo || !tileinfo.sizeX || !tileinfo.sizeY || !options || !options.baseUrl) { + return; + } + let maxTextureSize; + try { + maxTextureSize = layer.renderer()._maxTextureSize || layer.renderer().constructor._maxTextureSize; + } catch (err) { } + let w = tileinfo.sizeX, + h = tileinfo.sizeY, + numFrames = (tileinfo.frames || []).length || 1, + texSize = maxTextureSize || 16384, + textures = options.maxTextures || 1, + maxTotalPixels = options.maxTotalTexturePixels || 1073741824, + alignment = options.alignment || 16; + let frames = []; + for (let fidx = options.frameBase || 0; fidx < numFrames; fidx += options.frameStride || 1) { + frames.push(fidx); + } + numFrames = frames.length; + if (numFrames === 0 || !Object.getOwnPropertyDescriptor(layer, 'baseQuad')) { + return; + } + texSize = Math.min(texSize, options.maxTextureSize || texSize); + while (texSize ** 2 > maxTotalPixels) { + texSize /= 2; + } + while (textures && texSize ** 2 * textures > maxTotalPixels) { + textures -= 1; + } + let fw, fh, fhorz, fvert; + /* Iterate in case we can reduce the number of textures or the texture + * size */ + while (true) { + let f = Math.ceil(numFrames / textures); // frames per texture + let texScale2 = texSize ** 2 / f / w / h; + // frames across the texture + fhorz = Math.ceil(texSize / (Math.floor(w * texScale2 ** 0.5 / alignment) * alignment)); + fvert = Math.ceil(texSize / (Math.floor(h * texScale2 ** 0.5 / alignment) * alignment)); + // tile sizes + fw = Math.floor(texSize / fhorz / alignment) * alignment; + fh = Math.floor(texSize / fvert / alignment) * alignment; + if (options.maxFrameSize) { + let maxFrameSize = Math.floor(options.maxFrameSize / alignment) * alignment; + fw = Math.min(fw, maxFrameSize); + fh = Math.min(fh, maxFrameSize); + } + if (fw > w) { + fw = Math.ceil(w / alignment) * alignment; + } + if (fh > h) { + fh = Math.ceil(h / alignment) * alignment; + } + // shrink one dimension so account for aspect ratio + fw = Math.min(Math.ceil(fw * w / h / alignment) * alignment, fw); + fh = Math.min(Math.ceil(fw * h / w / alignment) * alignment, fh); + // recompute frames across the texture + fhorz = Math.floor(texSize / fw); + fvert = Math.min(Math.floor(texSize / fh), Math.ceil(numFrames / fhorz)); + // check if we are not using all textires or are using less than a + // quarter of one texture. If not, stop, if so, reduce and recalculate + if (textures > 1 && numFrames <= fhorz * fvert * (textures - 1)) { + textures -= 1; + continue; + } + if (fhorz >= 2 && Math.ceil(f / Math.floor(fhorz / 2)) * fh <= texSize / 2) { + texSize /= 2; + continue; + } + break; + } + // used area of each tile + let usedw = Math.floor(w / Math.max(w / fw, h / fh)), + usedh = Math.floor(h / Math.max(w / fw, h / fh)); + // get the set of texture images + let status = { + tileinfo: tileinfo, + options: options, + images: [], + src: [], + quads: [], + frames: frames, + framesToIdx: {} + }; + if (tileinfo.tileWidth && tileinfo.tileHeight) { + // report that tiles below this level are not needed + status.minLevel = Math.ceil(Math.log(Math.min(usedw / tileinfo.tileWidth, usedh / tileinfo.tileHeight)) / Math.log(2)); + } + frames.forEach((frame, idx) => { status.framesToIdx[frame] = idx; }); + for (let idx = 0; idx < textures; idx += 1) { + let img = new Image(); + if (options.baseUrl.indexOf(':') >= 0 && options.baseUrl.indexOf('/') === options.baseUrl.indexOf(':') + 1) { + img.crossOrigin = options.crossOrigin || 'anonymous'; + } + let frameList = frames.slice(idx * fhorz * fvert, (idx + 1) * fhorz * fvert); + let src = `${options.baseUrl}/tile_frames?framesAcross=${fhorz}&width=${fw}&height=${fh}&fill=corner:black&exact=false`; + if (frameList.length !== (tileinfo.frames || []).length) { + src += `&frameList=${frameList.join(',')}`; + } + src += '&' + (options.format || 'encoding=JPEG&jpegQuality=85&jpegSubsampling=1').replace(/(^&|^\?|\?$|&$)/g, ''); + if (options.query) { + src += '&' + options.query.replace(/(^&|^\?|\?$|&$)/g, ''); + } + status.src.push(src); + if (idx === textures - 1) { + img.onload = function () { + status.loaded = true; + if (layer._options && layer._options.minLevel !== undefined && (options.adjustMinLevel === undefined || options.adjustMinLevel) && status.minLevel && status.minLevel > layer._options.minLevel) { + layer._options.minLevel = Math.min(layer._options.maxLevel, status.minLevel); + } + }; + } else { + ((idx) => { + img.onload = function () { + status.images[idx + 1].src = status.src[idx + 1]; + }; + })(idx); + } + status.images.push(img); + // the last image can have fewer frames than the other images + let f = frameList.length; + let ivert = Math.ceil(f / fhorz), + ihorz = Math.min(f, fhorz); + frameList.forEach((frame, fidx) => { + let quad = { + // z = -1 to place under other tile layers + ul: {x: 0, y: 0, z: -1}, + // y coordinate is inverted + lr: {x: w, y: -h, z: -1}, + crop: { + x: w, + y: h, + left: (fidx % ihorz) * fw, + top: (ivert - Math.floor(fidx / ihorz)) * fh - usedh, + right: (fidx % ihorz) * fw + usedw, + bottom: (ivert - Math.floor(fidx / ihorz)) * fh + }, + image: img + }; + status.quads.push(quad); + }); + } + status.images[0].src = status.src[0]; + + layer.setFrameQuad = function (frame) { + if (status.framesToIdx[frame] !== undefined) { + layer.baseQuad = Object.assign({}, status.quads[status.framesToIdx[frame]]); + status.frame = frame; + } + }; + layer.setFrameQuad.status = status; +} + +export default setFrameQuad;