diff --git a/CHANGELOG.md b/CHANGELOG.md index 083aef860..1d05cbc23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Improvements - Fallback when server notification streams are turned off ([#967](../../pull/967)) -- Show and edit yaml and json files using codemirror ([#969](../../pull/969)) +- Show and edit yaml and json files using codemirror ([#969](../../pull/969), [#971](../../pull/971)) - Show configured item lists even if there are no large image s ([#972](../../pull/972)) ## 1.17.0 diff --git a/docs/config_options.rst b/docs/config_options.rst index 01c6ef9bd..88cc544ad 100644 --- a/docs/config_options.rst +++ b/docs/config_options.rst @@ -60,7 +60,7 @@ For the Girder plugin, these can also be set in the ``girder.cfg`` file in a ``l max_small_image_size = 4096 # The bioformats tilesource won't read files that end in a comma-separated # list of extensions - source_bioformats_ignored_extensions = '.jpg,.jpeg,.jpe,.png,.tif,.tiff,.ndpi' + source_bioformats_ignored_names = r'(^[!.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi))$' Logging from Python ------------------- diff --git a/docs/girder_config_options.rst b/docs/girder_config_options.rst index 44b48906f..39c4c78fa 100644 --- a/docs/girder_config_options.rst +++ b/docs/girder_config_options.rst @@ -128,3 +128,9 @@ This is used to specify how items appear in item lists. There are two settings, If there are no large images in a folder, none of the image columns will appear. +Editing Configuration Files +--------------------------- + +Some file types can be edited on their item page. This is detected based on the mime type associated with the file: ``application/json`` for json files and ``text/yaml`` or ``text/x-yaml`` for yaml files. If a user has enough permissions, these can be modified and saved. Note that this does not alter imported files; rather, on save it will create a new file in the assetstore and use that; this works fine for using the configuration files. + +For admins, there is also support for the ``application/x-girder-ini`` mime type for Girder configuration files. This has a special option to replace the existing Girder configuration and restart the server and should be used with due caution. diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 39c69fa18..12b225d1a 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -16,18 +16,23 @@ import concurrent.futures import datetime +import io import json +import os +import pprint import re +import shutil import sys import time +import cherrypy import psutil from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job from girder import logger from girder.api import access -from girder.api.describe import Description, describeRoute +from girder.api.describe import Description, autoDescribeRoute, describeRoute from girder.api.rest import Resource from girder.exceptions import RestException from girder.models.file import File @@ -219,6 +224,9 @@ def __init__(self): self.resourceName = 'large_image' self.route('GET', ('cache', ), self.cacheInfo) self.route('PUT', ('cache', 'clear'), self.cacheClear) + self.route('POST', ('config', 'format'), self.configFormat) + self.route('POST', ('config', 'validate'), self.configValidate) + self.route('POST', ('config', 'replace'), self.configReplace) self.route('GET', ('settings',), self.getPublicSettings) self.route('GET', ('sources',), self.listSources) self.route('GET', ('thumbnails',), self.countThumbnails) @@ -489,3 +497,170 @@ def deleteHistograms(self, params): File().remove(file) removed += 1 return removed + + def _configValidateException(self, exc, lineno=None): + """ + Report a config validation exception with a line number. + """ + try: + msg = str(exc) + matches = re.search(r'line: (\d+)', msg) + if not matches: + matches = re.search(r'\[line[ ]*(\d+)\]', msg) + if matches: + line = int(matches.groups()[0]) + msg = msg.split('\n')[0].strip() or 'General error' + msg = msg.rsplit(": ''", 1)[0].rsplit("''", 1)[-1].strip() + return [{'line': line - 1, 'message': msg}] + except Exception: + pass + if lineno is not None: + return [{'line': lineno, 'message': str(exc)}] + return [{'line': 0, 'message': 'General error'}] + + def _configValidate(self, config): + """ + Check if a Girder config file will validate. If not, return an + array of lines where it fails to validate. + + :param config: The string representation of the config file to + validate. + :returns: a list of errors, though usually only the first one. + """ + parser = cherrypy.lib.reprconf.Parser() + try: + parser.read_string(config) + except Exception as exc: + return self._configValidateException(exc) + err = None + try: + parser.as_dict() + return [] + except Exception as exc: + err = exc + try: + parser.as_dict(raw=True) + return self._configValidateException(exc) + except Exception: + pass + lines = io.StringIO(config).readlines() + for pos in range(len(lines), 0, -1): + try: + parser = cherrypy.lib.reprconf.Parser() + parser.read_string(''.join(lines[:pos])) + parser.as_dict() + return self._configValidateException('Config values must be valid Python.', pos) + except Exception: + pass + return self._configValidateException(err) + + @autoDescribeRoute( + Description('Validate a Girder config file') + .notes('Returns a list of errors found.') + .param('config', 'The contents of config file to validate.', + paramType='body') + ) + @access.admin + def configValidate(self, config): + config = config.read().decode('utf8') + return self._configValidate(config) + + @autoDescribeRoute( + Description('Reformat a Girder config file') + .param('config', 'The contents of config file to format.', + paramType='body') + ) + @access.admin + def configFormat(self, config): # noqa + config = config.read().decode('utf8') + if len(self._configValidate(config)): + return config + # reformat here + # collect comments + comments = ['[__comment__]\n'] + for line in io.StringIO(config): + if line.strip()[:1] in {'#', ';'}: + line = '__comment__%d = %r\n' % (len(comments), line) + # If a comment is in the middle of a value, hoist it up + for pos in range(len(comments), 0, -1): + try: + parser = cherrypy.lib.reprconf.Parser() + parser.read_string(''.join(comments[:pos])) + parser.as_dict(raw=True) + comments[pos:pos] = [line] + break + except Exception: + pass + else: + comments.append(line) + parser = cherrypy.lib.reprconf.Parser() + parser.read_string(''.join(comments)) + results = parser.as_dict(raw=True) + # Build results + out = [] + for section in results: + if section != '__comment__': + out.append('[%s]\n' % section) + for key, val in results[section].items(): + if not key.startswith('__comment__'): + valstr = repr(val) + if len(valstr) + len(key) + 3 >= 79: + try: + valstr = pprint.pformat( + val, width=79, indent=2, compact=True, sort_dicts=False) + except Exception: + # sort_dicts isn't an option before Python 3.8 + valstr = pprint.pformat( + val, width=79, indent=2, compact=True) + out.append('%s = %s\n' % (key, valstr)) + else: + out.append(val) + if section != '__comment__': + out.append('\n') + return ''.join(out) + + @autoDescribeRoute( + Description('Replace the existing Girder config file') + .param('restart', 'Whether to restart the server after updating the ' + 'config file', required=False, dataType='boolean', default=True) + .param('config', 'The new contents of config file.', + paramType='body') + ) + @access.admin + def configReplace(self, config, restart): + config = config.read().decode('utf8') + if len(self._configValidate(config)): + raise RestException('Invalid config file') + path = os.path.join(os.path.expanduser('~'), '.girder', 'girder.cfg') + if 'GIRDER_CONFIG' in os.environ: + path = os.environ['GIRDER_CONFIG'] + if os.path.exists(path): + contents = open(path).read() + if contents == config: + return {'status': 'no change'} + newpath = path + '.' + time.strftime( + '%Y%m%d-%H%M%S', time.localtime(os.stat(path).st_mtime)) + logger.info('Copying existing config file from %s to %s' % (path, newpath)) + shutil.copy2(path, newpath) + logger.warning('Replacing config file %s' % (path)) + open(path, 'w').write(config) + + class Restart(cherrypy.process.plugins.Monitor): + def __init__(self, bus, frequency=1): + cherrypy.process.plugins.Monitor.__init__( + self, bus, self.run, frequency) + + def start(self): + cherrypy.process.plugins.Monitor.start(self) + + def run(self): + self.bus.log('Restarting.') + self.thread.cancel() + self.bus.restart() + + if restart: + restart = Restart(cherrypy.engine) + restart.subscribe() + restart.start() + return {'restarted': datetime.datetime.utcnow()} + return {'status': 'updated', 'time': datetime.datetime.utcnow()} diff --git a/girder/girder_large_image/web_client/templates/itemViewCodemirror.pug b/girder/girder_large_image/web_client/templates/itemViewCodemirror.pug index e3aacc4cf..d6b1db554 100644 --- a/girder/girder_large_image/web_client/templates/itemViewCodemirror.pug +++ b/girder/girder_large_image/web_client/templates/itemViewCodemirror.pug @@ -1,6 +1,10 @@ .li-item-view-codemirror-header.g-item-info-header if accessLevel >= AccessType.WRITE .li-item-header-btn-group.pull-right + for but in buttonList + button.g-view-codemirror-general-button.btn.btn-sm.btn-default(title=but.title, button-key=(but.key || but.text)) + = but.text + = ' ' button.g-view-codemirror-revert-button.btn.btn-sm.btn-default Revert = ' ' button.g-view-codemirror-format-button.btn.btn-sm.btn-default(title='This may remove comments') Format diff --git a/girder/girder_large_image/web_client/views/itemViewCodemirror.js b/girder/girder_large_image/web_client/views/itemViewCodemirror.js index 14e771ffb..884b3ff52 100644 --- a/girder/girder_large_image/web_client/views/itemViewCodemirror.js +++ b/girder/girder_large_image/web_client/views/itemViewCodemirror.js @@ -1,5 +1,7 @@ import $ from 'jquery'; +import { getCurrentUser } from '@girder/core/auth'; import { AccessType } from '@girder/core/constants'; +import { confirm } from '@girder/core/dialog'; import events from '@girder/core/events'; import { restRequest } from '@girder/core/rest'; import { wrap } from '@girder/core/utilities/PluginUtils'; @@ -21,6 +23,7 @@ import 'codemirror/addon/lint/lint'; import 'codemirror/addon/lint/json-lint'; import 'codemirror/addon/lint/yaml-lint'; import 'codemirror/mode/javascript/javascript'; +import 'codemirror/mode/properties/properties'; import 'codemirror/mode/yaml/yaml'; import itemViewCodemirror from '../templates/itemViewCodemirror.pug'; @@ -48,12 +51,127 @@ const Formats = { * do so, or we could use sort of regex parsing, but both of those are * more work. */ format: (val) => jsyaml.dump(val, {lineWidth: -1, noRefs: true}) + }, + 'application/x-girder-ini': { + name: 'Configuration', + mode: 'properties', + buttons: [{ + key: 'replace', + text: 'Replace', + title: 'Replace existing configuration and restart', + action: (val, parent) => confirm({ + text: 'Are you sure you want to save this file, replace the existing configuration, and restart?', + yesText: 'Save, Replace, and Restart', + confirmCallback: () => { + parent.save(); + restRequest({ + url: 'system/version' + }).done((resp) => { + const lastStartDate = resp.serverStartDate; + restRequest({ + method: 'POST', + url: 'large_image/config/replace', + data: val, + contentType: 'application/x-girder-ini' + }).done((result) => { + if (result.restarted) { + events.trigger('g:alert', { + text: 'Restarting.', + type: 'warning', + timeout: 60000 + }); + + parent.wait = () => { + restRequest({ + url: 'system/version', + error: null + }).done((resp) => { + if (resp.serverStartDate !== lastStartDate) { + window.location.reload(); + } else { + window.setTimeout(parent.wait, 1000); + } + }).fail(() => { + window.setTimeout(parent.wait, 1000); + }); + }; + parent.wait(); + } else { + events.trigger('g:alert', { + text: 'Configuration unchanged.', + type: 'info' + }); + } + }); + }); + } + }) + }], + accessLevel: AccessType.ADMIN, + adminOnly: true, + validator: (val) => { + let promise = $.Deferred(); + restRequest({ + method: 'POST', + url: 'large_image/config/validate', + data: val, + contentType: 'application/x-girder-ini' + }).done((errors) => { + if (errors.length) { + promise.reject(errors[0].message); + return null; + } + promise.resolve(val); + return null; + }).fail((err) => { + promise.reject(err); + return null; + }); + return promise; + }, + format: (val) => { + let promise = $.Deferred(); + restRequest({ + method: 'POST', + url: 'large_image/config/format', + data: val, + contentType: 'application/x-girder-ini' + }).done((result) => { + promise.resolve(result); + return null; + }).fail((err) => { + promise.reject(err); + return null; + }); + return promise; + } } }; Formats['text/x-yaml'] = Formats['text/yaml']; +function lintGirderIni(text, callback) { + return restRequest({ + method: 'POST', + url: 'large_image/config/validate', + data: text, + contentType: 'application/x-girder-ini', + error: null + }).done((errorList) => { + callback(errorList.map((entry) => ({ + from: CodeMirror.Pos(entry.line), + to: CodeMirror.Pos(entry.line), + message: entry.message + }))); + return null; + }); +} +lintGirderIni.async = true; + +CodeMirror.registerHelper('lint', 'properties', lintGirderIni); + var CodemirrorEditWidget = View.extend({ events: { + 'click .g-view-codemirror-general-button': 'generalAction', 'click .g-view-codemirror-revert-button': 'revert', 'click .g-view-codemirror-format-button': 'format', 'click .g-view-codemirror-save-button': 'save' @@ -79,6 +197,7 @@ var CodemirrorEditWidget = View.extend({ this.$el.html(itemViewCodemirror({ formatName: Formats[this.mimeType].name, accessLevel: this.accessLevel, + buttonList: Formats[this.mimeType].buttons || [], AccessType: AccessType })); this.code = CodeMirror(this.$el.find('.editor')[0], { @@ -93,49 +212,97 @@ var CodemirrorEditWidget = View.extend({ }, format: function () { - let content = this.code.getValue(); - let validated; - try { - validated = Formats[this.mimeType].validator(content); - } catch (e) { - events.trigger('g:alert', { - text: 'Contents do not validate. ' + e, - type: 'warning' - }); + if (this._informat) { return; } + this._informat = true; + let content = this.code.getValue(); try { - content = Formats[this.mimeType].format(validated); + $.when(Formats[this.mimeType].validator(content)).done((validated) => { + try { + $.when(Formats[this.mimeType].format(validated)).done((content) => { + this.code.setValue(content); + this._informat = false; + return null; + }).fail(() => { + this._informat = false; + return null; + }); + } catch (e) { + events.trigger('g:alert', { + text: 'Contents do not format. ' + e, + type: 'warning' + }); + this._informat = false; + } + }).fail(() => { + this._informat = undefined; + }); } catch (e) { events.trigger('g:alert', { - text: 'Contents do not format. ' + e, + text: 'Contents do not validate. ' + e, type: 'warning' }); - return; + this._informat = false; } - this.code.setValue(content); }, revert: function () { this.code.setValue(this._contents); }, + generalAction: function (evt) { + const key = $(evt.target).attr('button-key'); + const button = Formats[this.mimeType].buttons.filter((but) => (but.key || but.name) === key); + if (button.length !== 1) { + return; + } + const content = this.code.getValue(); + button[0].action(content, this); + }, + save: function () { const content = this.code.getValue(); + if (content === this._lastSave) { + return; + } + if (this._insave) { + this._insave = 'again'; + return; + } + this._insave = true; try { - Formats[this.mimeType].validator(content); + $.when(Formats[this.mimeType].validator(content)).done(() => { + this.file.updateContents(content); + // functionally, this just marks the parent item's updated time + this.parentView.model._sendMetadata({}); + this._lastSave = content; + const lastInsave = this._insave; + this._insave = undefined; + if (lastInsave === 'again') { + this.save(); + } + }).fail((err) => { + events.trigger('g:alert', { + text: 'Contents do not validate. ' + err, + type: 'warning' + }); + const lastInsave = this._insave; + this._insave = undefined; + if (lastInsave === 'again') { + this.save(); + } + }); } catch (e) { events.trigger('g:alert', { text: 'Contents do not validate. ' + e, type: 'warning' }); - return; - } - if (content !== this._lastSave) { - this.file.updateContents(content); - // functional, this just marks the parent item's updated time - this.parentView.model._sendMetadata({}); - this._lastSave = content; + const lastInsave = this._insave; + this._insave = undefined; + if (lastInsave === 'again') { + this.save(); + } } } }); @@ -153,6 +320,12 @@ wrap(ItemView, 'render', function (render) { if (!Formats[mimeType] || firstFile.get('size') > 100000) { return; } + if (Formats[mimeType].accessLevel !== undefined && this.accessLevel < Formats[mimeType].accessLevel) { + return; + } + if (Formats[mimeType].adminOnly && !(getCurrentUser() && getCurrentUser().get('admin'))) { + return; + } this.codemirrorEditWidget = new CodemirrorEditWidget({ el: $('
', {class: 'g-codemirror-edit-container'}) .insertAfter(this.$('.g-item-files')), diff --git a/girder/test_girder/test_large_image.py b/girder/test_girder/test_large_image.py index b668dd510..f009c9714 100644 --- a/girder/test_girder/test_large_image.py +++ b/girder/test_girder/test_large_image.py @@ -1,5 +1,6 @@ import json import os +import tempfile import time from unittest import mock @@ -538,3 +539,70 @@ def testYAMLConfigFileInherit(server, admin, user, fsAssetstore): assert resp.json['keyB'] == 'value2' assert resp.json['keyC'] == 'value3' assert resp.json['keyE'] == 'value7' + + +@pytest.mark.singular +@pytest.mark.usefixtures('unbindLargeImage') +@pytest.mark.plugin('large_image') +def testConfigFileEndpoints(server, admin, fsAssetstore): + config1 = ( + '[global]\nA = "B"\nC = {\n# comment\n "key1": "value1"\n }\n' + 'D = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,' + '25,26,27,28,29,30,31,32,33,34,35,36,37,38,39]\n') + config2 = '[global]\nA = B' + config3 = 'A[global]\nA = "B"' + + resp = server.request( + method='POST', path='/large_image/config/validate', body=config1, + user=admin, type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert resp.json == [] + + for config in (config2, config3): + resp = server.request( + method='POST', path='/large_image/config/validate', body=config, + user=admin, type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 1 + + resp = server.request( + method='POST', path='/large_image/config/format', body=config1, + user=admin, type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert resp.json != config1 + formatted = resp.json + + resp = server.request( + method='POST', path='/large_image/config/format', body=config2, + user=admin, type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert resp.json == config2 + + oldGirderConfig = os.environ.pop('GIRDER_CONFIG', None) + try: + with tempfile.TemporaryDirectory() as tempDir: + os.environ['GIRDER_CONFIG'] = os.path.join(tempDir, 'girder.cfg') + resp = server.request( + method='POST', path='/large_image/config/replace', + params=dict(restart='false'), body=config1, user=admin, + type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert os.path.exists(os.environ['GIRDER_CONFIG']) + assert open(os.environ['GIRDER_CONFIG']).read() == config1 + + resp = server.request( + method='POST', path='/large_image/config/replace', + params=dict(restart='false'), body=config2, user=admin, + type='application/x-girder-ini') + assert utilities.respStatus(resp) == 400 + + resp = server.request( + method='POST', path='/large_image/config/replace', + params=dict(restart='false'), body=formatted, user=admin, + type='application/x-girder-ini') + assert utilities.respStatus(resp) == 200 + assert open(os.environ['GIRDER_CONFIG']).read() == formatted + finally: + os.environ.pop('GIRDER_CONFIG', None) + if oldGirderConfig is not None: + os.environ['GIRDER_CONFIG'] = oldGirderConfig diff --git a/girder/test_girder/web_client_specs/otherFeatures.js b/girder/test_girder/web_client_specs/otherFeatures.js index 492c38f04..ef0f4f8d2 100644 --- a/girder/test_girder/web_client_specs/otherFeatures.js +++ b/girder/test_girder/web_client_specs/otherFeatures.js @@ -73,4 +73,76 @@ describe('Test features added by the large image plugin but not directly related expect($('.li-item-view-codemirror>.editor').length).toBe(1); }); }); + it('Go up a folder', function () { + runs(function () { + $('a.g-item-breadcrumb-link:last').trigger('click'); + }); + girderTest.waitForLoad(); + waitsFor(function () { + return $('.g-folder-actions-button:visible').length === 1; + }, 'the folder to appear'); + }); + it('upload test file', function () { + girderTest.waitForLoad(); + runs(function () { + $('.g-folder-list-link:first').click(); + }); + girderTest.waitForLoad(); + runs(function () { + girderTest.binaryUpload('${large_image}/../../test/test_files/sample.girder.cfg'); + }); + girderTest.waitForLoad(); + }); + it('navigate to item and change the mime type', function () { + runs(function () { + $('a.g-item-list-link:last').click(); + }); + girderTest.waitForLoad(); + runs(function () { + $('.g-file-actions-container .g-update-info').click(); + }); + girderTest.waitForDialog(); + runs(function () { + $('#g-mimetype').val('application/x-girder-ini'); + $('button.g-save-file').click(); + }); + girderTest.waitForLoad(); + }); + it('navigate away and back', function () { + runs(function () { + $('a.g-item-breadcrumb-link:last').trigger('click'); + }); + girderTest.waitForLoad(); + waitsFor(function () { + return $('.g-folder-actions-button:visible').length === 1; + }, 'the folder to appear'); + runs(function () { + $('a.g-item-list-link:last').click(); + }); + girderTest.waitForLoad(); + }); + it('test buttons', function () { + runs(function () { + expect($('.li-item-view-codemirror>.editor').length).toBe(1); + }); + runs(function () { + $('.g-view-codemirror-format-button').click(); + $('.g-view-codemirror-save-button').click(); + $('.g-view-codemirror-revert-button').click(); + $('.g-view-codemirror-save-button').click(); + $('.g-view-codemirror-format-button').click(); + }); + girderTest.waitForLoad(); + }); + it('test replace button', function () { + runs(function () { + $('.g-view-codemirror-general-button[button-key="replace"]').click(); + }); + girderTest.waitForDialog(); + runs(function () { + expect($('#g-confirm-button').text()).toEqual('Save, Replace, and Restart'); + $('a.btn-default').click(); + }); + girderTest.waitForLoad(); + }); }); diff --git a/test/test_files/sample.girder.cfg b/test/test_files/sample.girder.cfg new file mode 100644 index 000000000..883129487 --- /dev/null +++ b/test/test_files/sample.girder.cfg @@ -0,0 +1,19 @@ +[large_image] +# cache_backend, used for caching tiles, is either "memcached" or "python" +cache_backend = "python" +# 'python' cache can use 1/(val) of the available memory +cache_python_memory_portion = 32 +# 'memcached' cache backend can specify the memcached server. +# cache_memcached_url may be a list +cache_memcached_url = "127.0.0.1" +cache_memcached_username = None +cache_memcached_password = None +# The tilesource cache uses the lesser of a value based on available file +# handles, the memory portion, and the maximum (if not 0) +cache_tilesource_memory_portion = 8 +cache_tilesource_maximum = 0 +# The PIL tilesource won't read images larger than the max small images size +max_small_image_size = 4096 +# The bioformats tilesource won't read files that end in a comma-separated +# list of extensions +source_bioformats_ignored_names = r'(^[!.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi))$'