Grafana service started not working (realized on May1st).
I start the service like this
--services=redmine-mailer,chatbot,grafana
below is what I get from docker logs grafana-submit-prod-agent-v2
Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.
logger=settings t=2026-05-04T12:28:02.102268873Z level=info msg="Starting Grafana" version=10.2.0 commit=895fbafb7a branch=HEAD compiled=2023-10-23T13:41:03Z
logger=settings t=2026-05-04T12:28:02.102580727Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini
logger=settings t=2026-05-04T12:28:02.102590199Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini
logger=settings t=2026-05-04T12:28:02.102596618Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana"
logger=settings t=2026-05-04T12:28:02.102601278Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana"
logger=settings t=2026-05-04T12:28:02.102605134Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins"
logger=settings t=2026-05-04T12:28:02.102608806Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning"
logger=settings t=2026-05-04T12:28:02.10261232Z level=info msg="Config overridden from command line" arg="default.log.mode=console"
logger=settings t=2026-05-04T12:28:02.102616581Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana"
logger=settings t=2026-05-04T12:28:02.102620701Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana"
logger=settings t=2026-05-04T12:28:02.102624213Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins"
logger=settings t=2026-05-04T12:28:02.102627787Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning"
logger=settings t=2026-05-04T12:28:02.102631632Z level=info msg=Target target=[all]
logger=settings t=2026-05-04T12:28:02.102639791Z level=info msg="Path Home" path=/usr/share/grafana
logger=settings t=2026-05-04T12:28:02.1026435Z level=info msg="Path Data" path=/var/lib/grafana
logger=settings t=2026-05-04T12:28:02.102647229Z level=info msg="Path Logs" path=/var/log/grafana
logger=settings t=2026-05-04T12:28:02.102650754Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins
logger=settings t=2026-05-04T12:28:02.102654682Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning
logger=settings t=2026-05-04T12:28:02.102658635Z level=info msg="App mode production"
logger=sqlstore t=2026-05-04T12:28:02.102719153Z level=info msg="Connecting to DB" dbtype=sqlite3
logger=migrator t=2026-05-04T12:28:02.103541459Z level=info msg="Starting DB migrations"
logger=migrator t=2026-05-04T12:28:02.122266672Z level=info msg="migrations completed" performed=0 skipped=608 duration=838.056µs
logger=licensing t=2026-05-04T12:28:02.123289516Z level=info msg="Validated license token" appURL=http://localhost:3000/ source=disk status=NotFound
logger=secrets t=2026-05-04T12:28:02.123734684Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1
logger=local.finder t=2026-05-04T12:28:02.218923097Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled
logger=plugins.registration t=2026-05-04T12:28:04.677511088Z level=info msg="Plugin registered" pluginId=grafana-simple-json-datasource
logger=plugins.registration t=2026-05-04T12:28:04.677612724Z level=info msg="Plugin registered" pluginId=marcusolsson-json-datasource
logger=plugins.registration t=2026-05-04T12:28:04.677691352Z level=info msg="Plugin registered" pluginId=yesoreyeram-infinity-datasource
logger=plugins.registration t=2026-05-04T12:28:04.691555072Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-app
logger=plugins.registration t=2026-05-04T12:28:04.69160425Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-datasource
logger=plugins.registration t=2026-05-04T12:28:04.699649697Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-triggers-panel
logger=plugins.registration t=2026-05-04T12:28:04.699698063Z level=info msg="Plugin registered" pluginId=grafana-image-renderer
logger=query_data t=2026-05-04T12:28:04.703909307Z level=info msg="Query Service initialization"
logger=live.push_http t=2026-05-04T12:28:04.711418393Z level=info msg="Live Push Gateway initialization"
logger=infra.usagestats.collector t=2026-05-04T12:28:06.968984365Z level=info msg="registering usage stat providers" usageStatsProvidersLen=4
logger=provisioning.alerting t=2026-05-04T12:28:07.000638406Z level=info msg="starting to provision alerting"
logger=provisioning.alerting t=2026-05-04T12:28:07.000656324Z level=info msg="finished to provision alerting"
logger=report t=2026-05-04T12:28:07.000858775Z level=warn msg="Scheduling and sending of reports disabled, SMTP is not configured and enabled. Configure SMTP to enable."
logger=caching.service t=2026-05-04T12:28:07.000902274Z level=warn msg="Caching service is disabled"
logger=ngalert.state.manager t=2026-05-04T12:28:07.000944859Z level=info msg="Warming state cache for startup"
logger=grafanaStorageLogger t=2026-05-04T12:28:07.001071691Z level=info msg="Storage starting"
logger=plugin.grafana-image-renderer t=2026-05-04T12:28:07.002834572Z level=error msg="plugin process exited" path=/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64 pid=98 error="exit status 127"
logger=server t=2026-05-04T12:28:07.003104126Z level=error msg="Stopped background service" service=*rendering.RenderingService reason="Unrecognized remote plugin message: \nThis usually means\n the plugin was not compiled for this architecture,\n the plugin is missing dynamic-link libraries necessary to run,\n the plugin is not executable by this process due to file permissions, or\n the plugin failed to negotiate the initial go-plugin protocol handshake\n\nAdditional notes about plugin:\n Path: /var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64\n Mode: -rwxr-xr-x\n Owner: 472 [grafana] (current: 472 [grafana])\n Group: 0 [root] (current: 0 [root])\n ELF architecture: EM_X86_64 (current architecture: amd64)\n"
logger=tracing t=2026-05-04T12:28:07.003224587Z level=info msg="Closing tracing"
logger=plugin.grafana-image-renderer t=2026-05-04T12:28:07.003363724Z level=warn msg="plugin failed to exit gracefully"
logger=grafana.update.checker t=2026-05-04T12:28:07.003364362Z level=error msg="Update check failed" error="failed to get latest.json repo from github.com: Get "https://raw.githubusercontent.com/grafana/grafana/main/latest.json\": context canceled" duration=2.412229ms
logger=plugin.alexanderzobnin-zabbix-datasource t=2026-05-04T12:28:07.0058878Z level=info msg="plugin process exited" path=/var/lib/grafana/plugins/alexanderzobnin-zabbix-app/datasource/gpx_zabbix-datasource_linux_amd64 pid=64
logger=plugin.yesoreyeram-infinity-datasource t=2026-05-04T12:28:07.006166241Z level=info msg="plugin process exited" path=/var/lib/grafana/plugins/yesoreyeram-infinity-datasource/gpx_infinity_linux_amd64 pid=50
logger=http.server t=2026-05-04T12:28:07.009682677Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=http subUrl= socket=
logger=ngalert.state.manager t=2026-05-04T12:28:07.064819751Z level=error msg="Unable to fetch orgIds" error="context canceled"
logger=ngalert.state.manager t=2026-05-04T12:28:07.064864256Z level=info msg="State cache has been initialized" states=0 duration=63.921049ms
logger=ngalert.scheduler t=2026-05-04T12:28:07.064902229Z level=info msg="Starting scheduler" tickInterval=10s
logger=ngalert.multiorg.alertmanager t=2026-05-04T12:28:07.064919837Z level=info msg="Starting MultiOrg Alertmanager"
logger=ticker t=2026-05-04T12:28:07.06497724Z level=info msg=starting first_tick=2026-05-04T12:28:10Z
logger=secret.migration t=2026-05-04T12:28:07.078940485Z level=error msg="Server lock for secret migration already exists"
logger=provisioning t=2026-05-04T12:28:07.082906414Z level=error msg="Failed to provision dashboard" error="Failed to create provisioner: Failed to read dashboards config: failed to provision dashboards with "archi" reader: failed to check whether org. with the given ID exists: context canceled"
logger=analytics.summaries t=2026-05-04T12:28:07.084757559Z level=error msg="error during daily rollup" err="got error when acquiring lock: context canceled"
logger=ngalert.migration t=2026-05-04T12:28:07.086604294Z level=warn msg="Server lock for alerting migration already exists"
logger=publicdashboards.metric t=2026-05-04T12:28:07.087192342Z level=error msg="error collecting background metrics" err="context canceled"
logger=serviceaccounts t=2026-05-04T12:28:07.08789602Z level=warn msg="Failed to get usage metrics" error="context canceled"
logger=plugin.signature.key_retriever t=2026-05-04T12:28:07.088169208Z level=error msg="Error downloading plugin manifest keys" error="kv get: context canceled"
Error: ✗ *rendering.RenderingService run error: Unrecognized remote plugin message:
This usually means
the plugin was not compiled for this architecture,
the plugin is missing dynamic-link libraries necessary to run,
the plugin is not executable by this process due to file permissions, or
the plugin failed to negotiate the initial go-plugin protocol handshake
Additional notes about plugin:
Path: /var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64
Mode: -rwxr-xr-x
Owner: 472 [grafana] (current: 472 [grafana])
Group: 0 [root] (current: 0 [root])
ELF architecture: EM_X86_64 (current architecture: amd64)
Grafana service started not working (realized on May1st).
I start the service like this
--services=redmine-mailer,chatbot,grafanabelow is what I get from
docker logs grafana-submit-prod-agent-v2Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.
logger=settings t=2026-05-04T12:28:02.102268873Z level=info msg="Starting Grafana" version=10.2.0 commit=895fbafb7a branch=HEAD compiled=2023-10-23T13:41:03Z
logger=settings t=2026-05-04T12:28:02.102580727Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini
logger=settings t=2026-05-04T12:28:02.102590199Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini
logger=settings t=2026-05-04T12:28:02.102596618Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana"
logger=settings t=2026-05-04T12:28:02.102601278Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana"
logger=settings t=2026-05-04T12:28:02.102605134Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins"
logger=settings t=2026-05-04T12:28:02.102608806Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning"
logger=settings t=2026-05-04T12:28:02.10261232Z level=info msg="Config overridden from command line" arg="default.log.mode=console"
logger=settings t=2026-05-04T12:28:02.102616581Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana"
logger=settings t=2026-05-04T12:28:02.102620701Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana"
logger=settings t=2026-05-04T12:28:02.102624213Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins"
logger=settings t=2026-05-04T12:28:02.102627787Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning"
logger=settings t=2026-05-04T12:28:02.102631632Z level=info msg=Target target=[all]
logger=settings t=2026-05-04T12:28:02.102639791Z level=info msg="Path Home" path=/usr/share/grafana
logger=settings t=2026-05-04T12:28:02.1026435Z level=info msg="Path Data" path=/var/lib/grafana
logger=settings t=2026-05-04T12:28:02.102647229Z level=info msg="Path Logs" path=/var/log/grafana
logger=settings t=2026-05-04T12:28:02.102650754Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins
logger=settings t=2026-05-04T12:28:02.102654682Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning
logger=settings t=2026-05-04T12:28:02.102658635Z level=info msg="App mode production"
logger=sqlstore t=2026-05-04T12:28:02.102719153Z level=info msg="Connecting to DB" dbtype=sqlite3
logger=migrator t=2026-05-04T12:28:02.103541459Z level=info msg="Starting DB migrations"
logger=migrator t=2026-05-04T12:28:02.122266672Z level=info msg="migrations completed" performed=0 skipped=608 duration=838.056µs
logger=licensing t=2026-05-04T12:28:02.123289516Z level=info msg="Validated license token" appURL=http://localhost:3000/ source=disk status=NotFound
logger=secrets t=2026-05-04T12:28:02.123734684Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1
logger=local.finder t=2026-05-04T12:28:02.218923097Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled
logger=plugins.registration t=2026-05-04T12:28:04.677511088Z level=info msg="Plugin registered" pluginId=grafana-simple-json-datasource
logger=plugins.registration t=2026-05-04T12:28:04.677612724Z level=info msg="Plugin registered" pluginId=marcusolsson-json-datasource
logger=plugins.registration t=2026-05-04T12:28:04.677691352Z level=info msg="Plugin registered" pluginId=yesoreyeram-infinity-datasource
logger=plugins.registration t=2026-05-04T12:28:04.691555072Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-app
logger=plugins.registration t=2026-05-04T12:28:04.69160425Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-datasource
logger=plugins.registration t=2026-05-04T12:28:04.699649697Z level=info msg="Plugin registered" pluginId=alexanderzobnin-zabbix-triggers-panel
logger=plugins.registration t=2026-05-04T12:28:04.699698063Z level=info msg="Plugin registered" pluginId=grafana-image-renderer
logger=query_data t=2026-05-04T12:28:04.703909307Z level=info msg="Query Service initialization"
logger=live.push_http t=2026-05-04T12:28:04.711418393Z level=info msg="Live Push Gateway initialization"
logger=infra.usagestats.collector t=2026-05-04T12:28:06.968984365Z level=info msg="registering usage stat providers" usageStatsProvidersLen=4
logger=provisioning.alerting t=2026-05-04T12:28:07.000638406Z level=info msg="starting to provision alerting"
logger=provisioning.alerting t=2026-05-04T12:28:07.000656324Z level=info msg="finished to provision alerting"
logger=report t=2026-05-04T12:28:07.000858775Z level=warn msg="Scheduling and sending of reports disabled, SMTP is not configured and enabled. Configure SMTP to enable."
logger=caching.service t=2026-05-04T12:28:07.000902274Z level=warn msg="Caching service is disabled"
logger=ngalert.state.manager t=2026-05-04T12:28:07.000944859Z level=info msg="Warming state cache for startup"
logger=grafanaStorageLogger t=2026-05-04T12:28:07.001071691Z level=info msg="Storage starting"
logger=plugin.grafana-image-renderer t=2026-05-04T12:28:07.002834572Z level=error msg="plugin process exited" path=/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64 pid=98 error="exit status 127"
logger=server t=2026-05-04T12:28:07.003104126Z level=error msg="Stopped background service" service=*rendering.RenderingService reason="Unrecognized remote plugin message: \nThis usually means\n the plugin was not compiled for this architecture,\n the plugin is missing dynamic-link libraries necessary to run,\n the plugin is not executable by this process due to file permissions, or\n the plugin failed to negotiate the initial go-plugin protocol handshake\n\nAdditional notes about plugin:\n Path: /var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64\n Mode: -rwxr-xr-x\n Owner: 472 [grafana] (current: 472 [grafana])\n Group: 0 [root] (current: 0 [root])\n ELF architecture: EM_X86_64 (current architecture: amd64)\n"
logger=tracing t=2026-05-04T12:28:07.003224587Z level=info msg="Closing tracing"
logger=plugin.grafana-image-renderer t=2026-05-04T12:28:07.003363724Z level=warn msg="plugin failed to exit gracefully"
logger=grafana.update.checker t=2026-05-04T12:28:07.003364362Z level=error msg="Update check failed" error="failed to get latest.json repo from github.com: Get "https://raw.githubusercontent.com/grafana/grafana/main/latest.json\": context canceled" duration=2.412229ms
logger=plugin.alexanderzobnin-zabbix-datasource t=2026-05-04T12:28:07.0058878Z level=info msg="plugin process exited" path=/var/lib/grafana/plugins/alexanderzobnin-zabbix-app/datasource/gpx_zabbix-datasource_linux_amd64 pid=64
logger=plugin.yesoreyeram-infinity-datasource t=2026-05-04T12:28:07.006166241Z level=info msg="plugin process exited" path=/var/lib/grafana/plugins/yesoreyeram-infinity-datasource/gpx_infinity_linux_amd64 pid=50
logger=http.server t=2026-05-04T12:28:07.009682677Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=http subUrl= socket=
logger=ngalert.state.manager t=2026-05-04T12:28:07.064819751Z level=error msg="Unable to fetch orgIds" error="context canceled"
logger=ngalert.state.manager t=2026-05-04T12:28:07.064864256Z level=info msg="State cache has been initialized" states=0 duration=63.921049ms
logger=ngalert.scheduler t=2026-05-04T12:28:07.064902229Z level=info msg="Starting scheduler" tickInterval=10s
logger=ngalert.multiorg.alertmanager t=2026-05-04T12:28:07.064919837Z level=info msg="Starting MultiOrg Alertmanager"
logger=ticker t=2026-05-04T12:28:07.06497724Z level=info msg=starting first_tick=2026-05-04T12:28:10Z
logger=secret.migration t=2026-05-04T12:28:07.078940485Z level=error msg="Server lock for secret migration already exists"
logger=provisioning t=2026-05-04T12:28:07.082906414Z level=error msg="Failed to provision dashboard" error="Failed to create provisioner: Failed to read dashboards config: failed to provision dashboards with "archi" reader: failed to check whether org. with the given ID exists: context canceled"
logger=analytics.summaries t=2026-05-04T12:28:07.084757559Z level=error msg="error during daily rollup" err="got error when acquiring lock: context canceled"
logger=ngalert.migration t=2026-05-04T12:28:07.086604294Z level=warn msg="Server lock for alerting migration already exists"
logger=publicdashboards.metric t=2026-05-04T12:28:07.087192342Z level=error msg="error collecting background metrics" err="context canceled"
logger=serviceaccounts t=2026-05-04T12:28:07.08789602Z level=warn msg="Failed to get usage metrics" error="context canceled"
logger=plugin.signature.key_retriever t=2026-05-04T12:28:07.088169208Z level=error msg="Error downloading plugin manifest keys" error="kv get: context canceled"
Error: ✗ *rendering.RenderingService run error: Unrecognized remote plugin message:
This usually means
the plugin was not compiled for this architecture,
the plugin is missing dynamic-link libraries necessary to run,
the plugin is not executable by this process due to file permissions, or
the plugin failed to negotiate the initial go-plugin protocol handshake
Additional notes about plugin:
Path: /var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64
Mode: -rwxr-xr-x
Owner: 472 [grafana] (current: 472 [grafana])
Group: 0 [root] (current: 0 [root])
ELF architecture: EM_X86_64 (current architecture: amd64)