UTF-8 support in metric and label names

.github/super-linter.env

@@ -14,8 +14,6 @@ VALIDATE_GO_MODULES=false
 VALIDATE_HTML=false
 # done by checkstyle
 VALIDATE_JAVA=false
-# contradicting with prettier
-VALIDATE_JAVASCRIPT_STANDARD=false
 # we have many duplicate code in our codebase for demo purposes
 VALIDATE_JSCPD=false
 VALIDATE_PYTHON_PYLINT=false

benchmarks/src/main/java/io/prometheus/metrics/benchmarks/TextFormatUtilBenchmark.java

@@ -1,5 +1,6 @@
 package io.prometheus.metrics.benchmarks;
 
+import io.prometheus.metrics.config.EscapingScheme;
 import io.prometheus.metrics.expositionformats.ExpositionFormatWriter;
 import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
 import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;
@@ -69,14 +70,15 @@ public OutputStream openMetricsWriteToByteArray(WriterState writerState) throws
     // avoid growing the array
     ByteArrayOutputStream byteArrayOutputStream = writerState.byteArrayOutputStream;
     byteArrayOutputStream.reset();
-    OPEN_METRICS_TEXT_FORMAT_WRITER.write(byteArrayOutputStream, SNAPSHOTS);
+    OPEN_METRICS_TEXT_FORMAT_WRITER.write(
+        byteArrayOutputStream, SNAPSHOTS, EscapingScheme.ALLOW_UTF8);
     return byteArrayOutputStream;
   }
 
   @Benchmark
   public OutputStream openMetricsWriteToNull() throws IOException {
     OutputStream nullOutputStream = NullOutputStream.INSTANCE;
-    OPEN_METRICS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS);
+    OPEN_METRICS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS, EscapingScheme.ALLOW_UTF8);
     return nullOutputStream;
   }
 
@@ -85,14 +87,15 @@ public OutputStream prometheusWriteToByteArray(WriterState writerState) throws I
     // avoid growing the array
     ByteArrayOutputStream byteArrayOutputStream = writerState.byteArrayOutputStream;
     byteArrayOutputStream.reset();
-    PROMETHEUS_TEXT_FORMAT_WRITER.write(byteArrayOutputStream, SNAPSHOTS);
+    PROMETHEUS_TEXT_FORMAT_WRITER.write(
+        byteArrayOutputStream, SNAPSHOTS, EscapingScheme.ALLOW_UTF8);
     return byteArrayOutputStream;
   }
 
   @Benchmark
   public OutputStream prometheusWriteToNull() throws IOException {
     OutputStream nullOutputStream = NullOutputStream.INSTANCE;
-    PROMETHEUS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS);
+    PROMETHEUS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS, EscapingScheme.ALLOW_UTF8);
     return nullOutputStream;
   }
 

docs/content/config/config.md

@@ -15,7 +15,7 @@ Future releases will add more options, like configuration via environment variab
 Example:
 
 ```properties
-io.prometheus.exporter.httpServer.port = 9401
+io.prometheus.exporter.httpServer.port=9401
 ```
 
 The property above changes the port for the
@@ -71,13 +71,13 @@ metric only by specifying the metric name. Example:
 Let's say you have a histogram named `latency_seconds`.
 
 ```properties
-io.prometheus.metrics.histogramClassicUpperBounds = 0.2, 0.4, 0.8, 1.0
+io.prometheus.metrics.histogramClassicUpperBounds=0.2, 0.4, 0.8, 1.0
 ```
 
 The line above sets histogram buckets for all histograms. However:
 
 ```properties
-io.prometheus.metrics.latency_seconds.histogramClassicUpperBounds = 0.2, 0.4, 0.8, 1.0
+io.prometheus.metrics.latency_seconds.histogramClassicUpperBounds=0.2, 0.4, 0.8, 1.0
 ```
 
 The line above sets histogram buckets only for the histogram named `latency_seconds`.
@@ -170,10 +170,15 @@ See Javadoc for details.
 
 <!-- editorconfig-checker-disable -->
 
-| Name                                       | Javadoc                                                                                                                                           | Note |
-| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------- | ---- |
-| io.prometheus.exporter.pushgateway.address | [PushGateway.Builder.address()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#address(java.lang.String)>) |      |
-| io.prometheus.exporter.pushgateway.scheme  | [PushGateway.Builder.scheme()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#scheme(java.lang.String)>)   |      |
-| io.prometheus.exporter.pushgateway.job     | [PushGateway.Builder.job()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#job(java.lang.String)>)         |      |
+| Name                                              | Javadoc                                                                                                                                                                                    | Note |
+| ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---- |
+| io.prometheus.exporter.pushgateway.address        | [PushGateway.Builder.address()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#address(java.lang.String)>)                                          |      |
+| io.prometheus.exporter.pushgateway.scheme         | [PushGateway.Builder.scheme()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#scheme(java.lang.String)>)                                            |      |
+| io.prometheus.exporter.pushgateway.job            | [PushGateway.Builder.job()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#job(java.lang.String)>)                                                  |      |
+| io.prometheus.exporter.pushgateway.escapingScheme | [PushGateway.Builder.escapingScheme()](</client_java/api/io/prometheus/metrics/exporter/pushgateway/PushGateway.Builder.html#escapingScheme(io.prometheus.metrics.config.EscapingScheme)>) | (1)  |
 
 <!-- editorconfig-checker-enable -->
+
+(1) Escaping scheme can be `allow-utf-8`, `underscores`, `dots`, or `values` as described in
+[escaping schemes](https://github.com/prometheus/docs/blob/main/docs/instrumenting/escaping_schemes.md#escaping-schemes) <!-- editorconfig-checker-disable-line -->
+and in the [Unicode documentation]({{< relref "../exporters/unicode.md" >}}).

docs/content/exporters/filter.md

@@ -1,6 +1,6 @@
 ---
 title: Filter
-weight: 2
+weight: 3
 ---
 
 All exporters support a `name[]` URL parameter for querying only specific metric names. Examples:

docs/content/exporters/httpserver.md

@@ -1,6 +1,6 @@
 ---
 title: HTTPServer
-weight: 3
+weight: 4
 ---
 
 The `HTTPServer` is a standalone server for exposing a metric endpoint. A minimal example

docs/content/exporters/pushgateway.md

@@ -1,6 +1,6 @@
 ---
 title: Pushgateway
-weight: 5
+weight: 6
 ---
 
 The [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) exists to allow ephemeral

docs/content/exporters/servlet.md

@@ -1,6 +1,6 @@
 ---
 title: Servlet
-weight: 4
+weight: 5
 ---
 
 The

docs/content/exporters/spring.md

@@ -1,6 +1,6 @@
 ---
 title: Spring
-weight: 5
+weight: 7
 ---
 
 ## Alternative: Use Spring's Built-in Metrics Library

docs/content/exporters/unicode.md

@@ -0,0 +1,29 @@
+---
+title: Unicode
+weight: 2
+---
+
+The Prometheus Java client library allows all Unicode characters, that can be encoded as UTF-8.
+
+At scrape time, some characters are replaced based on the `encoding` header according
+to
+the [Escaping scheme](https://github.com/prometheus/docs/blob/main/docs/instrumenting/escaping_schemes.md). <!-- editorconfig-checker-disable-line -->
+
+For example, if you use the `underscores` escaping scheme, dots in metric and label names are
+replaced with underscores, so that the metric name `http.server.duration` becomes
+`http_server_duration`.
+
+Prometheus servers that do not support Unicode at all will not pass the `encoding` header, and the
+Prometheus Java client library will replace dots, as well as any character that is not in the legacy
+character set (`a-zA-Z0-9_:`), with underscores by default.
+
+When `escaping=allow-utf-8` is passed, add valid UTF-8 characters to the metric and label names
+without replacing them. This allows you to use dots in metric and label names, as well as
+other UTF-8 characters, without any replacements.
+
+## PushGateway
+
+When using the [Pushgateway](/exporters/pushgateway/), Unicode support has to be enabled
+explicitly by setting `io.prometheus.exporter.pushgateway.escapingScheme` to `allow-utf-8` in the
+Pushgateway configuration file - see
+[Pushgateway configuration]({{< relref "/config/config.md#exporter-pushgateway-properties" >}})

docs/content/otel/names.md

@@ -17,7 +17,7 @@ the Prometheus server as if you had exposed Prometheus metrics directly.
 
 The main steps when converting OpenTelemetry metric names to Prometheus metric names are:
 
-- Replace dots with underscores.
+- Escape illegal characters as described in [Unicode support]
 - If the metric has a unit, append the unit to the metric name, like `_seconds`.
 - If the metric type has a suffix, append it, like `_total` for counters.
 
@@ -29,14 +29,8 @@ OpenTelemetry's [Semantic Conventions for HTTP Metrics](https://opentelemetry.io
 say that if you instrument an HTTP server with OpenTelemetry, you must have a histogram named
 `http.server.duration`.
 
-Most names defined in semantic conventions use dots. In the Prometheus server, the dot is an illegal
-character (this might change in future versions of the Prometheus server).
+Most names defined in semantic conventions use dots.
+Dots in metric and label names are now supported in the Prometheus Java client library as
+described in [Unicode support].
 
-The Prometheus Java client library allows dots, so that you can use metric names and label names as
-defined in OpenTelemetry's semantic conventions.
-The dots will automatically be replaced with underscores if you expose metrics in Prometheus format,
-but you will see the original names with dots if you push your metrics in OpenTelemetry format.
-
-That way, you can use OTel-compliant metric and label names today when instrumenting your
-application with the Prometheus Java client, and you are prepared in case your monitoring backend
-adds features in the future that require OTel-compliant instrumentation.
+[Unicode support]: {{< relref "../exporters/unicode.md" >}}

prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/EscapingScheme.java

@@ -0,0 +1,85 @@
+package io.prometheus.metrics.config;
+
+import javax.annotation.Nullable;
+
+public enum EscapingScheme {
+  /** NO_ESCAPING indicates that a name will not be escaped. */
+  ALLOW_UTF8("allow-utf-8"),
+
+  /** UNDERSCORE_ESCAPING replaces all legacy-invalid characters with underscores. */
+  UNDERSCORE_ESCAPING("underscores"),
+
+  /**
+   * DOTS_ESCAPING is similar to UNDERSCORE_ESCAPING, except that dots are converted to `_dot_` and
+   * pre-existing underscores are converted to `__`.
+   */
+  DOTS_ESCAPING("dots"),
+
+  /**
+   * VALUE_ENCODING_ESCAPING prepends the name with `U__` and replaces all invalid characters with
+   * the Unicode value, surrounded by underscores. Single underscores are replaced with double
+   * underscores.
+   */
+  VALUE_ENCODING_ESCAPING("values");
+
+  private static final String ESCAPING_KEY = "escaping";
+
+  /** Default escaping scheme for names when not specified. */
+  public static final EscapingScheme DEFAULT = UNDERSCORE_ESCAPING;
+
+  public final String getValue() {
+    return value;
+  }
+
+  private final String value;
+
+  EscapingScheme(String value) {
+    this.value = value;
+  }
+
+  /**
+   * fromAcceptHeader returns an EscapingScheme depending on the Accept header. Iff the header
+   * contains an escaping=allow-utf-8 term, it will select NO_ESCAPING. If a valid "escaping" term
+   * exists, that will be used. Otherwise, the global default will be returned.
+   */
+  public static EscapingScheme fromAcceptHeader(@Nullable String acceptHeader) {
+    if (acceptHeader != null) {
+      for (String p : acceptHeader.split(";")) {
+        String[] toks = p.split("=");
+        if (toks.length != 2) {
+          continue;
+        }
+        String key = toks[0].trim();
+        String value = toks[1].trim();
+        if (key.equals(ESCAPING_KEY)) {
+          try {
+            return EscapingScheme.forString(value);
+          } catch (IllegalArgumentException e) {
+            // If the escaping parameter is unknown, ignore it.
+            return DEFAULT;
+          }
+        }
+      }
+    }
+    return DEFAULT;
+  }
+
+  static EscapingScheme forString(String value) {
+    switch (value) {
+      case "allow-utf-8":
+        return ALLOW_UTF8;
+      case "underscores":
+        return UNDERSCORE_ESCAPING;
+      case "dots":
+        return DOTS_ESCAPING;
+      case "values":
+        return VALUE_ENCODING_ESCAPING;
+      default:
+        throw new IllegalArgumentException("Unknown escaping scheme: " + value);
+    }
+  }
+
+  public String toHeaderFormat() {
+    return "; " + ESCAPING_KEY + "=" + value;
+  }
+}

prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/ExporterPushgatewayProperties.java

@@ -8,16 +8,22 @@ public class ExporterPushgatewayProperties {
   private static final String ADDRESS = "address";
   private static final String JOB = "job";
   private static final String SCHEME = "scheme";
+  private static final String ESCAPING_SCHEME = "escapingScheme";
   private static final String PREFIX = "io.prometheus.exporter.pushgateway";
   @Nullable private final String scheme;
   @Nullable private final String address;
   @Nullable private final String job;
+  @Nullable private final EscapingScheme escapingScheme;
 
   private ExporterPushgatewayProperties(
-      @Nullable String address, @Nullable String job, @Nullable String scheme) {
+      @Nullable String address,
+      @Nullable String job,
+      @Nullable String scheme,
+      @Nullable EscapingScheme escapingScheme) {
     this.address = address;
     this.job = job;
     this.scheme = scheme;
+    this.escapingScheme = escapingScheme;
   }
 
   /** Address of the Pushgateway in the form {@code host:port}. Default is {@code localhost:9091} */
@@ -44,6 +50,12 @@ public String getScheme() {
     return scheme;
   }
 
+  /** Escaping scheme to be used when pushing metric data to the pushgateway. */
+  @Nullable
+  public EscapingScheme getEscapingScheme() {
+    return escapingScheme;
+  }
+
   /**
    * Note that this will remove entries from {@code properties}. This is because we want to know if
    * there are unused properties remaining after all properties have been loaded.
@@ -53,6 +65,8 @@ static ExporterPushgatewayProperties load(Map<Object, Object> properties)
     String address = Util.loadString(PREFIX + "." + ADDRESS, properties);
     String job = Util.loadString(PREFIX + "." + JOB, properties);
     String scheme = Util.loadString(PREFIX + "." + SCHEME, properties);
+    String escapingScheme = Util.loadString(PREFIX + "." + ESCAPING_SCHEME, properties);
+
     if (scheme != null) {
       if (!scheme.equals("http") && !scheme.equals("https")) {
         throw new PrometheusPropertiesException(
@@ -61,7 +75,31 @@ static ExporterPushgatewayProperties load(Map<Object, Object> properties)
                 PREFIX, SCHEME, scheme));
       }
     }
-    return new ExporterPushgatewayProperties(address, job, scheme);
+
+    return new ExporterPushgatewayProperties(
+        address, job, scheme, parseEscapingScheme(escapingScheme));
+  }
+
+  private static @Nullable EscapingScheme parseEscapingScheme(@Nullable String scheme) {
+    if (scheme == null) {
+      return null;
+    }
+    switch (scheme) {
+      case "allow-utf-8":
+        return EscapingScheme.ALLOW_UTF8;
+      case "values":
+        return EscapingScheme.VALUE_ENCODING_ESCAPING;
+      case "underscores":
+        return EscapingScheme.UNDERSCORE_ESCAPING;
+      case "dots":
+        return EscapingScheme.DOTS_ESCAPING;
+      default:
+        throw new PrometheusPropertiesException(
+            String.format(
+                "%s.%s: Illegal value. Expecting 'allow-utf-8', 'values', 'underscores', "
+                    + "or 'dots'. Found: %s",
+                PREFIX, ESCAPING_SCHEME, scheme));
+    }
   }
 
   public static Builder builder() {
@@ -72,6 +110,7 @@ public static class Builder {
     @Nullable private String address;
     @Nullable private String job;
     @Nullable private String scheme;
+    @Nullable private EscapingScheme escapingScheme;
 
     private Builder() {}
 
@@ -90,8 +129,13 @@ public Builder scheme(String scheme) {
       return this;
     }
 
+    public Builder escapingScheme(EscapingScheme escapingScheme) {
+      this.escapingScheme = escapingScheme;
+      return this;
+    }
+
     public ExporterPushgatewayProperties build() {
-      return new ExporterPushgatewayProperties(address, job, scheme);
+      return new ExporterPushgatewayProperties(address, job, scheme, escapingScheme);
     }
   }
 }