Align metrics reporter with default Datadog retry behavior

Author: Drarig29

packages/base/src/helpers/__tests__/metrics.test.ts

@@ -18,6 +18,11 @@ describe('metrics', () => {
   beforeEach(() => {
     jest.restoreAllMocks()
     mockedGetProxyDispatcher.mockReturnValue(fakeDispatcher)
+    delete process.env.DATADOG_API_KEY
+    delete process.env.DD_API_KEY
+    delete process.env.DATADOG_SITE
+    delete process.env.DD_SITE
+    delete process.env.DATADOG_API_HOST
     mockedHttpRequest.mockResolvedValue({
       config: {},
       data: {},
@@ -68,6 +73,29 @@ describe('metrics', () => {
       })
     })
 
+    test('should strip an app. prefix from the configured site', async () => {
+      const reporter = new UndiciDatadogReporter({
+        apiKey: 'test-api-key',
+        datadogSite: 'app.datadoghq.eu',
+      })
+
+      await reporter.report([gaugeMetric])
+
+      expect(mockedHttpRequest).toHaveBeenCalledWith({
+        data: {series: [gaugeMetric]},
+        dispatcher: fakeDispatcher,
+        headers: {'DD-API-KEY': 'test-api-key'},
+        method: 'POST',
+        url: 'https://api.datadoghq.eu/api/v1/series',
+      })
+    })
+
+    test('should throw when no Datadog API key is configured', () => {
+      expect(() => new UndiciDatadogReporter()).toThrow(
+        'Datadog API key not found. You must specify one via the `apiKey` configuration option or the DATADOG_API_KEY or DD_API_KEY environment variable.'
+      )
+    })
+
     test('should retry retryable request errors', async () => {
       jest.spyOn(global, 'setTimeout').mockImplementation((handler: Parameters<typeof setTimeout>[0]) => {
         if (typeof handler === 'function') {
@@ -99,9 +127,63 @@ describe('metrics', () => {
       expect(mockedHttpRequest).toHaveBeenCalledTimes(2)
     })
 
+    test('should use the retry-after header when retrying 429 responses', async () => {
+      const setTimeoutSpy = jest
+        .spyOn(global, 'setTimeout')
+        .mockImplementation((handler: Parameters<typeof setTimeout>[0]) => {
+          if (typeof handler === 'function') {
+            handler()
+          }
+
+          return {} as ReturnType<typeof setTimeout>
+        })
+
+      mockedHttpRequest
+        .mockRejectedValueOnce(
+          new RequestError('Request failed', {} as RequestConfig, {
+            data: {},
+            headers: {'retry-after': '3'},
+            status: 429,
+            statusText: 'Too Many Requests',
+          })
+        )
+        .mockResolvedValueOnce({
+          config: {},
+          data: {},
+          headers: {},
+          status: 202,
+          statusText: 'Accepted',
+        })
+
+      const reporter = new UndiciDatadogReporter({
+        apiKey: 'test-api-key',
+        datadogSite: 'datadoghq.com',
+      })
+
+      await reporter.report([gaugeMetric])
+
+      expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 3000)
+      expect(mockedHttpRequest).toHaveBeenCalledTimes(2)
+    })
+
+    test('should not retry unknown network errors without an upstream retryable code', async () => {
+      const unknownNetworkError = new RequestError('socket hang up', {} as RequestConfig)
+      ;(unknownNetworkError as RequestError & {code?: string}).code = 'UNKNOWN'
+
+      mockedHttpRequest.mockRejectedValue(unknownNetworkError)
+
+      const reporter = new UndiciDatadogReporter({
+        apiKey: 'test-api-key',
+        datadogSite: 'datadoghq.com',
+      })
+
+      await expect(reporter.report([gaugeMetric])).rejects.toThrow('socket hang up')
+      expect(mockedHttpRequest).toHaveBeenCalledTimes(1)
+    })
+
     test('should not retry non-retryable request errors', async () => {
       mockedHttpRequest.mockRejectedValue(
-        new RequestError('Request failed', {} as RequestConfig, {data: {}, status: 403, statusText: 'Forbidden'})
+        new RequestError('Request failed', {} as RequestConfig, {data: {}, status: 413, statusText: 'Too Large'})
       )
 
       const reporter = new UndiciDatadogReporter({
@@ -112,6 +194,21 @@ describe('metrics', () => {
       await expect(reporter.report([gaugeMetric])).rejects.toThrow('Request failed')
       expect(mockedHttpRequest).toHaveBeenCalledTimes(1)
     })
+
+    test('should wrap 403 responses with the default authorization error message', async () => {
+      mockedHttpRequest.mockRejectedValue(
+        new RequestError('Request failed', {} as RequestConfig, {data: {}, status: 403, statusText: 'Forbidden'})
+      )
+
+      const reporter = new UndiciDatadogReporter({
+        apiKey: 'test-api-key',
+        datadogSite: 'datadoghq.com',
+      })
+
+      await expect(reporter.report([gaugeMetric])).rejects.toThrow(
+        'Your Datadog API key is not authorized to send metrics.'
+      )
+    })
   })
 
   describe('getMetricsLogger', () => {

packages/base/src/helpers/metrics.ts

@@ -1,11 +1,15 @@
 import {BufferedMetricsLogger} from 'datadog-metrics'
 
-import {getApiUrl, getDatadogSite} from './api'
+import {DATADOG_SITE_US1} from '../constants'
+
+import {getDatadogSiteFromEnv} from './api'
 import {getProxyDispatcher, httpRequest, isRequestError} from './request'
 
 const DEFAULT_RETRIES = 2
 const DEFAULT_RETRY_BACKOFF_SECONDS = 1
-const NON_RETRYABLE_STATUS_CODES = new Set([400, 403, 413])
+const RETRYABLE_ERROR_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EPIPE', 'ETIMEDOUT'])
+const AUTHORIZATION_ERROR_MESSAGE =
+  'Your Datadog API key is not authorized to send metrics. Check to make sure the DATADOG_API_KEY or DD_API_KEY environment variable or the `apiKey` init option is set to a valid API key for your Datadog account, and that it is not an *application* key. For more, see: https://docs.datadoghq.com/account_management/api-app-keys/'
 
 interface MetricSeries {
   metric: string
@@ -15,43 +19,98 @@ interface MetricSeries {
   tags: string[]
 }
 
+interface UndiciDatadogReporterOptions {
+  apiKey?: string
+  datadogSite?: string
+  retries?: number
+  retryBackoff?: number
+}
+
 const sleep = async (milliseconds: number): Promise<void> =>
   new Promise((resolve) => {
     setTimeout(resolve, milliseconds)
   })
 
-const isRetryableError = (error: unknown): boolean => {
-  if (!isRequestError(error) || error.response === undefined) {
-    return true
+const getRetryableErrorCode = (error: unknown): string | undefined => {
+  if (typeof error === 'object' && error && 'code' in error && typeof (error as {code?: unknown}).code === 'string') {
+    return (error as {code: string}).code
   }
 
-  if (NON_RETRYABLE_STATUS_CODES.has(error.response.status)) {
-    return false
+  return undefined
+}
+
+const getRetryDelay = (error: unknown, attempt: number, retryBackoffSeconds: number): number => {
+  const retryAfterHeader =
+    isRequestError(error) && error.response?.status === 429
+      ? (error.response.headers?.['retry-after'] ?? error.response.headers?.['x-ratelimit-reset'])
+      : undefined
+
+  const retryAfterSeconds = retryAfterHeader ? Number.parseInt(retryAfterHeader, 10) : Number.NaN
+  if (!Number.isNaN(retryAfterSeconds) && retryAfterSeconds > 0) {
+    return retryAfterSeconds * 1000
   }
 
-  return error.response.status === 429 || error.response.status >= 500
+  return 2 ** attempt * retryBackoffSeconds * 1000
+}
+
+const isRetryableError = (error: unknown, attempt: number, retries: number): boolean => {
+  const status = isRequestError(error) ? error.response?.status : undefined
+  const code = getRetryableErrorCode(error)
+
+  return (
+    attempt < retries &&
+    (RETRYABLE_ERROR_CODES.has(code ?? '') || status === 429 || (status !== undefined && status >= 500))
+  )
 }
 
-const getRetryDelay = (attempt: number): number => 2 ** attempt * DEFAULT_RETRY_BACKOFF_SECONDS * 1000
+const getReporterSite = (site?: string): string => {
+  const resolvedSite = site || getDatadogSiteFromEnv() || process.env.DATADOG_API_HOST || DATADOG_SITE_US1
+
+  return resolvedSite.replace(/^app\./i, '')
+}
 
 export class UndiciDatadogReporter {
-  private apiKey?: string
+  private apiKey: string
   private apiUrl: string
   private dispatcher = getProxyDispatcher('')
+  private retries: number
+  private retryBackoffSeconds: number
+
+  constructor(options: UndiciDatadogReporterOptions = {}) {
+    this.apiKey = options.apiKey ?? process.env.DATADOG_API_KEY ?? process.env.DD_API_KEY ?? ''
+    if (!this.apiKey) {
+      throw new Error(
+        'Datadog API key not found. You must specify one via the `apiKey` configuration option or the DATADOG_API_KEY or DD_API_KEY environment variable.'
+      )
+    }
 
-  constructor(options: {apiKey?: string; datadogSite?: string}) {
-    this.apiKey = options.apiKey ?? process.env.DATADOG_API_KEY ?? process.env.DD_API_KEY
-    this.apiUrl = getApiUrl(options.datadogSite)
+    this.retries = options.retries !== undefined && options.retries >= 0 ? options.retries : DEFAULT_RETRIES
+    this.retryBackoffSeconds =
+      options.retryBackoff !== undefined && options.retryBackoff >= 0
+        ? options.retryBackoff
+        : DEFAULT_RETRY_BACKOFF_SECONDS
+    this.apiUrl = `https://api.${getReporterSite(options.datadogSite)}`
   }
 
   public async report(series: MetricSeries[]): Promise<void> {
     const metricsSeries = series.filter((metric) => metric.type !== 'distribution')
     const distributionsSeries = series.filter((metric) => metric.type === 'distribution')
-
-    await Promise.all([
+    const submissions = [
       this.sendSeries('/api/v1/series', metricsSeries),
       this.sendSeries('/api/v1/distribution_points', distributionsSeries),
-    ])
+    ]
+
+    try {
+      await Promise.all(submissions)
+    } catch (error) {
+      if (isRequestError(error) && error.response?.status === 403) {
+        const wrappedError = new Error(AUTHORIZATION_ERROR_MESSAGE)
+        ;(wrappedError as Error & {cause?: unknown}).cause = error
+        throw wrappedError
+      }
+
+      throw error
+    }
   }
 
   private async sendSeries(path: string, series: MetricSeries[]): Promise<void> {
@@ -73,11 +132,11 @@ export class UndiciDatadogReporter {
 
         return
       } catch (error) {
-        if (attempt >= DEFAULT_RETRIES || !isRetryableError(error)) {
+        if (!isRetryableError(error, attempt, this.retries)) {
           throw error
         }
 
-        await sleep(getRetryDelay(attempt))
+        await sleep(getRetryDelay(error, attempt, this.retryBackoffSeconds))
       }
     }
   }
@@ -96,7 +155,7 @@ export interface MetricsLoggerOptions {
 }
 
 export const getMetricsLogger = (opts: MetricsLoggerOptions): MetricsLogger => {
-  const datadogSite = getDatadogSite(opts.datadogSite)
+  const datadogSite = getReporterSite(opts.datadogSite)
   const apiUrl = `api.${datadogSite}`
 
   const metricsOpts = {

packages/base/src/helpers/request.ts

@@ -30,9 +30,13 @@ export interface RequestResponse<T = any> {
 export class RequestError extends Error {
   public config: RequestConfig
   public isRequestError = true as const
-  public response?: {data: any; status: number; statusText: string}
+  public response?: {data: any; headers?: Record<string, string>; status: number; statusText: string}
 
-  constructor(message: string, config: RequestConfig, response?: {data: any; status: number; statusText: string}) {
+  constructor(
+    message: string,
+    config: RequestConfig,
+    response?: {data: any; headers?: Record<string, string>; status: number; statusText: string}
+  ) {
     super(message)
     this.name = 'RequestError'
     this.config = config
@@ -183,6 +187,7 @@ export const httpRequest = async <T = any>(config: RequestConfig): Promise<Reque
     const message = `Request failed with status code ${response.status}`
     throw new RequestError(message, config, {
       data,
+      headers: responseHeaders,
       status: response.status,
       statusText: response.statusText,
     })