Add wasm-symbols upload command

.github/CODEOWNERS

@@ -28,6 +28,7 @@ packages/base/src/commands/flutter-symbols  @DataDog/datadog-ci-admins @DataDog/
 packages/base/src/commands/react-native     @DataDog/datadog-ci-admins @DataDog/rum-mobile @DataDog/rum-backend
 packages/base/src/commands/sourcemaps       @DataDog/datadog-ci-admins @DataDog/rum-browser @DataDog/rum-backend
 packages/base/src/commands/unity-symbols    @DataDog/datadog-ci-admins @DataDog/rum-mobile @DataDog/rum-backend
+packages/base/src/commands/wasm-symbols     @DataDog/datadog-ci-admins @DataDog/rum-browser @DataDog/rum-backend
 
 ## Serverless (label: serverless)
 packages/plugin-aas                                  @DataDog/datadog-ci-admins @DataDog/serverless-onboarding-and-enablement

bin/lint-packages.ts

@@ -25,6 +25,7 @@ const noPluginExceptions = new Set([
   'trace',
   'unity-symbols',
   'version',
+  'wasm-symbols',
 ])
 
 /**

packages/base/src/cli.ts

@@ -32,6 +32,7 @@ import {commands as terraformCommands} from './commands/terraform/cli'
 import {commands as traceCommands} from './commands/trace/cli'
 import {commands as unitySymbolsCommands} from './commands/unity-symbols/cli'
 import {commands as versionCommands} from './commands/version/cli'
+import {commands as wasmSymbolsCommands} from './commands/wasm-symbols/cli'
 
 // DO NOT EDIT MANUALLY. Update the source of truth in `bin/lint-packages.ts` instead.
 
@@ -66,6 +67,7 @@ export const commands = {
   'trace': traceCommands,
   'unity-symbols': unitySymbolsCommands,
   'version': versionCommands,
+  'wasm-symbols': wasmSymbolsCommands,
 } satisfies RecordWithKebabCaseKeys
 
 // DO NOT EDIT MANUALLY. Update the source of truth in `bin/lint-packages.ts` instead.
@@ -89,4 +91,5 @@ export const noPluginExceptions: Set<string> = new Set([
   'trace',
   'unity-symbols',
   'version',
+  'wasm-symbols',
 ]) satisfies Set<keyof typeof commands>

packages/base/src/commands/wasm-symbols/README.md

@@ -0,0 +1,55 @@
+## Overview
+
+Upload WebAssembly (`.wasm`) debug info files to Datadog to symbolicate WASM stack traces reported by the Datadog Browser SDK.
+
+## Setup
+
+You need to have `DD_API_KEY` in your environment.
+
+```bash
+# Environment setup
+export DD_API_KEY="<API KEY>"
+```
+
+You can configure the tool to use Datadog EU by defining the `DD_SITE` environment variable as `datadoghq.eu`. By default, the requests are sent to Datadog US.
+
+To make these variables available, Datadog recommends setting them in an encrypted `datadog-ci.json` file at the root of your project:
+
+```json
+{
+  "apiKey": "<API_KEY>",
+  "datadogSite": "<SITE>"
+}
+```
+
+To override the full URL for the intake endpoint, define the `DATADOG_SOURCEMAP_INTAKE_URL` environment variable.
+
+## Commands
+
+### `upload`
+
+**Warning:** The `wasm-symbols upload` command is in beta. It requires you to set `DD_BETA_COMMANDS_ENABLED=1`.
+
+This command will upload debug info from WASM files to Datadog in order to symbolicate your application's WASM stack traces.
+
+Run the following command to upload all the necessary files:
+
+```bash
+DD_BETA_COMMANDS_ENABLED=1 datadog-ci wasm-symbols upload ~/your/build/output/
+```
+
+If location is a directory, the command will scan it recursively looking for `.wasm` files. If location is a file, only that file is uploaded.
+
+A `.wasm` file is only uploaded if it carries debug information: either embedded DWARF debug sections (produced by e.g. `emcc -g`, `wasm-pack build --dev`) or a custom `external_debug_info` section pointing at a separate debug artifact.
+
+The module's identifier is read from a `build_id` custom section if present. If absent, it falls back to a SHA-256 hash of the module's code section, which the Datadog Browser SDK computes the same way at runtime — so lookups keep working even for toolchains that don't emit a `build_id` section.
+
+| Parameter | Condition | Description |
+|-----------|-----------|-------------|
+| `--dry-run` | Optional | Run the command without the final step of uploading. All other checks are performed. |
+| `--max-concurrency` | Optional | The number of concurrent uploads to the API. Defaults to 20. |
+| `--disable-git`    | Optional | Prevents the command from invoking Git in the current working directory and sending repository-related data to Datadog (such as the hash, remote URL, and paths within the repository of sources referenced in the source map). |
+| `--repository-url` | Optional | Overrides the remote repository with a custom URL. For example, `https://github.com/my-company/my-project`. |
+| `--replace-existing` | Optional | If symbol information with the same build ID is already present on Datadog side, discard it and use the newly uploaded information.<br>Default behavior is to only replace existing debug information if the newly uploaded information is considered a better source with the following ordering: embedded debug info > external debug info reference. |
+| `--arch` | Optional | The target WASM architecture: `wasm32` or `wasm64`. Defaults to `wasm32`. |
+| `--source-url` | Optional | The URL the module is served from in production. Used as an additional lookup key for engines that key module identity by fetch URL (e.g. `WebAssembly.instantiateStreaming`). |

packages/base/src/commands/wasm-symbols/__tests__/upload.test.ts

@@ -0,0 +1,250 @@
+import fs from 'fs'
+import os from 'os'
+
+import type {MultipartFileValue, MultipartPayload, MultipartStringValue} from '@datadog/datadog-ci-base/helpers/upload'
+
+import upath from 'upath'
+
+import {createCommand} from '@datadog/datadog-ci-base/helpers/__tests__/testing-tools'
+import {TrackedFilesMatcher} from '@datadog/datadog-ci-base/helpers/git/format-git-sourcemaps-data'
+import {UploadStatus} from '@datadog/datadog-ci-base/helpers/upload'
+import {cliVersion} from '@datadog/datadog-ci-base/version'
+
+import {uploadMultipartHelper} from '../helpers'
+import {renderArgumentMissingError, renderInvalidSymbolsLocation} from '../renderer'
+import {WasmSymbolsUploadCommand} from '../upload'
+import {WasmSectionId} from '../wasm-constants'
+
+import {buildCustomSection, buildSection, buildWasmModule} from './wasm-test-helpers'
+
+jest.mock('@datadog/datadog-ci-base/helpers/git/format-git-sourcemaps-data', () => ({
+  ...jest.requireActual('@datadog/datadog-ci-base/helpers/git/format-git-sourcemaps-data'),
+  getRepositoryData: jest.fn(),
+}))
+
+jest.mock('../helpers', () => ({
+  ...jest.requireActual('../helpers'),
+  uploadMultipartHelper: jest.fn(() => Promise.resolve(UploadStatus.Success)),
+}))
+
+const commonMetadata = {
+  cli_version: cliVersion,
+  origin: 'datadog-ci',
+  origin_version: cliVersion,
+  type: 'wasm',
+  overwrite: false,
+  source_url: undefined,
+}
+
+describe('wasm-symbols upload', () => {
+  let fixtureDir: string
+
+  beforeEach(() => {
+    fixtureDir = fs.mkdtempSync(upath.join(os.tmpdir(), 'wasm-upload-tests-'))
+    jest.clearAllMocks()
+  })
+
+  afterEach(() => {
+    fs.rmSync(fixtureDir, {recursive: true})
+  })
+
+  const writeWasmFile = (filename: string, sections: Buffer[]) => {
+    const filePath = upath.join(fixtureDir, filename)
+    fs.mkdirSync(upath.dirname(filePath), {recursive: true})
+    fs.writeFileSync(filePath, buildWasmModule(sections))
+
+    return filePath
+  }
+
+  const withDebugInfo = (buildId: string, codePayload: Buffer) => [
+    buildCustomSection('build_id', Buffer.from(buildId, 'hex')),
+    buildCustomSection('.debug_info', Buffer.from([0x00])),
+    buildSection(WasmSectionId.CODE, codePayload),
+  ]
+
+  const runCommand = async (prepFunction: (command: WasmSymbolsUploadCommand) => void) => {
+    const command = createCommand(WasmSymbolsUploadCommand)
+    prepFunction(command)
+
+    const exitCode = await command.execute()
+
+    return {exitCode, context: command.context}
+  }
+
+  describe('parameter validation', () => {
+    test('fails if symbols locations is empty', async () => {
+      const {exitCode, context} = await runCommand((cmd) => {
+        cmd['symbolsLocations'] = []
+      })
+      const errorOutput = context.stderr.toString()
+
+      expect(exitCode).not.toBe(0)
+      expect(errorOutput).toContain(renderArgumentMissingError('symbols locations'))
+    })
+
+    test('fails if symbols locations does not exist', async () => {
+      const nonExistentSymbolsLocation = upath.join(fixtureDir, 'does-not-exist')
+      const {exitCode, context} = await runCommand((cmd) => {
+        cmd['symbolsLocations'] = [nonExistentSymbolsLocation]
+      })
+      const errorOutput = context.stderr.toString()
+
+      expect(exitCode).not.toBe(0)
+      expect(errorOutput).toContain(renderInvalidSymbolsLocation(nonExistentSymbolsLocation))
+    })
+
+    test('fails on an unsupported --arch value', async () => {
+      writeWasmFile('a.wasm', withDebugInfo('aabbcc', Buffer.from([0x01])))
+      const {exitCode, context} = await runCommand((cmd) => {
+        cmd['symbolsLocations'] = [fixtureDir]
+        cmd['arch'] = 'x86_64'
+      })
+      const errorOutput = context.stderr.toString()
+
+      expect(exitCode).not.toBe(0)
+      expect(errorOutput).toContain('arch')
+    })
+  })
+
+  describe('getWasmSymbolFiles', () => {
+    test('finds wasm files with debug info recursively', async () => {
+      writeWasmFile('a.wasm', withDebugInfo('aabbcc', Buffer.from([0x01])))
+      writeWasmFile('nested/b.wasm', withDebugInfo('ddeeff', Buffer.from([0x02])))
+
+      const command = createCommand(WasmSymbolsUploadCommand)
+      const files = await command['getWasmSymbolFiles'](fixtureDir)
+
+      expect(files.map((f) => f.filename).sort()).toEqual(
+        [upath.join(fixtureDir, 'a.wasm'), upath.join(fixtureDir, 'nested/b.wasm')].sort()
+      )
+    })
+
+    test('skips wasm files without debug info', async () => {
+      writeWasmFile('no-debug.wasm', [buildSection(WasmSectionId.CODE, Buffer.from([0x01]))])
+      writeWasmFile('with-debug.wasm', withDebugInfo('aabbcc', Buffer.from([0x02])))
+
+      const command = createCommand(WasmSymbolsUploadCommand)
+      const files = await command['getWasmSymbolFiles'](fixtureDir)
+
+      expect(files.map((f) => f.filename)).toEqual([upath.join(fixtureDir, 'with-debug.wasm')])
+    })
+
+    test('throws when a single non-wasm file is given', async () => {
+      const filePath = upath.join(fixtureDir, 'not-wasm.txt')
+      fs.writeFileSync(filePath, 'hello')
+
+      const command = createCommand(WasmSymbolsUploadCommand)
+      await expect(command['getWasmSymbolFiles'](filePath)).rejects.toThrow()
+    })
+  })
+
+  describe('removeBuildIdDuplicates', () => {
+    test('prefers the entry that has embedded debug info', async () => {
+      writeWasmFile('external.wasm', [
+        buildCustomSection('build_id', Buffer.from('aabbcc', 'hex')),
+        buildCustomSection('external_debug_info', Buffer.from('external.debug.wasm')),
+        buildSection(WasmSectionId.CODE, Buffer.from([0x01])),
+      ])
+      writeWasmFile('embedded.wasm', withDebugInfo('aabbcc', Buffer.from([0x02])))
+
+      const command = createCommand(WasmSymbolsUploadCommand)
+      const files = await command['getWasmSymbolFiles'](fixtureDir)
+      const deduped = command['removeBuildIdDuplicates'](files)
+
+      expect(deduped).toHaveLength(1)
+      expect(deduped[0].filename).toBe(upath.join(fixtureDir, 'embedded.wasm'))
+    })
+  })
+
+  describe('upload', () => {
+    test('creates correct metadata payload', () => {
+      const command = createCommand(WasmSymbolsUploadCommand)
+      command['symbolsLocations'] = [fixtureDir]
+      command['gitData'] = {
+        hash: 'fake-git-hash',
+        remote: 'fake-git-remote',
+        trackedFilesMatcher: new TrackedFilesMatcher([]),
+      }
+
+      const wasmFileMetadata = {
+        filename: './a/b/c/fake-filename.wasm',
+        isWasm: true,
+        arch: 'wasm32',
+        buildId: 'fake-build-id',
+        fileHash: 'fake-file-hash',
+        hasDebugInfo: true,
+        hasExternalDebugInfo: false,
+        hasCode: true,
+      }
+      const metadata = command['getMappingMetadata'](wasmFileMetadata)
+
+      expect(metadata).toEqual({
+        ...commonMetadata,
+        arch: 'wasm32',
+        build_id: 'fake-build-id',
+        file_hash: 'fake-file-hash',
+        git_commit_sha: 'fake-git-hash',
+        git_repository_url: 'fake-git-remote',
+        symbol_source: 'debug_info',
+        filename: 'fake-filename.wasm',
+      })
+
+      command['replaceExisting'] = true
+      const metadataReplaceExisting = command['getMappingMetadata'](wasmFileMetadata)
+      expect(metadataReplaceExisting).toEqual({...metadata, overwrite: true})
+    })
+
+    test('uploads correct multipart payload', async () => {
+      writeWasmFile('a.wasm', withDebugInfo('aabbcc', Buffer.from([0x01, 0x02])))
+      writeWasmFile('b.wasm', withDebugInfo('ddeeff', Buffer.from([0x03, 0x04])))
+
+      const {exitCode} = await runCommand((cmd) => {
+        cmd['symbolsLocations'] = [fixtureDir]
+      })
+
+      expect(exitCode).toBe(0)
+      expect(uploadMultipartHelper).toHaveBeenCalledTimes(2)
+
+      const calls = (uploadMultipartHelper as jest.Mock).mock.calls.map((call) => {
+        const payload = call[1] as MultipartPayload
+        const fileItem = payload.content.get('wasm_symbol_file') as MultipartFileValue
+        expect(fileItem).toBeTruthy()
+        expect(fileItem.options.filename).toBe('wasm_symbol_file')
+
+        return JSON.parse((payload.content.get('event') as MultipartStringValue).value)
+      })
+      calls.sort((a, b) => (a.build_id as string).localeCompare(b.build_id as string))
+
+      expect(calls).toEqual([
+        {
+          ...commonMetadata,
+          arch: 'wasm32',
+          build_id: 'aabbcc',
+          file_hash: calls[0].file_hash,
+          filename: 'a.wasm',
+          symbol_source: 'debug_info',
+        },
+        {
+          ...commonMetadata,
+          arch: 'wasm32',
+          build_id: 'ddeeff',
+          file_hash: calls[1].file_hash,
+          filename: 'b.wasm',
+          symbol_source: 'debug_info',
+        },
+      ])
+    })
+
+    test('skips upload on dry run', async () => {
+      writeWasmFile('a.wasm', withDebugInfo('aabbcc', Buffer.from([0x01])))
+
+      const {exitCode} = await runCommand((cmd) => {
+        cmd['symbolsLocations'] = [fixtureDir]
+        cmd['dryRun'] = true
+      })
+
+      expect(uploadMultipartHelper).not.toHaveBeenCalled()
+      expect(exitCode).toBe(0)
+    })
+  })
+})

packages/base/src/commands/wasm-symbols/__tests__/wasm-test-helpers.ts

@@ -0,0 +1,35 @@
+import {WASM_MAGIC, WASM_VERSION, WasmSectionId} from '../wasm-constants'
+
+export const encodeUnsignedLEB128 = (value: number): Buffer => {
+  const bytes: number[] = []
+  let remaining = BigInt(value)
+  do {
+    // eslint-disable-next-line no-bitwise
+    let byte = Number(remaining & BigInt(0x7f))
+    // eslint-disable-next-line no-bitwise
+    remaining >>= BigInt(7)
+    if (remaining !== BigInt(0)) {
+      // eslint-disable-next-line no-bitwise
+      byte |= 0x80
+    }
+    bytes.push(byte)
+  } while (remaining !== BigInt(0))
+
+  return Buffer.from(bytes)
+}
+
+export const buildSection = (id: WasmSectionId, payload: Buffer): Buffer => {
+  const size = encodeUnsignedLEB128(payload.length)
+
+  return Buffer.concat([Buffer.from([id]), size, payload])
+}
+
+export const buildCustomSection = (name: string, payload: Buffer): Buffer => {
+  const nameBuffer = Buffer.from(name, 'utf8')
+  const nameLength = encodeUnsignedLEB128(nameBuffer.length)
+  const content = Buffer.concat([nameLength, nameBuffer, payload])
+
+  return buildSection(WasmSectionId.CUSTOM, content)
+}
+
+export const buildWasmModule = (sections: Buffer[]): Buffer => Buffer.concat([WASM_MAGIC, WASM_VERSION, ...sections])