fix(es): clarify bulk ingest source format is decorative (#521)

* docs(es): clarify bulk ingest source format is decorative

* test(es): pin bulk ingest source format behavior

* ci: add ecr fallback for trivy db download

* ci: fix mirror.gcr.io trivy db namespace

Author: margaretjgu

.mega-linter.yml

@@ -39,5 +39,14 @@ YAML_YAMLLINT_FILTER_REGEX_EXCLUDE: "(codegen/functional/test/fixtures/|node_mod
 
 COPYPASTE_JSCPD_CONFIG_FILE: .jscpd.json
 
+# trivy's default DB registries (mirror.gcr.io, ghcr.io) have been hitting
+# TOOMANYREQUESTS during vulnerability DB download. Add the AWS ECR Public
+# mirror as a third fallback (setting this overrides the defaults, so they're
+# listed explicitly too) since it draws from a separate rate-limit pool.
+# Note: mirror.gcr.io proxies Docker Hub, whose org for this image is
+# "aquasec" (not "aquasecurity" like GHCR/ECR); using the wrong namespace
+# here causes a permanent MANIFEST_UNKNOWN, not a transient rate limit.
+TRIVY_DB_REPOSITORY: "mirror.gcr.io/aquasec/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2,public.ecr.aws/aquasecurity/trivy-db:2"
+
 # shellcheck: warn only until active script work stabilises
 BASH_SHELLCHECK_DISABLE_ERRORS: true

package-lock.json

@@ -36,7 +36,7 @@ const inputSchema: Record<string, unknown> = {
     data_dir: { type: 'string', description: 'Path to directory of data files to ingest' },
     glob: { type: 'string', description: 'Glob pattern for --data-dir file matching (default: **/*.json, or **/*.csv when --source-format csv)' },
     no_recursive: { type: 'boolean', description: 'Do not recurse into subdirectories when using --data-dir' },
-    source_format: { type: 'string', enum: SOURCE_FORMATS, description: 'Input file format: ndjson, json, or csv', default: 'ndjson' },
+    source_format: { type: 'string', enum: SOURCE_FORMATS, description: 'Input file format: csv, or json-based (ndjson vs json array is auto-detected from content; "ndjson" and "json" behave identically here)', default: 'ndjson' },
     csv_delimiter: { type: 'string', description: 'CSV column delimiter (default: ",")' },
     csv_columns: { type: 'string', description: 'Comma-separated list of column names (overrides CSV header row)' },
     skip_header: { type: 'boolean', description: 'Skip the first row of a CSV file' },
@@ -338,6 +338,11 @@ async function streamBulkIngest (
     } else {
       // ndjson: line-by-line. json (JSON array): streamed element-by-element via
       // JsonArraySplitter, so a multi-GB array never gets buffered whole.
+      // `source_format` only distinguishes csv from everything else (see the
+      // `if` above); "ndjson" and "json" are treated identically here, and
+      // array-vs-line format is auto-detected from the first non-empty line
+      // rather than trusted from the flag, since either format may legitimately
+      // start a file either way regardless of which of the two was passed.
       const rl = createInterface({ input: stream, crlfDelay: Infinity })
       let isJsonArray: boolean | null = null // null = not yet determined
       const arraySplitter = new JsonArraySplitter()

src/es/helpers/bulk-ingest.ts

@@ -322,6 +322,60 @@ describe('bulk-ingest command', () => {
     assert.ok(body.includes('"a"') && body.includes('"b"') && body.includes('"c"'))
   })
 
+  it('splits a JSON array into separate documents regardless of --source-format (ndjson vs json is decorative)', async () => {
+    const docs = [{ title: 'doc1' }, { title: 'doc2' }]
+
+    for (const sourceFormat of ['ndjson', 'json'] as const) {
+      const tmpDir = mkdtempSync(join(tmpdir(), 'bulk-test-'))
+      writeFileSync(join(tmpDir, 'data.json'), JSON.stringify(docs))
+
+      const { transport, requests } = mockTransport([successResponse(2)])
+
+      await runCommand([
+        '--index', 'test-idx',
+        '--data-file', join(tmpDir, 'data.json'),
+        '--source-format', sourceFormat,
+        '--json'
+      ], makeDeps(transport))
+
+      assert.equal(requests.length, 1)
+      const body = requests[0]!.params.body as string
+      const docLines = body.trim().split('\n').filter((_, i) => i % 2 === 1)
+      assert.deepStrictEqual(
+        docLines.map((l) => JSON.parse(l)),
+        docs,
+        `expected two separate documents with --source-format ${sourceFormat}, not the array as one doc`
+      )
+    }
+  })
+
+  it('parses an NDJSON file the same way whether --source-format is ndjson or json', async () => {
+    const docs = [{ title: 'doc1' }, { title: 'doc2' }]
+
+    for (const sourceFormat of ['ndjson', 'json'] as const) {
+      const tmpDir = mkdtempSync(join(tmpdir(), 'bulk-test-'))
+      writeFileSync(join(tmpDir, 'data.ndjson'), docs.map((d) => JSON.stringify(d)).join('\n') + '\n')
+
+      const { transport, requests } = mockTransport([successResponse(2)])
+
+      await runCommand([
+        '--index', 'test-idx',
+        '--data-file', join(tmpDir, 'data.ndjson'),
+        '--source-format', sourceFormat,
+        '--json'
+      ], makeDeps(transport))
+
+      assert.equal(requests.length, 1)
+      const body = requests[0]!.params.body as string
+      const docLines = body.trim().split('\n').filter((_, i) => i % 2 === 1)
+      assert.deepStrictEqual(
+        docLines.map((l) => JSON.parse(l)),
+        docs,
+        `expected NDJSON to parse the same way with --source-format ${sourceFormat}`
+      )
+    }
+  })
+
   it('correctly splits JSON array elements containing commas and brackets inside strings', async () => {
     const tmpDir = mkdtempSync(join(tmpdir(), 'bulk-test-'))
     const docs = [