diff --git a/cli/cli/connection/bigquery.ts b/cli/cli/connection/bigquery.ts index 12af9f6c93..c5a664df97 100644 --- a/cli/cli/connection/bigquery.ts +++ b/cli/cli/connection/bigquery.ts @@ -22,7 +22,7 @@ function configKey(c: BigQueryCredentials): string { // PEM in a join key is wasteful. return [ c.projectId, - c.serviceAccountJson.client_email, + c.authType === 'adc' ? 'adc' : c.serviceAccountJson.client_email, c.location ?? '', c.defaultDataset ?? '' ].join('|'); diff --git a/cli/cli/connection/load-config.test.ts b/cli/cli/connection/load-config.test.ts index d036e09462..65f70b354b 100644 --- a/cli/cli/connection/load-config.test.ts +++ b/cli/cli/connection/load-config.test.ts @@ -129,12 +129,35 @@ describe('loadConnectionConfig', () => { ).toBe(validKeyfileJson.client_email); }); - it('rejects when neither keyfile nor keyfile_json present', async () => { + it('rejects when no auth method is present', async () => { await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\n`); await expect(loadConnectionConfig(workDir)).rejects.toThrow( - /Provide one of: keyfile_json, keyfile/ + /Provide one of: keyfile_json, keyfile, adc/ ); }); + + it('resolves adc: true without a key', async () => { + await writeYaml(`type: bigquery\nproject: p\nlocation: EU\ndatasets: [d]\nadc: true\n`); + const cfg = await loadConnectionConfig(workDir); + expect(cfg).toEqual({ + type: 'bigquery', + authType: 'adc', + projectId: 'p', + location: 'EU', + defaultDataset: undefined, + datasets: ['d'] + }); + }); + + it('rejects adc combined with a keyfile', async () => { + await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\nadc: true\nkeyfile: ./sa.json\n`); + await expect(loadConnectionConfig(workDir)).rejects.toThrow(/Provide only one of/); + }); + + it('rejects adc: false', async () => { + await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\nadc: false\n`); + await expect(loadConnectionConfig(workDir)).rejects.toThrow(); + }); }); describe('clickhouse', () => { diff --git a/cli/cli/init/connection-template.ts b/cli/cli/init/connection-template.ts index 2e6b703f2c..bc3b394cb9 100644 --- a/cli/cli/init/connection-template.ts +++ b/cli/cli/init/connection-template.ts @@ -44,7 +44,7 @@ database: "" const BIGQUERY_TEMPLATE = `# BigQuery direct connector. Docs: https://docs.evidence.dev/direct-connectors/bigquery type: bigquery project: "" -keyfile: ./service-account.json # or inline keyfile_json +keyfile: ./service-account.json # or inline keyfile_json, or adc: true datasets: # accessible datasets, required (at least one) - "" # location: US # default query location, optional diff --git a/cli/cli/launch.ts b/cli/cli/launch.ts index afb9072dbf..e5e24cc617 100644 --- a/cli/cli/launch.ts +++ b/cli/cli/launch.ts @@ -420,6 +420,11 @@ async function maybeUploadCredentials( // No connection.yaml → Evidence-managed; nothing to upload. return; } + if (config.type === 'bigquery' && config.authType === 'adc') { + console.log(" • connection.yaml uses ADC, which can't be uploaded."); + console.log(' Add a service-account key in Studio → Settings → Warehouse.'); + return; + } // Uploading writes org-wide warehouse settings, so require an explicit opt-in: // a confirm in a TTY, or `--upload-credentials` in a non-interactive run. Never diff --git a/core/src/connectors/bigquery/client-options.test.ts b/core/src/connectors/bigquery/client-options.test.ts index f5d7283caf..f10099bd67 100644 --- a/core/src/connectors/bigquery/client-options.test.ts +++ b/core/src/connectors/bigquery/client-options.test.ts @@ -33,6 +33,15 @@ describe('buildBigQueryClientOptions', () => { }); expect(opts.location).toBe('US'); }); + + it('omits credentials for ADC so the SDK resolves them', () => { + const opts = buildBigQueryClientOptions({ + authType: 'adc', + projectId: 'my-proj', + location: 'EU' + }); + expect(opts).toEqual({ projectId: 'my-proj', location: 'EU' }); + }); }); describe('normalizeCredentials', () => { diff --git a/core/src/connectors/bigquery/client-options.ts b/core/src/connectors/bigquery/client-options.ts index 563eaaefce..106e7d011d 100644 --- a/core/src/connectors/bigquery/client-options.ts +++ b/core/src/connectors/bigquery/client-options.ts @@ -8,22 +8,20 @@ import type { BigQueryCredentials } from './credentials'; */ export type BigQueryClientOptions = { projectId: string; - credentials: { client_email: string; private_key: string }; + /** Omitted for ADC; the SDK then resolves Application Default Credentials. */ + credentials?: { client_email: string; private_key: string }; location?: string; }; -/** - * Build the options object passed to `new BigQuery(...)`. Service-account JSON - * is the only auth path supported in v1. - */ +/** Build the options object passed to `new BigQuery(...)`. */ export function buildBigQueryClientOptions( credentials: BigQueryCredentials ): BigQueryClientOptions { - const { client_email, private_key } = credentials.serviceAccountJson; - const opts: BigQueryClientOptions = { - projectId: credentials.projectId, - credentials: { client_email, private_key } - }; + const opts: BigQueryClientOptions = { projectId: credentials.projectId }; + if (credentials.authType === 'service_account_json') { + const { client_email, private_key } = credentials.serviceAccountJson; + opts.credentials = { client_email, private_key }; + } if (credentials.location !== undefined) { opts.location = credentials.location; } diff --git a/core/src/connectors/bigquery/connection-schema.ts b/core/src/connectors/bigquery/connection-schema.ts index ced01e340b..617564768d 100644 --- a/core/src/connectors/bigquery/connection-schema.ts +++ b/core/src/connectors/bigquery/connection-schema.ts @@ -45,6 +45,20 @@ export const bigqueryBase = z.object({ }) ), + adc: z + .literal(true) + .optional() + .meta( + meta({ + label: 'Application Default Credentials', + description: + 'Use Application Default Credentials (gcloud auth application-default login, GOOGLE_APPLICATION_CREDENTIALS, or the GCP metadata server).', + category: 'credential', + cliOnly: true, + authGroup: 'bigquery-auth' + }) + ), + location: z .string() .optional() diff --git a/core/src/connectors/bigquery/credentials.ts b/core/src/connectors/bigquery/credentials.ts index 11ef2e6cd8..acc2411584 100644 --- a/core/src/connectors/bigquery/credentials.ts +++ b/core/src/connectors/bigquery/credentials.ts @@ -1,4 +1,4 @@ -export type BigQueryAuthType = 'service_account_json'; +export type BigQueryAuthType = 'service_account_json' | 'adc'; export type BigQueryServiceAccountJson = { client_email: string; @@ -18,7 +18,11 @@ export type BigQueryServiceAccountCredentials = BigQueryConnectionParams & { serviceAccountJson: BigQueryServiceAccountJson; }; -export type BigQueryCredentials = BigQueryServiceAccountCredentials; +export type BigQueryAdcCredentials = BigQueryConnectionParams & { + authType: 'adc'; +}; + +export type BigQueryCredentials = BigQueryServiceAccountCredentials | BigQueryAdcCredentials; /** * Coerce raw vault payload into BigQueryCredentials. @@ -26,7 +30,7 @@ export type BigQueryCredentials = BigQueryServiceAccountCredentials; * the right shape but assert the load-bearing keys here so a corrupted secret * fails with a readable error rather than a downstream SDK error. */ -export function normalizeCredentials(raw: unknown): BigQueryCredentials { +export function normalizeCredentials(raw: unknown): BigQueryServiceAccountCredentials { if (raw === null || raw === undefined || typeof raw !== 'object') { throw new Error('BigQuery credentials are missing or invalid'); } diff --git a/core/src/connectors/bigquery/resolve.ts b/core/src/connectors/bigquery/resolve.ts index a903fe1d9d..ef7bd10369 100644 --- a/core/src/connectors/bigquery/resolve.ts +++ b/core/src/connectors/bigquery/resolve.ts @@ -16,6 +16,16 @@ export async function resolveBigQueryCredentials( config: BigQueryConnection, opts: ResolveOpts ): Promise { + const params = { + projectId: config.project, + location: config.location, + defaultDataset: config.dataset + }; + + if (config.adc) { + return { authType: 'adc', ...params }; + } + let serviceAccountJson: BigQueryServiceAccountJson; if (config.keyfile_json) { @@ -37,14 +47,8 @@ export async function resolveBigQueryCredentials( } } else { // Schema's auth-group check should have caught this — defensive. - throw new Error('BigQuery credentials are missing keyfile_json and keyfile'); + throw new Error('BigQuery credentials are missing keyfile_json, keyfile and adc'); } - return { - authType: 'service_account_json', - projectId: config.project, - serviceAccountJson, - location: config.location, - defaultDataset: config.dataset - }; + return { authType: 'service_account_json', serviceAccountJson, ...params }; } diff --git a/docs/cli/connections.mdx b/docs/cli/connections.mdx index 685b42c346..7d931b339a 100644 --- a/docs/cli/connections.mdx +++ b/docs/cli/connections.mdx @@ -63,7 +63,7 @@ Fill in the placeholders with your own connection details. If `--warehouse` is o # location: US ``` - Provide exactly one of `keyfile` or `keyfile_json`. The `keyfile` path is resolved relative to `connection.yaml`. + Provide exactly one of `keyfile`, `keyfile_json`, or `adc: true`. The `keyfile` path is resolved relative to `connection.yaml`. With `adc: true`, Evidence uses [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), such as your `gcloud auth application-default login` session. See the [BigQuery direct connector](/direct-connectors/bigquery) for the full field reference and the service-account setup steps. diff --git a/docs/direct-connectors/bigquery.mdx b/docs/direct-connectors/bigquery.mdx index e8a7a4c7d6..4d0bdca7ff 100644 --- a/docs/direct-connectors/bigquery.mdx +++ b/docs/direct-connectors/bigquery.mdx @@ -70,6 +70,27 @@ gcloud iam service-accounts keys create /evidence-bq-primary.js +### Application Default Credentials (CLI and self-hosted) + +When running the CLI or a self-hosted `evidence serve`, you can skip the key file and use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) instead: + +```yaml +type: bigquery +project: my-gcp-project +adc: true +datasets: + - analytics +``` + +Evidence then authenticates with, in order: + +1. The key file at `GOOGLE_APPLICATION_CREDENTIALS`. +2. Your own login from `gcloud auth application-default login`. +3. The attached service account when running on GCP (Cloud Run, GCE, GKE). + +The identity needs the same **BigQuery Job User** and **BigQuery Data Viewer** roles described above. + + {/* GENERATED:CONNECTION-OPTIONS START */} ## Configuration reference @@ -122,7 +143,7 @@ gcloud iam service-accounts keys create /evidence-bq-primary.js #### Credentials - _Provide exactly one of `keyfile_json`, `keyfile`._ + _Provide exactly one of `keyfile_json`, `keyfile`, `adc`._ GCP project that owns the BigQuery datasets you want to query. @@ -133,6 +154,9 @@ gcloud iam service-accounts keys create /evidence-bq-primary.js Path to a service-account key JSON file, resolved relative to connection.yaml. + + Use Application Default Credentials (gcloud auth application-default login, GOOGLE_APPLICATION_CREDENTIALS, or the GCP metadata server). + Default query location (e.g. US, EU, us-central1). @@ -310,4 +334,4 @@ Enable [BigQuery audit logs](https://cloud.google.com/bigquery/docs/reference/au - `EXPORT DATA` jobs (data leaving the warehouse) - DDL or DCL statements run by Evidence's SAs (they shouldn't be running any) -- Unusually large scans by a single SA \ No newline at end of file +- Unusually large scans by a single SA