Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/cli/connection/bigquery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ function configKey(c: BigQueryCredentials): string {
// PEM in a join key is wasteful.
return [
c.projectId,
c.serviceAccountJson.client_email,
c.authType === 'adc' ? 'adc' : c.serviceAccountJson.client_email,
c.location ?? '',
c.defaultDataset ?? ''
].join('|');
Expand Down
27 changes: 25 additions & 2 deletions cli/cli/connection/load-config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,35 @@ describe('loadConnectionConfig', () => {
).toBe(validKeyfileJson.client_email);
});

it('rejects when neither keyfile nor keyfile_json present', async () => {
it('rejects when no auth method is present', async () => {
await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\n`);
await expect(loadConnectionConfig(workDir)).rejects.toThrow(
/Provide one of: keyfile_json, keyfile/
/Provide one of: keyfile_json, keyfile, adc/
);
});

it('resolves adc: true without a key', async () => {
await writeYaml(`type: bigquery\nproject: p\nlocation: EU\ndatasets: [d]\nadc: true\n`);
const cfg = await loadConnectionConfig(workDir);
expect(cfg).toEqual({
type: 'bigquery',
authType: 'adc',
projectId: 'p',
location: 'EU',
defaultDataset: undefined,
datasets: ['d']
});
});

it('rejects adc combined with a keyfile', async () => {
await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\nadc: true\nkeyfile: ./sa.json\n`);
await expect(loadConnectionConfig(workDir)).rejects.toThrow(/Provide only one of/);
});

it('rejects adc: false', async () => {
await writeYaml(`type: bigquery\nproject: p\ndatasets: [d]\nadc: false\n`);
await expect(loadConnectionConfig(workDir)).rejects.toThrow();
});
});

describe('clickhouse', () => {
Expand Down
2 changes: 1 addition & 1 deletion cli/cli/init/connection-template.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ database: "<database>"
const BIGQUERY_TEMPLATE = `# BigQuery direct connector. Docs: https://docs.evidence.dev/direct-connectors/bigquery
type: bigquery
project: "<project-id>"
keyfile: ./service-account.json # or inline keyfile_json
keyfile: ./service-account.json # or inline keyfile_json, or adc: true
datasets: # accessible datasets, required (at least one)
- "<dataset>"
# location: US # default query location, optional
Expand Down
5 changes: 5 additions & 0 deletions cli/cli/launch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,11 @@ async function maybeUploadCredentials(
// No connection.yaml → Evidence-managed; nothing to upload.
return;
}
if (config.type === 'bigquery' && config.authType === 'adc') {
console.log(" • connection.yaml uses ADC, which can't be uploaded.");
console.log(' Add a service-account key in Studio → Settings → Warehouse.');
return;
}

// Uploading writes org-wide warehouse settings, so require an explicit opt-in:
// a confirm in a TTY, or `--upload-credentials` in a non-interactive run. Never
Expand Down
9 changes: 9 additions & 0 deletions core/src/connectors/bigquery/client-options.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,15 @@ describe('buildBigQueryClientOptions', () => {
});
expect(opts.location).toBe('US');
});

it('omits credentials for ADC so the SDK resolves them', () => {
const opts = buildBigQueryClientOptions({
authType: 'adc',
projectId: 'my-proj',
location: 'EU'
});
expect(opts).toEqual({ projectId: 'my-proj', location: 'EU' });
});
});

describe('normalizeCredentials', () => {
Expand Down
18 changes: 8 additions & 10 deletions core/src/connectors/bigquery/client-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,20 @@ import type { BigQueryCredentials } from './credentials';
*/
export type BigQueryClientOptions = {
projectId: string;
credentials: { client_email: string; private_key: string };
/** Omitted for ADC; the SDK then resolves Application Default Credentials. */
credentials?: { client_email: string; private_key: string };
location?: string;
};

/**
* Build the options object passed to `new BigQuery(...)`. Service-account JSON
* is the only auth path supported in v1.
*/
/** Build the options object passed to `new BigQuery(...)`. */
export function buildBigQueryClientOptions(
credentials: BigQueryCredentials
): BigQueryClientOptions {
const { client_email, private_key } = credentials.serviceAccountJson;
const opts: BigQueryClientOptions = {
projectId: credentials.projectId,
credentials: { client_email, private_key }
};
const opts: BigQueryClientOptions = { projectId: credentials.projectId };
if (credentials.authType === 'service_account_json') {
const { client_email, private_key } = credentials.serviceAccountJson;
opts.credentials = { client_email, private_key };
}
if (credentials.location !== undefined) {
opts.location = credentials.location;
}
Expand Down
14 changes: 14 additions & 0 deletions core/src/connectors/bigquery/connection-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,20 @@ export const bigqueryBase = z.object({
})
),

adc: z
.literal(true)
.optional()
.meta(
meta({
label: 'Application Default Credentials',
description:
'Use Application Default Credentials (gcloud auth application-default login, GOOGLE_APPLICATION_CREDENTIALS, or the GCP metadata server).',
category: 'credential',
cliOnly: true,
authGroup: 'bigquery-auth'
})
),

location: z
.string()
.optional()
Expand Down
10 changes: 7 additions & 3 deletions core/src/connectors/bigquery/credentials.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export type BigQueryAuthType = 'service_account_json';
export type BigQueryAuthType = 'service_account_json' | 'adc';

export type BigQueryServiceAccountJson = {
client_email: string;
Expand All @@ -18,15 +18,19 @@ export type BigQueryServiceAccountCredentials = BigQueryConnectionParams & {
serviceAccountJson: BigQueryServiceAccountJson;
};

export type BigQueryCredentials = BigQueryServiceAccountCredentials;
export type BigQueryAdcCredentials = BigQueryConnectionParams & {
authType: 'adc';
};

export type BigQueryCredentials = BigQueryServiceAccountCredentials | BigQueryAdcCredentials;

/**
* Coerce raw vault payload into BigQueryCredentials.
* The Vault stores arbitrary JSON; we trust the Studio config flow to write
* the right shape but assert the load-bearing keys here so a corrupted secret
* fails with a readable error rather than a downstream SDK error.
*/
export function normalizeCredentials(raw: unknown): BigQueryCredentials {
export function normalizeCredentials(raw: unknown): BigQueryServiceAccountCredentials {
if (raw === null || raw === undefined || typeof raw !== 'object') {
throw new Error('BigQuery credentials are missing or invalid');
}
Expand Down
20 changes: 12 additions & 8 deletions core/src/connectors/bigquery/resolve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@ export async function resolveBigQueryCredentials(
config: BigQueryConnection,
opts: ResolveOpts
): Promise<BigQueryCredentials> {
const params = {
projectId: config.project,
location: config.location,
defaultDataset: config.dataset
};

if (config.adc) {
return { authType: 'adc', ...params };
}

let serviceAccountJson: BigQueryServiceAccountJson;

if (config.keyfile_json) {
Expand All @@ -37,14 +47,8 @@ export async function resolveBigQueryCredentials(
}
} else {
// Schema's auth-group check should have caught this — defensive.
throw new Error('BigQuery credentials are missing keyfile_json and keyfile');
throw new Error('BigQuery credentials are missing keyfile_json, keyfile and adc');
}

return {
authType: 'service_account_json',
projectId: config.project,
serviceAccountJson,
location: config.location,
defaultDataset: config.dataset
};
return { authType: 'service_account_json', serviceAccountJson, ...params };
}
2 changes: 1 addition & 1 deletion docs/cli/connections.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ Fill in the placeholders with your own connection details. If `--warehouse` is o
# location: US
```

Provide exactly one of `keyfile` or `keyfile_json`. The `keyfile` path is resolved relative to `connection.yaml`.
Provide exactly one of `keyfile`, `keyfile_json`, or `adc: true`. The `keyfile` path is resolved relative to `connection.yaml`. With `adc: true`, Evidence uses [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), such as your `gcloud auth application-default login` session.

See the [BigQuery direct connector](/direct-connectors/bigquery) for the full field reference and the service-account setup steps.
</Tab>
Expand Down
28 changes: 26 additions & 2 deletions docs/direct-connectors/bigquery.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,27 @@ gcloud iam service-accounts keys create <some/local/path>/evidence-bq-primary.js
</Step>
</Steps>

### Application Default Credentials (CLI and self-hosted)

When running the CLI or a self-hosted `evidence serve`, you can skip the key file and use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) instead:

```yaml
type: bigquery
project: my-gcp-project
adc: true
datasets:
- analytics
```

Evidence then authenticates with, in order:

1. The key file at `GOOGLE_APPLICATION_CREDENTIALS`.
2. Your own login from `gcloud auth application-default login`.
3. The attached service account when running on GCP (Cloud Run, GCE, GKE).

The identity needs the same **BigQuery Job User** and **BigQuery Data Viewer** roles described above.


Comment on lines +73 to +93

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Very open to feedback regarding how the docs should discuss this, especially given that ADC is not supported on the Studio side.

{/* GENERATED:CONNECTION-OPTIONS START */}

## Configuration reference
Expand Down Expand Up @@ -122,7 +143,7 @@ gcloud iam service-accounts keys create <some/local/path>/evidence-bq-primary.js

#### Credentials

_Provide exactly one of `keyfile_json`, `keyfile`._
_Provide exactly one of `keyfile_json`, `keyfile`, `adc`._

<ResponseField name="project" type="string" required>
GCP project that owns the BigQuery datasets you want to query.
Expand All @@ -133,6 +154,9 @@ gcloud iam service-accounts keys create <some/local/path>/evidence-bq-primary.js
<ResponseField name="keyfile" type="string">
Path to a service-account key JSON file, resolved relative to connection.yaml.
</ResponseField>
<ResponseField name="adc" type="true">
Use Application Default Credentials (gcloud auth application-default login, GOOGLE_APPLICATION_CREDENTIALS, or the GCP metadata server).
</ResponseField>
<ResponseField name="location" type="string">
Default query location (e.g. US, EU, us-central1).
</ResponseField>
Expand Down Expand Up @@ -310,4 +334,4 @@ Enable [BigQuery audit logs](https://cloud.google.com/bigquery/docs/reference/au

- `EXPORT DATA` jobs (data leaving the warehouse)
- DDL or DCL statements run by Evidence's SAs (they shouldn't be running any)
- Unusually large scans by a single SA
- Unusually large scans by a single SA