diff --git a/package.json b/package.json index cbfb27887..95e6c6e18 100644 --- a/package.json +++ b/package.json @@ -45,13 +45,16 @@ "test": "c8 mocha build/test" }, "dependencies": { + "big.js": "^7.0.1", "google-gax": "^5.0.0" }, "devDependencies": { + "@types/big.js": "^6.2.2", "@types/mocha": "^10.0.10", "@types/node": "^22.15.21", "@types/sinon": "^17.0.4", "c8": "^10.1.3", + "eslint-plugin-prettier": "^5.5.1", "gapic-tools": "^1.0.2", "gts": "^6.0.2", "jsdoc": "^4.0.4", diff --git a/src/index.ts b/src/index.ts index f86616dca..f92513e49 100644 --- a/src/index.ts +++ b/src/index.ts @@ -46,3 +46,4 @@ export { }; import * as protos from '../protos/protos'; export {protos}; +export * as query from './query'; diff --git a/src/query/builder.ts b/src/query/builder.ts new file mode 100644 index 000000000..d753fcf57 --- /dev/null +++ b/src/query/builder.ts @@ -0,0 +1,36 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {protos} from '../'; + +/** + * fromSQL creates a query configuration from a SQL string. + * @param {string} sql The SQL query. + * @returns {protos.google.cloud.bigquery.v2.IPostQueryRequest} + */ +export function fromSQL( + projectId: string, + sql: string, +): protos.google.cloud.bigquery.v2.IPostQueryRequest { + return { + queryRequest: { + query: sql, + useLegacySql: {value: false}, + formatOptions: { + useInt64Timestamp: true, + }, + }, + projectId, + }; +} diff --git a/src/query/client.ts b/src/query/client.ts new file mode 100644 index 000000000..e658263ab --- /dev/null +++ b/src/query/client.ts @@ -0,0 +1,172 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + BigQueryClient, + BigQueryClientOptions, +} from '../bigquery'; +import {QueryJob, CallOptions} from './job'; +import {protos} from '../'; +import {fromSQL as builderFromSQL} from './builder'; + +/** + * QueryClient is a client for running queries in BigQuery. + */ +export class QueryClient { + private client: BigQueryClient; + projectId: string; + private billingProjectId: string; + + /** + * @param {BigQueryClientOptions} options - The configuration object. + */ + constructor( + options?: BigQueryClientOptions, + ) { + this.client = new BigQueryClient(options); + this.projectId = ''; + this.billingProjectId = ''; + void this.initialize(); + } + + async getProjectId(): Promise { + if (this.projectId) { + return this.projectId; + } + const {jobClient} = this.getBigQueryClient(); + const projectId = await jobClient.getProjectId(); + this.projectId = projectId; + return projectId; + } + /** + * Initialize the client. + * Performs asynchronous operations (such as authentication) and prepares the client. + * This function will be called automatically when any class method is called for the + * first time, but if you need to initialize it before calling an actual method, + * feel free to call initialize() directly. + * + * You can await on this method if you want to make sure the client is initialized. + * + * @returns {Promise} A promise that resolves when auth is complete. + */ + initialize = async (): Promise => { + if (this.projectId) { + return; + } + const {jobClient} = this.getBigQueryClient(); + await jobClient.initialize(); + const projectId = await this.getProjectId(); + this.projectId = projectId; + if (this.billingProjectId !== '') { + this.billingProjectId = projectId; + } + }; + + setBillingProjectId(projectId: string) { + this.billingProjectId = projectId; + } + + /** + * fromSQL creates a query configuration from a SQL string. + * @param {string} sql The SQL query. + * @returns {protos.google.cloud.bigquery.v2.IPostQueryRequest} + */ + fromSQL(sql: string): protos.google.cloud.bigquery.v2.IPostQueryRequest { + const req = builderFromSQL(this.projectId, sql); + return req; + } + + /** + * Runs a query and returns a QueryJob handle. + * + * @param {protos.google.cloud.bigquery.v2.IPostQueryRequest} request + * The request object that will be sent. + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + * @returns {Promise} + */ + async startQuery( + request: protos.google.cloud.bigquery.v2.IPostQueryRequest, + options?: CallOptions, + ): Promise { + const [response] = await this.client.jobClient.query(request, options); + return new QueryJob(this, response); + } + + /** + * Runs a query and returns a QueryJob handle. + * + * @param {protos.google.cloud.bigquery.v2.IQueryRequest} request + * The request object that will be sent. + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + * @returns {Promise} + */ + async startQueryRequest( + request: protos.google.cloud.bigquery.v2.IQueryRequest, + options?: CallOptions, + ): Promise { + return this.startQuery( + { + queryRequest: request, + projectId: this.projectId, + }, + options, + ); + } + + /** + * Starts a new asynchronous job. + * + * @param {protos.google.cloud.bigquery.v2.IJob} job + * A job resource to insert + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + * @returns {Promise} + */ + async startQueryJob( + job: protos.google.cloud.bigquery.v2.IJob, + options?: CallOptions, + ): Promise { + const [response] = await this.client.jobClient.insertJob( + { + projectId: this.projectId, + job, + }, + options, + ); + return new QueryJob(this, {jobReference: response.jobReference}); + } + + /** + * Create a managed QueryJob from a job reference. + * + * @param {protos.google.cloud.bigquery.v2.IJob} job + * A job resource to insert + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + * @returns {Promise} + */ + async attachJob( + jobRef: protos.google.cloud.bigquery.v2.IJobReference, + ): Promise { + return new QueryJob(this, { + jobReference: jobRef, + }); + } + + getBigQueryClient(): BigQueryClient { + return this.client; + } +} diff --git a/src/query/format.ts b/src/query/format.ts new file mode 100644 index 000000000..420f15fc5 --- /dev/null +++ b/src/query/format.ts @@ -0,0 +1,47 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export function civilDateString(d: Date): string { + return d.toISOString().slice(0, 10); +} + +export function civilTimeString(value: string | Date): string { + if (value instanceof Date) { + const h = `${value.getHours()}`.padStart(2, '0'); + const m = `${value.getMinutes()}`.padStart(2, '0'); + const s = `${value.getSeconds()}`.padStart(2, '0'); + const f = `${value.getMilliseconds() * 1000}`.padStart(6, '0'); + return `${h}:${m}:${s}.${f}`; + } + return value; +} + +export function civilDateTimeString(value: Date | string): string { + if (value instanceof Date) { + let time; + if (value.getHours()) { + time = civilTimeString(value); + } + const y = `${value.getFullYear()}`.padStart(2, '0'); + const m = `${value.getMonth() + 1}`.padStart(2, '0'); + const d = `${value.getDate()}`.padStart(2, '0'); + time = time ? 'T' + time : ''; + return `${y}-${m}-${d}${time}`; + } + return value.replace(/^(.*)T(.*)Z$/, '$1 $2'); +} + +export function timestampString(ts: Date): string { + return ts.toISOString().replace('T', ' ').replace('Z', ''); +} diff --git a/src/query/index.ts b/src/query/index.ts new file mode 100644 index 000000000..d63621472 --- /dev/null +++ b/src/query/index.ts @@ -0,0 +1,19 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export {QueryClient} from './client'; +export {QueryJob} from './job'; +export {Row} from './row'; +export {RowIterator} from './iterator'; +export {fromSQL} from './builder'; diff --git a/src/query/iterator.ts b/src/query/iterator.ts new file mode 100644 index 000000000..d94230e29 --- /dev/null +++ b/src/query/iterator.ts @@ -0,0 +1,60 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {QueryJob} from './job'; +import {Row} from './row'; + +/** + * RowIterator iterates over the results of a query. + */ +export class RowIterator { + private job: QueryJob; + private pageToken?: string; + private rows: Row[] = []; + + constructor( + job: QueryJob, + opts?: { + rows?: Row[]; + pageToken?: string; + }, + ) { + this.job = job; + this.pageToken = opts?.pageToken; + this.rows = opts?.rows ?? []; + } + + async fetchRows() { + const [rows, _, pageToken] = await this.job._getRows(this.pageToken); + this.rows = rows; + this.pageToken = pageToken || undefined; + } + + /** + * Asynchronously iterates over the rows in the query result. + */ + async *[Symbol.asyncIterator](): AsyncGenerator { + if (this.rows.length > 0) { + for (const row of this.rows) { + yield row; + } + } + while (this.pageToken) { + await this.fetchRows(); + for (const row of this.rows) { + yield row; + } + } + } +} diff --git a/src/query/job.ts b/src/query/job.ts new file mode 100644 index 000000000..7ca175c6a --- /dev/null +++ b/src/query/job.ts @@ -0,0 +1,205 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {QueryClient} from './client'; +import {protos} from '../'; +import {RowIterator} from './iterator'; +import {Row} from './row'; +import {Schema} from './schema'; +import {convertRows} from './value'; +import {CallOptions as GaxCallOptions} from 'google-gax'; + +export interface CallOptions extends GaxCallOptions { + signal?: AbortSignal; +} + +/** + * Query represents a query job. + */ +export class QueryJob { + private client: QueryClient; + private jobComplete: boolean; + private projectId: string; + private jobId: string; + private location: string; + + private cachedRows: Row[]; + private cachedSchema: protos.google.cloud.bigquery.v2.ITableSchema; + private cachedPageToken: string; + private cachedTotalRows: number; + + constructor( + client: QueryClient, + response: protos.google.cloud.bigquery.v2.IQueryResponse, + ) { + this.client = client; + + this.cachedRows = []; + this.cachedSchema = {}; + this.cachedPageToken = ''; + this.cachedTotalRows = 0; + this.jobComplete = false; + + this.consumeQueryResponse({ + jobComplete: response.jobComplete, + schema: response.schema, + pageToken: response.pageToken, + totalRows: response.totalRows, + rows: response.rows, + }); + + this.jobId = ''; + this.location = response.location ?? ''; + this.projectId = ''; + if (response.jobReference) { + this.projectId = response.jobReference.projectId!; + this.jobId = response.jobReference.jobId!; + this.location = response.jobReference.location?.value || ''; + } + if (response.queryId) { + this.jobId = response.queryId; + } + } + + jobReference(): protos.google.cloud.bigquery.v2.IJobReference { + return { + jobId: this.jobId, + projectId: this.projectId, + location: {value: this.location}, + }; + } + + get schema(): protos.google.cloud.bigquery.v2.ITableSchema { + return this.cachedSchema; + } + + get complete(): boolean { + return this.jobComplete + } + + /** + * Waits for the query to complete. + * + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + */ + async wait(options?: CallOptions): Promise { + const signal = options?.signal; + while (!this.complete) { + if (signal?.aborted) { + throw new Error('The operation was aborted.'); + } + await this.checkStatus(options); + if (!this.complete) { + await this.waitFor(signal); + } + } + } + + /** + * Cancel a running query. + * + * @param {CallOptions} [options] + * Call options. See {@link https://googleapis.dev/nodejs/google-gax/latest/interfaces/CallOptions.html|CallOptions} for more details. + */ + async cancel(options?: CallOptions): Promise { + const {jobClient} = this.client.getBigQueryClient(); + const [response] = await jobClient.cancelJob({ + projectId: this.projectId, + jobId: this.jobId, + location: this.location, + }, options); + return response; + } + + private async waitFor(signal?: AbortSignal): Promise { + const delay = 1000; // TODO: backoff settings + return new Promise((resolve, reject) => { + const timeout = setTimeout(resolve, delay); + signal?.addEventListener('abort', () => { + clearTimeout(timeout); + reject(new Error('The operation was aborted.')); + }); + }); + } + + /** + * Returns a RowIterator for the query results. + * + * @returns {RowIterator} + */ + async read(): Promise { + const it = new RowIterator(this, { + pageToken: this.cachedPageToken, + rows: this.cachedRows, + }); + if (this.cachedRows.length === 0) { + await it.fetchRows(); + } + return it; + } + + /** + * @internal + */ + async _getRows( + pageToken?: string, + ): Promise< + [Row[], protos.google.cloud.bigquery.v2.ITableSchema | null, string | null] + > { + const {jobClient} = this.client.getBigQueryClient(); + const [response] = await jobClient.getQueryResults({ + projectId: this.projectId, + jobId: this.jobId, + location: this.location, + pageToken, + formatOptions: { + useInt64Timestamp: true, + }, + }); + + const rows = convertRows(response.rows || [], new Schema(response.schema!)); + return [rows, response.schema || null, response.pageToken || null]; + } + + private consumeQueryResponse( + response: protos.google.cloud.bigquery.v2.IGetQueryResultsResponse, + ) { + this.jobComplete = response.jobComplete?.value ?? false; + this.cachedSchema = response.schema!; + this.cachedPageToken = response.pageToken!; + this.cachedTotalRows = Number(response.totalRows); + this.cachedRows = convertRows( + response.rows || [], + new Schema(this.cachedSchema), + ); + } + + private async checkStatus(options?: CallOptions): Promise { + const {jobClient} = this.client.getBigQueryClient(); + const [response] = await jobClient.getQueryResults( + { + projectId: this.projectId, + jobId: this.jobId, + location: this.location, + maxResults: {value: 0}, + formatOptions: { + useInt64Timestamp: true, + }, + }, + options, + ); + this.consumeQueryResponse(response); + } +} diff --git a/src/query/row.ts b/src/query/row.ts new file mode 100644 index 000000000..fc0568e05 --- /dev/null +++ b/src/query/row.ts @@ -0,0 +1,126 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {protos} from '../'; +import {Schema} from './schema'; +import {Value} from './value'; + +/** + * Represents a row in a query result. + */ +export class Row { + private schema: Schema; + private value: protos.google.protobuf.Struct; + + constructor(schema: Schema) { + this.schema = schema; + this.value = protos.google.protobuf.Struct.create({ + fields: this.schema.pb.fields?.reduce( + (fields, f) => { + fields[f.name!] = {}; + return fields; + }, + {} as {[key: string]: protos.google.protobuf.IValue}, + ), + }); + } + + set(columnName: string, value: Value) { + if (this.value.fields) { + this.value.fields[columnName] = value; + } + } + + /** + * toJSON returns the row as a JSON object. + */ + toJSON(): {[key: string]: any} { + const value: {[key: string]: any} = {}; + for (const field of this.schema.pb.fields!) { + const fieldValue = this.value.fields[field.name!]; + value[field.name!] = this.fieldValueToJSON(field, fieldValue); + } + return value; + } + + private fieldValueToJSON( + field: protos.google.cloud.bigquery.v2.ITableFieldSchema, + value: protos.google.protobuf.IValue, + ): any { + if (value.structValue) { + const subrow = new Row(Schema.fromField(field)); + subrow.value = protos.google.protobuf.Struct.create(value.structValue); + return subrow.toJSON(); + } else if (value.listValue) { + const arr: any[] = []; + for (const row of value.listValue.values ?? []) { + const subvalue = this.fieldValueToJSON(field, row); + arr.push(subvalue); + } + return arr; + } else if (value.nullValue) { + return null; + } else if (value.numberValue) { + return value.numberValue; + } else if (value.boolValue) { + return value.boolValue; + } + return value.stringValue; + } + + /** + * toStruct returns the row as a protobuf Struct object. + */ + toStruct(): protos.google.protobuf.IStruct { + return this.value; + } + + /** + * toValues encodes the row into an array of Value. + */ + toValues(): any[] { + const values: any[] = []; + for (const field of this.schema.pb.fields!) { + const fieldValue = this.value.fields[field.name!]; + const value = this.fieldValueToValues(field, fieldValue); + values.push(value); + } + return values; + } + + private fieldValueToValues( + field: protos.google.cloud.bigquery.v2.ITableFieldSchema, + value: protos.google.protobuf.IValue, + ): any { + if (value.structValue) { + const subrow = new Row(Schema.fromField(field)); + subrow.value = protos.google.protobuf.Struct.create(value.structValue); + return subrow.toValues(); + } else if (value.listValue) { + const arr: any[] = []; + for (const row of value.listValue.values ?? []) { + const subvalue = this.fieldValueToValues(field, row); + arr.push(subvalue); + } + return arr; + } else if (value.nullValue) { + return null; + } else if (value.numberValue) { + return value.numberValue; + } else if (value.boolValue) { + return value.boolValue; + } + return value.stringValue; + } +} diff --git a/src/query/schema.ts b/src/query/schema.ts new file mode 100644 index 000000000..718efd37e --- /dev/null +++ b/src/query/schema.ts @@ -0,0 +1,31 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {protos} from '../'; + +export class Schema { + pb: protos.google.cloud.bigquery.v2.ITableSchema; + + constructor(pb: protos.google.cloud.bigquery.v2.ITableSchema) { + this.pb = pb; + } + + static fromField(field: protos.google.cloud.bigquery.v2.ITableFieldSchema) { + return new Schema({fields: field.fields}); + } + + get length() { + return this.pb.fields?.length || 0; + } +} diff --git a/src/query/value.ts b/src/query/value.ts new file mode 100644 index 000000000..80a3d899e --- /dev/null +++ b/src/query/value.ts @@ -0,0 +1,143 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {protos} from '../'; +import {Row} from './row'; +import {Schema} from './schema'; + +export type Value = protos.google.protobuf.IValue; + +export function convertRows( + rows: protos.google.protobuf.IStruct[], + schema: Schema, +): Row[] { + return rows.map(row => convertRow(row, schema)); +} + +function convertRow(row: protos.google.protobuf.IStruct, schema: Schema): Row { + const fields = getFieldList(row); + if (schema.length !== fields.length) { + throw new Error('Schema length does not match row length'); + } + const newRow = new Row(schema); + for (let i = 0; i < fields.length; i++) { + const cell = fields[i]; + const cellValue = getFieldValue(cell); + const fs = schema.pb.fields![i]; + const value = convertValue( + cellValue, + fs.type! as any, + Schema.fromField(fs), + ); + newRow.set(fs.name!, value); + } + return newRow; +} + +function convertValue( + val: protos.google.protobuf.IValue, + typ: string, + schema: Schema, +): Value { + if (val.nullValue !== undefined && val.nullValue !== null) { + return { + nullValue: 'NULL_VALUE', + }; + } + if (val.listValue) { + return convertRepeatedRecord(val.listValue, typ, schema); + } + if (val.structValue) { + return convertNestedRecord(val.structValue, schema); + } + if (val.stringValue) { + return convertBasicType(val.stringValue, typ); + } + throw new Error(`Got value ${val}; expected a value of type ${typ}`); +} + +function convertRepeatedRecord( + vals: protos.google.protobuf.IListValue, + typ: string, + schema: Schema, +): Value { + return { + listValue: { + values: vals.values!.map(cell => { + const val = getFieldValue(cell); + return convertValue(val, typ, schema); + }), + }, + }; +} + +function convertNestedRecord( + val: protos.google.protobuf.IStruct, + schema: Schema, +): Value { + const row = convertRow(val, schema); + return { + structValue: row.toStruct(), + }; +} + +function convertBasicType(val: string, typ: string): Value { + switch (typ) { + case 'STRING': + case 'GEOGRAPHY': + case 'JSON': + case 'TIMESTAMP': + case 'DATE': + case 'TIME': + case 'DATETIME': + case 'NUMERIC': + case 'BIGNUMERIC': + case 'INTERVAL': + return {stringValue: val}; + case 'BYTES': + return {stringValue: val}; + case 'INTEGER': + return {numberValue: parseInt(val, 10)}; + case 'FLOAT': + return {numberValue: parseFloat(val)}; + case 'BOOLEAN': + return {boolValue: val.toLowerCase() === 'true'}; + default: + throw new Error(`Unsupported type: ${typ}`); + } +} + +function getFieldList(row: protos.google.protobuf.IStruct): any[] { + const fieldValue = row.fields?.f; + if (!fieldValue) { + throw new Error('Missing fields in row'); + } + const fields = fieldValue.listValue; + if (!fields) { + throw new Error('Missing fields in row'); + } + return fields.values!; +} + +function getFieldValue(val: protos.google.protobuf.IValue): any { + const s = val.structValue; + if (!s) { + throw new Error('Missing value in a field row'); + } + const fieldValue = s.fields?.v; + if (!fieldValue) { + throw new Error('Missing value in a field row'); + } + return fieldValue; +} diff --git a/system-test/fixtures/query.ts b/system-test/fixtures/query.ts new file mode 100644 index 000000000..a25a04795 --- /dev/null +++ b/system-test/fixtures/query.ts @@ -0,0 +1,313 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + civilDateString, + civilDateTimeString, + civilTimeString, + timestampString, +} from '../../src/query/format'; +import {protos} from '../../src'; + +export interface QueryParameterTestCase { + name: string; + query: string; + parameters: protos.google.cloud.bigquery.v2.IQueryParameter[]; + wantRowJSON: {[key: string]: any}; + wantRowValues: any[]; +} + +export function queryParameterTestCases(): QueryParameterTestCase[] { + const d = new Date('2016-03-20'); + const tm = '15:04:05.003000'; // TODO use civil time type + const dtm = new Date('2016-03-20T15:04:05.003Z'); + const ts = new Date('2016-03-20T15:04:05Z'); + + return [ + { + name: 'Int64Param', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'INT64'}, + parameterValue: { + value: {value: '1'}, + }, + }, + ], + wantRowJSON: {f0_: 1}, + wantRowValues: [1], + }, + { + name: 'FloatParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'FLOAT64'}, + parameterValue: { + value: {value: '1.3'}, + }, + }, + ], + wantRowJSON: {f0_: 1.3}, + wantRowValues: [1.3], + }, + { + name: 'BoolParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'BOOL'}, + parameterValue: {value: {value: 'true'}}, + }, + ], + wantRowJSON: {f0_: true}, + wantRowValues: [true], + }, + { + name: 'StringParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'STRING'}, + parameterValue: {value: {value:'ABC'}}, + }, + ], + wantRowJSON: {f0_: 'ABC'}, + wantRowValues: ['ABC'], + }, + { + name: 'ByteParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'BYTES'}, + parameterValue: { + value: {value: Buffer.from('foo').toString('base64')}, + }, + }, + ], + wantRowJSON: {f0_: Buffer.from('foo').toString('base64')}, + wantRowValues: [Buffer.from('foo').toString('base64')], + }, + { + name: 'TimestampParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'TIMESTAMP'}, + parameterValue: {value: {value:timestampString(ts)}}, + }, + ], + wantRowJSON: {f0_: String(ts.valueOf() * 1000)}, + wantRowValues: [String(ts.valueOf() * 1000)], + }, + { + name: 'TimestampArrayParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: { + type: 'ARRAY', + arrayType: { + type: 'TIMESTAMP', + }, + }, + parameterValue: { + arrayValues: [{value: {value: timestampString(ts)}}, {value: {value: timestampString(ts)}}], + }, + }, + ], + wantRowJSON: {f0_: [String(ts.valueOf() * 1000), String(ts.valueOf() * 1000)]}, + wantRowValues: [[String(ts.valueOf() * 1000), String(ts.valueOf() * 1000)]], + }, + { + name: 'DatetimeParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'DATETIME'}, + parameterValue: {value: {value: civilDateTimeString(dtm)}}, + }, + ], + wantRowJSON: {f0_: civilDateTimeString(dtm)}, + wantRowValues: [civilDateTimeString(dtm)], + }, + { + name: 'DateParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'DATE'}, + parameterValue: {value: {value:civilDateString(d)}}, + }, + ], + wantRowJSON: {f0_: civilDateString(d)}, + wantRowValues: [civilDateString(d)], + }, + { + name: 'TimeParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'TIME'}, + parameterValue: {value: {value:civilTimeString(tm)}}, + }, + ], + wantRowJSON: {f0_: tm}, + wantRowValues: [tm], + }, + { + name: 'JsonParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: {type: 'JSON'}, + parameterValue: { + value: {value:'{"alpha":"beta"}'}, + }, + }, + ], + wantRowJSON: {f0_: '{"alpha":"beta"}'}, + wantRowValues: ['{"alpha":"beta"}'], + }, + { + name: 'NestedStructParam', + query: 'SELECT @val', + parameters: [ + { + name: 'val', + parameterType: { + type: 'STRUCT', + structTypes: [ + { + name: 'Datetime', + type: { + type: 'DATETIME', + }, + }, + { + name: 'StringArray', + type: { + type: 'ARRAY', + arrayType: { + type: 'STRING', + }, + }, + }, + { + name: 'SubStruct', + type: { + type: 'STRUCT', + structTypes: [ + { + name: 'String', + type: { + type: 'STRING', + }, + }, + ], + }, + }, + { + name: 'SubStructArray', + type: { + type: 'ARRAY', + arrayType: { + type: 'STRUCT', + structTypes: [ + { + name: 'String', + type: { + type: 'STRING', + }, + }, + ], + }, + }, + }, + ], + }, + parameterValue: { + structValues: { + Datetime: { + value: {value: civilDateTimeString(dtm)}, + }, + StringArray: { + arrayValues: [{value: {value: 'a'}}, {value: {value: 'b'}}], + }, + SubStruct: { + structValues: { + String: { + value: {value: 'c'}, + }, + }, + }, + SubStructArray: { + arrayValues: [ + { + structValues: { + String: { + value: {value: 'd'}, + }, + }, + }, + { + structValues: { + String: { + value: {value: 'e'}, + }, + }, + }, + ], + }, + }, + }, + }, + ], + wantRowJSON: { + f0_: { + Datetime: civilDateTimeString(dtm), + StringArray: ['a', 'b'], + SubStruct: { + String: 'c', + }, + SubStructArray: [ + { + String: 'd', + }, + { + String: 'e', + }, + ], + }, + }, + wantRowValues: [ + [civilDateTimeString(dtm), ['a', 'b'], ['c'], [['d'], ['e']]], + ], + }, + ]; +} + diff --git a/system-test/fixtures/transport.ts b/system-test/fixtures/transport.ts new file mode 100644 index 000000000..bc3f84257 --- /dev/null +++ b/system-test/fixtures/transport.ts @@ -0,0 +1,36 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe} from 'mocha'; +import {query} from '../../src'; +import { QueryClient } from '../../src/query'; + +export const describeWithBothTransports = (title: string, fn: (client: QueryClient) => void) => { + describe(title, () => { + describe("REST", () => { + const client = new query.QueryClient({ fallback: true }); + before(async () => { + await client.initialize(); + }); + fn(client); + }); + describe("GRPC", () => { + const client = new query.QueryClient({ fallback: false }); + before(async () => { + await client.initialize(); + }); + fn(client); + }); + }); +} \ No newline at end of file diff --git a/system-test/query/query.ts b/system-test/query/query.ts new file mode 100644 index 000000000..89e442017 --- /dev/null +++ b/system-test/query/query.ts @@ -0,0 +1,119 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import {it} from 'mocha'; + +import {protos} from '../../src'; +import {describeWithBothTransports} from '../fixtures/transport'; + +const sleep = (ms: number) => + new Promise(resolve => { + setTimeout(resolve, ms); + }); + +describeWithBothTransports('Run Query', client => { + let getQueryResultsSpy: sinon.SinonSpy; + + beforeEach(() => { + const {jobClient} = client.getBigQueryClient(); + getQueryResultsSpy = sinon.spy(jobClient, 'getQueryResults'); + }); + + afterEach(() => { + getQueryResultsSpy.restore(); + }); + + it('should run a stateless query', async () => { + const req = client.fromSQL( + 'SELECT CURRENT_TIMESTAMP() as foo, SESSION_USER() as bar', + ); + req.queryRequest!.jobCreationMode = + protos.google.cloud.bigquery.v2.QueryRequest.JobCreationMode.JOB_CREATION_OPTIONAL; + + const q = await client.startQuery(req); + await q.wait(); + + const it = await q.read(); + const rows = []; + for await (const row of it) { + rows.push(row.toJSON()); + } + assert.strictEqual(rows.length, 1); + }); + + it('should stop waiting for query to complete', async () => { + const req = client.fromSQL( + 'SELECT num FROM UNNEST(GENERATE_ARRAY(1,1000000)) as num', + ); + req.queryRequest!.useQueryCache = {value: false}; + req.queryRequest!.jobCreationMode = + protos.google.cloud.bigquery.v2.QueryRequest.JobCreationMode.JOB_CREATION_REQUIRED; + req.queryRequest!.timeoutMs = {value: 500}; + + const q = await client.startQuery(req); + const abortCtrl = new AbortController(); + q.wait({ + signal: abortCtrl.signal, + }).catch(err => { + assert(err, 'aborted'); + }); + await sleep(1000); + abortCtrl.abort(); + + assert(getQueryResultsSpy.callCount >= 1); + assert(getQueryResultsSpy.callCount <= 2); + }).timeout(5000); + + it('should read a query job without cache', async () => { + const req = client.fromSQL( + 'SELECT CURRENT_TIMESTAMP() as foo, SESSION_USER() as bar', + ); + req.queryRequest!.jobCreationMode = + protos.google.cloud.bigquery.v2.QueryRequest.JobCreationMode.JOB_CREATION_REQUIRED; + + let q = await client.startQuery(req); + await q.wait(); + + const jobRef = q.jobReference(); + q = await client.attachJob(jobRef); + + const it = await q.read(); + const rows = []; + for await (const row of it) { + rows.push(row.toJSON()); + } + assert.strictEqual(rows.length, 1); + }); + + it('should insert a query job', async () => { + const q = await client.startQueryJob({ + configuration: { + query: { + query: 'SELECT CURRENT_DATETIME() as foo, SESSION_USER() as bar', + useLegacySql: {value: false}, + }, + }, + }); + await q.wait(); + + const it = await q.read(); + const rows = []; + for await (const row of it) { + rows.push(row); + } + assert.strictEqual(rows.length, 1); + }); +}); diff --git a/system-test/query/value.ts b/system-test/query/value.ts new file mode 100644 index 000000000..57aa43f51 --- /dev/null +++ b/system-test/query/value.ts @@ -0,0 +1,66 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as assert from 'assert'; +import {describe, it} from 'mocha'; + +import {queryParameterTestCases} from '../fixtures/query'; +import {describeWithBothTransports} from '../fixtures/transport'; + +describeWithBothTransports('Read Query Values', client => { + describe('types', () => { + for (const tc of queryParameterTestCases()) { + it(tc.name, async () => { + const req = client.fromSQL(tc.query); + req.queryRequest!.queryParameters = tc.parameters; + + const q = await client.startQuery(req); + await q.wait(); + + const it = await q.read(); + const rows = []; + for await (const row of it) { + rows.push(row); + } + assert.deepStrictEqual(rows[0].toJSON(), tc.wantRowJSON); + assert.deepStrictEqual(rows[0].toValues(), tc.wantRowValues); + }); + } + }); + + it('should read nested objects', async () => { + const req = client.fromSQL( + "SELECT 40 as age, [STRUCT(STRUCT('1' as a, '2' as b) as object)] as nested", + ); + const q = await client.startQuery(req); + await q.wait(); + + const it = await q.read(); + const rows = []; + for await (const row of it) { + rows.push(row); + } + assert.deepStrictEqual(rows[0].toJSON(), { + age: 40, + nested: [ + { + object: { + a: '1', + b: '2', + }, + }, + ], + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 7ecdf9fc6..ba1dec360 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,11 +16,14 @@ "test/*.ts", "test/**/*.ts", "system-test/*.ts", + "system-test/**/*.ts", "src/**/*.json", "protos/protos.json", "benchmark/*.ts", "scripts/*.ts", "samples/**/*.json" - + ], + "exclude": [ + "system-test/fixtures/sample/**/*.ts" ] } \ No newline at end of file