From f41e3401654fa4f9bd371f87dca4d521ca1c4684 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 27 Mar 2024 12:34:18 -0400 Subject: [PATCH 1/3] chore: remove type any when possible --- src/bigquery.ts | 26 ++++++++++++++++---------- src/dataset.ts | 13 ++++++------- src/model.ts | 5 ++--- src/rowQueue.ts | 3 +-- src/table.ts | 44 +++++++++++++++++++++----------------------- 5 files changed, 46 insertions(+), 45 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 0df263dd..91d97dd1 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -1191,7 +1191,12 @@ export class BigQuery extends Service { ); } else if (typeName === 'STRUCT') { queryParameter.parameterValue!.structValues = Object.keys(value).reduce( - (structValues, prop) => { + ( + structValues: { + [key: string]: bigquery.IQueryParameterValue; + }, + prop: string + ) => { let nestedQueryParameter; if (providedType) { nestedQueryParameter = BigQuery.valueToQueryParameter_( @@ -1201,8 +1206,9 @@ export class BigQuery extends Service { } else { nestedQueryParameter = BigQuery.valueToQueryParameter_(value[prop]); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (structValues as any)[prop] = nestedQueryParameter.parameterValue; + if (nestedQueryParameter.parameterValue !== undefined) { + structValues[prop] = nestedQueryParameter.parameterValue; + } return structValues; }, {} @@ -1517,11 +1523,9 @@ export class BigQuery extends Service { delete query.params; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const reqOpts: any = { - configuration: { - query, - }, + const reqOpts: JobOptions = {}; + reqOpts.configuration = { + query, }; if (typeof query.jobTimeoutMs === 'number') { @@ -1792,7 +1796,7 @@ export class BigQuery extends Service { if (options.projectId) { reqOpts.projectId = options.projectId; } - this.request(reqOpts, (err, resp) => { + this.request(reqOpts, (err, resp: bigquery.IDatasetList) => { if (err) { callback!(err, null, null, resp); return; @@ -1806,7 +1810,6 @@ export class BigQuery extends Service { }); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any const datasets = (resp.datasets || []).map( (dataset: bigquery.IDataset) => { const dsOpts: DatasetOptions = { @@ -1954,6 +1957,9 @@ export class BigQuery extends Service { if (this.location) { options = extend({location: this.location}, options); } + if (this.projectId) { + options = extend({projectId: this.projectId}, options); + } return new Job(this, id, options); } diff --git a/src/dataset.ts b/src/dataset.ts index e2347436..50251d35 100644 --- a/src/dataset.ts +++ b/src/dataset.ts @@ -739,8 +739,7 @@ class Dataset extends ServiceObject { const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb; const body = Table.formatMetadata_(options as TableMetadata); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (body as any).tableReference = { + body.tableReference = { datasetId: this.id, projectId: this.projectId, tableId: id, @@ -1163,7 +1162,7 @@ class Dataset extends ServiceObject { uri: '/tables', qs: options, }, - (err, resp) => { + (err, resp: bigquery.ITableList) => { if (err) { callback!(err, null, null, resp); return; @@ -1176,10 +1175,10 @@ class Dataset extends ServiceObject { }); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const tables = (resp.tables || []).map((tableObject: any) => { - const table = this.table(tableObject.tableReference.tableId, { - location: tableObject.location, + const tables = (resp.tables || []).map(tableObject => { + const tableRef = tableObject.tableReference!; + const table = this.table(tableRef.tableId!, { + location: this.location, }); table.metadata = tableObject; return table; diff --git a/src/model.ts b/src/model.ts index ea0ea73a..bd478c05 100644 --- a/src/model.ts +++ b/src/model.ts @@ -26,7 +26,7 @@ import { RequestCallback, JobRequest, } from '.'; -import {JobMetadata} from './job'; +import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; // This is supposed to be a @google-cloud/storage `File` type. The storage npm @@ -424,8 +424,7 @@ class Model extends ServiceObject { } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { extract: extend(true, options, { sourceModel: { diff --git a/src/rowQueue.ts b/src/rowQueue.ts index 534921ce..55796b53 100644 --- a/src/rowQueue.ts +++ b/src/rowQueue.ts @@ -177,8 +177,7 @@ export class RowQueue { reason: error.reason, }; }), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - row: rows[(insertError as any).index], + row: rows[insertError.index], }; } ); diff --git a/src/table.ts b/src/table.ts index 4dfe0792..21ce62ea 100644 --- a/src/table.ts +++ b/src/table.ts @@ -50,7 +50,7 @@ import { } from '.'; import {GoogleErrorBody} from '@google-cloud/common/build/src/util'; import {Duplex, Writable} from 'stream'; -import {JobMetadata} from './job'; +import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; import {IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; @@ -923,8 +923,7 @@ class Table extends ServiceObject { const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { copy: extend(true, metadata, { destinationTable: { @@ -1045,8 +1044,7 @@ class Table extends ServiceObject { const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { copy: extend(true, metadata, { destinationTable: { @@ -1218,8 +1216,7 @@ class Table extends ServiceObject { delete options.gzip; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { extract: extend(true, options, { sourceTable: { @@ -1399,15 +1396,13 @@ class Table extends ServiceObject { return [jobResponse, jobResponse.metadata]; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { - configuration: { - load: { - destinationTable: { - projectId: this.dataset.projectId, - datasetId: this.dataset.id, - tableId: this.id, - }, + const body: JobOptions = {}; + body.configuration = { + load: { + destinationTable: { + projectId: this.dataset.projectId, + datasetId: this.dataset.id, + tableId: this.id, }, }, }; @@ -1438,7 +1433,9 @@ class Table extends ServiceObject { // to CSV. const format = FORMATS[path.extname(src.name).substr(1).toLowerCase()]; if (!metadata.sourceFormat && format) { - body.configuration.load.sourceFormat = format; + if (body.configuration && body.configuration.load) { + body.configuration.load.sourceFormat = format; + } } return 'gs://' + src.bucket.name + '/' + src.name; }), @@ -1550,10 +1547,11 @@ class Table extends ServiceObject { uri: `${this.bigQuery.apiEndpoint}/upload/bigquery/v2/projects/${this.dataset.projectId}/jobs`, }, }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (data: any) => { - const job = this.bigQuery.job(data.jobReference.jobId, { - location: data.jobReference.location, + (data: JobMetadata) => { + const jobRef = data.jobReference!; + const job = this.bigQuery.job(jobRef.jobId!, { + location: jobRef.location, + projectId: jobRef.projectId, }); job.metadata = data; dup.emit('job', job); @@ -2199,6 +2197,7 @@ class Table extends ServiceObject { const partialFailures = (resp.insertErrors || []).map( (insertError: GoogleErrorBody) => { + insertError.index; return { errors: insertError.errors!.map(error => { return { @@ -2206,8 +2205,7 @@ class Table extends ServiceObject { reason: error.reason, }; }), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - row: rows[(insertError as any).index], + row: rows[insertError.index], }; } ); From c6ae2374e20ae8ac44e558a0bb11d1dc957803d2 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 27 Mar 2024 12:37:58 -0400 Subject: [PATCH 2/3] fix: rollback googleerror index --- src/rowQueue.ts | 3 ++- src/table.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rowQueue.ts b/src/rowQueue.ts index 55796b53..534921ce 100644 --- a/src/rowQueue.ts +++ b/src/rowQueue.ts @@ -177,7 +177,8 @@ export class RowQueue { reason: error.reason, }; }), - row: rows[insertError.index], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + row: rows[(insertError as any).index], }; } ); diff --git a/src/table.ts b/src/table.ts index 21ce62ea..d78519ef 100644 --- a/src/table.ts +++ b/src/table.ts @@ -2197,7 +2197,6 @@ class Table extends ServiceObject { const partialFailures = (resp.insertErrors || []).map( (insertError: GoogleErrorBody) => { - insertError.index; return { errors: insertError.errors!.map(error => { return { @@ -2205,7 +2204,8 @@ class Table extends ServiceObject { reason: error.reason, }; }), - row: rows[insertError.index], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + row: rows[(insertError as any).index], }; } ); From 267cac2be30a868f1841792987a98b48a8076752 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 27 Mar 2024 13:48:40 -0400 Subject: [PATCH 3/3] fix: pass projectId by default for jobs and datasets --- src/bigquery.ts | 7 +++---- src/dataset.ts | 4 +--- test/bigquery.ts | 23 ++++++++++++++++++----- test/dataset.ts | 2 ++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index d55da7a7..8431ce1c 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -1935,7 +1935,8 @@ export class BigQuery extends Service { } const jobs = (resp.jobs || []).map((jobObject: bigquery.IJob) => { const job = this.job(jobObject.jobReference!.jobId!, { - location: jobObject.jobReference!.location!, + location: jobObject.jobReference?.location, + projectId: jobObject.jobReference?.projectId, }); job.metadata = jobObject!; return job; @@ -1965,9 +1966,7 @@ export class BigQuery extends Service { if (this.location) { options = extend({location: this.location}, options); } - if (this.projectId) { - options = extend({projectId: this.projectId}, options); - } + options = extend({projectId: this.projectId}, options); return new Job(this, id, options); } diff --git a/src/dataset.ts b/src/dataset.ts index 50251d35..3cb00202 100644 --- a/src/dataset.ts +++ b/src/dataset.ts @@ -1177,9 +1177,7 @@ class Dataset extends ServiceObject { const tables = (resp.tables || []).map(tableObject => { const tableRef = tableObject.tableReference!; - const table = this.table(tableRef.tableId!, { - location: this.location, - }); + const table = this.table(tableRef.tableId!); table.metadata = tableObject; return table; }); diff --git a/test/bigquery.ts b/test/bigquery.ts index 8efa158d..032a8748 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -150,6 +150,7 @@ describe('BigQuery', () => { const PROJECT_ID = 'test-project'; const ANOTHER_PROJECT_ID = 'another-test-project'; const LOCATION = 'asia-northeast1'; + const ANOTHER_LOCATION = 'us-central1'; // eslint-disable-next-line @typescript-eslint/no-explicit-any let BigQueryCached: any; @@ -2667,6 +2668,7 @@ describe('BigQuery', () => { jobReference: { jobId: JOB_ID, location: LOCATION, + projectId: PROJECT_ID, }, }, ], @@ -2682,7 +2684,10 @@ describe('BigQuery', () => { assert(job instanceof FakeJob); assert.strictEqual(args[0], bq); assert.strictEqual(args[1], JOB_ID); - assert.deepStrictEqual(args[2], {location: LOCATION}); + assert.deepStrictEqual(args[2], { + location: LOCATION, + projectId: PROJECT_ID, + }); done(); }); }); @@ -2766,19 +2771,27 @@ describe('BigQuery', () => { it('should pass the options object', () => { const options = {a: 'b'}; + const expectedOptions = extend({projectId: PROJECT_ID}, options); const job = bq.job(JOB_ID, options); - assert.strictEqual(job.calledWith_[2], options); + assert.deepStrictEqual(job.calledWith_[2], expectedOptions); }); - it('should pass in the user specified location', () => { + it('should pass in the user specified location and project', () => { const bq = new BigQuery({ projectId: PROJECT_ID, location: LOCATION, }); - const options = {a: 'b'}; - const expectedOptions = Object.assign({location: LOCATION}, options); + const options = { + a: 'b', + location: ANOTHER_LOCATION, + projectId: ANOTHER_PROJECT_ID, + }; + const expectedOptions = Object.assign( + {location: ANOTHER_LOCATION, projectId: ANOTHER_PROJECT_ID}, + options + ); const job = bq.job(JOB_ID, options); const args = job.calledWith_; diff --git a/test/dataset.ts b/test/dataset.ts index 5d77e33d..c7187596 100644 --- a/test/dataset.ts +++ b/test/dataset.ts @@ -909,6 +909,7 @@ describe('BigQuery/Dataset', () => { }; beforeEach(() => { + ds = new Dataset(BIGQUERY, DATASET_ID, {location: LOCATION}); ds.request = (reqOpts: DecorateRequestOptions, callback: Function) => { callback(null, apiResponse); }; @@ -929,6 +930,7 @@ describe('BigQuery/Dataset', () => { assert(table instanceof Table); assert.strictEqual(table.id, tableId); assert.strictEqual(table.location, LOCATION); + assert.strictEqual(table.projectId, BIGQUERY.location); assert.strictEqual(apiResponse_, apiResponse); done(); }