From 1a14da942ac82988efe861aaeec104b3cce8ffb9 Mon Sep 17 00:00:00 2001 From: ricardo Date: Wed, 7 Apr 2021 21:50:31 -0400 Subject: [PATCH 1/5] :sparkles: Google BigQuery node --- .../GoogleBigQueryOAuth2Api.credentials.ts | 25 ++ .../nodes/Google/BigQuery/GenericFunctions.ts | 79 ++++ .../Google/BigQuery/GoogleBigQuery.node.ts | 221 ++++++++++++ .../Google/BigQuery/RecordDescription.ts | 338 ++++++++++++++++++ .../nodes/Google/BigQuery/googleBigQuery.svg | 1 + packages/nodes-base/package.json | 2 + 6 files changed, 666 insertions(+) create mode 100644 packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts create mode 100644 packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts create mode 100644 packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts create mode 100644 packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts create mode 100644 packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg diff --git a/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts b/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts new file mode 100644 index 0000000000000..5349ceebba7f8 --- /dev/null +++ b/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts @@ -0,0 +1,25 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + +const scopes = [ + 'https://www.googleapis.com/auth/bigquery', +]; + +export class GoogleBigQueryOAuth2Api implements ICredentialType { + name = 'googleBigQueryOAuth2Api'; + extends = [ + 'googleOAuth2Api', + ]; + displayName = 'Google BigQuery OAuth2 API'; + documentationUrl = 'google'; + properties = [ + { + displayName: 'Scope', + name: 'scope', + type: 'hidden' as NodePropertyTypes, + default: scopes.join(' '), + }, + ]; +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts new file mode 100644 index 0000000000000..828f625f2e0f3 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts @@ -0,0 +1,79 @@ +import { + OptionsWithUri, +} from 'request'; + +import { + IExecuteFunctions, + IExecuteSingleFunctions, + ILoadOptionsFunctions, +} from 'n8n-core'; + +import { + IDataObject, +} from 'n8n-workflow'; + +export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise { // tslint:disable-line:no-any + const options: OptionsWithUri = { + headers: { + 'Content-Type': 'application/json', + }, + method, + body, + qs, + uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`, + json: true, + }; + try { + if (Object.keys(headers).length !== 0) { + options.headers = Object.assign({}, options.headers, headers); + } + if (Object.keys(body).length === 0) { + delete options.body; + } + //@ts-ignore + return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options); + } catch (error) { + if (error.response && error.response.body && error.response.body.error) { + + let errors = error.response.body.error.errors; + + errors = errors.map((e: IDataObject) => e.message); + // Try to return the error prettier + throw new Error( + `Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`, + ); + } + throw error; + } +} + +export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOptionsFunctions, propertyName: string, method: string, endpoint: string, body: any = {}, query: IDataObject = {}): Promise { // tslint:disable-line:no-any + + const returnData: IDataObject[] = []; + + let responseData; + query.maxResults = 100; + + do { + responseData = await googleApiRequest.call(this, method, endpoint, body, query); + query.pageToken = responseData['nextPageToken']; + returnData.push.apply(returnData, responseData[propertyName]); + } while ( + responseData['nextPageToken'] !== undefined && + responseData['nextPageToken'] !== '' + ); + + return returnData; +} + +export function simplify(rows: IDataObject[], fields: string[]) { + const results = []; + for (const row of rows) { + const record: IDataObject = {}; + for (const [index, field] of fields.entries()) { + record[field] = (row.f as IDataObject[])[index].v; + } + results.push(record); + } + return results; +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts new file mode 100644 index 0000000000000..3953b6e81af01 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -0,0 +1,221 @@ +import { + IExecuteFunctions, +} from 'n8n-core'; + +import { + IDataObject, + ILoadOptionsFunctions, + INodeExecutionData, + INodePropertyOptions, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { + googleApiRequest, + googleApiRequestAllItems, + simplify, +} from './GenericFunctions'; + +import { + recordFields, + recordOperations, +} from './RecordDescription'; + +import * as uuid from 'uuid'; + +export class GoogleBigQuery implements INodeType { + description: INodeTypeDescription = { + displayName: 'Google BigQuery', + name: 'googleBigQuery', + icon: 'file:googleBigQuery.svg', + group: ['input'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Consume Google BigQuery API.', + defaults: { + name: 'Google BigQuery', + color: '#3E87E4', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'googleBigQueryOAuth2Api', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + options: [ + { + name: 'Record', + value: 'record', + }, + ], + default: 'record', + description: 'The resource to operate on.', + }, + ...recordOperations, + ...recordFields, + ], + }; + + methods = { + loadOptions: { + async getProjects( + this: ILoadOptionsFunctions, + ): Promise { + const returnData: INodePropertyOptions[] = []; + const { projects } = await googleApiRequest.call( + this, + 'GET', + '/v2/projects', + ); + for (const project of projects) { + returnData.push({ + name: project.friendlyName as string, + value: project.id, + }); + } + return returnData; + }, + async getDatasets( + this: ILoadOptionsFunctions, + ): Promise { + const projectId = this.getCurrentNodeParameter('projectId'); + const returnData: INodePropertyOptions[] = []; + const { datasets } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets`, + ); + for (const dataset of datasets) { + returnData.push({ + name: dataset.datasetReference.datasetId as string, + value: dataset.datasetReference.datasetId, + }); + } + return returnData; + }, + async getTables( + this: ILoadOptionsFunctions, + ): Promise { + const projectId = this.getCurrentNodeParameter('projectId'); + const datasetId = this.getCurrentNodeParameter('datasetId'); + const returnData: INodePropertyOptions[] = []; + const { tables } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables`, + ); + for (const table of tables) { + returnData.push({ + name: table.tableReference.tableId as string, + value: table.tableReference.tableId, + }); + } + return returnData; + }, + }, + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: IDataObject[] = []; + const length = (items.length as unknown) as number; + const qs: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0) as string; + const operation = this.getNodeParameter('operation', 0) as string; + + if (resource === 'record') { + //https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll + if (operation === 'create') { + const projectId = this.getNodeParameter('projectId', 0) as string; + const datasetId = this.getNodeParameter('datasetId', 0) as string; + const tableId = this.getNodeParameter('tableId', 0) as string; + const rows: IDataObject[] = []; + const body: IDataObject = {}; + for (let i = 0; i < length; i++) { + const options = this.getNodeParameter('options', i) as IDataObject; + Object.assign(body, options); + if (body.traceId === undefined) { + body.traceId = uuid(); + } + const columns = this.getNodeParameter('columns', i) as string; + const columnList = columns.split(',').map(column => column.trim()); + const record: IDataObject = {}; + + for (const key of Object.keys(items[i].json)) { + if (columnList.includes(key)) { + record[`${key}`] = items[i].json[key]; + } + } + rows.push({ json: record }); + } + body.rows = rows; + responseData = await googleApiRequest.call( + this, + 'POST', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`, + body, + ); + returnData.push(responseData); + } + if (operation === 'getAll') { + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const projectId = this.getNodeParameter('projectId', 0) as string; + const datasetId = this.getNodeParameter('datasetId', 0) as string; + const tableId = this.getNodeParameter('tableId', 0) as string; + const simple = this.getNodeParameter('simple', 0) as boolean; + let fields; + if (simple === true) { + const { schema } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`, + {}, + ); + fields = schema.fields.map((field: IDataObject) => field.name); + } + + for (let i = 0; i < length; i++) { + const options = this.getNodeParameter('options', i) as IDataObject; + Object.assign(qs, options); + if (qs.useInt64Timestamp !== undefined) { + qs.formatOptions = { + useInt64Timestamp: qs.useInt64Timestamp, + }; + } + if (returnAll) { + responseData = await googleApiRequestAllItems.call( + this, + 'rows', + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData); + } else { + qs.maxResults = this.getNodeParameter('limit', i) as number; + responseData = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); + } + } + } + } + + return [this.helpers.returnJsonArray(returnData)]; + } +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts new file mode 100644 index 0000000000000..3e9301704a3c0 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -0,0 +1,338 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const recordOperations = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + displayOptions: { + show: { + resource: [ + 'record', + ], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a new record', + }, + { + name: 'Get All', + value: 'getAll', + description: 'Retrieve all records', + }, + ], + default: 'create', + description: 'The operation to perform.', + }, +] as INodeProperties[]; + +export const recordFields = [ + /* -------------------------------------------------------------------------- */ + /* record:create */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Project ID', + name: 'projectId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getProjects', + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Project ID', + }, + { + displayName: 'Dataset ID', + name: 'datasetId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getDatasets', + loadOptionsDependsOn: [ + 'projectId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Project ID', + }, + { + displayName: 'Table ID', + name: 'tableId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getTables', + loadOptionsDependsOn: [ + 'projectId', + 'datasetId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Table ID', + }, + { + displayName: 'Columns', + name: 'columns', + type: 'string', + displayOptions: { + show: { + resource: [ + 'record', + ], + operation: [ + 'create', + ], + }, + }, + default: '', + required: true, + placeholder: 'id,name,description', + description: 'Comma separated list of the properties which should used as columns for the new rows.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Options', + default: {}, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + options: [ + { + displayName: 'Ignore Unknown Values', + name: 'ignoreUnknownValues', + type: 'boolean', + default: false, + description: `Accept rows that contain values that do not match the schema. The unknown values are ignored`, + }, + { + displayName: 'Skip Invalid Rows', + name: 'skipInvalidRows', + type: 'boolean', + default: false, + description: `Insert all valid rows of a request, even if invalid rows exist`, + }, + { + displayName: 'Tempalte Suffix', + name: 'templateSuffix', + type: 'string', + default: '', + description: `If specified, treats the destination table as a base template, and inserts the rows into an instance table named "{destination}{templateSuffix}`, + }, + { + displayName: 'Trace ID', + name: 'traceId', + type: 'string', + default: '', + description: `Unique request trace id. Used for debugging purposes only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.`, + }, + ], + }, + /* -------------------------------------------------------------------------- */ + /* record:getAll */ + /* -------------------------------------------------------------------------- */ + { + displayName: 'Project ID', + name: 'projectId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getProjects', + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Project ID', + }, + { + displayName: 'Dataset ID', + name: 'datasetId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getDatasets', + loadOptionsDependsOn: [ + 'projectId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Project ID', + }, + { + displayName: 'Table ID', + name: 'tableId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getTables', + loadOptionsDependsOn: [ + 'projectId', + 'datasetId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'Table ID', + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, + { + displayName: 'Simple', + name: 'simple', + type: 'boolean', + displayOptions: { + show: { + resource: [ + 'record', + ], + operation: [ + 'getAll', + ], + }, + }, + default: true, + description: 'When set to true a simplify version of the response will be used else the raw data.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Options', + default: {}, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + options: [ + { + displayName: 'Fields', + name: 'selectedFields', + type: 'string', + default: '', + description: `Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f"`, + }, + { + displayName: 'Use Int64 Timestamp', + name: 'useInt64Timestamp', + type: 'boolean', + default: false, + description: `Output timestamp as usec int64. Default is false.`, + }, + ], + }, +] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg b/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg new file mode 100644 index 0000000000000..b006a05d6ab55 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index cb7e05155c8ac..87b9f9cda5588 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -95,6 +95,7 @@ "dist/credentials/GmailOAuth2Api.credentials.js", "dist/credentials/GoogleAnalyticsOAuth2Api.credentials.js", "dist/credentials/GoogleApi.credentials.js", + "dist/credentials/GoogleBigQueryOAuth2Api.credentials.js", "dist/credentials/GoogleBooksOAuth2Api.credentials.js", "dist/credentials/GoogleCalendarOAuth2Api.credentials.js", "dist/credentials/GoogleContactsOAuth2Api.credentials.js", @@ -361,6 +362,7 @@ "dist/nodes/Github/GithubTrigger.node.js", "dist/nodes/Gitlab/Gitlab.node.js", "dist/nodes/Gitlab/GitlabTrigger.node.js", + "dist/nodes/Google/BigQuery/GoogleBigQuery.node.js", "dist/nodes/Google/Books/GoogleBooks.node.js", "dist/nodes/Google/Analytics/GoogleAnalytics.node.js", "dist/nodes/Google/Calendar/GoogleCalendar.node.js", From 3311e6d13f84455e607cad687321bc2bb3360a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 9 Apr 2021 11:27:36 +0200 Subject: [PATCH 2/5] :hammer: Add comment dividers --- .../Google/BigQuery/GoogleBigQuery.node.ts | 20 ++++++++++++++++++- .../Google/BigQuery/RecordDescription.ts | 13 ++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts index 3953b6e81af01..135a56145b5d9 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -133,8 +133,19 @@ export class GoogleBigQuery implements INodeType { const operation = this.getNodeParameter('operation', 0) as string; if (resource === 'record') { - //https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll + + // ********************************************************************* + // record + // ********************************************************************* + if (operation === 'create') { + + // ---------------------------------- + // record: create + // ---------------------------------- + + // https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll + const projectId = this.getNodeParameter('projectId', 0) as string; const datasetId = this.getNodeParameter('datasetId', 0) as string; const tableId = this.getNodeParameter('tableId', 0) as string; @@ -167,6 +178,13 @@ export class GoogleBigQuery implements INodeType { returnData.push(responseData); } if (operation === 'getAll') { + + // ---------------------------------- + // record: getAll + // ---------------------------------- + + // https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; const projectId = this.getNodeParameter('projectId', 0) as string; const datasetId = this.getNodeParameter('datasetId', 0) as string; diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts index 3e9301704a3c0..e0ac50ceac3d9 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -32,9 +32,9 @@ export const recordOperations = [ ] as INodeProperties[]; export const recordFields = [ - /* -------------------------------------------------------------------------- */ - /* record:create */ - /* -------------------------------------------------------------------------- */ + // ---------------------------------- + // record: create + // ---------------------------------- { displayName: 'Project ID', name: 'projectId', @@ -171,9 +171,10 @@ export const recordFields = [ }, ], }, - /* -------------------------------------------------------------------------- */ - /* record:getAll */ - /* -------------------------------------------------------------------------- */ + + // ---------------------------------- + // record: getAll + // ---------------------------------- { displayName: 'Project ID', name: 'projectId', From 4a0531e66e55c05df1d5cd7f6f188fa6a20a4682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 9 Apr 2021 11:30:02 +0200 Subject: [PATCH 3/5] :zap: Add whitespace for readability --- .../nodes/Google/BigQuery/GoogleBigQuery.node.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts index 135a56145b5d9..8815fe437b89e 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -151,7 +151,9 @@ export class GoogleBigQuery implements INodeType { const tableId = this.getNodeParameter('tableId', 0) as string; const rows: IDataObject[] = []; const body: IDataObject = {}; + for (let i = 0; i < length; i++) { + const options = this.getNodeParameter('options', i) as IDataObject; Object.assign(body, options); if (body.traceId === undefined) { @@ -168,6 +170,7 @@ export class GoogleBigQuery implements INodeType { } rows.push({ json: record }); } + body.rows = rows; responseData = await googleApiRequest.call( this, @@ -176,8 +179,8 @@ export class GoogleBigQuery implements INodeType { body, ); returnData.push(responseData); - } - if (operation === 'getAll') { + + } else if (operation === 'getAll') { // ---------------------------------- // record: getAll @@ -191,6 +194,7 @@ export class GoogleBigQuery implements INodeType { const tableId = this.getNodeParameter('tableId', 0) as string; const simple = this.getNodeParameter('simple', 0) as boolean; let fields; + if (simple === true) { const { schema } = await googleApiRequest.call( this, @@ -204,11 +208,13 @@ export class GoogleBigQuery implements INodeType { for (let i = 0; i < length; i++) { const options = this.getNodeParameter('options', i) as IDataObject; Object.assign(qs, options); + if (qs.useInt64Timestamp !== undefined) { qs.formatOptions = { useInt64Timestamp: qs.useInt64Timestamp, }; } + if (returnAll) { responseData = await googleApiRequestAllItems.call( this, @@ -231,6 +237,7 @@ export class GoogleBigQuery implements INodeType { returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); } } + } } From eef7c609b4cb5d9b02e66f4fb371dad48f7ff532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 9 Apr 2021 11:41:40 +0200 Subject: [PATCH 4/5] :pencil2: Edit resource descriptions --- .../Google/BigQuery/RecordDescription.ts | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts index e0ac50ceac3d9..cb50ad9d5c308 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -18,16 +18,16 @@ export const recordOperations = [ { name: 'Create', value: 'create', - description: 'Create a new record', + description: 'Create a new record.', }, { name: 'Get All', value: 'getAll', - description: 'Retrieve all records', + description: 'Retrieve all records.', }, ], default: 'create', - description: 'The operation to perform.', + description: 'Operation to perform.', }, ] as INodeProperties[]; @@ -54,7 +54,7 @@ export const recordFields = [ }, }, default: '', - description: 'Project ID', + description: 'ID of the project to create the record in.', }, { displayName: 'Dataset ID', @@ -78,7 +78,7 @@ export const recordFields = [ }, }, default: '', - description: 'Project ID', + description: 'ID of the dataset to create the record in.', }, { displayName: 'Table ID', @@ -103,7 +103,7 @@ export const recordFields = [ }, }, default: '', - description: 'Table ID', + description: 'ID of the table to create the record in.', }, { displayName: 'Columns', @@ -122,7 +122,7 @@ export const recordFields = [ default: '', required: true, placeholder: 'id,name,description', - description: 'Comma separated list of the properties which should used as columns for the new rows.', + description: 'Comma-separated list of the item properties to use as columns.', }, { displayName: 'Options', @@ -146,28 +146,28 @@ export const recordFields = [ name: 'ignoreUnknownValues', type: 'boolean', default: false, - description: `Accept rows that contain values that do not match the schema. The unknown values are ignored`, + description: 'Ignore row values that do not match the schema.', }, { displayName: 'Skip Invalid Rows', name: 'skipInvalidRows', type: 'boolean', default: false, - description: `Insert all valid rows of a request, even if invalid rows exist`, + description: 'Skip rows with values that do not match the schema.', }, { - displayName: 'Tempalte Suffix', + displayName: 'Template Suffix', name: 'templateSuffix', type: 'string', default: '', - description: `If specified, treats the destination table as a base template, and inserts the rows into an instance table named "{destination}{templateSuffix}`, + description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named {destinationTable}{templateSuffix}.', }, { displayName: 'Trace ID', name: 'traceId', type: 'string', default: '', - description: `Unique request trace id. Used for debugging purposes only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.`, + description: 'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.', }, ], }, @@ -194,7 +194,7 @@ export const recordFields = [ }, }, default: '', - description: 'Project ID', + description: 'ID of the project to retrieve all rows from.', }, { displayName: 'Dataset ID', @@ -218,7 +218,7 @@ export const recordFields = [ }, }, default: '', - description: 'Project ID', + description: 'ID of the dataset to retrieve all rows from.', }, { displayName: 'Table ID', @@ -243,7 +243,7 @@ export const recordFields = [ }, }, default: '', - description: 'Table ID', + description: 'ID of the table to retrieve all rows from.', }, { displayName: 'Return All', @@ -301,7 +301,7 @@ export const recordFields = [ }, }, default: true, - description: 'When set to true a simplify version of the response will be used else the raw data.', + description: 'Return a simplified version of the response instead of the raw data.', }, { displayName: 'Options', @@ -325,14 +325,14 @@ export const recordFields = [ name: 'selectedFields', type: 'string', default: '', - description: `Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f"`, + description: 'Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f".', }, { displayName: 'Use Int64 Timestamp', name: 'useInt64Timestamp', type: 'boolean', default: false, - description: `Output timestamp as usec int64. Default is false.`, + description: 'Output timestamp as usec int64.', }, ], }, From ad3871eb1e3434bedff5d9821957abebe2372e9c Mon Sep 17 00:00:00 2001 From: ricardo Date: Tue, 13 Apr 2021 20:07:17 -0400 Subject: [PATCH 5/5] :zap: Improvements --- .../nodes/Google/BigQuery/GenericFunctions.ts | 1 + .../nodes/Google/BigQuery/GoogleBigQuery.node.ts | 14 +++++++++----- .../nodes/Google/BigQuery/RecordDescription.ts | 14 +++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts index 828f625f2e0f3..b212c0c981524 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts @@ -30,6 +30,7 @@ export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleF if (Object.keys(body).length === 0) { delete options.body; } + console.log(options); //@ts-ignore return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options); } catch (error) { diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts index 8815fe437b89e..9a848cd58fdff 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -209,10 +209,15 @@ export class GoogleBigQuery implements INodeType { const options = this.getNodeParameter('options', i) as IDataObject; Object.assign(qs, options); - if (qs.useInt64Timestamp !== undefined) { - qs.formatOptions = { - useInt64Timestamp: qs.useInt64Timestamp, - }; + // if (qs.useInt64Timestamp !== undefined) { + // qs.formatOptions = { + // useInt64Timestamp: qs.useInt64Timestamp, + // }; + // delete qs.useInt64Timestamp; + // } + + if (qs.selectedFields) { + fields = (qs.selectedFields as string).split(','); } if (returnAll) { @@ -237,7 +242,6 @@ export class GoogleBigQuery implements INodeType { returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); } } - } } diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts index cb50ad9d5c308..3f490ee648482 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -327,13 +327,13 @@ export const recordFields = [ default: '', description: 'Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f".', }, - { - displayName: 'Use Int64 Timestamp', - name: 'useInt64Timestamp', - type: 'boolean', - default: false, - description: 'Output timestamp as usec int64.', - }, + // { + // displayName: 'Use Int64 Timestamp', + // name: 'useInt64Timestamp', + // type: 'boolean', + // default: false, + // description: 'Output timestamp as usec int64.', + // }, ], }, ] as INodeProperties[];