Skip to content

Commit

Permalink
fix(Google BigQuery Node): Better error messages, transform timestamps (
Browse files Browse the repository at this point in the history
#9255)

Co-authored-by: Jonathan Bennetts <jonathan.bennetts@gmail.com>
  • Loading branch information
elsmr and Joffcom authored May 2, 2024
1 parent e896889 commit 7ff24f1
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ export class GoogleBigQuery extends VersionedNodeType {
group: ['input'],
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
description: 'Consume Google BigQuery API',
defaultVersion: 2,
defaultVersion: 2.1,
};

const nodeVersions: IVersionedNodeType['nodeVersions'] = {
1: new GoogleBigQueryV1(baseDescription),
2: new GoogleBigQueryV2(baseDescription),
2.1: new GoogleBigQueryV2(baseDescription),
};

super(nodeVersions, baseDescription);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import type { IDataObject, IExecuteFunctions } from 'n8n-workflow';
import type { IDataObject, IExecuteFunctions, INode } from 'n8n-workflow';
import { constructExecutionMetaData } from 'n8n-core';
import { mock } from 'jest-mock-extended';
import { prepareOutput } from '../../../v2/helpers/utils';

describe('Google BigQuery v2 Utils', () => {
it('should prepareOutput', () => {
const thisArg = mock<IExecuteFunctions>({ helpers: mock({ constructExecutionMetaData }) });
const thisArg = mock<IExecuteFunctions>({
getNode: () => ({ typeVersion: 2.1 }) as INode,
helpers: mock({ constructExecutionMetaData }),
});
const response: IDataObject = {
kind: 'bigquery#getQueryResultsResponse',
etag: 'e_tag',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -249,10 +249,16 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
returnData.push(...executionErrorData);
continue;
}
if ((error.message as string).includes('location')) {
if ((error.message as string).includes('location') || error.httpCode === '404') {
error.description =
"Are you sure your table is in that region? You can specify the region using the 'Location' parameter from options.";
}

if (error.httpCode === '403' && error.message.includes('Drive')) {
error.description =
'If your table(s) pull from a document in Google Drive, make sure that document is shared with your user';
}

throw new NodeOperationError(this.getNode(), error as Error, {
itemIndex: i,
description: error.description,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export const versionDescription: INodeTypeDescription = {
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
version: 2,
version: [2, 2.1],
subtitle: '={{$parameter["operation"]}}',
description: 'Consume Google BigQuery API',
defaults: {
Expand Down
29 changes: 24 additions & 5 deletions packages/nodes-base/nodes/Google/BigQuery/v2/helpers/utils.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { DateTime } from 'luxon';
import type { IDataObject, IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { jsonParse, NodeOperationError } from 'n8n-workflow';
import type { SchemaField, TableRawData, TableSchema } from './interfaces';

function getFieldValue(schemaField: SchemaField, field: IDataObject) {
function getFieldValue(schemaField: SchemaField, field: IDataObject, parseTimestamps = false) {
if (schemaField.type === 'RECORD') {
// eslint-disable-next-line @typescript-eslint/no-use-before-define
return simplify([field.v as TableRawData], schemaField.fields as unknown as SchemaField[]);
Expand All @@ -12,6 +13,9 @@ function getFieldValue(schemaField: SchemaField, field: IDataObject) {
try {
value = jsonParse(value as string);
} catch (error) {}
} else if (schemaField.type === 'TIMESTAMP' && parseTimestamps) {
const dt = DateTime.fromSeconds(Number(value));
value = dt.isValid ? dt.toISO() : value;
}
return value;
}
Expand All @@ -26,18 +30,27 @@ export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[
}));
}

export function simplify(data: TableRawData[], schema: SchemaField[], includeSchema = false) {
export function simplify(
data: TableRawData[],
schema: SchemaField[],
includeSchema = false,
parseTimestamps = false,
) {
const returnData: IDataObject[] = [];
for (const entry of data) {
const record: IDataObject = {};

for (const [index, field] of entry.f.entries()) {
if (schema[index].mode !== 'REPEATED') {
record[schema[index].name] = getFieldValue(schema[index], field);
record[schema[index].name] = getFieldValue(schema[index], field, parseTimestamps);
} else {
record[schema[index].name] = (field.v as unknown as IDataObject[]).flatMap(
(repeatedField) => {
return getFieldValue(schema[index], repeatedField as unknown as IDataObject);
return getFieldValue(
schema[index],
repeatedField as unknown as IDataObject,
parseTimestamps,
);
},
);
}
Expand Down Expand Up @@ -68,12 +81,18 @@ export function prepareOutput(
responseData = response;
} else {
const { rows, schema } = response;
const parseTimestamps = this.getNode().typeVersion >= 2.1;

if (rows !== undefined && schema !== undefined) {
const fields = (schema as TableSchema).fields;
responseData = rows;

responseData = simplify(responseData as TableRawData[], fields, includeSchema);
responseData = simplify(
responseData as TableRawData[],
fields,
includeSchema,
parseTimestamps,
);
} else if (schema && includeSchema) {
responseData = { success: true, _schema: schema };
} else {
Expand Down

0 comments on commit 7ff24f1

Please sign in to comment.