From 0582700a1e7e153c22aa941f264058ebb71857ae Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Thu, 18 Apr 2024 06:36:51 +0300 Subject: [PATCH 1/2] :zap: util to convert js arrays to postgress fromat --- .../v2/actions/database/insert.operation.ts | 3 ++ .../v2/actions/database/update.operation.ts | 3 ++ .../v2/actions/database/upsert.operation.ts | 3 ++ .../nodes/Postgres/v2/helpers/utils.ts | 53 ++++++++++++++++++- 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts index 0dcdf23f3fe47..b1e1c4da30911 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts @@ -19,6 +19,7 @@ import { configureTableSchemaUpdater, getTableSchema, prepareItem, + convertArraysToPostgresFormat, replaceEmptyStringsByNulls, } from '../../helpers/utils'; @@ -224,6 +225,8 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + values.push(checkItemAgainstSchema(this.getNode(), item, tableSchema, i)); const outputColumns = this.getNodeParameter('options.outputColumns', i, ['*']) as string[]; diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts index 844806221aedd..9673eb14d64f5 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts @@ -21,6 +21,7 @@ import { doesRowExist, getTableSchema, prepareItem, + convertArraysToPostgresFormat, replaceEmptyStringsByNulls, } from '../../helpers/utils'; @@ -301,6 +302,8 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); let values: QueryValues = [schema, table]; diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts index 3a4b85d900f55..aadfb32620170 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts @@ -21,6 +21,7 @@ import { prepareItem, replaceEmptyStringsByNulls, configureTableSchemaUpdater, + convertArraysToPostgresFormat, } from '../../helpers/utils'; import { optionsCollection } from '../common.descriptions'; @@ -270,6 +271,8 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); let values: QueryValues = [schema, table]; diff --git a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts index 7367a62af4711..9245d9e76e8bf 100644 --- a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts +++ b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts @@ -5,7 +5,7 @@ import type { INodeExecutionData, INodePropertyOptions, } from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; +import { NodeOperationError, jsonParse } from 'n8n-workflow'; import { generatePairedItemData } from '../../../../utils/utilities'; import type { @@ -510,3 +510,54 @@ export const configureTableSchemaUpdater = (initialSchema: string, initialTable: return tableSchema; }; }; + +/** + * If postgress column type is array we need to convert it to fornmat that postgres understands, original object data would be modified + * @param data the object with keys representing column names and values + * @param schema table schema + * @param node INode + * @param itemIndex the index of the current item + */ +export const convertArraysToPostgresFormat = ( + data: IDataObject, + schema: ColumnInfo[], + node: INode, + itemIndex = 0, +) => { + for (const columnInfo of schema) { + //in case column type is array we need to convert it to fornmat that postgres understands + if (columnInfo.data_type.toUpperCase() === 'ARRAY') { + let columnValue = data[columnInfo.column_name]; + + if (typeof columnValue === 'string') { + columnValue = jsonParse(columnValue); + } + + if (Array.isArray(columnValue)) { + const arrayEntries = columnValue.map((entry) => { + if (typeof entry === 'number') { + return entry; + } + + if (typeof entry === 'object') { + entry = JSON.stringify(entry); + } + + //escape double quotes + return `"${entry.replace(/"/g, '\\"')}"`; + }); + + //wrap in {} instead of [] as postgres does and join with , + data[columnInfo.column_name] = `{${arrayEntries.join(',')}}`; + } else { + throw new NodeOperationError( + node, + `Column '${columnInfo.column_name}' has to be an array`, + { + itemIndex, + }, + ); + } + } + } +}; From dd3158befcb7eb0674409c563423de283af84916 Mon Sep 17 00:00:00 2001 From: Michael Kret Date: Thu, 18 Apr 2024 09:01:49 +0300 Subject: [PATCH 2/2] :zap: bool array support, new node version, test --- .../nodes/Postgres/Postgres.node.ts | 3 +- .../nodes/Postgres/test/v2/utils.test.ts | 77 +++++++++++++++++++ .../v2/actions/database/insert.operation.ts | 6 +- .../v2/actions/database/update.operation.ts | 6 +- .../v2/actions/database/upsert.operation.ts | 6 +- .../Postgres/v2/actions/versionDescription.ts | 2 +- .../nodes/Postgres/v2/helpers/interfaces.ts | 2 +- .../nodes/Postgres/v2/helpers/utils.ts | 27 ++++--- 8 files changed, 111 insertions(+), 18 deletions(-) diff --git a/packages/nodes-base/nodes/Postgres/Postgres.node.ts b/packages/nodes-base/nodes/Postgres/Postgres.node.ts index 103c8a82e3ac5..05e09dff4dfcb 100644 --- a/packages/nodes-base/nodes/Postgres/Postgres.node.ts +++ b/packages/nodes-base/nodes/Postgres/Postgres.node.ts @@ -11,7 +11,7 @@ export class Postgres extends VersionedNodeType { name: 'postgres', icon: 'file:postgres.svg', group: ['input'], - defaultVersion: 2.3, + defaultVersion: 2.4, description: 'Get, add and update data in Postgres', parameterPane: 'wide', }; @@ -22,6 +22,7 @@ export class Postgres extends VersionedNodeType { 2.1: new PostgresV2(baseDescription), 2.2: new PostgresV2(baseDescription), 2.3: new PostgresV2(baseDescription), + 2.4: new PostgresV2(baseDescription), }; super(nodeVersions, baseDescription); diff --git a/packages/nodes-base/nodes/Postgres/test/v2/utils.test.ts b/packages/nodes-base/nodes/Postgres/test/v2/utils.test.ts index 8a707d592b720..39cdaf16ec005 100644 --- a/packages/nodes-base/nodes/Postgres/test/v2/utils.test.ts +++ b/packages/nodes-base/nodes/Postgres/test/v2/utils.test.ts @@ -10,7 +10,9 @@ import { prepareItem, replaceEmptyStringsByNulls, wrapData, + convertArraysToPostgresFormat, } from '../../v2/helpers/utils'; +import type { ColumnInfo } from '../../v2/helpers/interfaces'; const node: INode = { id: '1', @@ -373,3 +375,78 @@ describe('Test PostgresV2, checkItemAgainstSchema', () => { } }); }); + +describe('Test PostgresV2, convertArraysToPostgresFormat', () => { + it('should convert js arrays to postgres format', () => { + const item = { + jsonb_array: [ + { + key: 'value44', + }, + ], + json_array: [ + { + key: 'value54', + }, + ], + int_array: [1, 2, 5], + text_array: ['one', 't"w"o'], + bool_array: [true, false], + }; + + const schema: ColumnInfo[] = [ + { + column_name: 'id', + data_type: 'integer', + is_nullable: 'NO', + udt_name: 'int4', + column_default: "nextval('test_data_array_id_seq'::regclass)", + }, + { + column_name: 'jsonb_array', + data_type: 'ARRAY', + is_nullable: 'YES', + udt_name: '_jsonb', + column_default: null, + }, + { + column_name: 'json_array', + data_type: 'ARRAY', + is_nullable: 'YES', + udt_name: '_json', + column_default: null, + }, + { + column_name: 'int_array', + data_type: 'ARRAY', + is_nullable: 'YES', + udt_name: '_int4', + column_default: null, + }, + { + column_name: 'bool_array', + data_type: 'ARRAY', + is_nullable: 'YES', + udt_name: '_bool', + column_default: null, + }, + { + column_name: 'text_array', + data_type: 'ARRAY', + is_nullable: 'YES', + udt_name: '_text', + column_default: null, + }, + ]; + + convertArraysToPostgresFormat(item, schema, node, 0); + + expect(item).toEqual({ + jsonb_array: '{"{\\"key\\":\\"value44\\"}"}', + json_array: '{"{\\"key\\":\\"value54\\"}"}', + int_array: '{1,2,5}', + text_array: '{"one","t\\"w\\"o"}', + bool_array: '{"true","false"}', + }); + }); +}); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts index b1e1c4da30911..13927797c7d4c 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/insert.operation.ts @@ -136,7 +136,7 @@ const properties: INodeProperties[] = [ }, displayOptions: { show: { - '@version': [2.2, 2.3], + '@version': [{ _cnd: { gte: 2.2 } }], }, }, }, @@ -225,7 +225,9 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); - convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + if (nodeVersion >= 2.4) { + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + } values.push(checkItemAgainstSchema(this.getNode(), item, tableSchema, i)); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts index 9673eb14d64f5..6c5db8477062c 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/update.operation.ts @@ -173,7 +173,7 @@ const properties: INodeProperties[] = [ }, displayOptions: { show: { - '@version': [2.2, 2.3], + '@version': [{ _cnd: { gte: 2.2 } }], }, }, }, @@ -302,7 +302,9 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); - convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + if (nodeVersion >= 2.4) { + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + } item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts index aadfb32620170..47a0a944931e5 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/database/upsert.operation.ts @@ -172,7 +172,7 @@ const properties: INodeProperties[] = [ }, displayOptions: { show: { - '@version': [2.2, 2.3], + '@version': [{ _cnd: { gte: 2.2 } }], }, }, }, @@ -271,7 +271,9 @@ export async function execute( tableSchema = await updateTableSchema(db, tableSchema, schema, table); - convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + if (nodeVersion >= 2.4) { + convertArraysToPostgresFormat(item, tableSchema, this.getNode(), i); + } item = checkItemAgainstSchema(this.getNode(), item, tableSchema, i); diff --git a/packages/nodes-base/nodes/Postgres/v2/actions/versionDescription.ts b/packages/nodes-base/nodes/Postgres/v2/actions/versionDescription.ts index 722473f43a6d4..1687accf1cde4 100644 --- a/packages/nodes-base/nodes/Postgres/v2/actions/versionDescription.ts +++ b/packages/nodes-base/nodes/Postgres/v2/actions/versionDescription.ts @@ -8,7 +8,7 @@ export const versionDescription: INodeTypeDescription = { name: 'postgres', icon: 'file:postgres.svg', group: ['input'], - version: [2, 2.1, 2.2, 2.3], + version: [2, 2.1, 2.2, 2.3, 2.4], subtitle: '={{ $parameter["operation"] }}', description: 'Get, add and update data in Postgres', defaults: { diff --git a/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts b/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts index 5835579d3ac96..223996db6cdee 100644 --- a/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts +++ b/packages/nodes-base/nodes/Postgres/v2/helpers/interfaces.ts @@ -16,7 +16,7 @@ export type ColumnInfo = { data_type: string; is_nullable: string; udt_name?: string; - column_default?: string; + column_default?: string | null; is_generated?: 'ALWAYS' | 'NEVER'; identity_generation?: 'ALWAYS' | 'NEVER'; }; diff --git a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts index 9245d9e76e8bf..4698d31938000 100644 --- a/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts +++ b/packages/nodes-base/nodes/Postgres/v2/helpers/utils.ts @@ -539,24 +539,33 @@ export const convertArraysToPostgresFormat = ( return entry; } + if (typeof entry === 'boolean') { + entry = String(entry); + } + if (typeof entry === 'object') { entry = JSON.stringify(entry); } - //escape double quotes - return `"${entry.replace(/"/g, '\\"')}"`; + if (typeof entry === 'string') { + return `"${entry.replace(/"/g, '\\"')}"`; //escape double quotes + } + + return entry; }); //wrap in {} instead of [] as postgres does and join with , data[columnInfo.column_name] = `{${arrayEntries.join(',')}}`; } else { - throw new NodeOperationError( - node, - `Column '${columnInfo.column_name}' has to be an array`, - { - itemIndex, - }, - ); + if (columnInfo.is_nullable === 'NO') { + throw new NodeOperationError( + node, + `Column '${columnInfo.column_name}' has to be an array`, + { + itemIndex, + }, + ); + } } } }