From 2055e485583823c628b5cd85bd830de2294ce91f Mon Sep 17 00:00:00 2001 From: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com> Date: Thu, 2 Jun 2022 12:39:42 +0200 Subject: [PATCH] feat(core): Add support to import/export tags (#3130) * Export and Import Workflow Tags Support exporting and importing tags of workflows via frontend and cli. On export, all tag data is included in the json. - id - name - updatedAt - createdAt When importing a workflow json to n8n we: - first check if a tag with the same id and createdAt date exists in the database, then we can assume the tag is identical. Changes on the name of the tag are now preserved. - check if a tag with the same name exists on the database. - create a new tag with the given name. * clean up fe export * remove usage count * return updatedat, createdat * fix tags import * move logic from workflow package * refactor import * check for tags before import * update checks on type * fix on import * fix build issues * fix type issue * remove unnessary ? * update tag helpers so only name is required * fix tag import * add don't replace existing tags * fix build issue * address comments * fix with promise.all * update setting tags * update check * fix existing check * add helper * fix duplication * fix multiple same tags bug * fix db bugs * add more validation on workflow type * fix validation * disable importing tags on copy paste Co-authored-by: Luca Berneking --- packages/cli/commands/export/workflow.ts | 6 +- packages/cli/commands/import/workflow.ts | 40 ++++++++-- packages/cli/src/Interfaces.ts | 11 +++ packages/cli/src/Server.ts | 2 +- packages/cli/src/TagHelpers.ts | 73 ++++++++++++++++++- packages/editor-ui/src/Interface.ts | 2 + .../editor-ui/src/components/MainSidebar.vue | 13 +++- packages/editor-ui/src/modules/tags.ts | 12 +-- packages/editor-ui/src/store.ts | 4 + packages/editor-ui/src/views/NodeView.vue | 38 +++++++++- 10 files changed, 179 insertions(+), 22 deletions(-) diff --git a/packages/cli/commands/export/workflow.ts b/packages/cli/commands/export/workflow.ts index 958efe9c2db50a..ecfd173922efc5 100644 --- a/packages/cli/commands/export/workflow.ts +++ b/packages/cli/commands/export/workflow.ts @@ -110,8 +110,10 @@ export class ExportWorkflowsCommand extends Command { findQuery.id = flags.id; } - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const workflows = await Db.collections.Workflow.find(findQuery); + const workflows = await Db.collections.Workflow.find({ + where: findQuery, + relations: ['tags'], + }); if (workflows.length === 0) { throw new Error('No workflows found with specified filters.'); diff --git a/packages/cli/commands/import/workflow.ts b/packages/cli/commands/import/workflow.ts index bf87e680b7f23d..b165d0464181b0 100644 --- a/packages/cli/commands/import/workflow.ts +++ b/packages/cli/commands/import/workflow.ts @@ -17,15 +17,34 @@ import glob from 'fast-glob'; import { UserSettings } from 'n8n-core'; import { EntityManager, getConnection } from 'typeorm'; import { getLogger } from '../../src/Logger'; -import { Db, ICredentialsDb } from '../../src'; +import { Db, ICredentialsDb, IWorkflowToImport } from '../../src'; import { SharedWorkflow } from '../../src/databases/entities/SharedWorkflow'; import { WorkflowEntity } from '../../src/databases/entities/WorkflowEntity'; import { Role } from '../../src/databases/entities/Role'; import { User } from '../../src/databases/entities/User'; +import { setTagsForImport } from '../../src/TagHelpers'; const FIX_INSTRUCTION = 'Please fix the database by running ./packages/cli/bin/n8n user-management:reset'; +function assertHasWorkflowsToImport(workflows: unknown): asserts workflows is IWorkflowToImport[] { + if (!Array.isArray(workflows)) { + throw new Error( + 'File does not seem to contain workflows. Make sure the workflows are contained in an array.', + ); + } + + for (const workflow of workflows) { + if ( + typeof workflow !== 'object' || + !Object.prototype.hasOwnProperty.call(workflow, 'nodes') || + !Object.prototype.hasOwnProperty.call(workflow, 'connections') + ) { + throw new Error('File does not seem to contain valid workflows.'); + } + } +} + export class ImportWorkflowsCommand extends Command { static description = 'Import workflows'; @@ -82,7 +101,8 @@ export class ImportWorkflowsCommand extends Command { // Make sure the settings exist await UserSettings.prepareUserSettings(); - const credentials = (await Db.collections.Credentials.find()) ?? []; + const credentials = await Db.collections.Credentials.find(); + const tags = await Db.collections.Tag.find(); let totalImported = 0; @@ -111,6 +131,10 @@ export class ImportWorkflowsCommand extends Command { }); } + if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) { + await setTagsForImport(transactionManager, workflow, tags); + } + await this.storeWorkflow(workflow, user); } }); @@ -121,13 +145,9 @@ export class ImportWorkflowsCommand extends Command { const workflows = JSON.parse(fs.readFileSync(flags.input, { encoding: 'utf8' })); - totalImported = workflows.length; + assertHasWorkflowsToImport(workflows); - if (!Array.isArray(workflows)) { - throw new Error( - 'File does not seem to contain workflows. Make sure the workflows are contained in an array.', - ); - } + totalImported = workflows.length; await getConnection().transaction(async (transactionManager) => { this.transactionManager = transactionManager; @@ -139,6 +159,10 @@ export class ImportWorkflowsCommand extends Command { }); } + if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) { + await setTagsForImport(transactionManager, workflow, tags); + } + await this.storeWorkflow(workflow, user); } }); diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 07e598f9aa6386..2bc12e638fc85e 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -114,6 +114,13 @@ export interface ITagDb { updatedAt: Date; } +export interface ITagToImport { + id: string | number; + name: string; + createdAt?: string; + updatedAt?: string; +} + export type UsageCount = { usageCount: number; }; @@ -134,6 +141,10 @@ export interface IWorkflowDb extends IWorkflowBase { tags: ITagDb[]; } +export interface IWorkflowToImport extends IWorkflowBase { + tags: ITagToImport[]; +} + export interface IWorkflowResponse extends IWorkflowBase { id: string; } diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index e4786428092b26..82620691fcd306 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -1241,7 +1241,7 @@ class App { return TagHelpers.getTagsWithCountDb(tablePrefix); } - return Db.collections.Tag.find({ select: ['id', 'name'] }); + return Db.collections.Tag.find({ select: ['id', 'name', 'createdAt', 'updatedAt'] }); }, ), ); diff --git a/packages/cli/src/TagHelpers.ts b/packages/cli/src/TagHelpers.ts index 31e2a063f924ed..12165ceeb6587c 100644 --- a/packages/cli/src/TagHelpers.ts +++ b/packages/cli/src/TagHelpers.ts @@ -1,11 +1,11 @@ /* eslint-disable no-param-reassign */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ /* eslint-disable import/no-cycle */ -import { getConnection } from 'typeorm'; +import { EntityManager, getConnection } from 'typeorm'; import { TagEntity } from './databases/entities/TagEntity'; -import { ITagWithCountDb } from './Interfaces'; +import { ITagToImport, ITagWithCountDb, IWorkflowToImport } from './Interfaces'; // ---------------------------------- // utils @@ -38,6 +38,8 @@ export async function getTagsWithCountDb(tablePrefix: string): Promise => { + const tag = new TagEntity(); + tag.name = name; + return transactionManager.save(tag); +}; + +const findOrCreateTag = async ( + transactionManager: EntityManager, + importTag: ITagToImport, + tagsEntities: TagEntity[], +): Promise => { + // Assume tag is identical if createdAt date is the same to preserve a changed tag name + const identicalMatch = tagsEntities.find( + (existingTag) => + existingTag.id.toString() === importTag.id.toString() && + existingTag.createdAt && + importTag.createdAt && + existingTag.createdAt.getTime() === new Date(importTag.createdAt).getTime(), + ); + if (identicalMatch) { + return identicalMatch; + } + + const nameMatch = tagsEntities.find((existingTag) => existingTag.name === importTag.name); + if (nameMatch) { + return nameMatch; + } + + const created = await createTag(transactionManager, importTag.name); + tagsEntities.push(created); + return created; +}; + +const hasTags = (workflow: IWorkflowToImport) => + 'tags' in workflow && Array.isArray(workflow.tags) && workflow.tags.length > 0; + +/** + * Set tag IDs to use existing tags, creates a new tag if no matching tag could be found + */ +export async function setTagsForImport( + transactionManager: EntityManager, + workflow: IWorkflowToImport, + tags: TagEntity[], +): Promise { + if (!hasTags(workflow)) { + return; + } + + const workflowTags = workflow.tags; + const tagLookupPromises = []; + for (let i = 0; i < workflowTags.length; i++) { + if (workflowTags[i]?.name) { + const lookupPromise = findOrCreateTag(transactionManager, workflowTags[i], tags).then( + (tag) => { + workflowTags[i] = { + id: tag.id, + name: tag.name, + }; + }, + ); + tagLookupPromises.push(lookupPromise); + } + } + + await Promise.all(tagLookupPromises); +} diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index a09131392b18fb..15cc4a96c9ddb6 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -733,6 +733,8 @@ export interface ITag { id: string; name: string; usageCount?: number; + createdAt?: string; + updatedAt?: string; } export interface ITagRow { diff --git a/packages/editor-ui/src/components/MainSidebar.vue b/packages/editor-ui/src/components/MainSidebar.vue index fcde2e6854a3f8..10add3ce6d6d24 100644 --- a/packages/editor-ui/src/components/MainSidebar.vue +++ b/packages/editor-ui/src/components/MainSidebar.vue @@ -511,10 +511,21 @@ export default mixins( if (data.id && typeof data.id === 'string') { data.id = parseInt(data.id, 10); } - const blob = new Blob([JSON.stringify(data, null, 2)], { + + const exportData: IWorkflowDataUpdate = { + ...data, + tags: (tags || []).map(tagId => { + const {usageCount, ...tag} = this.$store.getters["tags/getTagById"](tagId); + + return tag; + }), + }; + + const blob = new Blob([JSON.stringify(exportData, null, 2)], { type: 'application/json;charset=utf-8', }); + let workflowName = this.$store.getters.workflowName || 'unsaved_workflow'; workflowName = workflowName.replace(/[^a-z0-9]/gi, '_'); diff --git a/packages/editor-ui/src/modules/tags.ts b/packages/editor-ui/src/modules/tags.ts index f0fbde557c1905..52063607c6375d 100644 --- a/packages/editor-ui/src/modules/tags.ts +++ b/packages/editor-ui/src/modules/tags.ts @@ -64,10 +64,10 @@ const module: Module = { }, }, actions: { - fetchAll: async (context: ActionContext, params?: { force?: boolean, withUsageCount?: boolean }) => { + fetchAll: async (context: ActionContext, params?: { force?: boolean, withUsageCount?: boolean }): Promise => { const { force = false, withUsageCount = false } = params || {}; if (!force && context.state.fetchedAll && context.state.fetchedUsageCount === withUsageCount) { - return context.state.tags; + return Object.values(context.state.tags); } context.commit('setLoading', true); @@ -77,7 +77,7 @@ const module: Module = { return tags; }, - create: async (context: ActionContext, name: string) => { + create: async (context: ActionContext, name: string): Promise => { const tag = await createTag(context.rootGetters.getRestApiContext, { name }); context.commit('upsertTags', [tag]); @@ -88,7 +88,7 @@ const module: Module = { context.commit('upsertTags', [tag]); return tag; - }, + }, delete: async (context: ActionContext, id: string) => { const deleted = await deleteTag(context.rootGetters.getRestApiContext, id); @@ -98,8 +98,8 @@ const module: Module = { } return deleted; - }, + }, }, }; -export default module; \ No newline at end of file +export default module; diff --git a/packages/editor-ui/src/store.ts b/packages/editor-ui/src/store.ts index 9d84df8429fad0..38107ed5fadf04 100644 --- a/packages/editor-ui/src/store.ts +++ b/packages/editor-ui/src/store.ts @@ -611,6 +611,10 @@ export const store = new Vuex.Store({ Vue.set(state.workflow, 'tags', tags); }, + addWorkflowTagIds (state, tags: string[]) { + Vue.set(state.workflow, 'tags', [...new Set([...(state.workflow.tags || []), ...tags])]); + }, + removeWorkflowTagId (state, tagId: string) { const tags = state.workflow.tags as string[]; const updated = tags.filter((id: string) => id !== tagId); diff --git a/packages/editor-ui/src/views/NodeView.vue b/packages/editor-ui/src/views/NodeView.vue index 28b4e926fd6221..bdf35408acc5e3 100644 --- a/packages/editor-ui/src/views/NodeView.vue +++ b/packages/editor-ui/src/views/NodeView.vue @@ -1232,7 +1232,7 @@ export default mixins( workflow_id: this.$store.getters.workflowId, }); - return this.importWorkflowData(workflowData!); + return this.importWorkflowData(workflowData!, false); }, // Returns the workflow data from a given URL. If no data gets found or @@ -1259,7 +1259,7 @@ export default mixins( }, // Imports the given workflow data into the current workflow - async importWorkflowData (workflowData: IWorkflowDataUpdate): Promise { + async importWorkflowData (workflowData: IWorkflowDataUpdate, importTags = true): Promise { // If it is JSON check if it looks on the first look like data we can use if ( !workflowData.hasOwnProperty('nodes') || @@ -1285,6 +1285,40 @@ export default mixins( this.nodeSelectedByName(node.name); }); }); + + const tagsEnabled = this.$store.getters['settings/areTagsEnabled']; + if (importTags && tagsEnabled && Array.isArray(workflowData.tags)) { + const allTags: ITag[] = await this.$store.dispatch('tags/fetchAll'); + const tagNames = new Set(allTags.map((tag) => tag.name)); + + const workflowTags = workflowData.tags as ITag[]; + const notFound = workflowTags.filter((tag) => !tagNames.has(tag.name)); + + const creatingTagPromises: Array> = []; + for (const tag of notFound) { + const creationPromise = this.$store.dispatch('tags/create', tag.name) + .then((tag: ITag) => { + allTags.push(tag); + return tag; + }); + + creatingTagPromises.push(creationPromise); + } + + await Promise.all(creatingTagPromises); + + const tagIds = workflowTags.reduce((accu: string[], imported: ITag) => { + const tag = allTags.find(tag => tag.name === imported.name); + if (tag) { + accu.push(tag.id); + } + + return accu; + }, []); + + this.$store.commit('addWorkflowTagIds', tagIds); + } + } catch (error) { this.$showError( error,