From e03cf8ff550c6e858bae0047139c02131d4d4ccc Mon Sep 17 00:00:00 2001 From: Samuel Lukes Date: Tue, 26 Sep 2023 08:02:26 +0200 Subject: [PATCH 1/2] chore: improve performance of fixtures iterator --- package.json | 1 + src/util/fixturesIterator.ts | 39 ++++++++++++++----------- test/unit/util/fixturesIterator.test.ts | 31 +++++++++++++++++++- yarn.lock | 5 ++++ 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index e587dd5..7c14543 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "opencollective-postinstall": "^2.0.3", "reflect-metadata": "^0.1.13", "resolve-from": "^5.0.0", + "typescript-collections": "^1.3.3", "yargs": "^17.5.1" }, "peerDependencies": { diff --git a/src/util/fixturesIterator.ts b/src/util/fixturesIterator.ts index 89d296b..37c98cc 100644 --- a/src/util/fixturesIterator.ts +++ b/src/util/fixturesIterator.ts @@ -1,25 +1,30 @@ -import { sum } from 'lodash'; import { IFixture } from '../interface'; - +import { PriorityQueue } from 'typescript-collections'; export function* fixturesIterator(fixtures: IFixture[]) { - const state: any = {}; + const processed = new Set(); + const queue = new PriorityQueue((a, b) => { + const aDependencies = a.dependencies.filter((dep) => !processed.has(dep)).length; + const bDependencies = b.dependencies.filter((dep) => !processed.has(dep)).length; + + return bDependencies - aDependencies; + }); - while (true) { - const result = fixtures.find((fixture) => { - return ( - sum( - fixture.dependencies.map((dependency: string) => { - return state[dependency] === undefined ? 0 : 1; - }), - ) === fixture.dependencies.length && !state[fixture.name] - ); - }); + for (const fixture of fixtures) { + queue.add(fixture); + } + + while (!queue.isEmpty()) { + const fixture = queue.dequeue(); + if (!fixture) { + // theoretically impossible, but keep the linter happy. + continue; + } - if (result) { - state[result.name] = true; - yield result; + if (fixture.dependencies.every((dep) => processed.has(dep))) { + processed.add(fixture.name); + yield fixture; } else { - return; + queue.add(fixture); } } } diff --git a/test/unit/util/fixturesIterator.test.ts b/test/unit/util/fixturesIterator.test.ts index f2d95f6..fdf85f5 100644 --- a/test/unit/util/fixturesIterator.test.ts +++ b/test/unit/util/fixturesIterator.test.ts @@ -1,6 +1,6 @@ import 'mocha'; import { expect } from 'chai'; -import { fixturesIterator } from '../../../src/util'; +import { fixturesIterator, niaveFixturesIterator } from '../../../src/util'; describe('Fixtures Iterator', () => { it('should be sort and iterate fixtures', () => { @@ -49,4 +49,33 @@ describe('Fixtures Iterator', () => { expect(fixtures[1].name).to.equal('post1'); expect(fixtures[2].name).to.equal('comment1'); }); + + it('should cope with 1 million fixtures in a reasonable period', () => { + const fixtures = []; + + // Generate 1 million fixtures with random dependencies + for (let i = 0; i < 1_000_000; i++) { + const dependencies: string[] = []; + fixtures.push({ + name: `fixture${i}`, + parameters: {}, + entity: 'Comment', + dependencies, + data: {}, + }); + } + + const start = Date.now(); + const iterator = fixturesIterator(fixtures); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for (const _fixture of iterator) { + // Do nothing + } + const end = Date.now(); + + // Check if the time taken is less than 2 seconds + // On macbook M1 this takes between 800 and 900ms + const timeTaken = end - start; + expect(timeTaken).to.be.lessThan(2000); + }); }); diff --git a/yarn.lock b/yarn.lock index 76fbf4b..71a84ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4649,6 +4649,11 @@ typeorm@^0.3.0: xml2js "^0.4.23" yargs "^17.3.1" +typescript-collections@^1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/typescript-collections/-/typescript-collections-1.3.3.tgz#62d50d93c018c094d425eabee649f00ec5cc0fea" + integrity sha512-7sI4e/bZijOzyURng88oOFZCISQPTHozfE2sUu5AviFYk5QV7fYGb6YiDl+vKjF/pICA354JImBImL9XJWUvdQ== + typescript@^4.6.2, typescript@^4.6.4: version "4.7.4" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" From 098f4e4b08599e6ca7a75fa70b562ae07cefd8fe Mon Sep 17 00:00:00 2001 From: Samuel Lukes Date: Tue, 26 Sep 2023 14:45:04 +0200 Subject: [PATCH 2/2] fix: use graphs instead --- package.json | 3 ++ src/util/fixturesIterator.ts | 36 +++++++++------------ test/unit/util/fixturesIterator.test.ts | 37 +++++++++++++++------ yarn.lock | 43 +++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index 7c14543..d6ecafd 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "coveralls": "^3.1.0", "cz-conventional-changelog": "^3.3.0", "eslint": "^8.17.0", + "graphology-types": "^0.24.7", "husky": "^8.0.0", "lint-staged": "^13.0.0", "mocha": "^10.0.0", @@ -82,6 +83,8 @@ "cli-progress": "^3.10.0", "ejs": "^3.1.5", "glob": "^8.0.1", + "graphology": "^0.25.4", + "graphology-dag": "^0.3.0", "joi": "^17.0.0", "js-yaml": "^4.0.0", "lodash": "^4.0.0", diff --git a/src/util/fixturesIterator.ts b/src/util/fixturesIterator.ts index 37c98cc..79f558d 100644 --- a/src/util/fixturesIterator.ts +++ b/src/util/fixturesIterator.ts @@ -1,30 +1,26 @@ import { IFixture } from '../interface'; -import { PriorityQueue } from 'typescript-collections'; -export function* fixturesIterator(fixtures: IFixture[]) { - const processed = new Set(); - const queue = new PriorityQueue((a, b) => { - const aDependencies = a.dependencies.filter((dep) => !processed.has(dep)).length; - const bDependencies = b.dependencies.filter((dep) => !processed.has(dep)).length; +import { DirectedGraph } from 'graphology'; +import { topologicalSort, willCreateCycle } from 'graphology-dag'; - return bDependencies - aDependencies; - }); +export function* fixturesIterator(fixtures: IFixture[]) { + const graph = new DirectedGraph({ allowSelfLoops: false }); + const fixturesByName = new Map(); for (const fixture of fixtures) { - queue.add(fixture); + fixturesByName.set(fixture.name, fixture); + graph.addNode(fixture.name, fixture); } - while (!queue.isEmpty()) { - const fixture = queue.dequeue(); - if (!fixture) { - // theoretically impossible, but keep the linter happy. - continue; + for (const fixture of fixtures) { + for (const dep of fixture.dependencies) { + if (willCreateCycle(graph, dep, fixture.name)) { + throw new Error(`There is a cycle between ${dep} and ${fixture.name}`); + } + graph.addDirectedEdge(dep, fixture.name); } + } - if (fixture.dependencies.every((dep) => processed.has(dep))) { - processed.add(fixture.name); - yield fixture; - } else { - queue.add(fixture); - } + for (const name of topologicalSort(graph)) { + yield fixturesByName.get(name)!; } } diff --git a/test/unit/util/fixturesIterator.test.ts b/test/unit/util/fixturesIterator.test.ts index fdf85f5..b980088 100644 --- a/test/unit/util/fixturesIterator.test.ts +++ b/test/unit/util/fixturesIterator.test.ts @@ -1,6 +1,6 @@ import 'mocha'; import { expect } from 'chai'; -import { fixturesIterator, niaveFixturesIterator } from '../../../src/util'; +import { fixturesIterator } from '../../../src/util'; describe('Fixtures Iterator', () => { it('should be sort and iterate fixtures', () => { @@ -45,16 +45,34 @@ describe('Fixtures Iterator', () => { fixtures.push(fixture); } - expect(fixtures[0].name).to.equal('user1'); - expect(fixtures[1].name).to.equal('post1'); - expect(fixtures[2].name).to.equal('comment1'); + expect(fixtures[0]?.name).to.equal('user1'); + expect(fixtures[1]?.name).to.equal('post1'); + expect(fixtures[2]?.name).to.equal('comment1'); }); - it('should cope with 1 million fixtures in a reasonable period', () => { + it('should throw if there is a cycle', () => { + fixturesIterator([ + { + parameters: {}, + entity: 'Post', + name: 'post1', + dependencies: ['post1'], + data: {}, + }, + { + parameters: {}, + entity: 'User', + name: 'user1', + dependencies: ['user1'], + data: {}, + }, + ]); + }); + + it('should cope with large number of fixtures in a reasonable period', () => { const fixtures = []; - // Generate 1 million fixtures with random dependencies - for (let i = 0; i < 1_000_000; i++) { + for (let i = 0; i < 100_000; i++) { const dependencies: string[] = []; fixtures.push({ name: `fixture${i}`, @@ -73,9 +91,8 @@ describe('Fixtures Iterator', () => { } const end = Date.now(); - // Check if the time taken is less than 2 seconds - // On macbook M1 this takes between 800 and 900ms + // On macbook M1 this takes between 100 and 200ms const timeTaken = end - start; - expect(timeTaken).to.be.lessThan(2000); + expect(timeTaken).to.be.lessThan(500); }); }); diff --git a/yarn.lock b/yarn.lock index 71a84ae..5d36d51 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1937,6 +1937,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@^2.0.1: version "2.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-2.1.0.tgz#e5d3ecd837d2a60ec50f3da78fd39767747bbe99" @@ -2429,6 +2434,32 @@ graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== +graphology-dag@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/graphology-dag/-/graphology-dag-0.3.0.tgz#e200640cbff89dc40a3525c1a8ad2254c1d2bfed" + integrity sha512-dg4JPb+/LDEbDinZIj7ezWzlEXDRokshdpTL8oAuftE9Uy0uTKGOKSmYULY8p3j/vw0HB31Wog9T/kpqprUQpg== + dependencies: + graphology-utils "^2.4.1" + mnemonist "^0.39.0" + +graphology-types@^0.24.7: + version "0.24.7" + resolved "https://registry.yarnpkg.com/graphology-types/-/graphology-types-0.24.7.tgz#7d630a800061666bfa70066310f56612e08b7bee" + integrity sha512-tdcqOOpwArNjEr0gNQKCXwaNCWnQJrog14nJNQPeemcLnXQUUGrsCWpWkVKt46zLjcS6/KGoayeJfHHyPDlvwA== + +graphology-utils@^2.4.1: + version "2.5.2" + resolved "https://registry.yarnpkg.com/graphology-utils/-/graphology-utils-2.5.2.tgz#4d30d6e567d27c01f105e1494af816742e8d2440" + integrity sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ== + +graphology@^0.25.4: + version "0.25.4" + resolved "https://registry.yarnpkg.com/graphology/-/graphology-0.25.4.tgz#e528a64555ac1f392a9d965321ada5b2b843efe1" + integrity sha512-33g0Ol9nkWdD6ulw687viS8YJQBxqG5LWII6FI6nul0pq6iM2t5EKquOTFDbyTblRB3O9I+7KX4xI8u5ffekAQ== + dependencies: + events "^3.3.0" + obliterator "^2.0.2" + handlebars@^4.7.7: version "4.7.7" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" @@ -3251,6 +3282,13 @@ mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mnemonist@^0.39.0: + version "0.39.5" + resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.39.5.tgz#5850d9b30d1b2bc57cc8787e5caa40f6c3420477" + integrity sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ== + dependencies: + obliterator "^2.0.1" + mocha@^10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9" @@ -3455,6 +3493,11 @@ object-inspect@^1.12.2: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== +obliterator@^2.0.1, obliterator@^2.0.2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-2.0.4.tgz#fa650e019b2d075d745e44f1effeb13a2adbe816" + integrity sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ== + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"