From 095f6f1a4a0c51ccd2c6b9e6372c4f0d45076832 Mon Sep 17 00:00:00 2001 From: Samika Kashyap Date: Tue, 23 Apr 2024 13:57:22 -0700 Subject: [PATCH 1/3] feat: add composedb correctness test that perfroms parallelization --- .../fast/composedb-model-sync.test.ts | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/suite/src/__tests__/fast/composedb-model-sync.test.ts b/suite/src/__tests__/fast/composedb-model-sync.test.ts index 25e96623..bd38e6c8 100644 --- a/suite/src/__tests__/fast/composedb-model-sync.test.ts +++ b/suite/src/__tests__/fast/composedb-model-sync.test.ts @@ -10,6 +10,12 @@ import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils' const ComposeDbUrls = String(process.env.COMPOSEDB_URLS).split(',') const adminSeeds = String(process.env.COMPOSEDB_ADMIN_DID_SEEDS).split(',') +const parallelIterations = process.env.TEST_ITERATIONS + ? parseInt(process.env.TEST_ITERATIONS, 10) + : 20 +const transactionIterations = process.env.TEST_ITERATIONS + ? parseInt(process.env.TEST_ITERATIONS, 10) + : 20 const timeoutMs = 60000 describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () => { @@ -109,4 +115,79 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () const queryResponseid = queryResponseObj?.data?.node?.id expect(queryResponseid).toBeDefined() }) + + test('Create 20 model instance documents in a second and query them from the same node using mult query in the same node, query 20 times ina second', async () => { + for (let i = 0; i < transactionIterations; i++) { + // Add whatever mutation gitcoin was doing + const createDocumentMutation = ` + mutation createBasicSchema($input: CreateBasicSchemaInput!) { + createBasicSchema(input: $input) { + document { + id + myData + } + } + } + ` + + // TODO : Generate 1 kb data and create 20 documents in a second + const createDocumentPromises = [] + for (let i = 0; i < parallelIterations; i++) { + createDocumentPromises.push( + (async () => { + const createDocumentVariables = { + input: { + content: { + myData: 'my data ' + Date.now() + ' ' + i, + }, + }, + } + + const response = await composeClient1.executeQuery( + createDocumentMutation, + createDocumentVariables, + ) + const responseObject = await JSON.parse(JSON.stringify(response)) + const documentId = responseObject?.data?.createBasicSchema?.document?.id + expect(documentId).toBeDefined() + return documentId + })(), + ) + } + + const documentIds = await Promise.all(createDocumentPromises) + documentIds.forEach((id) => expect(id).toBeDefined()) + + // TODO: Create a multi query instead of simple query + const getDocumentsByStreamIdQuery = ` + query GetBasicSchemaById($ids: [ID!]!) { + nodes(ids: $ids) { + ... on BasicSchema { + id + myData + } + } + } + ` + + const getDocumentsByStreamIdVariables = { + ids: documentIds, + } + + // Generate simultaneous query load of 20 + const queryPromises = [] + for (let i = 0; i < parallelIterations; i++) { + queryPromises.push( + composeClient1.executeQuery(getDocumentsByStreamIdQuery, getDocumentsByStreamIdVariables), + ) + } + + const queryResponses = await Promise.all(queryPromises) + queryResponses.forEach((queryResponse) => { + const queryResponseObj = JSON.parse(JSON.stringify(queryResponse)) + const nodes = queryResponseObj?.data?.nodes + expect(nodes).toHaveLength(parallelIterations) + }) + } + }) }) From cc0c27577850586f91e5efbb54ab6fbeb29105c7 Mon Sep 17 00:00:00 2001 From: Samika Kashyap Date: Tue, 23 Apr 2024 14:09:33 -0700 Subject: [PATCH 2/3] feat: use separate composite for gitcoin --- .../fast/composedb-model-sync.test.ts | 133 +++++++++--------- suite/src/graphql-schemas/basicSchema.ts | 6 + 2 files changed, 76 insertions(+), 63 deletions(-) diff --git a/suite/src/__tests__/fast/composedb-model-sync.test.ts b/suite/src/__tests__/fast/composedb-model-sync.test.ts index bd38e6c8..80986816 100644 --- a/suite/src/__tests__/fast/composedb-model-sync.test.ts +++ b/suite/src/__tests__/fast/composedb-model-sync.test.ts @@ -3,19 +3,15 @@ import { beforeAll, describe, test, expect } from '@jest/globals' import { Composite } from '@composedb/devtools' import { newCeramic } from '../../utils/ceramicHelpers.js' import { createDid } from '../../utils/didHelper.js' -import { BasicSchema } from '../../graphql-schemas/basicSchema' +import { BasicSchema, MultiQuerySchema } from '../../graphql-schemas/basicSchema' import { StreamID } from '@ceramicnetwork/streamid' import { waitForDocument } from '../../utils/composeDbHelpers.js' import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils' const ComposeDbUrls = String(process.env.COMPOSEDB_URLS).split(',') const adminSeeds = String(process.env.COMPOSEDB_ADMIN_DID_SEEDS).split(',') -const parallelIterations = process.env.TEST_ITERATIONS - ? parseInt(process.env.TEST_ITERATIONS, 10) - : 20 -const transactionIterations = process.env.TEST_ITERATIONS - ? parseInt(process.env.TEST_ITERATIONS, 10) - : 20 +const parallelIterations = process.env.CDB_ITRS_PARALLEL ? parseInt(process.env.CDB_ITRS_PARALLEL, 10) : 20; +const transactionIterations = process.env.CDB_TXN_ITRS ? parseInt(process.env.CDB_TXN_ITRS, 10) : 20; const timeoutMs = 60000 describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () => { @@ -34,12 +30,18 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () const ceramicInstance2 = await newCeramic(ComposeDbUrls[1], did2) const composite = await Composite.create({ ceramic: ceramicInstance1, schema: BasicSchema }) + const composite2 = await Composite.create({ ceramic: ceramicInstance1, schema: MultiQuerySchema }) composeClient1 = await new ComposeClient({ ceramic: ceramicInstance1, definition: composite.toRuntime(), }) composeClient1.setDID(did1) + composeClient2 = await new ComposeClient({ + ceramic: ceramicInstance1, + definition: composite2.toRuntime(), + }) + // CACAO resources URLs for the models the client interacts with const resources = composeClient1.resources // Fetch themodelId of the model created by the node @@ -117,77 +119,82 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () }) test('Create 20 model instance documents in a second and query them from the same node using mult query in the same node, query 20 times ina second', async () => { - for (let i = 0; i < transactionIterations; i++) { - // Add whatever mutation gitcoin was doing - const createDocumentMutation = ` - mutation createBasicSchema($input: CreateBasicSchemaInput!) { - createBasicSchema(input: $input) { + + for(let i = 0; i < transactionIterations; i++) { + // TODO : Add whatever mutation gitcoin was doing + const createDocumentMutation = ` + mutation MultiQuerySchema($input: CreateBasicSchemaInput!) { + MultiQuerySchema(input: $input) { document { id myData } } } - ` - - // TODO : Generate 1 kb data and create 20 documents in a second - const createDocumentPromises = [] - for (let i = 0; i < parallelIterations; i++) { - createDocumentPromises.push( - (async () => { - const createDocumentVariables = { - input: { - content: { - myData: 'my data ' + Date.now() + ' ' + i, - }, - }, - } + `; + + + // TODO : Generate 1 kb data and create 20 documents in a second + const createDocumentPromises = []; + for (let i = 0; i < parallelIterations; i++) { + createDocumentPromises.push((async () => { + const createDocumentVariables = { + input: { + content: { + myData: 'my data ' + Date.now() + ' ' + i, + }, + }, + }; + + const response = await composeClient2.executeQuery( + createDocumentMutation, + createDocumentVariables, + ); + const responseObject = await JSON.parse(JSON.stringify(response)); + const documentId = responseObject?.data?.createBasicSchema?.document?.id; + expect(documentId).toBeDefined(); + return documentId; + })()); + } - const response = await composeClient1.executeQuery( - createDocumentMutation, - createDocumentVariables, - ) - const responseObject = await JSON.parse(JSON.stringify(response)) - const documentId = responseObject?.data?.createBasicSchema?.document?.id - expect(documentId).toBeDefined() - return documentId - })(), - ) - } + const documentIds = await Promise.all(createDocumentPromises); + documentIds.forEach(id => expect(id).toBeDefined()); - const documentIds = await Promise.all(createDocumentPromises) - documentIds.forEach((id) => expect(id).toBeDefined()) - // TODO: Create a multi query instead of simple query - const getDocumentsByStreamIdQuery = ` - query GetBasicSchemaById($ids: [ID!]!) { + // TODO: Create a multi query instead of simple query + const getDocumentsByStreamIdQuery = ` + query MultiQuerySchemaById($ids: [ID!]!) { nodes(ids: $ids) { - ... on BasicSchema { + ... on MultiQuerySchema { id myData } } } - ` - - const getDocumentsByStreamIdVariables = { - ids: documentIds, - } - - // Generate simultaneous query load of 20 - const queryPromises = [] - for (let i = 0; i < parallelIterations; i++) { - queryPromises.push( - composeClient1.executeQuery(getDocumentsByStreamIdQuery, getDocumentsByStreamIdVariables), + `; + + const getDocumentsByStreamIdVariables = { + ids: documentIds, + }; + + // Generate simultaneous query load of 20 + const queryPromises = []; + for (let i = 0; i < parallelIterations; i++) { + queryPromises.push( + composeClient2.executeQuery( + getDocumentsByStreamIdQuery, + getDocumentsByStreamIdVariables, ) - } - - const queryResponses = await Promise.all(queryPromises) - queryResponses.forEach((queryResponse) => { - const queryResponseObj = JSON.parse(JSON.stringify(queryResponse)) - const nodes = queryResponseObj?.data?.nodes - expect(nodes).toHaveLength(parallelIterations) - }) + ); } - }) + + const queryResponses = await Promise.all(queryPromises); + queryResponses.forEach(queryResponse => { + const queryResponseObj = JSON.parse(JSON.stringify(queryResponse)); + const nodes = queryResponseObj?.data?.nodes; + expect(nodes).toHaveLength(parallelIterations); + }); + + } +}) }) diff --git a/suite/src/graphql-schemas/basicSchema.ts b/suite/src/graphql-schemas/basicSchema.ts index 477c6dc7..f4004c01 100644 --- a/suite/src/graphql-schemas/basicSchema.ts +++ b/suite/src/graphql-schemas/basicSchema.ts @@ -4,3 +4,9 @@ type BasicSchema @createModel(accountRelation: LIST, description: "A set of uniq myData: String! @string(maxLength: 500) } ` + +export const MultiQuerySchema = ` +type MultiQuerySchema @createModel(accountRelation: LIST, description: "A set of unique numbers") +@createIndex(fields: [{ path: "myData" }]){ + myData: String! @string(maxLength: 500) +}` From 83da21a1b1fed744f2b8356cfdfb51e97c5ee8c2 Mon Sep 17 00:00:00 2001 From: Samika Kashyap Date: Tue, 23 Apr 2024 14:43:01 -0700 Subject: [PATCH 3/3] fix: set did for the client --- .../fast/composedb-model-sync.test.ts | 124 +++++++++--------- 1 file changed, 63 insertions(+), 61 deletions(-) diff --git a/suite/src/__tests__/fast/composedb-model-sync.test.ts b/suite/src/__tests__/fast/composedb-model-sync.test.ts index 80986816..25c2555c 100644 --- a/suite/src/__tests__/fast/composedb-model-sync.test.ts +++ b/suite/src/__tests__/fast/composedb-model-sync.test.ts @@ -10,13 +10,16 @@ import { CommonTestUtils as TestUtils } from '@ceramicnetwork/common-test-utils' const ComposeDbUrls = String(process.env.COMPOSEDB_URLS).split(',') const adminSeeds = String(process.env.COMPOSEDB_ADMIN_DID_SEEDS).split(',') -const parallelIterations = process.env.CDB_ITRS_PARALLEL ? parseInt(process.env.CDB_ITRS_PARALLEL, 10) : 20; -const transactionIterations = process.env.CDB_TXN_ITRS ? parseInt(process.env.CDB_TXN_ITRS, 10) : 20; +const parallelIterations = process.env.CDB_ITRS_PARALLEL + ? parseInt(process.env.CDB_ITRS_PARALLEL, 10) + : 20 +const transactionIterations = process.env.CDB_TXN_ITRS ? parseInt(process.env.CDB_TXN_ITRS, 10) : 20 const timeoutMs = 60000 describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () => { let composeClient1: ComposeClient let composeClient2: ComposeClient + let composeClient3: ComposeClient beforeAll(async () => { const did1 = await createDid(adminSeeds[0]) @@ -30,17 +33,21 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () const ceramicInstance2 = await newCeramic(ComposeDbUrls[1], did2) const composite = await Composite.create({ ceramic: ceramicInstance1, schema: BasicSchema }) - const composite2 = await Composite.create({ ceramic: ceramicInstance1, schema: MultiQuerySchema }) + const composite2 = await Composite.create({ + ceramic: ceramicInstance1, + schema: MultiQuerySchema, + }) composeClient1 = await new ComposeClient({ ceramic: ceramicInstance1, definition: composite.toRuntime(), }) composeClient1.setDID(did1) - composeClient2 = await new ComposeClient({ + composeClient3 = await new ComposeClient({ ceramic: ceramicInstance1, definition: composite2.toRuntime(), }) + composeClient3.setDID(did1) // CACAO resources URLs for the models the client interacts with const resources = composeClient1.resources @@ -119,10 +126,9 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () }) test('Create 20 model instance documents in a second and query them from the same node using mult query in the same node, query 20 times ina second', async () => { - - for(let i = 0; i < transactionIterations; i++) { - // TODO : Add whatever mutation gitcoin was doing - const createDocumentMutation = ` + for (let i = 0; i < transactionIterations; i++) { + // TODO : Add whatever mutation gitcoin was doing + const createDocumentMutation = ` mutation MultiQuerySchema($input: CreateBasicSchemaInput!) { MultiQuerySchema(input: $input) { document { @@ -131,38 +137,38 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () } } } - `; - - - // TODO : Generate 1 kb data and create 20 documents in a second - const createDocumentPromises = []; - for (let i = 0; i < parallelIterations; i++) { - createDocumentPromises.push((async () => { - const createDocumentVariables = { - input: { - content: { - myData: 'my data ' + Date.now() + ' ' + i, - }, - }, - }; - - const response = await composeClient2.executeQuery( - createDocumentMutation, - createDocumentVariables, - ); - const responseObject = await JSON.parse(JSON.stringify(response)); - const documentId = responseObject?.data?.createBasicSchema?.document?.id; - expect(documentId).toBeDefined(); - return documentId; - })()); - } + ` + + // TODO : Generate 1 kb data and create 20 documents in a second + const createDocumentPromises = [] + for (let i = 0; i < parallelIterations; i++) { + createDocumentPromises.push( + (async () => { + const createDocumentVariables = { + input: { + content: { + myData: 'my data ' + Date.now() + ' ' + i, + }, + }, + } - const documentIds = await Promise.all(createDocumentPromises); - documentIds.forEach(id => expect(id).toBeDefined()); + const response = await composeClient3.executeQuery( + createDocumentMutation, + createDocumentVariables, + ) + const responseObject = await JSON.parse(JSON.stringify(response)) + const documentId = responseObject?.data?.createBasicSchema?.document?.id + expect(documentId).toBeDefined() + return documentId + })(), + ) + } + const documentIds = await Promise.all(createDocumentPromises) + documentIds.forEach((id) => expect(id).toBeDefined()) - // TODO: Create a multi query instead of simple query - const getDocumentsByStreamIdQuery = ` + // TODO: Create a multi query instead of simple query + const getDocumentsByStreamIdQuery = ` query MultiQuerySchemaById($ids: [ID!]!) { nodes(ids: $ids) { ... on MultiQuerySchema { @@ -171,30 +177,26 @@ describe('Sync Model and ModelInstanceDocument using ComposeDB GraphQL API', () } } } - `; - - const getDocumentsByStreamIdVariables = { - ids: documentIds, - }; - - // Generate simultaneous query load of 20 - const queryPromises = []; - for (let i = 0; i < parallelIterations; i++) { - queryPromises.push( - composeClient2.executeQuery( - getDocumentsByStreamIdQuery, - getDocumentsByStreamIdVariables, - ) - ); - } + ` - const queryResponses = await Promise.all(queryPromises); - queryResponses.forEach(queryResponse => { - const queryResponseObj = JSON.parse(JSON.stringify(queryResponse)); - const nodes = queryResponseObj?.data?.nodes; - expect(nodes).toHaveLength(parallelIterations); - }); + const getDocumentsByStreamIdVariables = { + ids: documentIds, + } - } -}) + // Generate simultaneous query load of 20 + const queryPromises = [] + for (let i = 0; i < parallelIterations; i++) { + queryPromises.push( + composeClient3.executeQuery(getDocumentsByStreamIdQuery, getDocumentsByStreamIdVariables), + ) + } + + const queryResponses = await Promise.all(queryPromises) + queryResponses.forEach((queryResponse) => { + const queryResponseObj = JSON.parse(JSON.stringify(queryResponse)) + const nodes = queryResponseObj?.data?.nodes + expect(nodes).toHaveLength(parallelIterations) + }) + } + }) })