From 8ca20ddedd723dc2b3bbe5a33563f48f8515defc Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 1 Mar 2022 09:53:52 -0800 Subject: [PATCH 01/61] [Docs] Indicate create*Query options.results is a Doc[] The `Connection`'s `createFetchQuery` and `createSubscribeQuery` take an optional existing result array as `options.results`. It should be an array of Doc instances: https://github.com/share/sharedb/blob/1cca122c63329665ebfdeceb1984921badb0cf4d/lib/client/connection.js#L563-L567 This PR updates the docs to indicate the specific type, instead of `Object[]`. --- docs/api/connection.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/connection.md b/docs/api/connection.md index 1b79a25dc..179514799 100644 --- a/docs/api/connection.md +++ b/docs/api/connection.md @@ -68,7 +68,7 @@ Optional > Default: `{}` -> `options.results` -- Object[] +> `options.results` -- [`Doc`]({{ site.baseurl }}{% link api/doc.md %})[] > > Prior query results, if available, such as from server rendering @@ -105,7 +105,7 @@ Optional > Default: `{}` -> `options.results` -- Object[] +> `options.results` -- [`Doc`]({{ site.baseurl }}{% link api/doc.md %})[] > > Prior query results, if available, such as from server rendering @@ -228,4 +228,4 @@ The document **must** be of a [type]({{ site.baseurl }}{% link types/index.md %} Return value -> A [`Presence`]({{ site.baseurl }}{% link api/presence.md %}) instance tied to the given document \ No newline at end of file +> A [`Presence`]({{ site.baseurl }}{% link api/presence.md %}) instance tied to the given document From 5337969664bcf9d7d12ed67d544d599810eb69eb Mon Sep 17 00:00:00 2001 From: Curran Kelleher <68416+curran@users.noreply.github.com> Date: Thu, 17 Mar 2022 11:57:45 -0400 Subject: [PATCH 02/61] Fix typos in query docs --- docs/queries.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/queries.md b/docs/queries.md index 8a8163b9c..930ef43b2 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -32,7 +32,7 @@ See the [API documentation]({{ site.baseurl }}{% link api/connection.md %}#creat [`Query`]({{ site.baseurl }}{% link api/query.md %}) instance, which can be used instead: ```js -const query = connection.createFetchQuery('my-connection', {userId: 1}) +const query = connection.createFetchQuery('my-collection', {userId: 1}) query.on('ready', () => { // results are now available in query.results }) @@ -48,7 +48,7 @@ A subscribed query will automatically cause any matched `Doc` instances to recei Subscribe queries can be created similarly to fetch queries, but you may also be interested in [other events]({{ site.baseurl }}{% link api/query.md %}#events): ```js -const query = connection.createSubscribeQuery('my-connection', {userId: 1}) +const query = connection.createSubscribeQuery('my-collection', {userId: 1}) query.on('ready', () => { // The initial results are available in query.results }) @@ -73,7 +73,7 @@ If a query can potentially return a large number of results, you may want to con For example, a `sharedb-mongo` limit might look like this: ```js -const query = connection.createSubscribeQuery('my-connection', {userId: 1, $skip: 10, $limit: 10}) +const query = connection.createSubscribeQuery('my-collection', {userId: 1, $skip: 10, $limit: 10}) ``` {: .info } From cb3aaedad05b69d622945e57bc68f88435812ec4 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 22 Mar 2022 11:20:40 +0000 Subject: [PATCH 03/61] =?UTF-8?q?=F0=9F=93=9D=20Remove=20`id`=20from=20`ge?= =?UTF-8?q?tOpsBulk()`=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/backend.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/api/backend.md b/docs/api/backend.md index c4f8ac0c5..b1db01078 100644 --- a/docs/api/backend.md +++ b/docs/api/backend.md @@ -388,10 +388,6 @@ backend.getOpsBulk(agent, index, fromMap, toMap [, options [, callback]]) > The name of the collection or [projection]({{ site.baseurl }}{% link projections.md %}) -`id` -- string - -> The document ID - `fromMap` -- Object > An object whose keys are the IDs of the target documents. The values are the first versions requested of each document (inclusive) From c49ad2119e33eed8cf85376f36231a33c2614688 Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 5 Apr 2022 09:06:26 -0700 Subject: [PATCH 04/61] More detail on create*Query prior results Co-authored-by: Curran Kelleher <68416+curran@users.noreply.github.com> --- docs/api/connection.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/connection.md b/docs/api/connection.md index 179514799..87df406ca 100644 --- a/docs/api/connection.md +++ b/docs/api/connection.md @@ -70,7 +70,7 @@ Optional > `options.results` -- [`Doc`]({{ site.baseurl }}{% link api/doc.md %})[] -> > Prior query results, if available, such as from server rendering +> > Prior query results, if available, such as from server rendering. This should be an array of Doc instances, as obtained from `connection.get(collection, id)`. If the docs' data is already available, invoke [`ingestSnapshot`]({{ site.baseurl }}{% link api/doc.md %}#ingestsnapshot) for each Doc instance beforehand to avoid re-transferring the data from the server. > `options.*` -- any @@ -107,7 +107,7 @@ Optional > `options.results` -- [`Doc`]({{ site.baseurl }}{% link api/doc.md %})[] -> > Prior query results, if available, such as from server rendering +> > Prior query results, if available, such as from server rendering. This should be an array of Doc instances, as obtained from `connection.get(collection, id)`. If the docs' data is already available, invoke [`ingestSnapshot`]({{ site.baseurl }}{% link api/doc.md %}#ingestsnapshot) for each Doc instance beforehand to avoid re-transferring the data from the server. > `options.*` -- any From c7e63b1b6c9206a3db29976ffd0adce2e4d8fdd7 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 3 May 2022 10:49:27 +0100 Subject: [PATCH 05/61] =?UTF-8?q?=F0=9F=92=A5=20Update=20Node.js=20test=20?= =?UTF-8?q?matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node.js 18 [has been released][1], and will be the next LTS version, so we add this to our test matrix. Node.js 12 has also been [end-of-lifed][2], and is dropped from the test matrix. [1]: https://nodejs.org/en/blog/announcements/v18-release-announce/ [2]: https://nodejs.org/en/about/releases/ --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3e08ba4a..4070a8d59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,9 +15,9 @@ jobs: strategy: matrix: node: - - 12 - 14 - 16 + - 18 services: mongodb: image: mongo:4.4 From 6b5b038d1b90e6c33dfd048a7636c69222cfa632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 May 2022 10:33:08 +0000 Subject: [PATCH 06/61] Bump nokogiri from 1.13.3 to 1.13.4 in /docs Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.3 to 1.13.4. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.3...v1.13.4) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index e232af6b8..c827a1c5b 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -232,7 +232,7 @@ GEM jekyll-seo-tag (~> 2.1) minitest (5.15.0) multipart-post (2.1.1) - nokogiri (1.13.3) + nokogiri (1.13.4) mini_portile2 (~> 2.8.0) racc (~> 1.4) octokit (4.22.0) From 7bf6781575433c190b6e04a5756eaf9b9c15d714 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 3 May 2022 17:08:59 +0100 Subject: [PATCH 07/61] 3.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6b9a04908..387e17cfc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "2.2.5", + "version": "3.0.0", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 99b551924b88e64ae30c6aa669458efaa15c8622 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:12:58 +0100 Subject: [PATCH 08/61] =?UTF-8?q?=F0=9F=91=B7=20Bump=20GH=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4070a8d59..c1eec7e7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,8 +25,8 @@ jobs: - 27017:27017 timeout-minutes: 10 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: ${{ matrix.node }} - name: Install From 16d7431c6d5314da3e4e5215721379618556b086 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:25:09 +0100 Subject: [PATCH 09/61] =?UTF-8?q?=F0=9F=91=B7=20Verbose=20GH=20Actions=20i?= =?UTF-8?q?nstallation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1eec7e7f..2363d7ec4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: with: node-version: ${{ matrix.node }} - name: Install - run: npm install + run: npm install --verbose - name: Lint run: npm run lint - name: Test From e71c91b88f69a2f49994141a9688a662d41e6612 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:17:38 +0100 Subject: [PATCH 10/61] =?UTF-8?q?=F0=9F=91=B7=20Increase=20GH=20Actions=20?= =?UTF-8?q?timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Node.js v14 `npm install` script keeps stalling. This change increases our timeout to try and let it pass. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2363d7ec4..316732d51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: image: mongo:4.4 ports: - 27017:27017 - timeout-minutes: 10 + timeout-minutes: 20 steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 From 400ec52326623a2125e8b6f5a19d8d9613eeffeb Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:21:55 +0100 Subject: [PATCH 11/61] =?UTF-8?q?=F0=9F=91=B7=20Use=20HTTPS=20to=20fetch?= =?UTF-8?q?=20`json0`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The older version of `npm` can have issues installing packages directly from Github over SSH. This change updates our `package.json` to explicitly fetch over HTTPS. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 387e17cfc..e46abc043 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "lolex": "^5.1.1", "mocha": "^8.2.1", "nyc": "^14.1.1", - "ot-json0-v2": "ottypes/json0", + "ot-json0-v2": "https://github.com/ottypes/json0/ottypes/json0", "ot-json1": "^0.3.0", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@=1.1.0", From b1c10d66be12d9fa2d92bb37198d7c341a368469 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:23:20 +0100 Subject: [PATCH 12/61] =?UTF-8?q?=F0=9F=91=B7=20Fix=20`json0`=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e46abc043..0e34d12b2 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "lolex": "^5.1.1", "mocha": "^8.2.1", "nyc": "^14.1.1", - "ot-json0-v2": "https://github.com/ottypes/json0/ottypes/json0", + "ot-json0-v2": "https://github.com/ottypes/json0", "ot-json1": "^0.3.0", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@=1.1.0", From df19240ff20e2c866ded26ca826855769e137860 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:24:31 +0100 Subject: [PATCH 13/61] =?UTF-8?q?Revert=20"=F0=9F=91=B7=20Increase=20GH=20?= =?UTF-8?q?Actions=20timeout"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e71c91b88f69a2f49994141a9688a662d41e6612. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 316732d51..2363d7ec4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: image: mongo:4.4 ports: - 27017:27017 - timeout-minutes: 20 + timeout-minutes: 10 steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 From 8ed35d5c417615d8e71aed148665052c6a9722a8 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:24:39 +0100 Subject: [PATCH 14/61] =?UTF-8?q?Revert=20"=F0=9F=91=B7=20Verbose=20GH=20A?= =?UTF-8?q?ctions=20installation"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 16d7431c6d5314da3e4e5215721379618556b086. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2363d7ec4..c1eec7e7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: with: node-version: ${{ matrix.node }} - name: Install - run: npm install --verbose + run: npm install - name: Lint run: npm run lint - name: Test From bec7e6d4accdccb37e5f0bd3440a7884eb147540 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:26:07 +0100 Subject: [PATCH 15/61] =?UTF-8?q?=F0=9F=93=8C=20Pin=20`json0`=20dev=20depe?= =?UTF-8?q?ndency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's best practice to specify a version when depending directly on a repo. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0e34d12b2..30d978c03 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "lolex": "^5.1.1", "mocha": "^8.2.1", "nyc": "^14.1.1", - "ot-json0-v2": "https://github.com/ottypes/json0", + "ot-json0-v2": "https://github.com/ottypes/json0#90a3ae26364c4fa3b19b6df34dad46707a704421", "ot-json1": "^0.3.0", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@=1.1.0", From 8ec52793270ffd2b4c43585b0a3247571650dfcb Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:58:35 +0100 Subject: [PATCH 16/61] =?UTF-8?q?=F0=9F=93=9D=20Update=20presence=20docume?= =?UTF-8?q?ntation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change updates our presence documentation to highlight that disconnections can be seen with a `null` value. --- docs/api/presence.md | 3 +++ docs/presence.md | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/api/presence.md b/docs/api/presence.md index 8d55b04c3..d8768a7a9 100644 --- a/docs/api/presence.md +++ b/docs/api/presence.md @@ -143,6 +143,9 @@ presence.on('receive', function(id, value) { ... }); > The presence value. The structure of this object will depend on the [type]({{ site.baseurl }}{% link types/index.md %}) +{: .info } +> A `null` value means the remote client is no longer present in the document (e.g. they disconnected) + ### `'error'` An error has occurred. diff --git a/docs/presence.md b/docs/presence.md index 7555d0994..9567446e2 100644 --- a/docs/presence.md +++ b/docs/presence.md @@ -22,6 +22,14 @@ In this case, clients just need to subscribe to a common channel using [`connect ```js const presence = connection.getPresence('my-channel') presence.subscribe() + +presence.on('receive', (presenceId, update) => { + if (update === null) { + // The remote client is no longer present in the document + } else { + // Handle the new value by updating UI, etc. + } +}) ``` In order to send presence information to other clients, a [`LocalPresence`]({{ site.baseurl }}{% link api/local-presence.md %}) should be created. The presence object can take any arbitrary value @@ -51,6 +59,14 @@ Clients subscribe to a particular [`Doc`]({{ site.baseurl }}{% link api/doc.md % ```js const presence = connection.getDocPresence(collection, id) presence.subscribe() + +presence.on('receive', (presenceId, update) => { + if (update === null) { + // The remote client is no longer present in the document + } else { + // Handle the new value by updating UI, etc. + } +}) ``` The shape of the presence value will be defined by the [type]({{ site.baseurl }}{% link types/index.md %}): From b6813660aa1d31dcce54c3c7627ff9e0aef4daad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:27:46 +0000 Subject: [PATCH 17/61] Bump nokogiri from 1.13.4 to 1.13.7 in /docs Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.4 to 1.13.7. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.4...v1.13.7) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index c827a1c5b..11f965e53 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -232,7 +232,7 @@ GEM jekyll-seo-tag (~> 2.1) minitest (5.15.0) multipart-post (2.1.1) - nokogiri (1.13.4) + nokogiri (1.13.7) mini_portile2 (~> 2.8.0) racc (~> 1.4) octokit (4.22.0) From 95297e5a8a3dafa59feab8513e1aee18cd199d61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Jul 2022 14:08:43 +0000 Subject: [PATCH 18/61] Bump tzinfo from 1.2.9 to 1.2.10 in /docs Bumps [tzinfo](https://github.com/tzinfo/tzinfo) from 1.2.9 to 1.2.10. - [Release notes](https://github.com/tzinfo/tzinfo/releases) - [Changelog](https://github.com/tzinfo/tzinfo/blob/master/CHANGES.md) - [Commits](https://github.com/tzinfo/tzinfo/compare/v1.2.9...v1.2.10) --- updated-dependencies: - dependency-name: tzinfo dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 11f965e53..a08eae2bf 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -265,7 +265,7 @@ GEM thread_safe (0.3.6) typhoeus (1.4.0) ethon (>= 0.9.0) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) tzinfo-data (1.2021.1) tzinfo (>= 1.0.0) From e91488d6dff741bd9f667d07560910b233e811d3 Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Mon, 8 Aug 2022 23:41:39 -0700 Subject: [PATCH 19/61] Refactor action code strings into an enum-like object --- lib/agent.js | 71 ++++++++++++++++++++--------------- lib/client/connection.js | 53 +++++++++++++------------- lib/shared/message-actions.js | 21 +++++++++++ 3 files changed, 88 insertions(+), 57 deletions(-) create mode 100644 lib/shared/message-actions.js diff --git a/lib/agent.js b/lib/agent.js index ebb3bc7d9..eca8057e2 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -1,8 +1,9 @@ var hat = require('hat'); +var ShareDBError = require('./error'); +var logger = require('./logger'); +var Action = require('./shared/message-actions').Action; var types = require('./types'); var util = require('./util'); -var logger = require('./logger'); -var ShareDBError = require('./error'); var ERROR_CODE = ShareDBError.CODES; @@ -58,7 +59,7 @@ function Agent(backend, stream) { this.custom = {}; // Send the legacy message to initialize old clients with the random agent Id - this.send(this._initMessage('init')); + this.send(this._initMessage(Action.InitLegacy)); } module.exports = Agent; @@ -174,7 +175,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query var agent = this; emitter.onExtra = function(extra) { - agent.send({a: 'q', id: queryId, extra: extra}); + agent.send({a: Action.QueryReply, id: queryId, extra: extra}); }; emitter.onDiff = function(diff) { @@ -186,7 +187,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query } // Consider stripping the collection out of the data we send here // if it matches the query's collection. - agent.send({a: 'q', id: queryId, diff: diff}); + agent.send({a: Action.QueryReply, id: queryId, diff: diff}); }; emitter.onError = function(err) { @@ -250,7 +251,7 @@ Agent.prototype.send = function(message) { Agent.prototype._sendOp = function(collection, id, op) { var message = { - a: 'op', + a: Action.Op, c: collection, d: id, v: op.v, @@ -354,22 +355,30 @@ Agent.prototype._open = function() { // Check a request to see if its valid. Returns an error if there's a problem. Agent.prototype._checkRequest = function(request) { - if (request.a === 'qf' || request.a === 'qs' || request.a === 'qu') { + if (request.a === Action.QueryFetch || request.a === Action.QuerySubscribe || request.a === Action.QueryUnsubscribe) { // Query messages need an ID property. if (typeof request.id !== 'number') return 'Missing query ID'; - } else if (request.a === 'op' || request.a === 'f' || request.a === 's' || request.a === 'u' || request.a === 'p') { + } else if (request.a === Action.Op || + request.a === Action.Fetch || + request.a === Action.Subscribe || + request.a === Action.Unsubscribe || + request.a === Action.Presence) { // Doc-based request. if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; if (request.d != null && typeof request.d !== 'string') return 'Invalid id'; - if (request.a === 'op' || request.a === 'p') { + if (request.a === Action.Op || request.a === Action.Presence) { if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version'; } - if (request.a === 'p') { + if (request.a === Action.Presence) { if (typeof request.id !== 'string') return 'Missing presence ID'; } - } else if (request.a === 'bf' || request.a === 'bs' || request.a === 'bu') { + } else if ( + request.a === Action.BulkFetch || + request.a === Action.BulkSubscribe || + request.a === Action.BulkUnsubscribe + ) { // Bulk request if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; if (typeof request.b !== 'object') return 'Invalid bulk subscribe data'; @@ -383,28 +392,28 @@ Agent.prototype._handleMessage = function(request, callback) { if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); switch (request.a) { - case 'hs': + case Action.Handshake: if (request.id) this.src = request.id; - return callback(null, this._initMessage('hs')); - case 'qf': + return callback(null, this._initMessage(Action.Handshake)); + case Action.QueryFetch: return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback); - case 'qs': + case Action.QuerySubscribe: return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback); - case 'qu': + case Action.QueryUnsubscribe: return this._queryUnsubscribe(request.id, callback); - case 'bf': + case Action.BulkFetch: return this._fetchBulk(request.c, request.b, callback); - case 'bs': + case Action.BulkSubscribe: return this._subscribeBulk(request.c, request.b, callback); - case 'bu': + case Action.BulkUnsubscribe: return this._unsubscribeBulk(request.c, request.b, callback); - case 'f': + case Action.Fetch: return this._fetch(request.c, request.d, request.v, callback); - case 's': + case Action.Subscribe: return this._subscribe(request.c, request.d, request.v, callback); - case 'u': + case Action.Unsubscribe: return this._unsubscribe(request.c, request.d, callback); - case 'op': + case Action.Op: // Normalize the properties submitted var op = createClientOp(request, this._src()); if (op.seq >= util.MAX_SAFE_INTEGER) { @@ -415,11 +424,11 @@ Agent.prototype._handleMessage = function(request, callback) { } if (!op) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid op message')); return this._submit(request.c, request.d, op, callback); - case 'nf': + case Action.SnapshotFetch: return this._fetchSnapshot(request.c, request.d, request.v, callback); - case 'nt': + case Action.SnapshotFetchByTimestamp: return this._fetchSnapshotByTimestamp(request.c, request.d, request.ts, callback); - case 'p': + case Action.Presence: if (!this.backend.presenceEnabled) return; var presence = this._createPresence(request); if (presence.t && !util.supportsPresence(types.map[presence.t])) { @@ -429,10 +438,10 @@ Agent.prototype._handleMessage = function(request, callback) { }); } return this._broadcastPresence(presence, callback); - case 'ps': + case Action.PresenceSubscribe: if (!this.backend.presenceEnabled) return; return this._subscribePresence(request.ch, request.seq, callback); - case 'pu': + case Action.PresenceUnsubscribe: return this._unsubscribePresence(request.ch, request.seq, callback); default: callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); @@ -761,7 +770,7 @@ Agent.prototype._broadcastPresence = function(presence, callback) { Agent.prototype._createPresence = function(request) { return { - a: 'p', + a: Action.Presence, ch: request.ch, src: this._src(), id: request.id, // Presence ID, not Doc ID (which is 'd') @@ -813,7 +822,7 @@ Agent.prototype._requestPresence = function(channel, callback) { Agent.prototype._handlePresenceData = function(presence) { if (presence.src === this._src()) return; - if (presence.r) return this.send({a: 'pr', ch: presence.ch}); + if (presence.r) return this.send({a: Action.PresenceRequest, ch: presence.ch}); var backend = this.backend; var context = { @@ -824,7 +833,7 @@ Agent.prototype._handlePresenceData = function(presence) { backend.trigger(backend.MIDDLEWARE_ACTIONS.sendPresence, this, context, function(error) { if (error) { if (backend.doNotForwardSendPresenceErrorsToClient) backend.errorHandler(error, {agent: agent}); - else agent.send({a: 'p', ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); + else agent.send({a: Action.Presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); return; } agent.send(presence); diff --git a/lib/client/connection.js b/lib/client/connection.js index d29570afb..1b551af12 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -6,6 +6,7 @@ var SnapshotVersionRequest = require('./snapshot-request/snapshot-version-reques var SnapshotTimestampRequest = require('./snapshot-request/snapshot-timestamp-request'); var emitter = require('../emitter'); var ShareDBError = require('../error'); +var Action = require('../shared/message-actions').Action; var types = require('../types'); var util = require('../util'); var logger = require('../logger'); @@ -194,25 +195,25 @@ Connection.prototype.handleMessage = function(message) { // Switch on the message action. Most messages are for documents and are // handled in the doc class. switch (message.a) { - case 'init': + case Action.InitLegacy: // Client initialization packet return this._handleLegacyInit(message); - case 'hs': + case Action.Handshake: return this._handleHandshake(err, message); - case 'qf': + case Action.QueryFetch: var query = this.queries[message.id]; if (query) query._handleFetch(err, message.data, message.extra); return; - case 'qs': + case Action.QuerySubscribe: var query = this.queries[message.id]; if (query) query._handleSubscribe(err, message.data, message.extra); return; - case 'qu': + case Action.QueryUnsubscribe: // Queries are removed immediately on calls to destroy, so we ignore // replies to query unsubscribes. Perhaps there should be a callback for // destroy, but this is currently unimplemented return; - case 'q': + case Action.QueryReply: // Query message. Pass this to the appropriate query object. var query = this.queries[message.id]; if (!query) return; @@ -221,36 +222,36 @@ Connection.prototype.handleMessage = function(message) { if (message.hasOwnProperty('extra')) query._handleExtra(message.extra); return; - case 'bf': + case Action.BulkFetch: return this._handleBulkMessage(err, message, '_handleFetch'); - case 'bs': - case 'bu': + case Action.BulkSubscribe: + case Action.BulkUnsubscribe: return this._handleBulkMessage(err, message, '_handleSubscribe'); - case 'nf': - case 'nt': + case Action.SnapshotFetch: + case Action.SnapshotFetchByTimestamp: return this._handleSnapshotFetch(err, message); - case 'f': + case Action.Fetch: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleFetch(err, message.data); return; - case 's': - case 'u': + case Action.Subscribe: + case Action.Unsubscribe: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleSubscribe(err, message.data); return; - case 'op': + case Action.Op: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleOp(err, message); return; - case 'p': + case Action.Presence: return this._handlePresence(err, message); - case 'ps': + case Action.PresenceSubscribe: return this._handlePresenceSubscribe(err, message); - case 'pu': + case Action.PresenceUnsubscribe: return this._handlePresenceUnsubscribe(err, message); - case 'pr': + case Action.PresenceRequest: return this._handlePresenceRequest(err, message); default: @@ -434,22 +435,22 @@ Connection.prototype._sendAction = function(action, doc, version) { }; Connection.prototype.sendFetch = function(doc) { - return this._sendAction('f', doc, doc.version); + return this._sendAction(Action.Fetch, doc, doc.version); }; Connection.prototype.sendSubscribe = function(doc) { - return this._sendAction('s', doc, doc.version); + return this._sendAction(Action.Subscribe, doc, doc.version); }; Connection.prototype.sendUnsubscribe = function(doc) { - return this._sendAction('u', doc); + return this._sendAction(Action.Unsubscribe, doc); }; Connection.prototype.sendOp = function(doc, op) { // Ensure the doc is registered so that it receives the reply message this._addDoc(doc); var message = { - a: 'op', + a: Action.Op, c: doc.collection, d: doc.id, v: doc.version, @@ -553,7 +554,7 @@ Connection.prototype._destroyQuery = function(query) { // The callback should have the signature function(error, results, extra) // where results is a list of Doc objects. Connection.prototype.createFetchQuery = function(collection, q, options, callback) { - return this._createQuery('qf', collection, q, options, callback); + return this._createQuery(Action.QueryFetch, collection, q, options, callback); }; // Create a subscribe query. Subscribe queries return with the initial data @@ -563,7 +564,7 @@ Connection.prototype.createFetchQuery = function(collection, q, options, callbac // If present, the callback should have the signature function(error, results, extra) // where results is a list of Doc objects. Connection.prototype.createSubscribeQuery = function(collection, q, options, callback) { - return this._createQuery('qs', collection, q, options, callback); + return this._createQuery(Action.QuerySubscribe, collection, q, options, callback); }; Connection.prototype.hasPending = function() { @@ -716,7 +717,7 @@ Connection.prototype._handleLegacyInit = function(message) { }; Connection.prototype._initializeHandshake = function() { - this.send({a: 'hs', id: this.id}); + this.send({a: Action.Handshake, id: this.id}); }; Connection.prototype._handleHandshake = function(error, message) { diff --git a/lib/shared/message-actions.js b/lib/shared/message-actions.js new file mode 100644 index 000000000..33beec7ec --- /dev/null +++ b/lib/shared/message-actions.js @@ -0,0 +1,21 @@ +exports.Action = { + InitLegacy: 'init', + Handshake: 'hs', + QueryFetch: 'qf', + QuerySubscribe: 'qs', + QueryUnsubscribe: 'qu', + QueryReply: 'q', + BulkFetch: 'bf', + BulkSubscribe: 'bs', + BulkUnsubscribe: 'bu', + Fetch: 'f', + Subscribe: 's', + Unsubscribe: 'u', + Op: 'op', + SnapshotFetch: 'nf', + SnapshotFetchByTimestamp: 'nt', + Presence: 'p', + PresenceSubscribe: 'ps', + PresenceUnsubscribe: 'pu', + PresenceRequest: 'pr' +}; From 461662d9df246d5e52023c08e58b6e9db0dcf9f2 Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 9 Aug 2022 09:17:59 -0700 Subject: [PATCH 20/61] Updates to message-actios from code review feedback --- lib/agent.js | 72 +++++++++++++++++------------------ lib/message-actions.js | 21 ++++++++++ lib/shared/message-actions.js | 21 ---------- 3 files changed, 57 insertions(+), 57 deletions(-) create mode 100644 lib/message-actions.js delete mode 100644 lib/shared/message-actions.js diff --git a/lib/agent.js b/lib/agent.js index eca8057e2..4ebd846f2 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -1,7 +1,7 @@ var hat = require('hat'); var ShareDBError = require('./error'); var logger = require('./logger'); -var Action = require('./shared/message-actions').Action; +var Action = require('./message-actions').ACTIONS; var types = require('./types'); var util = require('./util'); @@ -59,7 +59,7 @@ function Agent(backend, stream) { this.custom = {}; // Send the legacy message to initialize old clients with the random agent Id - this.send(this._initMessage(Action.InitLegacy)); + this.send(this._initMessage(Action.initLegacy)); } module.exports = Agent; @@ -175,7 +175,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query var agent = this; emitter.onExtra = function(extra) { - agent.send({a: Action.QueryReply, id: queryId, extra: extra}); + agent.send({a: Action.queryReply, id: queryId, extra: extra}); }; emitter.onDiff = function(diff) { @@ -187,7 +187,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query } // Consider stripping the collection out of the data we send here // if it matches the query's collection. - agent.send({a: Action.QueryReply, id: queryId, diff: diff}); + agent.send({a: Action.queryReply, id: queryId, diff: diff}); }; emitter.onError = function(err) { @@ -251,7 +251,7 @@ Agent.prototype.send = function(message) { Agent.prototype._sendOp = function(collection, id, op) { var message = { - a: Action.Op, + a: Action.op, c: collection, d: id, v: op.v, @@ -355,29 +355,29 @@ Agent.prototype._open = function() { // Check a request to see if its valid. Returns an error if there's a problem. Agent.prototype._checkRequest = function(request) { - if (request.a === Action.QueryFetch || request.a === Action.QuerySubscribe || request.a === Action.QueryUnsubscribe) { + if (request.a === Action.queryFetch || request.a === Action.querySubscribe || request.a === Action.queryUnsubscribe) { // Query messages need an ID property. if (typeof request.id !== 'number') return 'Missing query ID'; - } else if (request.a === Action.Op || - request.a === Action.Fetch || - request.a === Action.Subscribe || - request.a === Action.Unsubscribe || - request.a === Action.Presence) { + } else if (request.a === Action.op || + request.a === Action.fetch || + request.a === Action.subscribe || + request.a === Action.unsubscribe || + request.a === Action.presence) { // Doc-based request. if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; if (request.d != null && typeof request.d !== 'string') return 'Invalid id'; - if (request.a === Action.Op || request.a === Action.Presence) { + if (request.a === Action.op || request.a === Action.presence) { if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version'; } - if (request.a === Action.Presence) { + if (request.a === Action.presence) { if (typeof request.id !== 'string') return 'Missing presence ID'; } } else if ( - request.a === Action.BulkFetch || - request.a === Action.BulkSubscribe || - request.a === Action.BulkUnsubscribe + request.a === Action.bulkFetch || + request.a === Action.bulkSubscribe || + request.a === Action.bulkUnsubscribe ) { // Bulk request if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; @@ -392,28 +392,28 @@ Agent.prototype._handleMessage = function(request, callback) { if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); switch (request.a) { - case Action.Handshake: + case Action.handshake: if (request.id) this.src = request.id; - return callback(null, this._initMessage(Action.Handshake)); - case Action.QueryFetch: + return callback(null, this._initMessage(Action.handshake)); + case Action.queryFetch: return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback); - case Action.QuerySubscribe: + case Action.querySubscribe: return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback); - case Action.QueryUnsubscribe: + case Action.queryUnsubscribe: return this._queryUnsubscribe(request.id, callback); - case Action.BulkFetch: + case Action.bulkFetch: return this._fetchBulk(request.c, request.b, callback); - case Action.BulkSubscribe: + case Action.bulkSubscribe: return this._subscribeBulk(request.c, request.b, callback); - case Action.BulkUnsubscribe: + case Action.bulkUnsubscribe: return this._unsubscribeBulk(request.c, request.b, callback); - case Action.Fetch: + case Action.fetch: return this._fetch(request.c, request.d, request.v, callback); - case Action.Subscribe: + case Action.subscribe: return this._subscribe(request.c, request.d, request.v, callback); - case Action.Unsubscribe: + case Action.unsubscribe: return this._unsubscribe(request.c, request.d, callback); - case Action.Op: + case Action.op: // Normalize the properties submitted var op = createClientOp(request, this._src()); if (op.seq >= util.MAX_SAFE_INTEGER) { @@ -424,11 +424,11 @@ Agent.prototype._handleMessage = function(request, callback) { } if (!op) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid op message')); return this._submit(request.c, request.d, op, callback); - case Action.SnapshotFetch: + case Action.snapshotFetch: return this._fetchSnapshot(request.c, request.d, request.v, callback); - case Action.SnapshotFetchByTimestamp: + case Action.snapshotFetchByTimestamp: return this._fetchSnapshotByTimestamp(request.c, request.d, request.ts, callback); - case Action.Presence: + case Action.presence: if (!this.backend.presenceEnabled) return; var presence = this._createPresence(request); if (presence.t && !util.supportsPresence(types.map[presence.t])) { @@ -438,10 +438,10 @@ Agent.prototype._handleMessage = function(request, callback) { }); } return this._broadcastPresence(presence, callback); - case Action.PresenceSubscribe: + case Action.presenceSubscribe: if (!this.backend.presenceEnabled) return; return this._subscribePresence(request.ch, request.seq, callback); - case Action.PresenceUnsubscribe: + case Action.presenceUnsubscribe: return this._unsubscribePresence(request.ch, request.seq, callback); default: callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); @@ -770,7 +770,7 @@ Agent.prototype._broadcastPresence = function(presence, callback) { Agent.prototype._createPresence = function(request) { return { - a: Action.Presence, + a: Action.presence, ch: request.ch, src: this._src(), id: request.id, // Presence ID, not Doc ID (which is 'd') @@ -822,7 +822,7 @@ Agent.prototype._requestPresence = function(channel, callback) { Agent.prototype._handlePresenceData = function(presence) { if (presence.src === this._src()) return; - if (presence.r) return this.send({a: Action.PresenceRequest, ch: presence.ch}); + if (presence.r) return this.send({a: Action.presenceRequest, ch: presence.ch}); var backend = this.backend; var context = { @@ -833,7 +833,7 @@ Agent.prototype._handlePresenceData = function(presence) { backend.trigger(backend.MIDDLEWARE_ACTIONS.sendPresence, this, context, function(error) { if (error) { if (backend.doNotForwardSendPresenceErrorsToClient) backend.errorHandler(error, {agent: agent}); - else agent.send({a: Action.Presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); + else agent.send({a: Action.presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); return; } agent.send(presence); diff --git a/lib/message-actions.js b/lib/message-actions.js new file mode 100644 index 000000000..5e2b59517 --- /dev/null +++ b/lib/message-actions.js @@ -0,0 +1,21 @@ +exports.ACTIONS = { + initLegacy: 'init', + handshake: 'hs', + queryFetch: 'qf', + querySubscribe: 'qs', + queryUnsubscribe: 'qu', + queryReply: 'q', + bulkFetch: 'bf', + bulkSubscribe: 'bs', + bulkUnsubscribe: 'bu', + fetch: 'f', + subscribe: 's', + unsubscribe: 'u', + op: 'op', + snapshotFetch: 'nf', + snapshotFetchByTimestamp: 'nt', + presence: 'p', + presenceSubscribe: 'ps', + presenceUnsubscribe: 'pu', + presenceRequest: 'pr' +}; diff --git a/lib/shared/message-actions.js b/lib/shared/message-actions.js deleted file mode 100644 index 33beec7ec..000000000 --- a/lib/shared/message-actions.js +++ /dev/null @@ -1,21 +0,0 @@ -exports.Action = { - InitLegacy: 'init', - Handshake: 'hs', - QueryFetch: 'qf', - QuerySubscribe: 'qs', - QueryUnsubscribe: 'qu', - QueryReply: 'q', - BulkFetch: 'bf', - BulkSubscribe: 'bs', - BulkUnsubscribe: 'bu', - Fetch: 'f', - Subscribe: 's', - Unsubscribe: 'u', - Op: 'op', - SnapshotFetch: 'nf', - SnapshotFetchByTimestamp: 'nt', - Presence: 'p', - PresenceSubscribe: 'ps', - PresenceUnsubscribe: 'pu', - PresenceRequest: 'pr' -}; From c4e82d6d5eaad77fac26942af81dd169c012449d Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 9 Aug 2022 09:23:39 -0700 Subject: [PATCH 21/61] Update action refactoring missed in automated renames --- lib/agent.js | 72 ++++++++++++++++++++-------------------- lib/client/connection.js | 56 +++++++++++++++---------------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/lib/agent.js b/lib/agent.js index 4ebd846f2..d72c84aba 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -1,7 +1,7 @@ var hat = require('hat'); var ShareDBError = require('./error'); var logger = require('./logger'); -var Action = require('./message-actions').ACTIONS; +var ACTIONS = require('./message-actions').ACTIONS; var types = require('./types'); var util = require('./util'); @@ -59,7 +59,7 @@ function Agent(backend, stream) { this.custom = {}; // Send the legacy message to initialize old clients with the random agent Id - this.send(this._initMessage(Action.initLegacy)); + this.send(this._initMessage(ACTIONS.initLegacy)); } module.exports = Agent; @@ -175,7 +175,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query var agent = this; emitter.onExtra = function(extra) { - agent.send({a: Action.queryReply, id: queryId, extra: extra}); + agent.send({a: ACTIONS.queryReply, id: queryId, extra: extra}); }; emitter.onDiff = function(diff) { @@ -187,7 +187,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query } // Consider stripping the collection out of the data we send here // if it matches the query's collection. - agent.send({a: Action.queryReply, id: queryId, diff: diff}); + agent.send({a: ACTIONS.queryReply, id: queryId, diff: diff}); }; emitter.onError = function(err) { @@ -251,7 +251,7 @@ Agent.prototype.send = function(message) { Agent.prototype._sendOp = function(collection, id, op) { var message = { - a: Action.op, + a: ACTIONS.op, c: collection, d: id, v: op.v, @@ -355,29 +355,29 @@ Agent.prototype._open = function() { // Check a request to see if its valid. Returns an error if there's a problem. Agent.prototype._checkRequest = function(request) { - if (request.a === Action.queryFetch || request.a === Action.querySubscribe || request.a === Action.queryUnsubscribe) { + if (request.a === ACTIONS.queryFetch || request.a === ACTIONS.querySubscribe || request.a === ACTIONS.queryUnsubscribe) { // Query messages need an ID property. if (typeof request.id !== 'number') return 'Missing query ID'; - } else if (request.a === Action.op || - request.a === Action.fetch || - request.a === Action.subscribe || - request.a === Action.unsubscribe || - request.a === Action.presence) { + } else if (request.a === ACTIONS.op || + request.a === ACTIONS.fetch || + request.a === ACTIONS.subscribe || + request.a === ACTIONS.unsubscribe || + request.a === ACTIONS.presence) { // Doc-based request. if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; if (request.d != null && typeof request.d !== 'string') return 'Invalid id'; - if (request.a === Action.op || request.a === Action.presence) { + if (request.a === ACTIONS.op || request.a === ACTIONS.presence) { if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version'; } - if (request.a === Action.presence) { + if (request.a === ACTIONS.presence) { if (typeof request.id !== 'string') return 'Missing presence ID'; } } else if ( - request.a === Action.bulkFetch || - request.a === Action.bulkSubscribe || - request.a === Action.bulkUnsubscribe + request.a === ACTIONS.bulkFetch || + request.a === ACTIONS.bulkSubscribe || + request.a === ACTIONS.bulkUnsubscribe ) { // Bulk request if (request.c != null && typeof request.c !== 'string') return 'Invalid collection'; @@ -392,28 +392,28 @@ Agent.prototype._handleMessage = function(request, callback) { if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); switch (request.a) { - case Action.handshake: + case ACTIONS.handshake: if (request.id) this.src = request.id; - return callback(null, this._initMessage(Action.handshake)); - case Action.queryFetch: + return callback(null, this._initMessage(ACTIONS.handshake)); + case ACTIONS.queryFetch: return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback); - case Action.querySubscribe: + case ACTIONS.querySubscribe: return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback); - case Action.queryUnsubscribe: + case ACTIONS.queryUnsubscribe: return this._queryUnsubscribe(request.id, callback); - case Action.bulkFetch: + case ACTIONS.bulkFetch: return this._fetchBulk(request.c, request.b, callback); - case Action.bulkSubscribe: + case ACTIONS.bulkSubscribe: return this._subscribeBulk(request.c, request.b, callback); - case Action.bulkUnsubscribe: + case ACTIONS.bulkUnsubscribe: return this._unsubscribeBulk(request.c, request.b, callback); - case Action.fetch: + case ACTIONS.fetch: return this._fetch(request.c, request.d, request.v, callback); - case Action.subscribe: + case ACTIONS.subscribe: return this._subscribe(request.c, request.d, request.v, callback); - case Action.unsubscribe: + case ACTIONS.unsubscribe: return this._unsubscribe(request.c, request.d, callback); - case Action.op: + case ACTIONS.op: // Normalize the properties submitted var op = createClientOp(request, this._src()); if (op.seq >= util.MAX_SAFE_INTEGER) { @@ -424,11 +424,11 @@ Agent.prototype._handleMessage = function(request, callback) { } if (!op) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid op message')); return this._submit(request.c, request.d, op, callback); - case Action.snapshotFetch: + case ACTIONS.snapshotFetch: return this._fetchSnapshot(request.c, request.d, request.v, callback); - case Action.snapshotFetchByTimestamp: + case ACTIONS.snapshotFetchByTimestamp: return this._fetchSnapshotByTimestamp(request.c, request.d, request.ts, callback); - case Action.presence: + case ACTIONS.presence: if (!this.backend.presenceEnabled) return; var presence = this._createPresence(request); if (presence.t && !util.supportsPresence(types.map[presence.t])) { @@ -438,10 +438,10 @@ Agent.prototype._handleMessage = function(request, callback) { }); } return this._broadcastPresence(presence, callback); - case Action.presenceSubscribe: + case ACTIONS.presenceSubscribe: if (!this.backend.presenceEnabled) return; return this._subscribePresence(request.ch, request.seq, callback); - case Action.presenceUnsubscribe: + case ACTIONS.presenceUnsubscribe: return this._unsubscribePresence(request.ch, request.seq, callback); default: callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); @@ -770,7 +770,7 @@ Agent.prototype._broadcastPresence = function(presence, callback) { Agent.prototype._createPresence = function(request) { return { - a: Action.presence, + a: ACTIONS.presence, ch: request.ch, src: this._src(), id: request.id, // Presence ID, not Doc ID (which is 'd') @@ -822,7 +822,7 @@ Agent.prototype._requestPresence = function(channel, callback) { Agent.prototype._handlePresenceData = function(presence) { if (presence.src === this._src()) return; - if (presence.r) return this.send({a: Action.presenceRequest, ch: presence.ch}); + if (presence.r) return this.send({a: ACTIONS.presenceRequest, ch: presence.ch}); var backend = this.backend; var context = { @@ -833,7 +833,7 @@ Agent.prototype._handlePresenceData = function(presence) { backend.trigger(backend.MIDDLEWARE_ACTIONS.sendPresence, this, context, function(error) { if (error) { if (backend.doNotForwardSendPresenceErrorsToClient) backend.errorHandler(error, {agent: agent}); - else agent.send({a: Action.presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); + else agent.send({a: ACTIONS.presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); return; } agent.send(presence); diff --git a/lib/client/connection.js b/lib/client/connection.js index 1b551af12..592124bec 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -6,7 +6,7 @@ var SnapshotVersionRequest = require('./snapshot-request/snapshot-version-reques var SnapshotTimestampRequest = require('./snapshot-request/snapshot-timestamp-request'); var emitter = require('../emitter'); var ShareDBError = require('../error'); -var Action = require('../shared/message-actions').Action; +var ACTIONS = require('../message-actions').ACTIONS; var types = require('../types'); var util = require('../util'); var logger = require('../logger'); @@ -195,25 +195,25 @@ Connection.prototype.handleMessage = function(message) { // Switch on the message action. Most messages are for documents and are // handled in the doc class. switch (message.a) { - case Action.InitLegacy: + case ACTIONS.initLegacy: // Client initialization packet return this._handleLegacyInit(message); - case Action.Handshake: + case ACTIONS.handshake: return this._handleHandshake(err, message); - case Action.QueryFetch: + case ACTIONS.queryFetch: var query = this.queries[message.id]; if (query) query._handleFetch(err, message.data, message.extra); return; - case Action.QuerySubscribe: + case ACTIONS.querySubscribe: var query = this.queries[message.id]; if (query) query._handleSubscribe(err, message.data, message.extra); return; - case Action.QueryUnsubscribe: + case ACTIONS.queryUnsubscribe: // Queries are removed immediately on calls to destroy, so we ignore // replies to query unsubscribes. Perhaps there should be a callback for // destroy, but this is currently unimplemented return; - case Action.QueryReply: + case ACTIONS.queryReply: // Query message. Pass this to the appropriate query object. var query = this.queries[message.id]; if (!query) return; @@ -222,36 +222,36 @@ Connection.prototype.handleMessage = function(message) { if (message.hasOwnProperty('extra')) query._handleExtra(message.extra); return; - case Action.BulkFetch: + case ACTIONS.bulkFetch: return this._handleBulkMessage(err, message, '_handleFetch'); - case Action.BulkSubscribe: - case Action.BulkUnsubscribe: + case ACTIONS.bulkSubscribe: + case ACTIONS.bulkUnsubscribe: return this._handleBulkMessage(err, message, '_handleSubscribe'); - case Action.SnapshotFetch: - case Action.SnapshotFetchByTimestamp: + case ACTIONS.snapshotFetch: + case ACTIONS.snapshotFetchByTimestamp: return this._handleSnapshotFetch(err, message); - case Action.Fetch: + case ACTIONS.fetch: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleFetch(err, message.data); return; - case Action.Subscribe: - case Action.Unsubscribe: + case ACTIONS.subscribe: + case ACTIONS.unsubscribe: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleSubscribe(err, message.data); return; - case Action.Op: + case ACTIONS.op: var doc = this.getExisting(message.c, message.d); if (doc) doc._handleOp(err, message); return; - case Action.Presence: + case ACTIONS.presence: return this._handlePresence(err, message); - case Action.PresenceSubscribe: + case ACTIONS.presenceSubscribe: return this._handlePresenceSubscribe(err, message); - case Action.PresenceUnsubscribe: + case ACTIONS.presenceUnsubscribe: return this._handlePresenceUnsubscribe(err, message); - case Action.PresenceRequest: + case ACTIONS.presenceRequest: return this._handlePresenceRequest(err, message); default: @@ -417,7 +417,7 @@ Connection.prototype._sendBulk = function(action, collection, values) { } }; -Connection.prototype._sendAction = function(action, doc, version) { +Connection.prototype._sendActions = function(action, doc, version) { // Ensure the doc is registered so that it receives the reply message this._addDoc(doc); if (this.bulk) { @@ -435,22 +435,22 @@ Connection.prototype._sendAction = function(action, doc, version) { }; Connection.prototype.sendFetch = function(doc) { - return this._sendAction(Action.Fetch, doc, doc.version); + return this._sendActions(ACTIONS.fetch, doc, doc.version); }; Connection.prototype.sendSubscribe = function(doc) { - return this._sendAction(Action.Subscribe, doc, doc.version); + return this._sendActions(ACTIONS.subscribe, doc, doc.version); }; Connection.prototype.sendUnsubscribe = function(doc) { - return this._sendAction(Action.Unsubscribe, doc); + return this._sendActions(ACTIONS.unsubscribe, doc); }; Connection.prototype.sendOp = function(doc, op) { // Ensure the doc is registered so that it receives the reply message this._addDoc(doc); var message = { - a: Action.Op, + a: ACTIONS.op, c: doc.collection, d: doc.id, v: doc.version, @@ -554,7 +554,7 @@ Connection.prototype._destroyQuery = function(query) { // The callback should have the signature function(error, results, extra) // where results is a list of Doc objects. Connection.prototype.createFetchQuery = function(collection, q, options, callback) { - return this._createQuery(Action.QueryFetch, collection, q, options, callback); + return this._createQuery(ACTIONS.queryFetch, collection, q, options, callback); }; // Create a subscribe query. Subscribe queries return with the initial data @@ -564,7 +564,7 @@ Connection.prototype.createFetchQuery = function(collection, q, options, callbac // If present, the callback should have the signature function(error, results, extra) // where results is a list of Doc objects. Connection.prototype.createSubscribeQuery = function(collection, q, options, callback) { - return this._createQuery(Action.QuerySubscribe, collection, q, options, callback); + return this._createQuery(ACTIONS.querySubscribe, collection, q, options, callback); }; Connection.prototype.hasPending = function() { @@ -717,7 +717,7 @@ Connection.prototype._handleLegacyInit = function(message) { }; Connection.prototype._initializeHandshake = function() { - this.send({a: Action.Handshake, id: this.id}); + this.send({a: ACTIONS.handshake, id: this.id}); }; Connection.prototype._handleHandshake = function(error, message) { From 8879b28e1a6a8f314e1fd375b1fc587b31baaa4a Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 9 Aug 2022 09:28:07 -0700 Subject: [PATCH 22/61] Rename queryReply to queryUpdate, use shared action object from query.js --- lib/agent.js | 4 ++-- lib/client/connection.js | 2 +- lib/client/query.js | 5 +++-- lib/message-actions.js | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/agent.js b/lib/agent.js index d72c84aba..339b969ab 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -175,7 +175,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query var agent = this; emitter.onExtra = function(extra) { - agent.send({a: ACTIONS.queryReply, id: queryId, extra: extra}); + agent.send({a: ACTIONS.queryUpdate, id: queryId, extra: extra}); }; emitter.onDiff = function(diff) { @@ -187,7 +187,7 @@ Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query } // Consider stripping the collection out of the data we send here // if it matches the query's collection. - agent.send({a: ACTIONS.queryReply, id: queryId, diff: diff}); + agent.send({a: ACTIONS.queryUpdate, id: queryId, diff: diff}); }; emitter.onError = function(err) { diff --git a/lib/client/connection.js b/lib/client/connection.js index 592124bec..8e3209d2e 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -213,7 +213,7 @@ Connection.prototype.handleMessage = function(message) { // replies to query unsubscribes. Perhaps there should be a callback for // destroy, but this is currently unimplemented return; - case ACTIONS.queryReply: + case ACTIONS.queryUpdate: // Query message. Pass this to the appropriate query object. var query = this.queries[message.id]; if (!query) return; diff --git a/lib/client/query.js b/lib/client/query.js index 5815e977f..a8665dc8a 100644 --- a/lib/client/query.js +++ b/lib/client/query.js @@ -1,4 +1,5 @@ var emitter = require('../emitter'); +var ACTIONS = require('../message-actions').ACTIONS; var util = require('../util'); // Queries are live requests to the database for particular sets of fields. @@ -75,8 +76,8 @@ Query.prototype.send = function() { // Destroy the query object. Any subsequent messages for the query will be // ignored by the connection. Query.prototype.destroy = function(callback) { - if (this.connection.canSend && this.action === 'qs') { - this.connection.send({a: 'qu', id: this.id}); + if (this.connection.canSend && this.action === ACTIONS.querySubscribe) { + this.connection.send({a: ACTIONS.queryUnsubscribe, id: this.id}); } this.connection._destroyQuery(this); // There is a callback for consistency, but we don't actually wait for the diff --git a/lib/message-actions.js b/lib/message-actions.js index 5e2b59517..ad2b7536b 100644 --- a/lib/message-actions.js +++ b/lib/message-actions.js @@ -4,7 +4,7 @@ exports.ACTIONS = { queryFetch: 'qf', querySubscribe: 'qs', queryUnsubscribe: 'qu', - queryReply: 'q', + queryUpdate: 'q', bulkFetch: 'bf', bulkSubscribe: 'bs', bulkUnsubscribe: 'bu', From 55fa1f3e63e415e83484ccfcaa68a05400232f58 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 9 Aug 2022 17:31:37 +0100 Subject: [PATCH 23/61] Catch missed actions --- lib/agent.js | 6 +++++- lib/client/presence/local-presence.js | 3 ++- lib/client/presence/presence.js | 3 ++- lib/client/snapshot-request/snapshot-timestamp-request.js | 3 ++- lib/client/snapshot-request/snapshot-version-request.js | 3 ++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/agent.js b/lib/agent.js index 339b969ab..ce2e43a46 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -355,7 +355,11 @@ Agent.prototype._open = function() { // Check a request to see if its valid. Returns an error if there's a problem. Agent.prototype._checkRequest = function(request) { - if (request.a === ACTIONS.queryFetch || request.a === ACTIONS.querySubscribe || request.a === ACTIONS.queryUnsubscribe) { + if ( + request.a === ACTIONS.queryFetch || + request.a === ACTIONS.querySubscribe || + request.a === ACTIONS.queryUnsubscribe + ) { // Query messages need an ID property. if (typeof request.id !== 'number') return 'Missing query ID'; } else if (request.a === ACTIONS.op || diff --git a/lib/client/presence/local-presence.js b/lib/client/presence/local-presence.js index b9468d045..204007617 100644 --- a/lib/client/presence/local-presence.js +++ b/lib/client/presence/local-presence.js @@ -1,4 +1,5 @@ var emitter = require('../../emitter'); +var ACTIONS = require('../../message-actions').ACTIONS; var util = require('../../util'); module.exports = LocalPresence; @@ -59,7 +60,7 @@ LocalPresence.prototype._ack = function(error, presenceVersion) { LocalPresence.prototype._message = function() { return { - a: 'p', + a: ACTIONS.presence, ch: this.presence.channel, id: this.presenceId, p: this.value, diff --git a/lib/client/presence/presence.js b/lib/client/presence/presence.js index c0227f93e..107355e34 100644 --- a/lib/client/presence/presence.js +++ b/lib/client/presence/presence.js @@ -4,6 +4,7 @@ var RemotePresence = require('./remote-presence'); var util = require('../../util'); var async = require('async'); var hat = require('hat'); +var ACTIONS = require('../../message-actions').ACTIONS; module.exports = Presence; function Presence(connection, channel) { @@ -70,7 +71,7 @@ Presence.prototype.destroy = function(callback) { Presence.prototype._sendSubscriptionAction = function(wantSubscribe, callback) { this.wantSubscribe = !!wantSubscribe; - var action = this.wantSubscribe ? 'ps' : 'pu'; + var action = this.wantSubscribe ? ACTIONS.presenceSubscribe : ACTIONS.presenceUnsubscribe; var seq = this.connection._presenceSeq++; this._subscriptionCallbacksBySeq[seq] = callback; if (this.connection.canSend) { diff --git a/lib/client/snapshot-request/snapshot-timestamp-request.js b/lib/client/snapshot-request/snapshot-timestamp-request.js index 15789137b..8d4cec70d 100644 --- a/lib/client/snapshot-request/snapshot-timestamp-request.js +++ b/lib/client/snapshot-request/snapshot-timestamp-request.js @@ -1,5 +1,6 @@ var SnapshotRequest = require('./snapshot-request'); var util = require('../../util'); +var ACTIONS = require('../../message-actions').ACTIONS; module.exports = SnapshotTimestampRequest; @@ -17,7 +18,7 @@ SnapshotTimestampRequest.prototype = Object.create(SnapshotRequest.prototype); SnapshotTimestampRequest.prototype._message = function() { return { - a: 'nt', + a: ACTIONS.snapshotFetchByTimestamp, id: this.requestId, c: this.collection, d: this.id, diff --git a/lib/client/snapshot-request/snapshot-version-request.js b/lib/client/snapshot-request/snapshot-version-request.js index d352a676a..369bf82ee 100644 --- a/lib/client/snapshot-request/snapshot-version-request.js +++ b/lib/client/snapshot-request/snapshot-version-request.js @@ -1,5 +1,6 @@ var SnapshotRequest = require('./snapshot-request'); var util = require('../../util'); +var ACTIONS = require('../../message-actions').ACTIONS; module.exports = SnapshotVersionRequest; @@ -17,7 +18,7 @@ SnapshotVersionRequest.prototype = Object.create(SnapshotRequest.prototype); SnapshotVersionRequest.prototype._message = function() { return { - a: 'nf', + a: ACTIONS.snapshotFetch, id: this.requestId, c: this.collection, d: this.id, From bbeaeb9239e34d9185df05e561f62e2464dddc41 Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 9 Aug 2022 09:39:31 -0700 Subject: [PATCH 24/61] Add top-level export for MESSAGE_ACTIONS --- lib/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/index.js b/lib/index.js index df4b3f0a6..46fd974c7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -9,6 +9,7 @@ Backend.logger = require('./logger'); Backend.MemoryDB = require('./db/memory'); Backend.MemoryMilestoneDB = require('./milestone-db/memory'); Backend.MemoryPubSub = require('./pubsub/memory'); +Backend.MESSAGE_ACTIONS = require('./message-actions').ACTIONS; Backend.MilestoneDB = require('./milestone-db'); Backend.ot = require('./ot'); Backend.projections = require('./projections'); From 8b531bedf19860ffe0b37a3e1c8da7a32b5005bd Mon Sep 17 00:00:00 2001 From: Eric Hwang Date: Tue, 9 Aug 2022 10:21:27 -0700 Subject: [PATCH 25/61] Refactor message action codes into a shared file --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 30d978c03..6ec38a47d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.0.0", + "version": "3.1.0", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 56ac7a8b346310cfcc3f6954e4fdefd9d50bee60 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 27 Sep 2022 12:08:41 +0100 Subject: [PATCH 26/61] =?UTF-8?q?=E2=9C=A8=20Add=20ping/pong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds the ability for a client to send a "ping" to the server and receive a "pong" response. The motivation for this is that sometimes the socket underlying the `Connection` may not reliably report its connection state (for example if using a wrapper around a "vanilla" websocket that handles reconnection). The most bullet-proof way for a client to determine its connection state is to actually make a request to the server and assert that it receives a timely response. The implementation of ping/pong is arguably a websocket concern, especially since it's already [defined by the spec][1]. However, the browser JavaScript [`WebSocket`][2] does not expose this functionality, so we have to add our own ping/pong layer on top anyway. It's also worth noting that consumers of this library can't easily send their own ping messages down the socket, since ShareDB will [error][3] for unknown messages. Note that this change only adds the ability for a client to ping the server, and not the other way around. This is because: - the `Agent` is not an `Emitter`, and emitting a `pong` on the `Backend` is pretty meaningless - server-side implementations of WebSockets, such as `ws`, _do_ [expose a `ping` API][4] [1]: https://www.rfc-editor.org/rfc/rfc6455#section-5.5.2 [2]: https://developer.mozilla.org/en-US/docs/Web/API/WebSocket [3]: https://github.com/share/sharedb/blob/8b531bedf19860ffe0b37a3e1c8da7a32b5005bd/lib/agent.js#L451 [4]: https://github.com/websockets/ws/blob/966f9d47cd0ff5aa9db0b2aa262f9819d3f4d414/lib/websocket.js#L351 --- lib/agent.js | 10 ++++++++++ lib/client/connection.js | 20 ++++++++++++++++++++ lib/error.js | 1 + lib/message-actions.js | 1 + test/client/connection.js | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+) diff --git a/lib/agent.js b/lib/agent.js index ce2e43a46..2ad4d383d 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -447,6 +447,8 @@ Agent.prototype._handleMessage = function(request, callback) { return this._subscribePresence(request.ch, request.seq, callback); case ACTIONS.presenceUnsubscribe: return this._unsubscribePresence(request.ch, request.seq, callback); + case ACTIONS.pingPong: + return this._pingPong(callback); default: callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); } @@ -540,6 +542,14 @@ Agent.prototype._querySubscribe = function(queryId, collection, query, options, }); }; +Agent.prototype._pingPong = function(callback) { + var error = null; + var message = { + a: ACTIONS.pingPong + }; + callback(error, message); +}; + function getResultsData(results) { var items = []; for (var i = 0; i < results.length; i++) { diff --git a/lib/client/connection.js b/lib/client/connection.js index 8e3209d2e..ce5766ccf 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -253,6 +253,8 @@ Connection.prototype.handleMessage = function(message) { return this._handlePresenceUnsubscribe(err, message); case ACTIONS.presenceRequest: return this._handlePresenceRequest(err, message); + case ACTIONS.pingPong: + return this._handlePingPong(err); default: logger.warn('Ignoring unrecognized message', message); @@ -476,6 +478,19 @@ Connection.prototype.send = function(message) { this.socket.send(JSON.stringify(message)); }; +Connection.prototype.ping = function() { + if (!this.canSend) { + throw new ShareDBError( + ERROR_CODE.ERR_CANNOT_PING_OFFLINE, + 'Socket must be CONNECTED to ping' + ); + } + + var message = { + a: ACTIONS.pingPong + }; + this.send(message); +}; /** * Closes the socket and emits 'closed' @@ -725,6 +740,11 @@ Connection.prototype._handleHandshake = function(error, message) { this._initialize(message); }; +Connection.prototype._handlePingPong = function(error) { + if (error) return this.emit('error', error); + this.emit('pong'); +}; + Connection.prototype._initialize = function(message) { if (this.state !== 'connecting') return; diff --git a/lib/error.js b/lib/error.js index b278ad63c..929684643 100644 --- a/lib/error.js +++ b/lib/error.js @@ -16,6 +16,7 @@ ShareDBError.CODES = { ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT: 'ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT', ERR_APPLY_SNAPSHOT_NOT_PROVIDED: 'ERR_APPLY_SNAPSHOT_NOT_PROVIDED', ERR_CLIENT_ID_BADLY_FORMED: 'ERR_CLIENT_ID_BADLY_FORMED', + ERR_CANNOT_PING_OFFLINE: 'ERR_CANNOT_PING_OFFLINE', ERR_CONNECTION_SEQ_INTEGER_OVERFLOW: 'ERR_CONNECTION_SEQ_INTEGER_OVERFLOW', ERR_CONNECTION_STATE_TRANSITION_INVALID: 'ERR_CONNECTION_STATE_TRANSITION_INVALID', ERR_DATABASE_ADAPTER_NOT_FOUND: 'ERR_DATABASE_ADAPTER_NOT_FOUND', diff --git a/lib/message-actions.js b/lib/message-actions.js index ad2b7536b..baa242b2c 100644 --- a/lib/message-actions.js +++ b/lib/message-actions.js @@ -14,6 +14,7 @@ exports.ACTIONS = { op: 'op', snapshotFetch: 'nf', snapshotFetchByTimestamp: 'nt', + pingPong: 'pp', presence: 'p', presenceSubscribe: 'ps', presenceUnsubscribe: 'pu', diff --git a/test/client/connection.js b/test/client/connection.js index abf2cb970..505775776 100644 --- a/test/client/connection.js +++ b/test/client/connection.js @@ -124,6 +124,45 @@ describe('client connection', function() { }); }); + describe('ping/pong', function() { + it('pings the backend', function(done) { + var connection = this.backend.connect(); + + connection.on('pong', function() { + done(); + }); + + connection.on('connected', function() { + connection.ping(); + }); + }); + + it('handles errors', function(done) { + this.backend.use('receive', function(request, next) { + var error = request.data.a === 'pp' && new Error('bad'); + next(error); + }); + + var connection = this.backend.connect(); + + connection.on('error', function(error) { + expect(error.message).to.equal('bad'); + done(); + }); + + connection.on('connected', function() { + connection.ping(); + }); + }); + + it('throws if pinging offline', function() { + var connection = this.backend.connect(); + expect(function() { + connection.ping(); + }).to.throw('Socket must be CONNECTED to ping'); + }); + }); + describe('backend.agentsCount', function() { it('updates after connect and connection.close()', function(done) { var backend = this.backend; From 791937b760de2c7b9a20ce06fc7824ee21239ae2 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 27 Sep 2022 17:54:38 +0100 Subject: [PATCH 27/61] 3.2.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6ec38a47d..c77b71c08 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.1.0", + "version": "3.2.0", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From aa9bf9c495dee810cddca3ce122ceb231052e8a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 17:04:56 +0000 Subject: [PATCH 28/61] Bump commonmarker from 0.23.4 to 0.23.6 in /docs Bumps [commonmarker](https://github.com/gjtorikian/commonmarker) from 0.23.4 to 0.23.6. - [Release notes](https://github.com/gjtorikian/commonmarker/releases) - [Changelog](https://github.com/gjtorikian/commonmarker/blob/main/CHANGELOG.md) - [Commits](https://github.com/gjtorikian/commonmarker/compare/v0.23.4...v0.23.6) --- updated-dependencies: - dependency-name: commonmarker dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index a08eae2bf..0ab0bd830 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -14,7 +14,7 @@ GEM execjs coffee-script-source (1.11.1) colorator (1.1.0) - commonmarker (0.23.4) + commonmarker (0.23.6) concurrent-ruby (1.1.9) dnsruby (1.61.9) simpleidn (~> 0.1) From e3704a790436ed96550f18158a8e95e7e2f52b19 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 27 Sep 2022 18:05:58 +0100 Subject: [PATCH 29/61] =?UTF-8?q?=F0=9F=93=9D=20Add=20`connection.ping()`?= =?UTF-8?q?=20to=20the=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/connection.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/api/connection.md b/docs/api/connection.md index 87df406ca..bbf996331 100644 --- a/docs/api/connection.md +++ b/docs/api/connection.md @@ -229,3 +229,18 @@ The document **must** be of a [type]({{ site.baseurl }}{% link types/index.md %} Return value > A [`Presence`]({{ site.baseurl }}{% link api/presence.md %}) instance tied to the given document + +### ping() + +Send a short message to the server, which will respond with another short message, emitted on the connection as a +`'pong'` event. + +```js +connection.ping() +connection.on('pong', () => { + // The server is still there +}) +``` + +{: .warn } +Calling `ping()` when not connected will throw an error with code `ERR_CANNOT_PING_OFFLINE`. From d121a3e3066be34795223fca7ba150333664fe54 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:27:12 +0100 Subject: [PATCH 30/61] =?UTF-8?q?=F0=9F=90=9B=20Allow=20getting=20a=20`Doc?= =?UTF-8?q?`=20while=20it's=20being=20destroyed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's currently an edge case involving trying to get a doc in the middle of being destroyed. For example, consider these synchronous calls: ```js var doc = connection.get('collection', 'id') doc.destroy() var newDoc = connection.get('collection', 'id') ``` Since `Doc` objects are singletons, the above code is effectively the same as: ```js var doc = connection.get('collection', 'id') doc.destroy() var newDoc = doc ``` In other words, our "new" doc is actually already being destroyed, and will soon be removed from the `Connection` cache. One of the ways this could cause a bug is that the `doc` cannot be found in order to trigger callbacks when receiving responses from the server. `Connection` already tries to handle this case with [`_addDoc()`][1], but this could be problematic for a couple of reasons: 1. If we add new functionality to `Connection` we may forget to add a call to `_addDoc()` 2. This is susceptible to having multiple different instances of a `Doc` continuously trying to replace one another on the `Connection` cache We can reach multiple `Doc`s trying to replace one another with the following edge case: 1. Get a `Doc` 2. Destroy it 3. Synchronously get the same `Doc` (returning the instance in the process of being destroyed) 4. Wait for the `destroy()` to finish (ie the `Doc` has been removed from the `Connection` cache) 5. Get the `Doc` again - we now have a fresh, different instance 6. Try to submit ops on both docs at the same time 7. Depending on which one goes last, that one will be the `Doc` stored on the `Connection`, and will incorrectly receive both server responses This change attempts to address this edge case by adding a private `_wantsDestroy` to our `Doc` instances, which is: - `false` on construction - set to `true` when calling `destroy()` - reset to `false` if calling `connection.get()` We then check this flag when performing the `destroy()` tidy-up, and avoid tidying up if we have since reset the flag. The end result of this is that any calls to `connection.get()` before `destroy()` has resolved will return the same singleton `Doc` instance, and prevent tidy-up. [1]: https://github.com/share/sharedb/blob/5176283e2d1c9dc263f1fa3c2bc6a93c1afe2ad3/lib/client/connection.js#L452-L453 --- lib/client/connection.js | 3 +++ lib/client/doc.js | 5 +++-- test/client/doc.js | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/client/connection.js b/lib/client/connection.js index ce5766ccf..fece03b46 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -521,6 +521,7 @@ Connection.prototype.get = function(collection, id) { this.emit('doc', doc); } + doc._wantsDestroy = false; return doc; }; @@ -531,7 +532,9 @@ Connection.prototype.get = function(collection, id) { * @private */ Connection.prototype._destroyDoc = function(doc) { + if (!doc._wantsDestroy) return; util.digAndRemove(this.collections, doc.collection, doc.id); + doc.emit('destroy'); }; Connection.prototype._addDoc = function(doc) { diff --git a/lib/client/doc.js b/lib/client/doc.js index 3667b4895..cf4f08cc9 100644 --- a/lib/client/doc.js +++ b/lib/client/doc.js @@ -76,6 +76,8 @@ function Doc(connection, collection, id) { // Whether to re-establish the subscription on reconnect this.wantSubscribe = false; + this._wantsDestroy = false; + // The op that is currently roundtripping to the server, or null. // // When the connection reconnects, the inflight op is resubmitted. @@ -122,6 +124,7 @@ function Doc(connection, collection, id) { emitter.mixin(Doc); Doc.prototype.destroy = function(callback) { + this._wantsDestroy = true; var doc = this; doc.whenNothingPending(function() { if (doc.wantSubscribe) { @@ -131,12 +134,10 @@ Doc.prototype.destroy = function(callback) { return doc.emit('error', err); } doc.connection._destroyDoc(doc); - doc.emit('destroy'); if (callback) callback(); }); } else { doc.connection._destroyDoc(doc); - doc.emit('destroy'); if (callback) callback(); } }); diff --git a/test/client/doc.js b/test/client/doc.js index 6ea3666fc..fc9e771f8 100644 --- a/test/client/doc.js +++ b/test/client/doc.js @@ -43,11 +43,31 @@ describe('Doc', function() { if (err) return done(err); var doc2 = connection.get('dogs', 'fido'); expect(doc).not.equal(doc2); - expect(doc).eql(doc2); done(); }); }); + it('destroying then getting synchronously does not destroy the new doc', function(done) { + var connection = this.connection; + var doc = connection.get('dogs', 'fido'); + var doc2; + + doc.create({name: 'fido'}, function(error) { + if (error) return done(error); + + doc.destroy(function(error) { + if (error) return done(error); + var doc3 = connection.get('dogs', 'fido'); + async.parallel([ + doc2.submitOp.bind(doc2, [{p: ['snacks'], oi: true}]), + doc3.submitOp.bind(doc3, [{p: ['color'], oi: 'gray'}]) + ], done); + }); + + doc2 = connection.get('dogs', 'fido'); + }); + }); + it('doc.destroy() works without a callback', function() { var doc = this.connection.get('dogs', 'fido'); doc.destroy(); From c3f70301708c05786a2eef4041bcb1eb636ca3e7 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 18 Oct 2022 11:00:14 +0100 Subject: [PATCH 31/61] =?UTF-8?q?=F0=9F=90=9B=20Allow=20getting=20`Presenc?= =?UTF-8?q?e`=20while=20it's=20being=20destroyed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's currently an edge case involving trying to get presence in the middle of being destroyed. For example, consider these synchronous calls: ```js var presence = connection.getPresence('channel') presence.destroy() var newPresence = connection.getPresence('channel') ``` Since `Presence` objects are per-channel singletons, the above code is effectively the same as: ```js var presence = connection.getPresence('channel') presence.destroy() var newPresence = presence ``` In other words, our "new" presence is actually already being destroyed. What's worse, is that if we start trying to create local presence before the `destroy()` callback is triggered, that gets "cleaned up" by the initial call to `destroy()`: ```js var presence = connection.getPresence('channel') presence.destroy(() => { // by the time we reach here, my newly-created localPresence // will have been destroyed }) var newPresence = connection.getPresence('channel') var localPresence = newPresence.create('me') ``` This change attempts to address this edge case by adding a private `_wantsDestroy` to our `Presence` instances, which is: - `false` on construction - set to `true` when calling `destroy()` - reset to `false` if calling `connection.getPresence()` We then check this flag when performing the `destroy()` tidy-up, and avoid tidying up if we have since reset the flag. Note that there's a corner case within this edge case, surrounding `LocalPresence` instances created while a `destroy()` is in progress. Since we might create `LocalPresence` instances at any point during the asynchronous `destroy()`, we define the following behaviour: - any `LocalPresence` instances that exist at the time of invoking `destroy()` will be destroyed, **even if** we subsequently cancel the destroy (by invoking `connection.getPresence()`) - any `LocalPresence` instances that are created *after* invoking `destroy()` will be kept if the destroy is cancelled - creating a new `LocalPresence` while `_wantDestroy === true` will throw, because clients shouldn't put themselves in this situation and the behaviour when tidying this up is undefined --- lib/client/connection.js | 8 ++- lib/client/presence/presence.js | 18 ++++- test/client/presence/presence.js | 112 +++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/lib/client/connection.js b/lib/client/connection.js index fece03b46..4655c39a7 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -776,17 +776,21 @@ Connection.prototype._initialize = function(message) { Connection.prototype.getPresence = function(channel) { var connection = this; - return util.digOrCreate(this._presences, channel, function() { + var presence = util.digOrCreate(this._presences, channel, function() { return new Presence(connection, channel); }); + presence._wantsDestroy = false; + return presence; }; Connection.prototype.getDocPresence = function(collection, id) { var channel = DocPresence.channel(collection, id); var connection = this; - return util.digOrCreate(this._presences, channel, function() { + var presence = util.digOrCreate(this._presences, channel, function() { return new DocPresence(connection, collection, id); }); + presence._wantsDestroy = false; + return presence; }; Connection.prototype._sendPresenceAction = function(action, seq, presence) { diff --git a/lib/client/presence/presence.js b/lib/client/presence/presence.js index 107355e34..9aba6b60c 100644 --- a/lib/client/presence/presence.js +++ b/lib/client/presence/presence.js @@ -24,6 +24,7 @@ function Presence(connection, channel) { this._remotePresenceInstances = {}; this._subscriptionCallbacksBySeq = {}; + this._wantsDestroy = false; } emitter.mixin(Presence); @@ -36,6 +37,9 @@ Presence.prototype.unsubscribe = function(callback) { }; Presence.prototype.create = function(id) { + if (this._wantsDestroy) { + throw new Error('Presence is being destroyed'); + } id = id || hat(); var localPresence = this._createLocalPresence(id); this.localPresences[id] = localPresence; @@ -43,10 +47,15 @@ Presence.prototype.create = function(id) { }; Presence.prototype.destroy = function(callback) { + this._wantsDestroy = true; var presence = this; + // Store these at the time of destruction: any LocalPresence on this + // instance at this time will be destroyed, but if the destroy is + // cancelled, any future LocalPresence objects will be kept. + // See: https://github.com/share/sharedb/pull/579 + var localIds = Object.keys(presence.localPresences); this.unsubscribe(function(error) { if (error) return presence._callbackOrEmit(error, callback); - var localIds = Object.keys(presence.localPresences); var remoteIds = Object.keys(presence._remotePresenceInstances); async.parallel( [ @@ -56,13 +65,18 @@ Presence.prototype.destroy = function(callback) { }, next); }, function(next) { + // We don't bother stashing the RemotePresence instances because + // they're not really bound to our local state: if we want to + // destroy, we destroy them all, but if we cancel the destroy, + // we'll want to keep them all + if (!presence._wantsDestroy) return next(); async.each(remoteIds, function(presenceId, next) { presence._remotePresenceInstances[presenceId].destroy(next); }, next); } ], function(error) { - delete presence.connection._presences[presence.channel]; + if (presence._wantsDestroy) delete presence.connection._presences[presence.channel]; presence._callbackOrEmit(error, callback); } ); diff --git a/test/client/presence/presence.js b/test/client/presence/presence.js index 997097451..c2a59c495 100644 --- a/test/client/presence/presence.js +++ b/test/client/presence/presence.js @@ -98,6 +98,118 @@ describe('Presence', function() { ], done); }); + it('gets presence during a destroy', function(done) { + var localPresence1 = presence1.create('presence-1'); + var presence2a; + + async.series([ + presence2.subscribe.bind(presence2), + function(next) { + presence2.destroy(errorHandler(done)); + next(); + }, + function(next) { + presence2a = connection2.getPresence('test-channel'); + presence2a.subscribe(function(error) { + next(error); + }); + }, + function(next) { + localPresence1.submit({index: 5}, errorHandler(done)); + presence2a.once('receive', function() { + next(); + }); + } + ], done); + }); + + it('destroys old local presence but keeps new local presence when getting during destroy', function(done) { + presence2.create('presence-2'); + var presence2a; + + async.series([ + presence2.subscribe.bind(presence2), + function(next) { + presence2.destroy(function() { + expect(presence2).to.equal(presence2a); + expect(Object.keys(presence2.localPresences)).to.eql(['presence-2a']); + done(); + }); + next(); + }, + function(next) { + presence2a = connection2.getPresence('test-channel'); + presence2a.create('presence-2a'); + presence2a.subscribe(function(error) { + next(error); + }); + } + ], errorHandler(done)); + }); + + it('destroys old local presence but keeps new local presence when getting during destroy', function(done) { + presence2.create('presence-2'); + var presence2a; + + async.series([ + presence2.subscribe.bind(presence2), + function(next) { + presence2.destroy(function() { + expect(presence2).to.equal(presence2a); + expect(Object.keys(presence2.localPresences)).to.eql(['presence-2a']); + done(); + }); + next(); + }, + function(next) { + presence2a = connection2.getPresence('test-channel'); + presence2a.create('presence-2a'); + presence2a.subscribe(function(error) { + next(error); + }); + } + ], errorHandler(done)); + }); + + it('throws if trying to create local presence when wanting destroy', function(done) { + presence2.destroy(errorHandler(done)); + expect(function() { + presence2.create('presence-2'); + }).to.throw('Presence is being destroyed'); + done(); + }); + + it('gets presence after destroy unsubscribe', function(done) { + var localPresence2 = presence2.create('presence-2'); + var presence2a; + + var flushLocalPresence2Destroy; + sinon.stub(localPresence2, 'destroy').callsFake(function(callback) { + flushLocalPresence2Destroy = callback; + }); + + async.series([ + presence2.subscribe.bind(presence2), + function(next) { + presence2.destroy(function() { + expect(connection2.getPresence('test-channel')).to.equal(presence2a); + done(); + }); + next(); + }, + // Wait for the destroy unsubscribe callback to start, where we check + // _wantsDestroy for the first time + presence2.unsubscribe.bind(presence2), + function(next) { + presence2a = connection2.getPresence('test-channel'); + presence2a.subscribe(function(error) { + next(error); + }); + flushLocalPresence2Destroy(); + } + ], errorHandler(done)); + }); + it('requests existing presence from other subscribed clients when subscribing', function(done) { var localPresence1 = presence1.create('presence-1'); async.series([ From 2cdf0944f86c2a19e43430685bd45c9b9bbb3fb8 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 25 Oct 2022 17:33:04 +0100 Subject: [PATCH 32/61] 3.2.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c77b71c08..9656c98c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.2.0", + "version": "3.2.1", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 7bac0e9a6c6f7aef5c1ca3b091b86b39bd21811a Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 23 Nov 2022 14:30:58 +0000 Subject: [PATCH 33/61] =?UTF-8?q?=F0=9F=91=B7=E2=80=8D=E2=99=80=EF=B8=8F?= =?UTF-8?q?=20Move=20to=20`ruby/setup-ruby`=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The [`actions/setup-ruby`][1] action has been deprecated and advises moving to [`ruby/setup-ruby`][2] [1]: https://github.com/actions/setup-ruby [2]: https://github.com/ruby/setup-ruby --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eda3722e2..ea741568d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,10 +18,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-ruby@v1 + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.7 + ruby-version: '2.7' - name: Install run: cd docs && gem install bundler && bundle install - name: Build From 8a8447be5ea37f4b959665518777e5bd8ddf0bdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 17:06:11 +0000 Subject: [PATCH 34/61] Bump nokogiri from 1.13.7 to 1.13.9 in /docs Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.7 to 1.13.9. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.7...v1.13.9) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 0ab0bd830..28a997f39 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -232,7 +232,7 @@ GEM jekyll-seo-tag (~> 2.1) minitest (5.15.0) multipart-post (2.1.1) - nokogiri (1.13.7) + nokogiri (1.13.9) mini_portile2 (~> 2.8.0) racc (~> 1.4) octokit (4.22.0) From 3db70700f1bbb976ce514ec9477c0a8d0a3b7b30 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 23 Nov 2022 14:25:04 +0000 Subject: [PATCH 35/61] =?UTF-8?q?=E2=9C=A8=20Allow=20middleware=20to=20mut?= =?UTF-8?q?ate=20submitted=20ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds a new `request.$fixup()` method to the `SubmitRequest` class, which allows middleware to mutate ops in the `apply` middleware. Motivation ---------- There are times when we want the server to make changes to ops submitted by a client. There may be many reasons for wanting to do this, including distrusting a client (eg setting timestamp, or auth information); keeping sensitive business logic out of client-side code; etc. The only ways of doing this currently are: 1. Committing the client's op, and then submitting an amendment op from the server; or 2. Rejecting the client's op, and submitting the full amended op from the server Option 1 works, but possibly commits badly-formed, or untrustworthy data to the database, and broadcasts it to remote clients. It also makes it possible to fetch historic snapshots of this untrustworthy in-between state, which undermines the trustworthiness of our data. Option 2 also works, and might work with invertible types through a "soft" rollback, but can lose pending ops if we're not careful, or if the type isn't invertible. Approach -------- This change adds a `$fixup()` helper function to the `SubmitRequest` object, named after the `git` action for performing similar actions. The idea of the `$fixup()` function is to act as a middleware-friendly `submitRequest()`. Any ops submitted through `$fixup()` will be: - composed with the `request.op`, so downstream middleware can work on the fixed-up op - sent back to the original client, and applied - help to avoid infinite middleware loops, since we don't trigger any nested submission calls Note that in order to handle the case of a client disconnecting before getting their fixup, we now also store the fixup ops on the op `m` metadata field, and fetch this when getting ops to the current snapshot. Limitations ----------- This feature will only work: - on types which implement the optional `.compose()` function - in the `apply` middleware, where the snapshot's `type` is available, but before the op has actually been applied to the snapshot - on creation or edit ops (not deletion) --- docs/middleware/op-submission.md | 18 +- lib/agent.js | 3 +- lib/backend.js | 2 +- lib/client/doc.js | 18 ++ lib/error.js | 3 + lib/message-actions.js | 1 + lib/submit-request.js | 46 +++- test/client/submit.js | 48 +++++ test/middleware.js | 356 +++++++++++++++++++++++++++++++ 9 files changed, 487 insertions(+), 8 deletions(-) diff --git a/docs/middleware/op-submission.md b/docs/middleware/op-submission.md index 57a032af1..737fd914e 100644 --- a/docs/middleware/op-submission.md +++ b/docs/middleware/op-submission.md @@ -140,12 +140,20 @@ Since `'submitRequestEnd'` is an event -- not a middleware hook -- it provides n ## Mutating ops -{: .warn :} -Mutating ops in middleware is generally a **bad idea**, and should be avoided. +Ops may be amended in the `apply` middleware using the special `request.$fixup()` method: -The main reason for avoiding op mutation is that the client who submitted the op will not be informed of the mutation, so the client's doc will never receive the mutation. +```js +backend.use('apply', (request, next) => { + let error; + try { + request.$fixup([{p: ['meta'], oi: {timestamp: Date.now()}}]); + } catch (e) { + error = e; + } -The general workaround is to trigger a second op submission, rather than mutate the provided op. This obviously has the downside of op submissions being unatomic, but is the safest way to get the original client to receive the update. + next(error); +}); +``` {: .warn :} -When submitting ops from the middleware, set careful conditions under which you submit ops in order to avoid infinite op submission loops, where submitting an op recursively triggers infinite op submissions. +The `request.$fixup()` method may throw an error, which should be handled appropriately, usually by passing directly to the `next()` callback. diff --git a/lib/agent.js b/lib/agent.js index 2ad4d383d..7d1e3e5c7 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -718,9 +718,10 @@ Agent.prototype._unsubscribeBulk = function(collection, ids, callback) { Agent.prototype._submit = function(collection, id, op, callback) { var agent = this; - this.backend.submit(this, collection, id, op, null, function(err, ops) { + this.backend.submit(this, collection, id, op, null, function(err, ops, request) { // Message to acknowledge the op was successfully submitted var ack = {src: op.src, seq: op.seq, v: op.v}; + if (request._fixupOps.length) ack[ACTIONS.fixup] = request._fixupOps; if (err) { // Occasional 'Op already submitted' errors are expected to happen as // part of normal operation, since inflight ops need to be resent after diff --git a/lib/backend.js b/lib/backend.js index 0a4f6a884..3d73504c1 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -229,7 +229,7 @@ Backend.prototype.submit = function(agent, index, id, op, options, originalCallb var callback = function(error, ops) { backend.emit('submitRequestEnd', error, request); - originalCallback(error, ops); + originalCallback(error, ops, request); }; var err = ot.checkOp(op); diff --git a/lib/client/doc.js b/lib/client/doc.js index cf4f08cc9..d5a3d05b7 100644 --- a/lib/client/doc.js +++ b/lib/client/doc.js @@ -5,6 +5,7 @@ var types = require('../types'); var util = require('../util'); var clone = util.clone; var deepEqual = require('fast-deep-equal'); +var ACTIONS = require('../message-actions').ACTIONS; var ERROR_CODE = ShareDBError.CODES; @@ -960,6 +961,23 @@ Doc.prototype._opAcknowledged = function(message) { return this.fetch(); } + if (message[ACTIONS.fixup]) { + for (var i = 0; i < message[ACTIONS.fixup].length; i++) { + var fixupOp = message[ACTIONS.fixup][i]; + + for (var j = 0; j < this.pendingOps.length; j++) { + var transformErr = transformX(this.pendingOps[i], fixupOp); + if (transformErr) return this._hardRollback(transformErr); + } + + try { + this._otApply(fixupOp, false); + } catch (error) { + return this._hardRollback(error); + } + } + } + // The op was committed successfully. Increment the version number this.version++; diff --git a/lib/error.js b/lib/error.js index 929684643..5387749fb 100644 --- a/lib/error.js +++ b/lib/error.js @@ -15,6 +15,8 @@ ShareDBError.prototype.name = 'ShareDBError'; ShareDBError.CODES = { ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT: 'ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT', ERR_APPLY_SNAPSHOT_NOT_PROVIDED: 'ERR_APPLY_SNAPSHOT_NOT_PROVIDED', + ERR_FIXUP_IS_ONLY_VALID_ON_APPLY: 'ERR_FIXUP_IS_ONLY_VALID_ON_APPLY', + ERR_CANNOT_FIXUP_DELETION: 'ERR_CANNOT_FIXUP_DELETION', ERR_CLIENT_ID_BADLY_FORMED: 'ERR_CLIENT_ID_BADLY_FORMED', ERR_CANNOT_PING_OFFLINE: 'ERR_CANNOT_PING_OFFLINE', ERR_CONNECTION_SEQ_INTEGER_OVERFLOW: 'ERR_CONNECTION_SEQ_INTEGER_OVERFLOW', @@ -67,6 +69,7 @@ ShareDBError.CODES = { ERR_SNAPSHOT_READS_REJECTED: 'ERR_SNAPSHOT_READS_REJECTED', ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND: 'ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND', ERR_TYPE_CANNOT_BE_PROJECTED: 'ERR_TYPE_CANNOT_BE_PROJECTED', + ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE: 'ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE', ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE: 'ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE', ERR_UNKNOWN_ERROR: 'ERR_UNKNOWN_ERROR' }; diff --git a/lib/message-actions.js b/lib/message-actions.js index baa242b2c..e1a8e9942 100644 --- a/lib/message-actions.js +++ b/lib/message-actions.js @@ -9,6 +9,7 @@ exports.ACTIONS = { bulkSubscribe: 'bs', bulkUnsubscribe: 'bu', fetch: 'f', + fixup: 'fixup', subscribe: 's', unsubscribe: 'u', op: 'op', diff --git a/lib/submit-request.js b/lib/submit-request.js index 04517513b..2c8ecf668 100644 --- a/lib/submit-request.js +++ b/lib/submit-request.js @@ -1,6 +1,7 @@ var ot = require('./ot'); var projections = require('./projections'); var ShareDBError = require('./error'); +var types = require('./types'); var ERROR_CODE = ShareDBError.CODES; @@ -40,9 +41,47 @@ function SubmitRequest(backend, agent, index, id, op, options) { this.snapshot = null; this.ops = []; this.channels = null; + this._fixupOps = []; } module.exports = SubmitRequest; +SubmitRequest.prototype.$fixup = function(op) { + if (this.action !== this.backend.MIDDLEWARE_ACTIONS.apply) { + throw new ShareDBError( + ERROR_CODE.ERR_FIXUP_IS_ONLY_VALID_ON_APPLY, + 'fixup can only be called during the apply middleware' + ); + } + + if (this.op.del) { + throw new ShareDBError( + ERROR_CODE.ERR_CANNOT_FIXUP_DELETION, + 'fixup cannot be applied on deletion ops' + ); + } + + var typeId = this.op.create ? this.op.create.type : this.snapshot.type; + var type = types.map[typeId]; + if (typeof type.compose !== 'function') { + throw new ShareDBError( + ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE, + typeId + ' does not support compose' + ); + } + + if (this.op.create) this.op.create.data = type.apply(this.op.create.data, op); + else this.op.op = type.compose(this.op.op, op); + + var fixupOp = { + src: this.op.src, + seq: this.op.seq, + v: this.op.v, + op: op + }; + + this._fixupOps.push(fixupOp); +}; + SubmitRequest.prototype.submit = function(callback) { var request = this; var backend = this.backend; @@ -103,7 +142,7 @@ SubmitRequest.prototype.submit = function(callback) { // Transform the op up to the current snapshot version, then apply var from = op.v; - backend.db.getOpsToSnapshot(collection, id, from, snapshot, null, function(err, ops) { + backend.db.getOpsToSnapshot(collection, id, from, snapshot, {metadata: true}, function(err, ops) { if (err) return callback(err); if (ops.length !== snapshot.v - from) { @@ -134,6 +173,8 @@ SubmitRequest.prototype.apply = function(callback) { // Always set the channels before each attempt to apply. If the channels are // modified in a middleware and we retry, we want to reset to a new array this.channels = this.backend.getChannels(this.collection, this.id); + this._fixupOps = []; + delete this.op.m.fixup; var request = this; this.backend.trigger(this.backend.MIDDLEWARE_ACTIONS.apply, this.agent, this, function(err) { @@ -152,6 +193,7 @@ SubmitRequest.prototype.commit = function(callback) { var backend = this.backend; backend.trigger(backend.MIDDLEWARE_ACTIONS.commit, this.agent, this, function(err) { if (err) return callback(err); + if (request._fixupOps.length) request.op.m.fixup = request._fixupOps; // Try committing the operation and snapshot to the database atomically backend.db.commit( @@ -204,6 +246,7 @@ SubmitRequest.prototype._transformOp = function(ops) { // can happen in normal operation, such as a client resending an // unacknowledged operation at reconnect. It's important we don't apply // the same op twice + if (op.m.fixup) this._fixupOps = op.m.fixup; return this.alreadySubmittedError(); } @@ -213,6 +256,7 @@ SubmitRequest.prototype._transformOp = function(ops) { var err = ot.transform(type, this.op, op); if (err) return err; + delete op.m; this.ops.push(op); } }; diff --git a/test/client/submit.js b/test/client/submit.js index 15d9e3f49..37716dc41 100644 --- a/test/client/submit.js +++ b/test/client/submit.js @@ -4,6 +4,7 @@ var sinon = require('sinon'); var types = require('../../lib/types'); var deserializedType = require('./deserialized-type'); var numberType = require('./number-type'); +var errorHandler = require('../util').errorHandler; types.register(deserializedType.type); types.register(deserializedType.type2); types.register(numberType.type); @@ -1210,5 +1211,52 @@ module.exports = function() { }); }); }); + + describe('submitting when behind the server', function() { + var doc; + var remoteDoc; + + beforeEach(function(done) { + var connection = this.backend.connect(); + doc = connection.get('dogs', 'fido'); + var remoteConnection = this.backend.connect(); + remoteDoc = remoteConnection.get('dogs', 'fido'); + + async.series([ + doc.create.bind(doc, {name: 'fido'}), + remoteDoc.fetch.bind(remoteDoc), + remoteDoc.submitOp.bind(remoteDoc, [{p: ['tricks'], oi: ['fetch']}]), + function(next) { + expect(doc.data).to.eql({name: 'fido'}); + expect(remoteDoc.data).to.eql({name: 'fido', tricks: ['fetch']}); + next(); + } + ], done); + }); + + it('is sent ops it has missed when submitting, without calling fetch', function(done) { + sinon.spy(doc, 'fetch'); + + doc.submitOp([{p: ['age'], oi: 2}], errorHandler(done)); + + doc.once('op', function() { + expect(doc.data).to.eql({name: 'fido', tricks: ['fetch'], age: 2}); + expect(doc.fetch.called).to.be.false; + done(); + }); + }); + + it('does not expose op metadata in the middleware when sending missing ops', function(done) { + this.backend.use('apply', function(request) { + expect(request.ops).to.have.length(1); + var op = request.ops[0]; + expect(op.op).to.eql([{p: ['tricks'], oi: ['fetch']}]); + expect(op.m).to.be.undefined; + done(); + }); + + doc.submitOp([{p: ['age'], oi: 2}], errorHandler(done)); + }); + }); }); }; diff --git a/test/middleware.js b/test/middleware.js index a22316b56..19fbe19f1 100644 --- a/test/middleware.js +++ b/test/middleware.js @@ -3,6 +3,11 @@ var expect = require('chai').expect; var util = require('./util'); var types = require('../lib/types'); var errorHandler = util.errorHandler; +var ShareDBError = require('../lib/error'); +var sinon = require('sinon'); +var ACTIONS = require('../lib/message-actions').ACTIONS; + +var ERROR_CODE = ShareDBError.CODES; describe('middleware', function() { beforeEach(function() { @@ -417,4 +422,355 @@ describe('middleware', function() { doc.submitOp([{p: ['tricks', 1], li: 'stay'}], {source: 'c'}, errorHandler(done)); }); }); + + describe('$fixup', function() { + var connection; + var backend; + var doc; + + beforeEach(function(done) { + backend = this.backend; + connection = backend.connect(); + doc = connection.get('dogs', 'fido'); + + doc.create({name: 'fido'}, done); + }); + + it('applies a fixup op to the client that submitted it', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 1], li: 'stay'}]); + next(); + }); + + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], function(error) { + if (error) return done(error); + expect(doc.data.tricks).to.eql(['fetch', 'stay']); + expect(doc.version).to.equal(2); + done(); + }); + }); + + it('emits an op event for the fixup', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 1], li: 'stay'}]); + next(); + }); + + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], errorHandler(done)); + + doc.on('op', function() { + expect(doc.data.tricks).to.eql(['fetch', 'stay']); + done(); + }); + }); + + it('passes the fixed up op to future middleware', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 1], li: 'stay'}]); + next(); + }); + + backend.use('apply', function(request, next) { + expect(request.op.op).to.eql([ + {p: ['tricks'], oi: ['fetch']}, + {p: ['tricks', 1], li: 'stay'} + ]); + next(); + }); + + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], done); + }); + + it('applies the composed op to a remote client', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 1], li: 'stay'}]); + next(); + }); + + var remoteConnection = backend.connect(); + var remoteDoc = remoteConnection.get('dogs', 'fido'); + + remoteDoc.subscribe(function(error) { + if (error) return done(error); + + expect(remoteDoc.data).to.eql({name: 'fido'}); + + remoteDoc.on('op batch', function() { + expect(remoteDoc.data.tricks).to.eql(['fetch', 'stay']); + expect(doc.version).to.equal(remoteDoc.version); + done(); + }); + + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], errorHandler(done)); + }); + }); + + it('transforms pending ops by the fixup for remote clients', function(done) { + var applied = false; + backend.use('apply', function(request, next) { + if (applied) return next(); + applied = true; + request.$fixup([{p: ['tricks', 0], li: 'stay'}]); + next(); + }); + + var remoteConnection = backend.connect(); + var remoteDoc = remoteConnection.get('dogs', 'fido'); + + remoteDoc.subscribe(function(error) { + if (error) return done(error); + + expect(remoteDoc.data).to.eql({name: 'fido'}); + + remoteDoc.on('op batch', function() { + if (remoteDoc.version !== 3) return; + expect(remoteDoc.data.tricks).to.eql(['stay', 'fetch', 'sit']); + expect(remoteDoc.data).to.eql(doc.data); + done(); + }); + + doc.preventCompose = true; + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], errorHandler(done)); + doc.submitOp([{p: ['tricks', 1], li: 'sit'}], errorHandler(done)); + }); + }); + + it('transforms pending ops by the fixup for the local doc', function(done) { + var applied = false; + backend.use('apply', function(request, next) { + if (applied) return next(); + applied = true; + request.$fixup([{p: ['tricks', 0, 0], si: 'go '}]); + next(); + }); + + var remoteConnection = backend.connect(); + var remoteDoc = remoteConnection.get('dogs', 'fido'); + + remoteDoc.subscribe(function(error) { + if (error) return done(error); + + expect(remoteDoc.data).to.eql({name: 'fido'}); + + remoteDoc.on('op batch', function() { + if (remoteDoc.version !== 3) return; + expect(remoteDoc.data.tricks).to.eql(['stay', 'go fetch']); + expect(remoteDoc.data).to.eql(doc.data); + done(); + }); + + doc.preventCompose = true; + doc.submitOp([{p: ['tricks'], oi: ['fetch', 'stay']}], errorHandler(done)); + doc.submitOp([{p: ['tricks', 0], lm: 1}], errorHandler(done)); + }); + }); + + it('applies a fixup to a creation op', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['goodBoy'], oi: true}]); + next(); + }); + + doc = connection.get('dogs', 'rover'); + doc.create({name: 'rover'}, function(error) { + if (error) return done(error); + expect(doc.data.goodBoy).to.be.true; + done(); + }); + }); + + it('throws an error if trying to fixup a deletion', function(done) { + backend.use('apply', function(request, next) { + var error; + try { + request.$fixup([{p: ['tricks', 0], oi: ['stay']}]); + } catch (e) { + error = e; + } + next(error); + }); + + doc.del(function(error) { + expect(error.code).to.equal(ERROR_CODE.ERR_CANNOT_FIXUP_DELETION); + done(); + }); + }); + + it('throws an error if trying to fixup in commit middleware', function(done) { + backend.use('commit', function(request, next) { + var error; + try { + request.$fixup([{p: ['tricks', 0], oi: ['stay']}]); + } catch (e) { + error = e; + } + next(error); + }); + + doc.submitOp([{p: ['goodBoy'], oi: true}], function(error) { + expect(error.code).to.equal(ERROR_CODE.ERR_FIXUP_IS_ONLY_VALID_ON_APPLY); + done(); + }); + }); + + it('retry fixup', function(done) { + var flush; + backend.use('apply', function(request, next) { + expect(request.op.m.fixup).to.be.undefined; + if (flush) return next(); + flush = function() { + request.$fixup([{p: ['name', 0], si: 'fixup'}]); + next(); + }; + }); + + doc.subscribe(function(error) { + if (error) return done(error); + + var remoteConnection = backend.connect(); + var remoteDoc = remoteConnection.get('dogs', 'fido'); + + doc.submitOp([{p: ['name', 0], si: 'foo'}], function(error) { + if (error) return done(error); + expect(doc.data).to.eql({}); + done(); + }); + + remoteDoc.subscribe(function(error) { + if (error) return done(error); + remoteDoc.submitOp([{p: ['name'], od: 'fido'}], errorHandler(done)); + }); + + doc.once('op', function(op, source) { + if (source) return; + expect(doc.data).to.eql({}); + flush(); + }); + }); + }); + + it('fixup that ignores no-op', function(done) { + var flush; + backend.use('apply', function(request, next) { + if (request.op.src !== connection.id) return next(); + if (flush) { + request.$fixup([{p: ['name'], oi: 'fixup'}]); + return next(); + } + flush = function() { + request.$fixup([{p: ['name', 0], si: 'fixup'}]); + next(); + }; + }); + + doc.subscribe(function(error) { + if (error) return done(error); + + var remoteConnection = backend.connect(); + var remoteDoc = remoteConnection.get('dogs', 'fido'); + + var count = 0; + var callback = function() { + count++; + if (count !== 2) return; + expect(doc.data).to.eql({name: 'fixup'}); + expect(remoteDoc.data).to.eql(doc.data); + expect(doc.version).to.equal(remoteDoc.version); + done(); + }; + + doc.submitOp([{p: ['name', 0], si: 'foo'}], function(error) { + if (error) return done(error); + callback(); + }); + + remoteDoc.on('op', function(op, source) { + if (source) return; + callback(); + }); + + remoteDoc.subscribe(function(error) { + if (error) return done(error); + remoteDoc.submitOp([{p: ['name'], od: 'fido'}], errorHandler(done)); + }); + + doc.once('op', function(op, source) { + if (source) return; + expect(doc.data).to.eql({}); + flush(); + }); + }); + }); + + it('applies two fixups', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 0], li: 'sit'}]); + next(); + }); + + backend.use('apply', function(request, next) { + request.$fixup([{p: ['tricks', 0], li: 'stay'}]); + next(); + }); + + doc.submitOp([{p: ['tricks'], oi: ['fetch']}], function(error) { + if (error) return done(error); + expect(doc.data.tricks).to.eql(['stay', 'sit', 'fetch']); + done(); + }); + }); + + it('rolls the doc back if the fixup cannot be applied', function(done) { + backend.use('apply', function(request, next) { + request.$fixup([{p: ['stay'], oi: true}]); + next(); + }); + + backend.use('reply', function(request, next) { + if (request.reply[ACTIONS.fixup]) { + // Deliberately overwrite our fixup op to trigger a client rollback + request.reply[ACTIONS.fixup][0].op = [{p: ['fetch'], ld: 'bad'}]; + } + next(); + }); + + sinon.spy(doc, 'fetch'); + + doc.submitOp([{p: ['fetch'], oi: true}], function(error) { + expect(error).to.be.ok; + expect(doc.fetch.calledOnce).to.be.true; + done(); + }); + }); + + describe('no compose', function() { + var originalCompose; + + beforeEach(function() { + originalCompose = types.defaultType.compose; + delete types.defaultType.compose; + }); + + afterEach(function() { + types.defaultType.compose = originalCompose.bind(types.defaultType); + }); + + it('throws an error if trying to compose on a type that does not support it', function(done) { + backend.use('apply', function(request, next) { + var error; + try { + request.$fixup([{p: ['tricks', 0], oi: ['stay']}]); + } catch (e) { + error = e; + } + next(error); + }); + + doc.submitOp([{p: ['goodBoy'], oi: true}], function(error) { + expect(error.code).to.equal(ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE); + done(); + }); + }); + }); + }); }); From 2c04a08b122142a7b8cada5b06fd708b25a6b664 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:29:32 +0000 Subject: [PATCH 36/61] =?UTF-8?q?=F0=9F=94=87=20Fix=20sensitive=20memory?= =?UTF-8?q?=20leak=20warnings=20when=20using=20presence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/share/sharedb/issues/584 At the moment, we [attach `Doc` listeners][1] for every remote client in a document (as well as every local user). This means `Doc`s can quickly hit the [default of 10][2] maximum event listeners. This change adds a `DocPresenceEmitter` class, which only registers any given `Doc` listener once, regardless of how many presence instances there are. It then forwards events on through its own emitters, which have a much higher value of 1000 max listeners set, in order to avoid too much alert spam, but also keep *some* checking for memory leaks. [1]: https://github.com/share/sharedb/blob/cc0e3382bf3df4c38c7ccc6bc52da71aff8c9f74/lib/client/presence/remote-doc-presence.js#L42-L47 [2]: https://nodejs.org/api/events.html#eventsdefaultmaxlisteners --- lib/client/connection.js | 2 + lib/client/presence/doc-presence-emitter.js | 75 ++++++++++++++ lib/client/presence/local-doc-presence.js | 21 ++-- lib/client/presence/remote-doc-presence.js | 17 ++-- test/client/presence/doc-presence-emitter.js | 100 +++++++++++++++++++ test/client/presence/doc-presence.js | 10 ++ 6 files changed, 207 insertions(+), 18 deletions(-) create mode 100644 lib/client/presence/doc-presence-emitter.js create mode 100644 test/client/presence/doc-presence-emitter.js diff --git a/lib/client/connection.js b/lib/client/connection.js index 4655c39a7..0c3ad28f1 100644 --- a/lib/client/connection.js +++ b/lib/client/connection.js @@ -10,6 +10,7 @@ var ACTIONS = require('../message-actions').ACTIONS; var types = require('../types'); var util = require('../util'); var logger = require('../logger'); +var DocPresenceEmitter = require('./presence/doc-presence-emitter'); var ERROR_CODE = ShareDBError.CODES; @@ -51,6 +52,7 @@ function Connection(socket) { // Maps from channel -> presence objects this._presences = {}; + this._docPresenceEmitter = new DocPresenceEmitter(); // Map from snapshot request ID -> snapshot request this._snapshotRequests = {}; diff --git a/lib/client/presence/doc-presence-emitter.js b/lib/client/presence/doc-presence-emitter.js new file mode 100644 index 000000000..47ceb98fb --- /dev/null +++ b/lib/client/presence/doc-presence-emitter.js @@ -0,0 +1,75 @@ +var util = require('../../util'); +var EventEmitter = require('events').EventEmitter; + +var EVENTS = [ + 'create', + 'del', + 'destroy', + 'load', + 'op' +]; + +module.exports = DocPresenceEmitter; + +function DocPresenceEmitter() { + this._docs = {}; + this._forwarders = {}; + this._emitters = {}; +} + +DocPresenceEmitter.prototype.addEventListener = function(doc, event, listener) { + this._registerDoc(doc); + var emitter = util.dig(this._emitters, doc.collection, doc.id); + emitter.on(event, listener); +}; + +DocPresenceEmitter.prototype.removeEventListener = function(doc, event, listener) { + var emitter = util.dig(this._emitters, doc.collection, doc.id); + if (!emitter) return; + emitter.off(event, listener); + // We'll always have at least one, because of the destroy listener + if (emitter._eventsCount === 1) this._unregisterDoc(doc); +}; + +DocPresenceEmitter.prototype._registerDoc = function(doc) { + var alreadyRegistered = true; + util.digOrCreate(this._docs, doc.collection, doc.id, function() { + alreadyRegistered = false; + return doc; + }); + + if (alreadyRegistered) return; + + var emitter = util.digOrCreate(this._emitters, doc.collection, doc.id, function() { + var e = new EventEmitter(); + // Set a high limit to avoid unnecessary warnings, but still + // retain some degree of memory leak detection + e.setMaxListeners(1000); + return e; + }); + + var self = this; + EVENTS.forEach(function(event) { + var forwarder = util.digOrCreate(self._forwarders, doc.collection, doc.id, event, function() { + return emitter.emit.bind(emitter, event); + }); + + doc.on(event, forwarder); + }); + + this.addEventListener(doc, 'destroy', this._unregisterDoc.bind(this, doc)); +}; + +DocPresenceEmitter.prototype._unregisterDoc = function(doc) { + var forwarders = util.dig(this._forwarders, doc.collection, doc.id); + for (var event in forwarders) { + doc.off(event, forwarders[event]); + } + + var emitter = util.dig(this._emitters, doc.collection, doc.id); + emitter.removeAllListeners(); + + util.digAndRemove(this._forwarders, doc.collection, doc.id); + util.digAndRemove(this._emitters, doc.collection, doc.id); + util.digAndRemove(this._docs, doc.collection, doc.id); +}; diff --git a/lib/client/presence/local-doc-presence.js b/lib/client/presence/local-doc-presence.js index e6e84ba1d..ba752becf 100644 --- a/lib/client/presence/local-doc-presence.js +++ b/lib/client/presence/local-doc-presence.js @@ -11,6 +11,7 @@ function LocalDocPresence(presence, presenceId) { this.id = this.presence.id; this._doc = this.connection.get(this.collection, this.id); + this._emitter = this.connection._docPresenceEmitter; this._isSending = false; this._docDataVersionByPresenceVersion = {}; @@ -43,11 +44,11 @@ LocalDocPresence.prototype.submit = function(value, callback) { }; LocalDocPresence.prototype.destroy = function(callback) { - this._doc.removeListener('op', this._opHandler); - this._doc.removeListener('create', this._createOrDelHandler); - this._doc.removeListener('del', this._createOrDelHandler); - this._doc.removeListener('load', this._loadHandler); - this._doc.removeListener('destroy', this._destroyHandler); + this._emitter.removeEventListener(this._doc, 'op', this._opHandler); + this._emitter.removeEventListener(this._doc, 'create', this._createOrDelHandler); + this._emitter.removeEventListener(this._doc, 'del', this._createOrDelHandler); + this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); + this._emitter.removeEventListener(this._doc, 'destroy', this._destroyHandler); LocalPresence.prototype.destroy.call(this, callback); }; @@ -72,11 +73,11 @@ LocalDocPresence.prototype._sendPending = function() { }; LocalDocPresence.prototype._registerWithDoc = function() { - this._doc.on('op', this._opHandler); - this._doc.on('create', this._createOrDelHandler); - this._doc.on('del', this._createOrDelHandler); - this._doc.on('load', this._loadHandler); - this._doc.on('destroy', this._destroyHandler); + this._emitter.addEventListener(this._doc, 'op', this._opHandler); + this._emitter.addEventListener(this._doc, 'create', this._createOrDelHandler); + this._emitter.addEventListener(this._doc, 'del', this._createOrDelHandler); + this._emitter.addEventListener(this._doc, 'load', this._loadHandler); + this._emitter.addEventListener(this._doc, 'destroy', this._destroyHandler); }; LocalDocPresence.prototype._transformAgainstOp = function(op, source) { diff --git a/lib/client/presence/remote-doc-presence.js b/lib/client/presence/remote-doc-presence.js index 2fef91ffa..deb9a34a4 100644 --- a/lib/client/presence/remote-doc-presence.js +++ b/lib/client/presence/remote-doc-presence.js @@ -11,6 +11,7 @@ function RemoteDocPresence(presence, presenceId) { this.presenceVersion = null; this._doc = this.connection.get(this.collection, this.id); + this._emitter = this.connection._docPresenceEmitter; this._pending = null; this._opCache = null; this._pendingSetPending = false; @@ -31,19 +32,19 @@ RemoteDocPresence.prototype.receiveUpdate = function(message) { }; RemoteDocPresence.prototype.destroy = function(callback) { - this._doc.removeListener('op', this._opHandler); - this._doc.removeListener('create', this._createDelHandler); - this._doc.removeListener('del', this._createDelHandler); - this._doc.removeListener('load', this._loadHandler); + this._emitter.removeEventListener(this._doc, 'op', this._opHandler); + this._emitter.removeEventListener(this._doc, 'create', this._createDelHandler); + this._emitter.removeEventListener(this._doc, 'del', this._createDelHandler); + this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); RemotePresence.prototype.destroy.call(this, callback); }; RemoteDocPresence.prototype._registerWithDoc = function() { - this._doc.on('op', this._opHandler); - this._doc.on('create', this._createDelHandler); - this._doc.on('del', this._createDelHandler); - this._doc.on('load', this._loadHandler); + this._emitter.addEventListener(this._doc, 'op', this._opHandler); + this._emitter.addEventListener(this._doc, 'create', this._createDelHandler); + this._emitter.addEventListener(this._doc, 'del', this._createDelHandler); + this._emitter.addEventListener(this._doc, 'load', this._loadHandler); }; RemoteDocPresence.prototype._setPendingPresence = function() { diff --git a/test/client/presence/doc-presence-emitter.js b/test/client/presence/doc-presence-emitter.js new file mode 100644 index 000000000..b2bebe86f --- /dev/null +++ b/test/client/presence/doc-presence-emitter.js @@ -0,0 +1,100 @@ +var Backend = require('../../../lib/backend'); +var errorHandler = require('../../util').errorHandler; +var expect = require('chai').expect; + +describe('DocPresenceEmitter', function() { + var backend; + var connection; + var doc; + var emitter; + + beforeEach(function(done) { + backend = new Backend(); + connection = backend.connect(); + doc = connection.get('books', 'northern-lights'); + doc.create({title: 'Northern Lights'}, done); + emitter = connection._docPresenceEmitter; + }); + + it('listens to an op event', function(done) { + emitter.addEventListener(doc, 'op', function(op) { + expect(op).to.eql([{p: ['author'], oi: 'Philip Pullman'}]); + done(); + }); + + doc.submitOp([{p: ['author'], oi: 'Philip Pullman'}], errorHandler(done)); + }); + + it('stops listening to events', function(done) { + var listener = function() { + done(new Error('should not reach')); + }; + + emitter.addEventListener(doc, 'op', listener); + emitter.removeEventListener(doc, 'op', listener); + + doc.submitOp([{p: ['author'], oi: 'Philip Pullman'}], done); + }); + + it('removes the listener from the doc if there are no more listeners', function() { + expect(doc._eventsCount).to.equal(0); + var listener = function() {}; + + emitter.addEventListener(doc, 'op', listener); + + expect(doc._eventsCount).to.be.greaterThan(0); + expect(emitter._docs).not.to.be.empty; + expect(emitter._emitters).not.to.be.empty; + expect(emitter._forwarders).not.to.be.empty; + + emitter.removeEventListener(doc, 'op', listener); + + expect(doc._eventsCount).to.equal(0); + expect(emitter._docs).to.be.empty; + expect(emitter._emitters).to.be.empty; + expect(emitter._forwarders).to.be.empty; + }); + + it('only registers a single listener on the doc', function() { + expect(doc._eventsCount).to.equal(0); + var listener = function() { }; + emitter.addEventListener(doc, 'op', listener); + var count = doc._eventsCount; + emitter.addEventListener(doc, 'op', listener); + expect(doc._eventsCount).to.equal(count); + }); + + it('only triggers the given event', function(done) { + emitter.addEventListener(doc, 'op', function(op) { + expect(op).to.eql([{p: ['author'], oi: 'Philip Pullman'}]); + done(); + }); + + emitter.addEventListener(doc, 'del', function() { + done(new Error('should not reach')); + }); + + doc.submitOp([{p: ['author'], oi: 'Philip Pullman'}], errorHandler(done)); + }); + + it('removes listeners on destroy', function(done) { + expect(doc._eventsCount).to.equal(0); + var listener = function() { }; + + emitter.addEventListener(doc, 'op', listener); + + expect(doc._eventsCount).to.be.greaterThan(0); + expect(emitter._docs).not.to.be.empty; + expect(emitter._emitters).not.to.be.empty; + expect(emitter._forwarders).not.to.be.empty; + + doc.destroy(function(error) { + if (error) return done(error); + expect(doc._eventsCount).to.equal(0); + expect(emitter._docs).to.be.empty; + expect(emitter._emitters).to.be.empty; + expect(emitter._forwarders).to.be.empty; + done(); + }); + }); +}); diff --git a/test/client/presence/doc-presence.js b/test/client/presence/doc-presence.js index ea0bdfe71..1011ac74b 100644 --- a/test/client/presence/doc-presence.js +++ b/test/client/presence/doc-presence.js @@ -1025,4 +1025,14 @@ describe('DocPresence', function() { } ], done); }); + + it('does not trigger EventEmitter memory leak warnings', function() { + for (var i = 0; i < 100; i++) { + presence1.create(); + } + + expect(doc1._events.op.warned).not.to.be.ok; + var emitter = connection1._docPresenceEmitter._emitters[doc1.collection][doc1.id]; + expect(emitter._events.op.warned).not.to.be.ok; + }); }); From b84290f8a66ee42b3d2c75d6de780e125b99c514 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:04:52 +0000 Subject: [PATCH 37/61] 3.2.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9656c98c4..6f4f3695c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.2.1", + "version": "3.2.2", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 76810776e7d165de0129973e5a52b0665a792cc8 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Wed, 15 Feb 2023 11:12:21 +0000 Subject: [PATCH 38/61] =?UTF-8?q?=E2=9C=85=20Fix=20flaky=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/share/sharedb-mongo/issues/131 This change attempts to fix some tests that are flaky in `sharedb-mongo`. The flakiness can be reproduced locally by wrapping the `Agent._querySubscribe()` [call to `_fetchBulkOps()`][1] in a long `setTimeout()`: ```js if (options.fetchOps) { wait++; setTimeout(function() { console.log('fetch bulk ops'); agent._fetchBulkOps(collection, options.fetchOps, finish); }, 1000); } ``` This forces us into an edge case where the subscribe query triggers and returns the diff from a [`queryPoll()`][2], which triggers the tests' `'insert'` handlers, finishes the test, and closes the backend, all before the `_fetchBulkOps()` call is issued (which subsequently fails because the database has been closed). Handling this query subscribe actually triggers a variety of responses to be sent to the client at different times: 1. The actual `_querySubscribe()` [callback][3] (which ultimately triggers [`agent._reply()`][4] in response to the original request) 2. Ops sent [independently][5] by [`_fetchBulkOps()`][6] 3. The diff resulting from [`queryPoll()`][2] In order to reduce flakiness, this change adds checks that the query's [`'ready'` event][7] has been called, which will happen once the resubscribe request has been replied to by the `Agent`. This ensures we've waited for events 1 & 3 of the above list, although we notably aren't waiting for event 2 (which is where the error is actually coming from). Since no ops will actually be sent to the client, I'm not sure how best to wait for this. Hopefully waiting for the subscribe ack should be sufficient. [1]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L521-L524 [2]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L534 [3]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L511 [4]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L344 [5]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L265 [6]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/agent.js#L523 [7]: https://github.com/share/sharedb/blob/b84290f8a66ee42b3d2c75d6de780e125b99c514/lib/client/query.js#L148 --- test/client/query-subscribe.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test/client/query-subscribe.js b/test/client/query-subscribe.js index 5a3314655..eef9685f8 100644 --- a/test/client/query-subscribe.js +++ b/test/client/query-subscribe.js @@ -187,18 +187,25 @@ module.exports = function(options) { } ], function(err) { if (err) return done(err); + + var wait = 2; + function finish() { + if (--wait) return; + expect(util.pluck(query.results, 'id')).to.eql(['fido', 'spot', 'taco']); + done(); + } + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); connection.close(); connection2.get('dogs', 'taco').on('error', done).create({age: 2}); process.nextTick(function() { backend.connect(connection); + query.on('ready', finish); }); }); query.on('error', done); - query.on('insert', function() { - done(); - }); + query.on('insert', finish); }); }); @@ -234,11 +241,14 @@ module.exports = function(options) { ], function(error) { if (error) return done(error); backend.connect(connection); + query.once('ready', function() { + finish(); + }); }); }); }); - var wait = 2; + var wait = 3; function finish() { if (--wait) return; var results = util.sortById(query.results); From 9c0f1c6f8034c6e9078372fa53992b8aaec8c714 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Thu, 16 Feb 2023 08:11:20 +0000 Subject: [PATCH 39/61] 3.2.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6f4f3695c..70fed9e56 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.2.2", + "version": "3.2.3", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 36c8e65017459c464aedb99363f0e581f09ad475 Mon Sep 17 00:00:00 2001 From: Jim Fisher Date: Tue, 21 Feb 2023 08:51:28 +0000 Subject: [PATCH 40/61] Add missing import in hello world example --- docs/getting-started.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 8ea13b080..6a9c31d37 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,6 +38,7 @@ The ShareDB backend expects an instance of a [`Stream`](https://nodejs.org/api/s ```js var express = require('express') var WebSocket = require('ws') +var http = require('http') var ShareDB = require('sharedb') var WebSocketJSONStream = require('@teamwork/websocket-json-stream') From daf3a25e1906e007710481b279a5894a427a66ce Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:56:54 +0000 Subject: [PATCH 41/61] =?UTF-8?q?=E2=9C=85=20Remove=20order=20dependence?= =?UTF-8?q?=20from=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/client/query-subscribe.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/client/query-subscribe.js b/test/client/query-subscribe.js index eef9685f8..a414f1468 100644 --- a/test/client/query-subscribe.js +++ b/test/client/query-subscribe.js @@ -191,7 +191,7 @@ module.exports = function(options) { var wait = 2; function finish() { if (--wait) return; - expect(util.pluck(query.results, 'id')).to.eql(['fido', 'spot', 'taco']); + expect(util.pluck(query.results, 'id')).to.have.members(['fido', 'spot', 'taco']); done(); } From f4845fd10c3c8704506017cfcfc564821914f5bf Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:00:00 +0000 Subject: [PATCH 42/61] 3.2.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 70fed9e56..e0c05cbaa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.2.3", + "version": "3.2.4", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From ad23bf008969afb15cda80397f23e53ae64bbca2 Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 2 Mar 2023 03:30:45 -0500 Subject: [PATCH 43/61] Upgrade deps in examples. Closes #594 --- examples/counter-json1/package.json | 12 +++++------ examples/counter/package.json | 10 ++++----- examples/leaderboard/package.json | 26 ++++++++++++------------ examples/rich-text-presence/package.json | 14 ++++++------- examples/rich-text/package.json | 12 +++++------ examples/textarea/package.json | 10 ++++----- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/examples/counter-json1/package.json b/examples/counter-json1/package.json index f1a27d627..bb7221ebb 100644 --- a/examples/counter-json1/package.json +++ b/examples/counter-json1/package.json @@ -17,13 +17,13 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "express": "^4.14.0", - "ot-json1": "^1.0.0", - "reconnecting-websocket": "^4.2.0", - "sharedb": "^1.0.0-beta", - "ws": "^7.2.0" + "express": "^4.18.2", + "ot-json1": "^1.0.2", + "reconnecting-websocket": "^4.4.0", + "sharedb": "^3.2.4", + "ws": "^8.12.1" }, "devDependencies": { - "browserify": "^16.5.0" + "browserify": "^17.0.0" } } diff --git a/examples/counter/package.json b/examples/counter/package.json index 8cd2eea6f..953c991b5 100644 --- a/examples/counter/package.json +++ b/examples/counter/package.json @@ -16,12 +16,12 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "express": "^4.14.0", - "reconnecting-websocket": "^4.2.0", - "sharedb": "^1.0.0-beta", - "ws": "^7.2.0" + "express": "^4.18.2", + "reconnecting-websocket": "^4.4.0", + "sharedb": "^3.2.4", + "ws": "^8.12.1" }, "devDependencies": { - "browserify": "^16.5.0" + "browserify": "^17.0.0" } } diff --git a/examples/leaderboard/package.json b/examples/leaderboard/package.json index 15e9f3b6d..28f274888 100644 --- a/examples/leaderboard/package.json +++ b/examples/leaderboard/package.json @@ -16,20 +16,20 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "classnames": "^2.2.5", - "express": "^4.17.1", - "prop-types": "^15.7.2", - "react": "^16.11.0", - "react-dom": "^16.11.0", - "reconnecting-websocket": "^4.2.0", - "sharedb": "^1.0.0-beta", - "sharedb-mingo-memory": "^1.0.0-beta", - "underscore": "^1.8.3", - "ws": "^7.2.0" + "classnames": "^2.3.2", + "express": "^4.18.2", + "prop-types": "^15.8.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "reconnecting-websocket": "^4.4.0", + "sharedb": "^3.2.4", + "sharedb-mingo-memory": "^2.1.2", + "underscore": "^1.13.6", + "ws": "^8.12.1" }, "devDependencies": { - "babel-preset-react": "^6.5.0", - "babelify": "^7.3.0", - "browserify": "^16.5.0" + "babel-preset-react": "^6.24.1", + "babelify": "^10.0.0", + "browserify": "^17.0.0" } } diff --git a/examples/rich-text-presence/package.json b/examples/rich-text-presence/package.json index 736058e3c..9ebb29771 100644 --- a/examples/rich-text-presence/package.json +++ b/examples/rich-text-presence/package.json @@ -16,17 +16,17 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "bson-objectid": "^1.3.0", - "express": "^4.17.1", + "bson-objectid": "^2.0.4", + "express": "^4.18.2", "quill": "^1.3.7", - "quill-cursors": "^2.2.1", - "reconnecting-websocket": "^4.2.0", + "quill-cursors": "^4.0.2", + "reconnecting-websocket": "^4.4.0", "rich-text": "^4.1.0", "sharedb": "file:../../", - "tinycolor2": "^1.4.1", - "ws": "^7.2.0" + "tinycolor2": "^1.6.0", + "ws": "^8.12.1" }, "devDependencies": { - "browserify": "^16.5.0" + "browserify": "^17.0.0" } } diff --git a/examples/rich-text/package.json b/examples/rich-text/package.json index d4e0bfd2a..9504f8216 100644 --- a/examples/rich-text/package.json +++ b/examples/rich-text/package.json @@ -15,14 +15,14 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "express": "^4.17.1", + "express": "^4.18.2", "quill": "^1.3.7", - "reconnecting-websocket": "^4.2.0", - "rich-text": "^4.0.0", - "sharedb": "^1.0.0-beta", - "ws": "^7.2.0" + "reconnecting-websocket": "^4.4.0", + "rich-text": "^4.1.0", + "sharedb": "^3.2.4", + "ws": "^8.12.1" }, "devDependencies": { - "browserify": "^16.5.0" + "browserify": "^17.0.0" } } diff --git a/examples/textarea/package.json b/examples/textarea/package.json index 73a4cbcdc..6364d6c5e 100644 --- a/examples/textarea/package.json +++ b/examples/textarea/package.json @@ -15,13 +15,13 @@ "license": "MIT", "dependencies": { "@teamwork/websocket-json-stream": "^2.0.0", - "express": "^4.17.1", - "reconnecting-websocket": "^4.2.0", - "sharedb": "^1.0.0-beta", + "express": "^4.18.2", + "reconnecting-websocket": "^4.4.0", + "sharedb": "^3.2.4", "sharedb-string-binding": "^1.0.0", - "ws": "^7.2.0" + "ws": "^8.12.1" }, "devDependencies": { - "browserify": "^16.5.0" + "browserify": "^17.0.0" } } From 2690ecbdcbc4921165aeb5542d70b6441b28b48f Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 2 Mar 2023 03:43:19 -0500 Subject: [PATCH 44/61] Fix leaderboard example --- examples/leaderboard/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/leaderboard/package.json b/examples/leaderboard/package.json index 28f274888..dd74e0393 100644 --- a/examples/leaderboard/package.json +++ b/examples/leaderboard/package.json @@ -4,7 +4,7 @@ "description": "React Leaderboard backed by ShareDB", "main": "server.js", "scripts": { - "build": "browserify -t [ babelify --presets [ react ] ] client/index.jsx -o static/dist/bundle.js", + "build": "browserify -t [ babelify --presets [ @babel/react ] ] client/index.jsx -o static/dist/bundle.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "node server/index.js" }, @@ -28,7 +28,7 @@ "ws": "^8.12.1" }, "devDependencies": { - "babel-preset-react": "^6.24.1", + "@babel/preset-react": "^7.18.6", "babelify": "^10.0.0", "browserify": "^17.0.0" } From 9b4c9f49e9413819b231263e217c29bf5a677d87 Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 2 Mar 2023 03:53:26 -0500 Subject: [PATCH 45/61] Fix rich-text-presence example --- examples/rich-text-presence/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rich-text-presence/package.json b/examples/rich-text-presence/package.json index 9ebb29771..69b87ff33 100644 --- a/examples/rich-text-presence/package.json +++ b/examples/rich-text-presence/package.json @@ -22,7 +22,7 @@ "quill-cursors": "^4.0.2", "reconnecting-websocket": "^4.4.0", "rich-text": "^4.1.0", - "sharedb": "file:../../", + "sharedb": "^3.2.4", "tinycolor2": "^1.6.0", "ws": "^8.12.1" }, From 72e47b17ae514d84fb4a2f76871d11fbea3042ee Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 2 Mar 2023 04:22:11 -0500 Subject: [PATCH 46/61] Start on Vite example --- examples/counter-json1-vite/.gitignore | 24 ++++++++++++++ examples/counter-json1-vite/index.html | 14 ++++++++ examples/counter-json1-vite/main.js | 33 +++++++++++++++++++ examples/counter-json1-vite/package.json | 23 +++++++++++++ examples/counter-json1-vite/server.js | 41 ++++++++++++++++++++++++ 5 files changed, 135 insertions(+) create mode 100644 examples/counter-json1-vite/.gitignore create mode 100644 examples/counter-json1-vite/index.html create mode 100644 examples/counter-json1-vite/main.js create mode 100644 examples/counter-json1-vite/package.json create mode 100644 examples/counter-json1-vite/server.js diff --git a/examples/counter-json1-vite/.gitignore b/examples/counter-json1-vite/.gitignore new file mode 100644 index 000000000..a547bf36d --- /dev/null +++ b/examples/counter-json1-vite/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/examples/counter-json1-vite/index.html b/examples/counter-json1-vite/index.html new file mode 100644 index 000000000..8b3d96714 --- /dev/null +++ b/examples/counter-json1-vite/index.html @@ -0,0 +1,14 @@ + + + + + ShareDB Counter (ottype json1 with Vite) + + +
+ You clicked times. + +
+ + + diff --git a/examples/counter-json1-vite/main.js b/examples/counter-json1-vite/main.js new file mode 100644 index 000000000..ec2aada13 --- /dev/null +++ b/examples/counter-json1-vite/main.js @@ -0,0 +1,33 @@ +import ReconnectingWebSocket from 'reconnecting-websocket'; +import sharedb from 'sharedb/lib/client'; +import json1 from 'ot-json1'; + +// Open WebSocket connection to ShareDB server +var socket = new ReconnectingWebSocket('ws://' + window.location.host); +sharedb.types.register(json1.type); +var connection = new sharedb.Connection(socket); + +// Create local Doc instance mapped to 'examples' collection document with id 'counter' +var doc = connection.get('examples', 'counter'); + +// Get initial value of document and subscribe to changes +doc.subscribe(showNumbers); +// When document changes (by this client or any other, or the server), +// update the number on the page +doc.on('op', showNumbers); + +function showNumbers() { + document.querySelector('#num-clicks').textContent = doc.data.numClicks; +}; + +// When clicking on the '+1' button, change the number in the local +// document and sync the change to the server and other connected +// clients +function increment() { + // Increment `doc.data.numClicks`. See + // https://github.com/ottypes/json1/blob/master/spec.md for list of valid operations. + doc.submitOp(['numClicks', {ena: 1}]); +} + +// Expose to index.html +global.increment = increment; diff --git a/examples/counter-json1-vite/package.json b/examples/counter-json1-vite/package.json new file mode 100644 index 000000000..d2f8a6c75 --- /dev/null +++ b/examples/counter-json1-vite/package.json @@ -0,0 +1,23 @@ +{ + "name": "counter-json1-vite", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "start": "node server.js" + }, + "dependencies": { + "@teamwork/websocket-json-stream": "^2.0.0", + "express": "^4.18.2", + "ot-json1": "^1.0.2", + "reconnecting-websocket": "^4.4.0", + "sharedb": "^3.2.4", + "ws": "^8.12.1" + }, + "devDependencies": { + "vite": "^4.1.4" + } +} diff --git a/examples/counter-json1-vite/server.js b/examples/counter-json1-vite/server.js new file mode 100644 index 000000000..1c24334c4 --- /dev/null +++ b/examples/counter-json1-vite/server.js @@ -0,0 +1,41 @@ +import http from 'http'; +import express from 'express'; +import ShareDB from 'sharedb'; +import { WebSocketServer } from 'ws'; +import WebSocketJSONStream from '@teamwork/websocket-json-stream'; +import json1 from 'ot-json1'; + +ShareDB.types.register(json1.type); +var backend = new ShareDB(); +createDoc(startServer); + +// Create initial document then fire callback +function createDoc(callback) { + var connection = backend.connect(); + var doc = connection.get('examples', 'counter'); + doc.fetch(function(err) { + if (err) throw err; + if (doc.type === null) { + doc.create({numClicks: 0}, json1.type.uri, callback); + return; + } + callback(); + }); +} + +function startServer() { + // Create a web server to serve files and listen to WebSocket connections + var app = express(); + app.use(express.static('dist')); + var server = http.createServer(app); + + // Connect any incoming WebSocket connection to ShareDB + var wss = new WebSocketServer({server: server}); + wss.on('connection', function(ws) { + var stream = new WebSocketJSONStream(ws); + backend.listen(stream); + }); + + server.listen(8080); + console.log('Listening on http://localhost:8080'); +} From 280d681914ff588718f79152149fa6f0b5805d71 Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 2 Mar 2023 07:17:44 -0500 Subject: [PATCH 47/61] Fix CI --- .eslintrc.js | 9 +++++++-- .gitignore | 1 + examples/counter-json1-vite/server.js | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 50a30441f..cf96acb8d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -34,13 +34,18 @@ var SHAREDB_RULES = { // as-needed quote props are easier to write 'quote-props': ['error', 'as-needed'], 'require-jsdoc': 'off', - 'valid-jsdoc': 'off' + 'valid-jsdoc': 'off', + + // Required after upgrade to ecmaVersion: 6 + 'no-invalid-this': 'off' }; module.exports = { extends: 'google', parserOptions: { - ecmaVersion: 3 + // Support ES6 imports and exports + ecmaVersion: 6, + sourceType: 'module' }, rules: Object.assign( {}, diff --git a/.gitignore b/.gitignore index 7c27974e1..1086994ce 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ jspm_packages # Don't commit generated JS bundles examples/**/static/dist/bundle.js +examples/**/dist diff --git a/examples/counter-json1-vite/server.js b/examples/counter-json1-vite/server.js index 1c24334c4..68e628d2e 100644 --- a/examples/counter-json1-vite/server.js +++ b/examples/counter-json1-vite/server.js @@ -1,7 +1,7 @@ import http from 'http'; import express from 'express'; import ShareDB from 'sharedb'; -import { WebSocketServer } from 'ws'; +import {WebSocketServer} from 'ws'; import WebSocketJSONStream from '@teamwork/websocket-json-stream'; import json1 from 'ot-json1'; From 628de990e858439c1f3dbff4f65f95ef9a70d0a2 Mon Sep 17 00:00:00 2001 From: Dawid Kisielewski Date: Mon, 30 Jan 2023 12:22:09 +0100 Subject: [PATCH 48/61] =?UTF-8?q?=E2=9C=A8=20Add=20multi=20channel=20query?= =?UTF-8?q?=20subscription.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment we can only specify one channel the specific query, would be able to listen to, as per docs: ```javascript backend.use('query', (context, next) => { // Set our query to only listen for changes on our user-specific channel context.channel = userChannel(context) next() }) ``` However let's imagine the situation where the user wants to query all the posts, the where posted by him and all his friends. Now we would need new query every for friend separately to make sure the proper scalability is preserved and we do not receive all the changes to posts collection. This change allows to listen for multiple channels, so if we want to query all user friends posts. We can do it by this: ```javascript backend.use('query', (context, next) => { // Set our query to only listen for changes on our user-specific channel context.channels = [userChannel(context), friendChannel(context))] next() }) ``` Now this query would only listen to all the changes that were made to the user posts and his friends. --- docs/middleware/actions.md | 8 +- docs/queries.md | 2 +- lib/backend.js | 49 +- lib/error.js | 1 + lib/query-emitter.js | 57 +- package.json | 2 +- test/client/presence/presence.js | 25 - test/client/query-subscribe.js | 1274 ++++++++++++++++++------------ test/logger.js | 4 - test/setup.js | 5 + 10 files changed, 872 insertions(+), 555 deletions(-) diff --git a/docs/middleware/actions.md b/docs/middleware/actions.md index 0b9ab4965..bcd8ba71a 100644 --- a/docs/middleware/actions.md +++ b/docs/middleware/actions.md @@ -149,9 +149,13 @@ This action has these additional `context` properties: > The query's projection [fields]({{ site.baseurl }}{% link api/backend.md %}#addprojection) -`channel` -- string +`channel` -- string (deprecated) -> The [Pub/Sub]({{ site.baseurl }}{% link adapters/pub-sub.md %}) channel the query will subscribe to. Defaults to its collection channel. +> This property is deprecated use `channels` instead. The [Pub/Sub]({{ site.baseurl }}{% link adapters/pub-sub.md %}) channels the query will subscribe to. Defaults to its collection channel. + +`channels` -- string[] + +> The [Pub/Sub]({{ site.baseurl }}{% link adapters/pub-sub.md %}) channels the query will subscribe to. Defaults to its collection channel. `query` -- Object diff --git a/docs/queries.md b/docs/queries.md index 930ef43b2..bd7d9d3b6 100644 --- a/docs/queries.md +++ b/docs/queries.md @@ -160,7 +160,7 @@ backend.use('commit', (context, next) => { backend.use('query', (context, next) => { // Set our query to only listen for changes on our user-specific channel - context.channel = userChannel(context) + context.channels = [userChannel(context)] next() }) diff --git a/lib/backend.js b/lib/backend.js index 0a4f6a884..08cfa3480 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -661,10 +661,32 @@ Backend.prototype.querySubscribe = function(agent, index, query, options, callba 'DB does not support subscribe' )); } - backend.pubsub.subscribe(request.channel, function(err, stream) { - if (err) return callback(err); + + var channels = request.channels; + + if (request.channel) { + logger.warn( + '[DEPRECATED] "query" middleware\'s context.channel is deprecated, use context.channels instead. ' + + 'Read more: https://share.github.io/sharedb/middleware/actions#query' + ); + channels = [request.channel]; + } + + if (!channels || !channels.length) { + return callback(new ShareDBError(ERROR_CODE.ERR_QUERY_CHANNEL_MISSING, 'Required minimum one query channel.')); + } + + var streams = []; + + function destroyStreams() { + streams.forEach(function(stream) { + stream.destroy(); + }); + } + + function createQueryEmitter() { if (options.ids) { - var queryEmitter = new QueryEmitter(request, stream, options.ids); + var queryEmitter = new QueryEmitter(request, streams, options.ids); backend.emit('timing', 'querySubscribe.reconnect', Date.now() - start, request); callback(null, queryEmitter); return; @@ -672,14 +694,29 @@ Backend.prototype.querySubscribe = function(agent, index, query, options, callba // Issue query on db to get our initial results backend._query(agent, request, function(err, snapshots, extra) { if (err) { - stream.destroy(); + destroyStreams(); return callback(err); } var ids = pluckIds(snapshots); - var queryEmitter = new QueryEmitter(request, stream, ids, extra); + var queryEmitter = new QueryEmitter(request, streams, ids, extra); backend.emit('timing', 'querySubscribe.initial', Date.now() - start, request); callback(null, queryEmitter, snapshots, extra); }); + } + + channels.forEach(function(channel) { + backend.pubsub.subscribe(channel, function(err, stream) { + if (err) { + destroyStreams(); + return callback(err); + } + streams.push(stream); + + var subscribedToAllChannels = streams.length === channels.length; + if (subscribedToAllChannels) { + createQueryEmitter(); + } + }); }); }); }; @@ -693,7 +730,7 @@ Backend.prototype._triggerQuery = function(agent, index, query, options, callbac collection: collection, projection: projection, fields: fields, - channel: this.getCollectionChannel(collection), + channels: [this.getCollectionChannel(collection)], query: query, options: options, db: null, diff --git a/lib/error.js b/lib/error.js index 929684643..a93c84672 100644 --- a/lib/error.js +++ b/lib/error.js @@ -45,6 +45,7 @@ ShareDBError.CODES = { ERR_OT_OP_NOT_PROVIDED: 'ERR_OT_OP_NOT_PROVIDED', ERR_PRESENCE_TRANSFORM_FAILED: 'ERR_PRESENCE_TRANSFORM_FAILED', ERR_PROTOCOL_VERSION_NOT_SUPPORTED: 'ERR_PROTOCOL_VERSION_NOT_SUPPORTED', + ERR_QUERY_CHANNEL_MISSING: 'ERR_QUERY_CHANNEL_MISSING', ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED: 'ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED', /** * A special error that a "readSnapshots" middleware implementation can use to indicate that it diff --git a/lib/query-emitter.js b/lib/query-emitter.js index 79877bc31..cfbd930af 100644 --- a/lib/query-emitter.js +++ b/lib/query-emitter.js @@ -5,7 +5,7 @@ var util = require('./util'); var ERROR_CODE = ShareDBError.CODES; -function QueryEmitter(request, stream, ids, extra) { +function QueryEmitter(request, streams, ids, extra) { this.backend = request.backend; this.agent = request.agent; this.db = request.db; @@ -15,7 +15,7 @@ function QueryEmitter(request, stream, ids, extra) { this.fields = request.fields; this.options = request.options; this.snapshotProjection = request.snapshotProjection; - this.stream = stream; + this.streams = streams; this.ids = ids; this.extra = extra; @@ -23,10 +23,12 @@ function QueryEmitter(request, stream, ids, extra) { this.canPollDoc = this.db.canPollDoc(this.collection, this.query); this.pollDebounce = (typeof this.options.pollDebounce === 'number') ? this.options.pollDebounce : - (typeof this.db.pollDebounce === 'number') ? this.db.pollDebounce : 0; + (typeof this.db.pollDebounce === 'number') ? this.db.pollDebounce : + streams.length > 1 ? 1000 : 0; this.pollInterval = (typeof this.options.pollInterval === 'number') ? this.options.pollInterval : - (typeof this.db.pollInterval === 'number') ? this.db.pollInterval : 0; + (typeof this.db.pollInterval === 'number') ? this.db.pollInterval : + streams.length > 1 ? 1000 : 0; this._polling = false; this._pendingPoll = null; @@ -41,15 +43,19 @@ QueryEmitter.prototype._open = function() { this._defaultCallback = function(err) { if (err) emitter.onError(err); }; - emitter.stream.on('data', function(data) { - if (data.error) { - return emitter.onError(data.error); - } - emitter._update(data); - }); - emitter.stream.on('end', function() { - emitter.destroy(); + + emitter.streams.forEach(function(stream) { + stream.on('data', function(data) { + if (data.error) { + return emitter.onError(data.error); + } + emitter._update(data); + }); + stream.on('end', function() { + emitter.destroy(); + }); }); + // Make sure we start polling if pollInterval is being used this._flushPoll(); }; @@ -57,7 +63,12 @@ QueryEmitter.prototype._open = function() { QueryEmitter.prototype.destroy = function() { clearTimeout(this._pollDebounceId); clearTimeout(this._pollIntervalId); - this.stream.destroy(); + + var stream; + + while (stream = this.streams.pop()) { + stream.destroy(); + } }; QueryEmitter.prototype._emitTiming = function(action, start) { @@ -140,8 +151,8 @@ QueryEmitter.prototype._flushPoll = function() { if (this._pendingPoll) { this.queryPoll(); - // If a pollInterval is specified, poll if the query doesn't get polled in - // the time of the interval + // If a pollInterval is specified, poll if the query doesn't get polled in + // the time of the interval } else if (this.pollInterval) { var emitter = this; this._pollIntervalId = setTimeout(function() { @@ -301,14 +312,14 @@ QueryEmitter.prototype.queryPollDoc = function(id, callback) { // all messages are received and applied in order, so it is critical that none // are dropped. QueryEmitter.prototype.onError = -QueryEmitter.prototype.onDiff = -QueryEmitter.prototype.onExtra = -QueryEmitter.prototype.onOp = function() { - throw new ShareDBError( - ERROR_CODE.ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED, - 'Required QueryEmitter listener not assigned' - ); -}; + QueryEmitter.prototype.onDiff = + QueryEmitter.prototype.onExtra = + QueryEmitter.prototype.onOp = function() { + throw new ShareDBError( + ERROR_CODE.ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED, + 'Required QueryEmitter listener not assigned' + ); + }; function getInserted(diff) { var inserted = []; diff --git a/package.json b/package.json index e0c05cbaa..fe70b7b9a 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "ot-json1": "^0.3.0", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@=1.1.0", - "sinon": "^7.5.0" + "sinon": "^9.2.4" }, "files": [ "lib/", diff --git a/test/client/presence/presence.js b/test/client/presence/presence.js index c2a59c495..f7a52762b 100644 --- a/test/client/presence/presence.js +++ b/test/client/presence/presence.js @@ -38,7 +38,6 @@ describe('Presence', function() { }); afterEach(function(done) { - sinon.restore(); connection1.close(); connection2.close(); backend.close(done); @@ -147,30 +146,6 @@ describe('Presence', function() { ], errorHandler(done)); }); - it('destroys old local presence but keeps new local presence when getting during destroy', function(done) { - presence2.create('presence-2'); - var presence2a; - - async.series([ - presence2.subscribe.bind(presence2), - function(next) { - presence2.destroy(function() { - expect(presence2).to.equal(presence2a); - expect(Object.keys(presence2.localPresences)).to.eql(['presence-2a']); - done(); - }); - next(); - }, - function(next) { - presence2a = connection2.getPresence('test-channel'); - presence2a.create('presence-2a'); - presence2a.subscribe(function(error) { - next(error); - }); - } - ], errorHandler(done)); - }); - it('throws if trying to create local presence when wanting destroy', function(done) { presence2.destroy(errorHandler(done)); expect(function() { diff --git a/test/client/query-subscribe.js b/test/client/query-subscribe.js index a414f1468..95e286f9a 100644 --- a/test/client/query-subscribe.js +++ b/test/client/query-subscribe.js @@ -2,6 +2,7 @@ var expect = require('chai').expect; var async = require('async'); var util = require('../util'); var sinon = require('sinon'); +var ShareDBError = require('../../lib/error'); module.exports = function(options) { var getQuery = options.getQuery; @@ -11,596 +12,883 @@ module.exports = function(options) { this.matchAllDbQuery = getQuery({query: {}}); }); - it('creating a document updates a subscribed query', function(done) { - var connection = this.backend.connect(); - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err) { + afterEach(function() { + sinon.restore(); + }); + + commonTests(options); + + describe('custom channels', function() { + it('only informs subscribed channels', function(done) { + this.backend.use('connect', function(context, next) { + context.agent.custom = context.req; + next(); + }); + + this.backend.use('commit', function(context, next) { + var user = context.agent.custom; + + if (user === 'sending-user-1') { + context.channels.push('channel-1'); + } + if (user === 'sending-user-2') { + context.channels.push('channel-2'); + } + next(); + }); + + this.backend.use('query', function(context, next) { + var user = context.agent.custom; + if (user === 'receiving-user') { + context.channels = ['channel-1', 'channel-2']; + } else if (user === 'not-receiving-user') { + context.channels = ['different-channel']; + } + next(); + }); + + var receivingUserConnection = this.backend.connect(null, 'receiving-user'); + var notReceivingUserConnection = this.backend.connect(null, 'not-receiving-user'); + var sendingUser1Connection = this.backend.connect(null, 'sending-user-1'); + var sendingUser2Connection = this.backend.connect(null, 'sending-user-2'); + + var notReceivingQuery = notReceivingUserConnection.createSubscribeQuery( + 'dogs', + this.matchAllDbQuery, + null, + function(err) { + if (err) return done(err); + } + ); + + notReceivingQuery.on('error', done); + notReceivingQuery.on('insert', function() { + done('User who didn\'t subscribed to sending channels shouldn\'t get the message'); + }); + + var receivingQuery = receivingUserConnection.createSubscribeQuery( + 'dogs', + this.matchAllDbQuery, + null, + function(err) { + if (err) return done(err); + sendingUser1Connection.get('dogs', '1').on('error', done).create({}); + sendingUser2Connection.get('dogs', '2').on('error', done).create({}); + } + ); + var receivedDogsCount = 0; + receivingQuery.on('error', done); + receivingQuery.on('insert', function() { + receivedDogsCount++; + if (receivedDogsCount === 2) { + var allDocsIds = receivingQuery.results.map(function(doc) { + return doc.id; + }); + expect(allDocsIds.sort()).to.be.deep.equal(['1', '2']); + done(); + } else if (receivedDogsCount > 2) { + done('It should not duplicate messages'); + } + }); + }); + + describe('one common channel', function() { + beforeEach(function() { + this.backend.use('commit', function(context, next) { + context.channels.push('channel-1'); + context.channels.push('channel-3'); + next(); + }); + this.backend.use('query', function(context, next) { + context.channels = ['channel-1', 'channel-2']; + next(); + }); + }); + + commonCustomChannelsErrorHandlingTests(); + commonTests(options); + }); + + describe('multiple common channels', function() { + beforeEach(function() { + this.backend.use('commit', function(context, next) { + context.channels.push('channel-1'); + context.channels.push('channel-2'); + next(); + }); + this.backend.use('query', function(context, next) { + context.channels = ['channel-1', 'channel-2']; + next(); + }); + }); + + it('does not duplicate messages', function(done) { + var connection = this.backend.connect(); + var count = 0; + var query = connection.createSubscribeQuery( + 'dogs', + this.matchAllDbQuery, + {pollInterval: 0, pollDebounce: 0}, + function(err) { + if (err) return done(err); + connection.get('dogs', '1').on('error', done).create({}); + connection.get('dogs', '2').on('error', done).create({}); + connection.get('dogs', '3').on('error', done).create({}); + } + ); + query.on('error', done); + query.on('insert', function() { + count++; + if (count === 3) { + var allDocsIds = query.results.map(function(doc) { + return doc.id; + }); + expect(allDocsIds.sort()).to.be.deep.equal(['1', '2', '3']); + done(); + } else if (count > 3) { + done('It should not duplicate messages'); + } + }); + }); + + commonCustomChannelsErrorHandlingTests(); + commonTests(options); + }); + + describe('backward compatibility', function() { + beforeEach(function() { + this.backend.use('commit', function(context, next) { + context.channels.push('channel-1'); + next(); + }); + this.backend.use('query', function(context, next) { + context.channel = 'channel-1'; + next(); + }); + }); + commonTests(options); + }); + }); + }); +}; + +function commonCustomChannelsErrorHandlingTests() { + it('should throw if not channels provided in query', function(done) { + this.backend.use('query', function(context, next) { + context.channels = null; + next(); + }); + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { + if (!err) return done('Should throw Required minimum one query channel error'); + expect(err.message).to.be.equal('Required minimum one query channel.'); + expect(err.code).to.be.equal(ShareDBError.CODES.ERR_QUERY_CHANNEL_MISSING); + done(); + }); + }); + }); + + it('should throw if channels provided in query is an empty array', function(done) { + this.backend.use('query', function(context, next) { + context.channels = []; + next(); + }); + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { + if (!err) return done('Should throw Required minimum one query channel error'); + expect(err.message).to.be.equal('Required minimum one query channel.'); + expect(err.code).to.be.equal(ShareDBError.CODES.ERR_QUERY_CHANNEL_MISSING); + done(); + }); + }); + }); +} + +function commonTests(options) { + var getQuery = options.getQuery; + + it('creating a document updates a subscribed query', function(done) { + var connection = this.backend.connect(); + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err) { + if (err) return done(err); + connection.get('dogs', 'fido').on('error', done).create({age: 3}); + }); + query.on('error', done); + query.on('insert', function(docs, index) { + expect(util.pluck(docs, 'id')).eql(['fido']); + expect(util.pluck(docs, 'data')).eql([{age: 3}]); + expect(index).equal(0); + expect(util.pluck(query.results, 'id')).eql(['fido']); + expect(util.pluck(query.results, 'data')).eql([{age: 3}]); + done(); + }); + }); + + it('creating an additional document updates a subscribed query', function(done) { + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - connection.get('dogs', 'fido').on('error', done).create({age: 3}); + connection.get('dogs', 'taco').on('error', done).create({age: 2}); }); query.on('error', done); query.on('insert', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['fido']); - expect(util.pluck(docs, 'data')).eql([{age: 3}]); - expect(index).equal(0); - expect(util.pluck(query.results, 'id')).eql(['fido']); - expect(util.pluck(query.results, 'data')).eql([{age: 3}]); + expect(util.pluck(docs, 'id')).eql(['taco']); + expect(util.pluck(docs, 'data')).eql([{age: 2}]); + expect(query.results[index]).equal(docs[0]); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['fido', 'spot', 'taco']); + expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}, {age: 2}]); done(); }); }); + }); - it('creating an additional document updates a subscribed query', function(done) { - var connection = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('deleting a document updates a subscribed query', function(done) { + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { - if (err) return done(err); - connection.get('dogs', 'taco').on('error', done).create({age: 2}); - }); - query.on('error', done); - query.on('insert', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['taco']); - expect(util.pluck(docs, 'data')).eql([{age: 2}]); - expect(query.results[index]).equal(docs[0]); - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['fido', 'spot', 'taco']); - expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}, {age: 2}]); - done(); - }); + connection.get('dogs', 'fido').del(); + }); + query.on('error', done); + query.on('remove', function(docs, index) { + expect(util.pluck(docs, 'id')).eql(['fido']); + expect(util.pluck(docs, 'data')).eql([undefined]); + expect(index).a('number'); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['spot']); + expect(util.pluck(results, 'data')).eql([{age: 5}]); + done(); }); }); + }); - it('deleting a document updates a subscribed query', function(done) { - var connection = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('subscribed query removes document from results before sending delete op to other clients', function(done) { + var connection1 = this.backend.connect(); + var connection2 = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection1.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection1.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection2.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { - if (err) return done(err); - connection.get('dogs', 'fido').del(); - }); - query.on('error', done); - query.on('remove', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['fido']); - expect(util.pluck(docs, 'data')).eql([undefined]); - expect(index).a('number'); - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['spot']); - expect(util.pluck(results, 'data')).eql([{age: 5}]); - done(); - }); + connection1.get('dogs', 'fido').del(); + }); + query.on('error', done); + var removed = false; + connection2.get('dogs', 'fido').on('del', function() { + expect(removed).equal(true); + done(); + }); + query.on('remove', function(docs, index) { + removed = true; + expect(util.pluck(docs, 'id')).eql(['fido']); + expect(util.pluck(docs, 'data')).eql([{age: 3}]); + expect(index).a('number'); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['spot']); + expect(util.pluck(results, 'data')).eql([{age: 5}]); }); }); + }); - it('subscribed query removes document from results before sending delete op to other clients', function(done) { - var connection1 = this.backend.connect(); - var connection2 = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection1.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection1.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('subscribed query does not get updated after destroyed', function(done) { + var connection = this.backend.connect(); + var connection2 = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection2.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { + query.destroy(function(err) { if (err) return done(err); - connection1.get('dogs', 'fido').del(); - }); - query.on('error', done); - var removed = false; - connection2.get('dogs', 'fido').on('del', function() { - expect(removed).equal(true); - done(); - }); - query.on('remove', function(docs, index) { - removed = true; - expect(util.pluck(docs, 'id')).eql(['fido']); - expect(util.pluck(docs, 'data')).eql([{age: 3}]); - expect(index).a('number'); - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['spot']); - expect(util.pluck(results, 'data')).eql([{age: 5}]); + connection2.get('dogs', 'taco').on('error', done).create({age: 2}, done); }); }); + query.on('error', done); + query.on('insert', function() { + done(); + }); }); + }); - it('subscribed query does not get updated after destroyed', function(done) { - var connection = this.backend.connect(); - var connection2 = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('subscribed query does not get updated after connection is disconnected', function(done) { + var connection = this.backend.connect(); + var connection2 = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { - if (err) return done(err); - query.destroy(function(err) { - if (err) return done(err); - connection2.get('dogs', 'taco').on('error', done).create({age: 2}, done); - }); - }); - query.on('error', done); - query.on('insert', function() { - done(); - }); + connection.close(); + connection2.get('dogs', 'taco').on('error', done).create({age: 2}, done); + }); + query.on('error', done); + query.on('insert', function() { + done(); }); }); + }); - it('subscribed query does not get updated after connection is disconnected', function(done) { - var connection = this.backend.connect(); - var connection2 = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('subscribed query gets update after reconnecting', function(done) { + var backend = this.backend; + var connection = backend.connect(); + var connection2 = backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + + var wait = 2; + function finish() { + if (--wait) return; + expect(util.pluck(query.results, 'id')).to.have.members(['fido', 'spot', 'taco']); + done(); + } + + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { - if (err) return done(err); - connection.close(); - connection2.get('dogs', 'taco').on('error', done).create({age: 2}, done); - }); - query.on('error', done); - query.on('insert', function() { - done(); + connection.close(); + connection2.get('dogs', 'taco').on('error', done).create({age: 2}); + process.nextTick(function() { + backend.connect(connection); + query.on('ready', finish); }); }); + query.on('error', done); + query.on('insert', finish); }); + }); - it('subscribed query gets update after reconnecting', function(done) { - var backend = this.backend; - var connection = backend.connect(); - var connection2 = backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { - if (err) return done(err); + it('subscribed query gets simultaneous insert and remove after reconnecting', function(done) { + var backend = this.backend; + var connection = backend.connect(); + var connection2 = backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery); + query.on('error', done); - var wait = 2; - function finish() { - if (--wait) return; - expect(util.pluck(query.results, 'id')).to.have.members(['fido', 'spot', 'taco']); - done(); - } + query.once('ready', function() { + connection.close(); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { + connection2.get('dogs', 'fido').fetch(function(err) { if (err) return done(err); - connection.close(); - connection2.get('dogs', 'taco').on('error', done).create({age: 2}); - process.nextTick(function() { + async.parallel([ + function(cb) { + connection2.get('dogs', 'fido').del(cb); + }, + function(cb) { + connection2.get('dogs', 'taco').create({age: 2}, cb); + } + ], function(error) { + if (error) return done(error); backend.connect(connection); - query.on('ready', finish); - }); - }); - query.on('error', done); - query.on('insert', finish); - }); - }); - - it('subscribed query gets simultaneous insert and remove after reconnecting', function(done) { - var backend = this.backend; - var connection = backend.connect(); - var connection2 = backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { - if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery); - query.on('error', done); - - query.once('ready', function() { - connection.close(); - - connection2.get('dogs', 'fido').fetch(function(err) { - if (err) return done(err); - async.parallel([ - function(cb) { - connection2.get('dogs', 'fido').del(cb); - }, - function(cb) { - connection2.get('dogs', 'taco').create({age: 2}, cb); - } - ], function(error) { - if (error) return done(error); - backend.connect(connection); - query.once('ready', function() { - finish(); - }); + query.once('ready', function() { + finish(); }); }); }); + }); - var wait = 3; - function finish() { - if (--wait) return; - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['spot', 'taco']); - expect(util.pluck(results, 'data')).eql([{age: 5}, {age: 2}]); - done(); - } - query.once('insert', function(docs) { - expect(util.pluck(docs, 'id')).eql(['taco']); - expect(util.pluck(docs, 'data')).eql([{age: 2}]); - finish(); - }); - query.once('remove', function(docs) { - expect(util.pluck(docs, 'id')).eql(['fido']); - // We don't assert the value of data, because the del op could be - // applied by the client before or after the query result is removed. - // Order of ops & query result updates is not currently guaranteed - finish(); - }); + var wait = 3; + function finish() { + if (--wait) return; + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['spot', 'taco']); + expect(util.pluck(results, 'data')).eql([{age: 5}, {age: 2}]); + done(); + } + query.once('insert', function(docs) { + expect(util.pluck(docs, 'id')).eql(['taco']); + expect(util.pluck(docs, 'data')).eql([{age: 2}]); + finish(); + }); + query.once('remove', function(docs) { + expect(util.pluck(docs, 'id')).eql(['fido']); + // We don't assert the value of data, because the del op could be + // applied by the client before or after the query result is removed. + // Order of ops & query result updates is not currently guaranteed + finish(); }); }); + }); - it('creating an additional document updates a subscribed query', function(done) { - var connection = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('creating an additional document updates a subscribed query', function(done) { + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); - var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { - if (err) return done(err); - connection.get('dogs', 'taco').on('error', done).create({age: 2}); - }); - query.on('error', done); - query.on('insert', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['taco']); - expect(util.pluck(docs, 'data')).eql([{age: 2}]); - expect(query.results[index]).equal(docs[0]); - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['fido', 'spot', 'taco']); - expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}, {age: 2}]); - done(); - }); + connection.get('dogs', 'taco').on('error', done).create({age: 2}); }); - }); - - it('pollDebounce option reduces subsequent poll interval', function(done) { - var connection = this.backend.connect(); - this.backend.db.canPollDoc = function() { - return false; - }; - var query = connection.createSubscribeQuery('items', this.matchAllDbQuery, {pollDebounce: 1000}); query.on('error', done); - var batchSizes = []; - var total = 0; + query.on('insert', function(docs, index) { + expect(util.pluck(docs, 'id')).eql(['taco']); + expect(util.pluck(docs, 'data')).eql([{age: 2}]); + expect(query.results[index]).equal(docs[0]); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['fido', 'spot', 'taco']); + expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}, {age: 2}]); + done(); + }); + }); + }); - query.on('insert', function(docs) { - batchSizes.push(docs.length); - total += docs.length; - if (total === 1) { + it('pollDebounce option reduces subsequent poll interval', function(done) { + var clock = sinon.useFakeTimers(); + var connection = this.backend.connect(); + this.backend.db.canPollDoc = function() { + return false; + }; + var query = connection.createSubscribeQuery('items', this.matchAllDbQuery, {pollDebounce: 2000}); + query.on('error', done); + var batchSizes = []; + var total = 0; + + query.on('insert', function(docs) { + batchSizes.push(docs.length); + total += docs.length; + + if (total === 1) { // first write received by client. we're debouncing. create 9 // more documents. - for (var i = 1; i < 10; i++) { - connection.get('items', i.toString()).on('error', done).create({}); - } + var counter = 0; + for (var i = 1; i < 10; i++) { + connection.get('items', i.toString()).on('error', done).create({}, function(err) { + if (err) return done(err); + counter++; + if (counter === 9) clock.tickAsync(10000); + }); } - if (total === 10) { + } + if (total === 10) { // first document is its own batch; then subsequent creates - // are debounced until after all other 9 docs are created - expect(batchSizes).eql([1, 9]); - done(); - } - }); - - // create an initial document. this will lead to the 'insert' - // event firing the first time, while sharedb is definitely - // debouncing - connection.get('items', '0').on('error', done).create({}); + // are debounced and batched + expect(batchSizes[0]).eql(1); + batchSizes.shift(); + var sum = batchSizes.reduce(function(sum, batchSize) { + return sum + batchSize; + }, 0); + expect(batchSizes.length).to.lessThan(9); + expect(sum).eql(9); + done(); + } }); - it('db.pollDebounce option reduces subsequent poll interval', function(done) { - var connection = this.backend.connect(); - this.backend.db.canPollDoc = function() { - return false; - }; - this.backend.db.pollDebounce = 1000; - var query = connection.createSubscribeQuery('items', this.matchAllDbQuery); - query.on('error', done); - var batchSizes = []; - var total = 0; + // create an initial document. this will lead to the 'insert' + // event firing the first time, while sharedb is definitely + // debouncing + connection.get('items', '0').on('error', done).create({}, function() { + clock.tickAsync(3000); + }); + }); - query.on('insert', function(docs) { - batchSizes.push(docs.length); - total += docs.length; - if (total === 1) { + it('db.pollDebounce option reduces subsequent poll interval', function(done) { + var clock = sinon.useFakeTimers(); + var connection = this.backend.connect(); + this.backend.db.canPollDoc = function() { + return false; + }; + this.backend.db.pollDebounce = 2000; + var query = connection.createSubscribeQuery('items', this.matchAllDbQuery); + query.on('error', done); + var batchSizes = []; + var total = 0; + + query.on('insert', function(docs) { + batchSizes.push(docs.length); + total += docs.length; + + if (total === 1) { // first write received by client. we're debouncing. create 9 // more documents. - for (var i = 1; i < 10; i++) { - connection.get('items', i.toString()).on('error', done).create({}); - } + var counter = 0; + for (var i = 1; i < 10; i++) { + connection.get('items', i.toString()).on('error', done).create({}, function(err) { + if (err) return done(err); + counter++; + if (counter === 9) clock.tickAsync(10000); + }); } - if (total === 10) { + } + if (total === 10) { // first document is its own batch; then subsequent creates - // are debounced until after all other 9 docs are created - expect(batchSizes).eql([1, 9]); - done(); - } - }); + // are debounced and batched + expect(batchSizes[0]).eql(1); + batchSizes.shift(); + var sum = batchSizes.reduce(function(sum, batchSize) { + return sum + batchSize; + }, 0); + expect(batchSizes.length).to.lessThan(9); + expect(sum).eql(9); + done(); + } + }); - // create an initial document. this will lead to the 'insert' - // event firing the first time, while sharedb is definitely - // debouncing - connection.get('items', '0').on('error', done).create({}); + // create an initial document. this will lead to the 'insert' + // event firing the first time, while sharedb is definitely + // debouncing + connection.get('items', '0').on('error', done).create({}, function() { + clock.tickAsync(3000); }); + }); - it('pollInterval updates a subscribed query after an unpublished create', function(done) { - var connection = this.backend.connect(); - this.backend.suppressPublish = true; - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, {pollInterval: 50}, function(err) { + it('pollInterval updates a subscribed query after an unpublished create', function(done) { + var clock = sinon.useFakeTimers(); + var connection = this.backend.connect(); + this.backend.suppressPublish = true; + var query = connection.createSubscribeQuery( + 'dogs', + this.matchAllDbQuery, + {pollDebounce: 0, pollInterval: 50}, + function(err) { if (err) return done(err); - connection.get('dogs', 'fido').on('error', done).create({}); - }); - query.on('error', done); - query.on('insert', function(docs) { - expect(util.pluck(docs, 'id')).eql(['fido']); - done(); + connection.get('dogs', 'fido').on('error', done).create({}, function() { + clock.tickAsync(51); + }); + } + ); + query.on('error', done); + query.on('insert', function(docs) { + expect(util.pluck(docs, 'id')).eql(['fido']); + done(); + }); + }); + + it('db.pollInterval updates a subscribed query after an unpublished create', function(done) { + var clock = sinon.useFakeTimers(); + var connection = this.backend.connect(); + this.backend.suppressPublish = true; + this.backend.db.pollDebounce = 0; + this.backend.db.pollInterval = 50; + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err) { + if (err) return done(err); + connection.get('dogs', 'fido').on('error', done).create({}, function() { + clock.tickAsync(51); }); }); + query.on('error', done); + query.on('insert', function(docs) { + expect(util.pluck(docs, 'id')).eql(['fido']); + done(); + }); + }); - it('db.pollInterval updates a subscribed query after an unpublished create', function(done) { - var connection = this.backend.connect(); - this.backend.suppressPublish = true; - this.backend.db.pollInterval = 50; - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err) { - if (err) return done(err); - connection.get('dogs', 'fido').on('error', done).create({}); + it('pollInterval captures additional unpublished creates', function(done) { + var clock = sinon.useFakeTimers(); + var connection = this.backend.connect(); + this.backend.suppressPublish = true; + var count = 0; + + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, {pollInterval: 1000}, function(err) { + if (err) return done(err); + var doc = connection.get('dogs', count.toString()).on('error', done); + doc.create({}, function(e) { + if (e) return done(e); + clock.tickAsync(2000); }); - query.on('error', done); - query.on('insert', function(docs) { - expect(util.pluck(docs, 'id')).eql(['fido']); - done(); + }); + query.on('error', done); + query.on('insert', function() { + count++; + if (count === 3) return done(); + var doc = connection.get('dogs', count.toString()).on('error', done); + doc.create({}, function(e) { + if (e) return done(e); + clock.tickAsync(10000); }); }); + clock.tickAsync(1); + }); - it('pollInterval captures additional unpublished creates', function(done) { - var connection = this.backend.connect(); - this.backend.suppressPublish = true; - var count = 0; - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, {pollInterval: 50}, function(err) { - if (err) return done(err); - connection.get('dogs', count.toString()).on('error', done).create({}); + it('query extra is returned to client', function(done) { + var connection = this.backend.connect(); + this.backend.db.query = function(collection, query, fields, options, callback) { + process.nextTick(function() { + callback(null, [], {colors: ['brown', 'gold']}); }); - query.on('error', done); - query.on('insert', function() { - count++; - if (count === 3) return done(); - connection.get('dogs', count.toString()).on('error', done).create({}); + }; + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err, results, extra) { + if (err) return done(err); + expect(results).eql([]); + expect(extra).eql({colors: ['brown', 'gold']}); + expect(query.extra).eql({colors: ['brown', 'gold']}); + done(); + }); + query.on('error', done); + }); + + it('query extra is updated on change', function(done) { + var connection = this.backend.connect(); + this.backend.db.query = function(collection, query, fields, options, callback) { + process.nextTick(function() { + callback(null, [], 1); }); + }; + this.backend.db.queryPoll = function(collection, query, options, callback) { + process.nextTick(function() { + callback(null, [], 2); + }); + }; + this.backend.db.canPollDoc = function() { + return false; + }; + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err, results, extra) { + if (err) return done(err); + expect(extra).eql(1); + expect(query.extra).eql(1); + }); + query.on('error', done); + query.on('extra', function(extra) { + expect(extra).eql(2); + expect(query.extra).eql(2); + done(); }); + connection.get('dogs', 'fido').on('error', done).create({age: 3}); + }); - it('query extra is returned to client', function(done) { - var connection = this.backend.connect(); - this.backend.db.query = function(collection, query, fields, options, callback) { - process.nextTick(function() { - callback(null, [], {colors: ['brown', 'gold']}); - }); + describe('passing agent.custom to the DB adapter', function() { + var connection; + var expectedArg = { + agentCustom: {foo: 'bar'} + }; + beforeEach('set up', function() { + connection = this.backend.connect(); + connection.agent.custom = { + foo: 'bar' }; - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err, results, extra) { - if (err) return done(err); - expect(results).eql([]); - expect(extra).eql({colors: ['brown', 'gold']}); - expect(query.extra).eql({colors: ['brown', 'gold']}); + }); + + it('sends agentCustom to the db\'s getSnapshot call', function(done) { + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery); + var getSnapshotSpy = sinon.spy(this.backend.db, 'getSnapshot'); + + query.on('insert', function() { + // The first call to getSnapshot is when the document is created + // The seconds call is when the event is triggered, and is the one we are testing here + expect(getSnapshotSpy.callCount).to.equal(2); + expect(getSnapshotSpy.getCall(1).args[3]).to.deep.equal(expectedArg); done(); }); - query.on('error', done); + connection.get('dogs', 'fido').create({age: 3}); }); - it('query extra is updated on change', function(done) { - var connection = this.backend.connect(); - this.backend.db.query = function(collection, query, fields, options, callback) { - process.nextTick(function() { - callback(null, [], 1); - }); - }; - this.backend.db.queryPoll = function(collection, query, options, callback) { - process.nextTick(function() { - callback(null, [], 2); - }); - }; + it('sends agentCustom to the db\'s getSnapshotBulk call', function(done) { + // Ensures that getSnapshotBulk is called, instead of getSnapshot this.backend.db.canPollDoc = function() { return false; }; - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery, null, function(err, results, extra) { - if (err) return done(err); - expect(extra).eql(1); - expect(query.extra).eql(1); - }); - query.on('error', done); - query.on('extra', function(extra) { - expect(extra).eql(2); - expect(query.extra).eql(2); + + var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery); + var getSnapshotBulkSpy = sinon.spy(this.backend.db, 'getSnapshotBulk'); + + query.on('insert', function() { + expect(getSnapshotBulkSpy.callCount).to.equal(1); + expect(getSnapshotBulkSpy.getCall(0).args[3]).to.deep.equal(expectedArg); done(); }); - connection.get('dogs', 'fido').on('error', done).create({age: 3}); + connection.get('dogs', 'fido').create({age: 3}); }); + }); - describe('passing agent.custom to the DB adapter', function() { - var connection; - var expectedArg = { - agentCustom: {foo: 'bar'} - }; - beforeEach('set up', function() { - connection = this.backend.connect(); - connection.agent.custom = { - foo: 'bar' - }; - }); - - it('sends agentCustom to the db\'s getSnapshot call', function(done) { - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery); - var getSnapshotSpy = sinon.spy(this.backend.db, 'getSnapshot'); - - query.on('insert', function() { - // The first call to getSnapshot is when the document is created - // The seconds call is when the event is triggered, and is the one we are testing here - expect(getSnapshotSpy.callCount).to.equal(2); - expect(getSnapshotSpy.getCall(1).args[3]).to.deep.equal(expectedArg); - done(); - }); - connection.get('dogs', 'fido').create({age: 3}); + it('changing a filtered property removes from a subscribed query', function(done) { + var connection = this.backend.connect(); + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 3}, cb); + } + ], function(err) { + if (err) return done(err); + var dbQuery = getQuery({query: {age: 3}}); + var query = connection.createSubscribeQuery('dogs', dbQuery, null, function(err, results) { + if (err) return done(err); + var sorted = util.sortById(results); + expect(util.pluck(sorted, 'id')).eql(['fido', 'spot']); + expect(util.pluck(sorted, 'data')).eql([{age: 3}, {age: 3}]); + connection.get('dogs', 'fido').submitOp({p: ['age'], na: 2}); }); - - it('sends agentCustom to the db\'s getSnapshotBulk call', function(done) { - // Ensures that getSnapshotBulk is called, instead of getSnapshot - this.backend.db.canPollDoc = function() { - return false; - }; - - var query = connection.createSubscribeQuery('dogs', this.matchAllDbQuery); - var getSnapshotBulkSpy = sinon.spy(this.backend.db, 'getSnapshotBulk'); - - query.on('insert', function() { - expect(getSnapshotBulkSpy.callCount).to.equal(1); - expect(getSnapshotBulkSpy.getCall(0).args[3]).to.deep.equal(expectedArg); - done(); - }); - connection.get('dogs', 'fido').create({age: 3}); + query.on('error', done); + query.on('remove', function(docs, index) { + expect(util.pluck(docs, 'id')).eql(['fido']); + expect(util.pluck(docs, 'data')).eql([{age: 5}]); + expect(index).a('number'); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['spot']); + expect(util.pluck(results, 'data')).eql([{age: 3}]); + done(); }); }); + }); - it('changing a filtered property removes from a subscribed query', function(done) { - var connection = this.backend.connect(); - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 3}, cb); - } - ], function(err) { + it('changing a filtered property inserts to a subscribed query', function(done) { + var connection = this.backend.connect(); + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var dbQuery = getQuery({query: {age: 3}}); + var query = connection.createSubscribeQuery('dogs', dbQuery, null, function(err, results) { if (err) return done(err); - var dbQuery = getQuery({query: {age: 3}}); - var query = connection.createSubscribeQuery('dogs', dbQuery, null, function(err, results) { - if (err) return done(err); - var sorted = util.sortById(results); - expect(util.pluck(sorted, 'id')).eql(['fido', 'spot']); - expect(util.pluck(sorted, 'data')).eql([{age: 3}, {age: 3}]); - connection.get('dogs', 'fido').submitOp({p: ['age'], na: 2}); - }); - query.on('error', done); - query.on('remove', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['fido']); - expect(util.pluck(docs, 'data')).eql([{age: 5}]); - expect(index).a('number'); - var results = util.sortById(query.results); - expect(util.pluck(results, 'id')).eql(['spot']); - expect(util.pluck(results, 'data')).eql([{age: 3}]); - done(); - }); + var sorted = util.sortById(results); + expect(util.pluck(sorted, 'id')).eql(['fido']); + expect(util.pluck(sorted, 'data')).eql([{age: 3}]); + connection.get('dogs', 'spot').submitOp({p: ['age'], na: -2}); + }); + query.on('error', done); + query.on('insert', function(docs, index) { + expect(util.pluck(docs, 'id')).eql(['spot']); + expect(util.pluck(docs, 'data')).eql([{age: 3}]); + expect(index).a('number'); + var results = util.sortById(query.results); + expect(util.pluck(results, 'id')).eql(['fido', 'spot']); + expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 3}]); + done(); }); }); + }); - it('changing a filtered property inserts to a subscribed query', function(done) { - var connection = this.backend.connect(); - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { - if (err) return done(err); - var dbQuery = getQuery({query: {age: 3}}); - var query = connection.createSubscribeQuery('dogs', dbQuery, null, function(err, results) { + it('changing a sorted property moves in a subscribed query', function(done) { + var connection = this.backend.connect(); + + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var dbQuery = getQuery({query: {}, sort: [['age', 1]]}); + var query = connection.createSubscribeQuery( + 'dogs', + dbQuery, + null, + function(err, results) { if (err) return done(err); - var sorted = util.sortById(results); - expect(util.pluck(sorted, 'id')).eql(['fido']); - expect(util.pluck(sorted, 'data')).eql([{age: 3}]); - connection.get('dogs', 'spot').submitOp({p: ['age'], na: -2}); - }); - query.on('error', done); - query.on('insert', function(docs, index) { - expect(util.pluck(docs, 'id')).eql(['spot']); - expect(util.pluck(docs, 'data')).eql([{age: 3}]); - expect(index).a('number'); - var results = util.sortById(query.results); expect(util.pluck(results, 'id')).eql(['fido', 'spot']); - expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 3}]); - done(); + expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}]); + connection.get('dogs', 'spot').submitOp({p: ['age'], na: -3}); }); + query.on('error', done); + query.on('move', function(docs, from, to) { + expect(docs.length).eql(1); + expect(from).a('number'); + expect(to).a('number'); + expect(util.pluck(query.results, 'id')).eql(['spot', 'fido']); + expect(util.pluck(query.results, 'data')).eql([{age: 2}, {age: 3}]); + done(); }); }); + }); - it('changing a sorted property moves in a subscribed query', function(done) { - var connection = this.backend.connect(); - - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { - if (err) return done(err); - var dbQuery = getQuery({query: {}, sort: [['age', 1]]}); - var query = connection.createSubscribeQuery( - 'dogs', - dbQuery, - null, - function(err, results) { - if (err) return done(err); - expect(util.pluck(results, 'id')).eql(['fido', 'spot']); - expect(util.pluck(results, 'data')).eql([{age: 3}, {age: 5}]); - connection.get('dogs', 'spot').submitOp({p: ['age'], na: -3}); - }); - query.on('error', done); - query.on('move', function(docs, from, to) { - expect(docs.length).eql(1); - expect(from).a('number'); - expect(to).a('number'); - expect(util.pluck(query.results, 'id')).eql(['spot', 'fido']); - expect(util.pluck(query.results, 'data')).eql([{age: 2}, {age: 3}]); - done(); - }); - }); + it('returns pubSub error if fails to subscribe to channel', function(done) { + sinon.stub(this.backend.pubsub, 'subscribe').callsFake(function(_channel, callback) { + callback(new Error('TEST_ERROR')); }); + var connection = this.backend.connect(); + connection.createSubscribeQuery( + 'dogs', + this.matchAllDbQuery, + {pollInterval: 0, pollDebounce: 0}, + function(err) { + if (err) { + expect(err.message).to.be.equal('TEST_ERROR'); + return done(); + } else { + done('Should call callback with pubsub subscribe error'); + } + } + ); }); -}; +} diff --git a/test/logger.js b/test/logger.js index 72f75e909..193f6be5d 100644 --- a/test/logger.js +++ b/test/logger.js @@ -8,10 +8,6 @@ describe('Logger', function() { sinon.stub(console, 'warn'); }); - afterEach(function() { - sinon.restore(); - }); - it('logs to console by default', function() { var logger = new Logger(); logger.warn('warning'); diff --git a/test/setup.js b/test/setup.js index d8699ef3b..6c1c91f98 100644 --- a/test/setup.js +++ b/test/setup.js @@ -1,4 +1,5 @@ var logger = require('../lib/logger'); +var sinon = require('sinon'); if (process.env.LOGGING !== 'true') { // Silence the logger for tests by setting all its methods to no-ops @@ -8,3 +9,7 @@ if (process.env.LOGGING !== 'true') { error: function() {} }); } + +afterEach(function() { + sinon.restore(); +}); From c587181b356e5dcb8a93e022d0f1325255be4b41 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Mon, 6 Mar 2023 08:39:52 +0000 Subject: [PATCH 49/61] 3.3.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fe70b7b9a..8511ac267 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.2.4", + "version": "3.3.0", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From 0f4b21156ead6c07b8e9afa7c25020218aa6572a Mon Sep 17 00:00:00 2001 From: Curran Date: Wed, 8 Mar 2023 06:23:36 -0500 Subject: [PATCH 50/61] Got Vite example to work --- examples/counter-json1-vite/index.html | 2 +- examples/counter-json1-vite/main.js | 6 +++--- examples/counter-json1-vite/package.json | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/counter-json1-vite/index.html b/examples/counter-json1-vite/index.html index 8b3d96714..35ad9051a 100644 --- a/examples/counter-json1-vite/index.html +++ b/examples/counter-json1-vite/index.html @@ -7,7 +7,7 @@
You clicked times. - +
diff --git a/examples/counter-json1-vite/main.js b/examples/counter-json1-vite/main.js index ec2aada13..5c3450ac6 100644 --- a/examples/counter-json1-vite/main.js +++ b/examples/counter-json1-vite/main.js @@ -1,6 +1,6 @@ import ReconnectingWebSocket from 'reconnecting-websocket'; -import sharedb from 'sharedb/lib/client'; import json1 from 'ot-json1'; +import sharedb from 'sharedb-client-browser/dist/sharedb-client-umd.cjs'; // Open WebSocket connection to ShareDB server var socket = new ReconnectingWebSocket('ws://' + window.location.host); @@ -29,5 +29,5 @@ function increment() { doc.submitOp(['numClicks', {ena: 1}]); } -// Expose to index.html -global.increment = increment; +var button = document.querySelector('button.increment'); +button.addEventListener('click', increment); diff --git a/examples/counter-json1-vite/package.json b/examples/counter-json1-vite/package.json index d2f8a6c75..cadb5e3ec 100644 --- a/examples/counter-json1-vite/package.json +++ b/examples/counter-json1-vite/package.json @@ -15,6 +15,7 @@ "ot-json1": "^1.0.2", "reconnecting-websocket": "^4.4.0", "sharedb": "^3.2.4", + "sharedb-client-browser": "^4.2.0", "ws": "^8.12.1" }, "devDependencies": { From 265840d46ab6b0360cfd21c71d2a441fea52a3d0 Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 9 Mar 2023 00:46:01 -0500 Subject: [PATCH 51/61] Upgrade ShareDB in examples. Fix presence API warning. --- examples/counter-json1/package.json | 2 +- examples/counter/package.json | 2 +- examples/leaderboard/package.json | 2 +- examples/rich-text-presence/package.json | 2 +- examples/rich-text-presence/server.js | 5 ++++- examples/rich-text/package.json | 2 +- examples/textarea/package.json | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/counter-json1/package.json b/examples/counter-json1/package.json index bb7221ebb..7f9ce4fa0 100644 --- a/examples/counter-json1/package.json +++ b/examples/counter-json1/package.json @@ -20,7 +20,7 @@ "express": "^4.18.2", "ot-json1": "^1.0.2", "reconnecting-websocket": "^4.4.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "ws": "^8.12.1" }, "devDependencies": { diff --git a/examples/counter/package.json b/examples/counter/package.json index 953c991b5..30e49bca0 100644 --- a/examples/counter/package.json +++ b/examples/counter/package.json @@ -18,7 +18,7 @@ "@teamwork/websocket-json-stream": "^2.0.0", "express": "^4.18.2", "reconnecting-websocket": "^4.4.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "ws": "^8.12.1" }, "devDependencies": { diff --git a/examples/leaderboard/package.json b/examples/leaderboard/package.json index dd74e0393..263ac13d2 100644 --- a/examples/leaderboard/package.json +++ b/examples/leaderboard/package.json @@ -22,7 +22,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "reconnecting-websocket": "^4.4.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "sharedb-mingo-memory": "^2.1.2", "underscore": "^1.13.6", "ws": "^8.12.1" diff --git a/examples/rich-text-presence/package.json b/examples/rich-text-presence/package.json index 69b87ff33..d06e38021 100644 --- a/examples/rich-text-presence/package.json +++ b/examples/rich-text-presence/package.json @@ -22,7 +22,7 @@ "quill-cursors": "^4.0.2", "reconnecting-websocket": "^4.4.0", "rich-text": "^4.1.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "tinycolor2": "^1.6.0", "ws": "^8.12.1" }, diff --git a/examples/rich-text-presence/server.js b/examples/rich-text-presence/server.js index 91a7deea7..01be2e1f6 100644 --- a/examples/rich-text-presence/server.js +++ b/examples/rich-text-presence/server.js @@ -6,7 +6,10 @@ var WebSocket = require('ws'); var WebSocketJSONStream = require('@teamwork/websocket-json-stream'); ShareDB.types.register(richText.type); -var backend = new ShareDB({presence: true}); +var backend = new ShareDB({ + presence: true, + doNotForwardSendPresenceErrorsToClient: true +}); createDoc(startServer); // Create initial document then fire callback diff --git a/examples/rich-text/package.json b/examples/rich-text/package.json index 9504f8216..fe49a5ae5 100644 --- a/examples/rich-text/package.json +++ b/examples/rich-text/package.json @@ -19,7 +19,7 @@ "quill": "^1.3.7", "reconnecting-websocket": "^4.4.0", "rich-text": "^4.1.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "ws": "^8.12.1" }, "devDependencies": { diff --git a/examples/textarea/package.json b/examples/textarea/package.json index 6364d6c5e..b1a57caf6 100644 --- a/examples/textarea/package.json +++ b/examples/textarea/package.json @@ -17,7 +17,7 @@ "@teamwork/websocket-json-stream": "^2.0.0", "express": "^4.18.2", "reconnecting-websocket": "^4.4.0", - "sharedb": "^3.2.4", + "sharedb": "^3.3.0", "sharedb-string-binding": "^1.0.0", "ws": "^8.12.1" }, From f783dec63d8ab3cc837723c488b1880e4e49e5c7 Mon Sep 17 00:00:00 2001 From: Curran Date: Thu, 9 Mar 2023 01:09:39 -0500 Subject: [PATCH 52/61] Fix React warnings --- examples/leaderboard/client/index.jsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/leaderboard/client/index.jsx b/examples/leaderboard/client/index.jsx index 660d94923..2d65272e4 100644 --- a/examples/leaderboard/client/index.jsx +++ b/examples/leaderboard/client/index.jsx @@ -1,5 +1,7 @@ var Body = require('./Body.jsx'); var React = require('react'); -var ReactDOM = require('react-dom'); +var ReactDOM = require('react-dom/client'); -ReactDOM.render(, document.querySelector('#main')); +var container = document.getElementById('main'); +var root = ReactDOM.createRoot(container); +root.render(); From a7c7697b1b2d1dec2b1e04e0be722b7c311215ee Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 28 Mar 2023 17:39:28 +0100 Subject: [PATCH 53/61] =?UTF-8?q?=E2=9C=85=20Try=20to=20fix=20flaky=20Mile?= =?UTF-8?q?stone=20DB=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment we sometimes get some flaky Milestone Mongo test [failures][1]. This potentially happens because milestone snapshots are saved as a [fire-and-forget request][2], which means that we technically make no guarantee about the order that snapshots are written to the database. Since this is the case, the affected tests may write the v4 snapshot before the others, and then fail assertions about earlier snapshots. This change tweaks these tests to count the snapshots instead, which should be more robust to this race condition. [1]: https://github.com/share/sharedb-milestone-mongo/actions/runs/4545270938/jobs/8012439274 [2]: https://github.com/share/sharedb/blob/404cde5568bf88d8de12186a90ba6bc168920e23/lib/submit-request.js#L223 --- test/milestone-db.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/milestone-db.js b/test/milestone-db.js index 97190f50e..dc0acd107 100644 --- a/test/milestone-db.js +++ b/test/milestone-db.js @@ -576,8 +576,10 @@ module.exports = function(options) { }); it('only stores even-numbered versions', function(done) { - db.on('save', function(collection, snapshot) { - if (snapshot.v !== 4) return; + var snapshotCount = 0; + db.on('save', function() { + snapshotCount++; + if (snapshotCount < 2) return; async.waterfall([ db.getMilestoneSnapshot.bind(db, 'books', 'catcher-in-the-rye', 1), @@ -619,8 +621,10 @@ module.exports = function(options) { callback(); }); - db.on('save', function(collection, snapshot) { - if (snapshot.v !== 4) return; + var snapshotCount = 0; + db.on('save', function() { + snapshotCount++; + if (snapshotCount < 2) return; async.waterfall([ db.getMilestoneSnapshot.bind(db, 'books', 'catcher-in-the-rye', 1), From d21c26a8d6e59c50f575539b26a38d6176b294e4 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 28 Mar 2023 17:45:12 +0100 Subject: [PATCH 54/61] 3.3.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8511ac267..64e32010b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharedb", - "version": "3.3.0", + "version": "3.3.1", "description": "JSON OT database backend", "main": "lib/index.js", "dependencies": { From ff5d5152fea4ed3ecc885bc728b4811093c307d0 Mon Sep 17 00:00:00 2001 From: Curran Date: Wed, 29 Mar 2023 01:38:43 -0400 Subject: [PATCH 55/61] Use glob override for .eslintrc --- .eslintrc.js | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index cf96acb8d..eb5ad9671 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -34,18 +34,13 @@ var SHAREDB_RULES = { // as-needed quote props are easier to write 'quote-props': ['error', 'as-needed'], 'require-jsdoc': 'off', - 'valid-jsdoc': 'off', - - // Required after upgrade to ecmaVersion: 6 - 'no-invalid-this': 'off' + 'valid-jsdoc': 'off' }; module.exports = { extends: 'google', parserOptions: { - // Support ES6 imports and exports - ecmaVersion: 6, - sourceType: 'module' + ecmaVersion: 3 }, rules: Object.assign( {}, @@ -54,5 +49,17 @@ module.exports = { ), ignorePatterns: [ '/docs/' + ], + overrides: [ + { + files: ['examples/counter-json1-vite/*.js'], + parserOptions: { + ecmaVersion: 6, + sourceType: 'module' + }, + rules: { + quotes: ['error', 'single'] + } + } ] }; From c637650af4168de91e9cd64e7038f7f3f554117d Mon Sep 17 00:00:00 2001 From: Curran Date: Wed, 29 Mar 2023 01:58:04 -0400 Subject: [PATCH 56/61] Make it work --- examples/counter-json1-vite/main.js | 4 ++-- examples/counter-json1-vite/package.json | 8 ++++---- examples/counter-json1-vite/server.js | 2 +- examples/counter-json1-vite/vite.config.js | 13 +++++++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 examples/counter-json1-vite/vite.config.js diff --git a/examples/counter-json1-vite/main.js b/examples/counter-json1-vite/main.js index 5c3450ac6..d742a0658 100644 --- a/examples/counter-json1-vite/main.js +++ b/examples/counter-json1-vite/main.js @@ -1,9 +1,9 @@ import ReconnectingWebSocket from 'reconnecting-websocket'; -import json1 from 'ot-json1'; +import { json1 } from 'sharedb-client-browser/dist/ot-json1-umd.cjs'; import sharedb from 'sharedb-client-browser/dist/sharedb-client-umd.cjs'; // Open WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host + '/ws'); sharedb.types.register(json1.type); var connection = new sharedb.Connection(socket); diff --git a/examples/counter-json1-vite/package.json b/examples/counter-json1-vite/package.json index cadb5e3ec..534780718 100644 --- a/examples/counter-json1-vite/package.json +++ b/examples/counter-json1-vite/package.json @@ -14,11 +14,11 @@ "express": "^4.18.2", "ot-json1": "^1.0.2", "reconnecting-websocket": "^4.4.0", - "sharedb": "^3.2.4", - "sharedb-client-browser": "^4.2.0", - "ws": "^8.12.1" + "sharedb": "^3.3.1", + "sharedb-client-browser": "^4.2.1", + "ws": "^8.13.0" }, "devDependencies": { - "vite": "^4.1.4" + "vite": "^4.2.1" } } diff --git a/examples/counter-json1-vite/server.js b/examples/counter-json1-vite/server.js index 68e628d2e..966927467 100644 --- a/examples/counter-json1-vite/server.js +++ b/examples/counter-json1-vite/server.js @@ -30,7 +30,7 @@ function startServer() { var server = http.createServer(app); // Connect any incoming WebSocket connection to ShareDB - var wss = new WebSocketServer({server: server}); + var wss = new WebSocketServer({server: server, path:'/ws'}); wss.on('connection', function(ws) { var stream = new WebSocketJSONStream(ws); backend.listen(stream); diff --git a/examples/counter-json1-vite/vite.config.js b/examples/counter-json1-vite/vite.config.js new file mode 100644 index 000000000..487d0009c --- /dev/null +++ b/examples/counter-json1-vite/vite.config.js @@ -0,0 +1,13 @@ +import { defineConfig } from 'vite'; + +export default defineConfig({ + server: { + proxy: { + // Proxy websockets to ws://localhost:8080 for `npm run dev` + '/ws': { + target: 'ws://localhost:8080', + ws: true, + }, + }, + }, +}) From c219205cee29cab3a3cfa6b6f220204b617634b5 Mon Sep 17 00:00:00 2001 From: Curran Date: Wed, 29 Mar 2023 02:03:20 -0400 Subject: [PATCH 57/61] Lint --- examples/counter-json1-vite/main.js | 2 +- examples/counter-json1-vite/server.js | 2 +- examples/counter-json1-vite/vite.config.js | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/counter-json1-vite/main.js b/examples/counter-json1-vite/main.js index d742a0658..8fd401aab 100644 --- a/examples/counter-json1-vite/main.js +++ b/examples/counter-json1-vite/main.js @@ -1,5 +1,5 @@ import ReconnectingWebSocket from 'reconnecting-websocket'; -import { json1 } from 'sharedb-client-browser/dist/ot-json1-umd.cjs'; +import {json1} from 'sharedb-client-browser/dist/ot-json1-umd.cjs'; import sharedb from 'sharedb-client-browser/dist/sharedb-client-umd.cjs'; // Open WebSocket connection to ShareDB server diff --git a/examples/counter-json1-vite/server.js b/examples/counter-json1-vite/server.js index 966927467..b00bfa524 100644 --- a/examples/counter-json1-vite/server.js +++ b/examples/counter-json1-vite/server.js @@ -30,7 +30,7 @@ function startServer() { var server = http.createServer(app); // Connect any incoming WebSocket connection to ShareDB - var wss = new WebSocketServer({server: server, path:'/ws'}); + var wss = new WebSocketServer({server: server, path: '/ws'}); wss.on('connection', function(ws) { var stream = new WebSocketJSONStream(ws); backend.listen(stream); diff --git a/examples/counter-json1-vite/vite.config.js b/examples/counter-json1-vite/vite.config.js index 487d0009c..f124d1bf5 100644 --- a/examples/counter-json1-vite/vite.config.js +++ b/examples/counter-json1-vite/vite.config.js @@ -1,4 +1,4 @@ -import { defineConfig } from 'vite'; +import {defineConfig} from 'vite'; export default defineConfig({ server: { @@ -6,8 +6,8 @@ export default defineConfig({ // Proxy websockets to ws://localhost:8080 for `npm run dev` '/ws': { target: 'ws://localhost:8080', - ws: true, - }, - }, - }, -}) + ws: true + } + } + } +}); From 27dcfe3188a6fcaf0c1c757eaffc7807c2837c0b Mon Sep 17 00:00:00 2001 From: Zeus Lalkaka Date: Thu, 13 Apr 2023 19:21:14 -0400 Subject: [PATCH 58/61] Fix bad merge in test files --- test/client/query-subscribe.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/client/query-subscribe.js b/test/client/query-subscribe.js index cc21ecdb7..30afa9d08 100644 --- a/test/client/query-subscribe.js +++ b/test/client/query-subscribe.js @@ -504,17 +504,19 @@ function commonTests(options) { }); }); - it('creating an additional document updates a subscribed query', function(done) { - var connection = this.backend.connect(); - var matchAllDbQuery = this.matchAllDbQuery; - async.parallel([ - function(cb) { - connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); - }, - function(cb) { - connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); - } - ], function(err) { + it('creating an additional document updates a subscribed query', function(done) { + var connection = this.backend.connect(); + var matchAllDbQuery = this.matchAllDbQuery; + async.parallel([ + function(cb) { + connection.get('dogs', 'fido').on('error', done).create({age: 3}, cb); + }, + function(cb) { + connection.get('dogs', 'spot').on('error', done).create({age: 5}, cb); + } + ], function(err) { + if (err) return done(err); + var query = connection.createSubscribeQuery('dogs', matchAllDbQuery, null, function(err) { if (err) return done(err); connection.get('dogs', 'taco').on('error', done).create({age: 2}); }); @@ -530,6 +532,7 @@ function commonTests(options) { done(); }); }); + }); it('pollDebounce option reduces subsequent poll interval', function(done) { var clock = sinon.useFakeTimers(); From ac490e071ed9ca060543258bb20181f72dc03c42 Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Fri, 21 Apr 2023 08:27:08 +0100 Subject: [PATCH 59/61] =?UTF-8?q?=F0=9F=92=A5=20Drop=20Node.js=2014=20and?= =?UTF-8?q?=20add=20Node.js=2020?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to the Node.js [release schedule][1]: - v14 will be end-of-lifed on 30 April - v20 has been released This change drops v14 from our test matrix, and adds v20. [1]: https://nodejs.dev/en/about/releases/ --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1eec7e7f..6cdaae92b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,9 +15,9 @@ jobs: strategy: matrix: node: - - 14 - 16 - 18 + - 20 services: mongodb: image: mongo:4.4 From 3e1e7f0d521627feb0de1f68ec98268fd3a3b35d Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 2 May 2023 15:33:11 +0100 Subject: [PATCH 60/61] =?UTF-8?q?=F0=9F=93=9D=20Update=20docs=20around=20`?= =?UTF-8?q?reconnecting-websocket`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In our docs, we recommend using [`reconnecting-websocket`][1] to handle websocket reconnection. However, [by default][2] that library will [buffer][3] messages when its underlying socket is closed, which can lead to [undefined behaviour][4] when ShareDB reconnects, since it works under the assumption that all messages sent as the socket is closing have been lost (eg pushing inflight ops back onto the pending queue, etc.). This change updates our documentation and our examples to set `{maxEnqueuedMessages: 0}`, which disables this buffering, and should help to avoid ShareDB reaching undefined states when reconnecting using this library. [1]: https://www.npmjs.com/package/reconnecting-websocket [2]: https://github.com/pladaria/reconnecting-websocket/blob/05a2f7cb0e31f15dff5ff35ad53d07b1bec5e197/reconnecting-websocket.ts#L46 [3]: https://github.com/pladaria/reconnecting-websocket/blob/05a2f7cb0e31f15dff5ff35ad53d07b1bec5e197/reconnecting-websocket.ts#L260 [4]: https://github.com/share/sharedb/issues/605 --- docs/getting-started.md | 6 +++++- examples/counter-json1-vite/main.js | 6 +++++- examples/counter-json1/client.js | 6 +++++- examples/counter/client.js | 6 +++++- examples/leaderboard/client/connection.js | 6 +++++- examples/rich-text-presence/client.js | 6 +++++- examples/rich-text/client.js | 12 ++++++++++-- examples/textarea/client.js | 6 +++++- 8 files changed, 45 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 6a9c31d37..4e48c3b4c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -71,7 +71,11 @@ Try running the [working example](https://github.com/share/sharedb/tree/master/e var ReconnectingWebSocket = require('reconnecting-websocket') var Connection = require('sharedb/lib/client').Connection -var socket = new ReconnectingWebSocket('ws://localhost:8080') +var socket = new ReconnectingWebSocket('ws://localhost:8080', [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}) var connection = new Connection(socket) var doc = connection.get('doc-collection', 'doc-id') diff --git a/examples/counter-json1-vite/main.js b/examples/counter-json1-vite/main.js index 8fd401aab..dc15e5bc7 100644 --- a/examples/counter-json1-vite/main.js +++ b/examples/counter-json1-vite/main.js @@ -3,7 +3,11 @@ import {json1} from 'sharedb-client-browser/dist/ot-json1-umd.cjs'; import sharedb from 'sharedb-client-browser/dist/sharedb-client-umd.cjs'; // Open WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host + '/ws'); +var socket = new ReconnectingWebSocket('ws://' + window.location.host + '/ws', [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); sharedb.types.register(json1.type); var connection = new sharedb.Connection(socket); diff --git a/examples/counter-json1/client.js b/examples/counter-json1/client.js index 3ceac3f1a..00d908981 100644 --- a/examples/counter-json1/client.js +++ b/examples/counter-json1/client.js @@ -3,7 +3,11 @@ var sharedb = require('sharedb/lib/client'); var json1 = require('ot-json1'); // Open WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); sharedb.types.register(json1.type); var connection = new sharedb.Connection(socket); diff --git a/examples/counter/client.js b/examples/counter/client.js index 1067d2a44..913bdbdf9 100644 --- a/examples/counter/client.js +++ b/examples/counter/client.js @@ -2,7 +2,11 @@ var ReconnectingWebSocket = require('reconnecting-websocket'); var sharedb = require('sharedb/lib/client'); // Open WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); var connection = new sharedb.Connection(socket); // Create local Doc instance mapped to 'examples' collection document with id 'counter' diff --git a/examples/leaderboard/client/connection.js b/examples/leaderboard/client/connection.js index a90010504..636be0ad8 100644 --- a/examples/leaderboard/client/connection.js +++ b/examples/leaderboard/client/connection.js @@ -2,6 +2,10 @@ var ReconnectingWebSocket = require('reconnecting-websocket'); var sharedb = require('sharedb/lib/client'); // Expose a singleton WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); var connection = new sharedb.Connection(socket); module.exports = connection; diff --git a/examples/rich-text-presence/client.js b/examples/rich-text-presence/client.js index e10efdcc2..efe354423 100644 --- a/examples/rich-text-presence/client.js +++ b/examples/rich-text-presence/client.js @@ -22,7 +22,11 @@ var collection = 'examples'; var id = 'richtext'; var presenceId = new ObjectID().toString(); -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); var connection = new sharedb.Connection(socket); var doc = connection.get(collection, id); diff --git a/examples/rich-text/client.js b/examples/rich-text/client.js index 5785357dd..3e989306d 100644 --- a/examples/rich-text/client.js +++ b/examples/rich-text/client.js @@ -5,7 +5,11 @@ var Quill = require('quill'); sharedb.types.register(richText.type); // Open WebSocket connection to ShareDB server -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); var connection = new sharedb.Connection(socket); // For testing reconnection @@ -13,7 +17,11 @@ window.disconnect = function() { connection.close(); }; window.connect = function() { - var socket = new ReconnectingWebSocket('ws://' + window.location.host); + var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 + }); connection.bindToSocket(socket); }; diff --git a/examples/textarea/client.js b/examples/textarea/client.js index 20b2ffcc8..d8e39f616 100644 --- a/examples/textarea/client.js +++ b/examples/textarea/client.js @@ -3,7 +3,11 @@ var StringBinding = require('sharedb-string-binding'); // Open WebSocket connection to ShareDB server var ReconnectingWebSocket = require('reconnecting-websocket'); -var socket = new ReconnectingWebSocket('ws://' + window.location.host); +var socket = new ReconnectingWebSocket('ws://' + window.location.host, [], { + // ShareDB handles dropped messages, and buffering them while the socket + // is closed has undefined behavior + maxEnqueuedMessages: 0 +}); var connection = new sharedb.Connection(socket); var element = document.querySelector('textarea'); From 6ee645d14e3b773179504bd620088fb7491c9a8b Mon Sep 17 00:00:00 2001 From: Alec Gibson <12036746+alecgibson@users.noreply.github.com> Date: Tue, 2 May 2023 15:23:23 +0100 Subject: [PATCH 61/61] =?UTF-8?q?=F0=9F=94=8A=20Warn=20when=20messages=20a?= =?UTF-8?q?re=20sent=20before=20the=20v1.1=20handshake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment, it's possible for messages to be sent before the client- server handshake. Sending messages before the handshake has happened has undefined behaviour, and can result in errors such as in: https://github.com/share/sharedb/issues/605 We can't just ignore these messages, because old clients might potentially be on v1.0 of the client-server protocol, in which the server informs the client when it's ready, but not the other way around, so it's impossible to know when a client should be considered "ready", and its messages acceptable. Instead, we add a warning for clients on v1.1 who have sent a message before their handshake. In order to aid with debugging, we keep track of the first message received, and log it when the handshake is received (which means that v1.0 clients will never get such a warning). --- lib/agent.js | 20 ++++++++++++++ package.json | 3 ++- test/agent.js | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/setup.js | 4 +++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 test/agent.js diff --git a/lib/agent.js b/lib/agent.js index 7d1e3e5c7..3c6fa4161 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -58,6 +58,11 @@ function Agent(backend, stream) { // active, and it is passed to each middleware call this.custom = {}; + // The first message received over the connection. Stored to warn if messages + // are being sent before the handshake. + this._firstReceivedMessage = null; + this._handshakeReceived = false; + // Send the legacy message to initialize old clients with the random agent Id this.send(this._initMessage(ACTIONS.initLegacy)); } @@ -394,6 +399,7 @@ Agent.prototype._handleMessage = function(request, callback) { try { var errMessage = this._checkRequest(request); if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); + this._checkFirstMessage(request); switch (request.a) { case ACTIONS.handshake: @@ -855,6 +861,20 @@ Agent.prototype._handlePresenceData = function(presence) { }); }; +Agent.prototype._checkFirstMessage = function(request) { + if (this._handshakeReceived) return; + if (!this._firstReceivedMessage) this._firstReceivedMessage = request; + + if (request.a === ACTIONS.handshake) { + this._handshakeReceived = true; + if (this._firstReceivedMessage.a !== ACTIONS.handshake) { + logger.warn('Unexpected message received before handshake', this._firstReceivedMessage); + } + // Release memory + this._firstReceivedMessage = null; + } +}; + function createClientOp(request, clientId) { // src can be provided if it is not the same as the current agent, // such as a resubmission after a reconnect, but it usually isn't needed diff --git a/package.json b/package.json index 64e32010b..c18dc2af0 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "ot-json1": "^0.3.0", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@=1.1.0", - "sinon": "^9.2.4" + "sinon": "^9.2.4", + "sinon-chai": "^3.7.0" }, "files": [ "lib/", diff --git a/test/agent.js b/test/agent.js new file mode 100644 index 000000000..b67761487 --- /dev/null +++ b/test/agent.js @@ -0,0 +1,74 @@ +var Backend = require('../lib/backend'); +var logger = require('../lib/logger'); +var sinon = require('sinon'); +var StreamSocket = require('../lib/stream-socket'); +var expect = require('chai').expect; +var ACTIONS = require('../lib/message-actions').ACTIONS; +var Connection = require('../lib/client/connection'); +var LegacyConnection = require('sharedb-legacy/lib/client').Connection; + +describe('Agent', function() { + var backend; + + beforeEach(function() { + backend = new Backend(); + }); + + afterEach(function(done) { + backend.close(done); + }); + + describe('handshake', function() { + it('warns when messages are sent before the handshake', function(done) { + var socket = new StreamSocket(); + var stream = socket.stream; + backend.listen(stream); + sinon.spy(logger, 'warn'); + socket.send(JSON.stringify({a: ACTIONS.subscribe, c: 'dogs', d: 'fido'})); + var connection = new Connection(socket); + socket._open(); + connection.once('connected', function() { + expect(logger.warn).to.have.been.calledOnceWithExactly( + 'Unexpected message received before handshake', + {a: ACTIONS.subscribe, c: 'dogs', d: 'fido'} + ); + done(); + }); + }); + + it('does not warn when messages are sent after the handshake', function(done) { + var socket = new StreamSocket(); + var stream = socket.stream; + var agent = backend.listen(stream); + sinon.spy(logger, 'warn'); + var connection = new Connection(socket); + socket._open(); + connection.once('connected', function() { + socket.send(JSON.stringify({a: ACTIONS.subscribe, c: 'dogs', d: 'fido'})); + expect(logger.warn).not.to.have.been.called; + expect(agent._firstReceivedMessage).to.be.null; + done(); + }); + }); + + it('does not warn for clients on protocol v1.0', function(done) { + backend.use('receive', function(request, next) { + var error = null; + if (request.data.a === ACTIONS.handshake) error = new Error('Unexpected handshake'); + next(error); + }); + var socket = new StreamSocket(); + var stream = socket.stream; + backend.listen(stream); + sinon.spy(logger, 'warn'); + socket.send(JSON.stringify({a: ACTIONS.subscribe, c: 'dogs', d: 'fido'})); + var connection = new LegacyConnection(socket); + socket._open(); + connection.get('dogs', 'fido').fetch(function(error) { + if (error) return done(error); + expect(logger.warn).not.to.have.been.called; + done(); + }); + }); + }); +}); diff --git a/test/setup.js b/test/setup.js index 6c1c91f98..d00aeee38 100644 --- a/test/setup.js +++ b/test/setup.js @@ -1,5 +1,9 @@ var logger = require('../lib/logger'); var sinon = require('sinon'); +var sinonChai = require('sinon-chai'); +var chai = require('chai'); + +chai.use(sinonChai); if (process.env.LOGGING !== 'true') { // Silence the logger for tests by setting all its methods to no-ops