From b7949a539fa0b0d53ccfcee45f869045c5e835ed Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 31 Mar 2022 15:47:30 +0200 Subject: [PATCH 1/7] =?UTF-8?q?db/syncstrings:=20mark=20syncstrings=20with?= =?UTF-8?q?=20too=20large=20patches=20as=20"huge"=20and=20ignore=20them=20?= =?UTF-8?q?=E2=80=93=20for=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/packages/database/postgres-blobs.coffee | 52 +++++++++++++++------ src/packages/util/syncstring_schema.js | 4 ++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/packages/database/postgres-blobs.coffee b/src/packages/database/postgres-blobs.coffee index 9a18b7d3b2..39429ae103 100644 --- a/src/packages/database/postgres-blobs.coffee +++ b/src/packages/database/postgres-blobs.coffee @@ -609,7 +609,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext dbg("determine inactive syncstring ids") @_query query : 'SELECT string_id FROM syncstrings' - where : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL'] + where : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL', 'huge IS NOT TRUE'] limit : opts.limit timeout_s : TIMEOUT_LONG_S cb : all_results 'string_id', (err, v) => @@ -641,6 +641,9 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext # Offlines and archives the patch, unless the string is active very recently, in # which case this is a no-op. + # + # TODO: this ignores all syncstrings marked as "huge:true", because the patches are too large. + # come up with a better strategy (incremental?) to generate the blobs to avoid the problem. archive_patches: (opts) => opts = defaults opts, string_id : required @@ -649,13 +652,13 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext cutoff : misc.minutes_ago(30) # never touch anything this new cb : undefined dbg = @_dbg("archive_patches(string_id='#{opts.string_id}')") - syncstring = patches = blob_uuid = project_id = last_active = undefined + syncstring = patches = blob_uuid = project_id = last_active = huge = undefined where = {"string_id = $::CHAR(40)" : opts.string_id} async.series([ (cb) => - dbg("get project_id") + dbg("get syncstring info") @_query - query : "SELECT project_id, archived, last_active FROM syncstrings" + query : "SELECT project_id, archived, last_active, huge FROM syncstrings" where : where cb : one_result (err, x) => if err @@ -667,11 +670,16 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext else project_id = x.project_id last_active = x.last_active + huge = !!x.huge + dbg("got last_active=#{last_active} project_id=#{project_id} huge=#{huge}") cb() (cb) => if last_active? and last_active >= opts.cutoff dbg("excluding due to cutoff") cb(); return + if huge + dbg("excluding due to being huge") + cb(); return dbg("get patches") @export_patches string_id : opts.string_id @@ -681,6 +689,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext (cb) => if last_active? and last_active >= opts.cutoff cb(); return + if huge + cb(); return dbg("create blob from patches") try blob = Buffer.from(JSON.stringify(patches)) @@ -689,20 +699,30 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext # need to break patches up... # This is not exactly the end of the world as the entire point of all this is to # just save some space in the database... - cb(err) - return - dbg('save blob') - blob_uuid = misc_node.uuidsha1(blob) - @save_blob - uuid : blob_uuid - blob : blob - project_id : project_id - compress : opts.compress - level : opts.level - cb : cb + dbg('error creating blob, marking syncstring as being "huge": ' + err) + huge = true + @_query + query : "UPDATE syncstrings" + set : {huge : true} + where : where + cb : (err) => + cb(err) + return + if not huge + dbg('save blob') + blob_uuid = misc_node.uuidsha1(blob) + @save_blob + uuid : blob_uuid + blob : blob + project_id : project_id + compress : opts.compress + level : opts.level + cb : cb (cb) => if last_active? and last_active >= opts.cutoff cb(); return + if huge + cb(); return dbg("update syncstring to indicate patches have been archived in a blob") @_query query : "UPDATE syncstrings" @@ -712,6 +732,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext (cb) => if last_active? and last_active >= opts.cutoff cb(); return + if huge + cb(); return dbg("actually deleting patches") delete_patches(db:@, string_id: opts.string_id, cb:cb) ], (err) => opts.cb?(err)) diff --git a/src/packages/util/syncstring_schema.js b/src/packages/util/syncstring_schema.js index a129f52173..36f8b09461 100644 --- a/src/packages/util/syncstring_schema.js +++ b/src/packages/util/syncstring_schema.js @@ -88,6 +88,10 @@ schema.syncstrings = { desc: "Shared (by all users) configuration settings for editing this file (e.g., which spellcheck language to use).", }, + huge: { + type: "boolean", + desc: "If true, this syncstring contains too many or too large patches to be processed. Hence if this is set, it won't be processed. TODO: implement a better archiving mechanism and then process such 'huge' syncstrings.", + } }, pg_indexes: ["last_active", "archived"], From a57cd96447ea6e0f631617413c18aff07bfc827b Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Thu, 31 Mar 2022 17:46:46 +0200 Subject: [PATCH 2/7] database/patches: special case during epoch/time conversion --- src/packages/database/postgres-blobs.coffee | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/packages/database/postgres-blobs.coffee b/src/packages/database/postgres-blobs.coffee index 39429ae103..97477ca4ad 100644 --- a/src/packages/database/postgres-blobs.coffee +++ b/src/packages/database/postgres-blobs.coffee @@ -811,7 +811,12 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts.cb(err) else for p in patches - p.time = new Date(p.epoch) + # TODO why using epoch and then converting to Date, why not just taking time? + # Besides that: @hsy noticed in development that p.epoch could be a string, resulting in an invalid date. + if typeof p.epoch == 'string' + p.time = new Date(parseInt(p.epoch)) + else + p.time = new Date(p.epoch) delete p.epoch opts.cb(undefined, patches) From e806532f01c2c37e87208ca01bdc2b03cc55404d Mon Sep 17 00:00:00 2001 From: Harald Schilly Date: Fri, 1 Apr 2022 12:17:41 +0200 Subject: [PATCH 3/7] add reCAPTCHA to the list of 3rd parties --- .../next/pages/policies/thirdparties.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/packages/next/pages/policies/thirdparties.tsx b/src/packages/next/pages/policies/thirdparties.tsx index e446ebaed4..5925e12db4 100644 --- a/src/packages/next/pages/policies/thirdparties.tsx +++ b/src/packages/next/pages/policies/thirdparties.tsx @@ -74,6 +74,24 @@ export default function TermsOfService({ customize }) { +
  • + + reCAPTCHA + +
    • +
  • Zendesk
    • From 7076a411aa3d5ae38d1b0c658925c139fb3daf5e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Apr 2022 16:19:11 +0000 Subject: [PATCH 4/7] Publish new version of package packages/util to npmjs package repo. --- src/packages/util/package-lock.json | 4 ++-- src/packages/util/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/packages/util/package-lock.json b/src/packages/util/package-lock.json index abf66479f5..f7d4f56ff8 100644 --- a/src/packages/util/package-lock.json +++ b/src/packages/util/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cocalc/util", - "version": "1.32.1", + "version": "1.32.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@cocalc/util", - "version": "1.32.1", + "version": "1.32.2", "license": "SEE LICENSE.md", "workspaces": [ "." diff --git a/src/packages/util/package.json b/src/packages/util/package.json index f35719f5ed..cbff96159f 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -1,6 +1,6 @@ { "name": "@cocalc/util", - "version": "1.32.1", + "version": "1.32.2", "description": "CoCalc code shared between the frontend and the backend", "exports": { "./*": "./dist/*.js", From ac1dcff99e69a8d680106c69d94eb493486c2abf Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Apr 2022 16:19:23 +0000 Subject: [PATCH 5/7] Publish new version of package packages/hub to npmjs package repo. --- src/packages/hub/package-lock.json | 4 ++-- src/packages/hub/package.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/packages/hub/package-lock.json b/src/packages/hub/package-lock.json index cb31a081e8..604273db1c 100644 --- a/src/packages/hub/package-lock.json +++ b/src/packages/hub/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cocalc/hub", - "version": "1.51.0", + "version": "1.51.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@cocalc/hub", - "version": "1.51.0", + "version": "1.51.1", "license": "SEE LICENSE.md", "workspaces": [ "./", diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index eb34e89e5d..3e469bb057 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -1,6 +1,6 @@ { "name": "@cocalc/hub", - "version": "1.51.0", + "version": "1.51.1", "description": "CoCalc: Backend webserver component", "exports": { "./*": "./dist/*.js", @@ -22,12 +22,12 @@ "@cocalc/assets": "^1.6.0", "@cocalc/backend": "^1.12.1", "@cocalc/cdn": "^1.10.0", - "@cocalc/database": "^0.17.1", + "@cocalc/database": "^0.17.2", "@cocalc/frontend": "^1.41.0", - "@cocalc/next": "^0.39.0", + "@cocalc/next": "^0.39.1", "@cocalc/server": "^0.20.0", "@cocalc/static": "^1.74.0", - "@cocalc/util": "^1.32.1", + "@cocalc/util": "^1.32.2", "@passport-next/passport-google-oauth2": "^1.0.0", "@passport-next/passport-oauth2": "^2.1.1", "@sendgrid/client": "^7.4.6", From b032aca535c78e7e0f3fb4264f1d7d0b49f95ce5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Apr 2022 16:19:33 +0000 Subject: [PATCH 6/7] Publish new version of package packages/database to npmjs package repo. --- src/packages/database/package-lock.json | 4 ++-- src/packages/database/package.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/packages/database/package-lock.json b/src/packages/database/package-lock.json index d81ed97cee..7878c5d3c5 100644 --- a/src/packages/database/package-lock.json +++ b/src/packages/database/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cocalc/database", - "version": "0.17.1", + "version": "0.17.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@cocalc/database", - "version": "0.17.1", + "version": "0.17.2", "license": "SEE LICENSE.md", "workspaces": [ "./", diff --git a/src/packages/database/package.json b/src/packages/database/package.json index bcfc5e4595..89e6450353 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -1,6 +1,6 @@ { "name": "@cocalc/database", - "version": "0.17.1", + "version": "0.17.2", "description": "CoCalc: code for working with our PostgreSQL database", "exports": { ".": "./dist/index.js", @@ -19,8 +19,8 @@ ], "dependencies": { "@cocalc/backend": "^1.12.1", - "@cocalc/server": "^0.19.0", - "@cocalc/util": "^1.32.1", + "@cocalc/server": "^0.20.0", + "@cocalc/util": "^1.32.2", "@types/lodash": "^4.14.176", "@types/lru-cache": "^5.1.1", "@types/pg": "^8.6.1", From 6d963cca7cc76c7dca0caec68dda08b5b037e74f Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Apr 2022 16:22:42 +0000 Subject: [PATCH 7/7] Publish new version of package packages/next to npmjs package repo. --- src/packages/next/package-lock.json | 4 ++-- src/packages/next/package.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/packages/next/package-lock.json b/src/packages/next/package-lock.json index 34a0192ff3..f219ee4cad 100644 --- a/src/packages/next/package-lock.json +++ b/src/packages/next/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cocalc/next", - "version": "0.39.0", + "version": "0.39.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@cocalc/next", - "version": "0.39.0", + "version": "0.39.1", "license": "SEE LICENSE IN LICENSE.md", "workspaces": [ "../assets", diff --git a/src/packages/next/package.json b/src/packages/next/package.json index 3710bdbe52..220d0bd991 100644 --- a/src/packages/next/package.json +++ b/src/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "@cocalc/next", - "version": "0.39.0", + "version": "0.39.1", "description": "CoCalc's nextjs server", "main": "lib/init.js", "exports": { @@ -62,10 +62,10 @@ "@cocalc/assets": "^1.6.0", "@cocalc/backend": "^1.12.1", "@cocalc/cdn": "^1.10.0", - "@cocalc/database": "^0.17.1", + "@cocalc/database": "^0.17.2", "@cocalc/frontend": "^1.41.0", "@cocalc/server": "^0.20.0", - "@cocalc/util": "^1.32.1", + "@cocalc/util": "^1.32.2", "@types/express": "^4.17.13", "@types/react": "^17.0.33", "antd": "^4.18.2",