Merge branch 'master' into whiteboard

src/packages/database/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocalc/database",
-  "version": "0.17.1",
+  "version": "0.17.2",
   "description": "CoCalc: code for working with our PostgreSQL database",
   "exports": {
     ".": "./dist/index.js",
@@ -19,8 +19,8 @@
   ],
   "dependencies": {
     "@cocalc/backend": "^1.12.1",
-    "@cocalc/server": "^0.19.0",
-    "@cocalc/util": "^1.32.1",
+    "@cocalc/server": "^0.20.0",
+    "@cocalc/util": "^1.32.2",
     "@types/lodash": "^4.14.176",
     "@types/lru-cache": "^5.1.1",
     "@types/pg": "^8.6.1",

src/packages/database/postgres-blobs.coffee

@@ -609,7 +609,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                 dbg("determine inactive syncstring ids")
                 @_query
                     query    : 'SELECT string_id FROM syncstrings'
-                    where    : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL']
+                    where    : [{'last_active <= $::TIMESTAMP' : misc.days_ago(opts.age_days)}, 'archived IS NULL', 'huge IS NOT TRUE']
                     limit    : opts.limit
                     timeout_s : TIMEOUT_LONG_S
                     cb       : all_results 'string_id', (err, v) =>
@@ -641,6 +641,9 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
 
     # Offlines and archives the patch, unless the string is active very recently, in
     # which case this is a no-op.
+    #
+    # TODO: this ignores all syncstrings marked as "huge:true", because the patches are too large.
+    #       come up with a better strategy (incremental?) to generate the blobs to avoid the problem.
     archive_patches: (opts) =>
         opts = defaults opts,
             string_id : required
@@ -649,13 +652,13 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
             cutoff    : misc.minutes_ago(30)  # never touch anything this new
             cb        : undefined
         dbg = @_dbg("archive_patches(string_id='#{opts.string_id}')")
-        syncstring = patches = blob_uuid = project_id = last_active = undefined
+        syncstring = patches = blob_uuid = project_id = last_active = huge = undefined
         where = {"string_id = $::CHAR(40)" : opts.string_id}
         async.series([
             (cb) =>
-                dbg("get project_id")
+                dbg("get syncstring info")
                 @_query
-                    query : "SELECT project_id, archived, last_active FROM syncstrings"
+                    query : "SELECT project_id, archived, last_active, huge FROM syncstrings"
                     where : where
                     cb    : one_result (err, x) =>
                         if err
@@ -667,11 +670,16 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                         else
                             project_id = x.project_id
                             last_active = x.last_active
+                            huge = !!x.huge
+                            dbg("got last_active=#{last_active} project_id=#{project_id} huge=#{huge}")
                             cb()
             (cb) =>
                 if last_active? and last_active >= opts.cutoff
                     dbg("excluding due to cutoff")
                     cb(); return
+                if huge
+                    dbg("excluding due to being huge")
+                    cb(); return
                 dbg("get patches")
                 @export_patches
                     string_id : opts.string_id
@@ -681,6 +689,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
             (cb) =>
                 if last_active? and last_active >= opts.cutoff
                     cb(); return
+                if huge
+                    cb(); return
                 dbg("create blob from patches")
                 try
                     blob = Buffer.from(JSON.stringify(patches))
@@ -689,20 +699,30 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                     # need to break patches up...
                     # This is not exactly the end of the world as the entire point of all this is to
                     # just save some space in the database...
-                    cb(err)
-                    return
-                dbg('save blob')
-                blob_uuid = misc_node.uuidsha1(blob)
-                @save_blob
-                    uuid       : blob_uuid
-                    blob       : blob
-                    project_id : project_id
-                    compress   : opts.compress
-                    level      : opts.level
-                    cb         : cb
+                    dbg('error creating blob, marking syncstring as being "huge": ' + err)
+                    huge = true
+                    @_query
+                        query : "UPDATE syncstrings"
+                        set   : {huge : true}
+                        where : where
+                        cb    : (err) =>
+                            cb(err)
+                            return
+                if not huge
+                    dbg('save blob')
+                    blob_uuid = misc_node.uuidsha1(blob)
+                    @save_blob
+                        uuid       : blob_uuid
+                        blob       : blob
+                        project_id : project_id
+                        compress   : opts.compress
+                        level      : opts.level
+                        cb         : cb
             (cb) =>
                 if last_active? and last_active >= opts.cutoff
                     cb(); return
+                if huge
+                    cb(); return
                 dbg("update syncstring to indicate patches have been archived in a blob")
                 @_query
                     query : "UPDATE syncstrings"
@@ -712,6 +732,8 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
             (cb) =>
                 if last_active? and last_active >= opts.cutoff
                     cb(); return
+                if huge
+                    cb(); return
                 dbg("actually deleting patches")
                 delete_patches(db:@, string_id: opts.string_id, cb:cb)
         ], (err) => opts.cb?(err))
@@ -789,7 +811,12 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
                     opts.cb(err)
                 else
                     for p in patches
-                        p.time = new Date(p.epoch)
+                        # TODO why using epoch and then converting to Date, why not just taking time?
+                        # Besides that: @hsy noticed in development that p.epoch could be a string, resulting in an invalid date.
+                        if typeof p.epoch == 'string'
+                            p.time = new Date(parseInt(p.epoch))
+                        else
+                            p.time = new Date(p.epoch)
                         delete p.epoch
                     opts.cb(undefined, patches)
 

src/packages/hub/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocalc/hub",
-  "version": "1.51.0",
+  "version": "1.51.1",
   "description": "CoCalc: Backend webserver component",
   "exports": {
     "./*": "./dist/*.js",
@@ -22,12 +22,12 @@
     "@cocalc/assets": "^1.6.0",
     "@cocalc/backend": "^1.12.1",
     "@cocalc/cdn": "^1.10.0",
-    "@cocalc/database": "^0.17.1",
+    "@cocalc/database": "^0.17.2",
     "@cocalc/frontend": "^1.41.0",
-    "@cocalc/next": "^0.39.0",
+    "@cocalc/next": "^0.39.1",
     "@cocalc/server": "^0.20.0",
     "@cocalc/static": "^1.74.0",
-    "@cocalc/util": "^1.32.1",
+    "@cocalc/util": "^1.32.2",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@passport-next/passport-oauth2": "^2.1.1",
     "@sendgrid/client": "^7.4.6",

src/packages/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocalc/next",
-  "version": "0.39.0",
+  "version": "0.39.1",
   "description": "CoCalc's nextjs server",
   "main": "lib/init.js",
   "exports": {
@@ -62,10 +62,10 @@
     "@cocalc/assets": "^1.6.0",
     "@cocalc/backend": "^1.12.1",
     "@cocalc/cdn": "^1.10.0",
-    "@cocalc/database": "^0.17.1",
+    "@cocalc/database": "^0.17.2",
     "@cocalc/frontend": "^1.41.0",
     "@cocalc/server": "^0.20.0",
-    "@cocalc/util": "^1.32.1",
+    "@cocalc/util": "^1.32.2",
     "@types/express": "^4.17.13",
     "@types/react": "^17.0.33",
     "antd": "^4.18.2",

src/packages/next/pages/policies/thirdparties.tsx

@@ -74,6 +74,24 @@ export default function TermsOfService({ customize }) {
                   </A>
                 </li>
               </ul>
+              <li>
+                <A href={"https://www.google.com/recaptcha/about/"}>
+                  reCAPTCHA
+                </A>
+              </li>
+              <ul>
+                <li>usage: fraud and abuse protection</li>
+                <li>
+                  <A href={"https://policies.google.com/terms"}>
+                    Terms of Usage
+                  </A>
+                </li>
+                <li>
+                  <A href={"https://policies.google.com/privacy"}>
+                    Privacy Policy
+                  </A>
+                </li>
+              </ul>
               <li>
                 <A href="https://www.zendesk.com/">Zendesk</A>
               </li>

src/packages/util/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocalc/util",
-  "version": "1.32.1",
+  "version": "1.32.2",
   "description": "CoCalc code shared between the frontend and the backend",
   "exports": {
     "./*": "./dist/*.js",

src/packages/util/syncstring_schema.js

@@ -88,6 +88,10 @@ schema.syncstrings = {
       desc:
         "Shared (by all users) configuration settings for editing this file (e.g., which spellcheck language to use).",
     },
+    huge: {
+      type: "boolean",
+      desc: "If true, this syncstring contains too many or too large patches to be processed. Hence if this is set, it won't be processed. TODO: implement a better archiving mechanism and then process such 'huge' syncstrings.",
+    }
   },
 
   pg_indexes: ["last_active", "archived"],