feat: catalog (#322)

schemas/common.json

@@ -5,7 +5,7 @@
       "type": "string",
       "minLength": 1,
       "maxLength": 78
-    },    
+    },
     "uploadId": {
       "type": "string",
       "format": "uuid"
@@ -36,6 +36,17 @@
         "type": "number"
       }
     },
+    "categories": {
+      "type": "array",
+      "maxItems": 500,
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "minLength": 6,
+        "maxLength": 13,
+        "pattern": "^_s\\d+_c\\d+$"
+      }
+    },
     "fitToSquare": {
       "type": "boolean"
     },
@@ -1057,6 +1068,9 @@
               "const": ""
             }
           ]
+        },
+        "categories": {
+          "$ref": "common#/definitions/categories"
         }
       }
     },

schemas/list.json

@@ -166,6 +166,9 @@
         "modelType": {
             "type": "string",
             "enum": ["3d", "nft"]
+        },
+        "categories": {
+            "$ref": "common#/definitions/categories"
         }
     }
 }

src/actions/list.js

@@ -35,6 +35,7 @@ const {
   FILES_NFT_COLLECTION_FIELD,
   FILES_IS_CLONE_FIELD,
   FILES_LIST_SEARCH,
+  FILES_CATEGORIES_FIELD,
 } = require('../constant');
 
 const k404Error = new Error('ELIST404');
@@ -353,6 +354,10 @@ async function redisSearch(ctx) {
     query.push(`-@${FILES_DIRECT_ONLY_FIELD}:[1 1]`);
   }
 
+  if (ctx[FILES_CATEGORIES_FIELD]) {
+    query.push(`@${FILES_CATEGORIES_FIELD}:{ ${ctx[FILES_CATEGORIES_FIELD].join(' | ')} }`);
+  }
+
   if (ctx.hasTags) {
     for (const [idx, tag] of ctx.tags.sort().entries()) {
       // multi-word or tags containing punctuation will be broken into pieces
@@ -531,6 +536,7 @@ async function listFiles({ params }) {
     modelType,
     nftOwner,
     temp,
+    categories,
     expiration = 30000,
   } = params;
 
@@ -585,6 +591,7 @@ async function listFiles({ params }) {
     username: '',
     nftOwner,
     modelType,
+    categories,
   };
 
   try {

src/actions/update.js

@@ -38,6 +38,7 @@ const {
   FILES_REFERENCES_FIELD,
   FILES_NAME_FIELD,
   FILES_NAME_NORMALIZED_FIELD,
+  FILES_CATEGORIES_FIELD,
 } = require('../constant');
 
 const { call } = Function.prototype;
@@ -227,6 +228,10 @@ async function updateMeta(lock, ctx, params) {
       meta[FILES_NAME_NORMALIZED_FIELD] = normalizeForSearch(meta[FILES_NAME_FIELD]);
     }
 
+    if (meta[FILES_CATEGORIES_FIELD]) {
+      meta[FILES_CATEGORIES_FIELD] = meta[FILES_CATEGORIES_FIELD].join(', ');
+    }
+
     pipeline.hmset(key, meta);
   }
 

src/constant.js

@@ -73,6 +73,7 @@ module.exports = exports = Object.setPrototypeOf({
   FILES_REFERENCES_FIELD: 'references',
   FILES_IS_REFERENCED_FIELD: 'isReferenced',
   FILES_IS_IN_SHOWROOM_FIELD: 'isInShowroom',
+  FILES_CATEGORIES_FIELD: 'categories',
 
   // metatype of file
   FILES_TYPE_FIELD: 'type',
@@ -119,6 +120,10 @@ exports.FIELDS_TO_STRINGIFY = [
   exports.FILES_REFERENCES_FIELD,
 ];
 
+exports.FIELDS_TO_UNTAG = {
+  [exports.FILES_CATEGORIES_FIELD]: true,
+};
+
 exports.CAPPASITY_3D_MODEL = 'model';
 exports.CAPPASITY_IMAGE_MODEL = 'simple';
 exports.CAPPASITY_TYPE_MAP = Object.setPrototypeOf({
@@ -145,4 +150,4 @@ exports.UPLOAD_TYPE_GLB_EXTENDED = 'glb-extended';
 exports.UPLOAD_TYPE_PANORAMA_EQUIRECT = 'pano-equirect';
 exports.UPLOAD_TYPE_PANORAMA_CUBEMAP = 'pano-cubemap';
 
-exports.FILES_LIST_SEARCH = 'files-list-v9';
+exports.FILES_LIST_SEARCH = 'files-list-v10';

src/migrations/redis-search-stack-v2/index.js

@@ -33,6 +33,7 @@ const {
   FILES_IS_REFERENCED_FIELD,
   FILES_IS_IN_SHOWROOM_FIELD,
   FILES_LIST_SEARCH,
+  FILES_CATEGORIES_FIELD,
 } = require('../../constant');
 
 const FIELD_TO_TYPE = [
@@ -68,6 +69,7 @@ const FIELD_TO_TYPE = [
   [FILES_HAS_NFT_OWNER_FIELD, 'NUMERIC', 'SORTABLE'],
   [FILES_HAS_REFERENCES_FIELD, 'NUMERIC', 'SORTABLE'],
   [FILES_IS_REFERENCED_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_CATEGORIES_FIELD, 'TAG'],
 ];
 
 // https://redis.io/docs/stack/search/reference/aggregations/#filter-expressions
@@ -93,5 +95,5 @@ async function createSearchIndexV2(service) {
 module.exports = {
   script: createSearchIndexV2,
   min: 13,
-  final: 14,
+  final: 15,
 };

src/utils/fetch-data.js

@@ -6,7 +6,7 @@ const calcSlot = require('cluster-key-slot');
 const fs = require('fs');
 const is = require('is');
 const safeParse = require('./safe-parse');
-const { FIELDS_TO_STRINGIFY, FILES_TAGS_FIELD, FILE_MISSING_ERROR } = require('../constant');
+const { FIELDS_TO_STRINGIFY, FILES_TAGS_FIELD, FILE_MISSING_ERROR, FIELDS_TO_UNTAG } = require('../constant');
 
 /**
  * Helper constants
@@ -26,6 +26,8 @@ function remapData(field, index) {
     this.output[field] = safeParse(value, []);
   } else if (hasOwnProperty.call(STRINGIFY_FIELDS, field)) {
     this.output[field] = safeParse(value);
+  } else if (FIELDS_TO_UNTAG[field]) {
+    this.output[field] = value.split(', ');
   } else {
     this.output[field] = value;
   }

test/suites/list.js

@@ -842,6 +842,76 @@ for (const redisSearchEnabled of [true, false].values()) {
           assert.equal(list.pages, 0);
         });
       });
+
+      describe('catalog list', function catalogTestSuite() {
+        if (!redisSearchEnabled) {
+          return;
+        }
+        before('upload', async function pretest() {
+          const { uploadId } = await initAndUpload(modelData, false).call(this);
+          return this.amqp.publishAndWait(updateRoute, {
+            uploadId,
+            username,
+            meta: {
+              ...meta,
+              categories: ['_s1_c1', '_s1_c2'],
+            },
+          });
+        });
+
+        before('pre-upload file #2', async function pretest() {
+          const { uploadId } = await initAndUpload(modelData, false).call(this);
+
+          // this file should not be returned by search as tag search is using `AND`
+          return this.amqp.publishAndWait(updateRoute, {
+            uploadId,
+            username,
+            meta: {
+              ...meta,
+              categories: ['_s1_c3', '_s1_c4'],
+            },
+          });
+        });
+
+        it('should filter by catalog categories', async function test() {
+          const list = await this.amqp.publishAndWait('files.list', {
+            filter: {},
+            order: 'ASC',
+            offset: 0,
+            limit: 10,
+            owner: username,
+            categories: ['_s1_c2'],
+          });
+
+          assert.deepEqual(list.files[0].categories, ['_s1_c1', '_s1_c2']);
+        });
+
+        it('should return empty no categories', async function test() {
+          const list = await this.amqp.publishAndWait('files.list', {
+            filter: {},
+            order: 'ASC',
+            offset: 0,
+            limit: 10,
+            owner: username,
+            categories: ['_s222_c2'],
+          });
+
+          assert.equal(list.pages, 0);
+        });
+
+        it('should return both', async function test() {
+          const list = await this.amqp.publishAndWait('files.list', {
+            filter: {},
+            order: 'ASC',
+            offset: 0,
+            limit: 10,
+            owner: username,
+            categories: ['_s1_c2', '_s1_c4'],
+          });
+
+          assert.equal(list.files.length, 2);
+        });
+      });
     });
   });
 }