diff --git a/schemas/common.json b/schemas/common.json index c36303cc..7afd87b6 100644 --- a/schemas/common.json +++ b/schemas/common.json @@ -5,7 +5,7 @@ "type": "string", "minLength": 1, "maxLength": 78 - }, + }, "uploadId": { "type": "string", "format": "uuid" @@ -36,6 +36,17 @@ "type": "number" } }, + "categories": { + "type": "array", + "maxItems": 500, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 6, + "maxLength": 13, + "pattern": "^_s\\d+_c\\d+$" + } + }, "fitToSquare": { "type": "boolean" }, @@ -1057,6 +1068,9 @@ "const": "" } ] + }, + "categories": { + "$ref": "common#/definitions/categories" } } }, diff --git a/schemas/list.json b/schemas/list.json index f10244b9..2871429b 100644 --- a/schemas/list.json +++ b/schemas/list.json @@ -166,6 +166,9 @@ "modelType": { "type": "string", "enum": ["3d", "nft"] + }, + "categories": { + "$ref": "common#/definitions/categories" } } } diff --git a/src/actions/list.js b/src/actions/list.js index c3b6142d..f58c370e 100644 --- a/src/actions/list.js +++ b/src/actions/list.js @@ -35,6 +35,7 @@ const { FILES_NFT_COLLECTION_FIELD, FILES_IS_CLONE_FIELD, FILES_LIST_SEARCH, + FILES_CATEGORIES_FIELD, } = require('../constant'); const k404Error = new Error('ELIST404'); @@ -353,6 +354,10 @@ async function redisSearch(ctx) { query.push(`-@${FILES_DIRECT_ONLY_FIELD}:[1 1]`); } + if (ctx[FILES_CATEGORIES_FIELD]) { + query.push(`@${FILES_CATEGORIES_FIELD}:{ ${ctx[FILES_CATEGORIES_FIELD].join(' | ')} }`); + } + if (ctx.hasTags) { for (const [idx, tag] of ctx.tags.sort().entries()) { // multi-word or tags containing punctuation will be broken into pieces @@ -531,6 +536,7 @@ async function listFiles({ params }) { modelType, nftOwner, temp, + categories, expiration = 30000, } = params; @@ -585,6 +591,7 @@ async function listFiles({ params }) { username: '', nftOwner, modelType, + categories, }; try { diff --git a/src/actions/update.js b/src/actions/update.js index ac3f3f47..61856b3e 100644 --- a/src/actions/update.js +++ b/src/actions/update.js @@ -38,6 +38,7 @@ const { FILES_REFERENCES_FIELD, FILES_NAME_FIELD, FILES_NAME_NORMALIZED_FIELD, + FILES_CATEGORIES_FIELD, } = require('../constant'); const { call } = Function.prototype; @@ -227,6 +228,10 @@ async function updateMeta(lock, ctx, params) { meta[FILES_NAME_NORMALIZED_FIELD] = normalizeForSearch(meta[FILES_NAME_FIELD]); } + if (meta[FILES_CATEGORIES_FIELD]) { + meta[FILES_CATEGORIES_FIELD] = meta[FILES_CATEGORIES_FIELD].join(', '); + } + pipeline.hmset(key, meta); } diff --git a/src/constant.js b/src/constant.js index eda36258..a22a99a9 100644 --- a/src/constant.js +++ b/src/constant.js @@ -73,6 +73,7 @@ module.exports = exports = Object.setPrototypeOf({ FILES_REFERENCES_FIELD: 'references', FILES_IS_REFERENCED_FIELD: 'isReferenced', FILES_IS_IN_SHOWROOM_FIELD: 'isInShowroom', + FILES_CATEGORIES_FIELD: 'categories', // metatype of file FILES_TYPE_FIELD: 'type', @@ -119,6 +120,10 @@ exports.FIELDS_TO_STRINGIFY = [ exports.FILES_REFERENCES_FIELD, ]; +exports.FIELDS_TO_UNTAG = { + [exports.FILES_CATEGORIES_FIELD]: true, +}; + exports.CAPPASITY_3D_MODEL = 'model'; exports.CAPPASITY_IMAGE_MODEL = 'simple'; exports.CAPPASITY_TYPE_MAP = Object.setPrototypeOf({ @@ -145,4 +150,4 @@ exports.UPLOAD_TYPE_GLB_EXTENDED = 'glb-extended'; exports.UPLOAD_TYPE_PANORAMA_EQUIRECT = 'pano-equirect'; exports.UPLOAD_TYPE_PANORAMA_CUBEMAP = 'pano-cubemap'; -exports.FILES_LIST_SEARCH = 'files-list-v9'; +exports.FILES_LIST_SEARCH = 'files-list-v10'; diff --git a/src/migrations/redis-search-stack-v2/index.js b/src/migrations/redis-search-stack-v2/index.js index 17055b6a..ead90610 100644 --- a/src/migrations/redis-search-stack-v2/index.js +++ b/src/migrations/redis-search-stack-v2/index.js @@ -33,6 +33,7 @@ const { FILES_IS_REFERENCED_FIELD, FILES_IS_IN_SHOWROOM_FIELD, FILES_LIST_SEARCH, + FILES_CATEGORIES_FIELD, } = require('../../constant'); const FIELD_TO_TYPE = [ @@ -68,6 +69,7 @@ const FIELD_TO_TYPE = [ [FILES_HAS_NFT_OWNER_FIELD, 'NUMERIC', 'SORTABLE'], [FILES_HAS_REFERENCES_FIELD, 'NUMERIC', 'SORTABLE'], [FILES_IS_REFERENCED_FIELD, 'NUMERIC', 'SORTABLE'], + [FILES_CATEGORIES_FIELD, 'TAG'], ]; // https://redis.io/docs/stack/search/reference/aggregations/#filter-expressions @@ -93,5 +95,5 @@ async function createSearchIndexV2(service) { module.exports = { script: createSearchIndexV2, min: 13, - final: 14, + final: 15, }; diff --git a/src/utils/fetch-data.js b/src/utils/fetch-data.js index 4643f412..741ebf12 100644 --- a/src/utils/fetch-data.js +++ b/src/utils/fetch-data.js @@ -6,7 +6,7 @@ const calcSlot = require('cluster-key-slot'); const fs = require('fs'); const is = require('is'); const safeParse = require('./safe-parse'); -const { FIELDS_TO_STRINGIFY, FILES_TAGS_FIELD, FILE_MISSING_ERROR } = require('../constant'); +const { FIELDS_TO_STRINGIFY, FILES_TAGS_FIELD, FILE_MISSING_ERROR, FIELDS_TO_UNTAG } = require('../constant'); /** * Helper constants @@ -26,6 +26,8 @@ function remapData(field, index) { this.output[field] = safeParse(value, []); } else if (hasOwnProperty.call(STRINGIFY_FIELDS, field)) { this.output[field] = safeParse(value); + } else if (FIELDS_TO_UNTAG[field]) { + this.output[field] = value.split(', '); } else { this.output[field] = value; } diff --git a/test/suites/list.js b/test/suites/list.js index 26df2f12..3364e373 100644 --- a/test/suites/list.js +++ b/test/suites/list.js @@ -842,6 +842,76 @@ for (const redisSearchEnabled of [true, false].values()) { assert.equal(list.pages, 0); }); }); + + describe('catalog list', function catalogTestSuite() { + if (!redisSearchEnabled) { + return; + } + before('upload', async function pretest() { + const { uploadId } = await initAndUpload(modelData, false).call(this); + return this.amqp.publishAndWait(updateRoute, { + uploadId, + username, + meta: { + ...meta, + categories: ['_s1_c1', '_s1_c2'], + }, + }); + }); + + before('pre-upload file #2', async function pretest() { + const { uploadId } = await initAndUpload(modelData, false).call(this); + + // this file should not be returned by search as tag search is using `AND` + return this.amqp.publishAndWait(updateRoute, { + uploadId, + username, + meta: { + ...meta, + categories: ['_s1_c3', '_s1_c4'], + }, + }); + }); + + it('should filter by catalog categories', async function test() { + const list = await this.amqp.publishAndWait('files.list', { + filter: {}, + order: 'ASC', + offset: 0, + limit: 10, + owner: username, + categories: ['_s1_c2'], + }); + + assert.deepEqual(list.files[0].categories, ['_s1_c1', '_s1_c2']); + }); + + it('should return empty no categories', async function test() { + const list = await this.amqp.publishAndWait('files.list', { + filter: {}, + order: 'ASC', + offset: 0, + limit: 10, + owner: username, + categories: ['_s222_c2'], + }); + + assert.equal(list.pages, 0); + }); + + it('should return both', async function test() { + const list = await this.amqp.publishAndWait('files.list', { + filter: {}, + order: 'ASC', + offset: 0, + limit: 10, + owner: username, + categories: ['_s1_c2', '_s1_c4'], + }); + + assert.equal(list.files.length, 2); + }); + }); }); }); }