feat: add uploadType to redis search index (#329)

schemas/list.json

@@ -117,6 +117,9 @@
                 "isInShowroom": {
                     "$ref": "common#/definitions/filter-props"
                 },
+                "uploadType": {
+                    "$ref": "common#/definitions/filter-props"
+                },
                 "#": {
                     "$ref": "common#/definitions/filter-props"
                 },

src/actions/upload.js

@@ -13,26 +13,27 @@ const { normalizeForSearch } = require('../utils/normalize-name');
 const { FILES_NAME_FIELD, FILES_NAME_NORMALIZED_FIELD } = require('../constant');
 
 const {
-  STATUS_PENDING,
-  UPLOAD_DATA,
+  FIELDS_TO_STRINGIFY,
+  FILES_BUCKET_FIELD,
+  FILES_CONTENT_LENGTH_FIELD,
   FILES_DATA,
+  FILES_DIRECT_ONLY_FIELD,
+  FILES_HAS_NFT,
+  FILES_HAS_REFERENCES_FIELD,
+  FILES_ID_FIELD,
+  FILES_INDEX_TEMP,
+  FILES_NFT_FIELD,
+  FILES_OWNER_FIELD,
+  FILES_POST_ACTION,
   FILES_PUBLIC_FIELD,
+  FILES_REFERENCES_FIELD,
+  FILES_STATUS_FIELD,
   FILES_TEMP_FIELD,
-  FILES_BUCKET_FIELD,
-  FILES_OWNER_FIELD,
   FILES_UNLISTED_FIELD,
-  FILES_STATUS_FIELD,
-  FIELDS_TO_STRINGIFY,
-  FILES_INDEX_TEMP,
-  FILES_POST_ACTION,
-  FILES_DIRECT_ONLY_FIELD,
-  FILES_CONTENT_LENGTH_FIELD,
-  FILES_ID_FIELD,
   FILES_UPLOAD_STARTED_AT_FIELD,
-  FILES_REFERENCES_FIELD,
-  FILES_HAS_NFT,
-  FILES_NFT_FIELD,
-  FILES_HAS_REFERENCES_FIELD,
+  FILES_UPLOAD_TYPE_FIELD,
+  STATUS_PENDING,
+  UPLOAD_DATA,
 } = require('../constant');
 const { assertNotReferenced } = require('../utils/check-data');
 
@@ -50,16 +51,16 @@ const { assertNotReferenced } = require('../utils/check-data');
  */
 async function initFileUpload({ params }) {
   const {
+    [FILES_UPLOAD_TYPE_FIELD]: uploadType,
+    directOnly,
+    expires,
     meta,
-    username,
-    temp,
-    unlisted,
     origin,
-    resumable,
-    expires,
-    uploadType,
     postAction,
-    directOnly,
+    resumable,
+    temp,
+    unlisted,
+    username,
   } = params;
 
   const { redis, config: { uploadTTL } } = this;
@@ -208,7 +209,7 @@ async function initFileUpload({ params }) {
   }
 
   if (uploadType) {
-    fileData.uploadType = uploadType;
+    fileData[FILES_UPLOAD_TYPE_FIELD] = uploadType;
   }
 
   if (isPublic) {

src/constant.js

@@ -50,6 +50,7 @@ module.exports = exports = Object.setPrototypeOf({
   FILES_CONTENT_LENGTH_FIELD: 'contentLength',
   FILES_UPLOADED_AT_FIELD: 'uploadedAt',
   FILES_UPLOAD_STARTED_AT_FIELD: 'startedAt',
+  FILES_UPLOAD_TYPE_FIELD: 'uploadType',
   FILES_VERSION_FIELD: 'version',
   FILES_HAS_NFT: 'hn',
   FILES_NAME_FIELD: 'name',
@@ -150,4 +151,4 @@ exports.UPLOAD_TYPE_GLB_EXTENDED = 'glb-extended';
 exports.UPLOAD_TYPE_PANORAMA_EQUIRECT = 'pano-equirect';
 exports.UPLOAD_TYPE_PANORAMA_CUBEMAP = 'pano-cubemap';
 
-exports.FILES_LIST_SEARCH = 'files-list-v10';
+exports.FILES_LIST_SEARCH = 'files-list-v11';

src/migrations/redis-search-stack-v2/index.js

@@ -1,75 +1,77 @@
 const {
-  FILES_DATA,
   FILES_ALIAS_FIELD,
-  FILES_OWNER_FIELD,
-  FILES_PUBLIC_FIELD,
-  FILES_PACKED_FIELD,
-  FILES_TAGS_FIELD,
-  FILES_TEMP_FIELD,
-  FILES_STATUS_FIELD,
-  FILES_PARTS_FIELD,
-  FILES_UNLISTED_FIELD,
-  FILES_DIRECT_ONLY_FIELD,
-  FILES_UPLOADED_AT_FIELD,
-  FILES_ID_FIELD,
-  FILES_UPLOAD_STARTED_AT_FIELD,
+  FILES_CATEGORIES_FIELD,
+  FILES_CLONED_AT_FIELD,
   FILES_CONTENT_LENGTH_FIELD,
-  FILES_HAS_NFT,
-  FILES_NAME_FIELD,
-  FILES_NAME_NORMALIZED_FIELD,
+  FILES_DATA,
   FILES_DESCRIPTION_FIELD,
-  FILES_WEBSITE_FIELD,
-  FILES_PARENT_FIELD,
-  FILES_CLONED_AT_FIELD,
-  FILES_IMMUTABLE_FIELD,
+  FILES_DIRECT_ONLY_FIELD,
   FILES_HAS_CLONES_FIELD,
-  FILES_IS_CLONE_FIELD,
-  FILES_NFT_OWNER_FIELD,
-  FILES_NFT_COLLECTION_FIELD,
-  FILES_NFT_TOKEN_FIELD,
-  FILES_NFT_TOKEN_AMOUNT_FIELD,
   FILES_HAS_NFT_OWNER_FIELD,
+  FILES_HAS_NFT,
   FILES_HAS_REFERENCES_FIELD,
-  FILES_IS_REFERENCED_FIELD,
+  FILES_ID_FIELD,
+  FILES_IMMUTABLE_FIELD,
+  FILES_IS_CLONE_FIELD,
   FILES_IS_IN_SHOWROOM_FIELD,
+  FILES_IS_REFERENCED_FIELD,
   FILES_LIST_SEARCH,
-  FILES_CATEGORIES_FIELD,
+  FILES_NAME_FIELD,
+  FILES_NAME_NORMALIZED_FIELD,
+  FILES_NFT_COLLECTION_FIELD,
+  FILES_NFT_OWNER_FIELD,
+  FILES_NFT_TOKEN_AMOUNT_FIELD,
+  FILES_NFT_TOKEN_FIELD,
+  FILES_OWNER_FIELD,
+  FILES_PACKED_FIELD,
+  FILES_PARENT_FIELD,
+  FILES_PARTS_FIELD,
+  FILES_PUBLIC_FIELD,
+  FILES_STATUS_FIELD,
+  FILES_TAGS_FIELD,
+  FILES_TEMP_FIELD,
+  FILES_UNLISTED_FIELD,
+  FILES_UPLOAD_STARTED_AT_FIELD,
+  FILES_UPLOAD_TYPE_FIELD,
+  FILES_UPLOADED_AT_FIELD,
+  FILES_WEBSITE_FIELD,
 } = require('../../constant');
 
 const FIELD_TO_TYPE = [
-  [FILES_ID_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_ALIAS_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
   [FILES_ALIAS_FIELD, 'AS', 'alias_tag', 'TAG', 'SORTABLE'],
-  [FILES_OWNER_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_PUBLIC_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_PACKED_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_TAGS_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
-  [FILES_STATUS_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_PARTS_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_UNLISTED_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_DIRECT_ONLY_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_UPLOADED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_UPLOAD_STARTED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_ALIAS_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
+  [FILES_CATEGORIES_FIELD, 'TAG'],
+  [FILES_CLONED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
   [FILES_CONTENT_LENGTH_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_HAS_NFT, 'NUMERIC', 'SORTABLE'],
-  [FILES_TEMP_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_NAME_NORMALIZED_FIELD, 'AS', FILES_NAME_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
   [FILES_DESCRIPTION_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
-  [FILES_WEBSITE_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
-  [FILES_IS_IN_SHOWROOM_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_PARENT_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_CLONED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_IMMUTABLE_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_DIRECT_ONLY_FIELD, 'NUMERIC', 'SORTABLE'],
   [FILES_HAS_CLONES_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_IS_CLONE_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_NFT_OWNER_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_NFT_COLLECTION_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_NFT_TOKEN_FIELD, 'TAG', 'SORTABLE'],
-  [FILES_NFT_TOKEN_AMOUNT_FIELD, 'NUMERIC', 'SORTABLE'],
   [FILES_HAS_NFT_OWNER_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_HAS_NFT, 'NUMERIC', 'SORTABLE'],
   [FILES_HAS_REFERENCES_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_ID_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_IMMUTABLE_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_IS_CLONE_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_IS_IN_SHOWROOM_FIELD, 'TAG', 'SORTABLE'],
   [FILES_IS_REFERENCED_FIELD, 'NUMERIC', 'SORTABLE'],
-  [FILES_CATEGORIES_FIELD, 'TAG'],
+  [FILES_NAME_NORMALIZED_FIELD, 'AS', FILES_NAME_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
+  [FILES_NFT_COLLECTION_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_NFT_OWNER_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_NFT_TOKEN_AMOUNT_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_NFT_TOKEN_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_OWNER_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_PACKED_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_PARENT_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_PARTS_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_PUBLIC_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_STATUS_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_TAGS_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
+  [FILES_TEMP_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_UNLISTED_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_UPLOAD_STARTED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_UPLOAD_TYPE_FIELD, 'TAG', 'SORTABLE'],
+  [FILES_UPLOADED_AT_FIELD, 'NUMERIC', 'SORTABLE'],
+  [FILES_WEBSITE_FIELD, 'TEXT', 'NOSTEM', 'SORTABLE'],
 ];
 
 // https://redis.io/docs/stack/search/reference/aggregations/#filter-expressions
@@ -95,5 +97,5 @@ async function createSearchIndexV2(service) {
 module.exports = {
   script: createSearchIndexV2,
   min: 13,
-  final: 15,
+  final: 16,
 };

test/helpers/insert-data.js

@@ -3,28 +3,29 @@ const Promise = require('bluebird');
 const { faker } = require('@faker-js/faker');
 const uuid = require('uuid');
 const {
-  STATUS_PROCESSED,
+  FILES_ALIAS_FIELD,
+  FILES_CONTENT_LENGTH_FIELD,
   FILES_DATA,
-  FILES_INDEX,
+  FILES_DESCRIPTION_FIELD,
+  FILES_ID_FIELD,
   FILES_INDEX_PUBLIC,
+  FILES_INDEX_UAT_PUBLIC,
+  FILES_INDEX_UAT,
+  FILES_INDEX,
+  FILES_NAME_FIELD,
+  FILES_NAME_NORMALIZED_FIELD,
   FILES_OWNER_FIELD,
   FILES_PUBLIC_FIELD,
+  FILES_UPLOAD_STARTED_AT_FIELD,
+  FILES_UPLOAD_TYPE_FIELD,
+  FILES_UPLOADED_AT_FIELD,
   FILES_USER_INDEX_KEY,
   FILES_USER_INDEX_PUBLIC_KEY,
-  FILES_UPLOADED_AT_FIELD,
-  FILES_INDEX_UAT,
-  FILES_INDEX_UAT_PUBLIC,
   FILES_USER_INDEX_UAT_KEY,
   FILES_USER_INDEX_UAT_PUBLIC_KEY,
-  FILES_UPLOAD_STARTED_AT_FIELD,
-  FILES_ID_FIELD,
-  FILES_CONTENT_LENGTH_FIELD,
-  FILES_ALIAS_FIELD,
   FILES_USR_ALIAS_PTR,
-  FILES_NAME_FIELD,
-  FILES_NAME_NORMALIZED_FIELD,
-  FILES_DESCRIPTION_FIELD,
   FILES_WEBSITE_FIELD,
+  STATUS_PROCESSED,
 } = require('../../src/constant');
 const { normalizeForSearch } = require('../../src/utils/normalize-name');
 
@@ -38,7 +39,7 @@ function createFakeFile({ owners, statuses }) {
 
   let sku = false;
   while (!sku) {
-    const tempSku = faker.random.word();
+    const tempSku = faker.lorem.word();
     if (!skus.has(tempSku)) {
       skus.add(tempSku);
       sku = tempSku;
@@ -52,19 +53,20 @@ function createFakeFile({ owners, statuses }) {
   names.add(name);
 
   return {
+    [FILES_ALIAS_FIELD]: sku,
+    [FILES_CONTENT_LENGTH_FIELD]: ld.random(1, 2132311),
+    [FILES_DESCRIPTION_FIELD]: faker.commerce.productDescription(), // so it doesn't product unexpected results
     [FILES_ID_FIELD]: id,
-    status: ld.sample(statuses),
-    [FILES_UPLOAD_STARTED_AT_FIELD]: startedAt,
-    [FILES_UPLOADED_AT_FIELD]: startedAt + 1000,
     [FILES_NAME_FIELD]: name,
     [FILES_NAME_NORMALIZED_FIELD]: normalizeForSearch(name),
-    files: JSON.stringify([]), // can insert real files, but dont care
-    [FILES_CONTENT_LENGTH_FIELD]: ld.random(1, 2132311),
-    parts: ld.random(1, 4),
     [FILES_OWNER_FIELD]: owner,
-    [FILES_ALIAS_FIELD]: sku,
-    [FILES_DESCRIPTION_FIELD]: faker.commerce.productDescription(), // so it doesn't product unexpected results
+    [FILES_UPLOAD_STARTED_AT_FIELD]: startedAt,
+    [FILES_UPLOAD_TYPE_FIELD]: 'simple',
+    [FILES_UPLOADED_AT_FIELD]: startedAt + 1000,
     [FILES_WEBSITE_FIELD]: `https://${faker.internet.domainName()}`,
+    files: JSON.stringify([]), // can insert real files, but dont care
+    parts: ld.random(1, 4),
+    status: ld.sample(statuses),
   };
 }
 

test/helpers/utils.js

@@ -111,6 +111,7 @@ function modelMessage(model, textures, preview, owner) {
       ...texturesMessage,
       previewMessage,
     ],
+    uploadType: 'simple',
   };
 
   return {

test/suites/list.js

@@ -221,6 +221,15 @@ for (const redisSearchEnabled of [true, false].values()) {
           }
         });
       });
+
+      it('should be able to filter by uploadType', async function test() {
+        const data = await this.amqp.publishAndWait('files.list', {
+          filter: { uploadType: { ne: 'simple' } },
+          order: 'DESC',
+        });
+
+        assert.strictEqual(data.files.length, 0);
+      });
     });
 
     describe('owner-based list', function testSuite() {