From f24a8d560e009d172ed6eb436e9a80a050795c99 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Wed, 5 Jul 2017 18:15:40 -0300 Subject: [PATCH 1/4] Run avatar migration after server startup --- server/startup/migrations/v097.js | 116 +++++++++++++++--------------- 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/server/startup/migrations/v097.js b/server/startup/migrations/v097.js index 1ec5ae82c8ef..a283984d5bb4 100644 --- a/server/startup/migrations/v097.js +++ b/server/startup/migrations/v097.js @@ -117,75 +117,77 @@ RocketChat.Migrations.add({ const avatarsPath = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStorePath'}).value; const avatarStoreType = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStoreType'}).value; - Meteor.setTimeout(function() { - const avatarsFileStore = FileUpload.getStore('Avatars'); + Meteor.startup(function() { + Meteor.setTimeout(function() { + const avatarsFileStore = FileUpload.getStore('Avatars'); - const oldAvatarGridFS = new RocketChatFile.GridFS({ - name: 'avatars' - }); + const oldAvatarGridFS = new RocketChatFile.GridFS({ + name: 'avatars' + }); - const users = RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).fetch(); + const users = RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).fetch(); - const usersTotal = users.length; + const usersTotal = users.length; - log('Total users to migrate avatars ->', usersTotal); + log('Total users to migrate avatars ->', usersTotal); - let current = 0; + let current = 0; - batch(users, 300, (user) => { - const id = `${ user.username }.jpg`; + batch(users, 300, (user) => { + const id = `${ user.username }.jpg`; - const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); + const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); - log('Migrating', ++current, 'of', usersTotal); + log('Migrating', ++current, 'of', usersTotal); - if (gridFSAvatar) { - const details = { - userId: user._id, - type: gridFSAvatar.contentType, - size: gridFSAvatar.length - }; + if (gridFSAvatar) { + const details = { + userId: user._id, + type: gridFSAvatar.contentType, + size: gridFSAvatar.length + }; - return insertAvatar({ - details, - avatarsFileStore, - stream: gridFSAvatar.readStream, - callback() { - oldAvatarGridFS.deleteFile(id); - } - }); - } - if (avatarStoreType === 'FileSystem' && avatarsPath && avatarsPath.trim()) { - try { - const filePath = path.join(avatarsPath, id); - const stat = fs.statSync(filePath); - if (stat && stat.isFile()) { - const details = { - userId: user._id, - type: 'image/jpeg', - size: stat.size - }; - return insertAvatar({ - details, - avatarsFileStore, - stream: fs.createReadStream(filePath), - callback() { - fs.unlinkSync(filePath); - } - }); + return insertAvatar({ + details, + avatarsFileStore, + stream: gridFSAvatar.readStream, + callback() { + oldAvatarGridFS.deleteFile(id); + } + }); + } + if (avatarStoreType === 'FileSystem' && avatarsPath && avatarsPath.trim()) { + try { + const filePath = path.join(avatarsPath, id); + const stat = fs.statSync(filePath); + if (stat && stat.isFile()) { + const details = { + userId: user._id, + type: 'image/jpeg', + size: stat.size + }; + return insertAvatar({ + details, + avatarsFileStore, + stream: fs.createReadStream(filePath), + callback() { + fs.unlinkSync(filePath); + } + }); + } + } catch (e) { + logError('Error migrating old avatar', e); + return Promise.resolve(); } - } catch (e) { - logError('Error migrating old avatar', e); - return Promise.resolve(); } - } - }).then(() => { - const avatarsFiles = new Mongo.Collection('avatars.files'); - const avatarsChunks = new Mongo.Collection('avatars.chunks'); - avatarsFiles.rawCollection().drop(); - avatarsChunks.rawCollection().drop(); - }); - }, 1000); + }).then(() => { + const avatarsFiles = new Mongo.Collection('avatars.files'); + const avatarsChunks = new Mongo.Collection('avatars.chunks'); + avatarsFiles.rawCollection().drop(); + avatarsChunks.rawCollection().drop(); + }); + }, 1000); + }); RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); } From 892b469878cd08ec6915f5ff76c5792476c634c0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 5 Jul 2017 22:01:55 -0300 Subject: [PATCH 2/4] Move migration 97 to 99 --- packages/rocketchat-file/file.server.js | 1 + server/startup/migrations/v097.js | 190 +-------------------- server/startup/migrations/v099.js | 217 ++++++++++++++++++++++++ 3 files changed, 219 insertions(+), 189 deletions(-) create mode 100644 server/startup/migrations/v099.js diff --git a/packages/rocketchat-file/file.server.js b/packages/rocketchat-file/file.server.js index 500513a58e54..1fb0487184e6 100644 --- a/packages/rocketchat-file/file.server.js +++ b/packages/rocketchat-file/file.server.js @@ -105,6 +105,7 @@ RocketChatFile.GridFS = class { this.store = new Grid(db, mongo); this.findOneSync = Meteor.wrapAsync(this.store.collection(this.name).findOne.bind(this.store.collection(this.name))); this.removeSync = Meteor.wrapAsync(this.store.remove.bind(this.store)); + this.countSync = Meteor.wrapAsync(this.store._col.count.bind(this.store._col)); this.getFileSync = Meteor.wrapAsync(this.getFile.bind(this)); } diff --git a/server/startup/migrations/v097.js b/server/startup/migrations/v097.js index a283984d5bb4..bd6e421ac7a1 100644 --- a/server/startup/migrations/v097.js +++ b/server/startup/migrations/v097.js @@ -1,194 +1,6 @@ -import fs from 'fs'; -import path from 'path'; - -function log(...args) { - console.log('[AVATAR]', ...args); -} - -function logError(...args) { - console.error('[AVATAR]', ...args); -} - -function insertAvatar({ details, avatarsFileStore, stream, callback = () => {} }) { - return new Promise((resolve) => { - Meteor.defer(() => { - Meteor.runAsUser('rocket.cat', () => { - avatarsFileStore.insert(details, stream, (err) => { - if (err) { - logError({err}); - resolve(); - } else { - Meteor.setTimeout(() => { - callback(); - }, 200); - } - resolve(); - }); - }); - }); - }); -} - -function batch(arr, limit, fn) { - if (!arr.length) { - return Promise.resolve(); - } - return Promise.all(arr.splice(0, limit).map((item) => { - return fn(item); - })).then(() => { return batch(arr, limit, fn); }); -} - RocketChat.Migrations.add({ version: 97, up() { - log('Migrating avatars. This might take a while.'); - - const query = { - $or: [{ - 's3.path': { - $exists: true - } - }, { - 'googleCloudStorage.path': { - $exists: true - } - }] - }; - - RocketChat.models.Uploads.find(query).forEach((record) => { - if (record.s3) { - RocketChat.models.Uploads.model.direct.update({_id: record._id}, { - $set: { - 'store': 'AmazonS3:Uploads', - AmazonS3: { - path: record.s3.path + record._id - } - }, - $unset: { - s3: 1 - } - }, {multi: true}); - } else { - RocketChat.models.Uploads.model.direct.update({_id: record._id}, { - $set: { - store: 'GoogleCloudStorage:Uploads', - GoogleStorage: { - path: record.googleCloudStorage.path + record._id - } - }, - $unset: { - googleCloudStorage: 1 - } - }, {multi: true}); - } - }); - - RocketChat.models.Uploads.model.direct.update({ - store: 'fileSystem' - }, { - $set: { - store: 'FileSystem:Uploads' - } - }, { - multi: true - }); - RocketChat.models.Uploads.model.direct.update({ - store: 'rocketchat_uploads' - }, { - $set: { - store: 'GridFS:Uploads' - } - }, { - multi: true - }); - - const avatarOrigins = [ - 'upload', - 'gravatar', - 'facebook', - 'twitter', - 'github', - 'google', - 'url', - 'gitlab', - 'linkedin' - ]; - - const avatarsPath = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStorePath'}).value; - const avatarStoreType = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStoreType'}).value; - - Meteor.startup(function() { - Meteor.setTimeout(function() { - const avatarsFileStore = FileUpload.getStore('Avatars'); - - const oldAvatarGridFS = new RocketChatFile.GridFS({ - name: 'avatars' - }); - - const users = RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).fetch(); - - const usersTotal = users.length; - - log('Total users to migrate avatars ->', usersTotal); - - let current = 0; - - batch(users, 300, (user) => { - const id = `${ user.username }.jpg`; - - const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); - - log('Migrating', ++current, 'of', usersTotal); - - if (gridFSAvatar) { - const details = { - userId: user._id, - type: gridFSAvatar.contentType, - size: gridFSAvatar.length - }; - - return insertAvatar({ - details, - avatarsFileStore, - stream: gridFSAvatar.readStream, - callback() { - oldAvatarGridFS.deleteFile(id); - } - }); - } - if (avatarStoreType === 'FileSystem' && avatarsPath && avatarsPath.trim()) { - try { - const filePath = path.join(avatarsPath, id); - const stat = fs.statSync(filePath); - if (stat && stat.isFile()) { - const details = { - userId: user._id, - type: 'image/jpeg', - size: stat.size - }; - return insertAvatar({ - details, - avatarsFileStore, - stream: fs.createReadStream(filePath), - callback() { - fs.unlinkSync(filePath); - } - }); - } - } catch (e) { - logError('Error migrating old avatar', e); - return Promise.resolve(); - } - } - }).then(() => { - const avatarsFiles = new Mongo.Collection('avatars.files'); - const avatarsChunks = new Mongo.Collection('avatars.chunks'); - avatarsFiles.rawCollection().drop(); - avatarsChunks.rawCollection().drop(); - }); - }, 1000); - }); - RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); - RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); + // Migration moved to 099.js to fix a bug } }); diff --git a/server/startup/migrations/v099.js b/server/startup/migrations/v099.js new file mode 100644 index 000000000000..43344c5e5d03 --- /dev/null +++ b/server/startup/migrations/v099.js @@ -0,0 +1,217 @@ +/* globals SystemLogger */ + +import fs from 'fs'; +import path from 'path'; + +function log(...args) { + console.log('[AVATAR]', ...args); +} + +function logError(...args) { + console.error('[AVATAR]', ...args); +} + +function insertAvatar({ details, avatarsFileStore, stream, callback = () => {} }) { + return new Promise((resolve) => { + Meteor.defer(() => { + Meteor.runAsUser('rocket.cat', () => { + avatarsFileStore.insert(details, stream, (err) => { + if (err) { + logError({err}); + resolve(); + } else { + Meteor.setTimeout(() => { + callback(); + }, 200); + } + resolve(); + }); + }); + }); + }); +} + +function batch(arr, limit, fn) { + if (!arr.length) { + return Promise.resolve(); + } + return Promise.all(arr.splice(0, limit).map((item) => { + return fn(item); + })).then(() => { return batch(arr, limit, fn); }); +} + +RocketChat.Migrations.add({ + version: 99, + up() { + log('Migrating avatars. This might take a while.'); + + const query = { + $or: [{ + 's3.path': { + $exists: true + } + }, { + 'googleCloudStorage.path': { + $exists: true + } + }] + }; + + RocketChat.models.Uploads.find(query).forEach((record) => { + if (record.s3) { + RocketChat.models.Uploads.model.direct.update({_id: record._id}, { + $set: { + 'store': 'AmazonS3:Uploads', + AmazonS3: { + path: record.s3.path + record._id + } + }, + $unset: { + s3: 1 + } + }, {multi: true}); + } else { + RocketChat.models.Uploads.model.direct.update({_id: record._id}, { + $set: { + store: 'GoogleCloudStorage:Uploads', + GoogleStorage: { + path: record.googleCloudStorage.path + record._id + } + }, + $unset: { + googleCloudStorage: 1 + } + }, {multi: true}); + } + }); + + RocketChat.models.Uploads.model.direct.update({ + store: 'fileSystem' + }, { + $set: { + store: 'FileSystem:Uploads' + } + }, { + multi: true + }); + RocketChat.models.Uploads.model.direct.update({ + store: 'rocketchat_uploads' + }, { + $set: { + store: 'GridFS:Uploads' + } + }, { + multi: true + }); + + const avatarOrigins = [ + 'upload', + 'gravatar', + 'facebook', + 'twitter', + 'github', + 'google', + 'url', + 'gitlab', + 'linkedin' + ]; + + const avatarsPathRecord = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStorePath'}); + const avatarStoreTypeRecord = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStoreType'}); + + const avatarsPath = avatarsPathRecord ? avatarsPathRecord.value : process.env.AVATARS_PATH; + let avatarStoreType = avatarStoreTypeRecord && avatarStoreTypeRecord.value; + + const oldAvatarGridFS = new RocketChatFile.GridFS({ + name: 'avatars' + }); + + // 1. Novo usuário, não executa migration + // 2. Antigo, migra de antes da 97, vai ter storage type + // 3. Executou a 97, não vai ter storeType + // 3.1 Se ainda tem dados no gridfs, manda ver + // 3.2 Se não tem, quebra e pede a variável OU forçar a reexecutar a 97??? + + if (avatarStoreType == null) { + const count = oldAvatarGridFS.countSync(); + if (Match.test(count, Number) && count > 0) { + avatarStoreType = 'GridFS'; + } else if (Match.test(avatarsPath, String) && avatarsPath.length > 0) { + avatarStoreType = 'FileSystem'; + } else { + SystemLogger.error_box('Can\'t define the avatar\'s storage type.\nIf you have avatars missing and they was stored in your file system\nrun the process including the following environment variables: \n AVATARS_PATH=\'YOUR AVATAR\'S DIRECTORY\'\n MIGRATION_VERSION=98', 'WARNING'); + return; + } + } + + Meteor.startup(function() { + Meteor.setTimeout(function() { + const avatarsFileStore = FileUpload.getStore('Avatars'); + + const users = RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).fetch(); + + const usersTotal = users.length; + + log('Total users to migrate avatars ->', usersTotal); + + let current = 0; + + batch(users, 300, (user) => { + const id = `${ user.username }.jpg`; + + const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); + + log('Migrating', ++current, 'of', usersTotal); + + if (gridFSAvatar) { + const details = { + userId: user._id, + type: gridFSAvatar.contentType, + size: gridFSAvatar.length + }; + + return insertAvatar({ + details, + avatarsFileStore, + stream: gridFSAvatar.readStream, + callback() { + oldAvatarGridFS.deleteFile(id); + } + }); + } + if (avatarStoreType === 'FileSystem' && avatarsPath && avatarsPath.trim()) { + try { + const filePath = path.join(avatarsPath, id); + const stat = fs.statSync(filePath); + if (stat && stat.isFile()) { + const details = { + userId: user._id, + type: 'image/jpeg', + size: stat.size + }; + return insertAvatar({ + details, + avatarsFileStore, + stream: fs.createReadStream(filePath), + callback() { + fs.unlinkSync(filePath); + } + }); + } + } catch (e) { + logError('Error migrating old avatar', e); + return Promise.resolve(); + } + } + }).then(() => { + const avatarsFiles = new Mongo.Collection('avatars.files'); + const avatarsChunks = new Mongo.Collection('avatars.chunks'); + avatarsFiles.rawCollection().drop(); + avatarsChunks.rawCollection().drop(); + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); + }); + }, 1000); + }); + } +}); From f9ad156717ca86719cb6edf29eebdb56b1bef206 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 5 Jul 2017 22:17:18 -0300 Subject: [PATCH 3/4] Remove unecessary comments --- server/startup/migrations/v099.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/startup/migrations/v099.js b/server/startup/migrations/v099.js index 43344c5e5d03..3d77e4a5e765 100644 --- a/server/startup/migrations/v099.js +++ b/server/startup/migrations/v099.js @@ -126,12 +126,6 @@ RocketChat.Migrations.add({ name: 'avatars' }); - // 1. Novo usuário, não executa migration - // 2. Antigo, migra de antes da 97, vai ter storage type - // 3. Executou a 97, não vai ter storeType - // 3.1 Se ainda tem dados no gridfs, manda ver - // 3.2 Se não tem, quebra e pede a variável OU forçar a reexecutar a 97??? - if (avatarStoreType == null) { const count = oldAvatarGridFS.countSync(); if (Match.test(count, Number) && count > 0) { From 47dad0da400dea17a24d4c2e21d029f350a15dec Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 5 Jul 2017 22:41:03 -0300 Subject: [PATCH 4/4] Fix the migration `rerun` command --- packages/rocketchat-migrations/migrations.js | 2 +- server/startup/migrations/v099.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rocketchat-migrations/migrations.js b/packages/rocketchat-migrations/migrations.js index ee54c145e696..82885469da8b 100644 --- a/packages/rocketchat-migrations/migrations.js +++ b/packages/rocketchat-migrations/migrations.js @@ -239,7 +239,7 @@ Migrations._migrateTo = function(version, rerun) { if (rerun) { log.info('Rerunning version ' + version); - migrate('up', version); + migrate('up', this._findIndexByVersion(version)); log.info('Finished migrating.'); unlock(); return true; diff --git a/server/startup/migrations/v099.js b/server/startup/migrations/v099.js index 3d77e4a5e765..5e9334103326 100644 --- a/server/startup/migrations/v099.js +++ b/server/startup/migrations/v099.js @@ -133,7 +133,7 @@ RocketChat.Migrations.add({ } else if (Match.test(avatarsPath, String) && avatarsPath.length > 0) { avatarStoreType = 'FileSystem'; } else { - SystemLogger.error_box('Can\'t define the avatar\'s storage type.\nIf you have avatars missing and they was stored in your file system\nrun the process including the following environment variables: \n AVATARS_PATH=\'YOUR AVATAR\'S DIRECTORY\'\n MIGRATION_VERSION=98', 'WARNING'); + SystemLogger.error_box('Can\'t define the avatar\'s storage type.\nIf you have avatars missing and they was stored in your file system\nrun the process including the following environment variables: \n AVATARS_PATH=\'YOUR AVATAR\'S DIRECTORY\'\n MIGRATION_VERSION=99,rerun', 'WARNING'); return; } }