From 2bcca4f29c8d195099685905d9f769ece6ef8928 Mon Sep 17 00:00:00 2001 From: Semih Serhat Karakaya Date: Tue, 12 Jun 2018 17:30:59 +0300 Subject: [PATCH] [NEW] WebDAV(Nextcloud/ownCloud) Storage Server Option (#11027) Closes #9562, Part of #7791 Implementation of ownCloud/Nextcloud as file upload store feature. Since it is using generic webdav client, the store compatible with all WebDAV servers (NextCloud, ownCloud and so on). ![image](https://user-images.githubusercontent.com/14157973/41051962-3a63ef5e-69c0-11e8-9a8a-76ed36308dfb.png) --- package.json | 1 + .../server/config/Webdav.js | 62 ++++++++ .../server/config/_configUploadStorage.js | 1 + .../server/startup/settings.js | 49 +++++++ .../ufs/Webdav/client.js | 6 + .../ufs/Webdav/server.js | 135 ++++++++++++++++++ packages/rocketchat-i18n/i18n/en.i18n.json | 9 ++ 7 files changed, 263 insertions(+) create mode 100644 packages/rocketchat-file-upload/server/config/Webdav.js create mode 100644 packages/rocketchat-file-upload/ufs/Webdav/client.js create mode 100644 packages/rocketchat-file-upload/ufs/Webdav/server.js diff --git a/package.json b/package.json index fe302a67d62d..cde15f42e141 100644 --- a/package.json +++ b/package.json @@ -175,6 +175,7 @@ "ua-parser-js": "^0.7.18", "underscore": "^1.9.1", "underscore.string": "^3.3.4", + "webdav": "^1.5.2", "wolfy87-eventemitter": "^5.2.4", "xml-crypto": "^0.10.1", "xml2js": "^0.4.19", diff --git a/packages/rocketchat-file-upload/server/config/Webdav.js b/packages/rocketchat-file-upload/server/config/Webdav.js new file mode 100644 index 000000000000..1b2ce4d7449b --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/Webdav.js @@ -0,0 +1,62 @@ +/* globals FileUpload */ + +import _ from 'underscore'; +import { FileUploadClass } from '../lib/FileUpload'; +import '../../ufs/Webdav/server.js'; + +const get = function(file, req, res) { + this.store.getReadStream(file._id, file).pipe(res); +}; + +const copy = function(file, out) { + this.store.getReadStream(file._id, file).pipe(out); +}; + +const WebdavUploads = new FileUploadClass({ + name: 'Webdav:Uploads', + get, + copy + // store setted bellow +}); + +const WebdavAvatars = new FileUploadClass({ + name: 'Webdav:Avatars', + get, + copy + // store setted bellow +}); + +const WebdavUserDataFiles = new FileUploadClass({ + name: 'Webdav:UserDataFiles', + get, + copy + // store setted bellow +}); + +const configure = _.debounce(function() { + const uploadFolderPath = RocketChat.settings.get('FileUpload_Webdav_Upload_Folder_Path'); + const server = RocketChat.settings.get('FileUpload_Webdav_Server_URL'); + const username = RocketChat.settings.get('FileUpload_Webdav_Username'); + const password = RocketChat.settings.get('FileUpload_Webdav_Password'); + + if (!server || !username || !password) { + return; + } + + const config = { + connection: { + credentials: { + server, + username, + password + } + }, + uploadFolderPath + }; + + WebdavUploads.store = FileUpload.configureUploadsStore('Webdav', WebdavUploads.name, config); + WebdavAvatars.store = FileUpload.configureUploadsStore('Webdav', WebdavAvatars.name, config); + WebdavUserDataFiles.store = FileUpload.configureUploadsStore('Webdav', WebdavUserDataFiles.name, config); +}, 500); + +RocketChat.settings.get(/^FileUpload_Webdav_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js index 8855eb76fca8..44b0587a0740 100644 --- a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js +++ b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js @@ -5,6 +5,7 @@ import './AmazonS3.js'; import './FileSystem.js'; import './GoogleStorage.js'; import './GridFS.js'; +import './Webdav.js'; import './Slingshot_DEPRECATED.js'; const configStore = _.debounce(() => { diff --git a/packages/rocketchat-file-upload/server/startup/settings.js b/packages/rocketchat-file-upload/server/startup/settings.js index 9e134d98abc1..60bf6261be35 100644 --- a/packages/rocketchat-file-upload/server/startup/settings.js +++ b/packages/rocketchat-file-upload/server/startup/settings.js @@ -32,6 +32,9 @@ RocketChat.settings.addGroup('FileUpload', function() { }, { key: 'GoogleCloudStorage', i18nLabel: 'GoogleCloudStorage' + }, { + key: 'Webdav', + i18nLabel: 'WebDAV' }, { key: 'FileSystem', i18nLabel: 'FileSystem' @@ -180,6 +183,52 @@ RocketChat.settings.addGroup('FileUpload', function() { }); }); + this.section('WebDAV', function() { + this.add('FileUpload_Webdav_Upload_Folder_Path', '', { + type: 'string', + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + this.add('FileUpload_Webdav_Server_URL', '', { + type: 'string', + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + this.add('FileUpload_Webdav_Username', '', { + type: 'string', + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + this.add('FileUpload_Webdav_Password', '', { + type: 'password', + private: true, + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + this.add('FileUpload_Webdav_Proxy_Avatars', false, { + type: 'boolean', + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + this.add('FileUpload_Webdav_Proxy_Uploads', false, { + type: 'boolean', + enableQuery: { + _id: 'FileUpload_Storage_Type', + value: 'Webdav' + } + }); + }); + this.add('FileUpload_Enabled_Direct', true, { type: 'boolean', public: true diff --git a/packages/rocketchat-file-upload/ufs/Webdav/client.js b/packages/rocketchat-file-upload/ufs/Webdav/client.js new file mode 100644 index 000000000000..04808d27c501 --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/Webdav/client.js @@ -0,0 +1,6 @@ +import {UploadFS} from 'meteor/jalik:ufs'; + +export class WebdavStore extends UploadFS.Store {} + +// Add store to UFS namespace +UploadFS.store.Webdav = WebdavStore; diff --git a/packages/rocketchat-file-upload/ufs/Webdav/server.js b/packages/rocketchat-file-upload/ufs/Webdav/server.js new file mode 100644 index 000000000000..ea261e3784eb --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/Webdav/server.js @@ -0,0 +1,135 @@ +import {UploadFS} from 'meteor/jalik:ufs'; +import Webdav from 'webdav'; +import stream from 'stream'; +/** + * WebDAV store + * @param options + * @constructor + */ +export class WebdavStore extends UploadFS.Store { + + constructor(options) { + + super(options); + + + const client = new Webdav( + options.connection.credentials.server, + options.connection.credentials.username, + options.connection.credentials.password, + ); + + options.getPath = function(file) { + if (options.uploadFolderPath[options.uploadFolderPath.length-1] !== '/') { + options.uploadFolderPath += '/'; + } + return options.uploadFolderPath + file._id; + }; + + client.stat(options.uploadFolderPath).catch(function(err) { + if (err.status === '404') { + client.createDirectory(options.uploadFolderPath); + } + }); + + /** + * Returns the file path + * @param file + * @return {string} + */ + this.getPath = function(file) { + if (file.Webdav) { + return file.Webdav.path; + } + }; + + /** + * Creates the file in the col lection + * @param file + * @param callback + * @return {string} + */ + this.create = function(file, callback) { + check(file, Object); + + if (file._id == null) { + file._id = Random.id(); + } + + file.Webdav = { + path: options.getPath(file) + }; + + file.store = this.options.name; + return this.getCollection().insert(file, callback); + }; + + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + const file = this.getCollection().findOne({_id: fileId}); + client.deleteFile(this.getPath(file), (err, data) => { + if (err) { + console.error(err); + } + + callback && callback(err, data); + }); + }; + + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getReadStream = function(fileId, file, options = {}) { + const range = {}; + + if (options.start != null) { + range.start = options.start; + } + + if (options.end != null) { + range.end = options.end; + } + return client.createReadStream(this.getPath(file), options); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @return {*} + */ + this.getWriteStream = function(fileId, file) { + const writeStream = new stream.PassThrough(); + const webdavStream = client.createWriteStream(this.getPath(file)); + + //TODO remove timeout when UploadFS bug resolved + const newListenerCallback = (event, listener) => { + if (event === 'finish') { + process.nextTick(() => { + writeStream.removeListener(event, listener); + writeStream.removeListener('newListener', newListenerCallback); + writeStream.on(event, function() { + setTimeout(listener, 500); + }); + }); + } + }; + writeStream.on('newListener', newListenerCallback); + + writeStream.pipe(webdavStream); + return writeStream; + }; + + } +} + +// Add store to UFS namespace +UploadFS.store.Webdav = WebdavStore; diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index b00415579725..0eee2eb272d4 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -1141,6 +1141,15 @@ "FileUpload_S3_URLExpiryTimeSpan": "URLs Expiration Timespan", "FileUpload_S3_URLExpiryTimeSpan_Description": "Time after which Amazon S3 generated URLs will no longer be valid (in seconds). If set to less than 5 seconds, this field will be ignored.", "FileUpload_Storage_Type": "Storage Type", + "FileUpload_Webdav_Upload_Folder_Path": "Upload Folder Path", + "FileUpload_Webdav_Upload_Folder_Path_Description": "WebDAV folder path which the files should be uploaded to", + "FileUpload_Webdav_Server_URL": "WebDAV Server Access URL", + "FileUpload_Webdav_Username": "WebDAV Username", + "FileUpload_Webdav_Password": "WebDAV Password", + "FileUpload_Webdav_Proxy_Avatars": "Proxy Avatars", + "FileUpload_Webdav_Proxy_Avatars_Description": "Proxy avatar file transmissions through your server instead of direct access to the asset's URL", + "FileUpload_Webdav_Proxy_Uploads": "Proxy Uploads", + "FileUpload_Webdav_Proxy_Uploads_Description": "Proxy upload file transmissions through your server instead of direct access to the asset's URL", "Financial_Services": "Financial Services", "First_Channel_After_Login": "First Channel After Login", "Flags": "Flags",