feat: added manager options (path, params, getSignedUrl, filesize, aw…

…sAcl)

options.path is new, replacing options.bucketDir and options.filename

docs/image-plugin.md

@@ -10,11 +10,15 @@ To use the default image plugin shipped with monastery, you need to use the opti
 ```js
   let db = monastery('localhost/mydb', {
     imagePlugin: {
+      awsAcl: 'public-read', // default
       awsBucket: 'your-bucket-name',
       awsAccessKeyId: 'your-key-here',
       awsSecretAccessKey: 'your-key-here',
-      bucketDir: 'full', // default
-      formats: ['bmp', 'gif', 'jpg', 'jpeg', 'png', 'tiff'] // or 'any' to include everything
+      filesize: undefined, // default (max filesize in bytes)
+      formats: ['bmp', 'gif', 'jpg', 'jpeg', 'png', 'tiff'], // default (use 'any' to allow all extensions)
+      getSignedUrl: false, // default (get a S3 signed url after `model.find()`, can be defined per request)
+      path: (uid, basename, ext, file) => `/full/${uid}.${ext}`, // default
+      params: {}, // upload params (https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
     }
   })
 ```
@@ -25,15 +29,12 @@ Then in your model definition, e.g.
 let user = db.model('user', {
   fields: {
     logo:  {
-      type: 'image', // required
-      formats: ['bmp', 'gif', 'jpg', 'jpeg', 'png', 'tiff'],
-      filename: 'avatar',
-      filesize: 1000 * 1000 * 5, // max size in bytes
-      getSignedUrl: true, // get a s3 signed url after every `find()` operation (can be overridden per request)
-      params: {}, // upload params, https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
+      type: 'image',
+      // ...any imagePlugin option, excluding awsAccessKeyId and awsSecretAccessKey
     },
     logos: [{
-      type: 'image'
+      type: 'image',
+      // ...any imagePlugin option, excluding awsAccessKeyId and awsSecretAccessKey
     }],
   }
 })
@@ -61,6 +62,6 @@ user.update({
 
 ### File types
 
-Due to known limitations, we are inaccurately able to validate non-binary file types (e.g. txt, svg) before uploading to S3, and rely on their file processing to remove any malicious files.
+Due to known limitations, we are not able to verify the contents of non-binary files, only the filename extension (e.g. .txt, .svg) before uploading to S3
 
 ...to be continued

docs/manager/index.md

@@ -13,6 +13,8 @@ Monastery constructor, same as the [monk constructor](https://automattic.github.
 `uri` *(string\|array)*: A [mongo connection string URI](https://docs.mongodb.com/manual/reference/connection-string/). Replica sets can be an array or comma separated.
 
 [`options`] *(object)*:
+  - [`hideWarnings=false`] *(boolean)*: hide monastery warnings
+  - [`hideErrors=false`] *(boolean)*: hide monastery errors
   - [`defaultObjects=false`] *(boolean)*: when [inserting](../model/insert.html#defaults-example), undefined embedded documents and arrays are defined
   - [`nullObjects=false`] *(boolean)*: embedded documents and arrays can be set to null or an empty string (which gets converted to null). You can override this per field via `nullObject: true`.
   - [`timestamps=true`] *(boolean)*: whether to use [`createdAt` and `updatedAt`](../definition), this can be overridden per operation

docs/readme.md

@@ -65,12 +65,10 @@ db.user.insert({
 ```
 ## Debugging
 
-This package uses [Debug](https://github.com/visionmedia/debug) which allows you to see different levels of output:
+This package uses [debug](https://github.com/visionmedia/debug) which allows you to set different levels of output via the `DEBUG` environment variable. Due to known limations `monastery:warning` and `monastery:error` are forced on, you can however disable these via [manager settings](./manager).
 
 ```bash
-$ DEBUG=monastery:info
-# or show all levels of output, currently shows the same output as above
-$ DEBUG=monastery:*
+$ DEBUG=monastery:info # shows operation information
 ```
 
 To run isolated tests with Jest:

lib/index.js

@@ -16,13 +16,19 @@ module.exports = function(uri, opts, fn) {
    * @param {object} opts
    * @return monk manager
    */
+  if (!opts) opts = {}
   let monasteryOpts = [
-    'defaultObjects', 'imagePlugin', 'limit', 'nullObjects', 'timestamps', 'useMilliseconds'
+    'defaultObjects', 'hideWarnings', 'hideErrors', 'imagePlugin', 'limit', 'nullObjects',
+    'timestamps', 'useMilliseconds',
   ]
 
-  if (!opts) opts = {}
+  // Note: Debug doesn't allow debuggers to be enabled by default
+  let info = debug('monastery:info')
+  let warn = debug('monastery:warn' + (opts.hideWarnings ? '' : '*'))
+  let error = debug('monastery:error' + (opts.hideErrors ? '' : '*'))
+
   if (util.isDefined(opts.defaultFields)) {
-    var depreciationWarningDefaultField = true
+    warn('opts.defaultFields has been depreciated in favour of opts.timestamps')
     opts.timestamps = opts.defaultFields
     delete opts.defaultFields
   }
@@ -44,20 +50,15 @@ module.exports = function(uri, opts, fn) {
   manager.model = require('./model')
   manager.models = models
   manager.parseData = util.parseData.bind(util)
-  manager.warn = debug('monastery:warn*')
-  manager.error = debug('monastery:error*')
-  manager.info = debug('monastery:info')
+  manager.info = info
+  manager.warn = warn
+  manager.error = error
 
   // Add opts onto manager
   for (let key of monasteryOpts) {
     manager[key] = opts[key]
   }
 
-  // Depreciation warnings
-  if (depreciationWarningDefaultField) {
-    manager.error('opts.defaultFields has been depreciated in favour of opts.timestamps')
-  }
-
   // Initiate any plugins
   if (manager.imagePlugin) {
     manager.imagePluginFile.setup(manager, util.isObject(manager.imagePlugin)? manager.imagePlugin : {})

lib/model-validate.js

@@ -266,8 +266,9 @@ module.exports = {
 
   _ignoredRules: [ // todo: change name? i.e. 'specialFields'
     // Need to remove filesize and formats..
-    'default', 'defaultOverride', 'filename', 'filesize', 'fileSize', 'formats', 'image', 'index', 'insertOnly',
-    'model', 'nullObject', 'params', 'getSignedUrl', 'timestampField', 'type', 'virtual'
+    'awsAcl', 'awsBucket', 'default', 'defaultOverride', 'filename', 'filesize', 'fileSize', 'formats',
+    'image', 'index', 'insertOnly', 'model', 'nullObject', 'params', 'path', 'getSignedUrl', 'timestampField',
+    'type', 'virtual',
   ]
 
 }

package.json

@@ -17,7 +17,7 @@
     "mongo odm"
   ],
   "scripts": {
-    "dev": "npm run lint & DEBUG=-monastery:info jest --watchAll --runInBand --verbose=false",
+    "dev": "npm run lint & jest --watchAll --runInBand --verbose=false",
     "docs": "cd docs && bundle exec jekyll serve --livereload --livereload-port 4001",
     "lint": "eslint ./lib ./plugins ./test",
     "mong": "nodemon resources/mong.js",

plugins/images/index.js

@@ -18,13 +18,24 @@ let plugin = module.exports = {
      * @this plugin
      */
 
+    // Depreciation warnings
+    if (options.bucketDir) { // > 1.36.2
+      manager.warn('imagePlugin.bucketDir has been depreciated in favour of imagePlugin.path')
+      options.path = function (uid, basename, ext, file) { `${options.bucketDir}/${uid}.${ext}` }
+    }
+
     // Settings
+    this.awsAcl = options.awsAcl || 'public-read'
     this.awsBucket = options.awsBucket
     this.awsAccessKeyId = options.awsAccessKeyId
     this.awsSecretAccessKey = options.awsSecretAccessKey
-    this.bucketDir = options.bucketDir || 'full'
+    this.bucketDir = options.bucketDir || 'full' // depreciated > 1.36.2
+    this.filesize = options.filesize
     this.formats = options.formats || ['bmp', 'gif', 'jpg', 'jpeg', 'png', 'tiff']
+    this.getSignedUrl = options.getSignedUrl
     this.manager = manager
+    this.params = options.params ? util.deepCopy(options.params) : {},
+    this.path = options.path || function (uid, basename, ext, file) { return `full/${uid}.${ext}` }
 
     if (!options.awsBucket || !options.awsAccessKeyId || !options.awsSecretAccessKey) {
       manager.error('Monastery imagePlugin: awsBucket, awsAccessKeyId, or awsSecretAccessKey is not defined')
@@ -34,7 +45,7 @@ let plugin = module.exports = {
 
     // Create s3 'service' instance
     // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
-    manager.getSignedUrl = this._getSignedUrl
+    manager._getSignedUrl = this._getSignedUrl
     this.s3 = new S3({
       credentials: {
         accessKeyId: this.awsAccessKeyId,
@@ -114,9 +125,13 @@ let plugin = module.exports = {
     if (util.isArray(data)) {
       return Promise.reject('Adding images to mulitple data objects is not supported.')
 
+    // We currently don't support an array of data objects.
+    } else if (!util.isObject(data)) {
+      return Promise.reject('No creat e/ update data object passed to addImages?')
+
     // We require the update query OR data object to contain _id. This is because non-id queries
     // will not suffice when updating the document(s) against the same query again.
-    } else if (!data._id && (!query || !query._id)) {
+    } else if (!(data||{})._id && (!query || !query._id)) {
       return Promise.reject('Adding images requires the update operation to query via _id\'s.')
     }
 
@@ -126,48 +141,48 @@ let plugin = module.exports = {
         return Promise.all(filesArr.map(file => {
           return new Promise((resolve, reject) => {
             let uid = nanoid.nanoid()
-            let pathFilename = filesArr.imageField.filename ? '/' + filesArr.imageField.filename : ''
+            let path = filesArr.imageField.path || plugin.path
             let image = {
-              bucket: plugin.awsBucket,
+              bucket: filesArr.imageField.awsBucket || plugin.awsBucket,
               date: plugin.manager.useMilliseconds? Date.now() : Math.floor(Date.now() / 1000),
               filename: file.name,
               filesize: file.size,
-              path: `${plugin.bucketDir}/${uid}${pathFilename}.${file.ext}`,
+              path: path(uid, file.name, file.ext, file),
               // sizes: ['large', 'medium', 'small'],
               uid: uid,
             }
+            let s3Options = {
+              // ACL: Some IAM permissions "s3:PutObjectACL" must be included in the policy
+              // params: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
+              ACL: filesArr.imageField.awsAcl || plugin.awsAcl,
+              Body: file.data,
+              Bucket: image.bucket,
+              Key: image.path,
+              ...(filesArr.imageField.params || plugin.params),
+            }
             plugin.manager.info(
               `Uploading '${image.filename}' to '${image.bucket}/${image.path}'`
             )
             if (test) {
               plugin._addImageObjectsToData(filesArr.inputPath, data, image)
-              resolve()
+              resolve(s3Options)
             } else {
-              plugin.s3.upload({
-                Bucket: plugin.awsBucket,
-                Key: image.path,
-                Body: file.data,
-                // The IAM permission "s3:PutObjectACL" must be included in the appropriate policy
-                ACL: 'public-read',
-                // upload params,https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
-                ...filesArr.imageField.params,
-
-              }, (err, response) => {
+              plugin.s3.upload(s3Options, (err, response) => {
                 if (err) return reject(err)
                 plugin._addImageObjectsToData(filesArr.inputPath, data, image)
-                resolve()
+                resolve(s3Options)
               })
             }
           })
         }))
       }))
 
     // Save the data against the matching document(s)
-    }).then(() => {
-      // Remove update's _output object
+    }).then((s3Options) => {
       let prunedData = { ...data }
+      // Remove update's _output object
       delete prunedData._output
-      if (test) return [prunedData]
+      if (test) return [prunedData, s3Options]
       return model._update(
         idquery,
         { '$set': prunedData },
@@ -196,10 +211,11 @@ let plugin = module.exports = {
     // Find all image objects in data
     for (let doc of util.toArray(data)) {
       for (let imageField of this.imageFields) {
-        if (options.getSignedUrls || imageField.getSignedUrl) {
+        if (options.getSignedUrls
+            || (util.isDefined(imageField.getSignedUrl) ? imageField.getSignedUrl : plugin.getSignedUrl)) {
           let images = plugin._findImagesInData(doc, imageField, 0, '').filter(o => o.image)
           for (let image of images) {
-            image.image.signedUrl = plugin._getSignedUrl(image.image.path)
+            image.image.signedUrl = plugin._getSignedUrl(image.image.path, 3600, imageField.awsBucket)
           }
         }
       }
@@ -346,7 +362,10 @@ let plugin = module.exports = {
     // the file doesnt get deleted, we only delete from plugin.awsBucket.
     if (!unused.length) return
     await new Promise((resolve, reject) => {
-      plugin.s3.deleteObjects({ Bucket: plugin.awsBucket, Delete: { Objects: unused }}, (err, data) => {
+      plugin.s3.deleteObjects({
+        Bucket: plugin.awsBucket,
+        Delete: { Objects: unused }
+      }, (err, data) => {
         if (err) reject(err)
         resolve()
       })
@@ -410,7 +429,7 @@ let plugin = module.exports = {
       return Promise.all(filesArr.map((file, i) => {
         return new Promise((resolve, reject) => {
           fileType.fromBuffer(file.data).then(res => {
-            let maxSize = filesArr.imageField.filesize
+            let filesize = filesArr.imageField.filesize || plugin.filesize
             let formats = filesArr.imageField.formats || plugin.formats
             let allowAny = util.inArray(formats, 'any')
             file.format = res? res.ext : ''
@@ -421,9 +440,9 @@ let plugin = module.exports = {
               title: filesArr.inputPath + (i? `.${i}` : ''),
               detail: `The file size for '${file.nameClipped}' is too big.`
             })
-            else if (maxSize && maxSize < file.size) reject({ // file.size == bytes
+            else if (filesize && filesize < file.size) reject({ // file.size == bytes
               title: filesArr.inputPath + (i? `.${i}` : ''),
-              detail: `The file size for '${file.nameClipped}' is bigger than ${(maxSize/1000/1000).toFixed(1)}MB.`
+              detail: `The file size for '${file.nameClipped}' is bigger than ${(filesize/1000/1000).toFixed(1)}MB.`
             })
             else if (file.ext == 'unknown') reject({
               title: filesArr.inputPath + (i? `.${i}` : ''),
@@ -446,8 +465,10 @@ let plugin = module.exports = {
      * @param {object|array} fields
      * @param {string} path
      * @return [{}, ...]
+     * @this plugin
      */
     let list = []
+    let that = this
     util.forEach(fields, (field, fieldName) => {
       let path2 = `${path}.${fieldName}`.replace(/^\./, '')
       // let schema = field.schema || {}
@@ -464,11 +485,28 @@ let plugin = module.exports = {
 
       // Image field. Test for field.image as field.type may be 'any'
       } else if (field.type == 'image' || field.image) {
-        let formats = field.formats
-        let filesize = field.filesize || field.fileSize // old <= v1.31.7
-        let filename = field.filename
-        let getSignedUrl = field.getSignedUrl
-        let params = { ...field.params||{} }
+        if (field.fileSize) { // > v1.31.7
+          this.manager.warn(`${path2}.fileSize has been depreciated in favour of ${path2}.filesize`)
+          field.filesize = field.filesize || field.fileSize
+        }
+        if (field.filename) { // > v1.36.3
+          this.manager.warn(`${path2}.filename has been depreciated in favour of ${path2}.path()`)
+          field.path = field.path
+            || function(uid, basename, ext, file) { return `${that.bucketDir}/${uid}/${field.filename}.${ext}` }
+        }
+
+        list.push({
+          awsAcl: field.awsAcl,
+          awsBucket: field.awsBucket,
+          filename: field.filename,
+          filesize: field.filesize,
+          formats: field.formats,
+          fullPath: path2,
+          fullPathRegex: new RegExp('^' + path2.replace(/\.[0-9]+/g, '.[0-9]+').replace(/\./g, '\\.') + '$'),
+          getSignedUrl: field.getSignedUrl,
+          path: field.path,
+          params: field.params ? util.deepCopy(field.params) : undefined,
+        })
         // Convert image field to subdocument
         fields[fieldName] = {
           bucket: { type: 'string' },
@@ -479,15 +517,6 @@ let plugin = module.exports = {
           schema: { image: true, nullObject: true, isImageObject: true },
           uid: { type: 'string' },
         }
-        list.push({
-          fullPath: path2,
-          fullPathRegex: new RegExp('^' + path2.replace(/\.[0-9]+/g, '.[0-9]+').replace(/\./g, '\\.') + '$'),
-          formats: formats,
-          filesize: filesize,
-          filename: filename,
-          getSignedUrl: getSignedUrl,
-          params: params,
-        })
       }
     })
     return list
@@ -537,12 +566,17 @@ let plugin = module.exports = {
     return list
   },
 
-  _getSignedUrl: (path, expires=3600) => {
-    // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getSignedUrl-property
+  _getSignedUrl: (path, expires=3600, bucket) => {
+    /**
+     * @param {string} path - aws file path
+     * @param {number} <expires> - seconds
+     * @param {number} <bucket>
+     * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getSignedUrl-property
+     */
     let signedUrl = plugin.s3.getSignedUrl('getObject', {
-      Bucket: plugin.awsBucket,
+      Bucket: bucket || plugin.awsBucket,
+      Expires: expires,
       Key: path,
-      Expires: expires
     })
     return signedUrl
   },

test/model.js

@@ -117,8 +117,7 @@ module.exports = function(monastery, opendb) {
 
   test('model reserved rules', async () => {
     // Setup
-    let db = (await opendb(false, {})).db
-    db.error = () => {} // hiding debug error
+    let db = (await opendb(false, { hideErrors: true })).db // hide debug error
     let user = db.model('user', {
       fields: {
         name: {