diff --git a/.eslintignore b/.eslintignore index 06c5e16..88563a8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,3 @@ dist/** +node_modules/** +**/node_modules diff --git a/.gitignore b/.gitignore index 0f534be..a9a0271 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ coverage docs/.sass-cache docs/_site docs/Gemfile.lock + +**/.serverless diff --git a/circle.yml b/circle.yml index 46313ce..93ace02 100644 --- a/circle.yml +++ b/circle.yml @@ -10,12 +10,14 @@ dependencies: pre: - rm -rf node_modules - npm install + - cd prismic-backup-service && npm install - sudo pip install awsebcli --upgrade --ignore-installed six test: override: - npm run lint - npm run test-with-coverage + - cd prismic-backup-service && npm run test post: - cat ./coverage/lcov.info | ./node_modules/.bin/coveralls diff --git a/package.json b/package.json index ec7cbeb..6c1075a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "nodemon lib/index.js --exec babel-node", "build": "babel lib -d dist", - "lint": "eslint lib/", + "lint": "eslint lib/ prismic-backup-service/", "serve": "node dist/index.js", "test": "mocha", "test-watch": "mocha --watch --reporter min", @@ -67,11 +67,7 @@ "pre-commit": [ "npm run lint", "npm t" - ], - "pre-push": [], - "post-commit": [], - "post-checkout": [], - "post-merge": [] + ] } } } diff --git a/prismic-backup-service/README.md b/prismic-backup-service/README.md new file mode 100644 index 0000000..b1091b3 --- /dev/null +++ b/prismic-backup-service/README.md @@ -0,0 +1,12 @@ +Prismic backup service +============ + +Lambda that requests documents from prismic and stores the results in an S3 bucket. + +serverless 🙌 + +``` +npm install serverless@1.0.0-rc.1 -g +sls deploy +sls invoke --function backupPrismic +``` diff --git a/prismic-backup-service/bin/handler.js b/prismic-backup-service/bin/handler.js new file mode 100644 index 0000000..ff01c2b --- /dev/null +++ b/prismic-backup-service/bin/handler.js @@ -0,0 +1,21 @@ +'use strict'; // eslint-disable-line strict +const backup = require('../lib'); +const variables = require('../config/variables.json'); + +const environment = (cb, functionName) => { + switch (functionName) { + case 'prismic-backup-service-dev-backupPrismic': + return variables.dev; + case 'prismic-backup-service-prod-backupPrismic': + return variables.prod; + default: + return cb('Unable to determine environment.'); + } +}; + +module.exports.backupPrismic = (event, context, cb) => { + const env = environment(cb, context.functionName); + backup(env.bucketName) + .then(metadata => cb(null, metadata)) + .catch(err => cb(err)); +}; diff --git a/prismic-backup-service/config/variables.json b/prismic-backup-service/config/variables.json new file mode 100644 index 0000000..10d98bb --- /dev/null +++ b/prismic-backup-service/config/variables.json @@ -0,0 +1,8 @@ +{ + "dev": { + "bucketName": "prismic-backup-dev" + }, + "prod": { + "bucketName": "prismic-backup-prod" + } +} diff --git a/prismic-backup-service/lib/index.js b/prismic-backup-service/lib/index.js new file mode 100644 index 0000000..d6a52ff --- /dev/null +++ b/prismic-backup-service/lib/index.js @@ -0,0 +1,54 @@ +const fetch = require('node-fetch'); +const leftPad = require('./left-pad'); +const saveJson = require('./s3').saveJson; + +const prismicURL = 'https://rb-website-stage.prismic.io/api/documents/search?ref=V80_SyMAAKhGWsDT&page=1&pageSize=100'; +const timestamp = new Date().toISOString().substring(0, 10); + +function saveMetadata(bucketName, metadata, funcs) { + const name = `${timestamp}-prismic-backup/metadata.json`; + return funcs.saveJson(bucketName, name, metadata); +} + +function savePrismicData(bucketName, json, funcs) { + const name = `${timestamp}-prismic-backup/page-${leftPad(json.page, 3)}.json`; + return funcs.saveJson(bucketName, name, json); +} + +function updateMetadata(metadata, data) { + return { + totalDocuments: data.total_results_size, + seenDocuments: (metadata.seenDocuments || 0) + data.results.length, + totalPages: data.total_pages, + date: timestamp, + }; +} + +function getJson(url) { + return fetch(url) + .then(res => res.json()); +} + +function loop(bucketName, json, metadata, funcs) { + return savePrismicData(bucketName, json, funcs).then(() => { + const newMetadata = updateMetadata(metadata, json); + + if (json.next_page) { + return funcs.getJson(json.next_page) + .then((newJson) => loop(bucketName, newJson, newMetadata, funcs)); + } + return newMetadata; + }); +} + +const defaultFuncs = { + getJson, + saveJson, +}; + +module.exports = function backupPrismic(bucketName, passedFuncs) { + const funcs = passedFuncs || defaultFuncs; + return funcs.getJson(prismicURL) + .then(json => loop(bucketName, json, {}, funcs)) + .then(metadata => saveMetadata(bucketName, metadata, funcs)); +}; diff --git a/prismic-backup-service/lib/index.spec.js b/prismic-backup-service/lib/index.spec.js new file mode 100644 index 0000000..555e749 --- /dev/null +++ b/prismic-backup-service/lib/index.spec.js @@ -0,0 +1,70 @@ +const backupPrismic = require('./'); +const expect = require('chai').expect; +const describe = require('mocha').describe; +const it = require('mocha').it; + +function okPromise(fn) { + return new Promise(resolve => resolve(fn())); +} + +describe('backupPrismic', () => { + it('it saves all pages and metadata', (done) => { + const responses = [ + { + page: 1, + results_per_page: 100, + results_size: 100, + total_results_size: 132, + total_pages: 2, + next_page: 'https://r.prismic.io/api/documents/search?ref=V80&page=2&pageSize=100', + prev_page: null, + results: [ + 'we have results 1', + ], + }, + { + page: 2, + results_per_page: 100, + results_size: 32, + total_results_size: 132, + total_pages: 2, + next_page: null, + prev_page: 'https://r.prismic.io/api/documents/search?ref=V80&page=1&pageSize=100', + results: [ + 'we have results 2', + ], + }, + ]; + const responsesCopy = Object.assign({}, responses); + const saved = []; + const getJson = () => okPromise(() => responses.shift()); + const saveJson = (bucketName, name, data) => okPromise(() => { + saved.push({ bucketName, name, data }); + }); + const funcs = { getJson, saveJson }; + backupPrismic('bucketNameForTesting', funcs) + .then(() => { + expect(saved.length).to.equal(3, 'Expected 3 items to have been saved to S3'); + + expect(saved[0].name).to.match(/....-..-..-prismic-backup\/page-001.json/); + expect(saved[1].name).to.match(/....-..-..-prismic-backup\/page-002.json/); + expect(saved[2].name).to.match(/....-..-..-prismic-backup\/metadata.json/); + + expect(saved[0].bucketName).to.equal('bucketNameForTesting'); + expect(saved[1].bucketName).to.equal('bucketNameForTesting'); + expect(saved[2].bucketName).to.equal('bucketNameForTesting'); + + expect(saved[0].data).to.deep.equal(responsesCopy[0]); + expect(saved[1].data).to.deep.equal(responsesCopy[1]); + + const metadata = saved[2].data; + expect(metadata.totalDocuments).to.equal(132); + expect(metadata.seenDocuments).to.equal(2); + expect(metadata.totalPages).to.equal(2); + expect(metadata.date).to.match(/....-..-../); + + done(); + }) + .catch(done); + }); +}); diff --git a/prismic-backup-service/lib/left-pad.js b/prismic-backup-service/lib/left-pad.js new file mode 100644 index 0000000..f6fa90c --- /dev/null +++ b/prismic-backup-service/lib/left-pad.js @@ -0,0 +1,9 @@ +'use strict'; // eslint-disable-line strict + +module.exports = function leftPad(number, targetLength) { + let output = number.toString(); + while (output.length < targetLength) { + output = `0${output}`; + } + return output; +}; diff --git a/prismic-backup-service/lib/s3.js b/prismic-backup-service/lib/s3.js new file mode 100644 index 0000000..b819e4a --- /dev/null +++ b/prismic-backup-service/lib/s3.js @@ -0,0 +1,24 @@ +function saveJson(bucketName, key, data) { + const AWS = require('aws-sdk'); // eslint-disable-line import/no-unresolved, global-require + + AWS.config.region = 'eu-west-1'; + + const s3bucket = new AWS.S3({ params: { Bucket: bucketName } }); + + const index = { + Key: key, + Body: JSON.stringify(data), + ContentType: 'application/json', + }; + return new Promise((resolve, reject) => { + s3bucket.upload(index, (err) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); +} + +module.exports = { saveJson }; diff --git a/prismic-backup-service/package.json b/prismic-backup-service/package.json new file mode 100644 index 0000000..26cf33e --- /dev/null +++ b/prismic-backup-service/package.json @@ -0,0 +1,24 @@ +{ + "name": "prismic-backup-service", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "mocha", + "test-watch": "mocha --watch --reporter min" + + }, + "author": "", + "license": "Apache-2.0", + "dependencies": { + "node-fetch": "^1.6.0" + }, + "devDependencies": { + "chai": "^3.5.0", + "mocha": "^3.0.2" + }, + "config": { + "pre-git": { + } + } +} diff --git a/prismic-backup-service/serverless.env.yml b/prismic-backup-service/serverless.env.yml new file mode 100644 index 0000000..e53a857 --- /dev/null +++ b/prismic-backup-service/serverless.env.yml @@ -0,0 +1,20 @@ +# This is the Serverless Environment File +# +# It contains listing of your stages and their regions +# It also manages serverless variables at 3 levels: +# - common variables: variables that apply to all stages/regions +# - stage variables: variables that apply to a specific stage +# - region variables: variables that apply to a specific region + +vars: +stages: + dev: + vars: + regions: + eu-west-1: + vars: + prod: + vars: + regions: + eu-west-1: + vars: diff --git a/prismic-backup-service/serverless.yml b/prismic-backup-service/serverless.yml new file mode 100644 index 0000000..73e05c6 --- /dev/null +++ b/prismic-backup-service/serverless.yml @@ -0,0 +1,39 @@ +service: prismic-backup-service # NOTE: update this with your service name + +custom: + bucketName: prismic-backup-dev #${file(./config/variables.json)} + +provider: + name: aws + stage: dev + region: eu-west-1 + runtime: nodejs4.3 + iamRoleStatements: + - Effect: "Allow" + Action: + - "s3:ListBucket" + - "s3:PutObject" + Resource: + - "arn:aws:s3:::${self:custom.bucketName}" + - "arn:aws:s3:::${self:custom.bucketName}/*" + +functions: + backupPrismic: + handler: bin/handler.backupPrismic + +# you can add any of the following events +# events: +# - http: +# path: users/create +# method: get +# - s3: ${bucket} +# - schedule: rate(10 minutes) +# - sns: greeter-topic + +# you can add CloudFormation resource templates here +resources: + Resources: + BackupBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: ${self:custom.bucketName} diff --git a/prismic-backup-service/test/mocha.opts b/prismic-backup-service/test/mocha.opts new file mode 100644 index 0000000..e49f4d1 --- /dev/null +++ b/prismic-backup-service/test/mocha.opts @@ -0,0 +1 @@ +./{lib,test}/**/*spec.js diff --git a/test/mocha.opts b/test/mocha.opts index cdb0ab5..5e6e90b 100644 --- a/test/mocha.opts +++ b/test/mocha.opts @@ -2,4 +2,4 @@ --require test/helpers/globals --require babel-polyfill ---reporter mocha-trumpet-reporter -./{,!(node_modules|dist)/**/}*spec.js +./{lib,test}/**/*spec.js