diff --git a/README.md b/README.md index 038f6d8..9c23d1e 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ It was onle basic introduction to the `Package` class. To learn more let's take Factory method to instantiate `Package` class. This method is async and it should be used with await keyword or as a `Promise`. -- `descriptor (String/Object)` - data package descriptor as local path, url or object +- `descriptor (String/Object)` - data package descriptor as local path, url or object. If ththe path has a `zip` file extension it will be unzipped to the temp directory first. - `basePath (String)` - base path for all relative paths - `strict (Boolean)` - strict flag to alter validation behavior. Setting it to `true` leads to throwing errors on any operation with invalid descriptor - `(errors.DataPackageError)` - raises error if something goes wrong @@ -245,9 +245,7 @@ dataPackage.name // renamed-package #### `async package.save(target)` -> For now only descriptor will be saved. - -Save data package to target destination. +Save data package to target destination. If target path has a zip file extension the package will be zipped and saved entirely. If it has a json file extension only the descriptor will be saved. - `target (String)` - path where to save a data package - `(errors.DataPackageError)` - raises error if something goes wrong diff --git a/data/dp3-zip.zip b/data/dp3-zip.zip new file mode 100644 index 0000000..344962d Binary files /dev/null and b/data/dp3-zip.zip differ diff --git a/data/dp3-zip/data/countries.csv b/data/dp3-zip/data/countries.csv new file mode 100644 index 0000000..bebe322 --- /dev/null +++ b/data/dp3-zip/data/countries.csv @@ -0,0 +1,4 @@ +name,size +gb,100 +us,200 +cn,300 diff --git a/data/dp3-zip/datapackage.json b/data/dp3-zip/datapackage.json new file mode 100644 index 0000000..4f73480 --- /dev/null +++ b/data/dp3-zip/datapackage.json @@ -0,0 +1,22 @@ +{ + "name": "abc", + "resources": [ + { + "name": "countries", + "format": "csv", + "path": "data/countries.csv", + "schema": { + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "size", + "type": "number" + } + ] + } + } + ] +} diff --git a/package.json b/package.json index 9e1286a..0d34c28 100644 --- a/package.json +++ b/package.json @@ -38,10 +38,12 @@ "glob": "^7.1.2", "jschardet": "^1.5.1", "json-pointer": "^0.6.0", + "jszip": "^3.1.5", "lodash": "^4.13.1", "regenerator-runtime": "^0.11.0", "stream-to-async-iterator": "^0.2.0", "tableschema": "^1.5.1", + "tmp": "0.0.33", "tv4": "^1.2.7", "url-join": "^2.0.1" }, diff --git a/src/package.js b/src/package.js index 378738a..deb1c0f 100644 --- a/src/package.js +++ b/src/package.js @@ -1,8 +1,11 @@ const fs = require('fs') +const JSZip = require('jszip') const isEqual = require('lodash/isEqual') +const isString = require('lodash/isString') const isBoolean = require('lodash/isBoolean') const cloneDeep = require('lodash/cloneDeep') const isUndefined = require('lodash/isUndefined') +const {promisify} = require('util') const {Profile} = require('./profile') const {Resource} = require('./resource') const {DataPackageError} = require('./errors') @@ -21,6 +24,14 @@ class Package { */ static async load(descriptor={}, {basePath, strict=false}={}) { + // Extract zip + // TODO: + // it's first iteration of the zip loading implementation + // for now browser support and tempdir cleanup (not needed?) is not covered + if (isString(descriptor) && descriptor.endsWith('.zip')) { + descriptor = await extractZip(descriptor) + } + // Get base path if (isUndefined(basePath)) { basePath = helpers.locateDescriptor(descriptor) @@ -177,8 +188,43 @@ class Package { */ save(target) { return new Promise((resolve, reject) => { - const contents = JSON.stringify(this._currentDescriptor, null, 4) - fs.writeFile(target, contents, error => (!error) ? resolve() : reject(error)) + + // Save descriptor to json + if (target.endsWith('.json')) { + const contents = JSON.stringify(this._currentDescriptor, null, 4) + fs.writeFile(target, contents, error => (!error) ? resolve() : reject(error)) + + // Save package to zip + } else { + + // Not supported in browser + if (config.IS_BROWSER) { + throw new DataPackageError('Zip is not supported in browser') + } + + // Prepare zip + const zip = new JSZip() + const descriptor = cloneDeep(this._currentDescriptor) + for (const [index, resource] of this.resources.entries()) { + if (!resource.name) continue + if (!resource.local) continue + let path = `data/${resource.name}` + const format = resource.descriptor.format + if (format) path = `${path}.${format.toLowerCase()}` + descriptor.resources[index].path = path + zip.file(path, resource.rawRead()) + } + zip.file('datapackage.json', JSON.stringify(descriptor, null, 4)) + + // Write zip + zip + .generateNodeStream({type: 'nodebuffer', streamFiles: true}) + .pipe(fs.createWriteStream(target).on('error', error => reject(error))) + .on('error', error => reject(error)) + .on('finish', () => resolve(true)) + + } + }) } @@ -247,6 +293,49 @@ class Package { // Internal +async function extractZip(descriptor) { + + // Not supported in browser + if (config.IS_BROWSER) { + throw new DataPackageError('Zip is not supported in browser') + } + + // Load zip + const zip = JSZip() + const tempdir = await promisify(require('tmp').dir)() + await zip.loadAsync(promisify(fs.readFile)(descriptor)) + + // Validate zip + if (!zip.files['datapackage.json']) { + throw new DataPackageError('Invalid zip with data package') + } + + // Save zip to tempdir + for (const [name, item] of Object.entries(zip.files)) { + + // Get path/descriptor + const path = `${tempdir}/${name}` + if (path.endsWith('datapackage.json')) { + descriptor = path + } + + // Directory + if (item.dir) { + await promisify(fs.mkdir)(path) + + // File + } else { + const contents = await item.async('nodebuffer') + await promisify(fs.writeFile)(path, contents) + } + + } + + return descriptor + +} + + function findFiles(pattern, basePath) { const glob = require('glob') return new Promise((resolve, reject) => { diff --git a/test/package.js b/test/package.js index f53aec1..4dc9cd9 100644 --- a/test/package.js +++ b/test/package.js @@ -1,7 +1,9 @@ const fs = require('fs') +const JSZip = require('jszip') const axios = require('axios') const sinon = require('sinon') const {assert} = require('chai') +const {promisify} = require('util') const {catchError} = require('./helpers') const cloneDeep = require('lodash/cloneDeep') const AxiosMock = require('axios-mock-adapter') @@ -674,4 +676,51 @@ describe('Package', () => { }) + describe('#zip', () => { + + it('should load package from a zip', async function() { + if (process.env.USER_ENV === 'browser') this.skip() + const dp = await Package.load('data/dp3-zip.zip') + const countries = await dp.getResource('countries').read({keyed: true}) + assert.deepEqual(dp.descriptor.name, 'abc') + assert.deepEqual(countries, [ + {name: 'gb', size: 100}, + {name: 'us', size: 200}, + {name: 'cn', size: 300}, + ]) + }) + + it('should save package as a zip', async function() { + if (process.env.USER_ENV === 'browser') this.skip() + + // Save as a zip + const dp = await Package.load('data/dp3-zip/datapackage.json') + const target = await promisify(require('tmp').file)({postfix: '.zip'}) + const result = await dp.save(target) + assert.ok(result) + + // Assert file names + const zip = JSZip() + await zip.loadAsync(promisify(fs.readFile)(target)) + assert.deepEqual(zip.file('datapackage.json').name, 'datapackage.json') + assert.deepEqual(zip.file('data/countries.csv').name, 'data/countries.csv') + + // Assert contents + const descContents = await zip.file('datapackage.json').async('string') + const dataContents = await zip.file('data/countries.csv').async('string') + assert.deepEqual(JSON.parse(descContents), dp.descriptor) + assert.deepEqual(dataContents, 'name,size\ngb,100\nus,200\ncn,300\n') + + }) + + it('should raise saving package as a zip to the bad path', async function() { + if (process.env.USER_ENV === 'browser') this.skip() + const dp = await Package.load('data/dp3-zip/datapackage.json') + const error = await catchError(dp.save.bind(dp), 'non-existent/datapackage.zip') + assert.include(error.message, 'no such file or directory') + assert.include(error.message, 'non-existent/datapackage.zip') + }) + + }) + }) diff --git a/webpack.config.js b/webpack.config.js index de30fb4..95ff1a3 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -31,6 +31,7 @@ let webpackConfig = { fs: 'empty', http: 'empty', https: 'empty', + crypto: 'empty', } }