From 074e5b5d338cf57df2cb0e02ec654e3b5c3e080f Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 14 Sep 2021 14:23:37 -0600 Subject: [PATCH 1/2] Require node >=14 BREAKING CHANGE: Making some changes that require newer versions of node. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8ba57c3..07a496a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "3.0.2", "description": "Copy file globs, watching for changes.", "engines": { - "node": ">=6.5" + "node": ">=14" }, "main": "lib/index.js", "bin": { From be9680c7e8e84ebb1b43c089d53d9b6e3bbf4a4b Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 14 Sep 2021 18:57:07 -0600 Subject: [PATCH 2/2] Add ignore option and flag Add support for gitignore style ignore strings, either as an option or cli flag. This makes it a lot easier to perform folder exclusions against simpler globs. --- README.md | 3 ++ bin/help.js | 2 + bin/index.js | 2 + bin/main.js | 1 + lib/copy-sync.js | 1 + lib/copy.js | 1 + lib/utils/apply-action-sync.js | 4 +- lib/utils/apply-action.js | 78 ++++++---------------------------- lib/utils/normalize-options.js | 4 +- lib/utils/watcher.js | 28 +++++++++++- lib/watch.js | 1 + package.json | 8 ++-- test/copy.js | 66 ++++++++++++++++++++++++++++ test/watch.js | 55 ++++++++++++++++++++++++ 14 files changed, 183 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index e879497..f5479fd 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ Options: -c, --command A command text to transform each file. -C, --clean Clean files that matches like pattern in directory before the first copying. + -i --ignore A comma separated list of gitignore style ignore + patterns. -L, --dereference Follow symbolic links when copying from them. -h, --help Print usage information. --include-empty-dirs The flag to copy empty directories which is @@ -110,6 +112,7 @@ cpx.copy(source, dest, callback) - **options.preserve** `{boolean}` -- The flag to copy uid, gid, atime, and mtime of files. Default: `false`. - **options.transform** `{((filepath: string) => stream.Transform)[]}` -- Functions that creates a `stream.Transform` object to transform each copying file. - **options.update** `{boolean}` -- The flag to not overwrite files on destination if the source file is older. Default: `false`. + - **options.ignore** `{string|Array}` -- A gitignore style string or array of strings that make ignoring directory patterns easier. Default: [] - **callback** `{(err: Error|null) => void}` -- A function that is called at done. Copy files that matches with `source` glob to `dest` directory. diff --git a/bin/help.js b/bin/help.js index 9c5bcde..13adb3f 100644 --- a/bin/help.js +++ b/bin/help.js @@ -28,6 +28,8 @@ Options: -c, --command A command text to transform each file. -C, --clean Clean files that matches like pattern in directory before the first copying. + -i --ignore A comma separated list of gitignore style ignore + patterns. -L, --dereference Follow symbolic links when copying from them. -h, --help Print usage information. --include-empty-dirs The flag to copy empty directories which is diff --git a/bin/index.js b/bin/index.js index b961d05..ef50ab0 100644 --- a/bin/index.js +++ b/bin/index.js @@ -24,6 +24,7 @@ const args = subarg(process.argv.slice(2), { C: "clean", h: "help", includeEmptyDirs: "include-empty-dirs", + i: "ignore", L: "dereference", p: "preserve", t: "transform", @@ -44,6 +45,7 @@ const args = subarg(process.argv.slice(2), { "version", "watch", ], + string: ["ignore"], default: { initial: true }, unknown(arg) { if (arg[0] === "-") { diff --git a/bin/main.js b/bin/main.js index a29e512..bb98ab0 100644 --- a/bin/main.js +++ b/bin/main.js @@ -117,6 +117,7 @@ module.exports = function main(source, outDir, args) { initialCopy: args.initial, preserve: args.preserve, update: args.update, + ignore: args.ignore && args.ignore.split(","), }) if (args.clean) { diff --git a/lib/copy-sync.js b/lib/copy-sync.js index 5b2acdc..b5857ea 100644 --- a/lib/copy-sync.js +++ b/lib/copy-sync.js @@ -30,6 +30,7 @@ const removeFileSync = require("./utils/remove-file-sync") * @param {boolean} [options.initialCopy=true] The flag to copy files at the first time. * @param {boolean} [options.preserve=false] The flag to copy file attributes such as timestamps, users, and groups. * @param {boolean} [options.update=false] The flag to not overwrite newer files. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings * @returns {void} */ module.exports = function copySync(source, outputDir, options) { diff --git a/lib/copy.js b/lib/copy.js index 1a3ef40..ac1ff39 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -32,6 +32,7 @@ const removeFile = require("./utils/remove-file") * @param {boolean} [options.preserve=false] The flag to copy file attributes such as timestamps, users, and groups. * @param {(function(string):void)[]} [options.transform] The array of the factories of transform streams. * @param {boolean} [options.update=false] The flag to not overwrite newer files. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings * @param {function(Error):void} [callback] The callback function which will go fulfilled after done. * @returns {Promise} The promise which will go fulfilled after done. */ diff --git a/lib/utils/apply-action-sync.js b/lib/utils/apply-action-sync.js index 9c6ec16..03f3e8d 100644 --- a/lib/utils/apply-action-sync.js +++ b/lib/utils/apply-action-sync.js @@ -9,7 +9,7 @@ // Requirements //------------------------------------------------------------------------------ -const glob = require("glob") +const glob = require("glob-gitignore") //------------------------------------------------------------------------------ // Exports @@ -22,6 +22,7 @@ const glob = require("glob") * @param {object} options - The option object. * @param {boolean} [options.includeEmptyDirs=false] - The flag to include empty directories to copy. * @param {boolean} [options.dereference=false] - The flag to dereference symbolic links. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings * @param {function(string):void} action - The action function to apply. * @returns {Promise} The promise which will go fulfilled after done. * @private @@ -31,6 +32,7 @@ module.exports = function applyActionSync(pattern, options, action) { nodir: !options.includeEmptyDirs, silent: true, follow: Boolean(options.dereference), + ignore: options.ignore, } for (const sourcePath of glob.sync(pattern, globOptions)) { action(sourcePath) diff --git a/lib/utils/apply-action.js b/lib/utils/apply-action.js index 2619d7e..d26dd7e 100644 --- a/lib/utils/apply-action.js +++ b/lib/utils/apply-action.js @@ -9,7 +9,8 @@ // Requirements //------------------------------------------------------------------------------ -const glob = require("glob") +const glob = require("glob-gitignore") +const pMap = require("p-map") //------------------------------------------------------------------------------ // Exports @@ -22,73 +23,20 @@ const glob = require("glob") * @param {object} options - The option object. * @param {boolean} [options.includeEmptyDirs=false] - The flag to include empty directories to copy. * @param {boolean} [options.dereference=false] - The flag to dereference symbolic links. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings * @param {function(string):void} action - The action function to apply. * @returns {Promise} The promise which will go fulfilled after done. * @private */ -module.exports = function applyAction(pattern, options, action) { - return new Promise((resolve, reject) => { - let count = 0 - let done = false - let lastError = null +module.exports = async function applyAction(pattern, options, action) { + const globOptions = { + nodir: !options.includeEmptyDirs, + silent: true, + follow: Boolean(options.dereference), + nosort: true, + ignore: options.ignore, + } + const sourcePaths = await glob.glob(pattern, globOptions) - /** - * Calls the callback function if done. - * @returns {void} - */ - function next() { - if (done && count === 0) { - if (lastError == null) { - resolve() - } else { - reject(lastError) - } - } - } - - const globOptions = { - nodir: !options.includeEmptyDirs, - silent: true, - follow: Boolean(options.dereference), - nosort: true, - } - try { - new glob.Glob(pattern, globOptions) - .on("match", sourcePath => { - if (lastError != null) { - return - } - - count += 1 - try { - action(sourcePath).then( - () => { - count -= 1 - next() - }, - error => { - count -= 1 - lastError = lastError || error - next() - } - ) - } catch (error) { - count -= 1 - lastError = lastError || error - next() - } - }) - .on("end", () => { - done = true - next() - }) - .on("error", error => { - done = true - lastError = lastError || error - next() - }) - } catch (error) { - reject(error) - } - }) + return pMap(sourcePaths, action, { concurrency: 5 }) } diff --git a/lib/utils/normalize-options.js b/lib/utils/normalize-options.js index 4904a17..15edb0d 100644 --- a/lib/utils/normalize-options.js +++ b/lib/utils/normalize-options.js @@ -45,7 +45,8 @@ function getBasePath(source) { * @param {boolean} [options.preserve=false] The flag to copy file attributes such as timestamps, users, and groups. * @param {(function(string):stream.Transform)[]} [options.transform=null] The array of transform function's factories. * @param {boolean} [options.update=false] The flag to not overwrite newer files. - * @returns {{baseDir:string,clean:boolean,dereference:boolean,includeEmptyDirs:boolean,initialCopy:boolean,outputDir:string,preserve:boolean,source:string,transform:any[],toDestination:any,update:boolean}} The normalized options. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings + * @returns {{baseDir:string,clean:boolean,dereference:boolean,includeEmptyDirs:boolean,initialCopy:boolean,outputDir:string,preserve:boolean,source:string,transform:any[],toDestination:any,update:boolean,ignore:string|Array.}} The normalized options. * @private */ module.exports = function normalizeOptions(source, outputDir, options) { @@ -69,5 +70,6 @@ module.exports = function normalizeOptions(source, outputDir, options) { toDestination, transform: [].concat(options && options.transform).filter(Boolean), update: Boolean(options && options.update), + ignore: options && options.ignore, } } diff --git a/lib/utils/watcher.js b/lib/utils/watcher.js index 92afe73..d6b66b2 100644 --- a/lib/utils/watcher.js +++ b/lib/utils/watcher.js @@ -16,6 +16,7 @@ const debounce = require("debounce") const debug = require("debug")("cpx") const fs = require("fs-extra") const Minimatch = require("minimatch").Minimatch +const ignore = require("ignore") const copyFile = require("./copy-file") const normalizePath = require("./normalize-path") const removeFile = require("./remove-file") @@ -24,7 +25,7 @@ const removeFile = require("./remove-file") // Helpers //------------------------------------------------------------------------------ -const walkDirectories = co.wrap(function*(dirRoot, dereference, callback) { +const walkDirectories = co.wrap(function*(dirRoot, dereference, ig, callback) { const stack = [] // Check whether the root is a directory. @@ -52,6 +53,14 @@ const walkDirectories = co.wrap(function*(dirRoot, dereference, callback) { const childPath = normalizePath(path.join(entry.path, child)) const childStat = yield fs.stat(childPath) + if ( + ig.ignores( + childStat.isDirectory() ? `${childPath}/` : childPath + ) + ) { + continue + } + entry.files.set(childPath, childStat) if (childStat.isDirectory()) { @@ -90,6 +99,7 @@ module.exports = class Watcher extends EventEmitter { * @param {string} options.source The glob pattern of source files. * @param {(function(string):stream.Transform)[]} options.transform The array of transform function's factories. * @param {boolean} options.update The flag to not overwrite newer files. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings. */ constructor(options) { super() @@ -100,6 +110,7 @@ module.exports = class Watcher extends EventEmitter { this.includeEmptyDirs = options.includeEmptyDirs this.initialCopy = options.initialCopy this.matcher = new Minimatch(options.source) + this.ignore = ignore().add(options.ignore) this.outputDir = options.outputDir this.preserve = options.preserve this.source = options.source @@ -168,6 +179,7 @@ module.exports = class Watcher extends EventEmitter { return walkDirectories( dirRoot, this.dereference, + this.ignore, co.wrap( function*(dirPath, files) { if (this.trigger == null || this.watchers.has(dirPath)) { @@ -354,6 +366,10 @@ module.exports = class Watcher extends EventEmitter { return } + if (this.ignore.ignores(normalizedPath)) { + return + } + if (this.ready) { this.enqueueAdd(normalizedPath) } else if (this.initialCopy) { @@ -374,6 +390,11 @@ module.exports = class Watcher extends EventEmitter { onRemoved(sourcePath) { debug("Watcher#onRemoved", sourcePath) const normalizedPath = normalizePath(sourcePath) + + if (this.ignore.ignores(normalizedPath)) { + return + } + if (this.matcher.match(normalizedPath)) { this.enqueueRemove(normalizedPath) } @@ -388,6 +409,11 @@ module.exports = class Watcher extends EventEmitter { onChanged(sourcePath) { debug("Watcher#onChanged", sourcePath) const normalizedPath = normalizePath(sourcePath) + + if (this.ignore.ignores(normalizedPath)) { + return + } + if (this.matcher.match(normalizedPath)) { this.enqueueChange(normalizedPath) } diff --git a/lib/watch.js b/lib/watch.js index 3c5c024..5d2bfd3 100644 --- a/lib/watch.js +++ b/lib/watch.js @@ -32,6 +32,7 @@ const Watcher = require("./utils/watcher") * @param {boolean} [options.preserve=false] The flag to copy file attributes such as timestamps, users, and groups. * @param {(function(string):void)[]} [options.transform] The array of the factories of transform streams. * @param {boolean} [options.update=false] The flag to not overwrite newer files. + * @param {string|Array.} [options.ignore] - gitignore string or array of gitignore strings * @returns {Watcher} The watcher object which observes the files. */ module.exports = function watch(source, outputDir, options) { diff --git a/package.json b/package.json index 07a496a..c5b9ba0 100644 --- a/package.json +++ b/package.json @@ -30,22 +30,24 @@ "debug": "^4.1.1", "duplexer": "^0.1.1", "fs-extra": "^10.0.0", - "glob": "^7.1.4", + "glob-gitignore": "^1.0.14", "glob2base": "0.0.12", + "ignore": "^5.1.8", "minimatch": "^3.0.4", + "p-map": "^4.0.0", "resolve": "^1.12.0", "safe-buffer": "^5.2.0", "shell-quote": "^1.7.1", "subarg": "^1.0.0" }, "devDependencies": { - "auto-changelog": "^2.2.0", - "gh-release": "^6.0.0", "@babel/core": "^7.5.5", "@babel/register": "^7.5.5", "@mysticatea/eslint-plugin": "^13.0.0", + "auto-changelog": "^2.2.0", "babel-preset-power-assert": "^3.0.0", "eslint": "^7.9.0", + "gh-release": "^6.0.0", "mocha": "^9.0.3", "nyc": "15.1.0", "opener": "^1.5.1", diff --git a/test/copy.js b/test/copy.js index be888d0..6dc671a 100644 --- a/test/copy.js +++ b/test/copy.js @@ -78,6 +78,72 @@ describe("The copy method", () => { }) }) + describe("should copy specified files with globs and ignore strings:", () => { + beforeEach(() => + setupTestDir({ + "test-ws/untouchable.txt": "untouchable", + "test-ws/a/hello.txt": "Hello", + "test-ws/a/b/this-is.txt": "A pen", + "test-ws/a/b/that-is.txt": "A note", + "test-ws/a/b/no-copy.dat": "no-copy", + "test-ws/a/node_modules/no-copy.txt": "no-copy", + "test-ws/a/vscode/no-copy.txt": "no-copy", + }) + ) + afterEach(() => teardownTestDir("test-ws")) + + /** + * Verify. + * @returns {void} + */ + function verifyFiles() { + return verifyTestDir({ + "test-ws/untouchable.txt": "untouchable", + "test-ws/a/hello.txt": "Hello", + "test-ws/a/b/this-is.txt": "A pen", + "test-ws/a/b/that-is.txt": "A note", + "test-ws/a/b/no-copy.dat": "no-copy", + "test-ws/a/node_modules/no-copy.txt": "no-copy", + "test-ws/a/vscode/no-copy.txt": "no-copy", + "test-ws/b/untouchable.txt": null, + "test-ws/b/hello.txt": "Hello", + "test-ws/b/b/this-is.txt": "A pen", + "test-ws/b/b/that-is.txt": "A note", + "test-ws/b/b/no-copy.dat": null, + "test-ws/b/vscode/no-copy.txt": null, + }) + } + + const ignore = ["node_modules", "vscode"] + + it("lib async version.", done => { + cpx.copy("test-ws/a/**/*.txt", "test-ws/b", { ignore }, () => + verifyFiles().then(() => done(), done) + ) + }) + + it("lib async version (promise).", () => + cpx + .copy("test-ws/a/**/*.txt", "test-ws/b", { + ignore, + }) + .then(verifyFiles)) + + it("lib sync version.", () => { + cpx.copySync("test-ws/a/**/*.txt", "test-ws/b", { + ignore, + }) + return verifyFiles() + }) + + it("command version.", () => { + execCommandSync( + `"test-ws/a/**/*.txt" test-ws/b --ignore ${ignore.join(",")}` + ) + return verifyFiles() + }) + }) + describe("should clean and copy specified files with globs when give clean option:", () => { beforeEach(() => setupTestDir({ diff --git a/test/watch.js b/test/watch.js index 3579847..aa4ab17 100644 --- a/test/watch.js +++ b/test/watch.js @@ -539,6 +539,61 @@ describe("The watch method", () => { }) } + const patternsWithIgnore = [ + { + description: "should ignore ignored files:", + initialFiles: { + "test-ws/a/hello.txt": "Hello", + "test-ws/a/node_modules/dont-copy.txt": "no-copy", + }, + action() { + return writeFile("test-ws/a/b/added.txt", "added") + }, + verify: { + "test-ws/b/hello.txt": "Hello", + "test-ws/b/b/added.txt": "added", + "test-ws/a/node_modules/dont-copy.txt": "no-copy", + "test-ws/b/node_modules/dont-copy.txt": null, + }, + wait: waitForCopy, + ignore: ["node_modules"], + }, + ] + for (const pattern of patternsWithIgnore) { + //eslint-disable-next-line no-loop-func + ;(pattern.only ? describe.only : describe)(pattern.description, () => { + beforeEach(() => setupTestDir(pattern.initialFiles)) + + it( + "lib version.", + co.wrap(function*() { + watcher = cpx.watch("test-ws/a/**/*.txt", "test-ws/b", { + ignore: pattern.ignore, + }) + yield waitForReady() + yield pattern.action() + yield pattern.wait() + yield verifyTestDir(pattern.verify) + }) + ) + + it( + "command version.", + co.wrap(function*() { + command = execCommand( + `"test-ws/a/**/*.txt" test-ws/b --watch --verbose --ignore ${pattern.ignore.join( + "," + )}` + ) + yield waitForReady() + yield pattern.action() + yield pattern.wait() + yield verifyTestDir(pattern.verify) + }) + ) + }) + } + describe("should do reactions of multiple events:", () => { beforeEach(() => setupTestDir({