Error when doing large directory scans

[IamManifold]

I'm basically trying to make a directory compare tool that list all the files that are different between two directories. I can share my code if it helps.

Error: UNKNOWN: unknown error, lstat 'M:\User\xx\Node Projects\next-learn-demo\3-create-dynamic-pages\node_modules\core-js\fn\object\define.js'
    at Object.lstatSync (fs.js:943:3)
    at getStatIgnoreBrokenLink (C:\Node\fileDiff\node_modules\dir-compare\src\entry\entryBuilder.js:50:17)
    at Object.buildEntry (C:\Node\fileDiff\node_modules\dir-compare\src\entry\entryBuilder.js:32:15)
    at Object.buildDirEntries (C:\Node\fileDiff\node_modules\dir-compare\src\entry\entryBuilder.js:19:21)
    at C:\Node\fileDiff\node_modules\dir-compare\src\compareAsync.js:21:37
    at async Promise.all (index 1)
    at async Promise.all (index 9)
    at async Promise.all (index 6)
    at async Promise.all (index 92)
    at async Promise.all (index 3) {
  errno: -2146893818,
  syscall: 'lstat',
  code: 'UNKNOWN',
  path: 'M:\\User\\xx\\Node Projects\\next-learn-demo\\3-create-dynamic-pages\\node_modules\\core-js\\fn\\object\\define.js'
}

[gliviu]

Looks related to the way windows handles symlinks when using network drives or WSL.
Possible fixes below.
bmewburn/vscode-intelephense#260 (comment)
mrmlnc/fast-glob#161
https://stackoverflow.com/questions/29004843/unknown-error-on-package-installation

[IamManifold]

Thanks, I"m going to look into those fixes to see if it works.

I seem to be getting a handful of different errors during my test scans.
I think i'm going to work on the rest of my process and just test with smaller directories until i can figure out what's going on.

Error: EMFILE: too many open files, open 'C:\justin.miller\Code''Working'' code\VexcelDemo 4-5-2019\node_modules\forwarded\HISTORY.md'
at Object.openSync (fs.js:461:3)
at Object.fileBufferObject (C:\Node\fileDiff\node_modules\filecompare\lib\index.js:6:17)
at compare (C:\Node\fileDiff\node_modules\filecompare\index.js:8:16)
at C:\Node\fileDiff\app.js:90:17
at Array.forEach ()
at print (C:\Node\fileDiff\app.js:50:20)
at C:\Node\fileDiff\app.js:30:18 {
errno: -4066,
syscall: 'open',
code: 'EMFILE',
path: "C:\user\Code\''Working'' code\VexcelDemo 4-5-2019\node_modules\forwarded\HISTORY.md"
}

`

const dircompare = require('dir-compare');

//const options = { compareSize: true, compareContent: true};
var options = {
compareSize: true,
compareContent: true,
ignoreLineEnding: true,
ignoreWhiteSpaces: true,
skipSubdirs: false
};
//const oldFile = (__dirname, 'resources');
//const newFile = (__dirname, 'resources2');
const leftdir = ('C:\Users\JustXtreme\Desktop\Coast to Coast - Copy');
const rightdir = ('C:\Users\JustXtreme\Desktop\Coast to Coast');

// Multiple compare strategy can be used simultaneously - compareSize, compareContent, compareDate, compareSymlink.
// If one comparison fails for a pair of files, they are considered distinct.

// Synchronous
//const res = dircompare.compareSync(oldFile, newFile, options)
//print(res)
var Spinner = require('cli-spinner').Spinner;
var spinner = new Spinner('processing.. %s');
spinner.setSpinnerString('|/-\');
spinner.start();
// Asynchronous
dircompare.compare(leftdir, rightdir, options)
.then(res => print(res))
.catch(error => console.error(error));

function print(result) {
//console.log(result)

console.log('Directories are %s', result.same ? 'identical' : 'different')

//console.log('Statistics - equal entries: %s, distinct entries: %s, left only entries: %s, right only entries: %s, differences: %s',
//  result.equal, result.distinct, result.left, result.right, result.differences)


var counter = 0;
console.log('Left Dir: ' + leftdir)
console.log('Right Dir: ' + rightdir)

result.diffSet.forEach(dif => {

    counter++

    if (dif.state !== 'equal') {
        leftfile = dif.name1
        rightfile = dif.name2

        if (typeof leftfile === 'undefined' || leftfile === null) {
            console.log('Right File ' + dif.path2 + rightfile)
        } else if (typeof rightfile === 'undefined' || rightfile === null) {
            console.log('Left File ' + dif.path1 + leftfile)
        }

    }

})
spinner.stop();

}

`

Also,

Is it suppose to identify difference within files? let's say a text file for example.
same name different contents, is there a way to detect that with your code?

[gliviu]

Yes, it is possible. If you use compareContent: true option, it will check the files content, disregarding size of file.