From 191c21725df5d2cf545b8e2f19bc144cdcbab95b Mon Sep 17 00:00:00 2001 From: gliviu Date: Mon, 24 Dec 2018 19:58:04 +0200 Subject: [PATCH] Add option to ignore line endings and white space differences #10 --- .gitignore | 2 +- README.md | 5 + file_compare_handlers/common.js | 35 ++++++ file_compare_handlers/defaultFileCompare.js | 33 +----- file_compare_handlers/lineBasedFileCompare.js | 105 +++++++++++++----- package.json | 5 +- tests/runTests.js | 37 +++--- tests/testdir.tar | Bin 184320 -> 184320 bytes 8 files changed, 149 insertions(+), 73 deletions(-) diff --git a/.gitignore b/.gitignore index 1b2334d..908292a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ /tests/report.txt /tests/testdir /coverage/ -test.js +test*.js diff --git a/README.md b/README.md index 4f33d2f..f792bf6 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Included handlers are: * `dircompare.fileCompareHandlers.defaultFileCompare.compareSync` * `dircompare.fileCompareHandlers.defaultFileCompare.compareAsync` * `dircompare.fileCompareHandlers.lineBasedFileCompare.compareSync` +* `dircompare.fileCompareHandlers.lineBasedFileCompare.compareAsync` The line based comparator can be used to ignore line ending and white space differences. This comparator is not available in [CLI](#command-line) version. ```javascript @@ -138,6 +139,7 @@ var dircompare = require('dir-compare'); var options = { compareContent: true, compareFileSync: dircompare.fileCompareHandlers.lineBasedFileCompare.compareSync, + compareFileAsync: dircompare.fileCompareHandlers.lineBasedFileCompare.compareAsync, ignoreLineEnding: true, ignoreWhiteSpaces: true }; @@ -146,6 +148,9 @@ var path1 = '...'; var path2 = '...'; var res = dircompare.compareSync(path1, path2, options); console.log(res) + +dircompare.compare(path1, path2, options) +.then(res => console.log(res)) ``` ## Command line diff --git a/file_compare_handlers/common.js b/file_compare_handlers/common.js index 9b20f24..4fd7a71 100644 --- a/file_compare_handlers/common.js +++ b/file_compare_handlers/common.js @@ -1,3 +1,5 @@ +var fs = require('fs'); +var Promise = require('bluebird'); var alloc = function(bufSize) { if(Buffer.alloc){ @@ -6,6 +8,39 @@ var alloc = function(bufSize) { return new Buffer(bufSize); } +var wrapper = function(fdQueue) { + return { + open : Promise.promisify(fdQueue.open), + read : Promise.promisify(fs.read), + } +}; + +var closeFilesSync = function(fd1, fd2){ + if (fd1) { + fs.closeSync(fd1); + } + if (fd2) { + fs.closeSync(fd2); + } +} + +var closeFilesAsync = function(fd1, fd2, fdQueue){ + if (fd1) { + fdQueue.close(fd1, function(err){ + if(err){console.log(err);} + }); + } + if (fd2) { + fdQueue.close(fd2, function(err){ + if(err){console.log(err);} + }); + } +} + + module.exports = { alloc : alloc, + wrapper: wrapper, + closeFilesSync: closeFilesSync, + closeFilesAsync: closeFilesAsync }; diff --git a/file_compare_handlers/defaultFileCompare.js b/file_compare_handlers/defaultFileCompare.js index b1d773d..cd39d0a 100644 --- a/file_compare_handlers/defaultFileCompare.js +++ b/file_compare_handlers/defaultFileCompare.js @@ -4,6 +4,10 @@ var Promise = require('bluebird'); var FileDescriptorQueue = require('./fileDescriptorQueue'); var fdQueue = new FileDescriptorQueue(8); var alloc = require('./common').alloc; +var wrapper = require('./common').wrapper(fdQueue); +var closeFilesSync = require('./common').closeFilesSync; +var closeFilesAsync = require('./common').closeFilesAsync; + /** * Compares two partial buffers. */ @@ -44,11 +48,6 @@ var compareSync = function (path1, stat1, path2, stat2, options) { } }; -var wrapper = { - open : Promise.promisify(fdQueue.open), - read : Promise.promisify(fs.read), -}; - /** * Compares two files by content using bufSize as buffer length. */ @@ -80,35 +79,15 @@ var compareAsync = function (path1, stat1, path2, stat2, options) { }); }; return compareAsyncInternal().then(function (result) { - closeFilesAsync(fd1, fd2); + closeFilesAsync(fd1, fd2, fdQueue); return result; }); }).catch(function (error) { - closeFilesAsync(fd1, fd2); + closeFilesAsync(fd1, fd2, fdQueue); return error; }); }; -var closeFilesSync = function(fd1, fd2){ - if (fd1) { - fs.closeSync(fd1); - } - if (fd2) { - fs.closeSync(fd2); - } -} -var closeFilesAsync = function(fd1, fd2){ - if (fd1) { - fdQueue.close(fd1, function(err){ - if(err){console.log(err);} - }); - } - if (fd2) { - fdQueue.close(fd2, function(err){ - if(err){console.log(err);} - }); - } -} module.exports = { compareSync : compareSync, compareAsync : compareAsync diff --git a/file_compare_handlers/lineBasedFileCompare.js b/file_compare_handlers/lineBasedFileCompare.js index ed78101..60f4f5f 100644 --- a/file_compare_handlers/lineBasedFileCompare.js +++ b/file_compare_handlers/lineBasedFileCompare.js @@ -1,11 +1,17 @@ // Compare files line by line with options to ignore line endings and white space differencies. 'use strict' -var fs = require('fs') +var fs = require('fs'); +var bufferEqual = require('buffer-equal'); +var Promise = require('bluebird'); +var FileDescriptorQueue = require('./fileDescriptorQueue'); +var fdQueue = new FileDescriptorQueue(8); var alloc = require('./common').alloc; +var wrapper = require('./common').wrapper(fdQueue); +var closeFilesSync = require('./common').closeFilesSync; +var closeFilesAsync = require('./common').closeFilesAsync; var BUF_SIZE = 4096 var compareSync = function (path1, stat1, path2, stat2, options) { - var BUF_SIZE = 4096; var fd1, fd2; try { fd1 = fs.openSync(path1, 'r'); @@ -28,24 +34,11 @@ var compareSync = function (path1, stat1, path2, stat2, options) { else if(lines1.length !== lines2.length) { return false; } else { - var i - for(i = 0; i < lines1.length - 1; i++) { - var line1 = lines1[i] - var line2 = lines2[i] - if(options.ignoreLineEnding){ - line1 = removeLineEnding(line1) - line2 = removeLineEnding(line2) - } - if(options.ignoreWhiteSpaces){ - line1 = removeWhiteSpaces(line1) - line2 = removeWhiteSpaces(line2) - } - if(line1!==line2) { - return false - } + if(!compareLines(lines1, lines2, options)){ + return false } - last1 = lines1[i] - last2 = lines2[i] + last1 = lines1[lines1.length-1] + last2 = lines2[lines2.length-1] } } } finally { @@ -53,9 +46,53 @@ var compareSync = function (path1, stat1, path2, stat2, options) { } }; -var compareAsync = function (filePath1, fileStat1, filePath2, fileStat2, options) { - throw 'not implemented' -}; +var compareAsync = function(path1, stat1, path2, stat2, options) { + var fd1, fd2; + return Promise.all([wrapper.open(path1, 'r'), wrapper.open(path2, 'r')]) + .then(function(fds){ + fd1 = fds[0] + fd2 = fds[1] + var buf1 = alloc(BUF_SIZE); + var buf2 = alloc(BUF_SIZE); + var progress = 0; + var last1='', last2='' + var compareAsyncInternal = function () { + return Promise.all([ + wrapper.read(fd1, buf1, 0, BUF_SIZE, null), + wrapper.read(fd2, buf2, 0, BUF_SIZE, null) + ]).then(function(sizes){ + var size1 = sizes[0] + var size2 = sizes[1] + var chunk1 = buf1.toString('utf8', 0, size1) + var chunk2 = buf2.toString('utf8', 0, size2) + var lines1 = (last1+chunk1).split(/\n/) + var lines2 = (last2+chunk2).split(/\n/) + if(size1===0 && size2===0){ + // End of file reached + return true + } + else if(lines1.length !== lines2.length) { + return false; + } else { + if(!compareLines(lines1, lines2, options)){ + return false + } + last1 = lines1[lines1.length-1] + last2 = lines2[lines2.length-1] + return compareAsyncInternal() + } + }) + } + return compareAsyncInternal().then(function (result) { + closeFilesAsync(fd1, fd2, fdQueue); + return result; + }); + }) + .catch(function (error) { + closeFilesAsync(fd1, fd2, fdQueue); + return error; + }); +} var removeLineEnding = function (s) { return s.replace(/[\r]+$/g, ''); @@ -66,13 +103,25 @@ var removeWhiteSpaces = function (s) { return s.replace(/^[ \f\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+|[ \f\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+$/g, ''); }; -var closeFilesSync = function(fd1, fd2){ - if (fd1) { - fs.closeSync(fd1); - } - if (fd2) { - fs.closeSync(fd2); + +function compareLines(lines1, lines2, options){ + var i + for(i = 0; i < lines1.length - 1; i++) { + var line1 = lines1[i] + var line2 = lines2[i] + if(options.ignoreLineEnding){ + line1 = removeLineEnding(line1) + line2 = removeLineEnding(line2) + } + if(options.ignoreWhiteSpaces){ + line1 = removeWhiteSpaces(line1) + line2 = removeWhiteSpaces(line2) + } + if(line1!==line2) { + return false + } } + return true } module.exports = { diff --git a/package.json b/package.json index 86acf29..4cd1b6c 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "shelljs": "0.3.0", "tar-fs": "1.13.0", "temp": "0.8.1", - "istanbul": "0.4.4", - "memory-streams": "0.1.0" + "istanbul": "0.4.5", + "memory-streams": "0.1.0", + "semver": "5.6.0" }, "bin": { "dircompare": "dircompare.js" diff --git a/tests/runTests.js b/tests/runTests.js index 1063be4..6f2e391 100644 --- a/tests/runTests.js +++ b/tests/runTests.js @@ -16,7 +16,10 @@ var Streams = require('memory-streams'); var compareSync = require('../index').compareSync; var compareAsync = require('../index').compare; var untar = require('./untar'); -var linecomp = require('../index').fileCompareHandlers.lineBasedFileCompare +var semver = require('semver') + +var lineAsyncCompare = require('../index').fileCompareHandlers.lineBasedFileCompare.compareAsync +var lineSyncCompare = require('../index').fileCompareHandlers.lineBasedFileCompare.compareSync var count = 0, failed = 0, successful = 0; var syncCount = 0, syncFailed = 0, syncSuccessful = 0; @@ -80,6 +83,7 @@ function passed (value, type) { * * onlyCommandLine - Test is run only on command line. * * skipStatisticsCheck - Do not call checkStatistics() after each library test. * * onlySync - only apply for synchronous compare + * * nodeVersionSupport - limit test to specific node versions; ie. '>=2.5.0' */ var tests = [ { @@ -584,85 +588,85 @@ var tests = [ description: 'should ignore line endings', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreLineEnding: true, }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_2', path1: 'd35/crlf-spaces', path2: 'd35/lf-spaces', description: 'should not ignore line endings', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreLineEnding: false, }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_3', path1: 'd35/lf-spaces', path2: 'd35/lf-tabs', description: 'should ignore white spaces', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreWhiteSpaces: true }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_4', path1: 'd35/crlf-spaces', path2: 'd35/lf-tabs', description: 'should ignore white spaces and line endings', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreLineEnding: true, ignoreWhiteSpaces: true }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_5', path1: 'd35/lf-spaces', path2: 'd35/lf-tabs', description: 'should not ignore white spaces', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreWhiteSpaces: false }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_6', path1: 'd35/lf-spaces', path2: 'd35/lf-mix', description: 'should ignore mixed white spaces', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreWhiteSpaces: true }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, { name: 'test011_7', path1: 'd35/lf-tabs', path2: 'd35/lf-mix', description: 'should ignore mixed white spaces', options: { compareContent: true, - compareFileSync: linecomp.compareSync, + compareFileSync: lineSyncCompare, + compareFileAsync: lineAsyncCompare, ignoreWhiteSpaces: true }, displayOptions: {nocolors: true}, onlyLibrary: true, - onlySync: true }, ]; @@ -889,6 +893,7 @@ function executeTests (testDirPath, singleTestName, showResult) { var syncTestsPromises = []; tests.filter(function(test){return !test.onlyCommandLine;}) .filter(function(test){return singleTestName?test.name===singleTestName:true;}) + .filter(function(test){return test.nodeVersionSupport===undefined || semver.satisfies(process.version, test.nodeVersionSupport) }) .forEach(function(test){ syncTestsPromises.push(testSync(test, testDirPath, saveReport, showResult)); }); @@ -902,6 +907,7 @@ function executeTests (testDirPath, singleTestName, showResult) { var asyncTestsPromises = []; tests.filter(function(test){return !test.onlyCommandLine;}) .filter(function(test){return !test.onlySync;}) + .filter(function(test){return test.nodeVersionSupport===undefined || semver.satisfies(process.version, test.nodeVersionSupport) }) .filter(function(test){return singleTestName?test.name===singleTestName:true;}) .forEach(function(test){ asyncTestsPromises.push(testAsync(test, testDirPath, saveReport, showResult)); @@ -915,6 +921,7 @@ function executeTests (testDirPath, singleTestName, showResult) { // Run command line tests var commandLinePromises = []; tests.filter(function(test){return !test.onlyLibrary;}) + .filter(function(test){return test.nodeVersionSupport===undefined || semver.satisfies(process.version, test.nodeVersionSupport) }) .filter(function(test){return singleTestName?test.name===singleTestName:true;}) .forEach(function(test){ commandLinePromises.push(testCommandLine(test, testDirPath, saveReport, showResult)); diff --git a/tests/testdir.tar b/tests/testdir.tar index 638d8e2348e8418d4b5cffce8d672761d3f05b54..a539bc5a3c5bf67cdf3bf8df6caa1c3d0c32cb9d 100644 GIT binary patch delta 3553 zcmZ`*U5J!b7~bdX?ELOl+LDSv1JMs9_MJ1|kCl`f)nW+}g3y(&!{RSUMO3)DuDjE? zvxPXY9}G!*(S@RkI^cl_x(K{X2or?8kZQY-Fx`tKLaq0lZ)eUqpR^0h?40L)-sgSZ z_xy~_md0jF7p7lX7DizZ1(hC;bI*@@%0ZJqRc+(-9NPKy6VF=fUQprPbpUNvoysh? zqj3k%0(20?bh`L4BFpuRm}(=rgC<+aiam3&^LtrXb#LBaS5Y#Yc5Sl4u}!4 zL^!XCxak&AF*VMY(Cq(p%+48xrnU2rp!w+Uqt>$Ek*RUM(Pzo2M4alowb1LrO$B$C zA?|&E_nOtc&)b-Qn~ZlINI|D*((7T^EhjHZwG-Hd+V$rgi? zl8UzQ{g4A$9LZRv`=S!cRuKz1t1eF8cDr{qhd4io2`h+VDAgdPa>tueEseLt3N%(* zlwKe@#Yw!3Uc?z6&c=$m8@B@pQMZ!vAc$mZ2)hxpb(pfbqo6YvDzpJ7D&(E0^=?$q zohFGlw4&EiiY(d$upy;L*^nsOx8#&J$t?HMc5XWp;XP9c>a@H1E3(uakHS`DEwhQD-()a1JuR6%+OeL(!cLp%AKr z+Z;QD7t$zgdq|Yw-R|ZjMtiLfGW$Mi8CO+j5GA(jl6E*o2q6O5+TrHpK|}_Nlbfp& zyHY8G?m;JUKogFnJu(>&kPmTwHzr(3#)bs2*9G|8OnL&??*K$apPcdRpc;<-?r?CQ zckTftuzM037PF6>nWXVB!WBTWk1^$3 zP?4`sU@pHNBI9EAS(>18mPmZ62jGyiVd(BP-X(y;&V*FBM;qpFs#~6KbR|*a=_smr z+>N4Yj8Dh3m|9v)8c6Z z-=UsDnCAYd5m!*}cR5TWlPuRTm9-F@N5-cFF_#)aUq8Q+HGZOlwyxFw=uRV^gY9_ffaqfXm*LeumRa}xm+j| z;*EFkOi_QPKjlI^atCh?sE_3`eLcE{AQ$dn)#`FV{}w5RmefB*`m7?0@x?{FP*UHk zf9WTlTSU}HA1GMd#hOkoT*6K&BPVkUIKoy$Mk5{fFvN~M zX(2pn$ZT8)0?(M(`qG?BB!Gow1)ecLpKMuz$$1khu@oWQxNK>Zs=Ugt@P2 zt~Ql^4@-<{jzetHz71&(OqNky%NRRm4?F2F8|Jv?sI$ow!n|9OYEPu9^#h~hl=fpL zg|M@n(mK9+NQZGO;rQku4&(Y6*07(!oQG!1p3z>NPkUv(91_GOR?FL6jfC<02kpf7 QY%+x~ZYCP9uzQ970F^a1x&QzG delta 3504 zcmZ`*ZHSar6y9@pcIRVPQ(Mv~a40GbS>Jo#FD)rks>PC`1fgHLJG=M^rXqc~?)Nm# zY+*aFFAPbG^h2?v8t_5{{s{b;5F#xINwq;sn7cm~3D-LJzWdJokp3*g-se2$dCocS zy=QQ$ZE&jX>+$DS`kogs>NGi&IxJ|)`&GKOP=^=OP|s``eOfzre1s+E08l4|a%z6YuOEdjhAySPn1XuC`H#rM8Z~nnQg?Quxb8Y! ztYpn#zY)w^px%19_=tI~d|eagKNg90PN^R@;-|Av&HXg2AA63{Hl2fN+w~#+IHV|T zNxD3~F|=GMXQAiUYd7YhL;GJ43O9YU8}00^!*m5qLRS&CLv`8B0h2JO*KXQ`SOM0@ zmI!BriCdNl<#DDBs_nlH>Nx{XwRUDXR3G}iUpr>hR~~07Et(uR2+`l{e6tBx4(3)u z)O;V_iVAmO{ffr0F)vgZ=L||O^l81eFmDtk@YL)q3W5Jv0#%C_|2L!RY0W$J9961oX^1i%FscPH*TqyFn93Y)Otm!L99Bc+ zF^$skox~i6&Cm={$^+xW3L7780tld_C8fR_@LCt_gNST`vd$f{WG+N#1&k=qNusux zQNE+1wo64X*othxR$CF4Ay%}_;Kr(Br&%<01ora#+u##@44lwSriR0IXo0Bp6)0+c zFd9csMQ0Gv1x3@9D>56s)~+@VJ0NN&RFtMQ|E;=1myYZy1bDJ_vyj z9o%J?PGr>2T7b8{1I3i7Dj7r$yQQT)Mj;p`<+VL#PU`zuU^cO% zz@f)V={GJq4u>USpIsxB@i5jQ%It$tQ<5q}3^-r{+)zz=3^-%}7?U?=G}R%7q-5YzvWz6A8eAyy$+?Yum4<%dxI5`1j zNw-hM=oU*#*Fr_Pw8n&VW zbUbWEp&M1F5h2 z^&ONfglg^&5^)tu?Iwq6WSr$Xlv_0fE-9)`G2*5*f}UKz;vI=YjJaiEm`}y97u*O| zKsSfqpbV=VxtVwYu7Hj&##k;h;!t?`1BS?&AZVg4?m;dMg3v{v=LcxX!QvLG0RK+T zLOuFs0Xq4gv-o^{n$Jw$hPJ5fFBn96>S1swHq21_js$&_x($Pm`fe!c1<;W|S6<_i zp2Od;6{1(~z*EtQJMi+_Tt45wA(zeO^V!~a-|juW?pW_}x*?B$+4}s4x#-F)JRAM8 z04>qRIoKTkIywi>L{BWjIh2*=>3J9+tKB50BUGU!ybyIu3-A?3OK+f?R2+5}PLVZA zD;g7VxB+4BR4MMkd6Laa8T{vauujBZz7OMFMmHPl42Rc926(oMj6AL(u-0nq_Y={H zK2p*zcqeq!-0dgg2{=TICjb#@c8`%(;OV1e%;HdU*DpEVBi3>(zLBMg=$9NP$(a7Q z;bf{T{nE=I8PR7Kb*TA2EjdbL%s>R8+81KCXDqj#t8|=|^_a8}dX}?N$7c`dP!)?g zzIcE`Rs8~~=vytN_=zNyxFBnF$*z^Q2*ikQNim~$HFQ+HugFS#PsS{Ss%EV5D!G^a E54_ViHvj+t