diff --git a/lib/binary-extensions.json b/lib/binary-extensions.json deleted file mode 100644 index eb0137f5d..000000000 --- a/lib/binary-extensions.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "extensions": [ - "adp", - "au", - "mid", - "mp4a", - "mpga", - "oga", - "s3m", - "sil", - "eol", - "dra", - "dts", - "dtshd", - "lvp", - "pya", - "ecelp4800", - "ecelp7470", - "ecelp9600", - "rip", - "weba", - "aac", - "aif", - "caf", - "flac", - "mka", - "m3u", - "wax", - "wma", - "wav", - "xm", - "flac", - "3gp", - "3g2", - "h261", - "h263", - "h264", - "jpgv", - "jpm", - "mj2", - "mp4", - "mpeg", - "ogv", - "qt", - "uvh", - "uvm", - "uvp", - "uvs", - "dvb", - "fvt", - "mxu", - "pyv", - "uvu", - "viv", - "webm", - "f4v", - "fli", - "flv", - "m4v", - "mkv", - "mng", - "asf", - "vob", - "wm", - "wmv", - "wmx", - "wvx", - "movie", - "smv", - "bmp", - "cgm", - "g3", - "gif", - "ief", - "jpg", - "jpeg", - "ktx", - "png", - "btif", - "sgi", - "tiff", - "psd", - "uvi", - "sub", - "djvu", - "dwg", - "dxf", - "fbs", - "fpx", - "fst", - "mmr", - "rlc", - "mdi", - "wdp", - "npx", - "wbmp", - "xif", - "webp", - "3ds", - "ras", - "cmx", - "fh", - "ico", - "pcx", - "pic", - "pnm", - "pbm", - "pgm", - "ppm", - "rgb", - "tga", - "xbm", - "xpm", - "xwd", - "xz", - "zip", - "rar", - "tar", - "tbz2", - "tgz", - "txz", - "bz2", - "eot", - "ttf", - "woff", - "dat", - "nexe", - "pexe", - "epub", - "gz", - "mp3", - "ogg", - "swf", - "mem" - ] -} diff --git a/lib/preprocessor.js b/lib/preprocessor.js index 0a8c815d2..8da50338e 100644 --- a/lib/preprocessor.js +++ b/lib/preprocessor.js @@ -1,8 +1,7 @@ -var path = require('path') var fs = require('graceful-fs') var crypto = require('crypto') var mm = require('minimatch') -var extensions = require('./binary-extensions.json').extensions +var isBinaryFile = require('isbinaryfile') var log = require('./logger').create('preprocess') @@ -12,10 +11,30 @@ var sha1 = function (data) { return hash.digest('hex') } -var isBinary = Object.create(null) -extensions.forEach(function (extension) { - isBinary['.' + extension] = true -}) +var createNextProcessor = function (preprocessors, file, done) { + return function nextPreprocessor (error, content) { + // normalize B-C + if (arguments.length === 1 && typeof error === 'string') { + content = error + error = null + } + + if (error) { + file.content = null + file.contentPath = null + return done(error) + } + + if (!preprocessors.length) { + file.contentPath = null + file.content = content + file.sha = sha1(content) + return done() + } + + preprocessors.shift()(content, file, nextPreprocessor) + } +} var createPreprocessor = function (config, basePath, injector) { var alreadyDisplayedWarnings = {} @@ -51,64 +70,49 @@ var createPreprocessor = function (config, basePath, injector) { return function preprocess (file, done) { patterns = Object.keys(config) - var thisFileIsBinary = isBinary[path.extname(file.originalPath).toLowerCase()] - var preprocessors = [] - - var nextPreprocessor = function (error, content) { - // normalize B-C - if (arguments.length === 1 && typeof error === 'string') { - content = error - error = null - } - - if (error) { - file.content = null - file.contentPath = null - return done(error) - } - if (!preprocessors.length) { - file.contentPath = null - file.content = content - file.sha = sha1(content) - return done() + return fs.readFile(file.originalPath, function (err, buffer) { + if (err) { + throw err } - preprocessors.shift()(content, file, nextPreprocessor) - } - - for (var i = 0; i < patterns.length; i++) { - if (mm(file.originalPath, patterns[i], {dot: true})) { - if (thisFileIsBinary) { - log.warn('Ignoring preprocessing (%s) %s because it is a binary file.', - config[patterns[i]].join(', '), file.originalPath) - } else { - config[patterns[i]].forEach(function (name) { - var p = instances[name] - if (p == null) { - p = instantiatePreprocessor(name) - } + isBinaryFile(buffer, buffer.length, function (err, thisFileIsBinary) { + if (err) { + throw err + } - if (p == null) { - if (!alreadyDisplayedWarnings[name]) { - alreadyDisplayedWarnings[name] = true - log.warn('Failed to instantiate preprocessor %s', name) - } - return + var preprocessors = [] + var nextPreprocessor = createNextProcessor(preprocessors, file, done) + + for (var i = 0; i < patterns.length; i++) { + if (mm(file.originalPath, patterns[i], {dot: true})) { + if (thisFileIsBinary) { + log.warn('Ignoring preprocessing (%s) %s because it is a binary file.', + config[patterns[i]].join(', '), file.originalPath) + } else { + config[patterns[i]].forEach(function (name) { + var p = instances[name] + if (p == null) { + p = instantiatePreprocessor(name) + } + + if (p == null) { + if (!alreadyDisplayedWarnings[name]) { + alreadyDisplayedWarnings[name] = true + log.warn('Failed to instantiate preprocessor %s', name) + } + return + } + + instances[name] = p + preprocessors.push(p) + }) } - - instances[name] = p - preprocessors.push(p) - }) + } } - } - } - return fs.readFile(file.originalPath, function (err, buffer) { - if (err) { - throw err - } - nextPreprocessor(null, thisFileIsBinary ? buffer : buffer.toString()) + nextPreprocessor(null, thisFileIsBinary ? buffer : buffer.toString()) + }) }) } } diff --git a/package.json b/package.json index bf5315612..e0d249fc2 100644 --- a/package.json +++ b/package.json @@ -265,6 +265,7 @@ "glob": "^6.0.3", "graceful-fs": "^4.1.2", "http-proxy": "^1.11.1", + "isbinaryfile": "^3.0.0", "lodash": "^3.8.0", "log4js": "^0.6.28", "mime": "^1.3.4", diff --git a/test/unit/preprocessor.spec.js b/test/unit/preprocessor.spec.js index 1f3c10ba2..c0ba52d40 100644 --- a/test/unit/preprocessor.spec.js +++ b/test/unit/preprocessor.spec.js @@ -6,14 +6,17 @@ describe('preprocessor', () => { var m var mockFs + // mimic first few bytes of a pdf file + var binarydata = new Buffer([0x25, 0x50, 0x44, 0x66, 0x46, 0x00]) + beforeEach(() => { mockFs = mocks.fs.create({ some: { 'a.js': mocks.fs.file(0, 'content'), 'b.js': mocks.fs.file(0, 'content'), 'a.txt': mocks.fs.file(0, 'some-text'), - 'photo.png': mocks.fs.file(0, 'binary'), - 'CAM_PHOTO.JPG': mocks.fs.file(0, 'binary'), + 'photo.png': mocks.fs.file(0, binarydata), + 'CAM_PHOTO.JPG': mocks.fs.file(0, binarydata), '.dir': { 'a.js': mocks.fs.file(0, 'content') }