feat(normalization): archive

Also, normalization functions can now be async

bin/record.js

@@ -66,10 +66,10 @@ scenarios.reduce(async (promise, scenarioPath) => {
     ids: {}
   }
 
-  const newNormalizedFixtures = newRawFixtures
+  const newNormalizedFixtures = await Promise.all(newRawFixtures
     .map(cloneDeep)
     .filter(hasntIgnoreHeader)
-    .map(normalize.bind(null, scenarioState))
+    .map(normalize.bind(null, scenarioState)))
 
   const fixturesDiffs = diff(newNormalizedFixtures, oldNormalizedFixtures)
   if (!fixturesDiffs) {

lib/normalize/archive.js

@@ -0,0 +1,62 @@
+module.exports = normalizeArchive
+
+const zlib = require('zlib')
+
+const intoStream = require('into-stream')
+const getStream = require('get-stream')
+const tar = require('tar-stream')
+const gunzip = require('gunzip-maybe')
+
+const temporaryRepository = require('../temporary-repository')
+
+async function normalizeArchive (scenarioState, response, fixture) {
+  fixture.headers['content-disposition'] = fixture.headers['content-disposition']
+    // normalize folder name in file name
+    .replace(temporaryRepository.regex, '$1')
+    // zerofy sha
+    .replace(/archive-\w{7}/, 'archive-0000000')
+
+  const extract = tar.extract()
+  const pack = tar.pack()
+  const readStream = intoStream(Buffer.from(response, 'hex'))
+
+  // The response is the Repository folder with the README.md file inside. The
+  // folder name is always different, based on the repository name when recorded.
+  // That's why we have to untar/zip the response, change the folder name and
+  // retar/zip it again.
+
+  extract.on('entry', function (header, stream, callback) {
+    header.name = header.name
+      // normalize folder name in path
+      .replace(temporaryRepository.regex, '$1')
+      // zerofy sha in path
+      .replace(/-(\w){7}\//, '-0000000/')
+
+    // normalize mtime
+    header.mtime = {
+      getTime: () => 1507651200000
+    }
+
+    // write the new entry to the pack stream
+    stream.pipe(pack.entry(header, callback))
+  })
+
+  extract.on('finish', function () {
+    // all entries done - lets finalize it
+    pack.finalize()
+  })
+
+  // pipe the old tarball to the extractor
+  readStream.pipe(gunzip()).pipe(extract)
+
+  // pipe the new tarball the another stream
+  const writeStream = pack.pipe(zlib.createGzip())
+
+  const result = await getStream.buffer(writeStream).catch(console.log)
+  fixture.response = result.toString('hex')
+
+  // normalize across operating systems / extra flags
+  // see http://www.zlib.org/rfc-gzip.html#header-trailer
+  const normalizedHeader = '1f8b0800000000000003'
+  fixture.response = normalizedHeader + fixture.response.substr(normalizedHeader.length)
+}

lib/normalize/index.js

@@ -8,7 +8,7 @@ const headers = require('../headers')
 const setIfExists = require('../set-if-exists')
 const toEntityName = require('../to-entity-name')
 
-function normalize (scenarioState, fixture) {
+async function normalize (scenarioState, fixture) {
   // fixture.rawHeaders is an array in the form of ['key1', 'value1', 'key2', 'value2']
   // But the order of these can change, e.g. between local tests and CI on Travis.
   // That’s why we turn them into an object before storing the fixtures and turn
@@ -65,12 +65,12 @@ function normalize (scenarioState, fixture) {
   }
 
   const responses = Array.isArray(fixture.response) ? fixture.response : [fixture.response]
-  responses.forEach(response => {
+  await Promise.all(responses.map(async (response) => {
     const entityName = toEntityName(response, fixture)
     if (entityName) {
-      require(`./${entityName}`)(scenarioState, response, fixture)
+      await require(`./${entityName}`)(scenarioState, response, fixture)
     }
-  })
+  }))
 
   // remove headers added by proxy
   // see https://github.com/octokit/fixtures/pull/20#issuecomment-331558385
@@ -81,6 +81,10 @@ function normalize (scenarioState, fixture) {
     fixture.headers['content-length'] = String(calculateBodyLength(fixture.response))
   }
 
+  if (fixture.responseIsBinary) {
+    fixture.headers['content-length'] = Buffer.from(fixture.response, 'hex').length
+  }
+
   // remove `Transfer-Encoding: chunked` headers:
   // https://github.com/octokit/fixtures/issues/97octokit/fixtures#97
   if (fixture.headers['transfer-encoding'] === 'chunked') {

lib/to-entity-name.js

@@ -1,9 +1,15 @@
 module.exports = toEntityName
 
 function toEntityName (object, fixture) {
+  // object is binary response, so we check for it above the object check
+  if (/\/legacy\.(tar\.gz|zip)\/master$/.test(fixture.path)) {
+    return 'archive'
+  }
+
   if (typeof object !== 'object') {
     return
   }
+
   if (object.type === 'Organization') {
     return 'organization'
   }
@@ -52,6 +58,7 @@ function toEntityName (object, fixture) {
   if (/^\/search\/issues\?/.test(fixture.path)) {
     return 'search-issues'
   }
+
   if ('errors' in object) {
     return 'error'
   }