feat(reply): Response headers to more closely match Node's functional…

…ity. (#1564) * Response headers to more closely match Node. Updates the header handling in the `Interceptor` and `RequestOverrider` with the intention of mimicking the native behavior of `http.IncomingMessage.rawHeaders`. > The raw request/response headers list exactly as they were received. There are three fundamental changes in this changeset: 1) Header Input Type Previously, headers could be provided to: - `Scope.defaultReplyHeaders` as a plain object - `Interceptor.reply(status, body, headers)` as a plain object or an array of raw headers - `Interceptor.reply(() => [status, body, headers]` as a plain object Now, all three allow consistent inputs where the headers can be provided as a plain object, an array of raw headers, or a `Map`. 2) Duplicate Headers Folding This change deviates from the suggested guidelines laid out in #1553 because those guidelines didn't properly handle duplicate headers, especially when some are defined as defaults. This change was modeled to duplicate [Node's implementation](https://github.com/nodejs/node/blob/908292cf1f551c614a733d858528ffb13fb3a524/lib/_http_incoming.js#L245) ([relevant docs](https://nodejs.org/api/http.html#http_message_headers)). It specifically lays out how duplicate headers are handled depending on the field name. In the case of default headers, they are not included on the `Response` (not even in the raw headers) if the field name exists in the reply headers (using a case-insensitive comparison). 3) Raw Headers are the Source of Truth Previously, the `Interceptor` and `RequestOverrider` mostly keep track of headers as a plain object and the array of raw headers was created by looping that object. This was the cause for inconsistencies with the final result of the raw headers. The problem with that approach is that converting raw headers to an object is a lossy process, so going backwards makes it impossible to guarantee the correct results. This change reverses that logic and now the `Interceptor` and `RequestOverrider` maintain the header data in raw arrays. All additions to headers are only added to raw headers. The plain headers object is never mutated directly, and instead is [re]created from the raw headers as needed.
nock · Jun 8, 2019 · b687592 · b687592
1 parent 1fae9be
commit b687592
Show file tree

Hide file tree

Showing 10 changed files with 514 additions and 198 deletions.
diff --git a/lib/common.js b/lib/common.js
@@ -186,23 +186,19 @@ function stringifyRequest(options, body) {
 }
 
 function isContentEncoded(headers) {
-  const contentEncoding = _.get(headers, 'content-encoding')
+  const contentEncoding = headers['content-encoding']
   return _.isString(contentEncoding) && contentEncoding !== ''
 }
 
 function contentEncoding(headers, encoder) {
-  const contentEncoding = _.get(headers, 'content-encoding')
+  const contentEncoding = headers['content-encoding']
   return contentEncoding === encoder
 }
 
 function isJSONContent(headers) {
-  let contentType = _.get(headers, 'content-type')
-  if (Array.isArray(contentType)) {
-    contentType = contentType[0]
-  }
-  contentType = (contentType || '').toLocaleLowerCase()
-
-  return contentType === 'application/json'
+  // https://tools.ietf.org/html/rfc8259
+  const contentType = (headers['content-type'] || '').toLowerCase()
+  return contentType.startsWith('application/json')
 }
 
 const headersFieldNamesToLowerCase = function(headers) {
@@ -236,26 +232,139 @@ const headersFieldsArrayToLowerCase = function(headers) {
   )
 }
 
+/**
+ * Converts the various accepted formats of headers into a flat array representing "raw headers".
+ *
+ * Nock allows headers to be provided as a raw array, a plain object, or a Map.
+ *
+ * While all the header names are expected to be strings, the values are left intact as they can
+ * be functions, strings, or arrays of strings.
+ *
+ *  https://nodejs.org/api/http.html#http_message_rawheaders
+ */
+const headersInputToRawArray = function(headers) {
+  if (headers === undefined) {
+    return []
+  }
+
+  if (Array.isArray(headers)) {
+    // If the input is an array, assume it's already in the raw format and simply return a copy
+    // but throw an error if there aren't an even number of items in the array
+    if (headers.length % 2) {
+      throw new Error(
+        `Raw headers must be provided as an array with an even number of items. [fieldName, value, ...]`
+      )
+    }
+    return [...headers]
+  }
+
+  // [].concat(...) is used instead of Array.flat until v11 is the minimum Node version
+  if (_.isMap(headers)) {
+    return [].concat(...Array.from(headers, ([k, v]) => [k.toString(), v]))
+  }
+
+  if (_.isPlainObject(headers)) {
+    return [].concat(...Object.entries(headers))
+  }
+
+  throw new Error(
+    `Headers must be provided as an array of raw values, a Map, or a plain Object. ${headers}`
+  )
+}
+
+/**
+ * Converts an array of raw headers to an object, using the same rules as Nodes `http.IncomingMessage.headers`.
+ *
+ * Header names/keys are lower-cased.
+ */
 const headersArrayToObject = function(rawHeaders) {
   if (!Array.isArray(rawHeaders)) {
     throw Error('Expected a header array')
   }
 
-  const headers = {}
+  const accumulator = {}
 
-  for (let i = 0, len = rawHeaders.length; i < len; i = i + 2) {
-    const key = rawHeaders[i].toLowerCase()
-    const value = rawHeaders[i + 1]
+  forEachHeader(rawHeaders, (value, fieldName) => {
+    addHeaderLine(accumulator, fieldName, value)
+  })
+
+  return accumulator
+}
 
-    if (headers[key]) {
-      headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]]
-      headers[key].push(value)
+const noDuplicatesHeaders = new Set([
+  'age',
+  'authorization',
+  'content-length',
+  'content-type',
+  'etag',
+  'expires',
+  'from',
+  'host',
+  'if-modified-since',
+  'if-unmodified-since',
+  'last-modified',
+  'location',
+  'max-forwards',
+  'proxy-authorization',
+  'referer',
+  'retry-after',
+  'user-agent',
+])
+
+/**
+ * Set key/value data in accordance with Node's logic for folding duplicate headers.
+ *
+ * The `value` param should be a function, string, or array of strings.
+ *
+ * Node's docs and source:
+ * https://nodejs.org/api/http.html#http_message_headers
+ * https://github.com/nodejs/node/blob/908292cf1f551c614a733d858528ffb13fb3a524/lib/_http_incoming.js#L245
+ *
+ * Header names are lower-cased.
+ * Duplicates in raw headers are handled in the following ways, depending on the header name:
+ * - Duplicates of field names listed in `noDuplicatesHeaders` (above) are discarded.
+ * - `set-cookie` is always an array. Duplicates are added to the array.
+ * - For duplicate `cookie` headers, the values are joined together with '; '.
+ * - For all other headers, the values are joined together with ', '.
+ *
+ * Node's implementation is larger because it highly optimizes for not having to call `toLowerCase()`.
+ * We've opted to always call `toLowerCase` in exchange for a more concise function.
+ *
+ * While Node has the luxury of knowing `value` is always a string, we do an extra step of coercion at the top.
+ */
+const addHeaderLine = function(headers, name, value) {
+  let values // code below expects `values` to be an array of strings
+  if (typeof value === 'function') {
+    // Function values are evaluated towards the end of the response, before that we use a placeholder
+    // string just to designate that the header exists. Useful when `Content-Type` is set with a function.
+    values = [value.name]
+  } else if (Array.isArray(value)) {
+    values = value.map(String)
+  } else {
+    values = [String(value)]
+  }
+
+  const key = name.toLowerCase()
+  if (key === 'set-cookie') {
+    // Array header -- only Set-Cookie at the moment
+    if (headers['set-cookie'] === undefined) {
+      headers['set-cookie'] = values
     } else {
-      headers[key] = value
+      headers['set-cookie'].push(...values)
+    }
+  } else if (noDuplicatesHeaders.has(key)) {
+    if (headers[key] === undefined) {
+      // Drop duplicates
+      headers[key] = values[0]
+    }
+  } else {
+    if (headers[key] !== undefined) {
+      values = [headers[key], ...values]
     }
-  }
 
-  return headers
+    const separator = key === 'cookie' ? '; ' : ', '
+    headers[key] = values.join(separator)
+  }
 }
 
 /**
@@ -284,11 +393,25 @@ const deleteHeadersField = function(headers, fieldNameToDelete) {
       if (lowerCaseFieldName === lowerCaseFieldNameToDelete) {
         delete headers[fieldName]
         //  We don't stop here but continue in order to remove *all* matching field names
-        //  (even though if seen regorously there shouldn't be any)
+        //  (even though if seen rigorously there shouldn't be any)
       }
     })
 }
 
+/**
+ * Utility for iterating over a raw headers array.
+ *
+ * The callback is called with:
+ *  - The header value. string, array of strings, or a function
+ *  - The header field name. string
+ *  - Index of the header field in the raw header array.
+ */
+const forEachHeader = function(rawHeaders, callback) {
+  for (let i = 0; i < rawHeaders.length; i += 2) {
+    callback(rawHeaders[i + 1], rawHeaders[i], i)
+  }
+}
+
 function percentDecode(str) {
   try {
     return decodeURIComponent(str.replace(/\+/g, ' '))
@@ -391,7 +514,9 @@ exports.isJSONContent = isJSONContent
 exports.headersFieldNamesToLowerCase = headersFieldNamesToLowerCase
 exports.headersFieldsArrayToLowerCase = headersFieldsArrayToLowerCase
 exports.headersArrayToObject = headersArrayToObject
+exports.headersInputToRawArray = headersInputToRawArray
 exports.deleteHeadersField = deleteHeadersField
+exports.forEachHeader = forEachHeader
 exports.percentEncode = percentEncode
 exports.percentDecode = percentDecode
 exports.matchStringOrRegexp = matchStringOrRegexp

diff --git a/lib/interceptor.js b/lib/interceptor.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const mixin = require('./mixin')
 const matchBody = require('./match_body')
 const common = require('./common')
 const _ = require('lodash')
@@ -103,64 +102,50 @@ Interceptor.prototype.reply = function reply(statusCode, body, rawHeaders) {
 
   _.defaults(this.options, this.scope.scopeOptions)
 
-  // If needed, convert rawHeaders from Array to Object.
-  let headers = Array.isArray(rawHeaders)
-    ? common.headersArrayToObject(rawHeaders)
-    : rawHeaders
-
-  if (this.scope._defaultReplyHeaders) {
-    headers = headers || {}
-    // Because of this, this.rawHeaders gets lower-cased versions of the `rawHeaders` param.
-    // https://github.com/nock/nock/issues/1553
-    headers = common.headersFieldNamesToLowerCase(headers)
-    headers = mixin(this.scope._defaultReplyHeaders, headers)
-  }
+  this.rawHeaders = common.headersInputToRawArray(rawHeaders)
 
   if (this.scope.date) {
-    headers = headers || {}
-    headers['date'] = this.scope.date.toUTCString()
+    // https://tools.ietf.org/html/rfc7231#section-7.1.1.2
+    this.rawHeaders.push('Date', this.scope.date.toUTCString())
   }
 
-  if (headers !== undefined) {
-    this.rawHeaders = []
-
-    for (const key in headers) {
-      this.rawHeaders.push(key, headers[key])
-    }
-
-    // We use lower-case headers throughout Nock.
-    this.headers = common.headersFieldNamesToLowerCase(headers)
-
-    debug('reply.headers:', this.headers)
-    debug('reply.rawHeaders:', this.rawHeaders)
-  }
+  // Prepare the headers temporarily so we can make best guesses about content-encoding and content-type
+  // below as well as while the response is being processed in RequestOverrider.end().
+  // Including all the default headers is safe for our purposes because of the specific headers we introspect.
+  // A more thoughtful process is used to merge the default headers when the response headers are finally computed.
+  this.headers = common.headersArrayToObject(
+    this.rawHeaders.concat(this.scope._defaultReplyHeaders)
+  )
 
   //  If the content is not encoded we may need to transform the response body.
   //  Otherwise we leave it as it is.
   if (
     body &&
     typeof body !== 'string' &&
-    typeof body !== 'function' &&
     !Buffer.isBuffer(body) &&
     !common.isStream(body) &&
     !common.isContentEncoded(this.headers)
   ) {
     try {
       body = stringify(body)
-      if (!this.headers) {
-        this.headers = {}
-      }
-      if (!this.headers['content-type']) {
-        this.headers['content-type'] = 'application/json'
-      }
-      if (this.scope.contentLen) {
-        this.headers['content-length'] = body.length
-      }
     } catch (err) {
       throw new Error('Error encoding response body into JSON')
     }
+
+    if (!this.headers['content-type']) {
+      // https://tools.ietf.org/html/rfc7231#section-3.1.1.5
+      this.rawHeaders.push('Content-Type', 'application/json')
+    }
+
+    if (this.scope.contentLen) {
+      // https://tools.ietf.org/html/rfc7230#section-3.3.2
+      this.rawHeaders.push('Content-Length', body.length)
+    }
   }
 
+  debug('reply.headers:', this.headers)
+  debug('reply.rawHeaders:', this.rawHeaders)
+
   this.body = body
 
   this.scope.add(this._key, this, this.scope, this.scopeOptions)

diff --git a/lib/mixin.js b/lib/mixin.js
diff --git a/lib/recorder.js b/lib/recorder.js
@@ -46,7 +46,7 @@ const getBodyFromChunks = function(chunks, headers) {
   //  If we have headers and there is content-encoding it means that
   //  the body shouldn't be merged but instead persisted as an array
   //  of hex strings so that the responses can be mocked one by one.
-  if (common.isContentEncoded(headers)) {
+  if (headers && common.isContentEncoded(headers)) {
     return {
       body: _.map(chunks, chunk => chunk.toString('hex')),
     }