diff --git a/documentation/advanced-creation.md b/documentation/advanced-creation.md index 640b31943..482db6900 100644 --- a/documentation/advanced-creation.md +++ b/documentation/advanced-creation.md @@ -2,119 +2,6 @@ > Make calling REST APIs easier by creating niche-specific `got` instances. -#### got.create(settings) - -Example: [gh-got](https://github.com/sindresorhus/gh-got/blob/master/index.js) - -Configures a new `got` instance with the provided settings. You can access the resolved options with the `.defaults` property on the instance. - -**Note:** In contrast to [`got.extend()`](../readme.md#gotextendinstances), this method has no defaults. - -##### [options](readme.md#options) - -To inherit from the parent, set it to `got.defaults.options` or use [`got.mergeOptions(defaults.options, options)`](../readme.md#gotmergeoptionsparentoptions-newoptions).
-**Note:** Avoid using [object spread](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_syntax#Spread_in_object_literals) as it doesn't work recursively. - -##### mutableDefaults - -Type: `boolean`
-Default: `false` - -States if the defaults are mutable. It can be useful when you need to [update headers over time](readme.md#hooksafterresponse), for example, update an access token when it expires. - -##### handlers - -Type: `Function[]`
-Default: `[]` - -An array of functions. You execute them directly by calling `got()`. They are some sort of "global hooks" - these functions are called first. The last handler (*it's hidden*) is either [`asPromise`](../source/as-promise.ts) or [`asStream`](../source/as-stream.ts), depending on the `options.stream` property. - -To inherit from the parent, set it as `got.defaults.handlers`.
-To use the default handler, just omit specifying this. - -Each handler takes two arguments: - -###### [options](readme.md#options) - -**Note:** These options are [normalized](source/normalize-arguments.js). - -###### next() - -Returns a `Promise` or a `Stream` depending on [`options.stream`](readme.md#stream). - -```js -const settings = { - handlers: [ - (options, next) => { - if (options.stream) { - // It's a Stream, so we can perform stream-specific actions on it - return next(options) - .on('request', request => { - setTimeout(() => { - request.abort(); - }, 50); - }); - } - - // It's a Promise - return next(options); - } - ], - options: got.mergeOptions(got.defaults.options, { - responseType: 'json' - }) -}; - -const jsonGot = got.create(settings); -``` - -Sometimes you don't need to use `got.create(defaults)`. You should go for `got.extend(options)` if you don't want to overwrite the defaults: - -```js -const settings = { - handler: got.defaults.handler, - options: got.mergeOptions(got.defaults.options, { - headers: { - unicorn: 'rainbow' - } - }) -}; - -const unicorn = got.create(settings); - -// Same as: -const unicorn = got.extend({headers: {unicorn: 'rainbow'}}); -``` - -**Note:** Handlers can be asynchronous. The recommended approach is: - -```js -const handler = (options, next) => { - if (options.stream) { - // It's a Stream - return next(options); - } - - // It's a Promise - return (async () => { - try { - const response = await next(options); - - response.yourOwnProperty = true; - - return response; - } catch (error) { - // Every error will be replaced by this one. - // Before you receive any error here, - // it will be passed to the `beforeError` hooks first. - - // Note: this one won't be passed to `beforeError` hook. It's final. - throw new Error('Your very own error.'); - } - })(); -}; -``` - ### Merging instances Got supports composing multiple instances together. This is very powerful. You can create a client that limits download speed and then compose it with an instance that signs a request. It's like plugins without any of the plugin mess. You just create instances and then compose them together. diff --git a/documentation/examples/gh-got.js b/documentation/examples/gh-got.js index 083438292..51a234cf1 100644 --- a/documentation/examples/gh-got.js +++ b/documentation/examples/gh-got.js @@ -24,7 +24,7 @@ const instance = got.extend({ } // Don't touch streams - if (options.stream) { + if (options.isStream) { return next(options); } diff --git a/documentation/lets-make-a-plugin.md b/documentation/lets-make-a-plugin.md index 5ab435380..8ba883b2c 100644 --- a/documentation/lets-make-a-plugin.md +++ b/documentation/lets-make-a-plugin.md @@ -128,7 +128,7 @@ We should name our errors, just to know if the error is from the API response. S } // Don't touch streams - if (options.stream) { + if (options.isStream) { return next(options); } @@ -197,7 +197,7 @@ const getRateLimit = ({headers}) => ({ } // Don't touch streams - if (options.stream) { + if (options.isStream) { return next(options); } diff --git a/readme.md b/readme.md index 4897c4c1f..b182f0c6e 100644 --- a/readme.md +++ b/readme.md @@ -108,7 +108,7 @@ It's a `GET` request by default, but can be changed by using different methods o #### got([url], [options]) -Returns a Promise for a [`response` object](#response) or a [stream](#streams-1) if `options.stream` is set to true. +Returns a Promise for a [`response` object](#response) or a [stream](#streams-1) if `options.isStream` is set to true. ##### url @@ -138,12 +138,29 @@ Useful when used with `got.extend()` to create niche-specific Got-instances. **Note:** `prefixUrl` will be ignored if the `url` argument is a URL instance. +**Tip:** If the input URL still contains the initial `prefixUrl`, you can change it as many times as you want. Otherwise it will throw an error. + ```js const got = require('got'); (async () => { await got('unicorn', {prefixUrl: 'https://cats.com'}); //=> 'https://cats.com/unicorn' + + const instance = got.extend({ + prefixUrl: 'https://google.com' + }); + + await instance('unicorn', { + hooks: { + beforeRequest: [ + options => { + options.prefixUrl = 'https://cats.com'; + } + ] + } + }); + //=> 'https://cats.com/unicorn' })(); ``` @@ -154,9 +171,9 @@ Default: `{}` Request headers. -Existing headers will be overwritten. Headers set to `null` will be omitted. +Existing headers will be overwritten. Headers set to `undefined` will be omitted. -###### stream +###### isStream Type: `boolean`
Default: `false` @@ -225,11 +242,18 @@ const instance = got.extend({ ###### responseType Type: `string`
-Default: `text` +Default: `'default'` **Note:** When using streams, this option is ignored. -Parsing method used to retrieve the body from the response. Can be `text`, `json` or `buffer`. The promise has `.json()` and `.buffer()` and `.text()` functions which set this option automatically. +Parsing method used to retrieve the body from the response. + +- `'default'` - if `options.encoding` is `null`, the body will be a Buffer. Otherwise it will be a string unless it's overwritten in a `afterResponse` hook, +- `'text'` - will always give a string, no matter what's the `options.encoding` or if the body is a custom object, +- `'json'` - will always give an object, unless it's invalid JSON - then it will throw. +- `'buffer'` - will always give a Buffer, no matter what's the `options.encoding`. It will throw if the body is a custom object. + +The promise has `.json()` and `.buffer()` and `.text()` functions which set this option automatically. Example: @@ -246,11 +270,23 @@ When set to `true` the promise will return the [Response body](#body-1) instead ###### cookieJar -Type: [`tough.CookieJar` instance](https://github.com/salesforce/tough-cookie#cookiejar) +Type: `object` | [`tough.CookieJar` instance](https://github.com/salesforce/tough-cookie#cookiejar) **Note:** If you provide this option, `options.headers.cookie` will be overridden. -Cookie support. You don't have to care about parsing or how to store them. [Example.](#cookies) +Cookie support. You don't have to care about parsing or how to store them. [Example](#cookies). + +###### cookieJar.setCookie + +Type: `Function` + +The function takes two arguments: `rawCookie` (`string`) and `url` (`string`). + +###### cookieJar.getCookieString + +Type: `Function` + +The function takes one argument: `url` (`string`). ###### ignoreInvalidCookies @@ -280,8 +316,6 @@ If set to `true` and the `Content-Type` header is not set, it will be set to `ap Type: `string | object | URLSearchParams` -**Note:** The `query` option was renamed to `searchParams` in Got v10. The `query` option name is still functional, but is being deprecated and will be completely removed in Got v11. - Query string that will be added to the request URL. This will override the query string in `url`. If you need to pass in an array, you can do it using a `URLSearchParams` instance: @@ -518,6 +552,8 @@ got.post('https://example.com', { }); ``` +**Note:** When retrying in a `afterResponse` hook, all remaining `beforeRetry` hooks will be called without the `error` and `retryCount` arguments. + ###### hooks.afterResponse Type: `Function[]`
@@ -525,7 +561,7 @@ Default: `[]` **Note:** When using streams, this hook is ignored. -Called with [response object](#response) and a retry function. +Called with [response object](#response) and a retry function. Calling the retry function will trigger `beforeRetry` hooks. Each function should return the response. This is especially useful when you want to refresh an access token. Example: @@ -553,6 +589,11 @@ const instance = got.extend({ // No changes otherwise return response; } + ], + beforeRetry: [ + (options, error, retryCount) => { + // This will be called on `retryWithMergedOptions(...)` + } ] }, mutableDefaults: true @@ -677,7 +718,7 @@ The number of times the request was retried. #### got.stream(url, [options]) -Sets `options.stream` to `true`. +Sets `options.isStream` to `true`. Returns a [duplex stream](https://nodejs.org/api/stream.html#stream_class_stream_duplex) with additional events: @@ -789,8 +830,6 @@ client.get('/demo'); })(); ``` -**Tip:** Need more control over the behavior of Got? Check out the [`got.create()`](documentation/advanced-creation.md). - Additionally, `got.extend()` accepts two properties from the `defaults` object: `mutableDefaults` and `handlers`. Example: ```js @@ -808,6 +847,34 @@ const mergedHandlers = got.extend({ }); ``` +**Note:** Handlers can be asynchronous. The recommended approach is: + +```js +const handler = (options, next) => { + if (options.stream) { + // It's a Stream + return next(options); + } + + // It's a Promise + return (async () => { + try { + const response = await next(options); + response.yourOwnProperty = true; + return response; + } catch (error) { + // Every error will be replaced by this one. + // Before you receive any error here, + // it will be passed to the `beforeError` hooks first. + // Note: this one won't be passed to `beforeError` hook. It's final. + throw new Error('Your very own error.'); + } + })(); +}; + +const instance = got.extend({handlers: [handler]}); +``` + #### got.extend(...instances) Merges many instances into a single one: @@ -862,7 +929,57 @@ Options are deeply merged to a new object. The value of each key is determined a Type: `object` -The default Got options used in that instance. +The Got defaults used in that instance. + +##### [options](#options) + +##### handlers + +Type: `Function[]`
+Default: `[]` + +An array of functions. You execute them directly by calling `got()`. They are some sort of "global hooks" - these functions are called first. The last handler (*it's hidden*) is either [`asPromise`](source/as-promise.ts) or [`asStream`](source/as-stream.ts), depending on the `options.isStream` property. + +Each handler takes two arguments: + +###### [options](#options) + +###### next() + +Returns a `Promise` or a `Stream` depending on [`options.isStream`](#isstream). + +```js +const settings = { + handlers: [ + (options, next) => { + if (options.isStream) { + // It's a Stream, so we can perform stream-specific actions on it + return next(options) + .on('request', request => { + setTimeout(() => { + request.abort(); + }, 50); + }); + } + + // It's a Promise + return next(options); + } + ], + options: got.mergeOptions(got.defaults.options, { + responseType: 'json' + }) +}; + +const jsonGot = got.create(settings); +``` + +##### mutableDefaults + +Type: `boolean`
+Default: `false` + +A read-only boolean describing whether the defaults are mutable or not. If set to `true`, you can [update headers over time](#hooksafterresponse), for example, update an access token when it expires. ## Errors @@ -1229,7 +1346,7 @@ const got = require('got'); ### User Agent -It's a good idea to set the `'user-agent'` header so the provider can more easily see how their resource is used. By default, it's the URL to this repo. You can omit this header by setting it to `null`. +It's a good idea to set the `'user-agent'` header so the provider can more easily see how their resource is used. By default, it's the URL to this repo. You can omit this header by setting it to `undefined`. ```js const got = require('got'); @@ -1243,7 +1360,7 @@ got('https://sindresorhus.com', { got('https://sindresorhus.com', { headers: { - 'user-agent': null + 'user-agent': undefined } }); ``` diff --git a/source/as-promise.ts b/source/as-promise.ts index 68be175c7..df4d9a122 100644 --- a/source/as-promise.ts +++ b/source/as-promise.ts @@ -4,29 +4,36 @@ import getStream = require('get-stream'); import is from '@sindresorhus/is'; import PCancelable = require('p-cancelable'); import {NormalizedOptions, Response, CancelableRequest} from './utils/types'; -import {mergeOptions} from './merge'; import {ParseError, ReadError, HTTPError} from './errors'; -import {reNormalizeArguments} from './normalize-arguments'; -import requestAsEventEmitter from './request-as-event-emitter'; +import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; +import {normalizeArguments, mergeOptions} from './normalize-arguments'; -type ResponseReturn = Response | Buffer | string | any; +const parseBody = (body: Response['body'], responseType: NormalizedOptions['responseType'], statusCode: Response['statusCode']) => { + if (responseType === 'json' && is.string(body)) { + return statusCode === 204 ? '' : JSON.parse(body); + } -export const isProxiedSymbol: unique symbol = Symbol('proxied'); + if (responseType === 'buffer' && is.string(body)) { + return Buffer.from(body); + } -export default function asPromise(options: NormalizedOptions): CancelableRequest { - const proxy = new EventEmitter(); + if (responseType === 'text') { + return String(body); + } - const parseBody = (response: Response): void => { - if (options.responseType === 'json') { - response.body = JSON.parse(response.body); - } else if (options.responseType === 'buffer') { - response.body = Buffer.from(response.body); - } else if (options.responseType !== 'text' && !is.falsy(options.responseType)) { - throw new Error(`Failed to parse body of type '${options.responseType}'`); - } - }; + if (responseType === 'default') { + return body; + } + + throw new Error(`Failed to parse body of type '${typeof body}' as '${responseType}'`); +}; - const promise = new PCancelable((resolve, reject, onCancel) => { +export default function asPromise(options: NormalizedOptions) { + const proxy = new EventEmitter(); + let finalResponse: Pick; + + // @ts-ignore `.json()`, `.buffer()` and `.text()` are added later + const promise = new PCancelable((resolve, reject, onCancel) => { const emitter = requestAsEventEmitter(options); onCancel(emitter.abort); @@ -46,11 +53,10 @@ export default function asPromise(options: NormalizedOptions): CancelableRequest emitter.on('response', async (response: Response) => { proxy.emit('response', response); - const stream = is.null_(options.encoding) ? getStream.buffer(response) : getStream(response, {encoding: options.encoding}); + const streamAsPromise = is.null_(options.encoding) ? getStream.buffer(response) : getStream(response, {encoding: options.encoding}); - let data: Buffer | string; try { - data = await stream; + response.body = await streamAsPromise; } catch (error) { emitError(new ReadError(error, options)); return; @@ -61,15 +67,12 @@ export default function asPromise(options: NormalizedOptions): CancelableRequest return; } - const limitStatusCode = options.followRedirect ? 299 : 399; - - response.body = data; - try { for (const [index, hook] of options.hooks.afterResponse.entries()) { + // @ts-ignore // eslint-disable-next-line no-await-in-loop - response = await hook(response, updatedOptions => { - updatedOptions = reNormalizeArguments(mergeOptions(options, { + response = await hook(response, async (updatedOptions: NormalizedOptions) => { + updatedOptions = normalizeArguments(mergeOptions(options, { ...updatedOptions, retry: { calculateDelay: () => 0 @@ -83,7 +86,19 @@ export default function asPromise(options: NormalizedOptions): CancelableRequest // The loop continues. We don't want duplicates (asPromise recursion). updatedOptions.hooks.afterResponse = options.hooks.afterResponse.slice(0, index); - return asPromise(updatedOptions); + for (const hook of options.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(updatedOptions); + } + + const promise = asPromise(updatedOptions); + + onCancel(() => { + promise.catch(() => {}); + promise.cancel(); + }); + + return promise; }); } } catch (error) { @@ -93,18 +108,22 @@ export default function asPromise(options: NormalizedOptions): CancelableRequest const {statusCode} = response; - if (response.body) { - try { - parseBody(response); - } catch (error) { - if (statusCode >= 200 && statusCode < 300) { - const parseError = new ParseError(error, response, options); - emitError(parseError); - return; - } + finalResponse = { + body: response.body, + statusCode + }; + + try { + response.body = parseBody(response.body, options.responseType, response.statusCode); + } catch (error) { + if (statusCode >= 200 && statusCode < 300) { + const parseError = new ParseError(error, response, options); + emitError(parseError); + return; } } + const limitStatusCode = options.followRedirect ? 299 : 399; if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) { const error = new HTTPError(response, options); if (emitter.retry(error) === false) { @@ -124,44 +143,34 @@ export default function asPromise(options: NormalizedOptions): CancelableRequest emitter.once('error', reject); - const events = [ - 'request', - 'redirect', - 'uploadProgress', - 'downloadProgress' - ]; - - for (const event of events) { - emitter.on(event, (...args: unknown[]) => { - proxy.emit(event, ...args); - }); - } - }) as CancelableRequest; - - promise[isProxiedSymbol] = true; + proxyEvents(proxy, emitter); + }) as CancelableRequest; - promise.on = (name, fn) => { + promise.on = (name: string, fn: (...args: any[]) => void) => { proxy.on(name, fn); return promise; }; - promise.json = () => { - options.responseType = 'json'; - options.resolveBodyOnly = true; - return promise; - }; + const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { + // eslint-disable-next-line promise/prefer-await-to-then + const newPromise = promise.then(() => parseBody(finalResponse.body, responseType, finalResponse.statusCode)); - promise.buffer = () => { - options.responseType = 'buffer'; - options.resolveBodyOnly = true; - return promise; + Object.defineProperties(newPromise, Object.getOwnPropertyDescriptors(promise)); + + // @ts-ignore The missing properties are added above + return newPromise; }; - promise.text = () => { - options.responseType = 'text'; - options.resolveBodyOnly = true; - return promise; + promise.json = () => { + if (is.undefined(options.headers.accept)) { + options.headers.accept = 'application/json'; + } + + return shortcut('json'); }; + promise.buffer = () => shortcut('buffer'); + promise.text = () => shortcut('text'); + return promise; } diff --git a/source/as-stream.ts b/source/as-stream.ts index b03f69f8c..8a2fd906c 100644 --- a/source/as-stream.ts +++ b/source/as-stream.ts @@ -2,11 +2,11 @@ import {PassThrough as PassThroughStream, Duplex as DuplexStream} from 'stream'; import stream = require('stream'); import {IncomingMessage} from 'http'; import duplexer3 = require('duplexer3'); -import requestAsEventEmitter from './request-as-event-emitter'; +import requestAsEventEmitter, {proxyEvents} from './request-as-event-emitter'; import {HTTPError, ReadError} from './errors'; -import {NormalizedOptions, Response} from './utils/types'; +import {NormalizedOptions, Response, GotEvents} from './utils/types'; -export class ProxyStream extends DuplexStream { +export class ProxyStream extends DuplexStream implements GotEvents { isFromCache?: boolean; } @@ -14,7 +14,7 @@ export default function asStream(options: NormalizedOptions): ProxyStream { const input = new PassThroughStream(); const output = new PassThroughStream(); const proxy = duplexer3(input, output) as ProxyStream; - const piped = new Set(); // TODO: Should be `new Set();`. + const piped = new Set(); // TODO: Should be `new Set();`. let isFinished = false; options.retry.calculateDelay = () => 0; @@ -24,9 +24,16 @@ export default function asStream(options: NormalizedOptions): ProxyStream { proxy.destroy(); throw new Error('Got\'s stream is not writable when the `body` option is used'); }; + } else if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH') { + options.body = input; + } else { + proxy.write = () => { + proxy.destroy(); + throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); + }; } - const emitter = requestAsEventEmitter(options, input); + const emitter = requestAsEventEmitter(options); const emitError = async (error: Error): Promise => { try { @@ -94,19 +101,8 @@ export default function asStream(options: NormalizedOptions): ProxyStream { proxy.emit('response', response); }); - const events = [ - 'error', - 'request', - 'redirect', - 'uploadProgress', - 'downloadProgress' - ]; - - for (const event of events) { - emitter.on(event, (...args) => { - proxy.emit(event, ...args); - }); - } + proxyEvents(proxy, emitter); + emitter.on('error', (error: Error) => proxy.emit('error', error)); const pipe = proxy.pipe.bind(proxy); const unpipe = proxy.unpipe.bind(proxy); diff --git a/source/calculate-retry-delay.ts b/source/calculate-retry-delay.ts index 7f4bf3874..1c7f14b8f 100644 --- a/source/calculate-retry-delay.ts +++ b/source/calculate-retry-delay.ts @@ -16,8 +16,7 @@ const calculateRetryDelay: RetryFunction = ({attemptCount, retryOptions, error}) return 0; } - // TODO: This type coercion is not entirely correct as it makes `response` a guaranteed property, when it's in fact not. - const {response} = error as HTTPError | ParseError | MaxRedirectsError; + const {response} = error as HTTPError | ParseError | MaxRedirectsError | undefined; if (response && Reflect.has(response.headers, 'retry-after') && retryAfterStatusCodes.has(response.statusCode)) { let after = Number(response.headers['retry-after']); if (is.nan(after)) { diff --git a/source/create.ts b/source/create.ts index 8ca80ebf0..85366e906 100644 --- a/source/create.ts +++ b/source/create.ts @@ -1,3 +1,4 @@ +import {Merge} from 'type-fest'; import * as errors from './errors'; import { Options, @@ -6,16 +7,13 @@ import { Response, CancelableRequest, URLOrOptions, - URLArgument, HandlerFunction, - ExtendedOptions, - NormalizedDefaults + ExtendedOptions } from './utils/types'; import deepFreeze from './utils/deep-freeze'; -import merge, {mergeOptions} from './merge'; -import asPromise, {isProxiedSymbol} from './as-promise'; +import asPromise from './as-promise'; import asStream, {ProxyStream} from './as-stream'; -import {preNormalizeArguments, normalizeArguments} from './normalize-arguments'; +import {normalizeArguments, mergeOptions} from './normalize-arguments'; import {Hooks} from './known-hook-events'; export type HTTPAlias = @@ -26,13 +24,34 @@ export type HTTPAlias = | 'head' | 'delete'; -export type ReturnResponse = (url: URLArgument | Options & {stream?: false; url: URLArgument}, options?: Options & {stream?: false}) => CancelableRequest; -export type ReturnStream = (url: URLArgument | Options & {stream: true; url: URLArgument}, options?: Options & {stream: true}) => ProxyStream; +export type ReturnStream = (url: string | Options & {isStream: true}, options?: Options & {isStream: true}) => ProxyStream; export type GotReturn = ProxyStream | CancelableRequest; -const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.stream ? asStream(options) : asPromise(options); +const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? asStream(options) : asPromise(options); -export interface Got extends Record { +type OptionsOfDefaultResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType?: 'default'}; +type OptionsOfTextResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'text'}; +type OptionsOfJSONResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'json'}; +type OptionsOfBufferResponseBody = Options & {isStream?: false; resolveBodyOnly?: false; responseType: 'buffer'}; +type ResponseBodyOnly = {resolveBodyOnly: true}; + +interface GotFunctions { + // `asPromise` usage + (url: string | OptionsOfDefaultResponseBody, options?: OptionsOfDefaultResponseBody): CancelableRequest; + (url: string | OptionsOfTextResponseBody, options?: OptionsOfTextResponseBody): CancelableRequest>; + (url: string | OptionsOfJSONResponseBody, options?: OptionsOfJSONResponseBody): CancelableRequest>; + (url: string | OptionsOfBufferResponseBody, options?: OptionsOfBufferResponseBody): CancelableRequest>; + + (url: string | OptionsOfDefaultResponseBody & ResponseBodyOnly, options?: OptionsOfDefaultResponseBody & ResponseBodyOnly): CancelableRequest; + (url: string | OptionsOfTextResponseBody & ResponseBodyOnly, options?: OptionsOfTextResponseBody & ResponseBodyOnly): CancelableRequest; + (url: string | OptionsOfJSONResponseBody & ResponseBodyOnly, options?: OptionsOfJSONResponseBody & ResponseBodyOnly): CancelableRequest; + (url: string | OptionsOfBufferResponseBody & ResponseBodyOnly, options?: OptionsOfBufferResponseBody & ResponseBodyOnly): CancelableRequest; + + // `asStream` usage + (url: string | Options & {isStream: true}, options?: Options & {isStream: true}): ProxyStream; +} + +export interface Got extends Merge, GotFunctions> { stream: GotStream; defaults: Defaults | Readonly; GotError: typeof errors.GotError; @@ -46,10 +65,6 @@ export interface Got extends Record { TimeoutError: typeof errors.TimeoutError; CancelError: typeof errors.CancelError; - (url: URLArgument | Options & {stream?: false; url: URLArgument}, options?: Options & {stream?: false}): CancelableRequest; - (url: URLArgument | Options & {stream: true; url: URLArgument}, options?: Options & {stream: true}): ProxyStream; - (url: URLOrOptions, options?: Options): CancelableRequest | ProxyStream; - create(defaults: Defaults): Got; extend(...instancesOrOptions: Array): Got; mergeInstances(parent: Got, ...instances: Got[]): Got; mergeOptions(...sources: T[]): T & {hooks: Partial}; @@ -68,59 +83,43 @@ const aliases: readonly HTTPAlias[] = [ 'delete' ]; -const defaultHandler: HandlerFunction = (options, next) => next(options); +export const defaultHandler: HandlerFunction = (options, next) => next(options); -// `got.mergeInstances()` is deprecated -let hasShownDeprecation = false; +const create = (defaults: Defaults): Got => { + // Proxy properties from next handlers + defaults._rawHandlers = defaults.handlers; + defaults.handlers = defaults.handlers.map(fn => ((options, next) => { + let root: GotReturn; -const create = (nonNormalizedDefaults: Defaults): Got => { - const defaults: NormalizedDefaults = { - handlers: Reflect.has(nonNormalizedDefaults, 'handlers') ? merge([], nonNormalizedDefaults.handlers) : [defaultHandler], - options: preNormalizeArguments(mergeOptions(Reflect.has(nonNormalizedDefaults, 'options') ? nonNormalizedDefaults.options : {})), - mutableDefaults: Boolean(nonNormalizedDefaults.mutableDefaults) - }; + const result = fn(options, newOptions => { + root = next(newOptions); + return root; + }); + + if (result !== root && !options.isStream) { + Object.setPrototypeOf(result, Object.getPrototypeOf(root)); + Object.defineProperties(result, Object.getOwnPropertyDescriptors(root)); + } + + return result; + }) as HandlerFunction); // @ts-ignore Because the for loop handles it for us, as well as the other Object.defines const got: Got = (url: URLOrOptions, options?: Options): GotReturn => { - const isStream = options?.stream ?? false; - let iteration = 0; - const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { - let nextPromise: CancelableRequest; - const result = defaults.handlers[iteration++](newOptions, options => { - const fn = iteration === defaults.handlers.length ? getPromiseOrStream : iterateHandlers; - - if (isStream) { - return fn(options); - } - - // We need to remember the `next(options)` result. - nextPromise = fn(options) as CancelableRequest; - return nextPromise; - }); - - // Proxy the properties from the next handler to this one - if (!isStream && !Reflect.has(result, isProxiedSymbol)) { - for (const key of Object.keys(nextPromise)) { - Object.defineProperty(result, key, { - get: () => nextPromise[key], - set: (value: unknown) => { - nextPromise[key] = value; - } - }); - } - - (result as CancelableRequest).cancel = nextPromise.cancel; - result[isProxiedSymbol] = true; - } - - return result; + const iterateHandlers: HandlerFunction = newOptions => { + return defaults.handlers[iteration++]( + newOptions, + // @ts-ignore TS doesn't know that it calls `getPromiseOrStream` at the end + iteration === defaults.handlers.length ? getPromiseOrStream : iterateHandlers + ); }; try { - return iterateHandlers(normalizeArguments(url, options as NormalizedOptions, defaults)); + // @ts-ignore This handler takes only one parameter. + return iterateHandlers(normalizeArguments(url, options, defaults)); } catch (error) { - if (isStream) { + if (options?.isStream) { throw error; } else { // @ts-ignore It's an Error not a response, but TS thinks it's calling .resolve @@ -129,20 +128,20 @@ const create = (nonNormalizedDefaults: Defaults): Got => { } }; - got.create = create; got.extend = (...instancesOrOptions) => { - const options: Options[] = [defaults.options]; - const handlers: HandlerFunction[] = [...defaults.handlers]; + const optionsArray: Options[] = [defaults.options]; + let handlers: HandlerFunction[] = [...defaults._rawHandlers]; let mutableDefaults: boolean; for (const value of instancesOrOptions) { if (Reflect.has(value, 'defaults')) { - options.push((value as Got).defaults.options); - handlers.push(...(value as Got).defaults.handlers.filter(handler => handler !== defaultHandler)); + optionsArray.push((value as Got).defaults.options); + + handlers.push(...(value as Got).defaults._rawHandlers); mutableDefaults = (value as Got).defaults.mutableDefaults; } else { - options.push(value as Options); + optionsArray.push(value as ExtendedOptions); if (Reflect.has(value, 'handlers')) { handlers.push(...(value as ExtendedOptions).handlers); @@ -152,29 +151,25 @@ const create = (nonNormalizedDefaults: Defaults): Got => { } } - handlers.push(defaultHandler); + handlers = handlers.filter(handler => handler !== defaultHandler); + + if (handlers.length === 0) { + handlers.push(defaultHandler); + } return create({ - options: mergeOptions(...options), + options: mergeOptions(...optionsArray), handlers, mutableDefaults }); }; - got.mergeInstances = (parent, ...instances) => { - if (!hasShownDeprecation) { - console.warn('`got.mergeInstances()` is deprecated. We support it solely for compatibility - it will be removed in Got 11. Use `instance.extend(...instances)` instead.'); - hasShownDeprecation = true; - } - - return parent.extend(...instances); - }; - // @ts-ignore The missing methods because the for-loop handles it for us - got.stream = (url, options) => got(url, {...options, stream: true}); + got.stream = (url, options) => got(url, {...options, isStream: true}); for (const method of aliases) { - got[method] = (url, options) => got(url, {...options, method}); + // @ts-ignore + got[method] = (url: URLOrOptions, options?: Options): GotReturn => got(url, {...options, method}); got.stream[method] = (url, options) => got.stream(url, {...options, method}); } diff --git a/source/errors.ts b/source/errors.ts index 68248c1b0..b8807728a 100644 --- a/source/errors.ts +++ b/source/errors.ts @@ -1,4 +1,3 @@ -import {format} from 'url'; import is from '@sindresorhus/is'; import {Timings} from '@szmarczak/http-timer'; import {Response, NormalizedOptions} from './utils/types'; @@ -48,7 +47,7 @@ export class ParseError extends GotError { readonly response: Response; constructor(error: Error, response: Response, options: NormalizedOptions) { - super(`${error.message} in "${format(options as unknown as URL)}"`, error, options); + super(`${error.message} in "${options.url.toString()}"`, error, options); this.name = 'ParseError'; Object.defineProperty(this, 'response', { @@ -87,7 +86,7 @@ export class MaxRedirectsError extends GotError { export class UnsupportedProtocolError extends GotError { constructor(options: NormalizedOptions) { - super(`Unsupported protocol "${options.protocol}"`, {}, options); + super(`Unsupported protocol "${options.url.protocol}"`, {}, options); this.name = 'UnsupportedProtocolError'; } } diff --git a/source/get-response.ts b/source/get-response.ts index 57707963d..91b3001cd 100644 --- a/source/get-response.ts +++ b/source/get-response.ts @@ -1,7 +1,6 @@ import {IncomingMessage} from 'http'; import EventEmitter = require('events'); import stream = require('stream'); -import is from '@sindresorhus/is'; import decompressResponse = require('decompress-response'); import mimicResponse = require('mimic-response'); import {NormalizedOptions, Response} from './utils/types'; @@ -9,13 +8,12 @@ import {downloadProgress} from './progress'; export default (response: IncomingMessage, options: NormalizedOptions, emitter: EventEmitter) => { const downloadBodySize = Number(response.headers['content-length']) || undefined; - const progressStream = downloadProgress(response, emitter, downloadBodySize); + const progressStream = downloadProgress(emitter, downloadBodySize); mimicResponse(response, progressStream); const newResponse = ( - options.decompress === true && - is.function_(decompressResponse) && + options.decompress && options.method !== 'HEAD' ? decompressResponse(progressStream as unknown as IncomingMessage) : progressStream ) as Response; diff --git a/source/index.ts b/source/index.ts index 39a6171d2..21bee3f49 100644 --- a/source/index.ts +++ b/source/index.ts @@ -1,7 +1,7 @@ -import create from './create'; +import create, {defaultHandler} from './create'; import {Defaults} from './utils/types.js'; -const defaults: Partial = { +const defaults: Defaults = { options: { method: 'GET', retry: { @@ -32,8 +32,11 @@ const defaults: Partial = { 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' - ] + ], + maxRetryAfter: undefined, + calculateDelay: ({computedValue}) => computedValue }, + timeout: {}, headers: { 'user-agent': 'got (https://github.com/sindresorhus/got)' }, @@ -46,14 +49,16 @@ const defaults: Partial = { decompress: true, throwHttpErrors: true, followRedirect: true, - stream: false, + isStream: false, cache: false, dnsCache: false, useElectronNet: false, - responseType: 'text', + responseType: 'default', resolveBodyOnly: false, - maxRedirects: 10 + maxRedirects: 10, + prefixUrl: '' }, + handlers: [defaultHandler], mutableDefaults: false }; @@ -71,7 +76,6 @@ export * from './utils/types'; export { Got, GotStream, - ReturnResponse, ReturnStream, GotReturn } from './create'; diff --git a/source/known-hook-events.ts b/source/known-hook-events.ts index d3591cc6a..9baa49f0c 100644 --- a/source/known-hook-events.ts +++ b/source/known-hook-events.ts @@ -1,5 +1,4 @@ -import {Options, CancelableRequest, Response, NormalizedOptions} from './utils/types'; -import {HTTPError, GotError, ParseError, MaxRedirectsError} from './errors'; +import {CancelableRequest, Response, NormalizedOptions, GenericError} from './utils/types'; /** Called with plain request options, right before their normalization. This is especially useful in conjunction with got.extend() and got.create() when the input needs custom handling. @@ -8,7 +7,7 @@ Called with plain request options, right before their normalization. This is esp @see [Request migration guide](https://github.com/sindresorhus/got/blob/master/migration-guides.md#breaking-changes) for an example. */ -export type InitHook = (options: Options) => void; +export type InitHook = (options: NormalizedOptions) => void; /** Called with normalized [request options](https://github.com/sindresorhus/got#options). Got will make no further changes to the request before it is sent (except the body serialization). This is especially useful in conjunction with [`got.extend()`](https://github.com/sindresorhus/got#instances) and [`got.create()`](https://github.com/sindresorhus/got/blob/master/advanced-creation.md) when you want to create an API client that, for example, uses HMAC-signing. @@ -25,14 +24,14 @@ export type BeforeRedirectHook = (options: NormalizedOptions, response: Response /** Called with normalized [request options](https://github.com/sindresorhus/got#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. */ -export type BeforeRetryHook = (options: NormalizedOptions, error: Error | GotError | ParseError | HTTPError | MaxRedirectsError, retryCount: number) => void | Promise; +export type BeforeRetryHook = (options: NormalizedOptions, error?: GenericError, retryCount?: number) => void | Promise; /** Called with an `Error` instance. The error is passed to the hook right before it's thrown. This is especially useful when you want to have more detailed errors. **Note:** Errors thrown while normalizing input options are thrown directly and not part of this hook. */ -export type BeforeErrorHook = (error: ErrorLike) => Error | Promise; +export type BeforeErrorHook = (error: ErrorLike) => Error | Promise; /** Called with [response object](https://github.com/sindresorhus/got#response) and a retry function. diff --git a/source/merge.ts b/source/merge.ts deleted file mode 100644 index df6fb4d7f..000000000 --- a/source/merge.ts +++ /dev/null @@ -1,106 +0,0 @@ -import is from '@sindresorhus/is'; -import {Options} from './utils/types'; -import knownHookEvents, {Hooks, HookEvent, HookType} from './known-hook-events'; - -export default function merge, Source extends Record>(target: Target, ...sources: Source[]): Target & Source { - for (const source of sources) { - for (const [key, sourceValue] of Object.entries(source)) { - if (is.undefined(sourceValue)) { - continue; - } - - const targetValue = target[key]; - if (targetValue instanceof URLSearchParams && sourceValue instanceof URLSearchParams) { - const params = new URLSearchParams(); - - const append = (value: string, key: string): void => params.append(key, value); - targetValue.forEach(append); - sourceValue.forEach(append); - - // @ts-ignore https://github.com/microsoft/TypeScript/issues/31661 - target[key] = params; - } else if (is.urlInstance(targetValue) && (is.urlInstance(sourceValue) || is.string(sourceValue))) { - // @ts-ignore - target[key] = new URL(sourceValue as string, targetValue); - } else if (is.plainObject(sourceValue)) { - if (is.plainObject(targetValue)) { - // @ts-ignore - target[key] = merge({}, targetValue, sourceValue); - } else { - // @ts-ignore - target[key] = merge({}, sourceValue); - } - } else if (is.array(sourceValue)) { - // @ts-ignore - target[key] = sourceValue.slice(); - } else { - // @ts-ignore - target[key] = sourceValue; - } - } - } - - return target as Target & Source; -} - -export function mergeOptions(...sources: Array>): Partial { - sources = sources.map(source => { - if (!source) { - return {}; - } - - if (is.object(source.retry)) { - return source; - } - - return { - ...source, - retry: { - retries: source.retry - } - }; - }) as Array>; - - const mergedOptions = merge({}, ...sources); - - const hooks = knownHookEvents.reduce((accumulator, current) => ({...accumulator, [current]: []}), {}) as Record; - - for (const source of sources) { - // We need to check `source` to allow calling `.extend()` with no arguments. - if (!source) { - continue; - } - - if (Reflect.has(source, 'hooks')) { - for (const hook of knownHookEvents) { - hooks[hook] = hooks[hook].concat(source.hooks[hook] ?? []); - } - } - - if (Reflect.has(source, 'context')) { - Object.defineProperty(mergedOptions, 'context', { - writable: true, - configurable: true, - enumerable: false, - // @ts-ignore - value: source.context - }); - } - - if (Reflect.has(source, 'body')) { - mergedOptions.body = source.body; - } - - if (Reflect.has(source, 'json')) { - mergedOptions.json = source.json; - } - - if (Reflect.has(source, 'form')) { - mergedOptions.form = source.form; - } - } - - mergedOptions.hooks = hooks as Hooks; - - return mergedOptions; -} diff --git a/source/normalize-arguments.ts b/source/normalize-arguments.ts index 8515f3eef..b6be32415 100644 --- a/source/normalize-arguments.ts +++ b/source/normalize-arguments.ts @@ -1,57 +1,65 @@ +import {promisify} from 'util'; +import http = require('http'); import https = require('https'); -import {format} from 'url'; import CacheableLookup from 'cacheable-lookup'; +import CacheableRequest = require('cacheable-request'); import is from '@sindresorhus/is'; import lowercaseKeys = require('lowercase-keys'); +import toReadableStream = require('to-readable-stream'); import Keyv = require('keyv'); -import urlToOptions, {URLOptions} from './utils/url-to-options'; -import validateSearchParams from './utils/validate-search-params'; -import supportsBrotli from './utils/supports-brotli'; -import merge, {mergeOptions} from './merge'; +import optionsToUrl from './utils/options-to-url'; +import {UnsupportedProtocolError} from './errors'; +import merge from './utils/merge'; import knownHookEvents from './known-hook-events'; import { Options, NormalizedOptions, Method, - URLArgument, URLOrOptions, - NormalizedDefaults + Defaults } from './utils/types'; +import dynamicRequire from './utils/dynamic-require'; +import getBodySize from './utils/get-body-size'; +import isFormData from './utils/is-form-data'; +import supportsBrotli from './utils/supports-brotli'; -let hasShownDeprecation = false; - -// It's 2x faster than [...new Set(array)] -const uniqueArray = (array: T[]): T[] => array.filter((element, position) => array.indexOf(element) === position); +// `preNormalizeArguments` normalizes these options: `headers`, `prefixUrl`, `hooks`, `timeout`, `retry` and `method`. +// `normalizeArguments` is *only* called on `got(...)`. It normalizes the URL and performs `mergeOptions(...)`. +// `normalizeRequestArguments` converts Got options into HTTP options. -// `preNormalize` handles static options (e.g. headers). -// For example, when you create a custom instance and make a request -// with no static changes, they won't be normalized again. -// -// `normalize` operates on dynamic options - they cannot be saved. -// For example, `url` needs to be normalized every request. +type NonEnumerableProperty = 'context' | 'body' | 'json' | 'form'; +const nonEnumerableProperties: NonEnumerableProperty[] = [ + 'context', + 'body', + 'json', + 'form' +]; -// TODO: document this. export const preNormalizeArguments = (options: Options, defaults?: NormalizedOptions): NormalizedOptions => { // `options.headers` - if (is.nullOrUndefined(options.headers)) { + if (is.undefined(options.headers)) { options.headers = {}; } else { options.headers = lowercaseKeys(options.headers); } // `options.prefixUrl` - if (options.prefixUrl) { + if (is.urlInstance(options.prefixUrl) || is.string(options.prefixUrl)) { options.prefixUrl = options.prefixUrl.toString(); - if (!options.prefixUrl.endsWith('/')) { + if (options.prefixUrl.length !== 0 && !options.prefixUrl.endsWith('/')) { options.prefixUrl += '/'; } + } else { + options.prefixUrl = defaults ? defaults.prefixUrl : ''; } // `options.hooks` - if (is.nullOrUndefined(options.hooks)) { + if (is.undefined(options.hooks)) { options.hooks = {}; - } else if (is.object(options.hooks)) { + } + + if (is.object(options.hooks)) { for (const event of knownHookEvents) { if (Reflect.has(options.hooks, event)) { if (!is.array(options.hooks[event])) { @@ -65,6 +73,16 @@ export const preNormalizeArguments = (options: Options, defaults?: NormalizedOpt throw new TypeError(`Parameter \`hooks\` must be an Object, not ${is(options.hooks)}`); } + if (defaults) { + for (const event of knownHookEvents) { + // @ts-ignore TS is dumb. + options.hooks[event] = [ + ...defaults.hooks[event], + ...options.hooks[event] + ]; + } + } + // `options.timeout` if (is.number(options.timeout)) { options.timeout = {request: options.timeout}; @@ -75,7 +93,7 @@ export const preNormalizeArguments = (options: Options, defaults?: NormalizedOpt // `options.retry` const {retry} = options; - if (defaults && Reflect.has(defaults, 'retry')) { + if (defaults) { options.retry = {...defaults.retry}; } else { options.retry = { @@ -103,143 +121,309 @@ export const preNormalizeArguments = (options: Options, defaults?: NormalizedOpt ); } - options.retry.methods = uniqueArray(options.retry.methods.map(method => method.toUpperCase())) as Method[]; - options.retry.statusCodes = uniqueArray(options.retry.statusCodes); - options.retry.errorCodes = uniqueArray(options.retry.errorCodes); + options.retry.methods = [...new Set(options.retry.methods.map(method => method.toUpperCase()))] as Method[]; + options.retry.statusCodes = [...new Set(options.retry.statusCodes)]; + options.retry.errorCodes = [...new Set(options.retry.errorCodes)]; // `options.dnsCache` - if (options.dnsCache && !(options instanceof CacheableLookup)) { - options.dnsCache = new CacheableLookup({cacheAdapter: options.dnsCache as Keyv | undefined}); + if (options.dnsCache && !(options.dnsCache instanceof CacheableLookup)) { + options.dnsCache = new CacheableLookup({cacheAdapter: options.dnsCache as Keyv}); } - return options as NormalizedOptions; -}; - -export const normalizeArguments = (url: URLOrOptions, options: Options, defaults?: NormalizedDefaults): NormalizedOptions => { - let urlArgument: URLArgument; - if (is.plainObject(url)) { - options = {...url as Options, ...options}; - urlArgument = options.url ?? {}; - delete options.url; + // `options.method` + if (is.string(options.method)) { + options.method = options.method.toUpperCase() as Method; } else { - urlArgument = url; + options.method = defaults?.method || 'GET'; } - if (defaults) { - options = mergeOptions(defaults.options, options ? preNormalizeArguments(options, defaults.options) : {}); - } else { - options = merge({}, preNormalizeArguments(options)); + // Better memory management, so we don't have to generate a new object every time + if (options.cache) { + (options as NormalizedOptions).cacheableRequest = new CacheableRequest( + (options, handler) => options.request(options, handler), + options.cache as any + ); } - if (!is.string(urlArgument) && !is.object(urlArgument)) { - throw new TypeError(`Parameter \`url\` must be a string or an Object, not ${is(urlArgument)}`); + // `options.cookieJar` + if (is.object(options.cookieJar)) { + let {setCookie, getCookieString} = options.cookieJar; + + // Horrible `tough-cookie` check + if (setCookie.length === 4 && getCookieString.length === 0) { + if (!Reflect.has(setCookie, promisify.custom)) { + setCookie = promisify(setCookie.bind(options.cookieJar)); + getCookieString = promisify(getCookieString.bind(options.cookieJar)); + } + } else if (setCookie.length !== 2) { + throw new TypeError('`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); + } else if (getCookieString.length !== 1) { + throw new TypeError('`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); + } + + options.cookieJar = {setCookie, getCookieString}; } - let urlObj: https.RequestOptions | URLOptions; - if (is.string(urlArgument)) { - if (options.prefixUrl && urlArgument.startsWith('/')) { - throw new Error('`url` must not begin with a slash when using `prefixUrl`'); + return options as NormalizedOptions; +}; + +export const mergeOptions = (...sources: Options[]): NormalizedOptions => { + const mergedOptions = preNormalizeArguments({}); + + // Non enumerable properties shall not be merged + const properties = {}; + + for (const source of sources) { + if (!source) { + continue; } - if (options.prefixUrl) { - urlArgument = options.prefixUrl.toString() + urlArgument; + merge(mergedOptions, preNormalizeArguments(merge({}, source), mergedOptions)); + + for (const name of nonEnumerableProperties) { + if (!Reflect.has(source, name)) { + continue; + } + + properties[name] = { + writable: true, + configurable: true, + enumerable: false, + value: source[name] + }; } + } - urlArgument = urlArgument.replace(/^unix:/, 'http://$&'); + Object.defineProperties(mergedOptions, properties); - urlObj = urlToOptions(new URL(urlArgument)); - } else if (is.urlInstance(urlArgument)) { - urlObj = urlToOptions(urlArgument); - } else if (options.prefixUrl) { - urlObj = { - // @ts-ignore - ...urlToOptions(new URL(options.prefixUrl)), - ...urlArgument - }; - } else { - urlObj = urlArgument; + return mergedOptions; +}; + +export const normalizeArguments = (url: URLOrOptions, options?: Options, defaults?: Defaults): NormalizedOptions => { + // Merge options + if (typeof url === 'undefined') { + throw new TypeError('Missing `url` argument'); } - if (!Reflect.has(urlObj, 'protocol') && !Reflect.has(options, 'protocol')) { - throw new TypeError('No URL protocol specified'); + if (typeof options === 'undefined') { + options = {}; } - options = mergeOptions(urlObj, options); + if (is.urlInstance(url) || is.string(url)) { + options.url = url; - for (const hook of options.hooks.init) { - if (is.asyncFunction(hook)) { - throw new TypeError('The `init` hook must be a synchronous function'); + options = mergeOptions(defaults && defaults.options, options); + } else { + if (Reflect.has(url, 'resolve')) { + throw new Error('The legacy `url.Url` is deprecated. Use `URL` instead.'); } - // @ts-ignore TS is dumb. - hook(options); + options = mergeOptions(defaults && defaults.options, url, options); } - const {prefixUrl} = options; + // Normalize URL + // TODO: drop `optionsToUrl` in Got 12 + if (is.string(options.url)) { + options.url = (options.prefixUrl as string) + options.url; + options.url = options.url.replace(/^unix:/, 'http://$&'); + + if (options.searchParams || options.search) { + options.url = options.url.split('?')[0]; + } + + options.url = optionsToUrl({ + origin: options.url, + ...options + }); + } else if (!is.urlInstance(options.url)) { + options.url = optionsToUrl({origin: options.prefixUrl as string, ...options}); + } + + const normalizedOptions = options as NormalizedOptions; + + // Make it possible to change `options.prefixUrl` + let prefixUrl = options.prefixUrl as string; Object.defineProperty(options, 'prefixUrl', { - set: () => { - throw new Error('Failed to set prefixUrl. Options are normalized already.'); + set: (value: string) => { + if (!normalizedOptions.url.href.startsWith(value)) { + throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${normalizedOptions.url.href}`); + } + + normalizedOptions.url = new URL(value + normalizedOptions.url.href.slice(prefixUrl.length)); + prefixUrl = value; }, get: () => prefixUrl }); - let {searchParams} = options; - delete options.searchParams; + // Make it possible to remove default headers + for (const [key, value] of Object.entries(options.headers)) { + if (is.undefined(value)) { + delete options.headers[key]; + } else if (is.null_(value)) { + throw new TypeError('Use `undefined` instead of `null` to delete HTTP headers'); + } + } - // TODO: Remove this before Got v11 - if (options.query) { - if (!hasShownDeprecation) { - console.warn('`options.query` is deprecated. We support it solely for compatibility - it will be removed in Got 11. Use `options.searchParams` instead.'); - hasShownDeprecation = true; + for (const hook of options.hooks.init) { + if (is.asyncFunction(hook)) { + throw new TypeError('The `init` hook must be a synchronous function'); } - searchParams = options.query; - delete options.query; + // @ts-ignore TS is dumb. + hook(normalizedOptions); + } + + return normalizedOptions; +}; + +const withoutBody: ReadonlySet = new Set(['GET', 'HEAD']); + +export type NormalizedRequestArguments = https.RequestOptions & { + body: Pick; + url: Pick; +}; + +export const normalizeRequestArguments = async (options: NormalizedOptions): Promise => { + options = mergeOptions(options); + + let uploadBodySize: number | undefined; + + // Serialize body + const {headers} = options; + const isForm = !is.undefined(options.form); + const isJSON = !is.undefined(options.json); + const isBody = !is.undefined(options.body); + if ((isBody || isForm || isJSON) && withoutBody.has(options.method)) { + throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); + } + + if ([isBody, isForm, isJSON].filter(isTrue => isTrue).length > 1) { + throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); } - if (is.nonEmptyString(searchParams) || is.nonEmptyObject(searchParams) || (searchParams && searchParams instanceof URLSearchParams)) { - if (!is.string(searchParams)) { - if (!(searchParams instanceof URLSearchParams)) { - // @ts-ignore - validateSearchParams(searchParams); + if (isBody) { + if (is.object(options.body) && isFormData(options.body)) { + // Special case for https://github.com/form-data/form-data + if (!Reflect.has(headers, 'content-type')) { + headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; } + } else if (!is.nodeStream(options.body) && !is.string(options.body) && !is.buffer(options.body)) { + throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); + } + } else if (isForm) { + if (!is.object(options.form)) { + throw new TypeError('The `form` option must be an Object'); + } + + if (!Reflect.has(headers, 'content-type')) { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } + + options.body = (new URLSearchParams(options.form as Record)).toString(); + } else if (isJSON) { + if (!Reflect.has(headers, 'content-type')) { + headers['content-type'] = 'application/json'; + } + + options.body = JSON.stringify(options.json); + } + + // Convert buffer to stream to receive upload progress events (#322) + if (is.buffer(options.body)) { + uploadBodySize = options.body.length; + options.body = toReadableStream(options.body); + } else { + uploadBodySize = await getBodySize(options); + } - searchParams = (new URLSearchParams(searchParams as Record)).toString(); + // See https://tools.ietf.org/html/rfc7230#section-3.3.2 + // A user agent SHOULD send a Content-Length in a request message when + // no Transfer-Encoding is sent and the request method defines a meaning + // for an enclosed payload body. For example, a Content-Length header + // field is normally sent in a POST request even when the value is 0 + // (indicating an empty payload body). A user agent SHOULD NOT send a + // Content-Length header field when the request message does not contain + // a payload body and the method semantics do not anticipate such a + // body. + if (!Reflect.has(headers, 'content-length') && !Reflect.has(headers, 'transfer-encoding')) { + if ( + (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH') && + !is.undefined(uploadBodySize) + ) { + headers['content-length'] = String(uploadBodySize); } + } - options.path = `${options.path.split('?')[0]}?${searchParams}`; + if (!options.isStream && options.responseType === 'json' && is.undefined(headers.accept)) { + headers.accept = 'application/json'; } - if (options.hostname === 'unix') { - const matches = /(?.+?):(?.+)/.exec(options.path); + if (options.decompress && is.undefined(headers['accept-encoding'])) { + headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; + } + + // Validate URL + if (options.url.protocol !== 'http:' && options.url.protocol !== 'https:') { + throw new UnsupportedProtocolError(options); + } + + decodeURI(options.url.toString()); + + // Normalize request function + if (!is.function_(options.request)) { + options.request = options.url.protocol === 'https:' ? https.request : http.request; + } + + // UNIX sockets + if (options.url.hostname === 'unix') { + const matches = /(?.+?):(?.+)/.exec(options.url.pathname); if (matches?.groups) { const {socketPath, path} = matches.groups; + + // It's a bug! + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions options = { ...options, socketPath, path, host: '' - }; + } as NormalizedOptions; } } - const {headers} = options; - for (const [key, value] of Object.entries(headers)) { - if (is.nullOrUndefined(value)) { - delete headers[key]; - } + if (is.object(options.agent)) { + options.agent = options.agent[options.url.protocol.slice(0, -1)] || options.agent; } - if (options.decompress && is.undefined(headers['accept-encoding'])) { - headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'; + if (options.dnsCache) { + options.lookup = options.dnsCache.lookup; } - if (options.method) { - options.method = options.method.toUpperCase() as Method; + /* istanbul ignore next: electron.net is broken */ + // No point in typing process.versions correctly, as + // `process.version.electron` is used only once, right here. + if (options.useElectronNet && (process.versions as any).electron) { + const electron = dynamicRequire(module, 'electron') as any; // Trick webpack + options.request = electron.net.request ?? electron.remote.net.request; } - return options as NormalizedOptions; -}; + // Got's `timeout` is an object, http's `timeout` is a number, so they're not compatible. + delete options.timeout; -export const reNormalizeArguments = (options: Options): NormalizedOptions => normalizeArguments(format(options as unknown as URL | URLOptions), options); + // Set cookies + if (options.cookieJar) { + const cookieString = await options.cookieJar.getCookieString(options.url.toString()); + + if (is.nonEmptyString(cookieString)) { + options.headers.cookie = cookieString; + } else { + delete options.headers.cookie; + } + } + + // `http-cache-semantics` checks this + delete options.url; + + return options as unknown as NormalizedRequestArguments; +}; diff --git a/source/progress.ts b/source/progress.ts index 8c98db75e..806b6ec8e 100644 --- a/source/progress.ts +++ b/source/progress.ts @@ -1,12 +1,13 @@ -import {IncomingMessage, ClientRequest} from 'http'; +import {ClientRequest} from 'http'; import {Transform as TransformStream} from 'stream'; import {Socket} from 'net'; import EventEmitter = require('events'); +import is from '@sindresorhus/is'; -export function downloadProgress(_response: IncomingMessage, emitter: EventEmitter, downloadBodySize?: number): TransformStream { +export function downloadProgress(emitter: EventEmitter, downloadBodySize?: number): TransformStream { let downloadedBytes = 0; - return new TransformStream({ + const progressStream = new TransformStream({ transform(chunk, _encoding, callback) { downloadedBytes += chunk.length; @@ -34,6 +35,8 @@ export function downloadProgress(_response: IncomingMessage, emitter: EventEmitt callback(); } }); + + return progressStream; } export function uploadProgress(request: ClientRequest, emitter: EventEmitter, uploadBodySize?: number): void { @@ -51,6 +54,10 @@ export function uploadProgress(request: ClientRequest, emitter: EventEmitter, up clearInterval(progressInterval); }); + request.once('abort', () => { + clearInterval(progressInterval); + }); + request.once('response', () => { clearInterval(progressInterval); @@ -65,8 +72,20 @@ export function uploadProgress(request: ClientRequest, emitter: EventEmitter, up const onSocketConnect = (): void => { progressInterval = setInterval(() => { const lastUploadedBytes = uploadedBytes; - /* istanbul ignore next: see #490 (occurs randomly!) */ - const headersSize = (request as any)._header ? Buffer.byteLength((request as any)._header) : 0; + + /* istanbul ignore next: future versions of Node may not have this property */ + if (!is.string((request as any)._header)) { + clearInterval(progressInterval); + + const url = new URL('https://github.com/sindresorhus/got/issues/new'); + url.searchParams.set('title', '`request._header` is not present'); + url.searchParams.set('body', 'It causes `uploadProgress` to fail.'); + + console.warn('`request._header` is not present. Please report this as a bug:\n' + url.href); + return; + } + + const headersSize = Buffer.byteLength((request as any)._header); uploadedBytes = socket.bytesWritten - headersSize; // Don't emit events with unchanged progress and diff --git a/source/request-as-event-emitter.ts b/source/request-as-event-emitter.ts index aa572e5cc..48640928e 100644 --- a/source/request-as-event-emitter.ts +++ b/source/request-as-event-emitter.ts @@ -1,46 +1,34 @@ -import {format, UrlObject} from 'url'; -import {promisify} from 'util'; import stream = require('stream'); import EventEmitter = require('events'); -import {Transform as TransformStream} from 'stream'; import http = require('http'); -import https = require('https'); import CacheableRequest = require('cacheable-request'); -import toReadableStream = require('to-readable-stream'); import is from '@sindresorhus/is'; import timer, {Timings} from '@szmarczak/http-timer'; import timedOut, {TimeoutError as TimedOutTimeoutError} from './utils/timed-out'; -import getBodySize from './utils/get-body-size'; -import isFormData from './utils/is-form-data'; import calculateRetryDelay from './calculate-retry-delay'; import getResponse from './get-response'; +import {normalizeRequestArguments} from './normalize-arguments'; import {uploadProgress} from './progress'; -import {CacheError, UnsupportedProtocolError, MaxRedirectsError, RequestError, TimeoutError} from './errors'; +import {CacheError, MaxRedirectsError, RequestError, TimeoutError} from './errors'; import urlToOptions from './utils/url-to-options'; -import {RequestFunction, NormalizedOptions, Response, ResponseObject, AgentByProtocol} from './utils/types'; -import dynamicRequire from './utils/dynamic-require'; +import {NormalizedOptions, Response, ResponseObject} from './utils/types'; const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); -const withoutBody: ReadonlySet = new Set(['GET', 'HEAD']); export interface RequestAsEventEmitter extends EventEmitter { retry: (error: T) => boolean; abort: () => void; } -export default (options: NormalizedOptions, input?: TransformStream) => { +export default (options: NormalizedOptions) => { const emitter = new EventEmitter() as RequestAsEventEmitter; + + const requestURL = options.url.toString(); const redirects: string[] = []; - let currentRequest: http.ClientRequest; - let requestUrl: string; - let redirectString: string | undefined; - let uploadBodySize: number | undefined; let retryCount = 0; - let shouldAbort = false; - const setCookie = options.cookieJar && promisify(options.cookieJar.setCookie.bind(options.cookieJar)); - const getCookieString = options.cookieJar && promisify(options.cookieJar.getCookieString.bind(options.cookieJar)); - const agents = is.object(options.agent) && options.agent; + let currentRequest: http.ClientRequest; + let shouldAbort = false; const emitError = async (error: Error): Promise => { try { @@ -55,48 +43,11 @@ export default (options: NormalizedOptions, input?: TransformStream) => { } }; - const get = async (options: NormalizedOptions): Promise => { - const currentUrl = redirectString ?? requestUrl; - - if (options.protocol !== 'http:' && options.protocol !== 'https:') { - throw new UnsupportedProtocolError(options); - } - - // Validate the URL - decodeURI(currentUrl); - - let requestFn: RequestFunction; - if (is.function_(options.request)) { - requestFn = options.request; - } else { - requestFn = options.protocol === 'https:' ? https.request : http.request; - } - - if (agents) { - const protocolName = options.protocol === 'https:' ? 'https' : 'http'; - options.agent = (agents as AgentByProtocol)[protocolName] ?? options.agent; - } - - /* istanbul ignore next: electron.net is broken */ - // No point in typing process.versions correctly, as - // `process.version.electron` is used only once, right here. - if (options.useElectronNet && (process.versions as any).electron) { - const electron = dynamicRequire(module, 'electron') as any; // Trick Webpack - requestFn = electron.net.request ?? electron.remote.net.request; - } - - if (options.cookieJar) { - const cookieString = await getCookieString(currentUrl); - - if (is.nonEmptyString(cookieString)) { - options.headers.cookie = cookieString; - } - } + const get = async (): Promise => { + let httpOptions = await normalizeRequestArguments(options); let timings: Timings; const handleResponse = async (response: http.ServerResponse | ResponseObject): Promise => { - options.timeout = timeout; - try { /* istanbul ignore next: fixes https://github.com/electron/electron/blob/cbb460d47628a7a146adf4419ed48550a98b2923/lib/browser/api/net.js#L59-L65 */ if (options.useElectronNet) { @@ -116,8 +67,8 @@ export default (options: NormalizedOptions, input?: TransformStream) => { const typedResponse = response as Response; // This is intentionally using `||` over `??` so it can also catch empty status message. typedResponse.statusMessage = typedResponse.statusMessage || http.STATUS_CODES[statusCode]; - typedResponse.url = currentUrl; - typedResponse.requestUrl = requestUrl; + typedResponse.url = options.url.toString(); + typedResponse.requestUrl = requestURL; typedResponse.retryCount = retryCount; typedResponse.timings = timings; typedResponse.redirectUrls = redirects; @@ -131,10 +82,11 @@ export default (options: NormalizedOptions, input?: TransformStream) => { } const rawCookies = typedResponse.headers['set-cookie']; - if (options.cookieJar && rawCookies) { - let promises: Array> = rawCookies.map((rawCookie: string) => setCookie(rawCookie, typedResponse.url)); + if (Reflect.has(options, 'cookieJar') && rawCookies) { + let promises: Array> = rawCookies.map((rawCookie: string) => options.cookieJar.setCookie(rawCookie, typedResponse.url)); + if (options.ignoreInvalidCookies) { - promises = promises.map(async p => p.catch(() => {})); + promises = promises.map(p => p.catch(() => {})); } await Promise.all(promises); @@ -143,14 +95,16 @@ export default (options: NormalizedOptions, input?: TransformStream) => { if (options.followRedirect && Reflect.has(typedResponse.headers, 'location') && redirectCodes.has(statusCode)) { typedResponse.resume(); // We're being redirected, we don't care about the response. - if (statusCode === 303 && options.method !== 'GET' && options.method !== 'HEAD') { - // Server responded with "see other", indicating that the resource exists at another location, - // and the client should request it from that location via GET or HEAD. - options.method = 'GET'; + if (statusCode === 303) { + if (options.method !== 'GET' && options.method !== 'HEAD') { + // Server responded with "see other", indicating that the resource exists at another location, + // and the client should request it from that location via GET or HEAD. + options.method = 'GET'; + } + delete options.body; delete options.json; delete options.form; - delete options.body; } if (redirects.length >= options.maxRedirects) { @@ -159,31 +113,27 @@ export default (options: NormalizedOptions, input?: TransformStream) => { // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 const redirectBuffer = Buffer.from(typedResponse.headers.location, 'binary').toString(); - const redirectURL = new URL(redirectBuffer, currentUrl); - redirectString = redirectURL.toString(); + const redirectURL = new URL(redirectBuffer, options.url); - redirects.push(redirectString); + // Redirecting to a different site, clear cookies. + if (redirectURL.hostname !== options.url.hostname) { + delete options.headers.cookie; + } - const redirectOptions = { - ...options, - port: undefined, - auth: undefined, - ...urlToOptions(redirectURL) - }; + redirects.push(redirectURL.toString()); + options.url = redirectURL; for (const hook of options.hooks.beforeRedirect) { // eslint-disable-next-line no-await-in-loop - await hook(redirectOptions, typedResponse); + await hook(options, typedResponse); } - emitter.emit('redirect', response, redirectOptions); + emitter.emit('redirect', response, options); - await get(redirectOptions); + await get(); return; } - delete options.body; - getResponse(typedResponse, options, emitter); } catch (error) { emitError(error); @@ -197,14 +147,15 @@ export default (options: NormalizedOptions, input?: TransformStream) => { } currentRequest = request; - options.timeout = timeout; + + // `request.aborted` is a boolean since v11.0.0: https://github.com/nodejs/node/commit/4b00c4fafaa2ae8c41c1f78823c0feb810ae4723#diff-e3bc37430eb078ccbafe3aa3b570c91a + // We need to allow `TimedOutTimeoutError` here, because it `stream.pipeline(…)` aborts it automatically. + const isAborted = () => typeof request.aborted === 'number' || (request.aborted as unknown as boolean) === true; const onError = (error: Error): void => { const isTimedOutError = error instanceof TimedOutTimeoutError; - // `request.aborted` is a boolean since v11.0.0: https://github.com/nodejs/node/commit/4b00c4fafaa2ae8c41c1f78823c0feb810ae4723#diff-e3bc37430eb078ccbafe3aa3b570c91a - // We need to allow `TimedOutTimeoutError` here, because it `stream.pipeline(..)` aborts it automatically. - if (!isTimedOutError && (typeof request.aborted === 'number' || (request.aborted as unknown as boolean) === true)) { + if (!isTimedOutError && isAborted()) { return; } @@ -235,37 +186,34 @@ export default (options: NormalizedOptions, input?: TransformStream) => { request.on('error', onError); - timings = timer(request); + timings = timer(request); // TODO: Make `@szmarczak/http-timer` set `request.timings` and `response.timings` + const uploadBodySize = httpOptions.headers['content-length'] ? Number(httpOptions.headers['content-length']) : undefined; uploadProgress(request, emitter, uploadBodySize); - if (options.timeout) { - timedOut(request, options.timeout, options); - } + timedOut(request, options.timeout, options.url); emitter.emit('request', request); + if (isAborted()) { + return; + } + try { - if (is.nodeStream(options.body)) { - const {body} = options; - delete options.body; - - // `stream.pipeline(…)` handles `error` for us. - request.removeListener('error', onError); - - stream.pipeline( - body, - request, - uploadComplete - ); - } else if (options.body) { - request.end(options.body, uploadComplete); - } else if (input && (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH')) { - stream.pipeline( - input, - request, - uploadComplete - ); + if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH') { + if (is.nodeStream(httpOptions.body)) { + // `stream.pipeline(…)` handles `error` for us. + request.removeListener('error', onError); + + stream.pipeline( + // @ts-ignore Upgrade `@sindresorhus/is` + httpOptions.body, + request, + uploadComplete + ); + } else { + request.end(httpOptions.body, uploadComplete); + } } else { request.end(uploadComplete); } @@ -274,16 +222,14 @@ export default (options: NormalizedOptions, input?: TransformStream) => { } }; - const {timeout} = options; - delete options.timeout; - - if (options.dnsCache) { - options.lookup = options.dnsCache.lookup; - } - if (options.cache) { - const cacheableRequest = new CacheableRequest(requestFn, options.cache); - const cacheRequest = cacheableRequest(options as unknown as https.RequestOptions, handleResponse); + // `cacheable-request` doesn't support Node 10 API, fallback. + httpOptions = { + ...httpOptions, + ...urlToOptions(options.url) + }; + + const cacheRequest = options.cacheableRequest(httpOptions, handleResponse); cacheRequest.once('error', error => { if (error instanceof CacheableRequest.RequestError) { @@ -297,15 +243,16 @@ export default (options: NormalizedOptions, input?: TransformStream) => { } else { // Catches errors thrown by calling `requestFn(…)` try { - // @ts-ignore TS complains that URLSearchParams is not the same as URLSearchParams - handleRequest(requestFn(options as unknown as URL, handleResponse)); + // @ts-ignore 1. TS complains that URLSearchParams is not the same as URLSearchParams. + // 2. It doesn't notice that `options.timeout` is deleted above. + handleRequest(httpOptions.request(options.url, httpOptions, handleResponse)); } catch (error) { emitError(new RequestError(error, options)); } } }; - emitter.retry = (error): boolean => { + emitter.retry = error => { let backoff: number; retryCount++; @@ -335,7 +282,7 @@ export default (options: NormalizedOptions, input?: TransformStream) => { await hook(options, error, retryCount); } - await get(options); + await get(); } catch (error_) { emitError(error_); } @@ -363,59 +310,7 @@ export default (options: NormalizedOptions, input?: TransformStream) => { await hook(options); } - // Serialize body - const {body, headers} = options; - const isForm = !is.nullOrUndefined(options.form); - const isJSON = !is.nullOrUndefined(options.json); - const isBody = !is.nullOrUndefined(body); - if ((isBody || isForm || isJSON) && withoutBody.has(options.method)) { - throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); - } - - if (isBody) { - if (isForm || isJSON) { - throw new TypeError('The `body` option cannot be used with the `json` option or `form` option'); - } - - if (is.object(body) && isFormData(body)) { - // Special case for https://github.com/form-data/form-data - headers['content-type'] = headers['content-type'] ?? `multipart/form-data; boundary=${body.getBoundary()}`; - } else if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body)) { - throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); - } - } else if (isForm) { - if (!is.object(options.form)) { - throw new TypeError('The `form` option must be an Object'); - } - - headers['content-type'] = headers['content-type'] ?? 'application/x-www-form-urlencoded'; - options.body = (new URLSearchParams(options.form as Record)).toString(); - } else if (isJSON) { - headers['content-type'] = headers['content-type'] ?? 'application/json'; - options.body = JSON.stringify(options.json); - } - - // Convert buffer to stream to receive upload progress events (#322) - if (is.buffer(body)) { - options.body = toReadableStream(body); - uploadBodySize = body.length; - } else { - uploadBodySize = await getBodySize(options); - } - - if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding'])) { - if ((uploadBodySize > 0 || options.method === 'PUT') && !is.undefined(uploadBodySize)) { - headers['content-length'] = String(uploadBodySize); - } - } - - if (!options.stream && options.responseType === 'json' && is.undefined(headers.accept)) { - options.headers.accept = 'application/json'; - } - - requestUrl = options.href ?? (new URL(options.path, format(options as UrlObject))).toString(); - - await get(options); + await get(); } catch (error) { emitError(error); } @@ -423,3 +318,18 @@ export default (options: NormalizedOptions, input?: TransformStream) => { return emitter; }; + +export const proxyEvents = (proxy, emitter) => { + const events = [ + 'request', + 'redirect', + 'uploadProgress', + 'downloadProgress' + ]; + + for (const event of events) { + emitter.on(event, (...args: unknown[]) => { + proxy.emit(event, ...args); + }); + } +}; diff --git a/source/utils/get-body-size.ts b/source/utils/get-body-size.ts index 63a1e1b26..713bd1977 100644 --- a/source/utils/get-body-size.ts +++ b/source/utils/get-body-size.ts @@ -7,13 +7,13 @@ import {Options} from './types'; const statAsync = promisify(stat); export default async (options: Options): Promise => { - const {body, headers, stream} = options; + const {body, headers, isStream} = options; if (headers && 'content-length' in headers) { return Number(headers['content-length']); } - if (!body && !stream) { + if (!body && !isStream) { return 0; } diff --git a/source/utils/merge.ts b/source/utils/merge.ts new file mode 100644 index 000000000..b34e4c3e9 --- /dev/null +++ b/source/utils/merge.ts @@ -0,0 +1,30 @@ +import is from '@sindresorhus/is'; + +export default function merge, Source extends Record>(target: Target, ...sources: Source[]): Target & Source { + for (const source of sources) { + for (const [key, sourceValue] of Object.entries(source)) { + const targetValue = target[key]; + + if (is.urlInstance(targetValue) && is.string(sourceValue)) { + // @ts-ignore + target[key] = new URL(sourceValue, targetValue); + } else if (is.plainObject(sourceValue)) { + if (is.plainObject(targetValue)) { + // @ts-ignore + target[key] = merge({}, targetValue, sourceValue); + } else { + // @ts-ignore + target[key] = merge({}, sourceValue); + } + } else if (is.array(sourceValue)) { + // @ts-ignore + target[key] = sourceValue.slice(); + } else { + // @ts-ignore + target[key] = sourceValue; + } + } + } + + return target as Target & Source; +} diff --git a/source/utils/options-to-url.ts b/source/utils/options-to-url.ts new file mode 100644 index 000000000..1a0bd10a3 --- /dev/null +++ b/source/utils/options-to-url.ts @@ -0,0 +1,84 @@ +function validateSearchParams(searchParams: Record): asserts searchParams is Record { + for (const value of Object.values(searchParams)) { + if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean' && value !== null) { + throw new TypeError(`The \`searchParams\` value '${value}' must be a string, number, boolean or null`); + } + } +} + +export interface URLOptions { + href?: string; + origin?: string; + protocol?: string; + username?: string; + password?: string; + host?: string; + hostname?: string; + port?: string | number; + pathname?: string; + search?: string; + searchParams?: Record | URLSearchParams | string; + hash?: string; +} + +const keys = [ + 'protocol', + 'username', + 'password', + 'host', + 'hostname', + 'port', + 'pathname', + 'search', + 'hash' +]; + +export default (options: URLOptions): URL => { + let origin: string; + + if (Reflect.has(options, 'path')) { + throw new TypeError('Parameter `path` is deprecated. Use `pathname` instead.'); + } + + if (Reflect.has(options, 'auth')) { + throw new TypeError('Parameter `auth` is deprecated. Use `username`/`password` instead.'); + } + + if (options.search && options.searchParams) { + throw new TypeError('Parameters `search` and `searchParams` are mutually exclusive.'); + } + + if (options.href) { + return new URL(options.href); + } + + if (options.origin) { + origin = options.origin; + } else { + if (!options.protocol) { + throw new TypeError('No URL protocol specified'); + } + + origin = `${options.protocol}//${options.hostname || options.host}`; + } + + const url = new URL(origin); + + for (const key of keys) { + if (Reflect.has(options, key)) { + url[key] = options[key]; + } + } + + if (Reflect.has(options, 'searchParams')) { + if (typeof options.searchParams !== 'string' && !(options.searchParams instanceof URLSearchParams)) { + validateSearchParams(options.searchParams); + } + + (new URLSearchParams(options.searchParams as Record)).forEach((value, key) => { + url.searchParams.append(key, value); + }); + } + + return url; +}; diff --git a/source/utils/timed-out.ts b/source/utils/timed-out.ts index 27701e676..bd22cc5ea 100644 --- a/source/utils/timed-out.ts +++ b/source/utils/timed-out.ts @@ -40,7 +40,7 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions const timeout: NodeJS.Timeout = setTimeout(() => { // @ts-ignore https://github.com/microsoft/TypeScript/issues/26113 immediate = setImmediate(callback, delay, ...args); - immediate.unref(); + immediate.unref?.(); }, delay); timeout.unref?.(); @@ -100,8 +100,8 @@ export default (request: ClientRequest, delays: Delays, options: TimedOutOptions } once(request, 'socket', (socket: net.Socket): void => { - // TODO: There seems to not be a `socketPath` on the request, but there *is* a `socket.remoteAddress`. - const {socketPath} = request as any; + // @ts-ignore Node typings doesn't have this property + const {socketPath} = request; /* istanbul ignore next: hard to test */ if (socket.connecting) { diff --git a/source/utils/types.ts b/source/utils/types.ts index c7ca05580..571ab45d2 100644 --- a/source/utils/types.ts +++ b/source/utils/types.ts @@ -2,16 +2,18 @@ import http = require('http'); import https = require('https'); import ResponseLike = require('responselike'); import {Readable as ReadableStream} from 'stream'; +import {Except, Merge} from 'type-fest'; import PCancelable = require('p-cancelable'); -import {CookieJar} from 'tough-cookie'; -import {StorageAdapter} from 'cacheable-request'; -import {Except} from 'type-fest'; +import CacheableRequest = require('cacheable-request'); import CacheableLookup from 'cacheable-lookup'; import Keyv = require('keyv'); import {Timings} from '@szmarczak/http-timer/dist'; import {Hooks} from '../known-hook-events'; import {GotError, ParseError, HTTPError, MaxRedirectsError} from '../errors'; import {ProxyStream} from '../as-stream'; +import {URLOptions} from './options-to-url'; + +export type GenericError = Error | GotError | ParseError | HTTPError | MaxRedirectsError; export type Method = | 'GET' @@ -41,12 +43,10 @@ export type ErrorCode = | 'ENETUNREACH' | 'EAI_AGAIN'; -export type ResponseType = 'json' | 'buffer' | 'text'; - -export type URLArgument = string | https.RequestOptions | URL; +export type ResponseType = 'json' | 'buffer' | 'text' | 'default'; -export interface Response extends http.IncomingMessage { - body: Buffer | string | any; +export interface Response extends http.IncomingMessage { + body: BodyType; statusCode: number; /** @@ -79,7 +79,7 @@ export interface ResponseObject extends ResponseLike { export interface RetryObject { attemptCount: number; retryOptions: RetryOptions; - error: Error | GotError | ParseError | HTTPError | MaxRedirectsError; + error: GenericError; computedValue: number; } @@ -115,19 +115,22 @@ export interface Delays { export type Headers = Record; -// The library overrides the type definition of `agent`, `host`, 'headers and `timeout`. -export interface Options extends Except { - host?: string; +interface CookieJar { + getCookieString(url: string, callback: (error: Error, cookieHeader: string) => void): void; + getCookieString(url: string): Promise; + setCookie(rawCookie: string, url: string, callback: (error: Error, result: unknown) => void): void; + setCookie(rawCookie: string, url: string): Promise; +} + +// TODO: Missing lots of `http` options +export interface Options extends URLOptions { + url?: URL | string; body?: string | Buffer | ReadableStream; hostname?: string; - path?: string; socketPath?: string; - protocol?: string; - href?: string; - options?: Options; hooks?: Partial; decompress?: boolean; - stream?: boolean; + isStream?: boolean; encoding?: BufferEncoding | null; method?: Method; retry?: number | RetryOptions; @@ -136,7 +139,7 @@ export interface Options extends Except | Keyv | false; - url?: URL | string; - searchParams?: Record | URLSearchParams | string; - query?: Options['searchParams']; // Deprecated useElectronNet?: boolean; form?: Record; json?: Record; @@ -155,29 +155,20 @@ export interface Options extends Except { +export interface NormalizedOptions extends Except { // Normalized Got options headers: Headers; hooks: Hooks; timeout: Delays; dnsCache?: CacheableLookup | false; retry: Required; - readonly prefixUrl?: string; + prefixUrl: string; method: Method; + url: URL; + cacheableRequest?: (options: string | URL | http.RequestOptions, callback?: (response: http.ServerResponse | ResponseLike) => void) => CacheableRequest.Emitter; - // Normalized URL options - protocol: string; - hostname: string; - host: string; - hash: string; - search: string | null; - pathname: string; - href: string; - path: string; - port: number; - username: string; - password: string; - auth?: string; + // UNIX socket support + path?: string; } export interface ExtendedOptions extends Options { @@ -186,22 +177,29 @@ export interface ExtendedOptions extends Options { } export interface Defaults { - options?: Options; - handlers?: HandlerFunction[]; - mutableDefaults?: boolean; -} - -export interface NormalizedDefaults { - options: NormalizedOptions; + options: Except; handlers: HandlerFunction[]; + _rawHandlers?: HandlerFunction[]; mutableDefaults: boolean; } -export type URLOrOptions = URLArgument | (Options & {url: URLArgument}); +export type URLOrOptions = Options | string; + +export interface Progress { + percent: number; + transferred: number; + total?: number; +} + +export interface GotEvents { + on(name: 'request', listener: (request: http.ClientRequest) => void): T; + on(name: 'response', listener: (response: Response) => void): T; + on(name: 'redirect', listener: (response: Response, nextOptions: NormalizedOptions) => void): T; + on(name: 'uploadProgress' | 'downloadProgress', listener: (progress: Progress) => void): T; +} -export interface CancelableRequest extends PCancelable { - on(name: string, listener: () => void): CancelableRequest; +export interface CancelableRequest extends Merge, GotEvents>> { json(): CancelableRequest; - buffer(): CancelableRequest; - text(): CancelableRequest; + buffer(): CancelableRequest; + text(): CancelableRequest; } diff --git a/source/utils/url-to-options.ts b/source/utils/url-to-options.ts index 3cf6e2a56..2e274b06e 100644 --- a/source/utils/url-to-options.ts +++ b/source/utils/url-to-options.ts @@ -3,7 +3,7 @@ import is from '@sindresorhus/is'; // TODO: Deprecate legacy Url at some point -export interface URLOptions { +export interface LegacyURLOptions { protocol: string; hostname: string; host: string; @@ -16,11 +16,11 @@ export interface URLOptions { auth?: string; } -export default (url: URL | UrlWithStringQuery): URLOptions => { +export default (url: URL | UrlWithStringQuery): LegacyURLOptions => { // Cast to URL url = url as URL; - const options: URLOptions = { + const options: LegacyURLOptions = { protocol: url.protocol, hostname: url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname, host: url.host, diff --git a/source/utils/validate-search-params.ts b/source/utils/validate-search-params.ts deleted file mode 100644 index 561d703c6..000000000 --- a/source/utils/validate-search-params.ts +++ /dev/null @@ -1,9 +0,0 @@ -import is from '@sindresorhus/is'; - -export default (searchParams: Record): asserts searchParams is Record => { - for (const value of Object.values(searchParams)) { - if (!is.string(value) && !is.number(value) && !is.boolean(value) && !is.null_(value)) { - throw new TypeError(`The \`searchParams\` value '${value}' must be a string, number, boolean or null`); - } - } -}; diff --git a/test/arguments.ts b/test/arguments.ts index 14041f5e4..df3ea31e7 100644 --- a/test/arguments.ts +++ b/test/arguments.ts @@ -12,10 +12,10 @@ const echoUrl = (request, response) => { test('`url` is required', async t => { await t.throwsAsync( // @ts-ignore Manual tests - got(), + got(''), { instanceOf: TypeError, - message: 'Parameter `url` must be a string or an Object, not undefined' + message: 'No URL protocol specified' } ); }); @@ -29,6 +29,11 @@ test('`url` should be utf-8 encoded', async t => { ); }); +test('throws if no arguments provided', async t => { + // @ts-ignore This is on purpose. + await t.throwsAsync(got(), TypeError, 'Missing `url` argument'); +}); + test('throws an error if the protocol is not specified', async t => { await t.throwsAsync(got('example.com'), { instanceOf: TypeError, @@ -39,6 +44,11 @@ test('throws an error if the protocol is not specified', async t => { instanceOf: TypeError, message: 'No URL protocol specified' }); + + await t.throwsAsync(got({}), { + instanceOf: TypeError, + message: 'No URL protocol specified' + }); }); test('string url with searchParams is preserved', withServer, async (t, server, got) => { @@ -58,8 +68,7 @@ test('options are optional', withServer, async (t, server, got) => { test('methods are normalized', withServer, async (t, server, got) => { server.post('/test', echoUrl); - const instance = got.create({ - options: got.defaults.options, + const instance = got.extend({ handlers: [ (options, next) => { if (options.method === options.method.toUpperCase()) { @@ -76,16 +85,13 @@ test('methods are normalized', withServer, async (t, server, got) => { await instance('test', {method: 'post'}); }); -test('accepts url.parse object as first argument', withServer, async (t, server, got) => { +test('throws an error when legacy Url is passed', withServer, async (t, server, got) => { server.get('/test', echoUrl); - t.is((await got(parse(`${server.url}/test`))).body, '/test'); -}); - -test('requestUrl with url.parse object as first argument', withServer, async (t, server, got) => { - server.get('/test', echoUrl); - - t.is((await got(parse(`${server.url}/test`))).requestUrl, `${server.url}/test`); + await t.throwsAsync( + got(parse(`${server.url}/test`)), + 'The legacy `url.Url` is deprecated. Use `URL` instead.' + ); }); test('overrides `searchParams` from options', withServer, async (t, server, got) => { @@ -137,9 +143,9 @@ test('ignores empty searchParams object', withServer, async (t, server, got) => t.is((await got('test', {searchParams: {}})).requestUrl, `${server.url}/test`); }); -test('throws on invalid type of body', async t => { +test('throws when passing body with a non payload method', async t => { // @ts-ignore Manual tests - await t.throwsAsync(got('https://example.com', {body: false}), { + await t.throwsAsync(got('https://example.com', {body: 'asdf'}), { instanceOf: TypeError, message: 'The `GET` method cannot be used with a body' }); @@ -152,12 +158,12 @@ test('WHATWG URL support', withServer, async (t, server, got) => { await t.notThrowsAsync(got(wURL)); }); -test('returns streams when using stream option', withServer, async (t, server, got) => { +test('returns streams when using `isStream` option', withServer, async (t, server, got) => { server.get('/stream', (_request, response) => { response.end('ok'); }); - const data = await pEvent(got('stream', {stream: true}), 'data'); + const data = await pEvent(got('stream', {isStream: true}), 'data'); t.is(data.toString(), 'ok'); }); @@ -236,19 +242,36 @@ test('backslash in the end of `prefixUrl` option is optional', withServer, async t.is(body, '/test/foobar'); }); -test('throws when trying to modify `prefixUrl` after options got normalized', async t => { - const instanceA = got.create({ - options: {prefixUrl: 'https://example.com'}, +test('`prefixUrl` can be changed if the URL contains the old one', withServer, async (t, server) => { + server.get('/', echoUrl); + + const instanceA = got.extend({ + prefixUrl: `${server.url}/meh`, + handlers: [ + (options, next) => { + options.prefixUrl = server.url; + return next(options); + } + ] + }); + + const {body} = await instanceA(''); + t.is(body, '/'); +}); + +test('throws if cannot change `prefixUrl`', async t => { + const instanceA = got.extend({ + prefixUrl: 'https://example.com', handlers: [ (options, next) => { - // @ts-ignore Even though we know it's read only, we need to test it. - options.prefixUrl = 'https://google.com'; + options.url = new URL('https://google.pl'); + options.prefixUrl = 'https://example.com'; return next(options); } ] }); - await t.throwsAsync(instanceA(''), 'Failed to set prefixUrl. Options are normalized already.'); + await t.throwsAsync(instanceA(''), 'Cannot change `prefixUrl` from https://example.com/ to https://example.com: https://google.pl/'); }); test('throws if the `searchParams` value is invalid', async t => { @@ -313,67 +336,3 @@ test('`context` option is accessible when extending instances', t => { t.is(instance.defaults.options.context, context); t.false({}.propertyIsEnumerable.call(instance.defaults.options, 'context')); }); - -test('`options.body` is cleaned up when retrying - `options.json`', withServer, async (t, server, got) => { - let first = true; - server.post('/', (_request, response) => { - if (first) { - first = false; - - response.statusCode = 401; - } - - response.end(); - }); - - await t.notThrowsAsync(got.post('', { - hooks: { - afterResponse: [ - async (response, retryWithMergedOptions) => { - if (response.statusCode === 401) { - return retryWithMergedOptions(); - } - - t.is(response.request.options.body, undefined); - - return response; - } - ] - }, - json: { - some: 'data' - } - })); -}); - -test('`options.body` is cleaned up when retrying - `options.form`', withServer, async (t, server, got) => { - let first = true; - server.post('/', (_request, response) => { - if (first) { - first = false; - - response.statusCode = 401; - } - - response.end(); - }); - - await t.notThrowsAsync(got.post('', { - hooks: { - afterResponse: [ - async (response, retryWithMergedOptions) => { - if (response.statusCode === 401) { - return retryWithMergedOptions(); - } - - t.is(response.request.options.body, undefined); - - return response; - } - ] - }, - form: { - some: 'data' - } - })); -}); diff --git a/test/cache.ts b/test/cache.ts index b7c5d3030..648bf887b 100644 --- a/test/cache.ts +++ b/test/cache.ts @@ -92,14 +92,14 @@ test('cached response has got options', withServer, async (t, server, got) => { const cache = new Map(); const options = { - auth: 'foo:bar', + username: 'foo', cache }; await got(options); const secondResponse = await got(options); - t.is(secondResponse.request.options.auth, options.auth); + t.is(secondResponse.request.options.username, options.username); }); test('cache error throws `got.CacheError`', withServer, async (t, server, got) => { diff --git a/test/cancel.ts b/test/cancel.ts index 2ac55e304..306b5aefb 100644 --- a/test/cancel.ts +++ b/test/cancel.ts @@ -59,7 +59,7 @@ test.serial('does not retry after cancelation', withServerAndLolex, async (t, se const gotPromise = got('redirect', { retry: { - retries: () => { + calculateDelay: () => { t.fail('Makes a new try after cancelation'); } } diff --git a/test/cookies.ts b/test/cookies.ts index 4773df871..bb99e62af 100644 --- a/test/cookies.ts +++ b/test/cookies.ts @@ -130,8 +130,10 @@ test('no unhandled errors', async t => { const options = { cookieJar: { - setCookie: () => {}, - getCookieString: (_, cb) => cb(new Error(message)) + setCookie: async (_rawCookie, _url) => {}, + getCookieString: async _url => { + throw new Error(message); + } } }; @@ -142,3 +144,48 @@ test('no unhandled errors', async t => { server.close(); }); + +test('accepts custom `cookieJar` object', withServer, async (t, server, got) => { + server.get('/', (request, response) => { + response.setHeader('set-cookie', ['hello=world']); + response.end(request.headers.cookie); + }); + + const cookies = {}; + const cookieJar = { + async getCookieString(url) { + t.is(typeof url, 'string'); + + return cookies[url]; + }, + + async setCookie(rawCookie, url) { + cookies[url] = rawCookie; + } + }; + + const first = await got('', {cookieJar}); + const second = await got('', {cookieJar}); + + t.is(first.body, ''); + t.is(second.body, 'hello=world'); +}); + +test('throws on invalid `options.cookieJar.setCookie`', async t => { + await t.throwsAsync(got('https://example.com', { + cookieJar: { + // @ts-ignore + setCookie: () => {} + } + }), '`options.cookieJar.setCookie` needs to be an async function with 2 arguments'); +}); + +test('throws on invalid `options.cookieJar.getCookieString`', async t => { + await t.throwsAsync(got('https://example.com', { + cookieJar: { + setCookie: async (_rawCookie, _url) => {}, + // @ts-ignore + getCookieString: () => {} + } + }), '`options.cookieJar.getCookieString` needs to be an async function with 1 argument'); +}); diff --git a/test/create.ts b/test/create.ts index 494195b9e..06b75b3d9 100644 --- a/test/create.ts +++ b/test/create.ts @@ -103,28 +103,11 @@ test('extend merges URL instances', t => { t.is(b.defaults.options.custom.toString(), 'https://example.com/foo'); }); -test('create', withServer, async (t, server) => { - server.all('/', echoHeaders); - - const instance = got.create({ - options: {}, - handlers: [ - (options, next) => { - options.headers.unicorn = 'rainbow'; - return next(options); - } - ] - }); - const headers = await instance(server.url).json(); - t.is(headers.unicorn, 'rainbow'); - t.is(headers['user-agent'], undefined); -}); - test('hooks are merged on got.extend()', t => { const hooksA = [() => {}]; const hooksB = [() => {}]; - const instanceA = got.create({options: {hooks: {beforeRequest: hooksA}}}); + const instanceA = got.extend({hooks: {beforeRequest: hooksA}}); const extended = instanceA.extend({hooks: {beforeRequest: hooksB}}); t.deepEqual(extended.defaults.options.hooks.beforeRequest, hooksA.concat(hooksB)); @@ -140,33 +123,17 @@ test('custom endpoint with custom headers (extend)', withServer, async (t, serve }); test('no tampering with defaults', t => { - const instance = got.create({ - handlers: got.defaults.handlers, - options: got.mergeOptions(got.defaults.options, { - prefixUrl: 'example/' - }) - }); - - const instance2 = instance.create({ - handlers: instance.defaults.handlers, - options: instance.defaults.options - }); - - // Tamper Time t.throws(() => { - instance.defaults.options.prefixUrl = 'http://google.com'; + got.defaults.options.prefixUrl = 'http://google.com'; }); - t.is(instance.defaults.options.prefixUrl, 'example/'); - t.is(instance2.defaults.options.prefixUrl, 'example/'); + t.is(got.defaults.options.prefixUrl, ''); }); test('defaults can be mutable', t => { - const instance = got.create({ + const instance = got.extend({ mutableDefaults: true, - options: { - followRedirect: false - } + followRedirect: false }); t.notThrows(() => { @@ -201,7 +168,7 @@ test('only plain objects are freezed', withServer, async (t, server, got) => { test('defaults are cloned on instance creation', t => { const options = {foo: 'bar', hooks: {beforeRequest: [() => {}]}}; - const instance = got.create({options}); + const instance = got.extend(options); t.notThrows(() => { options.foo = 'foo'; diff --git a/test/error.ts b/test/error.ts index d429a4819..117c2636a 100644 --- a/test/error.ts +++ b/test/error.ts @@ -24,10 +24,7 @@ test('properties', withServer, async (t, server, got) => { t.false({}.propertyIsEnumerable.call(error, 'response')); t.false({}.hasOwnProperty.call(error, 'code')); t.is(error.message, 'Response code 404 (Not Found)'); - t.is(error.options.host, `${url.hostname}:${url.port}`); - t.is(error.options.method, 'GET'); - t.is(error.options.protocol, 'http:'); - t.is(error.options.url, error.options.requestUrl); + t.deepEqual(error.options.url, url); t.is(error.response.headers.connection, 'close'); t.is(error.response.body, 'not'); }); @@ -36,14 +33,14 @@ test('catches dns errors', async t => { const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); t.truthy(error); t.regex(error.message, /getaddrinfo ENOTFOUND/); - t.is(error.options.host, 'doesntexist'); + t.is(error.options.url.host, 'doesntexist'); t.is(error.options.method, 'GET'); }); test('`options.body` form error message', async t => { // @ts-ignore Manual tests await t.throwsAsync(got.post('https://example.com', {body: Buffer.from('test'), form: ''}), { - message: 'The `body` option cannot be used with the `json` option or `form` option' + message: 'The `body`, `json` and `form` options are mutually exclusive' }); }); @@ -108,12 +105,12 @@ test('contains Got options', withServer, async (t, server, got) => { }); const options = { - auth: 'foo:bar' + agent: false }; const error = await t.throwsAsync(got(options)); // @ts-ignore - t.is(error.options.auth, options.auth); + t.is(error.options.agent, options.agent); }); test('empty status message is overriden by the default one', withServer, async (t, server, got) => { @@ -197,7 +194,7 @@ test('catches error in mimicResponse', withServer, async (t, server) => { test('errors are thrown directly when options.stream is true', t => { t.throws(() => { // @ts-ignore Manual tests - got('https://example.com', {stream: true, hooks: false}); + got('https://example.com', {isStream: true, hooks: false}); }, { message: 'Parameter `hooks` must be an Object, not boolean' }); diff --git a/test/gzip.ts b/test/gzip.ts index 4af3f06c7..7f256146d 100644 --- a/test/gzip.ts +++ b/test/gzip.ts @@ -36,11 +36,10 @@ test('handles gzip error', withServer, async (t, server, got) => { response.end('Not gzipped content'); }); - const error = await t.throwsAsync(got(''), 'incorrect header check'); - - // @ts-ignore - t.is(error.options.path, '/'); - t.is(error.name, 'ReadError'); + await t.throwsAsync(got(''), { + name: 'ReadError', + message: 'incorrect header check' + }); }); test('handles gzip error - stream', withServer, async (t, server, got) => { @@ -49,11 +48,10 @@ test('handles gzip error - stream', withServer, async (t, server, got) => { response.end('Not gzipped content'); }); - const error = await t.throwsAsync(getStream(got.stream('')), 'incorrect header check'); - - // @ts-ignore - t.is(error.options.path, '/'); - t.is(error.name, 'ReadError'); + await t.throwsAsync(getStream(got.stream('')), { + name: 'ReadError', + message: 'incorrect header check' + }); }); test('decompress option opts out of decompressing', withServer, async (t, server, got) => { diff --git a/test/headers.ts b/test/headers.ts index 0834b9993..5da21cbdf 100644 --- a/test/headers.ts +++ b/test/headers.ts @@ -52,7 +52,8 @@ test('does not remove user headers from `url` object argument', withServer, asyn } })).body; - t.is(headers.accept, 'application/json'); + // TODO: The response is not typed, so we need to cast as any + t.is((headers as any).accept, 'application/json'); t.is(headers['user-agent'], 'got (https://github.com/sindresorhus/got)'); t.is(headers['accept-encoding'], supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate'); t.is(headers['x-request-id'], 'value'); @@ -183,19 +184,15 @@ test('buffer as `options.body` sets `content-length` header', withServer, async t.is(Number(headers['content-length']), buffer.length); }); -test('removes null value headers', withServer, async (t, server, got) => { - server.get('/', echoHeaders); - - const {body} = await got({ +test('throws on null value headers', async t => { + await t.throwsAsync(got({ headers: { 'user-agent': null } - }); - const headers = JSON.parse(body); - t.false(Reflect.has(headers, 'user-agent')); + }), TypeError, 'Use `undefined` instead of `null` to delete HTTP headers'); }); -test('setting a header to undefined keeps the old value', withServer, async (t, server, got) => { +test('removes undefined value headers', withServer, async (t, server, got) => { server.get('/', echoHeaders); const {body} = await got({ @@ -204,7 +201,7 @@ test('setting a header to undefined keeps the old value', withServer, async (t, } }); const headers = JSON.parse(body); - t.not(headers['user-agent'], undefined); + t.is(headers['user-agent'], undefined); }); test('non-existent headers set to undefined are omitted', withServer, async (t, server, got) => { diff --git a/test/hooks.ts b/test/hooks.ts index eebdab62b..396d31b45 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import getStream from 'get-stream'; import delay = require('delay'); import got from '../source'; import withServer from './helpers/with-server'; @@ -185,8 +186,8 @@ test('init is called with options', withServer, async (t, server, got) => { hooks: { init: [ options => { - t.is(options.path, '/'); - t.is(options.hostname, 'localhost'); + t.is(options.url.pathname, '/'); + t.is(options.url.hostname, 'localhost'); } ] } @@ -218,8 +219,8 @@ test('beforeRequest is called with options', withServer, async (t, server, got) hooks: { beforeRequest: [ options => { - t.is(options.path, '/'); - t.is(options.hostname, 'localhost'); + t.is(options.url.pathname, '/'); + t.is(options.url.hostname, 'localhost'); } ] } @@ -251,8 +252,8 @@ test('beforeRedirect is called with options and response', withServer, async (t, hooks: { beforeRedirect: [ (options, response) => { - t.is(options.path, '/'); - t.is(options.hostname, 'localhost'); + t.is(options.url.pathname, '/'); + t.is(options.url.hostname, 'localhost'); t.is(response.statusCode, 302); t.is(new URL(response.url).pathname, '/redirect'); @@ -291,7 +292,7 @@ test('beforeRetry is called with options', withServer, async (t, server, got) => hooks: { beforeRetry: [ (options, error, retryCount) => { - t.is(options.hostname, 'localhost'); + t.is(options.url.hostname, 'localhost'); t.truthy(error); t.true(retryCount >= 1); } @@ -381,6 +382,44 @@ test('afterResponse allows to retry', withServer, async (t, server, got) => { t.is(statusCode, 200); }); +test('afterResponse allows to retry - `beforeRetry` hook', withServer, async (t, server, got) => { + server.get('/', (request, response) => { + if (request.headers.token !== 'unicorn') { + response.statusCode = 401; + } + + response.end(); + }); + + let called = false; + + const {statusCode} = await got({ + hooks: { + afterResponse: [ + (response, retryWithMergedOptions) => { + if (response.statusCode === 401) { + return retryWithMergedOptions({ + headers: { + token: 'unicorn' + } + }); + } + + return response; + } + ], + beforeRetry: [ + options => { + t.truthy(options); + called = true; + } + ] + } + }); + t.is(statusCode, 200); + t.true(called); +}); + test('no infinity loop when retrying on afterResponse', withServer, async (t, server, got) => { server.get('/', (request, response) => { if (request.headers.token !== 'unicorn') { @@ -474,12 +513,59 @@ test('doesn\'t throw on afterResponse retry HTTP failure if throwHttpErrors is f t.is(statusCode, 500); }); -test('beforeError is called with an error', async t => { - await t.throwsAsync(got('https://example.com', { - request: () => { - throw error; - }, +test('throwing in a beforeError hook - promise', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + await t.throwsAsync(got({ + hooks: { + afterResponse: [ + () => { + throw error; + } + ], + beforeError: [ + () => { + throw new Error('foobar'); + }, + // @ts-ignore Assertion. + () => { + t.fail('This shouldn\'t be called at all'); + } + ] + } + }), 'foobar'); +}); + +test('throwing in a beforeError hook - stream', withServer, async (t, _server, got) => { + await t.throwsAsync(getStream(got.stream({ + hooks: { + beforeError: [ + () => { + throw new Error('foobar'); + }, + // @ts-ignore Assertion. + () => { + t.fail('This shouldn\'t be called at all'); + } + ] + } + })), 'foobar'); +}); + +test('beforeError is called with an error - promise', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + await t.throwsAsync(got({ hooks: { + afterResponse: [ + () => { + throw error; + } + ], beforeError: [error2 => { t.true(error2 instanceof Error); return error2; @@ -488,6 +574,17 @@ test('beforeError is called with an error', async t => { }), errorString); }); +test('beforeError is called with an error - stream', withServer, async (t, _server, got) => { + await t.throwsAsync(getStream(got.stream({ + hooks: { + beforeError: [error2 => { + t.true(error2 instanceof Error); + return error2; + }] + } + })), 'Response code 404 (Not Found)'); +}); + test('beforeError allows modifications', async t => { const errorString2 = 'foobar'; diff --git a/test/http.ts b/test/http.ts index 4e5ca5055..f13cd6e7a 100644 --- a/test/http.ts +++ b/test/http.ts @@ -114,10 +114,10 @@ test('response contains got options', withServer, async (t, server, got) => { }); const options = { - auth: 'foo:bar' + username: 'foo' }; - t.is((await got(options)).request.options.auth, options.auth); + t.is((await got(options)).request.options.username, options.username); }); test('socket destroyed by the server throws ECONNRESET', withServer, async (t, server, got) => { diff --git a/test/merge-instances.ts b/test/merge-instances.ts index 88ded05c6..0e276121b 100644 --- a/test/merge-instances.ts +++ b/test/merge-instances.ts @@ -20,27 +20,11 @@ test('merging instances', withServer, async (t, server) => { t.not(headers['user-agent'], undefined); }); -test('works even if no default handler in the end', withServer, async (t, server) => { - server.get('/', echoHeaders); - - const instanceA = got.create({ - options: {} - }); - - const instanceB = got.create({ - options: {} - }); - - const merged = instanceA.extend(instanceB); - await t.notThrowsAsync(merged(server.url)); -}); - test('merges default handlers & custom handlers', withServer, async (t, server) => { server.get('/', echoHeaders); const instanceA = got.extend({headers: {unicorn: 'rainbow'}}); - const instanceB = got.create({ - options: {}, + const instanceB = got.extend({ handlers: [ (options, next) => { options.headers.cat = 'meow'; @@ -115,50 +99,7 @@ test('hooks are merged', t => { t.deepEqual(getBeforeRequestHooks(merged), getBeforeRequestHooks(instanceA).concat(getBeforeRequestHooks(instanceB))); }); -test('hooks are passed by though other instances don\'t have them', t => { - const instanceA = got.extend({hooks: { - beforeRequest: [ - options => { - options.headers.dog = 'woof'; - } - ] - }}); - const instanceB = got.create({ - options: {} - }); - const instanceC = got.create({ - options: {hooks: {}} - }); - - const merged = instanceA.extend(instanceB, instanceC); - t.deepEqual(merged.defaults.options.hooks.beforeRequest, instanceA.defaults.options.hooks.beforeRequest); -}); - -test('URLSearchParams instances are merged', t => { - const instanceA = got.extend({ - searchParams: new URLSearchParams({a: '1'}) - }); - - const instanceB = got.extend({ - searchParams: new URLSearchParams({b: '2'}) - }); - - const merged = instanceA.extend(instanceB); - // @ts-ignore Manual tests - t.is(merged.defaults.options.searchParams.get('a'), '1'); - // @ts-ignore Manual tests - t.is(merged.defaults.options.searchParams.get('b'), '2'); -}); - -// TODO: remove this before Got v11 -test('`got.mergeInstances()` works', t => { - const instance = got.mergeInstances(got, got.create({ - options: { - headers: { - 'user-agent': null - } - } - })); - - t.is(instance.defaults.options.headers['user-agent'], null); +test('default handlers are not duplicated', t => { + const instance = got.extend(got); + t.is(instance.defaults.handlers.length, 1); }); diff --git a/test/options-to-url.ts b/test/options-to-url.ts new file mode 100644 index 000000000..4219fb5b6 --- /dev/null +++ b/test/options-to-url.ts @@ -0,0 +1,122 @@ +import test from 'ava'; +import is from '@sindresorhus/is'; +import optionsToUrl from '../source/utils/options-to-url'; + +test('`path` is deprecated', t => { + t.throws(() => { + // @ts-ignore + optionsToUrl({path: ''}); + }, 'Parameter `path` is deprecated. Use `pathname` instead.'); +}); + +test('`auth` is deprecated', t => { + t.throws(() => { + // @ts-ignore + optionsToUrl({auth: ''}); + }, 'Parameter `auth` is deprecated. Use `username`/`password` instead.'); +}); + +test('`search` and `searchParams` are mutually exclusive', t => { + t.throws(() => { + // @ts-ignore + optionsToUrl({search: 'a', searchParams: {}}); + }, 'Parameters `search` and `searchParams` are mutually exclusive.'); +}); + +test('`href` option', t => { + const href = 'https://google.com/'; + + const url = optionsToUrl({href}); + t.is(url.href, href); + t.true(is.urlInstance(url)); +}); + +test('`origin` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin}); + t.is(url.href, `${origin}/`); + t.true(is.urlInstance(url)); +}); + +test('throws if no protocol specified', t => { + t.throws(() => { + optionsToUrl({}); + }, 'No URL protocol specified'); +}); + +test('`port` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin, port: 8888}); + t.is(url.href, `${origin}:8888/`); + t.true(is.urlInstance(url)); +}); + +test('`protocol` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin, protocol: 'http:'}); + t.is(url.href, 'http://google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`username` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin, username: 'username'}); + t.is(url.href, 'https://username@google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`password` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin, password: 'password'}); + t.is(url.href, 'https://:password@google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`username` option combined with `password` option', t => { + const origin = 'https://google.com'; + + const url = optionsToUrl({origin, username: 'username', password: 'password'}); + t.is(url.href, 'https://username:password@google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`host` option', t => { + const url = optionsToUrl({protocol: 'https:', host: 'google.com'}); + t.is(url.href, 'https://google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`hostname` option', t => { + const url = optionsToUrl({protocol: 'https:', hostname: 'google.com'}); + t.is(url.href, 'https://google.com/'); + t.true(is.urlInstance(url)); +}); + +test('`pathname` option', t => { + const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', pathname: '/foobar'}); + t.is(url.href, 'https://google.com/foobar'); + t.true(is.urlInstance(url)); +}); + +test('`search` option', t => { + const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', search: '?a=1'}); + t.is(url.href, 'https://google.com/?a=1'); + t.true(is.urlInstance(url)); +}); + +test('`hash` option', t => { + const url = optionsToUrl({protocol: 'https:', hostname: 'google.com', hash: 'foobar'}); + t.is(url.href, 'https://google.com/#foobar'); + t.true(is.urlInstance(url)); +}); + +test('merges provided `searchParams`', t => { + const url = optionsToUrl({origin: 'https://google.com/?a=1', searchParams: {b: 2}}); + t.is(url.href, 'https://google.com/?a=1&b=2'); + t.true(is.urlInstance(url)); +}); diff --git a/test/post.ts b/test/post.ts index 17e5d13c9..1c42e3819 100644 --- a/test/post.ts +++ b/test/post.ts @@ -22,6 +22,14 @@ test('GET cannot have body', withServer, async (t, server, got) => { await t.throwsAsync(got.get({body: 'hi'}), 'The `GET` method cannot be used with a body'); }); +test('invalid body', async t => { + await t.throwsAsync( + got('https://example.com', {body: {} as any}), + TypeError, + 'The `body` option must be a stream.Readable, string or Buffer' + ); +}); + test('sends strings', withServer, async (t, server, got) => { server.post('/', defaultEndpoint); diff --git a/test/promise.ts b/test/promise.ts index 8269466eb..f5a3c8c9c 100644 --- a/test/promise.ts +++ b/test/promise.ts @@ -6,7 +6,7 @@ import withServer from './helpers/with-server'; test('emits request event as promise', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.statusCode = 200; - response.end(); + response.end('null'); }); await got('').json().on('request', request => { @@ -17,7 +17,7 @@ test('emits request event as promise', withServer, async (t, server, got) => { test('emits response event as promise', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.statusCode = 200; - response.end(); + response.end('null'); }); await got('').json().on('response', response => { diff --git a/test/query.ts b/test/query.ts deleted file mode 100644 index 786ff6e3f..000000000 --- a/test/query.ts +++ /dev/null @@ -1,84 +0,0 @@ -import test from 'ava'; -import withServer from './helpers/with-server'; - -// TODO: Remove this file before the Got v11 release together with completely removing the `query` option - -const echoUrl = (request, response) => { - response.end(request.url); -}; - -test('overrides query from options', withServer, async (t, server, got) => { - server.get('/', echoUrl); - - const {body} = await got( - '?drop=this', - { - query: { - test: 'wow' - }, - cache: { - get(key) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - }, - set(key) { - t.is(key, `cacheable-request:GET:${server.url}/?test=wow`); - } - } - } - ); - - t.is(body, '/?test=wow'); -}); - -test('escapes query parameter values', withServer, async (t, server, got) => { - server.get('/', echoUrl); - - const {body} = await got({ - query: { - test: 'it’s ok' - } - }); - - t.is(body, '/?test=it%E2%80%99s+ok'); -}); - -test('the `query` option can be a URLSearchParams', withServer, async (t, server, got) => { - server.get('/', echoUrl); - - const query = new URLSearchParams({test: 'wow'}); - const {body} = await got({query}); - t.is(body, '/?test=wow'); -}); - -test('should ignore empty query object', withServer, async (t, server, got) => { - server.get('/', echoUrl); - - t.is((await got({query: {}})).requestUrl, `${server.url}/`); -}); - -test('query option', withServer, async (t, server, got) => { - server.get('/', (request, response) => { - t.is(request.query.recent, 'true'); - response.end('recent'); - }); - - t.is((await got({query: {recent: true}})).body, 'recent'); - t.is((await got({query: 'recent=true'})).body, 'recent'); -}); - -test('query in options are not breaking redirects', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.end('reached'); - }); - - server.get('/relativeQuery', (request, response) => { - t.is(request.query.bang, '1'); - - response.writeHead(302, { - location: '/' - }); - response.end(); - }); - - t.is((await got('relativeQuery', {query: 'bang=1'})).body, 'reached'); -}); diff --git a/test/redirects.ts b/test/redirects.ts index 0146b75b8..3142a02f0 100644 --- a/test/redirects.ts +++ b/test/redirects.ts @@ -4,7 +4,12 @@ import nock = require('nock'); import withServer from './helpers/with-server'; const reachedHandler = (_request, response) => { - response.end('reached'); + const body = 'reached'; + + response.writeHead(200, { + 'content-length': body.length + }); + response.end(body); }; const finiteHandler = (_request, response) => { @@ -110,13 +115,13 @@ test('searchParams are not breaking redirects', withServer, async (t, server, go t.is((await got('relativeSearchParam', {searchParams: 'bang=1'})).body, 'reached'); }); -test('hostname + path are not breaking redirects', withServer, async (t, server, got) => { +test('hostname + pathname are not breaking redirects', withServer, async (t, server, got) => { server.get('/', reachedHandler); server.get('/relative', relativeHandler); t.is((await got('relative', { hostname: server.hostname, - path: '/relative' + pathname: '/relative' })).body, 'reached'); }); @@ -146,6 +151,22 @@ test('redirects POST requests', withServer, async (t, server, got) => { }); }); +test('redirects on 303 if GET or HEAD', withServer, async (t, server, got) => { + server.get('/', reachedHandler); + + server.head('/seeOther', (_request, response) => { + response.writeHead(303, { + location: '/' + }); + response.end(); + }); + + const {url, headers, request} = await got.head('seeOther'); + t.is(url, `${server.url}/`); + t.is(headers['content-length'], 'reached'.length.toString()); + t.is(request.options.method, 'HEAD'); +}); + test('redirects on 303 response even on post, put, delete', withServer, async (t, server, got) => { server.get('/', reachedHandler); diff --git a/test/response-parse.ts b/test/response-parse.ts index 5b53066ed..3fcf58972 100644 --- a/test/response-parse.ts +++ b/test/response-parse.ts @@ -14,6 +14,15 @@ test('`options.resolveBodyOnly` works', withServer, async (t, server, got) => { t.deepEqual(await got({responseType: 'json', resolveBodyOnly: true}), dog); }); +test('`options.resolveBodyOnly` combined with `options.throwHttpErrors`', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.statusCode = 404; + response.end('/'); + }); + + t.is(await got({resolveBodyOnly: true, throwHttpErrors: false}), '/'); +}); + test('JSON response', withServer, async (t, server, got) => { server.get('/', defaultHandler); @@ -50,23 +59,28 @@ test('Text response - promise.text()', withServer, async (t, server, got) => { t.is(await got('').text(), jsonResponse); }); -test('throws an error on invalid response type', withServer, async (t, server, got) => { +test('Text response - promise.json().text()', withServer, async (t, server, got) => { server.get('/', defaultHandler); - const error = await t.throwsAsync(got({responseType: 'invalid'}), /^Failed to parse body of type 'invalid'/); - // @ts-ignore - t.true(error.message.includes(error.options.hostname)); - // @ts-ignore - t.is(error.options.path, '/'); + t.is(await got('').json().text(), jsonResponse); }); -test('doesn\'t parse responses without a body', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.end(); - }); +test('works if promise has been already resolved', withServer, async (t, server, got) => { + server.get('/', defaultHandler); - const body = await got('').json(); - t.is(body, ''); + const promise = got('').text(); + t.is(await promise, jsonResponse); + t.deepEqual(await promise.json(), dog); +}); + +test('throws an error on invalid response type', withServer, async (t, server, got) => { + server.get('/', defaultHandler); + + const error = await t.throwsAsync(got({responseType: 'invalid'}), /^Failed to parse body of type 'string' as 'invalid'/); + // @ts-ignore + t.true(error.message.includes(error.options.url.hostname)); + // @ts-ignore + t.is(error.options.url.pathname, '/'); }); test('wraps parsing errors', withServer, async (t, server, got) => { @@ -76,9 +90,9 @@ test('wraps parsing errors', withServer, async (t, server, got) => { const error = await t.throwsAsync(got({responseType: 'json'}), got.ParseError); // @ts-ignore - t.true(error.message.includes(error.options.hostname)); + t.true(error.message.includes(error.options.url.hostname)); // @ts-ignore - t.is(error.options.path, '/'); + t.is(error.options.url.pathname, '/'); }); test('parses non-200 responses', withServer, async (t, server, got) => { @@ -106,7 +120,7 @@ test('ignores errors on invalid non-200 responses', withServer, async (t, server // @ts-ignore t.is(error.response.body, 'Internal error'); // @ts-ignore - t.is(error.options.path, '/'); + t.is(error.options.url.pathname, '/'); }); test('parse errors have `response` property', withServer, async (t, server, got) => { @@ -129,3 +143,13 @@ test('sets correct headers', withServer, async (t, server, got) => { t.is(headers['content-type'], 'application/json'); t.is(headers.accept, 'application/json'); }); + +test('doesn\'t throw on 204 No Content', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.statusCode = 204; + response.end(); + }); + + const body = await got('').json(); + t.is(body, ''); +}); diff --git a/test/retry.ts b/test/retry.ts index 60f21ffc9..c074db4fa 100644 --- a/test/retry.ts +++ b/test/retry.ts @@ -327,3 +327,23 @@ test('does not retry on POST', withServer, async (t, server, got) => { } }), got.TimeoutError); }); + +test('does not break on redirect', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.statusCode = 500; + response.end(); + }); + + let tries = 0; + server.get('/redirect', (_request, response) => { + tries++; + + response.writeHead(302, { + location: '/' + }); + response.end(); + }); + + await t.throwsAsync(got('redirect'), 'Response code 500 (Internal Server Error)'); + t.is(tries, 1); +}); diff --git a/test/stream.ts b/test/stream.ts index 848d92f12..d3e4094ec 100644 --- a/test/stream.ts +++ b/test/stream.ts @@ -72,6 +72,14 @@ test('throws on write if body is specified', withServer, (t, server, got) => { }, 'Got\'s stream is not writable when the `body` option is used'); }); +test('throws on write if no payload method is present', withServer, (t, server, got) => { + server.post('/', postHandler); + + t.throws(() => { + got.stream.get('').end('wow'); + }, 'The `GET` method cannot be used with a body'); +}); + test('has request event', withServer, async (t, server, got) => { server.get('/', defaultHandler);