From 6c18a848d181848a4caa511a4c2c7fd727ea48ab Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Thu, 8 Nov 2018 12:25:05 +1100 Subject: [PATCH] chore: refactor run-sift (#9764) * refactor run-sift * update yarn.lock --- packages/gatsby/src/redux/run-sift.js | 479 ++++++++++++++------------ yarn.lock | 12 +- 2 files changed, 267 insertions(+), 224 deletions(-) diff --git a/packages/gatsby/src/redux/run-sift.js b/packages/gatsby/src/redux/run-sift.js index 9fefcb2666e57..558c7f9d48769 100644 --- a/packages/gatsby/src/redux/run-sift.js +++ b/packages/gatsby/src/redux/run-sift.js @@ -18,100 +18,78 @@ const enhancedNodeCacheId = ({ node, args }) => }) : null -function awaitSiftField(fields, node, k) { - const field = fields[k] - if (field.resolve) { - return field.resolve( - node, - {}, - {}, - { - fieldName: k, - } - ) - } else if (node[k] !== undefined) { - return node[k] - } - - return undefined -} - const nodesCache = new Map() -/** - * Filters a list of nodes using mongodb-like syntax. - * - * @param args raw graphql query filter as an object - * @param nodes The nodes array to run sift over - * @param type gqlType - * @param typeName - * @param firstOnly true if you want to return only the first result - * found. This will return a collection of size 1. Not a single - * element - * @returns Collection of results. Collection will be limited to size - * if `firstOnly` is true - */ -module.exports = ({ queryArgs, gqlType, firstOnly = false }: Object) => { - // Clone args as for some reason graphql-js removes the constructor - // from nested objects which breaks a check in sift.js. - const clonedArgs = JSON.parse(JSON.stringify(queryArgs)) - - // this caching can be removed if we move to loki +function loadNodes(type) { let nodes - if (process.env.NODE_ENV === `production` && nodesCache.has(gqlType.name)) { - nodes = nodesCache.get(gqlType.name) + // this caching can be removed if we move to loki + if (process.env.NODE_ENV === `production` && nodesCache.has(type)) { + nodes = nodesCache.get(type) } else { - nodes = getNodesByType(gqlType.name) - nodesCache.set(gqlType.name, nodes) + nodes = getNodesByType(type) + nodesCache.set(type, nodes) } + return nodes +} - const siftifyArgs = object => { - const newObject = {} - _.each(object, (v, k) => { - if (_.isPlainObject(v)) { - if (k === `elemMatch`) { - k = `$elemMatch` - } - newObject[k] = siftifyArgs(v) - } else { - // Compile regex first. - if (k === `regex`) { - newObject[`$regex`] = prepareRegex(v) - } else if (k === `glob`) { - const Minimatch = require(`minimatch`).Minimatch - const mm = new Minimatch(v) - newObject[`$regex`] = mm.makeRe() - } else { - newObject[`$${k}`] = v - } - } - }) - return newObject - } +///////////////////////////////////////////////////////////////////// +// Parse filter +///////////////////////////////////////////////////////////////////// - // Build an object that excludes the innermost leafs, - // this avoids including { eq: x } when resolving fields. - function extractFieldsToSift(prekey, key, preobj, obj, val) { - if (_.isPlainObject(val)) { - _.forEach((val: any), (v, k) => { - if (k === `elemMatch`) { - // elemMatch is operator for arrays and not field we want to prepare - // so we need to skip it - extractFieldsToSift(prekey, key, preobj, obj, v) - return - } - preobj[prekey] = obj - extractFieldsToSift(key, k, obj, {}, v) - }) +function siftifyArgs(object) { + const newObject = {} + _.each(object, (v, k) => { + if (_.isPlainObject(v)) { + if (k === `elemMatch`) { + k = `$elemMatch` + } + newObject[k] = siftifyArgs(v) } else { - preobj[prekey] = true + // Compile regex first. + if (k === `regex`) { + newObject[`$regex`] = prepareRegex(v) + } else if (k === `glob`) { + const Minimatch = require(`minimatch`).Minimatch + const mm = new Minimatch(v) + newObject[`$regex`] = mm.makeRe() + } else { + newObject[`$${k}`] = v + } } + }) + return newObject +} + +// Build an object that excludes the innermost leafs, +// this avoids including { eq: x } when resolving fields. +function extractFieldsToSift(prekey, key, preobj, obj, val) { + if (_.isPlainObject(val)) { + _.forEach((val: any), (v, k) => { + if (k === `elemMatch`) { + // elemMatch is operator for arrays and not field we want to prepare + // so we need to skip it + extractFieldsToSift(prekey, key, preobj, obj, v) + return + } + preobj[prekey] = obj + extractFieldsToSift(key, k, obj, {}, v) + }) + } else { + preobj[prekey] = true } +} +/** + * Parse filter and returns an object with two fields: + * - siftArgs: the filter in a format that sift understands + * - fieldsToSift: filter with operate leaves (e.g { eq: 3 }) + * removed. Used later to resolve all filter fields + */ +function parseFilter(filter) { const siftArgs = [] const fieldsToSift = {} - if (clonedArgs.filter) { - _.each(clonedArgs.filter, (v, k) => { + if (filter) { + _.each(filter, (v, k) => { // Ignore connection and sorting args. if (_.includes([`skip`, `limit`, `sort`], k)) return @@ -123,68 +101,212 @@ module.exports = ({ queryArgs, gqlType, firstOnly = false }: Object) => { extractFieldsToSift(``, k, {}, fieldsToSift, v) }) } + return { siftArgs, fieldsToSift } +} - // Resolves every field used in the node. - function resolveRecursive(node, siftFieldsObj, gqFields) { - return Promise.all( - _.keys(siftFieldsObj).map(k => - Promise.resolve(awaitSiftField(gqFields, node, k)) - .then(v => { - const innerSift = siftFieldsObj[k] - const innerGqConfig = gqFields[k] - if ( - _.isObject(innerSift) && - v != null && - innerGqConfig && - innerGqConfig.type +///////////////////////////////////////////////////////////////////// +// Resolve nodes +///////////////////////////////////////////////////////////////////// + +function isEqId(firstOnly, fieldsToSift, siftArgs) { + return ( + firstOnly && + Object.keys(fieldsToSift).length === 1 && + Object.keys(fieldsToSift)[0] === `id` && + Object.keys(siftArgs[0].id).length === 1 && + Object.keys(siftArgs[0].id)[0] === `$eq` + ) +} + +function awaitSiftField(fields, node, k) { + const field = fields[k] + if (field.resolve) { + return field.resolve( + node, + {}, + {}, + { + fieldName: k, + } + ) + } else if (node[k] !== undefined) { + return node[k] + } + + return undefined +} + +// Resolves every field used in the node. +function resolveRecursive(node, siftFieldsObj, gqFields) { + return Promise.all( + _.keys(siftFieldsObj).map(k => + Promise.resolve(awaitSiftField(gqFields, node, k)) + .then(v => { + const innerSift = siftFieldsObj[k] + const innerGqConfig = gqFields[k] + if ( + _.isObject(innerSift) && + v != null && + innerGqConfig && + innerGqConfig.type + ) { + if (_.isFunction(innerGqConfig.type.getFields)) { + // this is single object + return resolveRecursive( + v, + innerSift, + innerGqConfig.type.getFields() + ) + } else if ( + _.isArray(v) && + innerGqConfig.type.ofType && + _.isFunction(innerGqConfig.type.ofType.getFields) ) { - if (_.isFunction(innerGqConfig.type.getFields)) { - // this is single object - return resolveRecursive( - v, - innerSift, - innerGqConfig.type.getFields() - ) - } else if ( - _.isArray(v) && - innerGqConfig.type.ofType && - _.isFunction(innerGqConfig.type.ofType.getFields) - ) { - // this is array - return Promise.all( - v.map(item => - resolveRecursive( - item, - innerSift, - innerGqConfig.type.ofType.getFields() - ) + // this is array + return Promise.all( + v.map(item => + resolveRecursive( + item, + innerSift, + innerGqConfig.type.ofType.getFields() ) ) - } + ) } + } + + return v + }) + .then(v => [k, v]) + ) + ).then(resolvedFields => { + const myNode = { + ...node, + } + resolvedFields.forEach(([k, v]) => (myNode[k] = v)) + return myNode + }) +} + +function resolveNodes(nodes, typeName, firstOnly, fieldsToSift, gqlFields) { + const nodesCacheKey = JSON.stringify({ + // typeName + count being the same is a pretty good + // indication that the nodes are the same. + typeName, + firstOnly, + nodesLength: nodes.length, + ...fieldsToSift, + }) + if ( + process.env.NODE_ENV === `production` && + resolvedNodesCache.has(nodesCacheKey) + ) { + return Promise.resolve(resolvedNodesCache.get(nodesCacheKey)) + } else { + return Promise.all( + nodes.map(node => { + const cacheKey = enhancedNodeCacheId({ + node, + args: fieldsToSift, + }) + if (cacheKey && enhancedNodeCache.has(cacheKey)) { + return Promise.resolve(enhancedNodeCache.get(cacheKey)) + } else if (cacheKey && enhancedNodePromiseCache.has(cacheKey)) { + return enhancedNodePromiseCache.get(cacheKey) + } - return v + const enhancedNodeGenerationPromise = new Promise(resolve => { + resolveRecursive(node, fieldsToSift, gqlFields).then(resolvedNode => { + trackInlineObjectsInRootNode(resolvedNode) + if (cacheKey) { + enhancedNodeCache.set(cacheKey, resolvedNode) + } + resolve(resolvedNode) }) - .then(v => [k, v]) - ) - ).then(resolvedFields => { - const myNode = { - ...node, - } - resolvedFields.forEach(([k, v]) => (myNode[k] = v)) - return myNode + }) + enhancedNodePromiseCache.set(cacheKey, enhancedNodeGenerationPromise) + return enhancedNodeGenerationPromise + }) + ).then(resolvedNodes => { + resolvedNodesCache.set(nodesCacheKey, resolvedNodes) + return resolvedNodes }) } +} + +///////////////////////////////////////////////////////////////////// +// Run Sift +///////////////////////////////////////////////////////////////////// + +function handleFirst(siftArgs, nodes) { + const index = _.isEmpty(siftArgs) + ? 0 + : sift.indexOf( + { + $and: siftArgs, + }, + nodes + ) + + if (index !== -1) { + return [nodes[index]] + } else { + return [] + } +} + +function handleMany(siftArgs, nodes, sort) { + let result = _.isEmpty(siftArgs) + ? nodes + : sift( + { + $and: siftArgs, + }, + nodes + ) + + if (!result || !result.length) return null + + // Sort results. + if (sort) { + // create functions that return the item to compare on + // uses _.get so nested fields can be retrieved + const convertedFields = sort.fields + .map(field => field.replace(/___/g, `.`)) + .map(field => v => _.get(v, field)) + + result = _.orderBy(result, convertedFields, sort.order) + } + return result +} + +/** + * Filters a list of nodes using mongodb-like syntax. + * + * @param args raw graphql query filter as an object + * @param nodes The nodes array to run sift over (Optional + * will load itself if not present) + * @param type gqlType. Created in build-node-types + * @param firstOnly true if you want to return only the first result + * found. This will return a collection of size 1. Not a single + * element + * @returns Collection of results. Collection will be limited to size + * if `firstOnly` is true + */ +module.exports = (args: Object) => { + const { queryArgs, gqlType, firstOnly = false } = args + // Clone args as for some reason graphql-js removes the constructor + // from nested objects which breaks a check in sift.js. + const clonedArgs = JSON.parse(JSON.stringify(queryArgs)) + + // If nodes weren't provided, then load them from the DB + const nodes = args.nodes || loadNodes(gqlType.name) + + const { siftArgs, fieldsToSift } = parseFilter(clonedArgs.filter) // If the the query for single node only has a filter for an "id" // using "eq" operator, then we'll just grab that ID and return it. - if ( - firstOnly && - Object.keys(fieldsToSift).length === 1 && - Object.keys(fieldsToSift)[0] === `id` && - Object.keys(siftArgs[0].id).length === 1 && - Object.keys(siftArgs[0].id)[0] === `$eq` - ) { + if (isEqId(firstOnly, fieldsToSift, siftArgs)) { return resolveRecursive( getNode(siftArgs[0].id[`$eq`]), fieldsToSift, @@ -192,94 +314,17 @@ module.exports = ({ queryArgs, gqlType, firstOnly = false }: Object) => { ).then(node => (node ? [node] : [])) } - const nodesPromise = () => { - const nodesCacheKey = JSON.stringify({ - // typeName + count being the same is a pretty good - // indication that the nodes are the same. - typeName: gqlType.name, - firstOnly, - nodesLength: nodes.length, - ...fieldsToSift, - }) - if ( - process.env.NODE_ENV === `production` && - resolvedNodesCache.has(nodesCacheKey) - ) { - return Promise.resolve(resolvedNodesCache.get(nodesCacheKey)) - } else { - return Promise.all( - nodes.map(node => { - const cacheKey = enhancedNodeCacheId({ - node, - args: fieldsToSift, - }) - if (cacheKey && enhancedNodeCache.has(cacheKey)) { - return Promise.resolve(enhancedNodeCache.get(cacheKey)) - } else if (cacheKey && enhancedNodePromiseCache.has(cacheKey)) { - return enhancedNodePromiseCache.get(cacheKey) - } - - const enhancedNodeGenerationPromise = new Promise(resolve => { - resolveRecursive(node, fieldsToSift, gqlType.getFields()).then( - resolvedNode => { - trackInlineObjectsInRootNode(resolvedNode) - if (cacheKey) { - enhancedNodeCache.set(cacheKey, resolvedNode) - } - resolve(resolvedNode) - } - ) - }) - enhancedNodePromiseCache.set(cacheKey, enhancedNodeGenerationPromise) - return enhancedNodeGenerationPromise - }) - ).then(resolvedNodes => { - resolvedNodesCache.set(nodesCacheKey, resolvedNodes) - return resolvedNodes - }) - } - } - const tempPromise = nodesPromise().then(myNodes => { + return resolveNodes( + nodes, + gqlType.name, + firstOnly, + fieldsToSift, + gqlType.getFields() + ).then(resolvedNodes => { if (firstOnly) { - const index = _.isEmpty(siftArgs) - ? 0 - : sift.indexOf( - { - $and: siftArgs, - }, - myNodes - ) - - if (index !== -1) { - return [myNodes[index]] - } else { - return [] - } + return handleFirst(siftArgs, resolvedNodes) } else { - let result = _.isEmpty(siftArgs) - ? myNodes - : sift( - { - $and: siftArgs, - }, - myNodes - ) - - if (!result || !result.length) return null - - // Sort results. - if (clonedArgs.sort) { - // create functions that return the item to compare on - // uses _.get so nested fields can be retrieved - const convertedFields = clonedArgs.sort.fields - .map(field => field.replace(/___/g, `.`)) - .map(field => v => _.get(v, field)) - - result = _.orderBy(result, convertedFields, clonedArgs.sort.order) - } - return result + return handleMany(siftArgs, resolvedNodes, clonedArgs.sort) } }) - - return tempPromise } diff --git a/yarn.lock b/yarn.lock index 389886b55b1c7..8a0b3ae8e7707 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2957,13 +2957,6 @@ babel-plugin-lodash@^3.2.11: lodash "^4.17.10" require-package-name "^2.0.1" -babel-plugin-macros@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/babel-plugin-macros/-/babel-plugin-macros-2.4.0.tgz#6c5f9836e1f6c0a9743b3bab4af29f73e437e544" - integrity sha512-flIBfrqAdHWn+4l2cS/4jZEyl+m5EaBHVzTb0aOF+eu/zR7E41/MoCFHPhDNL8Wzq1nyelnXeT+vcL2byFLSZw== - dependencies: - cosmiconfig "^5.0.5" - babel-plugin-macros@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/babel-plugin-macros/-/babel-plugin-macros-2.4.2.tgz#21b1a2e82e2130403c5ff785cba6548e9b644b28" @@ -11509,6 +11502,11 @@ json-stable-stringify@^1.0.0: dependencies: jsonify "~0.0.0" +json-stream-stringify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/json-stream-stringify/-/json-stream-stringify-2.0.1.tgz#8bc0e65ff94567d9010e14c27c043a951cb14939" + integrity sha512-5XymtJXPmzRWZ1UdLQQQXbjHV/E7NAanSClikEqORbkZKOYLSYLNHqRuooyju9W90kJUzknFhX2xvWn4cHluHQ== + json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"