From edafe936aa69f4052e6d5df85f1739fe3bb70f53 Mon Sep 17 00:00:00 2001 From: Ciaran Schutte Date: Mon, 16 Dec 2024 01:23:30 -0500 Subject: [PATCH] cleanup network resolvers fix resp --- .../src/network/resolvers/aggregations.ts | 271 ++++-------------- modules/server/src/network/resolvers/fetch.ts | 71 +++++ modules/server/src/network/resolvers/index.ts | 80 +++--- .../src/network/resolvers/networkNode.ts | 16 -- modules/server/src/network/resolvers/query.ts | 105 +++++++ .../server/src/network/resolvers/response.ts | 6 +- 6 files changed, 283 insertions(+), 266 deletions(-) create mode 100644 modules/server/src/network/resolvers/fetch.ts delete mode 100644 modules/server/src/network/resolvers/networkNode.ts create mode 100644 modules/server/src/network/resolvers/query.ts diff --git a/modules/server/src/network/resolvers/aggregations.ts b/modules/server/src/network/resolvers/aggregations.ts index 83baffd04..e994ec533 100644 --- a/modules/server/src/network/resolvers/aggregations.ts +++ b/modules/server/src/network/resolvers/aggregations.ts @@ -1,21 +1,16 @@ -import { gql } from 'apollo-server-core'; -import axios, { AxiosError } from 'axios'; -import { DocumentNode } from 'graphql'; -import { isEmpty } from 'lodash'; - +import { Aggregations } from '@/mapping/resolveAggregations'; import { AggregationAccumulator } from '../aggregations/AggregationAccumulator'; -import { fetchGql } from '../gql'; -import { failure, isSuccess, Result, success } from '../result'; +import { failure, isSuccess } from '../result'; +import { NodeConfig } from '../setup/query'; import { Hits } from '../types/hits'; -import { AllAggregations, NodeConfig } from '../types/types'; -import { ASTtoString, RequestedFieldsMap } from '../utils/gql'; -import { CONNECTION_STATUS, NetworkNode } from './networkNode'; +import { RequestedFieldsMap } from '../utils/gql'; +import { fetchData } from './fetch'; +import { createNetworkQuery } from './query'; -type NetworkQuery = { - url: string; - gqlQuery: DocumentNode; - queryVariables: QueryVariables; -}; +export const CONNECTION_STATUS = { + OK: 'OK', + ERROR: 'ERROR', +} as const; type QueryVariables = { filters?: object; @@ -23,169 +18,22 @@ type QueryVariables = { include_missing?: boolean; }; -/** - * Query remote connections and handle network responses - * - * @param query - * @returns - */ - -// narrows type -const isAxiosError = (error: unknown): error is AxiosError => axios.isAxiosError(error); - -const fetchData = async ( - query: NetworkQuery, -): Promise> => { - const { url, gqlQuery, queryVariables } = query; - - console.log(`Fetch data starting for ${url}`); - - try { - const response = await fetchGql({ - url, - gqlQuery: ASTtoString(gqlQuery), - variables: queryVariables, - }); - - // axios response "data" field, graphql response "data" field - const responseData = response.data?.data; - if (response.status === 200 && response.statusText === 'OK') { - console.log(`Fetch data completing for ${query.url}`); - return success(responseData); - } - } catch (error) { - if (axios.isCancel(error)) { - console.log(`Fetch data cancelled for ${query.url}`); - return failure(CONNECTION_STATUS.ERROR, `Request cancelled: ${url}`); - } - - if (axios.isAxiosError(error)) { - const errorResponse = error as AxiosError<{ errors: { message: string }[] }>; - - if (errorResponse.code === 'ECONNREFUSED') { - console.error(`Network failure: ${url}`); - return failure(CONNECTION_STATUS.ERROR, `Network failure: ${url}`); - } - - if (error.response) { - const errors = - errorResponse.response && - errorResponse.response.data.errors.map((gqlError) => gqlError.message).join('\n'); - console.error(errors); - return failure(CONNECTION_STATUS.ERROR, 'errors'); - } - } - return failure(CONNECTION_STATUS.ERROR, `Unknown error`); - } - // TS would like a return value outside of try/catch handling - return failure(CONNECTION_STATUS.ERROR, `Unknown error`); -}; - -/** - * Converts info field object into string - * @param requestedFields - * - * @example - * ### Input - * ``` - * { donors: { - * buckets: { - * bucket_count: {}, - * } - * }} - * ``` - * - * ### Output - * ``` - * ` - * { donors { - * buckets { - * bucket_count - * } - * }} - * ` - * ``` - */ -const convertFieldsToString = (requestedFields: RequestedFieldsMap) => { - const gqlFieldsString = JSON.stringify(requestedFields) - .replaceAll('"', '') - .replaceAll(':', '') - .replaceAll('{}', '') - .replaceAll(',', ' ') - .replaceAll('\\', ' '); +type SuccessResponse = Record; - return gqlFieldsString; +export type NetworkNode = { + name: string; + hits: number; + status: keyof typeof CONNECTION_STATUS; + errors: string; + aggregations: { name: string; type: string }[]; }; -/** - * Creates individual GQL query string for a node. - * Includes aggregation GQL arguments (actual data is provided alongside query, not here) - * Requested fields are converted to GQL style strings - * - * @param documentName - * @param requestedFields - * @returns constructed query string - */ -export const createNodeQueryString = ( - documentName: string, - requestedFields: RequestedFieldsMap, -) => { - const fields = convertFieldsToString(requestedFields); - const aggregationsString = !isEmpty(fields) ? `aggregations ${fields}` : ''; - const gqlString = `query nodeQuery {${documentName} { hits { total } ${aggregationsString} }}`; - return gqlString; -}; - -/** - * Creates a GQL query for fields with query arguments. - * Only adds requested fields that are available on a node. - * - * @param config - * @param requestedFields - * @returns a GQL document node or undefined if a valid GQL document node cannot be created - */ -export const createNetworkQuery = ( - config: NodeConfig, - requestedFields: RequestedFieldsMap, -): DocumentNode | undefined => { - const availableFields = config.aggregations; - const documentName = config.documentName; - - // ensure requested field is available to query on node - const fieldsToRequest = Object.keys(requestedFields).reduce((acc, requestedFieldKey) => { - const field = requestedFields[requestedFieldKey]; - if (availableFields.some((field) => field.name === requestedFieldKey)) { - return { ...acc, [requestedFieldKey]: field }; - } else { - return acc; - } - }, {}); - - const gqlString = createNodeQueryString(documentName, fieldsToRequest); - - /* - * convert string to AST object - * not needed if gqlString is formatted correctly but this acts as a validity check - */ - try { - const gqlQuery = gql` - ${gqlString} - `; - return gqlQuery; - } catch (err) { - console.error('invalid gql', err); - return undefined; - } -}; - -type SuccessResponse = { [k: string]: { hits: Hits; aggregations: AllAggregations } }; - /** * Query each remote connection * * @param queries - Query for each remote connection - * @param requestedAggregationFields - * @returns + * @param requestedAggregationFields - Fields requested + * @returns Resolved aggregation and node info */ export const aggregationPipeline = async ( configs: NodeConfig[], @@ -196,45 +44,48 @@ export const aggregationPipeline = async ( const totalAgg = new AggregationAccumulator(requestedAggregationFields); - const aggregationResultPromises = configs.map(async (config) => { - const gqlQuery = createNetworkQuery(config, requestedAggregationFields); - const response = gqlQuery - ? await fetchData({ - url: config.graphqlUrl, - gqlQuery, - queryVariables, - }) - : failure(CONNECTION_STATUS.ERROR, 'Invalid GQL query'); - - const nodeName = config.displayName; - - if (isSuccess(response)) { - const documentName = config.documentName; - const responseData = response.data[documentName]; - const aggregations = responseData?.aggregations || {}; - const hits = responseData?.hits || { total: 0 }; - - totalAgg.resolve({ aggregations, hits }); - - nodeInfo.push({ - name: nodeName, - hits: hits.total, - status: CONNECTION_STATUS.OK, - errors: '', - aggregations: config.aggregations, - }); - } else { - nodeInfo.push({ - name: nodeName, - hits: 0, - status: CONNECTION_STATUS.ERROR, - errors: response?.message || 'Error', - aggregations: config.aggregations, - }); - } - }); + Promise.allSettled( + configs.map(async (config) => { + // create node query + const gqlQuery = createNetworkQuery(config, requestedAggregationFields); + + // query node + const response = gqlQuery + ? await fetchData({ + url: config.graphqlUrl, + gqlQuery, + queryVariables, + }) + : failure(CONNECTION_STATUS.ERROR, 'Invalid GQL query'); + + const nodeName = config.displayName; + + if (isSuccess(response)) { + const documentName = config.documentName; + const responseData = response.data[documentName]; + const aggregations = responseData?.aggregations || {}; + const hits = responseData?.hits || { total: 0 }; + + totalAgg.resolve({ aggregations, hits }); + + nodeInfo.push({ + name: nodeName, + hits: hits.total, + status: CONNECTION_STATUS.OK, + errors: '', + aggregations: config.aggregations, + }); + } else { + nodeInfo.push({ + name: nodeName, + hits: 0, + status: CONNECTION_STATUS.ERROR, + errors: response?.message || 'Error', + aggregations: config.aggregations, + }); + } + }), + ); - // return accumulated results - await Promise.allSettled(aggregationResultPromises); return { aggregationResults: totalAgg.result(), nodeInfo }; }; diff --git a/modules/server/src/network/resolvers/fetch.ts b/modules/server/src/network/resolvers/fetch.ts new file mode 100644 index 000000000..8c2f4f128 --- /dev/null +++ b/modules/server/src/network/resolvers/fetch.ts @@ -0,0 +1,71 @@ +import axios, { AxiosError } from 'axios'; +import { DocumentNode } from 'graphql'; + +import { fetchGql } from '../gql'; +import { failure, Result, success } from '../result'; +import { ASTtoString } from '../utils/gql'; +import { CONNECTION_STATUS } from './aggregations'; + +type NetworkQuery = { + url: string; + gqlQuery: DocumentNode; + queryVariables: { + filters?: object; + aggregations_filter_themselves?: boolean; + include_missing?: boolean; + }; +}; + +/** + * Query remote connections and handle network responses + * + * @param query + * @returns + */ +export const fetchData = async ( + query: NetworkQuery, +): Promise> => { + const { url, gqlQuery, queryVariables } = query; + + console.log(`Fetch data starting for ${url}..`); + + try { + const response = await fetchGql({ + url, + gqlQuery: ASTtoString(gqlQuery), + variables: queryVariables, + }); + + // axios response "data" field, graphql response "data" field + const responseData = response.data?.data; + if (response.status === 200 && response.statusText === 'OK') { + console.log(`Fetch data completing for ${query.url}`); + return success(responseData); + } + } catch (error) { + if (axios.isCancel(error)) { + console.log(`Fetch data cancelled for ${query.url}`); + return failure(CONNECTION_STATUS.ERROR, `Request cancelled: ${url}`); + } + + if (axios.isAxiosError(error)) { + const errorResponse = error as AxiosError<{ errors: { message: string }[] }>; + + if (errorResponse.code === 'ECONNREFUSED') { + console.error(`Network failure: ${url}`); + return failure(CONNECTION_STATUS.ERROR, `Network failure: ${url}`); + } + + if (error.response) { + const errors = + errorResponse.response && + errorResponse.response.data.errors.map((gqlError) => gqlError.message).join('\n'); + console.error(errors); + return failure(CONNECTION_STATUS.ERROR, 'errors'); + } + } + return failure(CONNECTION_STATUS.ERROR, `Unknown error`); + } + // TS would like a return value outside of try/catch handling + return failure(CONNECTION_STATUS.ERROR, `Unknown error`); +}; diff --git a/modules/server/src/network/resolvers/index.ts b/modules/server/src/network/resolvers/index.ts index 147b6afb4..6dc5a340b 100644 --- a/modules/server/src/network/resolvers/index.ts +++ b/modules/server/src/network/resolvers/index.ts @@ -1,13 +1,12 @@ -import { type GraphQLResolveInfo } from 'graphql'; +import { Resolver } from '@/gqlServer'; import { isSuccess } from '../result'; -import { NodeConfig } from '../types/types'; +import { NodeConfig } from '../setup/query'; import { resolveInfoToMap } from '../utils/gql'; import { convertToSqon } from '../utils/sqon'; -import { aggregationPipeline } from './aggregations'; -import { NetworkNode } from './networkNode'; +import { aggregationPipeline, NetworkNode } from './aggregations'; import { createResponse } from './response'; -export type NetworkSearchRoot = { +type NetworkSearchRoot = { nodes: NetworkNode[]; aggregations: Record; }; @@ -15,14 +14,15 @@ export type NetworkSearchRoot = { /* * Type should match the "Network" GQL type definition arg types */ -export type NetworkAggregationArgs = { +// TODO: shared? +type NetworkAggregationArgs = { filters?: object; aggregations_filter_themselves?: boolean; include_missing?: boolean; }; /** - * Create GQL resolvers. + * Resolvers for network search. * * It's important to have both remote connection data and aggregations under a single field * as remote connection data is dependant on aggregations query @@ -32,38 +32,44 @@ export type NetworkAggregationArgs = { * @returns */ export const createResolvers = (configs: NodeConfig[]) => { - return { - Root: { - network: async ( - parent: NetworkSearchRoot, - // as mentioned above, type should match gql typedefs - args: NetworkAggregationArgs, - context: unknown, - info: GraphQLResolveInfo, - ) => { - const requestedFieldsMap = resolveInfoToMap(info, 'aggregations'); + const network: Resolver = async ( + _unusedParentObj, + args, + _unusedContext, + info, + ) => { + const requestedFieldsMap = resolveInfoToMap(info, 'aggregations'); - /* - * checks validity of SQON - * for now we will pass through the non SQON object to the pipeline - * TODO: resolve Arranger / SQONBuilder SQON outer wrapper conflict - * ie. {"content": [{...}], "op": "and"} - */ - if ('filters' in args) { - const result = convertToSqon(args.filters); - if (!isSuccess(result)) { - throw new Error(`${result.status} : ${result.message}`); - } - } - const queryVariables = { ...args }; + /* + * Checks validity of SQON + * For now we will pass through the non SQON object to the pipeline + * + * TODO: resolve Arranger / SQONBuilder SQON outer wrapper conflict + * {"content": [{...}], "op": "and"} + */ + if ('filters' in args) { + const result = convertToSqon(args.filters); + if (!isSuccess(result)) { + throw new Error(`${result.status} : ${result.message}`); + } + } + const queryVariables = { ...args }; - const { aggregationResults, nodeInfo } = await aggregationPipeline( - configs, - requestedFieldsMap, - queryVariables, - ); - return createResponse({ aggregationResults, nodeInfo }); - }, + /* + * Aggregation pipeline entrypoint + */ + const { aggregationResults, nodeInfo } = await aggregationPipeline( + configs, + requestedFieldsMap, + queryVariables, + ); + + return createResponse({ aggregationResults, nodeInfo }); + }; + + return { + Root: { + network, }, }; }; diff --git a/modules/server/src/network/resolvers/networkNode.ts b/modules/server/src/network/resolvers/networkNode.ts deleted file mode 100644 index 1c7b34db4..000000000 --- a/modules/server/src/network/resolvers/networkNode.ts +++ /dev/null @@ -1,16 +0,0 @@ -export const CONNECTION_STATUS = { - OK: 'OK', - ERROR: 'ERROR', -} as const; - -/* - * Querying from resolvers remote connections - */ - -export type NetworkNode = { - name: string; - hits: number; - status: keyof typeof CONNECTION_STATUS; - errors: string; - aggregations: { name: string; type: string }[]; -}; diff --git a/modules/server/src/network/resolvers/query.ts b/modules/server/src/network/resolvers/query.ts new file mode 100644 index 000000000..ba115d256 --- /dev/null +++ b/modules/server/src/network/resolvers/query.ts @@ -0,0 +1,105 @@ +import { gql } from 'apollo-server-core'; +import { DocumentNode } from 'graphql'; +import { isEmpty } from 'lodash'; + +import { NodeConfig } from '../setup/query'; +import { RequestedFieldsMap } from '../utils/gql'; + +/** + * Converts info field object into string + * + * @param requestedFields - Query fields object + * @returns GQL string + * + * @example + * ### Input + * ``` + * { donors: { + * buckets: { + * bucket_count: {}, + * } + * }} + * ``` + * + * ### Output + * ``` + * ` + * { donors { + * buckets { + * bucket_count + * } + * }} + * ` + * ``` + */ +const convertFieldsToString = (requestedFields: RequestedFieldsMap) => { + const gqlFieldsString = JSON.stringify(requestedFields) + .replaceAll('"', '') + .replaceAll(':', '') + .replaceAll('{}', '') + .replaceAll(',', ' ') + .replaceAll('\\', ' '); + + return gqlFieldsString; +}; + +/** + * Creates individual GQL query string for a node. + * Includes aggregation GQL arguments (actual data is sent with query in additional param) + * Requested fields are converted to GQL style strings + * + * This hardcodes an aggregation query for network search + * + * @param documentName - File type document name configured on node + * @param requestedFields - Query fields object + * @returns Fully constructed query string + */ +const createFileGQLQuery = (documentName: string, requestedFields: RequestedFieldsMap) => { + const fields = convertFieldsToString(requestedFields); + const queryArgs = `(filters: $filters, aggregations_filter_themselves: $aggregations_filter_themselves, include_missing: $include_missing)`; + const aggregationsString = !isEmpty(fields) ? `aggregations${queryArgs} ${fields}` : ''; + const gqlString = `query nodeQuery${queryArgs} {${documentName} { hits { total } ${aggregationsString} }}`; + return gqlString; +}; + +/** + * Creates a GQL query for fields with query arguments. + * Only adds requested fields that are available on a node. + * + * @param config - Node config + * @param requestedFields - Fields from query + * @returns a GQL document node or undefined if a valid GQL document node cannot be created + */ +export const createNetworkQuery = ( + config: NodeConfig, + requestedFields: RequestedFieldsMap, +): DocumentNode | undefined => { + const availableFields = config.aggregations; + const documentName = config.documentName; + + // ensure requested field is available to query on node + const fieldsToRequest = Object.keys(requestedFields).reduce((fields, requestedFieldKey) => { + const field = requestedFields[requestedFieldKey]; + if (availableFields.some((field) => field.name === requestedFieldKey)) { + return { ...fields, [requestedFieldKey]: field }; + } else { + return fields; + } + }, {}); + + const gqlString = createFileGQLQuery(documentName, fieldsToRequest); + + /* + * convert string to AST object + * not needed if gqlString is formatted correctly but this acts as a validity check + */ + try { + const gqlQuery = gql` + ${gqlString} + `; + return gqlQuery; + } catch (err) { + console.error('invalid gql', err); + return undefined; + } +}; diff --git a/modules/server/src/network/resolvers/response.ts b/modules/server/src/network/resolvers/response.ts index b4e6cede7..cb95e103e 100644 --- a/modules/server/src/network/resolvers/response.ts +++ b/modules/server/src/network/resolvers/response.ts @@ -1,5 +1,5 @@ -import { AllAggregations } from '../types/types'; -import { NetworkNode } from './networkNode'; +import { Aggregations } from '@/mapping/resolveAggregations'; +import { NetworkNode } from './aggregations'; /** * Format response object to match GQL type defs @@ -8,7 +8,7 @@ export const createResponse = ({ aggregationResults, nodeInfo, }: { - aggregationResults: AllAggregations; + aggregationResults: Aggregations; nodeInfo: NetworkNode[]; }) => { return { nodes: nodeInfo, aggregations: aggregationResults };