From 97c5be307a77e97f3ac3244d127193e5a03309ea Mon Sep 17 00:00:00 2001 From: rjawesome Date: Wed, 19 Jun 2024 15:22:59 -0700 Subject: [PATCH 1/6] create smartapi test files --- src/controllers/cron/index.ts | 2 + src/controllers/cron/test_smartapi.ts | 132 ++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/controllers/cron/test_smartapi.ts diff --git a/src/controllers/cron/index.ts b/src/controllers/cron/index.ts index 4d01aeb..a727cc4 100644 --- a/src/controllers/cron/index.ts +++ b/src/controllers/cron/index.ts @@ -1,7 +1,9 @@ import smartapiCron from "./update_local_smartapi"; +import testCron from "./test_smartapi"; import cacheClearCron from "./clear_edge_cache"; export default function scheduleCronJobs() { smartapiCron(); cacheClearCron(); + testCron(); } diff --git a/src/controllers/cron/test_smartapi.ts b/src/controllers/cron/test_smartapi.ts new file mode 100644 index 0000000..8c465b3 --- /dev/null +++ b/src/controllers/cron/test_smartapi.ts @@ -0,0 +1,132 @@ +import MetaKG, { SmartAPIKGOperationObject, TestExampleObject } from "@biothings-explorer/smartapi-kg"; +import { QEdge2APIEdgeHandler, QEdge } from "@biothings-explorer/query_graph_handler"; +import CallAPI from "@biothings-explorer/call-apis"; +import { Telemetry, redisClient } from "@biothings-explorer/utils"; +import Debug from "debug"; +const debug = Debug("bte:biothings-explorer-trapi:cron"); +import cron from "node-cron"; +import path from "path"; +import { stdout } from "process"; +import { spanStatusfromHttpCode } from "@sentry/node"; +const smartAPIPath = path.resolve( + __dirname, + process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/smartapi_specs.json` : "../../../data/smartapi_specs.json", +); +const predicatesPath = path.resolve( + __dirname, + process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/predicates.json` : "../../../data/predicates.json", +); + +interface OpError { + op: string; + issue: Error; +} + +function generateEdge(op: SmartAPIKGOperationObject, ex: TestExampleObject) { + return { + subject: { + categories: [op.association.input_type], + ids: [ex.qInput], + id: "n0" + }, + object: { + categories: [op.association.output_type], + ids: [ex.oneOutput], + id: "n1" + }, + predicates: ["biolink:" + op.association.predicate], + id: "e01", + frozen: true, + }; +} + +function generateId(op: SmartAPIKGOperationObject, ex: TestExampleObject) { + return `${op.association.api_name} [${ex.qInput}-${op.association.predicate}-${ex.oneOutput}]`; +} + +async function runTests(debug = false): Promise<{errors: OpError[], opsCount: number }> { + let errors = []; + let opsCount = 0; + const metakg: MetaKG = global.metakg ? global.metakg : new MetaKG(smartAPIPath, predicatesPath); + if (!global.metakg) { + metakg.constructMetaKGSync(false); + } + const ops = metakg.ops; + for (const op of ops) { + if (op.testExamples && op.testExamples.length > 0) { + opsCount++; + } + } + if (debug) console.log(`Operation Count: ${opsCount}`); + let curCount = 0; + let errCount = 0; + for (const op of ops) { + if (op.testExamples && op.testExamples.length > 0) { + curCount++; + for (const example of op.testExamples) { + try { + const newMeta = new MetaKG(undefined, undefined, [op]); + const edge = new QEdge(generateEdge(op, example)); + edge.subject.setEquivalentIDs({ [example.qInput]: { primaryID: example.qInput, equivalentIDs: [example.qInput], label: example.qInput, labelAliases: [], primaryTypes: [op.association.input_type], semanticTypes: [op.association.input_type] }}) + const edgeConverter = new QEdge2APIEdgeHandler([edge], newMeta); + const APIEdges = await edgeConverter.convert([edge]); + const executor = new CallAPI(APIEdges, {}, redisClient); + const records = await executor.query(false, {}); + if (records.filter(r => r.object.original === example.oneOutput).length <= 0) { + errors.push({ op: generateId(op, example), issue: new Error("Record is missing") }); + errCount++; + } + } catch (error) { + errors.push({ op: generateId(op, example), issue: error }); + errCount++; + } + } + if (debug) stdout.write("\r\r\r\r\r\r\r\r\r\r\r" + curCount.toString().padStart(4, '0') + " (" + errCount.toString().padStart(4, '0') + ")"); + } + } + if (debug) console.log(""); + + return { errors, opsCount } +} + +export default function testSmartApi() { + // Env set in manual sync script + const sync_and_exit = process.env.SYNC_AND_EXIT === "true"; + if (sync_and_exit) { + console.log("Testing SmartAPI specs with subsequent exit..."); + runTests(true).then(data => { + if (data.errors.length === 0) { + console.log(`Testing SmartAPI specs successful. ${data.opsCount} operations tested.`); + } + else { + console.log(`Testing SmartAPI specs failed. ${data.errors.length} operations failed.`); + data.errors.forEach(err => { + console.log(`${err.op}: ${err.issue.message}${err.issue.message = "Record is missing" ? "" : "\n"+err.issue.stack}`); + }); + } + process.exit(0); + }) + return; + } + + cron.schedule("0 0 * * *", async () => { + debug(`Testing SmartAPI specs now at ${new Date().toUTCString()}!`); + const span = Telemetry.startSpan({ description: "smartapiTest" }); + try { + const results = await runTests(false); + if (results.errors.length === 0) { + debug(`Testing SmartAPI specs successful. ${results.opsCount} operations tested.`); + } + else { + debug(`Testing SmartAPI specs failed. ${results.errors.length} operations failed (${results.opsCount} tested).`); + results.errors.forEach(err => { + debug(`${err.op}: ${err.issue.message}${err.issue.message = "Record is missing" ? "" : "\n"+err.issue.stack}`); + Telemetry.captureException(err.issue); + }); + } + } catch (err) { + debug(`Testing SmartAPI specs failed! The error message is ${err.toString()}`); + } + span.finish(); + }); +} \ No newline at end of file From f0f2d318ab062d86861aa043cb04e21d7d5a5a1a Mon Sep 17 00:00:00 2001 From: rjawesome Date: Wed, 19 Jun 2024 15:24:10 -0700 Subject: [PATCH 2/6] add depedency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 6c10459..7d6ed5a 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "@biothings-explorer/smartapi-kg": "workspace:../smartapi-kg", "@biothings-explorer/types": "workspace:../types", "@biothings-explorer/utils": "workspace:../utils", + "@biothings-explorer/call-apis": "workspace:../call-apis", "@bull-board/api": "^5.9.1", "@bull-board/express": "^5.9.1", "@opentelemetry/api": "^1.7.0", From c69f6715e32e35d9182e3548ac58ffeaf6dd1be7 Mon Sep 17 00:00:00 2001 From: rjawesome Date: Thu, 20 Jun 2024 13:49:42 -0700 Subject: [PATCH 3/6] fix debugs test smartapi --- src/controllers/cron/test_smartapi.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/controllers/cron/test_smartapi.ts b/src/controllers/cron/test_smartapi.ts index 8c465b3..2d0095b 100644 --- a/src/controllers/cron/test_smartapi.ts +++ b/src/controllers/cron/test_smartapi.ts @@ -113,7 +113,9 @@ export default function testSmartApi() { debug(`Testing SmartAPI specs now at ${new Date().toUTCString()}!`); const span = Telemetry.startSpan({ description: "smartapiTest" }); try { + let dbg_namespaces = Debug.disable(); const results = await runTests(false); + Debug.enable(dbg_namespaces) if (results.errors.length === 0) { debug(`Testing SmartAPI specs successful. ${results.opsCount} operations tested.`); } From 9449fc3e3b990b439861b98420718853929064c5 Mon Sep 17 00:00:00 2001 From: rjawesome Date: Thu, 20 Jun 2024 14:18:36 -0700 Subject: [PATCH 4/6] sentry breadcrumbs for testing smartapi --- src/controllers/cron/test_smartapi.ts | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/controllers/cron/test_smartapi.ts b/src/controllers/cron/test_smartapi.ts index 2d0095b..e0650c4 100644 --- a/src/controllers/cron/test_smartapi.ts +++ b/src/controllers/cron/test_smartapi.ts @@ -121,10 +121,32 @@ export default function testSmartApi() { } else { debug(`Testing SmartAPI specs failed. ${results.errors.length} operations failed (${results.opsCount} tested).`); + const recMissingList = []; results.errors.forEach(err => { - debug(`${err.op}: ${err.issue.message}${err.issue.message = "Record is missing" ? "" : "\n"+err.issue.stack}`); - Telemetry.captureException(err.issue); + debug(`${err.op}: ${err.issue.message}${err.issue.message == "Record is missing" ? "" : "\n"+err.issue.stack}`); + if (err.issue.message == "Record is missing") { + recMissingList.push(err.op); + } else { + Telemetry.addBreadcrumb({ + type: 'error', + data: { + op: err.op + }, + message: 'SmartAPI Operation Failed!' + }); + Telemetry.captureException(err.issue); + } }); + if (recMissingList.length > 0) { + Telemetry.addBreadcrumb({ + type: 'error', + data: { + missingRecords: recMissingList + }, + message: 'Records Missing for SmartAPI Operations!' + }); + Telemetry.captureException(new Error(`Records missing for SmartAPI operations`)); + } } } catch (err) { debug(`Testing SmartAPI specs failed! The error message is ${err.toString()}`); From 685c3b3d0ddaaa437827c32c214bf3a7119664a6 Mon Sep 17 00:00:00 2001 From: rjawesome Date: Thu, 20 Jun 2024 14:29:30 -0700 Subject: [PATCH 5/6] capture smartapi testing ex --- src/controllers/cron/test_smartapi.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/controllers/cron/test_smartapi.ts b/src/controllers/cron/test_smartapi.ts index e0650c4..1946480 100644 --- a/src/controllers/cron/test_smartapi.ts +++ b/src/controllers/cron/test_smartapi.ts @@ -150,6 +150,7 @@ export default function testSmartApi() { } } catch (err) { debug(`Testing SmartAPI specs failed! The error message is ${err.toString()}`); + Telemetry.captureException(err); } span.finish(); }); From 39435c0cfa61f2352487f04d539dd588eeaf000c Mon Sep 17 00:00:00 2001 From: rjawesome Date: Mon, 22 Jul 2024 13:40:54 -0700 Subject: [PATCH 6/6] separate errors in smartapi testing; check if api is avaliable --- src/controllers/cron/test_smartapi.ts | 106 ++++++++++++++------------ 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/src/controllers/cron/test_smartapi.ts b/src/controllers/cron/test_smartapi.ts index 1946480..ac31a66 100644 --- a/src/controllers/cron/test_smartapi.ts +++ b/src/controllers/cron/test_smartapi.ts @@ -7,7 +7,8 @@ const debug = Debug("bte:biothings-explorer-trapi:cron"); import cron from "node-cron"; import path from "path"; import { stdout } from "process"; -import { spanStatusfromHttpCode } from "@sentry/node"; +import API_LIST from "../../config/api_list"; +import axios from "axios"; const smartAPIPath = path.resolve( __dirname, process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/smartapi_specs.json` : "../../../data/smartapi_specs.json", @@ -17,9 +18,11 @@ const predicatesPath = path.resolve( process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/predicates.json` : "../../../data/predicates.json", ); -interface OpError { - op: string; - issue: Error; +class SmartapiSpecError extends Error { + constructor(message: string) { + super(message); + this.name = "SmartapiSpecError"; + } } function generateEdge(op: SmartAPIKGOperationObject, ex: TestExampleObject) { @@ -41,28 +44,48 @@ function generateEdge(op: SmartAPIKGOperationObject, ex: TestExampleObject) { } function generateId(op: SmartAPIKGOperationObject, ex: TestExampleObject) { - return `${op.association.api_name} [${ex.qInput}-${op.association.predicate}-${ex.oneOutput}]`; + return `${op.association.api_name} ${ex.qInput}-${op.association.predicate}-${ex.oneOutput}`; } -async function runTests(debug = false): Promise<{errors: OpError[], opsCount: number }> { +async function runTests(debug = false): Promise<{errors: Error[], opsCount: number }> { let errors = []; - let opsCount = 0; const metakg: MetaKG = global.metakg ? global.metakg : new MetaKG(smartAPIPath, predicatesPath); if (!global.metakg) { - metakg.constructMetaKGSync(false); + metakg.constructMetaKGSync(true); } const ops = metakg.ops; - for (const op of ops) { - if (op.testExamples && op.testExamples.length > 0) { - opsCount++; - } - } - if (debug) console.log(`Operation Count: ${opsCount}`); - let curCount = 0; + + let found = {}; + let opsCount = 0; let errCount = 0; for (const op of ops) { + const includedAPI = API_LIST.include.find(x => x.id === op.association.smartapi.id); + if (!includedAPI || API_LIST.exclude.find(x => x.id === op.association.smartapi.id)) { + continue; + } + + // API is unreachable + if (found[op.association.smartapi.id] === false) { + continue; + } + + // check if API is unreachable + if (!(op.association.smartapi.id in found)) { + try { + await axios.get(op.query_operation.server, { validateStatus: () => true, timeout: 5000, maxRedirects: 0 }); + found[op.association.smartapi.id] = true; + } catch (e) { + console.log('fun') + console.log(op.association.api_name) + console.log(e) + found[op.association.smartapi.id] = false; + errors.push(new SmartapiSpecError(`[${includedAPI.name}]: API is unreachable`)); + continue; + } + } + if (op.testExamples && op.testExamples.length > 0) { - curCount++; + opsCount++; for (const example of op.testExamples) { try { const newMeta = new MetaKG(undefined, undefined, [op]); @@ -73,19 +96,30 @@ async function runTests(debug = false): Promise<{errors: OpError[], opsCount: nu const executor = new CallAPI(APIEdges, {}, redisClient); const records = await executor.query(false, {}); if (records.filter(r => r.object.original === example.oneOutput).length <= 0) { - errors.push({ op: generateId(op, example), issue: new Error("Record is missing") }); + errors.push(new SmartapiSpecError(`[${generateId(op, example)}]: Record is missing`)); errCount++; } } catch (error) { - errors.push({ op: generateId(op, example), issue: error }); + if (!error.message) error.message = "Error"; + error.message = `[${generateId(op, example)}]: ${error.message}`; + errors.push(error); errCount++; } } - if (debug) stdout.write("\r\r\r\r\r\r\r\r\r\r\r" + curCount.toString().padStart(4, '0') + " (" + errCount.toString().padStart(4, '0') + ")"); + if (debug) stdout.write("\r\r\r\r\r\r\r\r\r\r\r" + opsCount.toString().padStart(4, '0') + " (" + errCount.toString().padStart(4, '0') + ")"); } } if (debug) console.log(""); + for (const api of API_LIST.include) { + if (API_LIST.exclude.find(x => x.id === api.id)) { + continue; + } + if (!(api.id in found)) { + errors.push(new SmartapiSpecError(`[${api.name}]: API does not have a spec`)); + } + } + return { errors, opsCount } } @@ -99,9 +133,9 @@ export default function testSmartApi() { console.log(`Testing SmartAPI specs successful. ${data.opsCount} operations tested.`); } else { - console.log(`Testing SmartAPI specs failed. ${data.errors.length} operations failed.`); + console.log(`Testing SmartAPI specs failed. ${data.errors.length} operations/APIs failed.`); data.errors.forEach(err => { - console.log(`${err.op}: ${err.issue.message}${err.issue.message = "Record is missing" ? "" : "\n"+err.issue.stack}`); + console.log(`${err.message}${err instanceof SmartapiSpecError ? "" : "\n"+err.stack}`); }); } process.exit(0); @@ -109,7 +143,7 @@ export default function testSmartApi() { return; } - cron.schedule("0 0 * * *", async () => { + cron.schedule("* * * * *", async () => { debug(`Testing SmartAPI specs now at ${new Date().toUTCString()}!`); const span = Telemetry.startSpan({ description: "smartapiTest" }); try { @@ -120,33 +154,11 @@ export default function testSmartApi() { debug(`Testing SmartAPI specs successful. ${results.opsCount} operations tested.`); } else { - debug(`Testing SmartAPI specs failed. ${results.errors.length} operations failed (${results.opsCount} tested).`); - const recMissingList = []; + debug(`Testing SmartAPI specs failed. ${results.errors.length} operations/APIs failed.`); results.errors.forEach(err => { - debug(`${err.op}: ${err.issue.message}${err.issue.message == "Record is missing" ? "" : "\n"+err.issue.stack}`); - if (err.issue.message == "Record is missing") { - recMissingList.push(err.op); - } else { - Telemetry.addBreadcrumb({ - type: 'error', - data: { - op: err.op - }, - message: 'SmartAPI Operation Failed!' - }); - Telemetry.captureException(err.issue); - } + debug(`${err.message}${err instanceof SmartapiSpecError ? "" : "\n"+err.stack}`); + Telemetry.captureException(err); }); - if (recMissingList.length > 0) { - Telemetry.addBreadcrumb({ - type: 'error', - data: { - missingRecords: recMissingList - }, - message: 'Records Missing for SmartAPI Operations!' - }); - Telemetry.captureException(new Error(`Records missing for SmartAPI operations`)); - } } } catch (err) { debug(`Testing SmartAPI specs failed! The error message is ${err.toString()}`);