@W-18964528@ Added test coverage for run_code_analyzer

Author: jfeingold35

packages/mcp/test/evals/run_code_analyzer.eval.ts

@@ -0,0 +1,23 @@
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+import { describeEval } from 'vitest-evals';
+import { TaskRunner } from './utils.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const pathToTarget: string = path.join(__dirname, '..', 'fixtures', 'sample-targets', 'SampleTarget1.cls');
+
+describeEval('run_code_analyzer', {
+    data: async () => [{
+        input: `Run code analysis against ${pathToTarget}, and tell me the number of violations in that file using the response format "There are X violations".` ,
+        expected: [6]
+    }],
+    task: TaskRunner(),
+    scorers: [(opts: {output: string, expected: number}) => {
+        const score: number = opts.output === `There are ${opts.expected} violations.` ? 1 : 0;
+        return {score};
+    }],
+    threshold: 0.9,
+    timeout: 60_000
+});

packages/mcp/test/evals/utils.ts

@@ -80,11 +80,14 @@ export function TaskRunner(model: LanguageModel = defaultModel) {
 
 export function outputIncludesExpectationArray(opts: {input: string, output: string, expected: string[]}) {
     let score: number = 0;
+    //console.log(`output is ${opts.output}`);
     const increment: number = 1/opts.expected.length;
     for (const expected of opts.expected) {
         if (opts.output.toLowerCase().includes(expected.toLowerCase())) {
+            //console.log(`contained ${expected}, icnrementing`);
             score += increment;
         }
+        //console.log(`score is now ${score}`)
     }
     return {
         score

packages/mcp/test/fixtures/sample-targets/SampleTarget1.cls

@@ -0,0 +1,9 @@
+public class SampleTarget1 {
+    public static boolean doSomething() {
+        Integer i = 0;
+        for (i = 0; i < 10; i++) {
+            Account[] accs = [SELECT Name FROM Account WITH SYSTEM_MODE];
+        }
+        return false;
+    }
+}