fix: filter out markdown delimiters from next edit responses (#8056)

* fix: filter out markdown delimiters from next edit responses

* fix: silence noisy console logs (debug now) and capture accept/reject feedback

* fix: console.log -> debug

* fix: console.log -> debug

* fix feedback

* fix: respect Telemetry

* requestId

* fix: lint

* bump

Author: sestinj

core/autocomplete/postprocessing/index.test.ts

@@ -0,0 +1,118 @@
+import { postprocessCompletion } from "./index";
+
+describe("postprocessCompletion - removeBackticks", () => {
+  const mockLLM = { model: "test-model" } as any;
+
+  it("should remove first line starting with ``` and last line that is ```", () => {
+    const completion =
+      "```typescript\nfunction hello() {\n  return 'world';\n}\n```";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("function hello() {\n  return 'world';\n}");
+  });
+
+  it("should remove only first line if it starts with ```", () => {
+    const completion = "```javascript\nconst x = 5;\nconsole.log(x);";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("const x = 5;\nconsole.log(x);");
+  });
+
+  it("should remove only last line if it is ```", () => {
+    const completion = "const y = 10;\nconsole.log(y);\n```";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("const y = 10;\nconsole.log(y);");
+  });
+
+  it("should not modify completion without backticks", () => {
+    const completion = "function test() {\n  return true;\n}";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("function test() {\n  return true;\n}");
+  });
+
+  it("should handle completion with backticks in the middle", () => {
+    const completion = "const str = `template ${literal}`;\nconsole.log(str);";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe(
+      "const str = `template ${literal}`;\nconsole.log(str);",
+    );
+  });
+
+  it("should handle first line with leading whitespace before ```", () => {
+    const completion = "  ```python\ndef hello():\n  pass\n```";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("def hello():\n  pass");
+  });
+
+  it("should handle last line with whitespace around ```", () => {
+    const completion = "```\ncode here\n  ```  ";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("code here");
+  });
+
+  it("should handle single line completion", () => {
+    const completion = "const x = 5;";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("const x = 5;");
+  });
+
+  it("should handle empty completion", () => {
+    const completion = "";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBeUndefined();
+  });
+
+  it("should not remove ``` if it's not on its own line at the end", () => {
+    const completion = "```typescript\nconst x = 5; // end```";
+    const result = postprocessCompletion({
+      completion,
+      llm: mockLLM,
+      prefix: "",
+      suffix: "",
+    });
+    expect(result).toBe("const x = 5; // end```");
+  });
+});

core/autocomplete/postprocessing/index.ts

@@ -52,6 +52,38 @@ function isBlank(completion: string): boolean {
   return completion.trim().length === 0;
 }
 
+/**
+ * Removes markdown code block delimiters from completion.
+ * Removes the first line if it starts with "```" and the last line if it is exactly "```".
+ */
+function removeBackticks(completion: string): string {
+  const lines = completion.split("\n");
+
+  if (lines.length === 0) {
+    return completion;
+  }
+
+  let startIdx = 0;
+  let endIdx = lines.length;
+
+  // Remove first line if it starts with ```
+  if (lines[0].trimStart().startsWith("```")) {
+    startIdx = 1;
+  }
+
+  // Remove last line if it is exactly ```
+  if (lines.length > startIdx && lines[lines.length - 1].trim() === "```") {
+    endIdx = lines.length - 1;
+  }
+
+  // If we removed lines, return the modified completion
+  if (startIdx > 0 || endIdx < lines.length) {
+    return lines.slice(startIdx, endIdx).join("\n");
+  }
+
+  return completion;
+}
+
 export function postprocessCompletion({
   completion,
   llm,
@@ -156,5 +188,8 @@ export function postprocessCompletion({
     completion = completion.slice(1);
   }
 
+  // Remove markdown code block delimiters
+  completion = removeBackticks(completion);
+
   return completion;
 }

core/config/types.ts

@@ -78,8 +78,9 @@ declare global {
   
   export interface ILLM extends LLMOptions {
     get providerName(): string;
-  
+
     uniqueId: string;
+    lastRequestId?: string;
     model: string;
   
     title?: string;

core/core.ts

@@ -929,7 +929,7 @@ export class Core {
     });
 
     on("files/closed", async ({ data }) => {
-      console.log("deleteChain called from files/closed");
+      console.debug("deleteChain called from files/closed");
       await NextEditProvider.getInstance().deleteChain();
 
       try {

core/index.d.ts

@@ -99,6 +99,8 @@ export interface ILLM
 
   autocompleteOptions?: Partial<TabAutocompleteOptions>;
 
+  lastRequestId?: string;
+
   complete(
     prompt: string,
     signal: AbortSignal,

core/llm/index.ts

@@ -193,12 +193,15 @@ export abstract class BaseLLM implements ILLM {
 
   isFromAutoDetect?: boolean;
 
+  lastRequestId: string | undefined;
+
   private _llmOptions: LLMOptions;
 
   protected openaiAdapter?: BaseLlmApi;
 
   constructor(_options: LLMOptions) {
     this._llmOptions = _options;
+    this.lastRequestId = undefined;
 
     // Set default options
     const options = {
@@ -594,6 +597,7 @@ export abstract class BaseLLM implements ILLM {
     signal: AbortSignal,
     options: LLMFullCompletionOptions = {},
   ): AsyncGenerator<string> {
+    this.lastRequestId = undefined;
     const { completionOptions, logEnabled } =
       this._parseCompletionOptions(options);
     const interaction = logEnabled
@@ -623,6 +627,9 @@ export abstract class BaseLLM implements ILLM {
           signal,
         );
         for await (const chunk of stream) {
+          if (!this.lastRequestId && typeof (chunk as any).id === "string") {
+            this.lastRequestId = (chunk as any).id;
+          }
           const result = fromChatCompletionChunk(chunk);
           if (result) {
             const content = renderChatMessage(result);
@@ -706,6 +713,7 @@ export abstract class BaseLLM implements ILLM {
     signal: AbortSignal,
     options: LLMFullCompletionOptions = {},
   ) {
+    this.lastRequestId = undefined;
     const { completionOptions, logEnabled, raw } =
       this._parseCompletionOptions(options);
     const interaction = logEnabled
@@ -745,6 +753,7 @@ export abstract class BaseLLM implements ILLM {
             { ...toCompleteBody(prompt, completionOptions), stream: false },
             signal,
           );
+          this.lastRequestId = response.id ?? this.lastRequestId;
           completion = response.choices[0]?.text ?? "";
           yield completion;
         } else {
@@ -756,6 +765,9 @@ export abstract class BaseLLM implements ILLM {
             },
             signal,
           )) {
+            if (!this.lastRequestId && typeof (chunk as any).id === "string") {
+              this.lastRequestId = (chunk as any).id;
+            }
             const content = chunk.choices[0]?.text ?? "";
             completion += content;
             interaction?.logItem({
@@ -835,6 +847,7 @@ export abstract class BaseLLM implements ILLM {
     signal: AbortSignal,
     options: LLMFullCompletionOptions = {},
   ) {
+    this.lastRequestId = undefined;
     const { completionOptions, logEnabled, raw } =
       this._parseCompletionOptions(options);
     const interaction = logEnabled
@@ -876,6 +889,7 @@ export abstract class BaseLLM implements ILLM {
           },
           signal,
         );
+        this.lastRequestId = result.id ?? this.lastRequestId;
         completion = result.choices[0].text;
       } else {
         completion = await this._complete(prompt, signal, completionOptions);
@@ -985,6 +999,7 @@ export abstract class BaseLLM implements ILLM {
     options: LLMFullCompletionOptions = {},
     messageOptions?: MessageOption,
   ): AsyncGenerator<ChatMessage, PromptLog> {
+    this.lastRequestId = undefined;
     let { completionOptions, logEnabled } =
       this._parseCompletionOptions(options);
     const interaction = logEnabled
@@ -1054,6 +1069,7 @@ export abstract class BaseLLM implements ILLM {
               { ...body, stream: false },
               signal,
             );
+            this.lastRequestId = response.id ?? this.lastRequestId;
             const msg = fromChatResponse(response);
             yield msg;
             completion = this._formatChatMessage(msg);
@@ -1071,6 +1087,12 @@ export abstract class BaseLLM implements ILLM {
               signal,
             );
             for await (const chunk of stream) {
+              if (
+                !this.lastRequestId &&
+                typeof (chunk as any).id === "string"
+              ) {
+                this.lastRequestId = (chunk as any).id;
+              }
               const result = fromChatCompletionChunk(chunk);
               if (result) {
                 completion += this._formatChatMessage(result);

core/nextEdit/NextEditLoggingService.ts

@@ -1,5 +1,7 @@
 import { COUNT_COMPLETION_REJECTED_AFTER } from "../util/parameters";
 
+import { fetchwithRequestOptions } from "@continuedev/fetch";
+import { getControlPlaneEnvSync } from "../control-plane/env";
 import { DataLogger } from "../data/log";
 import { Telemetry } from "../util/posthog";
 import { NextEditOutcome } from "./types";
@@ -93,6 +95,9 @@ export class NextEditLoggingService {
       outcome.accepted = true;
       outcome.aborted = false;
       this.logNextEditOutcome(outcome);
+      if (outcome.requestId) {
+        void this.logAcceptReject(outcome.requestId, true);
+      }
       this._outcomes.delete(completionId);
       return outcome;
     }
@@ -111,6 +116,9 @@ export class NextEditLoggingService {
       outcome.accepted = false;
       outcome.aborted = false;
       this.logNextEditOutcome(outcome);
+      if (outcome.requestId) {
+        void this.logAcceptReject(outcome.requestId, false);
+      }
       this._outcomes.delete(completionId);
       return outcome;
     }
@@ -121,6 +129,7 @@ export class NextEditLoggingService {
       clearTimeout(this._logRejectionTimeouts.get(completionId)!);
       this._logRejectionTimeouts.delete(completionId);
     }
+
     if (this._outcomes.has(completionId)) {
       this._outcomes.delete(completionId);
     }
@@ -142,6 +151,9 @@ export class NextEditLoggingService {
       outcome.accepted = false;
       outcome.aborted = false;
       this.logNextEditOutcome(outcome);
+      if (outcome.requestId) {
+        void this.logAcceptReject(outcome.requestId, false);
+      }
       this._logRejectionTimeouts.delete(completionId);
       this._outcomes.delete(completionId);
     }, COUNT_COMPLETION_REJECTED_AFTER);
@@ -245,4 +257,30 @@ export class NextEditLoggingService {
     // const { prompt, completion, prefix, suffix, ...restOfOutcome } = outcome;
     void Telemetry.capture("nextEditOutcome", outcome, true);
   }
+
+  private async logAcceptReject(
+    requestId: string,
+    accepted: boolean,
+  ): Promise<void> {
+    try {
+      if (!Telemetry.client) {
+        return;
+      }
+
+      const controlPlaneEnv = getControlPlaneEnvSync("production");
+      await fetchwithRequestOptions(
+        new URL("model-proxy/v1/feedback", controlPlaneEnv.CONTROL_PLANE_URL),
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            requestId,
+            accepted,
+          }),
+        },
+      );
+    } catch (error) {}
+  }
 }

core/nextEdit/providers/BaseNextEditProvider.ts

@@ -309,6 +309,7 @@ export abstract class BaseNextEditModelProvider {
         outcomeCtx.completionId || outcomeCtx.helper.input.completionId,
       gitRepo: await outcomeCtx.ide.getRepoName(outcomeCtx.helper.filepath),
       uniqueId: await outcomeCtx.ide.getUniqueId(),
+      requestId: outcomeCtx.llm.lastRequestId,
       timestamp: Date.now(),
       fileUri: outcomeCtx.helper.filepath,
       workspaceDirUri:

core/nextEdit/types.ts

@@ -50,6 +50,7 @@ export interface NextEditOutcome extends TabAutocompleteOptions {
   completionId: string;
   gitRepo?: string;
   uniqueId: string;
+  requestId?: string;
   timestamp: number;
 
   // New for Next Edit.