-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Allow filerting backticks in AI code completion (#14777)
* Allow filerting backticks in AI code completion fixed #14461 Signed-off-by: Jonas Helming <[email protected]>
- Loading branch information
1 parent
d59071a
commit d9a6525
Showing
5 changed files
with
166 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
84 changes: 84 additions & 0 deletions
84
packages/ai-code-completion/src/browser/code-completion-postprocessor.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // ***************************************************************************** | ||
| // Copyright (C) 2024 EclipseSource GmbH. | ||
| // | ||
| // This program and the accompanying materials are made available under the | ||
| // terms of the Eclipse Public License v. 2.0 which is available at | ||
| // http://www.eclipse.org/legal/epl-2.0. | ||
| // | ||
| // This Source Code may also be made available under the following Secondary | ||
| // Licenses when the conditions for such availability set forth in the Eclipse | ||
| // Public License v. 2.0 are satisfied: GNU General Public License, version 2 | ||
| // with the GNU Classpath Exception which is available at | ||
| // https://www.gnu.org/software/classpath/license.html. | ||
| // | ||
| // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 | ||
| // ***************************************************************************** | ||
|
|
||
| import { enableJSDOM } from '@theia/core/lib/browser/test/jsdom'; | ||
| let disableJSDOM = enableJSDOM(); | ||
| import { FrontendApplicationConfigProvider } from '@theia/core/lib/browser/frontend-application-config-provider'; | ||
| FrontendApplicationConfigProvider.set({}); | ||
|
|
||
| import { expect } from 'chai'; | ||
| import { DefaultCodeCompletionPostProcessor } from './code-completion-postprocessor'; | ||
|
|
||
| disableJSDOM(); | ||
|
|
||
| describe('CodeCompletionAgentImpl', () => { | ||
| let codeCompletionProcessor: DefaultCodeCompletionPostProcessor; | ||
| before(() => { | ||
| disableJSDOM = enableJSDOM(); | ||
| codeCompletionProcessor = new DefaultCodeCompletionPostProcessor(); | ||
| }); | ||
|
|
||
| after(() => { | ||
| // Disable JSDOM after all tests | ||
| disableJSDOM(); | ||
| }); | ||
|
|
||
| describe('stripBackticks', () => { | ||
|
|
||
| it('should remove surrounding backticks and language (TypeScript)', () => { | ||
| const input = '```TypeScript\nconsole.log(\"Hello, World!\");```'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('console.log("Hello, World!");'); | ||
| }); | ||
|
|
||
| it('should remove surrounding backticks and language (md)', () => { | ||
| const input = '```md\nconsole.log(\"Hello, World!\");```'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('console.log("Hello, World!");'); | ||
| }); | ||
|
|
||
| it('should remove all text after second occurrence of backticks', () => { | ||
| const input = '```js\nlet x = 10;\n```\nTrailing text should be removed'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('let x = 10;'); | ||
| }); | ||
|
|
||
| it('should return the text unchanged if no surrounding backticks', () => { | ||
| const input = 'console.log(\"Hello, World!\");'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('console.log("Hello, World!");'); | ||
| }); | ||
|
|
||
| it('should remove surrounding backticks without language', () => { | ||
| const input = '```\nconsole.log(\"Hello, World!\");```'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('console.log("Hello, World!");'); | ||
| }); | ||
|
|
||
| it('should handle text starting with backticks but no second delimiter', () => { | ||
| const input = '```python\nprint(\"Hello, World!\")'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('print("Hello, World!")'); | ||
| }); | ||
|
|
||
| it('should handle multiple internal backticks correctly', () => { | ||
| const input = '```\nFoo```Bar```FooBar```'; | ||
| const output = codeCompletionProcessor.stripBackticks(input); | ||
| expect(output).to.equal('Foo```Bar```FooBar'); | ||
| }); | ||
|
|
||
| }); | ||
| }); |
48 changes: 48 additions & 0 deletions
48
packages/ai-code-completion/src/browser/code-completion-postprocessor.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // ***************************************************************************** | ||
| // Copyright (C) 2024 EclipseSource GmbH. | ||
| // | ||
| // This program and the accompanying materials are made available under the | ||
| // terms of the Eclipse Public License v. 2.0 which is available at | ||
| // http://www.eclipse.org/legal/epl-2.0. | ||
| // | ||
| // This Source Code may also be made available under the following Secondary | ||
| // Licenses when the conditions for such availability set forth in the Eclipse | ||
| // Public License v. 2.0 are satisfied: GNU General Public License, version 2 | ||
| // with the GNU Classpath Exception which is available at | ||
| // https://www.gnu.org/software/classpath/license.html. | ||
| // | ||
| // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 | ||
| // ***************************************************************************** | ||
|
|
||
| import { inject, injectable } from '@theia/core/shared/inversify'; | ||
| import { PreferenceService } from '@theia/core/lib/browser'; | ||
| import { PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS } from './ai-code-completion-preference'; | ||
|
|
||
| export interface CodeCompletionPostProcessor { | ||
| postProcess(text: string): string; | ||
| } | ||
| export const CodeCompletionPostProcessor = Symbol('CodeCompletionPostProcessor'); | ||
|
|
||
| @injectable() | ||
| export class DefaultCodeCompletionPostProcessor { | ||
|
|
||
| @inject(PreferenceService) | ||
| protected readonly preferenceService: PreferenceService; | ||
|
|
||
| public postProcess(text: string): string { | ||
| if (this.preferenceService.get<boolean>(PREF_AI_INLINE_COMPLETION_STRIP_BACKTICKS, true)) { | ||
| return this.stripBackticks(text); | ||
| } | ||
| return text; | ||
| } | ||
|
|
||
| public stripBackticks(text: string): string { | ||
| if (text.startsWith('```')) { | ||
| // Remove the first backticks and any language identifier | ||
| const startRemoved = text.slice(3).replace(/^\w*\n/, ''); | ||
| const lastBacktickIndex = startRemoved.lastIndexOf('```'); | ||
| return lastBacktickIndex !== -1 ? startRemoved.slice(0, lastBacktickIndex).trim() : startRemoved.trim(); | ||
| } | ||
| return text; | ||
| } | ||
| } |