feat(embedding): implement ILanguageModelProvider in LocalEmbeddingsProvider

The LocalEmbeddingsProvider class now implements the ILanguageModelProvider interface. This includes the addition of several methods such as provideChatResponse, provideCompletionResponse, provideEmbedDocuments, and provideEmbedQuery. Also, a log message was added to indicate the completion of LanceDB indexing.

Author: phodal

src/base/common/language-models/languageModelsService.ts

@@ -10,6 +10,7 @@ import { OllamaLanguageModelProvider } from './providers/ollamaProvider';
 import { OpenAILanguageModelProvider } from './providers/openaiProvider';
 import { TongyiLanguageModelProvider } from './providers/TongyiProvider';
 import { WenxinLanguageModelProvider } from './providers/WenxinProvider';
+import { LocalEmbeddingsProvider } from "../../../code-search/embedding/LocalEmbeddingsProvider";
 
 export interface LanguageModelSelector {
 	/**
@@ -39,6 +40,7 @@ export class LanguageModelsService {
 			['tongyi', new TongyiLanguageModelProvider(configService)],
 			['ollama', new OllamaLanguageModelProvider(configService)],
 			['transformers', new HuggingFaceTransformersLanguageModelProvider(configService)],
+			// ['transformers', LocalEmbeddingsProvider.getInstance()],
 		]);
 	}
 

src/code-search/embedding/LocalEmbeddingsProvider.ts

@@ -5,6 +5,8 @@ import { Embedding } from './_base/Embedding';
 import { EmbeddingsProvider } from './_base/EmbeddingsProvider';
 import { mean_pooling, mergedTensor, normalize_, reshape, tensorData } from './_base/EmbeddingUtils';
 import { logger } from 'base/common/log/log';
+import type { ILanguageModelProvider } from "base/common/language-models/languageModels";
+import { CancellationToken } from 'vscode';
 
 // @ts-expect-error
 const ortPromise = import('onnxruntime-node');
@@ -20,7 +22,7 @@ const InferenceSessionCreate = (...args: any[]) => {
  *
  * @deprecated Please use LanguageModelsService instead.
  */
-export class LocalEmbeddingsProvider implements EmbeddingsProvider {
+export class LocalEmbeddingsProvider implements EmbeddingsProvider, ILanguageModelProvider {
 	id: string = 'local';
 	env: any;
 	tokenizer: any;
@@ -30,12 +32,25 @@ export class LocalEmbeddingsProvider implements EmbeddingsProvider {
 
 	private static instance: LocalEmbeddingsProvider;
 
-	private constructor() {}
+	private constructor() {
+	}
+
+	identifier: string = 'local';
+
+	async provideChatResponse(): Promise<never> {
+		throw new Error('This method is not implemented');
+	}
+
+	async provideCompletionResponse(): Promise<never> {
+		throw new Error('This method is not implemented');
+	}
 
 	static getInstance(): LocalEmbeddingsProvider {
 		if (!LocalEmbeddingsProvider.instance) {
 			LocalEmbeddingsProvider.instance = new LocalEmbeddingsProvider();
+			LocalEmbeddingsProvider.instance.init();
 		}
+
 		return LocalEmbeddingsProvider.instance;
 	}
 
@@ -64,6 +79,22 @@ export class LocalEmbeddingsProvider implements EmbeddingsProvider {
 		logger.appendLine("'hello' text's first 10 values" + value[0].slice(0, 10).join(', '));
 	}
 
+	async provideEmbedDocuments(
+			texts: string[],
+			options: { [name: string]: any },
+			token?: CancellationToken,
+	): Promise<number[][]> {
+		return this.embed(texts);
+	}
+
+	async provideEmbedQuery(
+			input: string | string[],
+			options: { [name: string]: any },
+			token?: CancellationToken,
+	): Promise<number[]> {
+		return (await this.embed(Array.isArray(input) ? input : [input]))[0];
+	}
+
 	async embed(chunks: string[]): Promise<Embedding[]> {
 		if (chunks.length === 0) {
 			return [];

src/code-search/indexing/LanceDbIndex.ts

@@ -267,6 +267,7 @@ export class LanceDbIndex implements CodebaseIndex {
 		}
 
 		markComplete(results.del, IndexResultType.Delete);
+		logger.appendLine('Completed LanceDB Indexing');
 		yield { progress: 1, desc: 'Completed Calculating Embeddings' };
 	}
 

src/extension.ts

@@ -15,6 +15,8 @@ import { LanguageModelsService } from './base/common/language-models/languageMod
 import { CommandsService } from './commands/commandsService';
 import { ChatViewService } from './editor/views/chat/chatViewService';
 
+(globalThis as any).self = globalThis;
+
 export async function activate(context: ExtensionContext) {
 	const instantiationService = new InstantiationService();