diff --git a/.changeset/polite-coats-return.md b/.changeset/polite-coats-return.md new file mode 100644 index 0000000000..2a11866908 --- /dev/null +++ b/.changeset/polite-coats-return.md @@ -0,0 +1,8 @@ +--- +"@llamaindex/e2e": patch +"@llamaindex/core": patch +"llamaindex": patch +"pg-vector-store": patch +--- + +refactor: @llamaindex/postgres diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e5162b206..bc6c6ede69 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,7 +150,7 @@ jobs: done - name: Pack provider packages run: | - for dir in packages/providers/*; do + for dir in packages/providers/* packages/providers/storage/*; do if [ -d "$dir" ] && [ -f "$dir/package.json" ]; then echo "Packing $dir" pnpm pack --pack-destination ${{ runner.temp }} -C $dir diff --git a/e2e/node/vector-store/pg-vector-store.e2e.ts b/e2e/node/vector-store/pg-vector-store.e2e.ts index b1725a1a8f..8bcb259766 100644 --- a/e2e/node/vector-store/pg-vector-store.e2e.ts +++ b/e2e/node/vector-store/pg-vector-store.e2e.ts @@ -1,6 +1,5 @@ import { config } from "dotenv"; -import { Document, VectorStoreQueryMode } from "llamaindex"; -import { PGVectorStore } from "llamaindex/vector-store/PGVectorStore"; +import { Document, PGVectorStore, VectorStoreQueryMode } from "llamaindex"; import assert from "node:assert"; import { test } from "node:test"; import pg from "pg"; diff --git a/examples/vector-store/pg/README.md b/examples/vector-store/pg/README.md index dc636a4ffc..b769196cd0 100644 --- a/examples/vector-store/pg/README.md +++ b/examples/vector-store/pg/README.md @@ -37,7 +37,7 @@ Read and follow the instructions in the README.md file located one directory up To import documents and save the embedding vectors to your database: -> `npx tsx pg-vector-store/load-docs.ts data` +> `npx tsx vector-store/pg/load-docs.ts data` where data is the directory containing your input files. Using the `data` directory in the example above will read all of the files in that directory using the LlamaIndexTS default readers for each file type. @@ -45,6 +45,23 @@ where data is the directory containing your input files. Using the `data` direct To query using the resulting vector store: -> `npx tsx pg-vector-store/query.ts` +> `npx tsx vector-store/pg/query.ts` The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query. + +## Supabase + +You can try the supabase example by running: + +> `npx tsx vector-store/pg/supabase.ts` + +This will use the `POSTGRES_URL` environment variable to connect to your Supabase database. +Get one from the Supabase project settings page. See more details here: https://supabase.com/docs/guides/database/connecting-to-postgres#direct-connection + +## Vercel + +You can try the vercel example by running: + +> `npx tsx vector-store/pg/vercel.ts` + +For more information on Vercel Postgres, see: https://vercel.com/docs/storage/vercel-postgres/sdk diff --git a/examples/vector-store/pg/supabase.ts b/examples/vector-store/pg/supabase.ts new file mode 100644 index 0000000000..748b48362d --- /dev/null +++ b/examples/vector-store/pg/supabase.ts @@ -0,0 +1,34 @@ +import dotenv from "dotenv"; +import { + PGVectorStore, + SimpleDirectoryReader, + storageContextFromDefaults, + VectorStoreIndex, +} from "llamaindex"; + +dotenv.config(); + +// Get direct connection string from Supabase and set it as POSTGRES_URL environment variable +// https://supabase.com/docs/guides/database/connecting-to-postgres#direct-connection + +const sourceDir = "../data"; +const connectionString = process.env.POSTGRES_URL; + +const rdr = new SimpleDirectoryReader(); +const docs = await rdr.loadData({ directoryPath: sourceDir }); +const pgvs = new PGVectorStore({ clientConfig: { connectionString } }); +pgvs.setCollection(sourceDir); + +const ctx = await storageContextFromDefaults({ vectorStore: pgvs }); + +const index = await VectorStoreIndex.fromDocuments(docs, { + storageContext: ctx, +}); + +const queryEngine = index.asQueryEngine(); + +const results = await queryEngine.query({ + query: "Information about the planet", +}); + +console.log(results); diff --git a/packages/core/package.json b/packages/core/package.json index 07edc30642..acca9b55aa 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -390,6 +390,8 @@ }, "devDependencies": { "@edge-runtime/vm": "^4.0.4", + "@types/lodash.get": "^4.4.9", + "@types/lodash.clone": "^4.5.9", "ajv": "^8.17.1", "bunchee": "6.2.0", "happy-dom": "^15.11.6", @@ -398,6 +400,8 @@ "dependencies": { "@llamaindex/env": "workspace:*", "@types/node": "^22.9.0", + "lodash.get": "^4.4.2", + "lodash.clone": "^4.5.0", "magic-bytes.js": "^1.10.0", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.3" diff --git a/packages/core/src/global/settings.ts b/packages/core/src/global/settings.ts index 3a3af7ccf7..3cb97b880f 100644 --- a/packages/core/src/global/settings.ts +++ b/packages/core/src/global/settings.ts @@ -1,5 +1,6 @@ import { getEnv } from "@llamaindex/env"; import type { Tokenizer } from "@llamaindex/env/tokenizers"; +import type { BaseEmbedding } from "../embeddings"; import type { LLM } from "../llms"; import { type CallbackManager, @@ -12,6 +13,11 @@ import { setChunkSize, withChunkSize, } from "./settings/chunk-size"; +import { + getEmbeddedModel, + setEmbeddedModel, + withEmbeddedModel, +} from "./settings/embedModel"; import { getLLM, setLLM, withLLM } from "./settings/llm"; import { getTokenizer, @@ -29,6 +35,15 @@ export const Settings = { withLLM(llm: LLM, fn: () => Result): Result { return withLLM(llm, fn); }, + get embedModel() { + return getEmbeddedModel(); + }, + set embedModel(embedModel) { + setEmbeddedModel(embedModel); + }, + withEmbedModel(embedModel: BaseEmbedding, fn: () => Result): Result { + return withEmbeddedModel(embedModel, fn); + }, get tokenizer() { return getTokenizer(); }, diff --git a/packages/llamaindex/src/internal/settings/EmbedModel.ts b/packages/core/src/global/settings/embedModel.ts similarity index 67% rename from packages/llamaindex/src/internal/settings/EmbedModel.ts rename to packages/core/src/global/settings/embedModel.ts index d912d20b9c..deb80a2981 100644 --- a/packages/llamaindex/src/internal/settings/EmbedModel.ts +++ b/packages/core/src/global/settings/embedModel.ts @@ -1,15 +1,18 @@ import type { BaseEmbedding } from "@llamaindex/core/embeddings"; import { AsyncLocalStorage } from "@llamaindex/env"; -import { OpenAIEmbedding } from "@llamaindex/openai"; const embeddedModelAsyncLocalStorage = new AsyncLocalStorage(); let globalEmbeddedModel: BaseEmbedding | null = null; export function getEmbeddedModel(): BaseEmbedding { - if (globalEmbeddedModel === null) { - globalEmbeddedModel = new OpenAIEmbedding(); + const currentEmbeddedModel = + embeddedModelAsyncLocalStorage.getStore() ?? globalEmbeddedModel; + if (!currentEmbeddedModel) { + throw new Error( + "Cannot find Embedding, please set `Settings.embedModel = ...` on the top of your code", + ); } - return embeddedModelAsyncLocalStorage.getStore() ?? globalEmbeddedModel; + return currentEmbeddedModel; } export function setEmbeddedModel(embeddedModel: BaseEmbedding) { diff --git a/packages/core/src/storage/doc-store/base-document-store.ts b/packages/core/src/storage/doc-store/base-document-store.ts new file mode 100644 index 0000000000..ffb6725ded --- /dev/null +++ b/packages/core/src/storage/doc-store/base-document-store.ts @@ -0,0 +1,167 @@ +import { path } from "@llamaindex/env"; +import { + DEFAULT_DOC_STORE_PERSIST_FILENAME, + DEFAULT_PERSIST_DIR, +} from "../../global"; +import type { StoredValue } from "../../schema"; +import { BaseNode, Document, ObjectType, TextNode } from "../../schema"; + +const TYPE_KEY = "__type__"; +const DATA_KEY = "__data__"; + +export interface Serializer { + toPersistence(data: Record): T; + + fromPersistence(data: T): Record; +} + +export const jsonSerializer: Serializer = { + toPersistence(data) { + return JSON.stringify(data); + }, + fromPersistence(data) { + return JSON.parse(data); + }, +}; + +export const noneSerializer: Serializer> = { + toPersistence(data) { + return data; + }, + fromPersistence(data) { + return data; + }, +}; + +type DocJson = { + [TYPE_KEY]: ObjectType; + [DATA_KEY]: Data; +}; + +export function isValidDocJson( + docJson: StoredValue | null | undefined, +): docJson is DocJson { + return ( + typeof docJson === "object" && + docJson !== null && + docJson[TYPE_KEY] !== undefined && + docJson[DATA_KEY] !== undefined + ); +} + +export function docToJson( + doc: BaseNode, + serializer: Serializer, +): DocJson { + return { + [DATA_KEY]: serializer.toPersistence(doc.toJSON()), + [TYPE_KEY]: doc.type, + }; +} + +export function jsonToDoc( + docDict: DocJson, + serializer: Serializer, +): BaseNode { + const docType = docDict[TYPE_KEY]; + // fixme: zod type check this + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const dataDict: any = serializer.fromPersistence(docDict[DATA_KEY]); + let doc: BaseNode; + + if (docType === ObjectType.DOCUMENT) { + doc = new Document({ + text: dataDict.text, + id_: dataDict.id_, + embedding: dataDict.embedding, + hash: dataDict.hash, + metadata: dataDict.metadata, + }); + } else if (docType === ObjectType.TEXT) { + doc = new TextNode({ + text: dataDict.text, + id_: dataDict.id_, + hash: dataDict.hash, + metadata: dataDict.metadata, + relationships: dataDict.relationships, + }); + } else { + throw new Error(`Unknown doc type: ${docType}`); + } + + return doc; +} + +const DEFAULT_PERSIST_PATH = path.join( + DEFAULT_PERSIST_DIR, + DEFAULT_DOC_STORE_PERSIST_FILENAME, +); + +export interface RefDocInfo { + nodeIds: string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + extraInfo: Record; +} + +export abstract class BaseDocumentStore { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + serializer: Serializer = jsonSerializer; + + // Save/load + persist(persistPath: string = DEFAULT_PERSIST_PATH): void { + // Persist the docstore to a file. + } + + // Main interface + abstract docs(): Promise>; + + abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise; + + abstract getDocument( + docId: string, + raiseError: boolean, + ): Promise; + + abstract deleteDocument(docId: string, raiseError: boolean): Promise; + + abstract documentExists(docId: string): Promise; + + // Hash + abstract setDocumentHash(docId: string, docHash: string): Promise; + + abstract getDocumentHash(docId: string): Promise; + + abstract getAllDocumentHashes(): Promise>; + + // Ref Docs + abstract getAllRefDocInfo(): Promise | undefined>; + + abstract getRefDocInfo(refDocId: string): Promise; + + abstract deleteRefDoc(refDocId: string, raiseError: boolean): Promise; + + // Nodes + getNodes(nodeIds: string[], raiseError: boolean = true): Promise { + return Promise.all( + nodeIds.map((nodeId) => this.getNode(nodeId, raiseError)), + ); + } + + async getNode(nodeId: string, raiseError: boolean = true): Promise { + const doc = await this.getDocument(nodeId, raiseError); + if (!(doc instanceof BaseNode)) { + throw new Error(`Document ${nodeId} is not a Node.`); + } + return doc; + } + + async getNodeDict(nodeIdDict: { + [index: number]: string; + }): Promise> { + const result: Record = {}; + for (const index in nodeIdDict) { + result[index] = await this.getNode(nodeIdDict[index]!); + } + return result; + } +} diff --git a/packages/core/src/storage/doc-store/index.ts b/packages/core/src/storage/doc-store/index.ts index ffb6725ded..bec1bbc408 100644 --- a/packages/core/src/storage/doc-store/index.ts +++ b/packages/core/src/storage/doc-store/index.ts @@ -1,167 +1,2 @@ -import { path } from "@llamaindex/env"; -import { - DEFAULT_DOC_STORE_PERSIST_FILENAME, - DEFAULT_PERSIST_DIR, -} from "../../global"; -import type { StoredValue } from "../../schema"; -import { BaseNode, Document, ObjectType, TextNode } from "../../schema"; - -const TYPE_KEY = "__type__"; -const DATA_KEY = "__data__"; - -export interface Serializer { - toPersistence(data: Record): T; - - fromPersistence(data: T): Record; -} - -export const jsonSerializer: Serializer = { - toPersistence(data) { - return JSON.stringify(data); - }, - fromPersistence(data) { - return JSON.parse(data); - }, -}; - -export const noneSerializer: Serializer> = { - toPersistence(data) { - return data; - }, - fromPersistence(data) { - return data; - }, -}; - -type DocJson = { - [TYPE_KEY]: ObjectType; - [DATA_KEY]: Data; -}; - -export function isValidDocJson( - docJson: StoredValue | null | undefined, -): docJson is DocJson { - return ( - typeof docJson === "object" && - docJson !== null && - docJson[TYPE_KEY] !== undefined && - docJson[DATA_KEY] !== undefined - ); -} - -export function docToJson( - doc: BaseNode, - serializer: Serializer, -): DocJson { - return { - [DATA_KEY]: serializer.toPersistence(doc.toJSON()), - [TYPE_KEY]: doc.type, - }; -} - -export function jsonToDoc( - docDict: DocJson, - serializer: Serializer, -): BaseNode { - const docType = docDict[TYPE_KEY]; - // fixme: zod type check this - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const dataDict: any = serializer.fromPersistence(docDict[DATA_KEY]); - let doc: BaseNode; - - if (docType === ObjectType.DOCUMENT) { - doc = new Document({ - text: dataDict.text, - id_: dataDict.id_, - embedding: dataDict.embedding, - hash: dataDict.hash, - metadata: dataDict.metadata, - }); - } else if (docType === ObjectType.TEXT) { - doc = new TextNode({ - text: dataDict.text, - id_: dataDict.id_, - hash: dataDict.hash, - metadata: dataDict.metadata, - relationships: dataDict.relationships, - }); - } else { - throw new Error(`Unknown doc type: ${docType}`); - } - - return doc; -} - -const DEFAULT_PERSIST_PATH = path.join( - DEFAULT_PERSIST_DIR, - DEFAULT_DOC_STORE_PERSIST_FILENAME, -); - -export interface RefDocInfo { - nodeIds: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - extraInfo: Record; -} - -export abstract class BaseDocumentStore { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - serializer: Serializer = jsonSerializer; - - // Save/load - persist(persistPath: string = DEFAULT_PERSIST_PATH): void { - // Persist the docstore to a file. - } - - // Main interface - abstract docs(): Promise>; - - abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise; - - abstract getDocument( - docId: string, - raiseError: boolean, - ): Promise; - - abstract deleteDocument(docId: string, raiseError: boolean): Promise; - - abstract documentExists(docId: string): Promise; - - // Hash - abstract setDocumentHash(docId: string, docHash: string): Promise; - - abstract getDocumentHash(docId: string): Promise; - - abstract getAllDocumentHashes(): Promise>; - - // Ref Docs - abstract getAllRefDocInfo(): Promise | undefined>; - - abstract getRefDocInfo(refDocId: string): Promise; - - abstract deleteRefDoc(refDocId: string, raiseError: boolean): Promise; - - // Nodes - getNodes(nodeIds: string[], raiseError: boolean = true): Promise { - return Promise.all( - nodeIds.map((nodeId) => this.getNode(nodeId, raiseError)), - ); - } - - async getNode(nodeId: string, raiseError: boolean = true): Promise { - const doc = await this.getDocument(nodeId, raiseError); - if (!(doc instanceof BaseNode)) { - throw new Error(`Document ${nodeId} is not a Node.`); - } - return doc; - } - - async getNodeDict(nodeIdDict: { - [index: number]: string; - }): Promise> { - const result: Record = {}; - for (const index in nodeIdDict) { - result[index] = await this.getNode(nodeIdDict[index]!); - } - return result; - } -} +export * from "./base-document-store"; +export * from "./kv-document-store"; diff --git a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts b/packages/core/src/storage/doc-store/kv-document-store.ts similarity index 84% rename from packages/llamaindex/src/storage/docStore/KVDocumentStore.ts rename to packages/core/src/storage/doc-store/kv-document-store.ts index 6657a38a22..bdc6ed714e 100644 --- a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts +++ b/packages/core/src/storage/doc-store/kv-document-store.ts @@ -1,15 +1,15 @@ -import { DEFAULT_NAMESPACE } from "@llamaindex/core/global"; -import type { BaseNode } from "@llamaindex/core/schema"; -import { ObjectType } from "@llamaindex/core/schema"; -import type { RefDocInfo } from "@llamaindex/core/storage/doc-store"; +import clone from "lodash.clone"; +import get from "lodash.get"; +import { DEFAULT_NAMESPACE } from "../../global"; +import { BaseNode, ObjectType, type StoredValue } from "../../schema"; +import type { BaseKVStore } from "../kv-store"; import { BaseDocumentStore, docToJson, isValidDocJson, jsonToDoc, -} from "@llamaindex/core/storage/doc-store"; -import type { BaseKVStore } from "@llamaindex/core/storage/kv-store"; -import _ from "lodash"; + type RefDocInfo, +} from "./base-document-store"; type DocMetaData = { docHash: string; refDocId?: string }; @@ -68,7 +68,7 @@ export class KVDocumentStore extends BaseDocumentStore { extraInfo: {}, }; refDocInfo.nodeIds.push(doc.id_); - if (_.isEmpty(refDocInfo.extraInfo)) { + if (Object.keys(refDocInfo.extraInfo).length === 0) { refDocInfo.extraInfo = {}; } await this.kvstore.put( @@ -88,7 +88,7 @@ export class KVDocumentStore extends BaseDocumentStore { raiseError: boolean = true, ): Promise { const json = await this.kvstore.get(docId, this.nodeCollection); - if (_.isNil(json)) { + if (this.isNil(json)) { if (raiseError) { throw new Error(`docId ${docId} not found.`); } else { @@ -103,23 +103,23 @@ export class KVDocumentStore extends BaseDocumentStore { async getRefDocInfo(refDocId: string): Promise { const refDocInfo = await this.kvstore.get(refDocId, this.refDocCollection); - return refDocInfo ? (_.clone(refDocInfo) as RefDocInfo) : undefined; + return refDocInfo ? (clone(refDocInfo) as RefDocInfo) : undefined; } async getAllRefDocInfo(): Promise | undefined> { const refDocInfos = await this.kvstore.getAll(this.refDocCollection); - if (_.isNil(refDocInfos)) { + if (this.isNil(refDocInfos)) { return; } return refDocInfos as Record; } async refDocExists(refDocId: string): Promise { - return !_.isNil(await this.getRefDocInfo(refDocId)); + return !this.isNil(await this.getRefDocInfo(refDocId)); } async documentExists(docId: string): Promise { - return !_.isNil(await this.kvstore.get(docId, this.nodeCollection)); + return !this.isNil(await this.kvstore.get(docId, this.nodeCollection)); } private async removeRefDocNode(docId: string): Promise { @@ -129,13 +129,13 @@ export class KVDocumentStore extends BaseDocumentStore { } const refDocId = metadata.refDocId; - if (_.isNil(refDocId)) { + if (this.isNil(refDocId)) { return; } const refDocInfo = await this.kvstore.get(refDocId, this.refDocCollection); - if (!_.isNil(refDocInfo)) { - if (refDocInfo.nodeIds.length > 0) { + if (!this.isNil(refDocInfo)) { + if (refDocInfo!.nodeIds.length > 0) { await this.kvstore.put(refDocId, refDocInfo, this.refDocCollection); } await this.kvstore.delete(refDocId, this.metadataCollection); @@ -164,7 +164,7 @@ export class KVDocumentStore extends BaseDocumentStore { raiseError: boolean = true, ): Promise { const refDocInfo = await this.getRefDocInfo(refDocId); - if (_.isNil(refDocInfo)) { + if (this.isNil(refDocInfo)) { if (raiseError) { throw new Error(`ref_doc_id ${refDocId} not found.`); } else { @@ -172,7 +172,7 @@ export class KVDocumentStore extends BaseDocumentStore { } } - for (const docId of refDocInfo.nodeIds) { + for (const docId of refDocInfo!.nodeIds) { await this.deleteDocument(docId, false, false); } @@ -187,7 +187,7 @@ export class KVDocumentStore extends BaseDocumentStore { async getDocumentHash(docId: string): Promise { const metadata = await this.kvstore.get(docId, this.metadataCollection); - return _.get(metadata, "docHash"); + return get(metadata, "docHash"); } async getAllDocumentHashes(): Promise> { @@ -201,4 +201,8 @@ export class KVDocumentStore extends BaseDocumentStore { } return hashes; } + + private isNil(value: RefDocInfo | StoredValue | undefined): boolean { + return value === null || value === undefined; + } } diff --git a/packages/core/src/vector-store/index.ts b/packages/core/src/vector-store/index.ts index 41542521db..151e89fa73 100644 --- a/packages/core/src/vector-store/index.ts +++ b/packages/core/src/vector-store/index.ts @@ -1,3 +1,7 @@ +import type { BaseEmbedding } from "../embeddings/base.js"; +import { Settings } from "../global"; +import type { BaseNode, ModalityType } from "../schema/node.js"; + /** * should compatible with npm:pg and npm:postgres */ @@ -12,3 +16,106 @@ export interface IsomorphicDB { close: () => Promise; onCloseEvent: (listener: () => void) => void; } + +export interface VectorStoreQueryResult { + nodes?: BaseNode[]; + similarities: number[]; + ids: string[]; +} + +export enum VectorStoreQueryMode { + DEFAULT = "default", + SPARSE = "sparse", + HYBRID = "hybrid", + // fit learners + SVM = "svm", + LOGISTIC_REGRESSION = "logistic_regression", + LINEAR_REGRESSION = "linear_regression", + // maximum marginal relevance + MMR = "mmr", + + // for Azure AI Search + SEMANTIC_HYBRID = "semantic_hybrid", +} + +export enum FilterOperator { + EQ = "==", // default operator (string, number) + IN = "in", // In array (string or number) + GT = ">", // greater than (number) + LT = "<", // less than (number) + NE = "!=", // not equal to (string, number) + GTE = ">=", // greater than or equal to (number) + LTE = "<=", // less than or equal to (number) + NIN = "nin", // Not in array (string or number) + ANY = "any", // Contains any (array of strings) + ALL = "all", // Contains all (array of strings) + TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field) + CONTAINS = "contains", // metadata array contains value (string or number) + IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array) +} + +export enum FilterCondition { + AND = "and", + OR = "or", +} + +export type MetadataFilterValue = string | number | string[] | number[]; + +export interface MetadataFilter { + key: string; + value?: MetadataFilterValue; + operator: `${FilterOperator}`; // ==, any, all,... +} + +export interface MetadataFilters { + filters: Array; + condition?: `${FilterCondition}`; // and, or +} + +export interface MetadataInfo { + name: string; + type: string; + description: string; +} + +export interface VectorStoreInfo { + metadataInfo: MetadataInfo[]; + contentInfo: string; +} + +export interface VectorStoreQuery { + queryEmbedding?: number[]; + similarityTopK: number; + docIds?: string[]; + queryStr?: string; + mode: VectorStoreQueryMode; + alpha?: number; + filters?: MetadataFilters | undefined; + mmrThreshold?: number; +} + +// Supported types of vector stores (for each modality) +export type VectorStoreByType = { + [P in ModalityType]?: BaseVectorStore; +}; + +export type VectorStoreBaseParams = { + embeddingModel?: BaseEmbedding | undefined; +}; + +export abstract class BaseVectorStore { + embedModel: BaseEmbedding; + abstract storesText: boolean; + isEmbeddingQuery?: boolean; + abstract client(): Client; + abstract add(embeddingResults: BaseNode[]): Promise; + abstract delete(refDocId: string, deleteOptions?: object): Promise; + abstract query( + query: VectorStoreQuery, + options?: object, + ): Promise; + + protected constructor(params?: VectorStoreBaseParams) { + this.embedModel = params?.embeddingModel ?? Settings.embedModel; + } +} diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json index 56995fbab1..01d250cac0 100644 --- a/packages/llamaindex/package.json +++ b/packages/llamaindex/package.json @@ -46,13 +46,13 @@ "@llamaindex/readers": "workspace:*", "@llamaindex/replicate": "workspace:*", "@llamaindex/vllm": "workspace:*", + "@llamaindex/postgres": "workspace:*", "@mistralai/mistralai": "^1.3.4", "@mixedbread-ai/sdk": "^2.2.11", "@pinecone-database/pinecone": "^4.0.0", "@qdrant/js-client-rest": "^1.11.0", "@types/lodash": "^4.17.7", "@types/node": "^22.9.0", - "@types/pg": "^8.11.8", "@upstash/vector": "^1.1.5", "@zilliz/milvus2-sdk-node": "^2.4.6", "ajv": "^8.17.1", @@ -74,27 +74,11 @@ "wink-nlp": "^2.3.0", "zod": "^3.23.8" }, - "peerDependencies": { - "pg": "^8.12.0", - "pgvector": "0.2.0" - }, - "peerDependenciesMeta": { - "pg": { - "optional": true - }, - "pgvector": { - "optional": true - } - }, "devDependencies": { "@swc/cli": "^0.5.0", "@swc/core": "^1.9.2", - "@vercel/postgres": "^0.10.0", "concurrently": "^9.1.0", "glob": "^11.0.0", - "pg": "^8.12.0", - "pgvector": "0.2.0", - "postgres": "^3.4.4", "typescript": "^5.7.2" }, "engines": { diff --git a/packages/llamaindex/src/Settings.ts b/packages/llamaindex/src/Settings.ts index 21b31e3438..89676036be 100644 --- a/packages/llamaindex/src/Settings.ts +++ b/packages/llamaindex/src/Settings.ts @@ -13,11 +13,6 @@ import { } from "@llamaindex/core/node-parser"; import { AsyncLocalStorage } from "@llamaindex/env"; import type { ServiceContext } from "./ServiceContext.js"; -import { - getEmbeddedModel, - setEmbeddedModel, - withEmbeddedModel, -} from "./internal/settings/EmbedModel.js"; export type PromptConfig = { llm?: string; @@ -84,15 +79,15 @@ class GlobalSettings implements Config { } get embedModel(): BaseEmbedding { - return getEmbeddedModel(); + return CoreSettings.embedModel; } set embedModel(embedModel: BaseEmbedding) { - setEmbeddedModel(embedModel); + CoreSettings.embedModel = embedModel; } withEmbedModel(embedModel: BaseEmbedding, fn: () => Result): Result { - return withEmbeddedModel(embedModel, fn); + return CoreSettings.withEmbedModel(embedModel, fn); } get nodeParser(): NodeParser { diff --git a/packages/llamaindex/src/index.edge.ts b/packages/llamaindex/src/index.edge.ts index 15f3242aef..302c1fb913 100644 --- a/packages/llamaindex/src/index.edge.ts +++ b/packages/llamaindex/src/index.edge.ts @@ -1,12 +1,15 @@ //#region initial setup for OpenAI -import { OpenAI } from "@llamaindex/openai"; +import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai"; import { Settings } from "./Settings.js"; try { // eslint-disable-next-line @typescript-eslint/no-unused-expressions Settings.llm; + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + Settings.embedModel; } catch { Settings.llm = new OpenAI(); + Settings.embedModel = new OpenAIEmbedding(); } //#endregion diff --git a/packages/llamaindex/src/indices/vectorStore/index.ts b/packages/llamaindex/src/indices/vectorStore/index.ts index 52bafb239c..bdbafd62c4 100644 --- a/packages/llamaindex/src/indices/vectorStore/index.ts +++ b/packages/llamaindex/src/indices/vectorStore/index.ts @@ -19,6 +19,7 @@ import { } from "@llamaindex/core/schema"; import type { BaseIndexStore } from "@llamaindex/core/storage/index-store"; import { extractText } from "@llamaindex/core/utils"; +import { VectorStoreQueryMode } from "@llamaindex/core/vector-store"; import type { ServiceContext } from "../../ServiceContext.js"; import { nodeParserFromSettingsOrContext } from "../../Settings.js"; import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js"; @@ -38,7 +39,6 @@ import type { VectorStoreByType, VectorStoreQueryResult, } from "../../vector-store/index.js"; -import { VectorStoreQueryMode } from "../../vector-store/types.js"; import type { BaseIndexInit } from "../BaseIndex.js"; import { BaseIndex } from "../BaseIndex.js"; diff --git a/packages/llamaindex/src/ingestion/IngestionPipeline.ts b/packages/llamaindex/src/ingestion/IngestionPipeline.ts index bf9cb3a0b6..96142fc29b 100644 --- a/packages/llamaindex/src/ingestion/IngestionPipeline.ts +++ b/packages/llamaindex/src/ingestion/IngestionPipeline.ts @@ -10,7 +10,7 @@ import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store"; import type { BaseVectorStore, VectorStoreByType, -} from "../vector-store/types.js"; +} from "@llamaindex/core/vector-store"; import { IngestionCache, getTransformationHash } from "./IngestionCache.js"; import { DocStoreStrategy, diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts index f36dc7cd71..f328c71be6 100644 --- a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts +++ b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts @@ -1,6 +1,6 @@ import { BaseNode } from "@llamaindex/core/schema"; import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store"; -import type { BaseVectorStore } from "../../vector-store/types.js"; +import type { BaseVectorStore } from "@llamaindex/core/vector-store"; import { classify } from "./classify.js"; import { RollbackableTransformComponent } from "./rollback.js"; diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts index 370994cd87..7a53e0dcfa 100644 --- a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts +++ b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts @@ -1,6 +1,6 @@ import { BaseNode } from "@llamaindex/core/schema"; import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store"; -import type { BaseVectorStore } from "../../vector-store/types.js"; +import type { BaseVectorStore } from "@llamaindex/core/vector-store"; import { classify } from "./classify.js"; import { RollbackableTransformComponent } from "./rollback.js"; diff --git a/packages/llamaindex/src/ingestion/strategies/index.ts b/packages/llamaindex/src/ingestion/strategies/index.ts index a6ba3573f6..01e1198cf0 100644 --- a/packages/llamaindex/src/ingestion/strategies/index.ts +++ b/packages/llamaindex/src/ingestion/strategies/index.ts @@ -1,5 +1,5 @@ import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store"; -import type { BaseVectorStore } from "../../vector-store/types.js"; +import type { BaseVectorStore } from "@llamaindex/core/vector-store"; import { DuplicatesStrategy } from "./DuplicatesStrategy.js"; import { UpsertsAndDeleteStrategy } from "./UpsertsAndDeleteStrategy.js"; import { UpsertsStrategy } from "./UpsertsStrategy.js"; diff --git a/packages/llamaindex/src/storage/StorageContext.ts b/packages/llamaindex/src/storage/StorageContext.ts index b6a1cbe68d..3eae21ace4 100644 --- a/packages/llamaindex/src/storage/StorageContext.ts +++ b/packages/llamaindex/src/storage/StorageContext.ts @@ -9,13 +9,13 @@ import { BaseIndexStore, SimpleIndexStore, } from "@llamaindex/core/storage/index-store"; -import { path } from "@llamaindex/env"; -import type { ServiceContext } from "../ServiceContext.js"; -import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js"; import type { BaseVectorStore, VectorStoreByType, -} from "../vector-store/types.js"; +} from "@llamaindex/core/vector-store"; +import { path } from "@llamaindex/env"; +import type { ServiceContext } from "../ServiceContext.js"; +import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js"; import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js"; export interface StorageContext { diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts b/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts index f9fbe83ff5..e2b165fd7b 100644 --- a/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts @@ -1,6 +1,6 @@ +import { KVDocumentStore } from "@llamaindex/core/storage/doc-store"; import { MongoClient } from "mongodb"; import { AzureCosmosVCoreKVStore } from "../kvStore/AzureCosmosMongovCoreKVStore.js"; -import { KVDocumentStore } from "./KVDocumentStore.js"; const DEFAULT_DATABASE = "DocumentStoreDB"; const DEFAULT_COLLECTION = "DocumentStoreCollection"; diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts b/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts index abf6bb1e21..caab0e1142 100644 --- a/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts @@ -1,10 +1,10 @@ +import { KVDocumentStore } from "@llamaindex/core/storage/doc-store"; import { AzureCosmosNoSqlKVStore, type AadTokenOptions, type AccountAndKeyOptions, type ConnectionStringOptions, } from "../kvStore/AzureCosmosNoSqlKVStore.js"; -import { KVDocumentStore } from "./KVDocumentStore.js"; const DEFAULT_DATABASE = "DocumentStoreDB"; const DEFAULT_CONTAINER = "DocumentStoreContainer"; diff --git a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts index 7a057737c9..95482a4b38 100644 --- a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts @@ -3,13 +3,13 @@ import { DEFAULT_NAMESPACE, DEFAULT_PERSIST_DIR, } from "@llamaindex/core/global"; +import { KVDocumentStore } from "@llamaindex/core/storage/doc-store"; import { BaseInMemoryKVStore, SimpleKVStore, } from "@llamaindex/core/storage/kv-store"; import { path } from "@llamaindex/env"; import _ from "lodash"; -import { KVDocumentStore } from "./KVDocumentStore.js"; // eslint-disable-next-line @typescript-eslint/no-explicit-any type SaveDict = Record; diff --git a/packages/llamaindex/src/storage/index.ts b/packages/llamaindex/src/storage/index.ts index 7ffd634539..5ed611e6e1 100644 --- a/packages/llamaindex/src/storage/index.ts +++ b/packages/llamaindex/src/storage/index.ts @@ -2,17 +2,19 @@ export * from "@llamaindex/core/storage/chat-store"; export * from "@llamaindex/core/storage/doc-store"; export * from "@llamaindex/core/storage/index-store"; export * from "@llamaindex/core/storage/kv-store"; +export { + PostgresDocumentStore, + PostgresIndexStore, + PostgresKVStore, +} from "@llamaindex/postgres"; export * from "./chatStore/AzureCosmosMongovCoreChatStore.js"; export * from "./chatStore/AzureCosmosNoSqlChatStore.js"; export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js"; export * from "./docStore/AzureCosmosNoSqlDocumentStore.js"; -export { PostgresDocumentStore } from "./docStore/PostgresDocumentStore.js"; export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js"; export * from "./FileSystem.js"; export * from "./indexStore/AzureCosmosMongovCoreIndexStore.js"; export * from "./indexStore/AzureCosmosNoSqlIndexStore.js"; -export { PostgresIndexStore } from "./indexStore/PostgresIndexStore.js"; export * from "./kvStore/AzureCosmosMongovCoreKVStore.js"; export * from "./kvStore/AzureCosmosNoSqlKVStore.js"; -export { PostgresKVStore } from "./kvStore/PostgresKVStore.js"; export * from "./StorageContext.js"; diff --git a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts index 4af05bccf2..3f366305f3 100644 --- a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts +++ b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts @@ -8,7 +8,6 @@ import { } from "@datastax/astra-db-ts"; import type { BaseNode } from "@llamaindex/core/schema"; import { MetadataMode } from "@llamaindex/core/schema"; -import { getEnv } from "@llamaindex/env"; import { BaseVectorStore, FilterCondition, @@ -18,7 +17,8 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { getEnv } from "@llamaindex/env"; import { metadataDictToNode, nodeToMetadata, diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts b/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts index afddb59f9d..4870273267 100644 --- a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts +++ b/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts @@ -1,13 +1,13 @@ import type { BaseNode } from "@llamaindex/core/schema"; import { MetadataMode } from "@llamaindex/core/schema"; -import { getEnv } from "@llamaindex/env"; -import { Collection, Db, MongoClient } from "mongodb"; import { BaseVectorStore, type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { getEnv } from "@llamaindex/env"; +import { Collection, Db, MongoClient } from "mongodb"; import { metadataDictToNode, nodeToMetadata } from "./utils.js"; /** Azure Cosmos DB for MongoDB vCore Similarity type. */ diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts b/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts index 29d01de504..6118fef753 100644 --- a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts +++ b/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts @@ -20,7 +20,7 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; /** Azure Cosmos DB for NoSQL database creation options. */ export type AzureCosmosDBNoSqlCreateDatabaseOptions = Partial< diff --git a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts index fbaf2a263a..e8b9426f11 100644 --- a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts +++ b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts @@ -1,14 +1,5 @@ import type { BaseNode } from "@llamaindex/core/schema"; import { MetadataMode } from "@llamaindex/core/schema"; -import { - ChromaClient, - type ChromaClientParams, - type DeleteParams, - type QueryRecordsParams, - type QueryResponse, - type Where, - type WhereDocument, -} from "chromadb"; import { BaseVectorStore, FilterCondition, @@ -18,7 +9,16 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { + ChromaClient, + type ChromaClientParams, + type DeleteParams, + type QueryRecordsParams, + type QueryResponse, + type Where, + type WhereDocument, +} from "chromadb"; import { metadataDictToNode, nodeToMetadata } from "./utils.js"; type ChromaDeleteOptions = { diff --git a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts index 1dc475871e..97b7029f21 100644 --- a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts +++ b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts @@ -1,5 +1,12 @@ import type { ChannelOptions } from "@grpc/grpc-js"; import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema"; +import { + BaseVectorStore, + type MetadataFilters, + type VectorStoreBaseParams, + type VectorStoreQuery, + type VectorStoreQueryResult, +} from "@llamaindex/core/vector-store"; import { getEnv } from "@llamaindex/env"; import { DataType, @@ -9,13 +16,6 @@ import { type RowData, type SearchSimpleReq, } from "@zilliz/milvus2-sdk-node"; -import { - BaseVectorStore, - type MetadataFilters, - type VectorStoreBaseParams, - type VectorStoreQuery, - type VectorStoreQueryResult, -} from "./types.js"; import { metadataDictToNode, nodeToMetadata, diff --git a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts index da9cab0086..3aaef9dda8 100644 --- a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts +++ b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts @@ -1,9 +1,6 @@ import type { BaseEmbedding } from "@llamaindex/core/embeddings"; import type { BaseNode } from "@llamaindex/core/schema"; import { MetadataMode } from "@llamaindex/core/schema"; -import { getEnv } from "@llamaindex/env"; -import type { BulkWriteOptions, Collection } from "mongodb"; -import { MongoClient } from "mongodb"; import { BaseVectorStore, FilterCondition, @@ -13,7 +10,10 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { getEnv } from "@llamaindex/env"; +import type { BulkWriteOptions, Collection } from "mongodb"; +import { MongoClient } from "mongodb"; import { metadataDictToNode, nodeToMetadata } from "./utils.js"; // define your Atlas Search index. See detail https://www.mongodb.com/docs/atlas/atlas-search/field-types/knn-vector/ diff --git a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts index e2dca1494e..e5cce0bff4 100644 --- a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts +++ b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts @@ -7,7 +7,7 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; import type { BaseNode, Metadata } from "@llamaindex/core/schema"; import { getEnv } from "@llamaindex/env"; diff --git a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts index f531adb2bd..7a7c083f5a 100644 --- a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts +++ b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts @@ -7,7 +7,7 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; import type { QdrantClientParams, Schemas } from "@qdrant/js-client-rest"; import { QdrantClient } from "@qdrant/js-client-rest"; diff --git a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts index b8564e81c0..a0ab5b3f53 100644 --- a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts +++ b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts @@ -5,8 +5,6 @@ import { } from "@llamaindex/core/embeddings"; import { DEFAULT_PERSIST_DIR } from "@llamaindex/core/global"; import type { BaseNode } from "@llamaindex/core/schema"; -import { fs, path } from "@llamaindex/env"; -import { exists } from "../storage/FileSystem.js"; import { BaseVectorStore, FilterOperator, @@ -16,7 +14,9 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { fs, path } from "@llamaindex/env"; +import { exists } from "../storage/FileSystem.js"; import { nodeToMetadata, parseArrayValue, diff --git a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts index f7575dc17b..5101617beb 100644 --- a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts +++ b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts @@ -5,7 +5,7 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; import type { BaseNode, Metadata, TextNode } from "@llamaindex/core/schema"; import { getEnv } from "@llamaindex/env"; diff --git a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts index 66132be971..c85dccaf07 100644 --- a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts +++ b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts @@ -8,8 +8,6 @@ import weaviate, { type WeaviateNonGenericObject, } from "weaviate-client"; -import { getEnv } from "@llamaindex/env"; -import type { BaseHybridOptions } from "weaviate-client"; import { BaseVectorStore, VectorStoreQueryMode, @@ -18,7 +16,9 @@ import { type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; +} from "@llamaindex/core/vector-store"; +import { getEnv } from "@llamaindex/env"; +import type { BaseHybridOptions } from "weaviate-client"; import { metadataDictToNode, nodeToMetadata, diff --git a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts b/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts index 51cc814db8..5dfb226861 100644 --- a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts +++ b/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts @@ -27,7 +27,6 @@ import { ManagedIdentityCredential, } from "@azure/identity"; import { type BaseNode, MetadataMode } from "@llamaindex/core/schema"; -import { consoleLogger, getEnv } from "@llamaindex/env"; import { BaseVectorStore, FilterCondition, @@ -37,7 +36,8 @@ import { type VectorStoreQuery, VectorStoreQueryMode, type VectorStoreQueryResult, -} from "../types.js"; +} from "@llamaindex/core/vector-store"; +import { consoleLogger, getEnv } from "@llamaindex/env"; import { metadataDictToNode, nodeToMetadata } from "../utils.js"; import { AzureAISearchVectorStoreConfig, diff --git a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts b/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts index 1f39cfe014..721e23f6d6 100644 --- a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts +++ b/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts @@ -4,11 +4,11 @@ import { type VectorizedQuery, } from "@azure/search-documents"; import { - TextNode, type VectorStoreQuery, type VectorStoreQueryResult, -} from "llamaindex"; +} from "@llamaindex/core/vector-store"; +import type { TextNode } from "@llamaindex/core/schema"; import { consoleLogger } from "@llamaindex/env"; import { metadataDictToNode } from "../utils.js"; import { diff --git a/packages/llamaindex/src/vector-store/index.ts b/packages/llamaindex/src/vector-store/index.ts index 86554d2301..030319e3e1 100644 --- a/packages/llamaindex/src/vector-store/index.ts +++ b/packages/llamaindex/src/vector-store/index.ts @@ -1,3 +1,11 @@ +export * from "@llamaindex/core/vector-store"; +export { + DEFAULT_DIMENSIONS, + PGVECTOR_SCHEMA, + PGVECTOR_TABLE, + PGVectorStore, + type PGVectorStoreConfig, +} from "@llamaindex/postgres"; export * from "./AstraDBVectorStore.js"; export * from "./azure/AzureAISearchVectorStore.js"; export * from "./AzureCosmosDBMongoVectorStore.js"; @@ -5,9 +13,7 @@ export * from "./AzureCosmosDBNoSqlVectorStore.js"; export * from "./ChromaVectorStore.js"; export * from "./MilvusVectorStore.js"; export * from "./MongoDBAtlasVectorStore.js"; -export * from "./PGVectorStore.js"; export * from "./PineconeVectorStore.js"; export * from "./QdrantVectorStore.js"; export * from "./SimpleVectorStore.js"; -export * from "./types.js"; export * from "./WeaviateVectorStore.js"; diff --git a/packages/llamaindex/src/vector-store/types.ts b/packages/llamaindex/src/vector-store/types.ts deleted file mode 100644 index a5563b2846..0000000000 --- a/packages/llamaindex/src/vector-store/types.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { BaseEmbedding } from "@llamaindex/core/embeddings"; -import type { BaseNode, ModalityType } from "@llamaindex/core/schema"; -import { getEmbeddedModel } from "../internal/settings/EmbedModel.js"; - -export interface VectorStoreQueryResult { - nodes?: BaseNode[]; - similarities: number[]; - ids: string[]; -} - -export enum VectorStoreQueryMode { - DEFAULT = "default", - SPARSE = "sparse", - HYBRID = "hybrid", - // fit learners - SVM = "svm", - LOGISTIC_REGRESSION = "logistic_regression", - LINEAR_REGRESSION = "linear_regression", - // maximum marginal relevance - MMR = "mmr", - - // for Azure AI Search - SEMANTIC_HYBRID = "semantic_hybrid", -} - -export enum FilterOperator { - EQ = "==", // default operator (string, number) - IN = "in", // In array (string or number) - GT = ">", // greater than (number) - LT = "<", // less than (number) - NE = "!=", // not equal to (string, number) - GTE = ">=", // greater than or equal to (number) - LTE = "<=", // less than or equal to (number) - NIN = "nin", // Not in array (string or number) - ANY = "any", // Contains any (array of strings) - ALL = "all", // Contains all (array of strings) - TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field) - CONTAINS = "contains", // metadata array contains value (string or number) - IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array) -} - -export enum FilterCondition { - AND = "and", - OR = "or", -} - -export type MetadataFilterValue = string | number | string[] | number[]; - -export interface MetadataFilter { - key: string; - value?: MetadataFilterValue; - operator: `${FilterOperator}`; // ==, any, all,... -} - -export interface MetadataFilters { - filters: Array; - condition?: `${FilterCondition}`; // and, or -} - -export interface MetadataInfo { - name: string; - type: string; - description: string; -} - -export interface VectorStoreInfo { - metadataInfo: MetadataInfo[]; - contentInfo: string; -} - -export interface VectorStoreQuery { - queryEmbedding?: number[]; - similarityTopK: number; - docIds?: string[]; - queryStr?: string; - mode: VectorStoreQueryMode; - alpha?: number; - filters?: MetadataFilters | undefined; - mmrThreshold?: number; -} - -// Supported types of vector stores (for each modality) -export type VectorStoreByType = { - [P in ModalityType]?: BaseVectorStore; -}; - -export type VectorStoreBaseParams = { - embeddingModel?: BaseEmbedding | undefined; -}; - -export abstract class BaseVectorStore { - embedModel: BaseEmbedding; - abstract storesText: boolean; - isEmbeddingQuery?: boolean; - abstract client(): Client; - abstract add(embeddingResults: BaseNode[]): Promise; - abstract delete(refDocId: string, deleteOptions?: object): Promise; - abstract query( - query: VectorStoreQuery, - options?: object, - ): Promise; - - protected constructor(params?: VectorStoreBaseParams) { - this.embedModel = params?.embeddingModel ?? getEmbeddedModel(); - } -} diff --git a/packages/llamaindex/src/vector-store/utils.ts b/packages/llamaindex/src/vector-store/utils.ts index 79e02062f7..ac50a43e0d 100644 --- a/packages/llamaindex/src/vector-store/utils.ts +++ b/packages/llamaindex/src/vector-store/utils.ts @@ -1,6 +1,6 @@ import type { BaseNode, Metadata } from "@llamaindex/core/schema"; import { ObjectType, jsonToNode } from "@llamaindex/core/schema"; -import type { MetadataFilterValue } from "./types.js"; +import type { MetadataFilterValue } from "@llamaindex/core/vector-store"; const DEFAULT_TEXT_KEY = "text"; diff --git a/packages/llamaindex/tests/StorageContext.test.ts b/packages/llamaindex/tests/StorageContext.test.ts index 95dfc7c1e7..4b53dfa594 100644 --- a/packages/llamaindex/tests/StorageContext.test.ts +++ b/packages/llamaindex/tests/StorageContext.test.ts @@ -1,7 +1,4 @@ -import { - storageContextFromDefaults, - type StorageContext, -} from "llamaindex/storage/StorageContext"; +import { storageContextFromDefaults, type StorageContext } from "llamaindex"; import { existsSync, rmSync } from "node:fs"; import { mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; diff --git a/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts b/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts index cc028bcece..3d82e753cf 100644 --- a/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts +++ b/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts @@ -1,6 +1,6 @@ import type { BaseNode } from "@llamaindex/core/schema"; +import { AzureCosmosDBNoSqlVectorStore } from "llamaindex"; import type { Mocked } from "vitest"; -import { AzureCosmosDBNoSqlVectorStore } from "../../src/vector-store.js"; export class TestableAzureCosmosDBNoSqlVectorStore extends AzureCosmosDBNoSqlVectorStore { public nodes: BaseNode[] = []; diff --git a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts index 965144f965..23f7416fdf 100644 --- a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts +++ b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts @@ -1,5 +1,5 @@ import type { BaseNode } from "@llamaindex/core/schema"; -import { QdrantVectorStore } from "llamaindex/vector-store"; +import { QdrantVectorStore } from "llamaindex"; export class TestableQdrantVectorStore extends QdrantVectorStore { public nodes: BaseNode[] = []; diff --git a/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts b/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts index c9686d53cd..d94a333dd4 100644 --- a/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts +++ b/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { SearchClient, SearchIndexClient } from "@azure/search-documents"; +import { AzureAISearchVectorStore } from "llamaindex"; import { afterEach, beforeEach } from "node:test"; import { describe, expect, it, vi } from "vitest"; -import { AzureAISearchVectorStore } from "../../src/vector-store.js"; // We test only for the initialization of the store, and the search and index clients, will variants of the options provided const MOCK_ENDPOINT = "https://test-endpoint.com"; diff --git a/packages/providers/storage/postgres/package.json b/packages/providers/storage/postgres/package.json new file mode 100644 index 0000000000..f35b94dae4 --- /dev/null +++ b/packages/providers/storage/postgres/package.json @@ -0,0 +1,66 @@ +{ + "name": "@llamaindex/postgres", + "description": "PostgreSQL Storage for LlamaIndex", + "version": "0.0.29", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "exports": { + ".": { + "edge-light": { + "types": "./dist/index.edge-light.d.ts", + "default": "./dist/index.edge-light.js" + }, + "workerd": { + "types": "./dist/index.edge-light.d.ts", + "default": "./dist/index.edge-light.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + }, + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://github.com/run-llama/LlamaIndexTS.git", + "directory": "packages/providers/storage/postgres" + }, + "scripts": { + "build": "bunchee", + "dev": "bunchee --watch" + }, + "devDependencies": { + "bunchee": "6.2.0", + "@types/pg": "^8.11.8", + "@vercel/postgres": "^0.10.0", + "pg": "^8.12.0", + "pgvector": "0.2.0", + "postgres": "^3.4.4" + }, + "dependencies": { + "@llamaindex/core": "workspace:*", + "@llamaindex/env": "workspace:*", + "pg": "^8.11.3", + "pg-promise": "^11.5.4" + }, + "peerDependencies": { + "pg": "^8.12.0", + "pgvector": "0.2.0" + }, + "peerDependenciesMeta": { + "pg": { + "optional": true + }, + "pgvector": { + "optional": true + } + } +} diff --git a/packages/llamaindex/src/vector-store/PGVectorStore.ts b/packages/providers/storage/postgres/src/PGVectorStore.ts similarity index 98% rename from packages/llamaindex/src/vector-store/PGVectorStore.ts rename to packages/providers/storage/postgres/src/PGVectorStore.ts index 4524a3f3c3..a46275044b 100644 --- a/packages/llamaindex/src/vector-store/PGVectorStore.ts +++ b/packages/providers/storage/postgres/src/PGVectorStore.ts @@ -1,20 +1,18 @@ import type pg from "pg"; -import type { IsomorphicDB } from "@llamaindex/core/vector-store"; -import type { VercelPool } from "@vercel/postgres"; -import type { Sql } from "postgres"; import { BaseVectorStore, FilterCondition, FilterOperator, + type IsomorphicDB, type MetadataFilter, type MetadataFilterValue, type VectorStoreBaseParams, type VectorStoreQuery, type VectorStoreQueryResult, -} from "./types.js"; - -import { escapeLikeString } from "./utils.js"; +} from "@llamaindex/core/vector-store"; +import type { VercelPool } from "@vercel/postgres"; +import type { Sql } from "postgres"; import type { BaseEmbedding } from "@llamaindex/core/embeddings"; import { DEFAULT_COLLECTION } from "@llamaindex/core/global"; @@ -487,7 +485,7 @@ export class PGVectorStore extends BaseVectorStore { } if (filter.operator === FilterOperator.TEXT_MATCH) { - const escapedValue = escapeLikeString(filter.value as string); + const escapedValue = this.escapeLikeString(filter.value as string); return { clause: `metadata->>'${filter.key}' LIKE $${paramIndex}`, param: `%${escapedValue}%`, @@ -592,4 +590,8 @@ export class PGVectorStore extends BaseVectorStore { persist(persistPath: string): Promise { return Promise.resolve(); } + + private escapeLikeString(value: string) { + return value.replace(/[%_\\]/g, "\\$&"); + } } diff --git a/packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts b/packages/providers/storage/postgres/src/PostgresDocumentStore.ts similarity index 80% rename from packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts rename to packages/providers/storage/postgres/src/PostgresDocumentStore.ts index e15ad6c6c3..0ec67354a2 100644 --- a/packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts +++ b/packages/providers/storage/postgres/src/PostgresDocumentStore.ts @@ -1,10 +1,9 @@ import { DEFAULT_NAMESPACE } from "@llamaindex/core/global"; -import { noneSerializer } from "@llamaindex/core/storage/doc-store"; import { - PostgresKVStore, - type PostgresKVStoreConfig, -} from "../kvStore/PostgresKVStore.js"; -import { KVDocumentStore } from "./KVDocumentStore.js"; + KVDocumentStore, + noneSerializer, +} from "@llamaindex/core/storage/doc-store"; +import { PostgresKVStore, type PostgresKVStoreConfig } from "./PostgresKVStore"; const DEFAULT_TABLE_NAME = "llamaindex_doc_store"; diff --git a/packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts b/packages/providers/storage/postgres/src/PostgresIndexStore.ts similarity index 84% rename from packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts rename to packages/providers/storage/postgres/src/PostgresIndexStore.ts index 6e4420743b..f925ba56e8 100644 --- a/packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts +++ b/packages/providers/storage/postgres/src/PostgresIndexStore.ts @@ -1,9 +1,6 @@ import { DEFAULT_NAMESPACE } from "@llamaindex/core/global"; -import { - PostgresKVStore, - type PostgresKVStoreConfig, -} from "../kvStore/PostgresKVStore.js"; -import { KVIndexStore } from "./KVIndexStore.js"; +import { KVIndexStore } from "@llamaindex/core/storage/index-store"; +import { PostgresKVStore, type PostgresKVStoreConfig } from "./PostgresKVStore"; const DEFAULT_TABLE_NAME = "llamaindex_index_store"; diff --git a/packages/llamaindex/src/storage/kvStore/PostgresKVStore.ts b/packages/providers/storage/postgres/src/PostgresKVStore.ts similarity index 100% rename from packages/llamaindex/src/storage/kvStore/PostgresKVStore.ts rename to packages/providers/storage/postgres/src/PostgresKVStore.ts diff --git a/packages/providers/storage/postgres/src/index.ts b/packages/providers/storage/postgres/src/index.ts new file mode 100644 index 0000000000..71d1fb05be --- /dev/null +++ b/packages/providers/storage/postgres/src/index.ts @@ -0,0 +1,4 @@ +export * from "./PGVectorStore"; +export * from "./PostgresDocumentStore"; +export * from "./PostgresIndexStore"; +export * from "./PostgresKVStore"; diff --git a/packages/providers/storage/postgres/tsconfig.json b/packages/providers/storage/postgres/tsconfig.json new file mode 100644 index 0000000000..4607e18608 --- /dev/null +++ b/packages/providers/storage/postgres/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "outDir": "./lib", + "tsBuildInfoFile": "./lib/.tsbuildinfo" + }, + "include": ["./src"], + "references": [ + { + "path": "../../../core/tsconfig.json" + }, + { + "path": "../../../env/tsconfig.json" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6c4ebca286..fa41cf7561 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -827,6 +827,12 @@ importers: '@types/node': specifier: ^22.9.0 version: 22.9.0 + lodash.clone: + specifier: ^4.5.0 + version: 4.5.0 + lodash.get: + specifier: ^4.4.2 + version: 4.4.2 magic-bytes.js: specifier: ^1.10.0 version: 1.10.0 @@ -840,6 +846,12 @@ importers: '@edge-runtime/vm': specifier: ^4.0.4 version: 4.0.4 + '@types/lodash.clone': + specifier: ^4.5.9 + version: 4.5.9 + '@types/lodash.get': + specifier: ^4.4.9 + version: 4.4.9 ajv: specifier: ^8.17.1 version: 8.17.1 @@ -1001,6 +1013,9 @@ importers: '@llamaindex/portkey-ai': specifier: workspace:* version: link:../providers/portkey-ai + '@llamaindex/postgres': + specifier: workspace:* + version: link:../providers/storage/postgres '@llamaindex/readers': specifier: workspace:* version: link:../readers @@ -1028,9 +1043,6 @@ importers: '@types/node': specifier: ^22.9.0 version: 22.9.0 - '@types/pg': - specifier: ^8.11.8 - version: 8.11.8 '@upstash/vector': specifier: ^1.1.5 version: 1.1.5 @@ -1098,24 +1110,12 @@ importers: '@swc/core': specifier: ^1.9.2 version: 1.9.2(@swc/helpers@0.5.15) - '@vercel/postgres': - specifier: ^0.10.0 - version: 0.10.0 concurrently: specifier: ^9.1.0 version: 9.1.0 glob: specifier: ^11.0.0 version: 11.0.0 - pg: - specifier: ^8.12.0 - version: 8.12.0 - pgvector: - specifier: 0.2.0 - version: 0.2.0 - postgres: - specifier: ^3.4.4 - version: 3.4.4 typescript: specifier: ^5.7.2 version: 5.7.2 @@ -1328,6 +1328,37 @@ importers: specifier: 6.2.0 version: 6.2.0(typescript@5.7.2) + packages/providers/storage/postgres: + dependencies: + '@llamaindex/core': + specifier: workspace:* + version: link:../../../core + '@llamaindex/env': + specifier: workspace:* + version: link:../../../env + pg: + specifier: ^8.11.3 + version: 8.13.1 + pg-promise: + specifier: ^11.5.4 + version: 11.10.2(pg-query-stream@4.7.1(pg@8.13.1)) + devDependencies: + '@types/pg': + specifier: ^8.11.8 + version: 8.11.8 + '@vercel/postgres': + specifier: ^0.10.0 + version: 0.10.0 + bunchee: + specifier: 6.2.0 + version: 6.2.0(typescript@5.7.2) + pgvector: + specifier: 0.2.0 + version: 0.2.0 + postgres: + specifier: ^3.4.4 + version: 3.4.4 + packages/providers/vercel: dependencies: '@llamaindex/core': @@ -2054,10 +2085,6 @@ packages: resolution: {integrity: sha512-IzD+hfqGqFtXymHXm4RzrZW2MsSH2M7RLmZsKaKVi7SUxbeYTUeX+ALk8gVzkM8ykb7EzlDLWCNErKfAa57rYQ==} engines: {node: '>=18.0.0'} - '@babel/code-frame@7.22.5': - resolution: {integrity: sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==} - engines: {node: '>=6.9.0'} - '@babel/code-frame@7.26.2': resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} engines: {node: '>=6.9.0'} @@ -2100,10 +2127,6 @@ packages: resolution: {integrity: sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==} engines: {node: '>=6.9.0'} - '@babel/helper-validator-identifier@7.22.5': - resolution: {integrity: sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==} - engines: {node: '>=6.9.0'} - '@babel/helper-validator-identifier@7.25.9': resolution: {integrity: sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==} engines: {node: '>=6.9.0'} @@ -2116,10 +2139,6 @@ packages: resolution: {integrity: sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==} engines: {node: '>=6.9.0'} - '@babel/highlight@7.22.5': - resolution: {integrity: sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==} - engines: {node: '>=6.9.0'} - '@babel/parser@7.22.7': resolution: {integrity: sha512-7NF8pOkHP5o2vpmGgNGcfAeCvOYhGLyA3Z4eBQkT1RJlWu47n63bCs93QfJ2hIAFCil7L5P2IWhs1oToVgrL0Q==} engines: {node: '>=6.0.0'} @@ -3210,9 +3229,6 @@ packages: '@jridgewell/sourcemap-codec@1.4.14': resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} - '@jridgewell/sourcemap-codec@1.4.15': - resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} - '@jridgewell/sourcemap-codec@1.5.0': resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} @@ -4929,6 +4945,12 @@ packages: '@types/lodash-es@4.17.12': resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} + '@types/lodash.clone@4.5.9': + resolution: {integrity: sha512-euFSUq+8csIliszqTBSMh7eU41/by1JPQhTNa2gq3dncxC2Z6s87bbzVB9UUBdYDbV+FV5whFDGxz6hOKTY/EQ==} + + '@types/lodash.get@4.4.9': + resolution: {integrity: sha512-J5dvW98sxmGnamqf+/aLP87PYXyrha9xIgc2ZlHl6OHMFR2Ejdxep50QfU0abO1+CH6+ugx+8wEUN1toImAinA==} + '@types/lodash@4.17.13': resolution: {integrity: sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==} @@ -5583,6 +5605,10 @@ packages: engines: {node: '>=16', npm: '>=7'} hasBin: true + assert-options@0.8.2: + resolution: {integrity: sha512-XaXoMxY0zuwAb0YuZjxIm8FeWvNq0aWNIbrzHhFjme8Smxw4JlPoyrAKQ6808k5UvQdhvnWqHZCphq5mXd4TDA==} + engines: {node: '>=10.0.0'} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -5965,10 +5991,6 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} - cli-spinners@2.9.0: - resolution: {integrity: sha512-4/aL9X3Wh0yiMQlE+eeRhWP6vclO3QRtw1JHKIT0FFUs5FjpFmESqtMvYZ0+lbzBw900b95mS0hohy+qn2VK/g==} - engines: {node: '>=6'} - cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} @@ -7171,9 +7193,6 @@ packages: tailwindcss: optional: true - function-bind@1.1.1: - resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} @@ -7266,9 +7285,6 @@ packages: resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==} engines: {node: '>= 0.4'} - get-tsconfig@4.5.0: - resolution: {integrity: sha512-MjhiaIWCJ1sAU4pIQ5i5OfOuHHxVo1oYeNsWTON7jxYkod8pHocXeh+SSbmu5OZZZK73B6cbJ2XADzXehLyovQ==} - get-tsconfig@4.8.1: resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==} @@ -7720,9 +7736,6 @@ packages: resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==} engines: {node: '>= 0.4'} - is-core-module@2.12.1: - resolution: {integrity: sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==} - is-core-module@2.16.0: resolution: {integrity: sha512-urTSINYfAYgcbLb0yDQ6egFm6h3Mo1DcF9EkyXSRjjzdHbsulg01qhwWuXdOoUBuTkbQ80KDboXa0vFJ+BDH+g==} engines: {node: '>= 0.4'} @@ -8215,9 +8228,15 @@ packages: lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.clone@4.5.0: + resolution: {integrity: sha512-GhrVeweiTD6uTmmn5hV/lzgCQhccwReIVRLHp7LT4SopOjqEZ5BbX8b5WWEtAKasjmy8hR7ZPwsYlxRCku5odg==} + lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + lodash.get@4.4.2: + resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} + lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} @@ -9445,10 +9464,19 @@ packages: pg-connection-string@2.7.0: resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==} + pg-cursor@2.12.1: + resolution: {integrity: sha512-V13tEaA9Oq1w+V6Q3UBIB/blxJrwbbr35/dY54r/86soBJ7xkP236bXaORUTVXUPt9B6Ql2BQu+uwQiuMfRVgg==} + peerDependencies: + pg: ^8 + pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} + pg-minify@1.6.5: + resolution: {integrity: sha512-u0UE8veaCnMfJmoklqneeBBopOAPG3/6DHqGVHYAhz8DkJXh9dnjPlz25fRxn4e+6XVzdOp7kau63Rp52fZ3WQ==} + engines: {node: '>=14.0.0'} + pg-numeric@1.0.2: resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} engines: {node: '>=4'} @@ -9458,9 +9486,20 @@ packages: peerDependencies: pg: '>=8.0' + pg-promise@11.10.2: + resolution: {integrity: sha512-wK4yjxZdfxBmAMcs40q6IsC1SOzdLilc1yNvJqlbOjtm2syayqLDCt1JQ9lhS6yNSgVlGOQZT88yb/SADJmEBw==} + engines: {node: '>=14.0'} + peerDependencies: + pg-query-stream: 4.7.1 + pg-protocol@1.7.0: resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} + pg-query-stream@4.7.1: + resolution: {integrity: sha512-UMgsgn/pOIYsIifRySp59vwlpTpLADMK9HWJtq5ff0Z3MxBnPMGnCQeaQl5VuL+7ov4F96mSzIRIcz+Duo6OiQ==} + peerDependencies: + pg: ^8 + pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} @@ -10157,10 +10196,6 @@ packages: engines: {node: '>= 0.4'} hasBin: true - resolve@1.22.2: - resolution: {integrity: sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==} - hasBin: true - resolve@1.22.9: resolution: {integrity: sha512-QxrmX1DzraFIi9PxdG5VkRfRwIgjwyud+z/iBwfRRrVmHc+P9Q7u2lSSpQ6bjr2gy5lrqIiU9vb6iAeGf2400A==} hasBin: true @@ -10547,6 +10582,10 @@ packages: spdx-license-ids@3.0.20: resolution: {integrity: sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==} + spex@3.4.0: + resolution: {integrity: sha512-8JeZJ7QlEBnSj1W1fKXgbB2KUPA8k4BxFMf6lZX/c1ZagU/1b9uZWZK0yD6yjfzqAIuTNG4YlRmtMpQiXuohsg==} + engines: {node: '>=14.0.0'} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -13310,7 +13349,7 @@ snapshots: '@azure/abort-controller@1.1.0': dependencies: - tslib: 2.6.0 + tslib: 2.8.1 '@azure/abort-controller@2.1.2': dependencies: @@ -13433,10 +13472,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/code-frame@7.22.5': - dependencies: - '@babel/highlight': 7.22.5 - '@babel/code-frame@7.26.2': dependencies: '@babel/helper-validator-identifier': 7.25.9 @@ -13503,8 +13538,6 @@ snapshots: '@babel/helper-string-parser@7.25.9': {} - '@babel/helper-validator-identifier@7.22.5': {} - '@babel/helper-validator-identifier@7.25.9': {} '@babel/helper-validator-option@7.25.9': {} @@ -13514,12 +13547,6 @@ snapshots: '@babel/template': 7.25.9 '@babel/types': 7.26.3 - '@babel/highlight@7.22.5': - dependencies: - '@babel/helper-validator-identifier': 7.22.5 - chalk: 2.4.2 - js-tokens: 4.0.0 - '@babel/parser@7.22.7': dependencies: '@babel/types': 7.22.5 @@ -13563,7 +13590,7 @@ snapshots: '@babel/types@7.22.5': dependencies: '@babel/helper-string-parser': 7.22.5 - '@babel/helper-validator-identifier': 7.22.5 + '@babel/helper-validator-identifier': 7.25.9 to-fast-properties: 2.0.0 '@babel/types@7.26.3': @@ -14401,13 +14428,13 @@ snapshots: '@jridgewell/gen-mapping@0.3.3': dependencies: '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@jridgewell/trace-mapping': 0.3.18 '@jridgewell/gen-mapping@0.3.8': dependencies: '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@jridgewell/trace-mapping': 0.3.25 '@jridgewell/resolve-uri@3.1.0': {} @@ -14425,8 +14452,6 @@ snapshots: '@jridgewell/sourcemap-codec@1.4.14': {} - '@jridgewell/sourcemap-codec@1.4.15': {} - '@jridgewell/sourcemap-codec@1.5.0': {} '@jridgewell/trace-mapping@0.3.18': @@ -14437,12 +14462,12 @@ snapshots: '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@js-sdsl/ordered-map@4.4.2': {} @@ -16262,6 +16287,14 @@ snapshots: dependencies: '@types/lodash': 4.17.13 + '@types/lodash.clone@4.5.9': + dependencies: + '@types/lodash': 4.17.13 + + '@types/lodash.get@4.4.9': + dependencies: + '@types/lodash': 4.17.13 + '@types/lodash@4.17.13': {} '@types/lodash@4.17.7': {} @@ -17103,6 +17136,8 @@ snapshots: binaryen: 116.0.0-nightly.20240114 long: 5.2.3 + assert-options@0.8.2: {} + assertion-error@2.0.1: {} ast-module-types@6.0.0: {} @@ -17347,7 +17382,7 @@ snapshots: call-bind@1.0.2: dependencies: - function-bind: 1.1.1 + function-bind: 1.1.2 get-intrinsic: 1.2.0 call-bind@1.0.8: @@ -17521,8 +17556,6 @@ snapshots: dependencies: restore-cursor: 5.1.0 - cli-spinners@2.9.0: {} - cli-spinners@2.9.2: {} cli-truncate@4.0.0: @@ -18374,15 +18407,15 @@ snapshots: eslint-import-resolver-node@0.3.7: dependencies: debug: 3.2.7 - is-core-module: 2.12.1 - resolve: 1.22.2 + is-core-module: 2.16.1 + resolve: 1.22.10 transitivePeerDependencies: - supports-color eslint-import-resolver-node@0.3.9: dependencies: debug: 3.2.7 - is-core-module: 2.16.0 + is-core-module: 2.16.1 resolve: 1.22.10 transitivePeerDependencies: - supports-color @@ -18394,9 +18427,9 @@ snapshots: eslint: 9.16.0(jiti@2.4.2) eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2)) eslint-plugin-import: 2.31.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2)) - get-tsconfig: 4.5.0 + get-tsconfig: 4.8.1 globby: 13.1.4 - is-core-module: 2.12.1 + is-core-module: 2.16.1 is-glob: 4.0.3 synckit: 0.8.5 transitivePeerDependencies: @@ -18440,7 +18473,7 @@ snapshots: eslint-import-resolver-node: 0.3.9 eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2)) hasown: 2.0.2 - is-core-module: 2.16.0 + is-core-module: 2.16.1 is-glob: 4.0.3 minimatch: 3.1.2 object.fromentries: 2.0.8 @@ -19088,8 +19121,6 @@ snapshots: - algoliasearch - supports-color - function-bind@1.1.1: {} - function-bind@1.1.2: {} function.prototype.name@1.1.5: @@ -19158,7 +19189,7 @@ snapshots: get-intrinsic@1.2.0: dependencies: - function-bind: 1.1.1 + function-bind: 1.1.2 has: 1.0.3 has-symbols: 1.0.3 @@ -19205,8 +19236,6 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.2.6 - get-tsconfig@4.5.0: {} - get-tsconfig@4.8.1: dependencies: resolve-pkg-maps: 1.0.0 @@ -19432,7 +19461,7 @@ snapshots: has@1.0.3: dependencies: - function-bind: 1.1.1 + function-bind: 1.1.2 hasown@2.0.2: dependencies: @@ -19796,10 +19825,6 @@ snapshots: is-callable@1.2.7: {} - is-core-module@2.12.1: - dependencies: - has: 1.0.3 - is-core-module@2.16.0: dependencies: hasown: 2.0.2 @@ -20273,8 +20298,12 @@ snapshots: lodash.camelcase@4.3.0: {} + lodash.clone@4.5.0: {} + lodash.debounce@4.0.8: {} + lodash.get@4.4.2: {} + lodash.includes@4.3.0: {} lodash.isboolean@3.0.3: {} @@ -21920,7 +21949,7 @@ snapshots: bl: 4.1.0 chalk: 4.1.2 cli-cursor: 3.1.0 - cli-spinners: 2.9.0 + cli-spinners: 2.9.2 is-interactive: 1.0.0 is-unicode-supported: 0.1.0 log-symbols: 4.1.0 @@ -22012,7 +22041,7 @@ snapshots: parse-json@5.2.0: dependencies: - '@babel/code-frame': 7.22.5 + '@babel/code-frame': 7.26.2 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 @@ -22082,8 +22111,14 @@ snapshots: pg-connection-string@2.7.0: {} + pg-cursor@2.12.1(pg@8.13.1): + dependencies: + pg: 8.13.1 + pg-int8@1.0.1: {} + pg-minify@1.6.5: {} + pg-numeric@1.0.2: {} pg-pool@3.7.0(pg@8.12.0): @@ -22094,8 +22129,23 @@ snapshots: dependencies: pg: 8.13.1 + pg-promise@11.10.2(pg-query-stream@4.7.1(pg@8.13.1)): + dependencies: + assert-options: 0.8.2 + pg: 8.13.1 + pg-minify: 1.6.5 + pg-query-stream: 4.7.1(pg@8.13.1) + spex: 3.4.0 + transitivePeerDependencies: + - pg-native + pg-protocol@1.7.0: {} + pg-query-stream@4.7.1(pg@8.13.1): + dependencies: + pg: 8.13.1 + pg-cursor: 2.12.1(pg@8.13.1) + pg-types@2.2.0: dependencies: pg-int8: 1.0.1 @@ -22960,12 +23010,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - resolve@1.22.2: - dependencies: - is-core-module: 2.12.1 - path-parse: 1.0.7 - supports-preserve-symlinks-flag: 1.0.0 - resolve@1.22.9: dependencies: is-core-module: 2.16.0 @@ -22974,7 +23018,7 @@ snapshots: resolve@2.0.0-next.5: dependencies: - is-core-module: 2.16.0 + is-core-module: 2.16.1 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 @@ -23428,6 +23472,8 @@ snapshots: spdx-license-ids@3.0.20: {} + spex@3.4.0: {} + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -24793,7 +24839,7 @@ snapshots: miniflare: 3.20241106.0(bufferutil@4.0.8) nanoid: 3.3.6 path-to-regexp: 6.3.0 - resolve: 1.22.9 + resolve: 1.22.10 resolve.exports: 2.0.2 selfsigned: 2.1.1 source-map: 0.6.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 28ee54f28a..235afafd8d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,7 @@ packages: - "e2e/examples/*" - "packages/*" - "packages/providers/*" + - "packages/providers/storage/*" - "packages/core/tests" - "packages/llamaindex/tests" - "packages/autotool/examples/*" diff --git a/tsconfig.json b/tsconfig.json index 9f2ad80af8..5e20e187d0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -124,6 +124,9 @@ }, { "path": "./packages/experimental/tsconfig.json" + }, + { + "path": "./packages/providers/storage/postgres/tsconfig.json" } ] }