diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index b47948758..2250e360f 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -3,6 +3,7 @@ ot = "ot" OT = "OT" typ = "typ" Typ = "Typ" +Rabit = "Rabit" [files] extend-exclude = ["**/*.test.txt"] \ No newline at end of file diff --git a/components/indexer/milvus/README.md b/components/indexer/milvus/README.md index d02f85178..340094214 100644 --- a/components/indexer/milvus/README.md +++ b/components/indexer/milvus/README.md @@ -6,6 +6,8 @@ An Milvus 2.x indexer implementation for [Eino](https://github.com/cloudwego/ein interface. This enables seamless integration with Eino's vector storage and retrieval system for enhanced semantic search capabilities. +> **Note**: This package supports **Milvus 2.4.x**. For Milvus 2.5+ features (BM25, server-side functions, hybrid search), use the [`milvus2`](../milvus2) package instead. + ## Quick Start ### Installation diff --git a/components/indexer/milvus/README_zh.md b/components/indexer/milvus/README_zh.md index 506cb5d31..afffe16dc 100644 --- a/components/indexer/milvus/README_zh.md +++ b/components/indexer/milvus/README_zh.md @@ -5,6 +5,8 @@ 基于 Milvus 2.x 的向量存储实现，为 [Eino](https://github.com/cloudwego/eino) 提供了符合 `Indexer` 接口的存储方案。该组件可无缝集成 Eino 的向量存储和检索系统，增强语义搜索能力。 +> **注意**: 此包支持 **Milvus 2.4.x**。如需使用 Milvus 2.5+ 的新功能（BM25、服务端函数、混合检索），请使用 [`milvus2`](../milvus2) 包。 + ## 快速开始 ### 安装 diff --git a/components/indexer/milvus2/README.md b/components/indexer/milvus2/README.md new file mode 100644 index 000000000..383743700 --- /dev/null +++ b/components/indexer/milvus2/README.md @@ -0,0 +1,361 @@ +# Milvus 2.x Indexer + +English | [中文](./README_zh.md) + +This package provides a Milvus 2.x (V2 SDK) indexer implementation for the EINO framework. It enables document storage and vector indexing in Milvus. + +> **Note**: This package requires **Milvus 2.5+** for server-side function support (e.g., BM25). + +## Features + +- **Milvus V2 SDK**: Uses the latest `milvus-io/milvus/client/v2` SDK +- **Flexible Index Types**: Supports comprehensive index types including Auto, HNSW, IVF variants, SCANN, DiskANN, GPU indexes, and RaBitQ (Milvus 2.6+) +- **Hybrid Search Ready**: Native support for Sparse Vectors (BM25/SPLADE) alongside Dense Vectors +- **Service-side Vector Generation**: Automatically generate sparse vectors using Milvus Functions (BM25) +- **Auto Management**: Handles collection schema creation, index building, and loading automatically +- **Field Analysis**: Configurable text analyzers (English, Chinese, Standard, etc.) +- **Custom Document Conversion**: Flexible mapping from Eino documents to Milvus columns + +## Installation + +```bash +go get github.com/cloudwego/eino-ext/components/indexer/milvus2 +``` + +## Quick Start + +```go +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino-ext/components/embedding/ark" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + // Get the environment variables + addr := os.Getenv("MILVUS_ADDR") + username := os.Getenv("MILVUS_USERNAME") + password := os.Getenv("MILVUS_PASSWORD") + arkApiKey := os.Getenv("ARK_API_KEY") + arkModel := os.Getenv("ARK_MODEL") + + ctx := context.Background() + + // Create an embedding model + emb, err := ark.NewEmbedder(ctx, &ark.EmbeddingConfig{ + APIKey: arkApiKey, + Model: arkModel, + }) + if err != nil { + log.Fatalf("Failed to create embedding: %v", err) + return + } + + // Create an indexer + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: addr, + Username: username, + Password: password, + }, + Collection: "my_collection", + + Vector: &milvus2.VectorConfig{ + Dimension: 1024, // Match your embedding model dimension + MetricType: milvus2.COSINE, + IndexBuilder: milvus2.NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200), + }, + Embedding: emb, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + return + } + log.Printf("Indexer created successfully") + + // Store documents + docs := []*schema.Document{ + { + ID: "doc1", + Content: "Milvus is an open-source vector database", + MetaData: map[string]any{ + "category": "database", + "year": 2021, + }, + }, + { + ID: "doc2", + Content: "EINO is a framework for building AI applications", + }, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + return + } + log.Printf("Store success, ids: %v", ids) +} +``` + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `Client` | `*milvusclient.Client` | - | Pre-configured Milvus client (optional) | +| `ClientConfig` | `*milvusclient.ClientConfig` | - | Client configuration (required if Client is nil) | +| `Collection` | `string` | `"eino_collection"` | Collection name | +| `Vector` | `*VectorConfig` | - | Dense vector configuration (Dimension, MetricType, IndexBuilder) | +| `Sparse` | `*SparseVectorConfig` | - | Sparse vector configuration (MetricType, FieldName) | +| `Embedding` | `embedding.Embedder` | - | Embedder for vectorization (optional). If nil, documents must have vectors (BYOV). | +| `DocumentConverter` | `func` | default converter | Custom document to Milvus column converter | +| `ConsistencyLevel` | `ConsistencyLevel` | `ConsistencyLevelDefault` | Consistency level (`ConsistencyLevelDefault` uses Milvus default: Bounded; stays at collection level if not explicitly set) | +| `PartitionName` | `string` | - | Default partition for insertion | +| `EnableDynamicSchema` | `bool` | `false` | Enable dynamic field support | +| `Functions` | `[]*entity.Function` | - | Schema functions (e.g., BM25) for server-side processing | +| `FieldParams` | `map[string]map[string]string` | - | Parameters for fields (e.g., enable_analyzer) | + +### Vector Configuration (`VectorConfig`) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `Dimension` | `int64` | - | Vector dimension (Required) | +| `MetricType` | `MetricType` | `L2` | Similarity metric (L2, IP, COSINE, etc.) | +| `IndexBuilder` | `IndexBuilder` | `AutoIndexBuilder` | Index type builder (HNSW, IVF, etc.) | +| `VectorField` | `string` | `"vector"` | Field name for dense vector | + +### Sparse Vector Configuration (`SparseVectorConfig`) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `VectorField` | `string` | `"sparse_vector"` | Field name for sparse vector | +| `MetricType` | `MetricType` | `BM25` | Similarity metric | +| `Method` | `SparseMethod` | `SparseMethodAuto` | Generation method (`SparseMethodAuto` or `SparseMethodPrecomputed`) | +| `IndexBuilder` | `SparseIndexBuilder` | `SparseInvertedIndex` | Index builder (`NewSparseInvertedIndexBuilder` or `NewSparseWANDIndexBuilder`) | + +> **Note**: `Method` defaults to `Auto` only if `MetricType` is `BM25`. `Auto` implies using Milvus server-side functions (remote function). For other metrics (e.g., `IP`), it defaults to `Precomputed`. + +## Index Builders + +### Dense Index Builders + +| Builder | Description | Key Parameters | +|---------|-------------|----------------| +| `NewAutoIndexBuilder()` | Milvus auto-selects optimal index | - | +| `NewHNSWIndexBuilder()` | Graph-based with excellent performance | `M`, `EfConstruction` | +| `NewIVFFlatIndexBuilder()` | Cluster-based search | `NList` | +| `NewIVFPQIndexBuilder()` | Memory-efficient with product quantization | `NList`, `M`, `NBits` | +| `NewIVFSQ8IndexBuilder()` | Scalar quantization | `NList` | +| `NewIVFRabitQIndexBuilder()` | IVF + RaBitQ binary quantization (Milvus 2.6+) | `NList` | +| `NewFlatIndexBuilder()` | Brute-force exact search | - | +| `NewDiskANNIndexBuilder()` | Disk-based for large datasets | - | +| `NewSCANNIndexBuilder()` | Fast with high recall | `NList`, `WithRawDataEnabled` | +| `NewBinFlatIndexBuilder()` | Brute-force for binary vectors | - | +| `NewBinIVFFlatIndexBuilder()` | Cluster-based for binary vectors | `NList` | +| `NewGPUBruteForceIndexBuilder()` | GPU-accelerated brute-force | - | +| `NewGPUIVFFlatIndexBuilder()` | GPU-accelerated IVF_FLAT | - | +| `NewGPUIVFPQIndexBuilder()` | GPU-accelerated IVF_PQ | - | +| `NewGPUCagraIndexBuilder()` | GPU-accelerated graph-based (CAGRA) | `IntermediateGraphDegree`, `GraphDegree` | + +### Sparse Index Builders + +| Builder | Description | Key Parameters | +|---------|-------------|----------------| +| `NewSparseInvertedIndexBuilder()` | Inverted index for sparse vectors | `DropRatioBuild` | +| `NewSparseWANDIndexBuilder()` | WAND algorithm for sparse vectors | `DropRatioBuild` | + +### Example: HNSW Index + +```go +indexBuilder := milvus2.NewHNSWIndexBuilder(). + WithM(16). // Max connections per node (4-64) + WithEfConstruction(200) // Index build search width (8-512) +``` + +### Example: IVF_FLAT Index + +```go +indexBuilder := milvus2.NewIVFFlatIndexBuilder(). + WithNList(256) // Number of cluster units (1-65536) +``` + +### Example: IVF_PQ Index (Memory-efficient) + +```go +indexBuilder := milvus2.NewIVFPQIndexBuilder(). + WithNList(256). // Number of cluster units + WithM(16). // Number of subquantizers + WithNBits(8) // Bits per subquantizer (1-16) +``` + +### Example: SCANN Index (Fast with high recall) + +```go +indexBuilder := milvus2.NewSCANNIndexBuilder(). + WithNList(256). // Number of cluster units + WithRawDataEnabled(true) // Enable raw data for reranking +``` + +### Example: DiskANN Index (Large datasets) + +```go +indexBuilder := milvus2.NewDiskANNIndexBuilder() // Disk-based, no extra params +``` + +### Example: Sparse Inverted Index + +```go +indexBuilder := milvus2.NewSparseInvertedIndexBuilder(). + WithDropRatioBuild(0.2) // Drop ratio for small values (0.0-1.0) +``` + +### Dense Vector Metrics +| Metric | Description | +|--------|-------------| +| `L2` | Euclidean distance | +| `IP` | Inner Product | +| `COSINE` | Cosine similarity | + +### Sparse Vector Metrics +| Metric | Description | +|--------|-------------| +| `BM25` | Okapi BM25 (Required for `SparseMethodAuto`) | +| `IP` | Inner Product (Suitable for precomputed sparse vectors) | + +### Binary Vector Metrics +| Metric | Description | +|--------|-------------| +| `HAMMING` | Hamming distance | +| `JACCARD` | Jaccard distance | +| `TANIMOTO` | Tanimoto distance | +| `SUBSTRUCTURE` | Substructure search | +| `SUPERSTRUCTURE` | Superstructure search | + +## Sparse Vector Support + +The indexer supports two modes for sparse vectors: **Auto-Generation** and **Precomputed**. + +### 1. Auto-Generation (BM25) + +Uses Milvus server-side functions to automatically generate sparse vectors from the content field. + +- **Requirement**: Milvus 2.5+ +- **Configuration**: Set `MetricType: milvus2.BM25`. + +```go +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + // ... basic config ... + Collection: "hybrid_collection", + + Sparse: &milvus2.SparseVectorConfig{ + VectorField: "sparse_vector", + MetricType: milvus2.BM25, + // Method defaults to SparseMethodAuto for BM25 + }, + + // Analyzer configuration for BM25 + FieldParams: map[string]map[string]string{ + "content": { + "enable_analyzer": "true", + "analyzer_params": `{"type": "standard"}`, + }, + }, +}) +``` + +### 2. Precomputed (SPLADE, BGE-M3, etc.) + +Allows storing sparse vectors generated by external models (e.g., SPLADE, BGE-M3) or custom logic. + +- **Configuration**: Set `MetricType` (usually `IP`) and `Method: milvus2.SparseMethodPrecomputed`. +- **Usage**: Provide sparse vectors via `doc.WithSparseVector()`. + +```go +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + Collection: "sparse_collection", + + Sparse: &milvus2.SparseVectorConfig{ + VectorField: "sparse_vector", + MetricType: milvus2.IP, + Method: milvus2.SparseMethodPrecomputed, + }, +}) + +// Store documents with sparse vectors +doc := &schema.Document{ID: "1", Content: "..."} +doc.WithSparseVector(map[int]float64{ + 1024: 0.5, + 2048: 0.3, +}) +indexer.Store(ctx, []*schema.Document{doc}) +``` + +## Bring Your Own Vectors (BYOV) + +You can use the indexer without an embedder if your documents already have vectors. + +```go +// Create indexer without embedding +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "my_collection", + Vector: &milvus2.VectorConfig{ + Dimension: 128, + MetricType: milvus2.L2, + }, + // Embedding: nil, // Leave nil +}) + +// Store documents with pre-computed vectors +docs := []*schema.Document{ + { + ID: "doc1", + Content: "Document with existing vector", + }, +} + +// Attach dense vector to document +// Vector dimension must match the collection dimension +vector := []float64{0.1, 0.2, ...} +docs[0].WithDenseVector(vector) + +// Attach sparse vector (optional, if Sparse is configured) +// Sparse vectors are maps of index -> weight +sparseVector := map[int]float64{ + 10: 0.5, + 25: 0.8, +} +docs[0].WithSparseVector(sparseVector) + +ids, err := indexer.Store(ctx, docs) +``` + +For sparse vectors in BYOV mode, configured the sparse vector as **Precomputed** (see above). + +## Examples + +See the [examples](./examples) directory for complete working examples: + +- [demo](./examples/demo) - Basic collection setup with HNSW index +- [hnsw](./examples/hnsw) - HNSW index example +- [ivf_flat](./examples/ivf_flat) - IVF_FLAT index example +- [rabitq](./examples/rabitq) - IVF_RABITQ index example (Milvus 2.6+) +- [auto](./examples/auto) - AutoIndex example +- [diskann](./examples/diskann) - DISKANN index example +- [hybrid](./examples/hybrid) - Hybrid search setup (Dense + BM25 sparse) (Milvus 2.5+) +- [hybrid_chinese](./examples/hybrid_chinese) - Hybrid search with Chinese analyzer (Milvus 2.5+) +- [sparse](./examples/sparse) - Sparse-only index example (BM25) +- [byov](./examples/byov) - Bring Your Own Vectors example + +## License + +Apache License 2.0 diff --git a/components/indexer/milvus2/README_zh.md b/components/indexer/milvus2/README_zh.md new file mode 100644 index 000000000..26c40186c --- /dev/null +++ b/components/indexer/milvus2/README_zh.md @@ -0,0 +1,360 @@ +# Milvus 2.x Indexer + +[English](./README.md) | 中文 + +本包为 EINO 框架提供 Milvus 2.x (V2 SDK) 索引器实现，支持文档存储和向量索引。 + +> **注意**: 本包需要 **Milvus 2.5+** 以支持服务器端函数（如 BM25）。 + +## 功能特性 + +- **Milvus V2 SDK**: 使用最新的 `milvus-io/milvus/client/v2` SDK +- **灵活的索引类型**: 支持多种索引构建器，包括 Auto, HNSW, IVF 系列, SCANN, DiskANN, GPU 索引以及 RaBitQ (Milvus 2.6+) +- **混合搜索就绪**: 原生支持稀疏向量 (BM25/SPLADE) 与稠密向量的混合存储 +- **服务端向量生成**: 使用 Milvus Functions (BM25) 自动生成稀疏向量 +- **自动化管理**: 自动处理集合 Schema 创建、索引构建和加载 +- **字段分析**: 可配置的文本分析器（支持中文 Jieba、英文、Standard 等） +- **自定义文档转换**: Eino 文档到 Milvus 列的灵活映射 + +## 安装 + +```bash +go get github.com/cloudwego/eino-ext/components/indexer/milvus2 +``` + +## 快速开始 + +```go +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino-ext/components/embedding/ark" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + // 获取环境变量 + addr := os.Getenv("MILVUS_ADDR") + username := os.Getenv("MILVUS_USERNAME") + password := os.Getenv("MILVUS_PASSWORD") + arkApiKey := os.Getenv("ARK_API_KEY") + arkModel := os.Getenv("ARK_MODEL") + + ctx := context.Background() + + // 创建 embedding 模型 + emb, err := ark.NewEmbedder(ctx, &ark.EmbeddingConfig{ + APIKey: arkApiKey, + Model: arkModel, + }) + if err != nil { + log.Fatalf("Failed to create embedding: %v", err) + return + } + + // 创建索引器 + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: addr, + Username: username, + Password: password, + }, + Collection: "my_collection", + + Vector: &milvus2.VectorConfig{ + Dimension: 1024, // 与 embedding 模型维度匹配 + MetricType: milvus2.COSINE, + IndexBuilder: milvus2.NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200), + }, + Embedding: emb, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + return + } + log.Printf("Indexer created successfully") + + // 存储文档 + docs := []*schema.Document{ + { + ID: "doc1", + Content: "Milvus is an open-source vector database", + MetaData: map[string]any{ + "category": "database", + "year": 2021, + }, + }, + { + ID: "doc2", + Content: "EINO is a framework for building AI applications", + }, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + return + } + log.Printf("Store success, ids: %v", ids) +} +``` + +## 配置选项 + +| 字段 | 类型 | 默认值 | 描述 | +|------|------|--------|------| +| `Client` | `*milvusclient.Client` | - | 预配置的 Milvus 客户端（可选） | +| `ClientConfig` | `*milvusclient.ClientConfig` | - | 客户端配置（Client 为空时必需） | +| `Collection` | `string` | `"eino_collection"` | 集合名称 | +| `Vector` | `*VectorConfig` | - | 稠密向量配置 (维度, MetricType, 字段名) | +| `Sparse` | `*SparseVectorConfig` | - | 稀疏向量配置 (MetricType, 字段名) | +| `IndexBuilder` | `IndexBuilder` | `AutoIndexBuilder` | 索引类型构建器 | +| `Embedding` | `embedding.Embedder` | - | 用于向量化的 Embedder（可选）。如果为空，文档必须包含向量 (BYOV)。 | +| `ConsistencyLevel` | `ConsistencyLevel` | `ConsistencyLevelDefault` | 一致性级别 (`ConsistencyLevelDefault` 使用 Milvus 默认: Bounded; 如果未显式设置，则保持集合级别设置) | +| `PartitionName` | `string` | - | 插入数据的默认分区 | +| `EnableDynamicSchema` | `bool` | `false` | 启用动态字段支持 | +| `Functions` | `[]*entity.Function` | - | Schema 函数定义（如 BM25），用于服务器端处理 | +| `FieldParams` | `map[string]map[string]string` | - | 字段参数配置（如 enable_analyzer） | + +### 稠密向量配置 (`VectorConfig`) + +| 字段 | 类型 | 默认值 | 描述 | +|------|------|--------|------| +| `Dimension` | `int64` | - | 向量维度 (必需) | +| `MetricType` | `MetricType` | `L2` | 相似度度量类型 (L2, IP, COSINE 等) | +| `VectorField` | `string` | `"vector"` | 稠密向量字段名 | + +### 稀疏向量配置 (`SparseVectorConfig`) + +| 字段 | 类型 | 默认值 | 描述 | +|------|------|--------|------| +| `VectorField` | `string` | `"sparse_vector"` | 稀疏向量字段名 | +| `MetricType` | `MetricType` | `BM25` | 相似度度量类型 | +| `Method` | `SparseMethod` | `SparseMethodAuto` | 生成方法 (`SparseMethodAuto` 或 `SparseMethodPrecomputed`) | +| `IndexBuilder` | `SparseIndexBuilder` | `SparseInvertedIndex` | 索引构建器 (`NewSparseInvertedIndexBuilder` 或 `NewSparseWANDIndexBuilder`) | + +> **注意**: 仅当 `MetricType` 为 `BM25` 时，`Method` 默认为 `Auto`。`Auto` 意味着使用 Milvus 服务器端函数（远程函数）。对于其他度量类型（如 `IP`），默认为 `Precomputed`。 + +## 索引构建器 + +### 稠密索引构建器 (Dense) + +| 构建器 | 描述 | 关键参数 | +|--------|------|----------| +| `NewAutoIndexBuilder()` | Milvus 自动选择最优索引 | - | +| `NewHNSWIndexBuilder()` | 基于图的高性能索引 | `M`, `EfConstruction` | +| `NewIVFFlatIndexBuilder()` | 基于聚类的搜索 | `NList` | +| `NewIVFPQIndexBuilder()` | 乘积量化，内存高效 | `NList`, `M`, `NBits` | +| `NewIVFSQ8IndexBuilder()` | 标量量化 | `NList` | +| `NewIVFRabitQIndexBuilder()` | IVF + RaBitQ 二进制量化 (Milvus 2.6+) | `NList` | +| `NewFlatIndexBuilder()` | 暴力精确搜索 | - | +| `NewDiskANNIndexBuilder()` | 面向大数据集的磁盘索引 | - | +| `NewSCANNIndexBuilder()` | 高召回率的快速搜索 | `NList`, `WithRawDataEnabled` | +| `NewBinFlatIndexBuilder()` | 二进制向量的暴力搜索 | - | +| `NewBinIVFFlatIndexBuilder()` | 二进制向量的聚类搜索 | `NList` | +| `NewGPUBruteForceIndexBuilder()` | GPU 加速暴力搜索 | - | +| `NewGPUIVFFlatIndexBuilder()` | GPU 加速 IVF_FLAT | - | +| `NewGPUIVFPQIndexBuilder()` | GPU 加速 IVF_PQ | - | +| `NewGPUCagraIndexBuilder()` | GPU 加速图索引 (CAGRA) | `IntermediateGraphDegree`, `GraphDegree` | + +### 稀疏索引构建器 (Sparse) + +| 构建器 | 描述 | 关键参数 | +|--------|------|----------| +| `NewSparseInvertedIndexBuilder()` | 稀疏向量倒排索引 | `DropRatioBuild` | +| `NewSparseWANDIndexBuilder()` | 稀疏向量 WAND 算法 | `DropRatioBuild` | + +### 示例：HNSW 索引 + +```go +indexBuilder := milvus2.NewHNSWIndexBuilder(). + WithM(16). // 每个节点的最大连接数 (4-64) + WithEfConstruction(200) // 索引构建时的搜索宽度 (8-512) +``` + +### 示例：IVF_FLAT 索引 + +```go +indexBuilder := milvus2.NewIVFFlatIndexBuilder(). + WithNList(256) // 聚类单元数量 (1-65536) +``` + +### 示例：IVF_PQ 索引（内存高效） + +```go +indexBuilder := milvus2.NewIVFPQIndexBuilder(). + WithNList(256). // 聚类单元数量 + WithM(16). // 子量化器数量 + WithNBits(8) // 每个子量化器的位数 (1-16) +``` + +### 示例：SCANN 索引（高召回率快速搜索） + +```go +indexBuilder := milvus2.NewSCANNIndexBuilder(). + WithNList(256). // 聚类单元数量 + WithRawDataEnabled(true) // 启用原始数据进行重排序 +``` + +### 示例：DiskANN 索引（大数据集） + +```go +indexBuilder := milvus2.NewDiskANNIndexBuilder() // 基于磁盘，无额外参数 +``` + +### 示例：Sparse Inverted Index (稀疏倒排索引) + +```go +indexBuilder := milvus2.NewSparseInvertedIndexBuilder(). + WithDropRatioBuild(0.2) // 构建时忽略小值的比例 (0.0-1.0) +``` + +### 稠密向量度量 (Dense) +| 度量类型 | 描述 | +|----------|------| +| `L2` | 欧几里得距离 | +| `IP` | 内积 | +| `COSINE` | 余弦相似度 | + +### 稀疏向量度量 (Sparse) +| 度量类型 | 描述 | +|----------|------| +| `BM25` | Okapi BM25 (`SparseMethodAuto` 必需) | +| `IP` | 内积 (适用于预计算的稀疏向量) | + +### 二进制向量度量 (Binary) +| 度量类型 | 描述 | +|----------|------| +| `HAMMING` | 汉明距离 | +| `JACCARD` | 杰卡德距离 | +| `TANIMOTO` | Tanimoto 距离 | +| `SUBSTRUCTURE` | 子结构搜索 | +| `SUPERSTRUCTURE` | 超结构搜索 | + +## 稀疏向量支持 + +索引器支持两种稀疏向量模式：**自动生成 (Auto-Generation)** 和 **预计算 (Precomputed)**。 + +### 1. 自动生成 (BM25) + +使用 Milvus 服务器端函数从内容字段自动生成稀疏向量。 + +- **要求**: Milvus 2.5+ +- **配置**: 设置 `MetricType: milvus2.BM25`。 + +```go +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + // ... 基础配置 ... + Collection: "hybrid_collection", + + Sparse: &milvus2.SparseVectorConfig{ + VectorField: "sparse_vector", + MetricType: milvus2.BM25, + // BM25 时 Method 默认为 SparseMethodAuto + }, + + // BM25 的分析器配置 + FieldParams: map[string]map[string]string{ + "content": { + "enable_analyzer": "true", + "analyzer_params": `{"type": "standard"}`, // 中文可使用 {"type": "chinese"} + }, + }, +}) +``` + +### 2. 预计算 (SPLADE, BGE-M3 等) + +允许存储由外部模型（如 SPLADE, BGE-M3）或自定义逻辑生成的稀疏向量。 + +- **配置**: 设置 `MetricType`（通常为 `IP`）和 `Method: milvus2.SparseMethodPrecomputed`。 +- **用法**: 通过 `doc.WithSparseVector()` 传入稀疏向量。 + +```go +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + Collection: "sparse_collection", + + Sparse: &milvus2.SparseVectorConfig{ + VectorField: "sparse_vector", + MetricType: milvus2.IP, + Method: milvus2.SparseMethodPrecomputed, + }, +}) + +// 存储包含稀疏向量的文档 +doc := &schema.Document{ID: "1", Content: "..."} +doc.WithSparseVector(map[int]float64{ + 1024: 0.5, + 2048: 0.3, +}) +indexer.Store(ctx, []*schema.Document{doc}) +``` + +## 自带向量 (Bring Your Own Vectors) + +如果您的文档已经包含向量，可以不配置 Embedder 使用 Indexer。 + +```go +// 创建不带 embedding 的 indexer +indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "my_collection", + Vector: &milvus2.VectorConfig{ + Dimension: 128, + MetricType: milvus2.L2, + }, + // Embedding: nil, // 留空 +}) + +// 存储带有预计算向量的文档 +docs := []*schema.Document{ + { + ID: "doc1", + Content: "Document with existing vector", + }, +} + +// 附加稠密向量到文档 +// 向量维度必须与集合维度匹配 +vector := []float64{0.1, 0.2, ...} +docs[0].WithDenseVector(vector) + +// 附加稀疏向量（可选，如果配置了 Sparse） +// 稀疏向量是 index -> weight 的映射 +sparseVector := map[int]float64{ + 10: 0.5, + 25: 0.8, +} +docs[0].WithSparseVector(sparseVector) + +ids, err := indexer.Store(ctx, docs) +``` + +对于 BYOV 模式下的稀疏向量，请参考上文 **预计算 (Precomputed)** 部分进行配置。 + +## 示例 + +查看 [examples](./examples) 目录获取完整的示例代码： + +- [demo](./examples/demo) - 使用 HNSW 索引的基础集合设置 +- [hnsw](./examples/hnsw) - HNSW 索引示例 +- [ivf_flat](./examples/ivf_flat) - IVF_FLAT 索引示例 +- [rabitq](./examples/rabitq) - IVF_RABITQ 索引示例 (Milvus 2.6+) +- [auto](./examples/auto) - AutoIndex 示例 +- [diskann](./examples/diskann) - DISKANN 索引示例 +- [hybrid](./examples/hybrid) - 混合搜索设置 (稠密 + BM25 稀疏) (Milvus 2.5+) +- [hybrid_chinese](./examples/hybrid_chinese) - 中文混合搜索示例 (Milvus 2.5+) +- [sparse](./examples/sparse) - 纯稀疏索引示例 (BM25) +- [byov](./examples/byov) - 自带向量示例 + +## 许可证 + +Apache License 2.0 diff --git a/components/indexer/milvus2/consts.go b/components/indexer/milvus2/consts.go new file mode 100644 index 000000000..7c996a3a2 --- /dev/null +++ b/components/indexer/milvus2/consts.go @@ -0,0 +1,31 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +const ( + typ = "Milvus2" + + defaultCollection = "eino_collection" + defaultDescription = "the collection for eino" + defaultVectorField = "vector" + defaultSparseVectorField = "sparse_vector" + defaultContentField = "content" + defaultMetadataField = "metadata" + defaultIDField = "id" + defaultMaxContentLen = 65535 + defaultMaxIDLen = 512 +) diff --git a/components/indexer/milvus2/examples/auto/auto.go b/components/indexer/milvus2/examples/auto/auto.go new file mode 100644 index 000000000..65c092803 --- /dev/null +++ b/components/indexer/milvus2/examples/auto/auto.go @@ -0,0 +1,85 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates using AUTOINDEX for vector storage. +// AUTOINDEX lets Milvus automatically choose the optimal index type +// based on data characteristics - simplest to use. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create an indexer with AUTOINDEX (default when IndexBuilder is nil) + // Milvus will automatically select the best index type + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_auto", + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: 128, + MetricType: milvus2.IP, // Inner Product + // IndexBuilder: nil means AUTOINDEX is used + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + log.Println("AUTOINDEX indexer created successfully") + + // Store sample documents + docs := []*schema.Document{ + {ID: "auto-1", Content: "AUTOINDEX automatically selects the best index"}, + {ID: "auto-2", Content: "Simplest option for getting started"}, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + log.Printf("Stored documents: %v", ids) +} + +// mockEmbedding generates random embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(i+j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/byov/byov.go b/components/indexer/milvus2/examples/byov/byov.go new file mode 100644 index 000000000..1a9c05233 --- /dev/null +++ b/components/indexer/milvus2/examples/byov/byov.go @@ -0,0 +1,101 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "log" + "math/rand" + "os" + + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + collectionName := "eino_byov_test" + dim := 128 + + // Create a new Milvus client + cli, err := milvusclient.New(ctx, &milvusclient.ClientConfig{Address: addr}) + if err != nil { + log.Fatalf("failed to create milvus client: %v", err) + } + defer cli.Close(ctx) + + // Clean up existing collection + _ = cli.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) + + // Create Indexer WITHOUT an Embedder + // This configuration allows "Bring Your Own Vectors" (BYOV) + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + Client: cli, + Collection: collectionName, + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: int64(dim), + }, + // Embedding field is omitted/nil + }) + if err != nil { + log.Fatalf("failed to create indexer: %v", err) + } + + // Prepare documents with pre-computed vectors + docs := make([]*schema.Document, 0, 10) + for i := 0; i < 10; i++ { + // Generate variable content + doc := &schema.Document{ + ID: fmt.Sprintf("doc_%d", i), + Content: fmt.Sprintf("This is document number %d with a pre-computed vector.", i), + MetaData: map[string]any{ + "source": "manual_vector_generation", + }, + } + + // Generate a random vector + // In a real scenario, this would come from an external model or API + vector := make([]float64, dim) + for j := 0; j < dim; j++ { + vector[j] = rand.Float64() + } + + // Attach the vector to the document using WithDenseVector + doc.WithDenseVector(vector) + + docs = append(docs, doc) + } + + // Store the documents + // The indexer will detect the nil Embedder and look for vectors in the document metadata + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("failed to store documents: %v", err) + } + + log.Printf("Successfully stored %d documents with custom vectors!", len(ids)) + log.Printf("Stored IDs: %v", ids) +} diff --git a/components/indexer/milvus2/examples/demo/demo.go b/components/indexer/milvus2/examples/demo/demo.go new file mode 100644 index 000000000..75619ac0a --- /dev/null +++ b/components/indexer/milvus2/examples/demo/demo.go @@ -0,0 +1,203 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example creates the "demo_milvus2" collection used by multiple retriever examples. +// Run this first, then run any of the retriever examples: +// - retriever/milvus2/examples/grouping +// - retriever/milvus2/examples/iterator +// - retriever/milvus2/examples/scalar +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/column" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + collectionName := "demo_milvus2" + vectorField := "vector" + dim := 128 + itemCount := 60 // Enough for iterator (50+) and grouping tests + + // Create client + cli, err := milvusclient.New(ctx, &milvusclient.ClientConfig{Address: addr}) + if err != nil { + log.Fatal(err) + } + defer cli.Close(ctx) + + // Drop existing collection + _ = cli.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) + + // Create schema with category field for grouping + s := entity.NewSchema(). + WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeVarChar).WithMaxLength(65535).WithIsPrimaryKey(true)). + WithField(entity.NewField().WithName(vectorField).WithDataType(entity.FieldTypeFloatVector).WithDim(int64(dim))). + WithField(entity.NewField().WithName("content").WithDataType(entity.FieldTypeVarChar).WithMaxLength(65535)). + WithField(entity.NewField().WithName("metadata").WithDataType(entity.FieldTypeJSON)). + WithField(entity.NewField().WithName("category").WithDataType(entity.FieldTypeVarChar).WithMaxLength(255)) + + err = cli.CreateCollection(ctx, milvusclient.NewCreateCollectionOption(collectionName, s)) + if err != nil { + log.Fatal(err) + } + log.Printf("Created collection: %s", collectionName) + + // Create HNSW index with COSINE metric + idx := milvus2.NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200).Build(milvus2.COSINE) + t, err := cli.CreateIndex(ctx, milvusclient.NewCreateIndexOption(collectionName, vectorField, idx)) + if err != nil { + log.Fatal(err) + } + if err := t.Await(ctx); err != nil { + log.Fatal(err) + } + log.Println("Created HNSW index with COSINE metric") + + // Load collection + loadTask, err := cli.LoadCollection(ctx, milvusclient.NewLoadCollectionOption(collectionName)) + if err != nil { + log.Fatal(err) + } + if err := loadTask.Await(ctx); err != nil { + log.Fatal(err) + } + + // Create indexer with custom DocumentConverter that extracts category for grouping + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + Client: cli, + Collection: collectionName, + Vector: &milvus2.VectorConfig{ + VectorField: vectorField, + Dimension: int64(dim), + MetricType: milvus2.COSINE, + IndexBuilder: milvus2.NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200), + }, + Embedding: &mockEmbedding{dim: dim}, + DocumentConverter: func(ctx context.Context, docs []*schema.Document, vectors [][]float64) ([]column.Column, error) { + ids := make([]string, 0, len(docs)) + contents := make([]string, 0, len(docs)) + vecs := make([][]float32, 0, len(docs)) + metadatas := make([][]byte, 0, len(docs)) + categories := make([]string, 0, len(docs)) + + for idx, doc := range docs { + ids = append(ids, doc.ID) + contents = append(contents, doc.Content) + + // Convert float64 vector to float32 + vec := make([]float32, len(vectors[idx])) + for i, v := range vectors[idx] { + vec[i] = float32(v) + } + vecs = append(vecs, vec) + + // Marshal metadata to JSON + metadata, err := json.Marshal(doc.MetaData) + if err != nil { + return nil, fmt.Errorf("failed to marshal metadata: %w", err) + } + metadatas = append(metadatas, metadata) + + // Extract category for grouping search + cat := "" + if c, ok := doc.MetaData["category"].(string); ok { + cat = c + } + categories = append(categories, cat) + } + + vecDim := 0 + if len(vecs) > 0 { + vecDim = len(vecs[0]) + } + + return []column.Column{ + column.NewColumnVarChar("id", ids), + column.NewColumnVarChar("content", contents), + column.NewColumnFloatVector("vector", vecDim, vecs), + column.NewColumnJSONBytes("metadata", metadatas), + column.NewColumnVarChar("category", categories), + }, nil + }, + }) + if err != nil { + log.Fatal(err) + } + + // Insert documents with categories and metadata for all example scenarios + categories := []string{"electronics", "books", "clothing"} + tags := []string{"coding", "life", "travel"} + + var docs []*schema.Document + for i := 0; i < itemCount; i++ { + cat := categories[i%len(categories)] + tag := tags[i%len(tags)] + year := 2023 + (i % 2) + + docs = append(docs, &schema.Document{ + ID: fmt.Sprintf("doc_%d", i), + Content: fmt.Sprintf("Content %d in category %s about %s", i, cat, tag), + MetaData: map[string]any{ + "category": cat, + "tag": tag, + "year": year, + }, + }) + } + + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatal(err) + } + log.Printf("Stored %d documents", len(ids)) + log.Printf(" Categories: %v", categories) + log.Printf(" Tags: %v", tags) + log.Printf(" Years: 2023, 2024") + log.Println("Collection ready for retriever examples (grouping, iterator, scalar)") +} + +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + // Distinct vectors based on index for iterator test + vec[j] = float64(i) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/diskann/diskann.go b/components/indexer/milvus2/examples/diskann/diskann.go new file mode 100644 index 000000000..cfb4a2e21 --- /dev/null +++ b/components/indexer/milvus2/examples/diskann/diskann.go @@ -0,0 +1,85 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates using a DISKANN index for vector storage. +// DISKANN stores the index on disk, enabling handling of very large +// datasets that don't fit in memory. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create an indexer with DISKANN index + // Best for large datasets that exceed memory capacity + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_diskann", + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: 128, + MetricType: milvus2.L2, + IndexBuilder: milvus2.NewDiskANNIndexBuilder(), + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + log.Println("DISKANN indexer created successfully") + + // Store sample documents + docs := []*schema.Document{ + {ID: "disk-1", Content: "DISKANN enables disk-based vector search"}, + {ID: "disk-2", Content: "Handles billions of vectors without memory issues"}, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + log.Printf("Stored documents: %v", ids) +} + +// mockEmbedding generates random embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(i+j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/hnsw/hnsw.go b/components/indexer/milvus2/examples/hnsw/hnsw.go new file mode 100644 index 000000000..5ca1b6215 --- /dev/null +++ b/components/indexer/milvus2/examples/hnsw/hnsw.go @@ -0,0 +1,88 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates using an HNSW index for vector storage. +// HNSW (Hierarchical Navigable Small World) is a graph-based index +// that offers excellent query performance with tunable parameters. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create an indexer with HNSW index + // M: maximum connections per node (higher = better recall, more memory) + // EfConstruction: search width during building (higher = better index quality, slower build) + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_hnsw", + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: 128, + MetricType: milvus2.COSINE, + IndexBuilder: milvus2.NewHNSWIndexBuilder(). + WithM(16). + WithEfConstruction(200), + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + log.Println("HNSW indexer created successfully") + + // Store sample documents + docs := []*schema.Document{ + {ID: "hnsw-1", Content: "HNSW provides excellent query performance"}, + {ID: "hnsw-2", Content: "Graph-based index for high-dimensional vectors"}, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + log.Printf("Stored documents: %v", ids) +} + +// mockEmbedding generates random embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(i+j) * 0.01 // Simple deterministic values + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/hybrid/hybrid.go b/components/indexer/milvus2/examples/hybrid/hybrid.go new file mode 100644 index 000000000..3e3d36e97 --- /dev/null +++ b/components/indexer/milvus2/examples/hybrid/hybrid.go @@ -0,0 +1,109 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "log" + + "github.com/cloudwego/eino-ext/components/indexer/milvus2" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// This example demonstrates Hybrid Search indexing - creating a collection with +// both Dense vectors (for semantic search) and Sparse vectors (BM25 for keyword search). +// +// The BM25 function tokenizes text and generates term frequency vectors automatically +// on the server side, enabling keyword-based search without client-side sparse embedding. +// +// Prerequisites: +// - Milvus 2.5+ (required for server-side functions) +// +// The collection created here can be used with the retriever hybrid example +// for combined Dense + Sparse (BM25) hybrid search. + +func main() { + ctx := context.Background() + collectionName := "eino_hybrid_test" + milvusAddr := "localhost:19530" + denseField := "vector" + sparseField := "sparse_vector" + + // Create indexer with BM25 function + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: milvusAddr, + }, + Collection: collectionName, + Vector: &milvus2.VectorConfig{ + VectorField: denseField, + Dimension: 128, // Dense vector dimension + }, + + // BM25 requires analyzer on content field. + // Analyzer options (built-in): + // - {"type": "standard"} - general-purpose, tokenization + lowercase + // - {"type": "english"} - English with stopwords support + // - {"type": "chinese"} - Chinese with Jieba segmentation + // - Custom: {"tokenizer": "...", "filter": [...]} + // See: https://milvus.io/docs/analyzer-overview.md + FieldParams: map[string]map[string]string{ + "content": { + "enable_analyzer": "true", + "analyzer_params": `{"type": "standard"}`, // Use {"type": "chinese"} for Chinese text + }, + }, + // Functions: Auto-generated for BM25 when Sparse is set + Sparse: &milvus2.SparseVectorConfig{ + VectorField: sparseField, + MetricType: milvus2.BM25, + Method: milvus2.SparseMethodAuto, + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + + // Store documents - only dense vectors needed, sparse is generated by Milvus + ids, err := indexer.Store(ctx, []*schema.Document{ + {ID: "1", Content: "Milvus is a highly scalable vector database for AI applications."}, + {ID: "2", Content: "Eino framework simplifies building AI-powered applications in Go."}, + {ID: "3", Content: "Vector search enables semantic similarity matching."}, + }) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + + fmt.Printf("Stored %d documents: %v\n", len(ids), ids) + fmt.Println("Use the retriever hybrid example to search this collection with BM25.") +} + +// mockEmbedding for demo purposes +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + vecs := make([][]float64, len(texts)) + for i := range texts { + vecs[i] = make([]float64, m.dim) + vecs[i][0] = 0.1 + } + return vecs, nil +} diff --git a/components/indexer/milvus2/examples/hybrid_chinese/hybrid_chinese.go b/components/indexer/milvus2/examples/hybrid_chinese/hybrid_chinese.go new file mode 100644 index 000000000..699673ba8 --- /dev/null +++ b/components/indexer/milvus2/examples/hybrid_chinese/hybrid_chinese.go @@ -0,0 +1,108 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "log" + + "github.com/cloudwego/eino-ext/components/indexer/milvus2" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// This example demonstrates Hybrid Search indexing with Chinese text, creating a +// collection with both Dense vectors (semantic) and Sparse vectors (BM25 keyword). +// +// Uses Milvus's built-in Chinese analyzer (Jieba-based segmentation). +// +// Prerequisites: +// - Milvus 2.5+ (required for server-side functions) +// +// The collection created here can be used with the retriever hybrid_chinese example +// for Chinese text hybrid search. + +func main() { + ctx := context.Background() + collectionName := "eino_hybrid_chinese" + milvusAddr := "localhost:19530" + denseField := "vector" + sparseField := "sparse_vector" + + // Create indexer with Chinese analyzer + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: milvusAddr, + }, + Collection: collectionName, + Vector: &milvus2.VectorConfig{ + VectorField: denseField, + Dimension: 128, // Dense vector dimension + }, + + // BM25 requires analyzer on content field. + // Analyzer options (built-in): + // - {"type": "standard"} - general-purpose, tokenization + lowercase + // - {"type": "english"} - English with stopwords support + // - {"type": "chinese"} - Chinese with Jieba segmentation + // - Custom: {"tokenizer": "...", "filter": [...]} + // See: https://milvus.io/docs/analyzer-overview.md + FieldParams: map[string]map[string]string{ + "content": { + "enable_analyzer": "true", + "analyzer_params": `{"type": "chinese"}`, // Use Chinese analyzer + }, + }, + // Functions: Auto-generated for BM25 when Sparse is set + Sparse: &milvus2.SparseVectorConfig{ + VectorField: sparseField, + MetricType: milvus2.BM25, + Method: milvus2.SparseMethodAuto, + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + + // Store Chinese documents - only dense vectors needed, sparse is generated by Milvus + ids, err := indexer.Store(ctx, []*schema.Document{ + {ID: "1", Content: "Milvus 是一个高度可扩展的向量数据库，用于人工智能应用。"}, + {ID: "2", Content: "Eino 框架简化了在 Go 中构建 AI 驱动应用程序的过程。"}, + {ID: "3", Content: "向量搜索支持语义相似度匹配。"}, + }) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + + fmt.Printf("Stored %d Chinese documents: %v\n", len(ids), ids) + fmt.Println("Use the retriever hybrid_chinese example to search this collection.") +} + +// mockEmbedding for demo purposes +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + vecs := make([][]float64, len(texts)) + for i := range texts { + vecs[i] = make([]float64, m.dim) + vecs[i][0] = 0.1 + } + return vecs, nil +} diff --git a/components/indexer/milvus2/examples/ivf_flat/ivf_flat.go b/components/indexer/milvus2/examples/ivf_flat/ivf_flat.go new file mode 100644 index 000000000..60d7d1110 --- /dev/null +++ b/components/indexer/milvus2/examples/ivf_flat/ivf_flat.go @@ -0,0 +1,86 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates using an IVF_FLAT index for vector storage. +// IVF_FLAT (Inverted File with Flat) partitions data into clusters +// for faster search with good accuracy. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create an indexer with IVF_FLAT index + // NList: number of clusters (higher = better recall but slower) + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_ivf_flat", + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: 128, + MetricType: milvus2.L2, // Euclidean distance + IndexBuilder: milvus2.NewIVFFlatIndexBuilder(). + WithNList(128), + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + log.Println("IVF_FLAT indexer created successfully") + + // Store sample documents + docs := []*schema.Document{ + {ID: "ivf-1", Content: "IVF_FLAT uses cluster-based search"}, + {ID: "ivf-2", Content: "Quantization-free storage with exact distances"}, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + log.Printf("Stored documents: %v", ids) +} + +// mockEmbedding generates random embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(i+j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/rabitq/rabitq.go b/components/indexer/milvus2/examples/rabitq/rabitq.go new file mode 100644 index 000000000..a7d1c03c3 --- /dev/null +++ b/components/indexer/milvus2/examples/rabitq/rabitq.go @@ -0,0 +1,85 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates using an IVF_RABITQ index for vector storage. +// IVF_RABITQ combines IVF clustering with RaBitQ binary quantization +// for excellent storage efficiency. Requires Milvus 2.6+. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create an indexer with IVF_RABITQ index + // NList: number of clusters (recommended: 32-4096) + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_rabitq", + Vector: &milvus2.VectorConfig{ + VectorField: "vector", + Dimension: 128, + MetricType: milvus2.L2, + IndexBuilder: milvus2.NewIVFRabitQIndexBuilder().WithNList(128), + }, + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + log.Println("IVF_RABITQ indexer created successfully") + + // Store sample documents + docs := []*schema.Document{ + {ID: "rabitq-1", Content: "IVF_RABITQ provides excellent storage efficiency"}, + {ID: "rabitq-2", Content: "Binary quantization for compact vector representation"}, + } + ids, err := indexer.Store(ctx, docs) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + log.Printf("Stored documents: %v", ids) +} + +// mockEmbedding generates deterministic embeddings for demonstration. +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(i+j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/indexer/milvus2/examples/sparse/sparse.go b/components/indexer/milvus2/examples/sparse/sparse.go new file mode 100644 index 000000000..33d455300 --- /dev/null +++ b/components/indexer/milvus2/examples/sparse/sparse.go @@ -0,0 +1,91 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "log" + + "github.com/cloudwego/eino-ext/components/indexer/milvus2" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// This example demonstrates Sparse-only indexing using BM25. +// Milvus will automatically generate sparse vectors from the text content. +// No dense vector configuration is provided. +// +// Prerequisites: +// - Milvus 2.5+ (required for server-side functions) + +func main() { + ctx := context.Background() + collectionName := "eino_sparse_test" + milvusAddr := "localhost:19530" + sparseField := "sparse_vector" + + // Create indexer with BM25 function + indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: milvusAddr, + }, + Collection: collectionName, + // Vector: nil, // Explicitly nil implies no dense vector + + // BM25 requires analyzer on content field. + FieldParams: map[string]map[string]string{ + "content": { + "enable_analyzer": "true", + "analyzer_params": `{"type": "standard"}`, + }, + }, + // Functions: Auto-generated for BM25 when Sparse is set + Sparse: &milvus2.SparseVectorConfig{ + VectorField: sparseField, + MetricType: milvus2.BM25, + Method: milvus2.SparseMethodAuto, + }, + // Embedding is required by interface but not used if Vector is nil + Embedding: &mockEmbedding{}, + }) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + + // Store documents + ids, err := indexer.Store(ctx, []*schema.Document{ + {ID: "1", Content: "Milvus is a vector database."}, + {ID: "2", Content: "BM25 is a ranking function used in information retrieval."}, + {ID: "3", Content: "Sparse vectors are good for keyword search."}, + }) + if err != nil { + log.Fatalf("Failed to store: %v", err) + } + + fmt.Printf("Stored %d documents: %v\n", len(ids), ids) + fmt.Println("Use the retriever sparse example to search this collection.") +} + +// mockEmbedding for demo purposes (satisfies interface only) +type mockEmbedding struct{} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + // Not used for sparse-only indexing + return make([][]float64, len(texts)), nil +} diff --git a/components/indexer/milvus2/go.mod b/components/indexer/milvus2/go.mod new file mode 100644 index 000000000..6ae7ca547 --- /dev/null +++ b/components/indexer/milvus2/go.mod @@ -0,0 +1,142 @@ +module github.com/cloudwego/eino-ext/components/indexer/milvus2 + +go 1.24.6 + +require ( + github.com/bytedance/mockey v1.4.0 + github.com/bytedance/sonic v1.14.1 + github.com/cloudwego/eino v0.6.0 + github.com/milvus-io/milvus/client/v2 v2.6.1 + github.com/smartystreets/goconvey v1.8.1 +) + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.11.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect + github.com/cockroachdb/redact v1.1.3 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.2 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/getsentry/sentry-go v0.12.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3 // indirect + github.com/milvus-io/milvus/pkg/v2 v2.6.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/panjf2000/ants/v2 v2.11.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/samber/lo v1.27.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.etcd.io/bbolt v1.3.8 // indirect + go.etcd.io/etcd/api/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/v2 v2.305.10 // indirect + go.etcd.io/etcd/client/v3 v3.5.10 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/raft/v3 v3.5.10 // indirect + go.etcd.io/etcd/server/v3 v3.5.10 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.32.3 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/components/indexer/milvus2/go.sum b/components/indexer/milvus2/go.sum new file mode 100644 index 000000000..1314946b2 --- /dev/null +++ b/components/indexer/milvus2/go.sum @@ -0,0 +1,679 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.4.0 h1:xwuZ3rr4mpbGkkBOYoSM+cO112dvzQ/sY0cVdP9FBSA= +github.com/bytedance/mockey v1.4.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= +github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.6.0 h1:pobGKMOfcQHVNhD9UT/HrvO0eYG6FC2ML/NKY2Eb9+Q= +github.com/cloudwego/eino v0.6.0/go.mod h1:JNapfU+QUrFFpboNDrNOFvmz0m9wjBFHHCr77RH6a50= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eino-contrib/jsonschema v1.0.2 h1:HaxruBMUdnXa7Lg/lX8g0Hk71ZIfdTZXmBQz0e3esr8= +github.com/eino-contrib/jsonschema v1.0.2/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3 h1:w7IBrU25KULWNlHKoKwx6ruTsDAmzrWknotIc6A4ys4= +github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus/client/v2 v2.6.1 h1:JGV+2JoZypc0ORnVj41ZWLdz9EpBGcwXCliIFXFW1f4= +github.com/milvus-io/milvus/client/v2 v2.6.1/go.mod h1:MnickP646pUKhfOS4JQD3uMUukDXhJKpdTXk467MXuU= +github.com/milvus-io/milvus/pkg/v2 v2.6.3 h1:WDf4mXFWL5Sk/V87yLwRKq24MYMkjS2YA6qraXbLbJA= +github.com/milvus-io/milvus/pkg/v2 v2.6.3/go.mod h1:49umaGHK9nKHJNtgBlF/iB24s1sZ/SG5/Q7iLj/Gc14= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= +github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= +github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ= +github.com/samber/lo v1.27.0/go.mod h1:it33p9UtPMS7z72fP4gw/EIfQB2eI8ke7GR2wc6+Rhg= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4= +go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.etcd.io/etcd/pkg/v3 v3.5.10 h1:WPR8K0e9kWl1gAhB5A7gEa5ZBTNkT9NdNWrR8Qpo1CM= +go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs= +go.etcd.io/etcd/raft/v3 v3.5.10 h1:cgNAYe7xrsrn/5kXMSaH8kM/Ky8mAdMqGOxyYwpP0LA= +go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc= +go.etcd.io/etcd/server/v3 v3.5.10 h1:4NOGyOwD5sUZ22PiWYKmfxqoeh72z6EhYjNosKGLmZg= +go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= +golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e h1:YA5lmSs3zc/5w+xsRcHqpETkaYyK63ivEPzNTcUUlSA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/components/indexer/milvus2/index_builder.go b/components/indexer/milvus2/index_builder.go new file mode 100644 index 000000000..25034a638 --- /dev/null +++ b/components/indexer/milvus2/index_builder.go @@ -0,0 +1,452 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "github.com/milvus-io/milvus/client/v2/index" +) + +// IndexBuilder defines the interface for building Milvus vector indexes. +// It provides different index types with their specific parameters. +type IndexBuilder interface { + // Build creates a Milvus index configured with the specified metric type. + Build(metricType MetricType) index.Index +} + +// SparseIndexBuilder defines the interface for building Milvus sparse vector indexes. +type SparseIndexBuilder interface { + // Build creates a Milvus sparse index configured with the specified metric type. + Build(metricType MetricType) index.Index +} + +// AutoIndexBuilder creates an AUTOINDEX that allows Milvus to automatically +// select the optimal index type based on data characteristics. +type AutoIndexBuilder struct{} + +// NewAutoIndexBuilder creates a new AutoIndexBuilder instance. +func NewAutoIndexBuilder() *AutoIndexBuilder { + return &AutoIndexBuilder{} +} + +// Build creates an AUTOINDEX using the specified metric type. +func (b *AutoIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewAutoIndex(metricType.toEntity()) +} + +// FlatIndexBuilder creates a FLAT index for brute-force exact search. +// It guarantees 100% recall accuracy but is slowest for large datasets. +type FlatIndexBuilder struct{} + +// NewFlatIndexBuilder creates a new FlatIndexBuilder instance. +func NewFlatIndexBuilder() *FlatIndexBuilder { + return &FlatIndexBuilder{} +} + +// Build creates a FLAT index using the specified metric type. +func (b *FlatIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewFlatIndex(metricType.toEntity()) +} + +// HNSWIndexBuilder creates an HNSW (Hierarchical Navigable Small World) index. +// It provides a graph-based index with excellent query performance. +type HNSWIndexBuilder struct { + // M specifies the maximum degree of nodes in graph layers (recommended: 4-64). + M int + // EfConstruction specifies the search width during index building (recommended: 8-512). + EfConstruction int +} + +// NewHNSWIndexBuilder creates a new HNSWIndexBuilder with default parameters. +// Default: M=16, EfConstruction=200 +func NewHNSWIndexBuilder() *HNSWIndexBuilder { + return &HNSWIndexBuilder{ + M: 16, + EfConstruction: 200, + } +} + +// WithM sets the maximum degree of nodes in graph layers. +func (b *HNSWIndexBuilder) WithM(m int) *HNSWIndexBuilder { + b.M = m + return b +} + +// WithEfConstruction sets the search width during index building. +func (b *HNSWIndexBuilder) WithEfConstruction(ef int) *HNSWIndexBuilder { + b.EfConstruction = ef + return b +} + +// Build creates an HNSW index using the specified metric type. +func (b *HNSWIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewHNSWIndex(metricType.toEntity(), b.M, b.EfConstruction) +} + +// IVFFlatIndexBuilder creates an IVF_FLAT (Inverted File with Flat) index. +// It partitions data into clusters for faster search. +type IVFFlatIndexBuilder struct { + // NList is the number of cluster units (recommended: 1-65536). + NList int +} + +// NewIVFFlatIndexBuilder creates a new IVFFlatIndexBuilder with default parameters. +// Default: NList=128 +func NewIVFFlatIndexBuilder() *IVFFlatIndexBuilder { + return &IVFFlatIndexBuilder{ + NList: 128, + } +} + +// WithNList sets the NList parameter. +func (b *IVFFlatIndexBuilder) WithNList(nlist int) *IVFFlatIndexBuilder { + b.NList = nlist + return b +} + +// Build creates an IVF_FLAT index. +func (b *IVFFlatIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewIvfFlatIndex(metricType.toEntity(), b.NList) +} + +// IVFPQIndexBuilder creates an IVF_PQ (Inverted File with Product Quantization) index. +// It provides significant memory savings with good search quality. +type IVFPQIndexBuilder struct { + // NList is the number of cluster units (recommended: 1-65536). + NList int + // M is the number of subquantizers (recommended: 1-div). + M int + // NBits is the number of bits for each subquantizer (recommended: 1-16). + NBits int +} + +// NewIVFPQIndexBuilder creates a new IVFPQIndexBuilder with default parameters. +// Default: NList=128, M=16, NBits=8 +func NewIVFPQIndexBuilder() *IVFPQIndexBuilder { + return &IVFPQIndexBuilder{ + NList: 128, + M: 16, + NBits: 8, + } +} + +// WithNList sets the NList parameter. +func (b *IVFPQIndexBuilder) WithNList(nlist int) *IVFPQIndexBuilder { + b.NList = nlist + return b +} + +// WithM sets the M parameter. +func (b *IVFPQIndexBuilder) WithM(m int) *IVFPQIndexBuilder { + b.M = m + return b +} + +// WithNBits sets the NBits parameter. +func (b *IVFPQIndexBuilder) WithNBits(nbits int) *IVFPQIndexBuilder { + b.NBits = nbits + return b +} + +// Build creates an IVF_PQ index. +func (b *IVFPQIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewIvfPQIndex(metricType.toEntity(), b.NList, b.M, b.NBits) +} + +// IVFSQ8IndexBuilder creates an IVF_SQ8 (Inverted File with Scalar Quantization) index. +// It uses scalar quantization for memory efficiency. +type IVFSQ8IndexBuilder struct { + // NList is the number of cluster units (recommended: 1-65536). + NList int +} + +// NewIVFSQ8IndexBuilder creates a new IVFSQ8IndexBuilder with default parameters. +// Default: NList=128 +func NewIVFSQ8IndexBuilder() *IVFSQ8IndexBuilder { + return &IVFSQ8IndexBuilder{ + NList: 128, + } +} + +// WithNList sets the NList parameter. +func (b *IVFSQ8IndexBuilder) WithNList(nlist int) *IVFSQ8IndexBuilder { + b.NList = nlist + return b +} + +// Build creates an IVF_SQ8 index. +func (b *IVFSQ8IndexBuilder) Build(metricType MetricType) index.Index { + return index.NewIvfSQ8Index(metricType.toEntity(), b.NList) +} + +// DiskANNIndexBuilder creates a DISKANN index for disk-based ANN search. +// It is optimized for large datasets that don't fit in memory. +type DiskANNIndexBuilder struct{} + +// NewDiskANNIndexBuilder creates a new DiskANNIndexBuilder. +func NewDiskANNIndexBuilder() *DiskANNIndexBuilder { + return &DiskANNIndexBuilder{} +} + +// Build creates a DISKANN index. +func (b *DiskANNIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewDiskANNIndex(metricType.toEntity()) +} + +// SCANNIndexBuilder creates a SCANN (Scalable Approximate Nearest Neighbors) index. +type SCANNIndexBuilder struct { + // NList is the number of cluster units. + NList int + // WithRawData determines whether to include raw data for reranking. + WithRawData bool +} + +// NewSCANNIndexBuilder creates a new SCANNIndexBuilder with default parameters. +// Default: NList=128, WithRawData=true +func NewSCANNIndexBuilder() *SCANNIndexBuilder { + return &SCANNIndexBuilder{ + NList: 128, + WithRawData: true, + } +} + +// WithNList sets the NList parameter. +func (b *SCANNIndexBuilder) WithNList(nlist int) *SCANNIndexBuilder { + b.NList = nlist + return b +} + +// WithRawDataEnabled sets whether to include raw data for reranking. +func (b *SCANNIndexBuilder) WithRawDataEnabled(enabled bool) *SCANNIndexBuilder { + b.WithRawData = enabled + return b +} + +// Build creates a SCANN index. +func (b *SCANNIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewSCANNIndex(metricType.toEntity(), b.NList, b.WithRawData) +} + +// BinFlatIndexBuilder creates a BIN_FLAT index for binary vectors. +type BinFlatIndexBuilder struct{} + +// NewBinFlatIndexBuilder creates a new BinFlatIndexBuilder. +func NewBinFlatIndexBuilder() *BinFlatIndexBuilder { + return &BinFlatIndexBuilder{} +} + +// Build creates a BIN_FLAT index. +func (b *BinFlatIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewBinFlatIndex(metricType.toEntity()) +} + +// BinIVFFlatIndexBuilder creates a BIN_IVF_FLAT index for binary vectors. +type BinIVFFlatIndexBuilder struct { + // NList is the number of cluster units. + NList int +} + +// NewBinIVFFlatIndexBuilder creates a new BinIVFFlatIndexBuilder with default parameters. +// Default: NList=128 +func NewBinIVFFlatIndexBuilder() *BinIVFFlatIndexBuilder { + return &BinIVFFlatIndexBuilder{ + NList: 128, + } +} + +// WithNList sets the NList parameter. +func (b *BinIVFFlatIndexBuilder) WithNList(nlist int) *BinIVFFlatIndexBuilder { + b.NList = nlist + return b +} + +// Build creates a BIN_IVF_FLAT index. +func (b *BinIVFFlatIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewBinIvfFlatIndex(metricType.toEntity(), b.NList) +} + +// GPUBruteForceIndexBuilder creates a GPU-accelerated brute-force index. +type GPUBruteForceIndexBuilder struct{} + +// NewGPUBruteForceIndexBuilder creates a new GPUBruteForceIndexBuilder. +func NewGPUBruteForceIndexBuilder() *GPUBruteForceIndexBuilder { + return &GPUBruteForceIndexBuilder{} +} + +// Build creates a GPU brute-force index. +func (b *GPUBruteForceIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewGPUBruteForceIndex(metricType.toEntity()) +} + +// GPUIVFFlatIndexBuilder creates a GPU-accelerated IVF_FLAT index. +type GPUIVFFlatIndexBuilder struct{} + +// NewGPUIVFFlatIndexBuilder creates a new GPUIVFFlatIndexBuilder. +func NewGPUIVFFlatIndexBuilder() *GPUIVFFlatIndexBuilder { + return &GPUIVFFlatIndexBuilder{} +} + +// Build creates a GPU IVF_FLAT index. +func (b *GPUIVFFlatIndexBuilder) Build(metricType MetricType) index.Index { + // NewGPUIVPFlatIndex is a typo in Milvus SDK (should be NewGPUIVFFlatIndex). + // We keep using it until SDK fixes it. + return index.NewGPUIVPFlatIndex(metricType.toEntity()) +} + +// GPUIVFPQIndexBuilder creates a GPU-accelerated IVF_PQ index. +// It provides high-speed approximate nearest neighbor search with memory efficiency. +// Note: The current Milvus SDK implementation does not expose NList, M, NBits parameters. +type GPUIVFPQIndexBuilder struct{} + +// NewGPUIVFPQIndexBuilder creates a new GPUIVFPQIndexBuilder. +func NewGPUIVFPQIndexBuilder() *GPUIVFPQIndexBuilder { + return &GPUIVFPQIndexBuilder{} +} + +// Build creates a GPU IVF_PQ index. +func (b *GPUIVFPQIndexBuilder) Build(metricType MetricType) index.Index { + // NewGPUIVPPQIndex is a typo in Milvus SDK (should be NewGPUIVFPQIndex). + // We keep using it until SDK fixes it. + return index.NewGPUIVPPQIndex(metricType.toEntity()) +} + +// GPUCagraIndexBuilder creates a GPU CAGRA index. +type GPUCagraIndexBuilder struct { + // IntermediateGraphDegree is the degree of the intermediate graph. + IntermediateGraphDegree int + // GraphDegree is the degree of the final graph. + GraphDegree int +} + +// NewGPUCagraIndexBuilder creates a new GPUCagraIndexBuilder with default parameters. +// Default: IntermediateGraphDegree=128, GraphDegree=64 +func NewGPUCagraIndexBuilder() *GPUCagraIndexBuilder { + return &GPUCagraIndexBuilder{ + IntermediateGraphDegree: 128, + GraphDegree: 64, + } +} + +// WithIntermediateGraphDegree sets the intermediate graph degree. +func (b *GPUCagraIndexBuilder) WithIntermediateGraphDegree(degree int) *GPUCagraIndexBuilder { + b.IntermediateGraphDegree = degree + return b +} + +// WithGraphDegree sets the final graph degree. +func (b *GPUCagraIndexBuilder) WithGraphDegree(degree int) *GPUCagraIndexBuilder { + b.GraphDegree = degree + return b +} + +// Build creates a GPU CAGRA index. +func (b *GPUCagraIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewGPUCagraIndex(metricType.toEntity(), b.IntermediateGraphDegree, b.GraphDegree) +} + +// IVFRabitQIndexBuilder creates an IVF_RABITQ index (Milvus 2.6+). +// It combines IVF clustering with RaBitQ binary quantization for storage efficiency. +type IVFRabitQIndexBuilder struct { + // NList is the number of cluster units (recommended: 32-4096). + NList int +} + +// NewIVFRabitQIndexBuilder creates a new IVFRabitQIndexBuilder with default parameters. +// Default: NList=128 +func NewIVFRabitQIndexBuilder() *IVFRabitQIndexBuilder { + return &IVFRabitQIndexBuilder{ + NList: 128, + } +} + +// WithNList sets the number of cluster units. +func (b *IVFRabitQIndexBuilder) WithNList(nlist int) *IVFRabitQIndexBuilder { + b.NList = nlist + return b +} + +// Build creates an IVF_RABITQ index. +func (b *IVFRabitQIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewIvfRabitQIndex(metricType.toEntity(), b.NList) +} + +// SparseInvertedIndexBuilder creates a SPARSE_INVERTED_INDEX. +type SparseInvertedIndexBuilder struct { + DropRatioBuild float64 +} + +// NewSparseInvertedIndexBuilder creates a new SparseInvertedIndexBuilder. +func NewSparseInvertedIndexBuilder() *SparseInvertedIndexBuilder { + return &SparseInvertedIndexBuilder{ + DropRatioBuild: 0.2, + } +} + +// WithDropRatioBuild sets the drop ratio for building the index. +func (b *SparseInvertedIndexBuilder) WithDropRatioBuild(ratio float64) *SparseInvertedIndexBuilder { + b.DropRatioBuild = ratio + return b +} + +// Build creates a SPARSE_INVERTED_INDEX. +func (b *SparseInvertedIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewSparseInvertedIndex(metricType.toEntity(), b.DropRatioBuild) +} + +// SparseWANDIndexBuilder creates a SPARSE_WAND index. +type SparseWANDIndexBuilder struct { + DropRatioBuild float64 +} + +// NewSparseWANDIndexBuilder creates a new SparseWANDIndexBuilder. +func NewSparseWANDIndexBuilder() *SparseWANDIndexBuilder { + return &SparseWANDIndexBuilder{ + DropRatioBuild: 0.2, + } +} + +// WithDropRatioBuild sets the drop ratio for building the index. +func (b *SparseWANDIndexBuilder) WithDropRatioBuild(ratio float64) *SparseWANDIndexBuilder { + b.DropRatioBuild = ratio + return b +} + +// Build creates a SPARSE_WAND index. +func (b *SparseWANDIndexBuilder) Build(metricType MetricType) index.Index { + return index.NewSparseWANDIndex(metricType.toEntity(), b.DropRatioBuild) +} + +// Ensure all builders implement IndexBuilder or SparseIndexBuilder +var ( + _ IndexBuilder = (*AutoIndexBuilder)(nil) + _ IndexBuilder = (*FlatIndexBuilder)(nil) + _ IndexBuilder = (*HNSWIndexBuilder)(nil) + _ IndexBuilder = (*IVFFlatIndexBuilder)(nil) + _ IndexBuilder = (*IVFPQIndexBuilder)(nil) + _ IndexBuilder = (*IVFSQ8IndexBuilder)(nil) + _ IndexBuilder = (*DiskANNIndexBuilder)(nil) + _ IndexBuilder = (*SCANNIndexBuilder)(nil) + _ IndexBuilder = (*BinFlatIndexBuilder)(nil) + _ IndexBuilder = (*BinIVFFlatIndexBuilder)(nil) + _ IndexBuilder = (*GPUBruteForceIndexBuilder)(nil) + _ IndexBuilder = (*GPUIVFFlatIndexBuilder)(nil) + _ IndexBuilder = (*GPUCagraIndexBuilder)(nil) + _ IndexBuilder = (*GPUIVFPQIndexBuilder)(nil) + _ IndexBuilder = (*IVFRabitQIndexBuilder)(nil) + + _ SparseIndexBuilder = (*SparseInvertedIndexBuilder)(nil) + _ SparseIndexBuilder = (*SparseWANDIndexBuilder)(nil) +) diff --git a/components/indexer/milvus2/index_builder_test.go b/components/indexer/milvus2/index_builder_test.go new file mode 100644 index 000000000..0b2f4500c --- /dev/null +++ b/components/indexer/milvus2/index_builder_test.go @@ -0,0 +1,369 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestAutoIndexBuilder(t *testing.T) { + convey.Convey("test AutoIndexBuilder", t, func() { + convey.Convey("test NewAutoIndexBuilder", func() { + builder := NewAutoIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with L2", func() { + builder := NewAutoIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with IP", func() { + builder := NewAutoIndexBuilder() + idx := builder.Build(IP) + convey.So(idx, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with COSINE", func() { + builder := NewAutoIndexBuilder() + idx := builder.Build(COSINE) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestFlatIndexBuilder(t *testing.T) { + convey.Convey("test FlatIndexBuilder", t, func() { + convey.Convey("test NewFlatIndexBuilder", func() { + builder := NewFlatIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with L2", func() { + builder := NewFlatIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with IP", func() { + builder := NewFlatIndexBuilder() + idx := builder.Build(IP) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestHNSWIndexBuilder(t *testing.T) { + convey.Convey("test HNSWIndexBuilder", t, func() { + convey.Convey("test NewHNSWIndexBuilder default values", func() { + builder := NewHNSWIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.M, convey.ShouldEqual, 16) + convey.So(builder.EfConstruction, convey.ShouldEqual, 200) + }) + + convey.Convey("test WithM", func() { + builder := NewHNSWIndexBuilder().WithM(32) + convey.So(builder.M, convey.ShouldEqual, 32) + }) + + convey.Convey("test WithEfConstruction", func() { + builder := NewHNSWIndexBuilder().WithEfConstruction(500) + convey.So(builder.EfConstruction, convey.ShouldEqual, 500) + }) + + convey.Convey("test chained methods", func() { + builder := NewHNSWIndexBuilder().WithM(8).WithEfConstruction(100) + convey.So(builder.M, convey.ShouldEqual, 8) + convey.So(builder.EfConstruction, convey.ShouldEqual, 100) + }) + + convey.Convey("test Build", func() { + builder := NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200) + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestIVFFlatIndexBuilder(t *testing.T) { + convey.Convey("test IVFFlatIndexBuilder", t, func() { + convey.Convey("test NewIVFFlatIndexBuilder default values", func() { + builder := NewIVFFlatIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + }) + + convey.Convey("test WithNList", func() { + builder := NewIVFFlatIndexBuilder().WithNList(256) + convey.So(builder.NList, convey.ShouldEqual, 256) + }) + + convey.Convey("test Build", func() { + builder := NewIVFFlatIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestIVFPQIndexBuilder(t *testing.T) { + convey.Convey("test IVFPQIndexBuilder", t, func() { + convey.Convey("test NewIVFPQIndexBuilder default values", func() { + builder := NewIVFPQIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + convey.So(builder.M, convey.ShouldEqual, 16) + convey.So(builder.NBits, convey.ShouldEqual, 8) + }) + + convey.Convey("test WithNList", func() { + builder := NewIVFPQIndexBuilder().WithNList(256) + convey.So(builder.NList, convey.ShouldEqual, 256) + }) + + convey.Convey("test WithM", func() { + builder := NewIVFPQIndexBuilder().WithM(32) + convey.So(builder.M, convey.ShouldEqual, 32) + }) + + convey.Convey("test WithNBits", func() { + builder := NewIVFPQIndexBuilder().WithNBits(4) + convey.So(builder.NBits, convey.ShouldEqual, 4) + }) + + convey.Convey("test chained methods", func() { + builder := NewIVFPQIndexBuilder().WithNList(64).WithM(8).WithNBits(16) + convey.So(builder.NList, convey.ShouldEqual, 64) + convey.So(builder.M, convey.ShouldEqual, 8) + convey.So(builder.NBits, convey.ShouldEqual, 16) + }) + + convey.Convey("test Build", func() { + builder := NewIVFPQIndexBuilder() + idx := builder.Build(IP) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestIVFSQ8IndexBuilder(t *testing.T) { + convey.Convey("test IVFSQ8IndexBuilder", t, func() { + convey.Convey("test NewIVFSQ8IndexBuilder default values", func() { + builder := NewIVFSQ8IndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + }) + + convey.Convey("test WithNList", func() { + builder := NewIVFSQ8IndexBuilder().WithNList(64) + convey.So(builder.NList, convey.ShouldEqual, 64) + }) + + convey.Convey("test Build", func() { + builder := NewIVFSQ8IndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestDiskANNIndexBuilder(t *testing.T) { + convey.Convey("test DiskANNIndexBuilder", t, func() { + convey.Convey("test NewDiskANNIndexBuilder", func() { + builder := NewDiskANNIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build", func() { + builder := NewDiskANNIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestSCANNIndexBuilder(t *testing.T) { + convey.Convey("test SCANNIndexBuilder", t, func() { + convey.Convey("test NewSCANNIndexBuilder default values", func() { + builder := NewSCANNIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + convey.So(builder.WithRawData, convey.ShouldBeTrue) + }) + + convey.Convey("test WithNList", func() { + builder := NewSCANNIndexBuilder().WithNList(256) + convey.So(builder.NList, convey.ShouldEqual, 256) + }) + + convey.Convey("test WithRawDataEnabled", func() { + builder := NewSCANNIndexBuilder().WithRawDataEnabled(false) + convey.So(builder.WithRawData, convey.ShouldBeFalse) + }) + + convey.Convey("test Build", func() { + builder := NewSCANNIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestBinFlatIndexBuilder(t *testing.T) { + convey.Convey("test BinFlatIndexBuilder", t, func() { + convey.Convey("test NewBinFlatIndexBuilder", func() { + builder := NewBinFlatIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with HAMMING", func() { + builder := NewBinFlatIndexBuilder() + idx := builder.Build(HAMMING) + convey.So(idx, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build with JACCARD", func() { + builder := NewBinFlatIndexBuilder() + idx := builder.Build(JACCARD) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestBinIVFFlatIndexBuilder(t *testing.T) { + convey.Convey("test BinIVFFlatIndexBuilder", t, func() { + convey.Convey("test NewBinIVFFlatIndexBuilder default values", func() { + builder := NewBinIVFFlatIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + }) + + convey.Convey("test WithNList", func() { + builder := NewBinIVFFlatIndexBuilder().WithNList(64) + convey.So(builder.NList, convey.ShouldEqual, 64) + }) + + convey.Convey("test Build", func() { + builder := NewBinIVFFlatIndexBuilder() + idx := builder.Build(HAMMING) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestGPUBruteForceIndexBuilder(t *testing.T) { + convey.Convey("test GPUBruteForceIndexBuilder", t, func() { + convey.Convey("test NewGPUBruteForceIndexBuilder", func() { + builder := NewGPUBruteForceIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build", func() { + builder := NewGPUBruteForceIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestGPUIVFFlatIndexBuilder(t *testing.T) { + convey.Convey("test GPUIVFFlatIndexBuilder", t, func() { + convey.Convey("test NewGPUIVFFlatIndexBuilder", func() { + builder := NewGPUIVFFlatIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build", func() { + builder := NewGPUIVFFlatIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestGPUIVFPQIndexBuilder(t *testing.T) { + convey.Convey("test GPUIVFPQIndexBuilder", t, func() { + convey.Convey("test NewGPUIVFPQIndexBuilder", func() { + builder := NewGPUIVFPQIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + }) + + convey.Convey("test Build", func() { + builder := NewGPUIVFPQIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestGPUCagraIndexBuilder(t *testing.T) { + convey.Convey("test GPUCagraIndexBuilder", t, func() { + convey.Convey("test NewGPUCagraIndexBuilder default values", func() { + builder := NewGPUCagraIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.IntermediateGraphDegree, convey.ShouldEqual, 128) + convey.So(builder.GraphDegree, convey.ShouldEqual, 64) + }) + + convey.Convey("test WithIntermediateGraphDegree", func() { + builder := NewGPUCagraIndexBuilder().WithIntermediateGraphDegree(256) + convey.So(builder.IntermediateGraphDegree, convey.ShouldEqual, 256) + }) + + convey.Convey("test WithGraphDegree", func() { + builder := NewGPUCagraIndexBuilder().WithGraphDegree(32) + convey.So(builder.GraphDegree, convey.ShouldEqual, 32) + }) + + convey.Convey("test chained methods", func() { + builder := NewGPUCagraIndexBuilder().WithIntermediateGraphDegree(64).WithGraphDegree(16) + convey.So(builder.IntermediateGraphDegree, convey.ShouldEqual, 64) + convey.So(builder.GraphDegree, convey.ShouldEqual, 16) + }) + + convey.Convey("test Build", func() { + builder := NewGPUCagraIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} + +func TestIVFRabitQIndexBuilder(t *testing.T) { + convey.Convey("test IVFRabitQIndexBuilder", t, func() { + convey.Convey("test NewIVFRabitQIndexBuilder default values", func() { + builder := NewIVFRabitQIndexBuilder() + convey.So(builder, convey.ShouldNotBeNil) + convey.So(builder.NList, convey.ShouldEqual, 128) + }) + + convey.Convey("test WithNList", func() { + builder := NewIVFRabitQIndexBuilder().WithNList(256) + convey.So(builder.NList, convey.ShouldEqual, 256) + }) + + convey.Convey("test Build", func() { + builder := NewIVFRabitQIndexBuilder() + idx := builder.Build(L2) + convey.So(idx, convey.ShouldNotBeNil) + }) + }) +} diff --git a/components/indexer/milvus2/indexer.go b/components/indexer/milvus2/indexer.go new file mode 100644 index 000000000..2871602c1 --- /dev/null +++ b/components/indexer/milvus2/indexer.go @@ -0,0 +1,687 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "context" + "fmt" + "log" + "sort" + + "github.com/bytedance/sonic" + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/indexer" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/column" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/index" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// IndexerConfig contains configuration for the Milvus2 indexer. +type IndexerConfig struct { + // Client is an optional pre-configured Milvus client. + // If not provided, the component will create one using ClientConfig. + Client *milvusclient.Client + + // ClientConfig for creating Milvus client if Client is not provided. + ClientConfig *milvusclient.ClientConfig + + // Collection is the collection name in Milvus. + // Default: "eino_collection" + Collection string + + // Description is the description for the collection. + // Default: "the collection for eino" + Description string + + // PartitionName is the default partition for insertion. + // Optional. + PartitionName string + + // ConsistencyLevel for Milvus operations. + // Default: ConsistencyLevelBounded + ConsistencyLevel ConsistencyLevel + + // EnableDynamicSchema enables dynamic field support for flexible metadata. + // Default: false + EnableDynamicSchema bool + + // Vector defines the configuration for dense vector index. + // Optional. + Vector *VectorConfig + + // Sparse defines the configuration for sparse vector index. + // Optional. + Sparse *SparseVectorConfig + + // DocumentConverter converts EINO documents to Milvus columns. + // If nil, uses default conversion (id, content, vector, metadata as JSON). + DocumentConverter func(ctx context.Context, docs []*schema.Document, vectors [][]float64) ([]column.Column, error) + + // Embedding is the embedder for vectorization. + // Required. + Embedding embedding.Embedder + + // Functions defines the Milvus built-in functions (e.g. BM25) to be added to the schema. + // Optional. + Functions []*entity.Function + + // FieldParams defines extra parameters for fields (e.g. "enable_analyzer": "true"). + // Key is field name, value is a map of parameter key-value pairs. + // Optional. + FieldParams map[string]map[string]string +} + +// VectorConfig contains configuration for dense vector index. +type VectorConfig struct { + // Dimension is the vector dimension. + // Required when auto-creating the collection. + Dimension int64 + + // MetricType is the metric type for vector similarity. + // Default: L2 + MetricType MetricType + + // IndexBuilder specifies how to build the vector index. + // If nil, uses AutoIndex (Milvus automatically selects the best index). + // Use NewHNSWIndexBuilder(), NewIVFFlatIndexBuilder(), etc. for specific index types. + IndexBuilder IndexBuilder + + // VectorField is the name of the vector field in the collection. + // Default: "vector" + VectorField string +} + +// SparseMethod defines the method for sparse vector generation. +type SparseMethod string + +const ( + // SparseMethodAuto indicates that Milvus will automatically generate sparse vectors + // (e.g. via built-in BM25 function) on the server side. + SparseMethodAuto SparseMethod = "Auto" + // SparseMethodPrecomputed indicates that the client will provide precomputed sparse vectors + // (e.g. from document.SparseVector()). + SparseMethodPrecomputed SparseMethod = "Precomputed" +) + +// SparseVectorConfig contains configuration for sparse vector index. +type SparseVectorConfig struct { + // IndexBuilder specifies how to build the sparse vector index. + // Optional. If nil, uses SPARSE_INVERTED_INDEX by default. + IndexBuilder SparseIndexBuilder + + // VectorField is the name of the sparse vector field in the collection. + // Optional. Default: "sparse_vector" + VectorField string + + // MetricType is the metric type for sparse vector similarity. + // Optional. Default: BM25. + MetricType MetricType + + // Method specifies the method for sparse vector generation. + // Optional. Default: SparseMethodAuto if MetricType is BM25, otherwise SparseMethodPrecomputed. + Method SparseMethod +} + +// Indexer implements the indexer.Indexer interface for Milvus 2.x using the V2 SDK. +type Indexer struct { + client *milvusclient.Client + config *IndexerConfig +} + +// NewIndexer creates a new Milvus2 indexer with the provided configuration. +// It returns an error if the configuration is invalid. +func NewIndexer(ctx context.Context, conf *IndexerConfig) (*Indexer, error) { + if err := conf.validate(); err != nil { + return nil, err + } + + cli, err := initClient(ctx, conf) + if err != nil { + return nil, err + } + + if err := initCollection(ctx, cli, conf); err != nil { + return nil, err + } + + return &Indexer{ + client: cli, + config: conf, + }, nil +} + +func initClient(ctx context.Context, conf *IndexerConfig) (*milvusclient.Client, error) { + if conf.Client != nil { + return conf.Client, nil + } + + if conf.ClientConfig == nil { + return nil, fmt.Errorf("[NewIndexer] either Client or ClientConfig must be provided") + } + + cli, err := milvusclient.New(ctx, conf.ClientConfig) + if err != nil { + return nil, fmt.Errorf("[NewIndexer] failed to create milvus client: %w", err) + } + + return cli, nil +} + +func initCollection(ctx context.Context, cli *milvusclient.Client, conf *IndexerConfig) error { + hasCollection, err := cli.HasCollection(ctx, milvusclient.NewHasCollectionOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewIndexer] failed to check collection: %w", err) + } + + if !hasCollection { + // Dimension is required only if Vector config is present or we want default behavior. + // However, in this new design, if Collection doesn't exist, we need to know schema. + // Schema builder checks conf.Vector. So we check it here too if needed? + // Actually buildSchema checks conf.Vector. + if conf.Vector != nil && conf.Vector.Dimension <= 0 { + return fmt.Errorf("[NewIndexer] vector dimension is required when collection does not exist") + } + if err := createCollection(ctx, cli, conf); err != nil { + return err + } + } + + loadState, err := cli.GetLoadState(ctx, milvusclient.NewGetLoadStateOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewIndexer] failed to get load state: %w", err) + } + if loadState.State != entity.LoadStateLoaded { + // Try to create indexes. Ignore "already exists" errors. + if err := createIndex(ctx, cli, conf); err != nil { + return err + } + + loadTask, err := cli.LoadCollection(ctx, milvusclient.NewLoadCollectionOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewIndexer] failed to load collection: %w", err) + } + if err := loadTask.Await(ctx); err != nil { + return fmt.Errorf("[NewIndexer] failed to await collection load: %w", err) + } + } + + return nil +} + +// Store adds the provided documents to the Milvus collection. +// It returns the list of IDs for the stored documents or an error. +func (i *Indexer) Store(ctx context.Context, docs []*schema.Document, opts ...indexer.Option) (ids []string, err error) { + co := indexer.GetCommonOptions(&indexer.Options{ + Embedding: i.config.Embedding, + }, opts...) + io := indexer.GetImplSpecificOptions(&ImplOptions{ + Partition: i.config.PartitionName, + }, opts...) + + ctx = callbacks.EnsureRunInfo(ctx, i.GetType(), components.ComponentOfIndexer) + ctx = callbacks.OnStart(ctx, &indexer.CallbackInput{ + Docs: docs, + }) + defer func() { + if err != nil { + callbacks.OnError(ctx, err) + } + }() + + vectors, err := i.embedDocuments(ctx, co.Embedding, docs) + if err != nil { + return nil, err + } + + upsertResult, err := i.upsertDocuments(ctx, docs, vectors, io.Partition) + if err != nil { + return nil, err + } + + callbacks.OnEnd(ctx, &indexer.CallbackOutput{ + IDs: upsertResult, + }) + + return upsertResult, nil +} + +func (i *Indexer) embedDocuments(ctx context.Context, emb embedding.Embedder, docs []*schema.Document) ([][]float64, error) { + if emb == nil { + return nil, nil // Return nil vectors if no embedder + } + + texts := make([]string, 0, len(docs)) + for _, doc := range docs { + texts = append(texts, doc.Content) + } + + vectors, err := emb.EmbedStrings(i.makeEmbeddingCtx(ctx, emb), texts) + if err != nil { + return nil, fmt.Errorf("[Indexer.Store] failed to embed documents: %w", err) + } + if len(vectors) != len(docs) { + return nil, fmt.Errorf("[Indexer.Store] embedding result length mismatch: need %d, got %d", len(docs), len(vectors)) + } + return vectors, nil +} + +func (i *Indexer) upsertDocuments(ctx context.Context, docs []*schema.Document, vectors [][]float64, partition string) ([]string, error) { + columns, err := i.config.DocumentConverter(ctx, docs, vectors) + if err != nil { + return nil, fmt.Errorf("[Indexer.Store] failed to convert documents: %w", err) + } + + insertOpt := milvusclient.NewColumnBasedInsertOption(i.config.Collection) + if partition != "" { + insertOpt = insertOpt.WithPartition(partition) + } + for _, col := range columns { + insertOpt = insertOpt.WithColumns(col) + } + + result, err := i.client.Upsert(ctx, insertOpt) + if err != nil { + return nil, fmt.Errorf("[Indexer.Store] failed to upsert documents: %w", err) + } + + ids := make([]string, 0, result.IDs.Len()) + for idx := 0; idx < result.IDs.Len(); idx++ { + idStr, err := result.IDs.GetAsString(idx) + if err != nil { + return nil, fmt.Errorf("[Indexer.Store] failed to get id: %w", err) + } + ids = append(ids, idStr) + } + + return ids, nil +} + +// GetType returns the type of the indexer. +func (i *Indexer) GetType() string { + return typ +} + +// IsCallbacksEnabled checks if callbacks are enabled for this indexer. +func (i *Indexer) IsCallbacksEnabled() bool { + return true +} + +// validate checks the configuration and sets default values. +func (c *IndexerConfig) validate() error { + if c.Client == nil && c.ClientConfig == nil { + return fmt.Errorf("[NewIndexer] milvus client or client config not provided") + } + + // Ensure at least one vector config is present + if c.Vector == nil && c.Sparse == nil { + return fmt.Errorf("[NewIndexer] at least one vector field (dense or sparse) is required") + } + + if c.Collection == "" { + c.Collection = defaultCollection + } + if c.Description == "" { + c.Description = defaultDescription + } + + // Dense vector defaults + if c.Vector != nil { + if c.Vector.MetricType == "" { + c.Vector.MetricType = L2 + } + if c.Vector.VectorField == "" { + c.Vector.VectorField = defaultVectorField + } + } + + // Sparse vector defaults + if c.Sparse != nil { + if c.Sparse.VectorField == "" { + c.Sparse.VectorField = defaultSparseVectorField + } + if c.Sparse.MetricType == "" { + c.Sparse.MetricType = BM25 + } + + if c.Sparse.Method == "" { + // Auto uses Milvus server-side functions (e.g. BM25). + // Precomputed requires the user to provide the sparse vector in the document. + // Currently, Milvus only provides built-in Auto support for BM25. + // For other metrics (e.g. IP), default to Precomputed. + if c.Sparse.MetricType == BM25 { + c.Sparse.Method = SparseMethodAuto + } else { + c.Sparse.Method = SparseMethodPrecomputed + } + } + + c.addDefaultBM25Function() + } + + if c.DocumentConverter == nil { + c.DocumentConverter = defaultDocumentConverter(c.Vector, c.Sparse) + } + return nil +} + +func (c *IndexerConfig) addDefaultBM25Function() { + if c.Sparse != nil && c.Sparse.Method == SparseMethodAuto { + hasSparseFunc := false + for _, fn := range c.Functions { + for _, output := range fn.OutputFieldNames { + if output == c.Sparse.VectorField { + hasSparseFunc = true + break + } + } + if hasSparseFunc { + break + } + } + + if !hasSparseFunc { + bm25Fn := entity.NewFunction(). + WithName("bm25_auto"). + WithType(entity.FunctionTypeBM25). + WithInputFields(defaultContentField). + WithOutputFields(c.Sparse.VectorField) + c.Functions = append(c.Functions, bm25Fn) + } + } +} + +// createCollection creates a new Milvus collection with the default schema. +func createCollection(ctx context.Context, cli *milvusclient.Client, conf *IndexerConfig) error { + sch, err := buildSchema(conf) + if err != nil { + return err + } + + createOpt := milvusclient.NewCreateCollectionOption(conf.Collection, sch) + if conf.ConsistencyLevel != ConsistencyLevelDefault { + createOpt = createOpt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + if err := cli.CreateCollection(ctx, createOpt); err != nil { + return fmt.Errorf("[NewIndexer] failed to create collection: %w", err) + } + + return nil +} + +func buildSchema(conf *IndexerConfig) (*entity.Schema, error) { + // Helper to apply field params + applyParams := func(f *entity.Field, name string) { + if params, ok := conf.FieldParams[name]; ok { + for k, v := range params { + f.WithTypeParams(k, v) + } + } + } + + idField := entity.NewField(). + WithName(defaultIDField). + WithDataType(entity.FieldTypeVarChar). + WithMaxLength(defaultMaxIDLen). + WithIsPrimaryKey(true) + applyParams(idField, defaultIDField) + + contentField := entity.NewField(). + WithName(defaultContentField). + WithDataType(entity.FieldTypeVarChar). + WithMaxLength(defaultMaxContentLen) + applyParams(contentField, defaultContentField) + + metadataField := entity.NewField(). + WithName(defaultMetadataField). + WithDataType(entity.FieldTypeJSON) + applyParams(metadataField, defaultMetadataField) + + sch := entity.NewSchema(). + WithField(idField). + WithField(contentField). + WithField(metadataField). + WithDynamicFieldEnabled(conf.EnableDynamicSchema) + + if conf.Vector != nil { + vecField := entity.NewField(). + WithName(conf.Vector.VectorField). + WithDataType(entity.FieldTypeFloatVector). + WithDim(conf.Vector.Dimension) + applyParams(vecField, conf.Vector.VectorField) + sch.WithField(vecField) + } else if conf.Sparse == nil { + // Should not happen if validation passed, but safety check: at least one vector field required + return nil, fmt.Errorf("[NewIndexer] at least one vector field (dense or sparse) is required") + } + + if conf.Sparse != nil { + sparseField := entity.NewField(). + WithName(conf.Sparse.VectorField). + WithDataType(entity.FieldTypeSparseVector) + applyParams(sparseField, conf.Sparse.VectorField) + sch.WithField(sparseField) + } + + // Add functions to schema + for _, fn := range conf.Functions { + sch.WithFunction(fn) + } + + return sch, nil +} + +// createIndex creates indexes on fields if they don't exist. +func createIndex(ctx context.Context, cli *milvusclient.Client, conf *IndexerConfig) error { + if conf.Vector != nil { + if err := createVectorIndex(ctx, cli, conf.Vector.VectorField, conf.Vector, conf.Collection); err != nil { + return err + } + } + + if conf.Sparse != nil { + if err := createSparseIndex(ctx, cli, conf.Sparse, conf.Collection); err != nil { + return err + } + } + + return nil +} + +func createVectorIndex(ctx context.Context, cli *milvusclient.Client, vectorField string, vectorConf *VectorConfig, collection string) error { + var idx index.Index + if vectorConf.IndexBuilder != nil { + idx = vectorConf.IndexBuilder.Build(vectorConf.MetricType) + } else { + idx = index.NewAutoIndex(vectorConf.MetricType.toEntity()) + } + + descOpts := milvusclient.NewDescribeIndexOption(collection, vectorField) + _, err := cli.DescribeIndex(ctx, descOpts) + if err == nil { + log.Printf("[NewIndexer] vector index for field %s already exists, skipping creation", vectorField) + return nil + } + + createIndexOpt := milvusclient.NewCreateIndexOption(collection, vectorField, idx) + + createTask, err := cli.CreateIndex(ctx, createIndexOpt) + if err != nil { + return fmt.Errorf("[NewIndexer] failed to create index: %w", err) + } + + if err := createTask.Await(ctx); err != nil { + return fmt.Errorf("[NewIndexer] failed to await index creation: %w", err) + } + return nil +} + +func createSparseIndex(ctx context.Context, cli *milvusclient.Client, sparseConf *SparseVectorConfig, collection string) error { + var sparseIdx index.Index + if sparseConf.IndexBuilder != nil { + sparseIdx = sparseConf.IndexBuilder.Build(sparseConf.MetricType) + } else { + sparseIdx = NewSparseInvertedIndexBuilder().Build(sparseConf.MetricType) + } + + descOpts := milvusclient.NewDescribeIndexOption(collection, sparseConf.VectorField) + _, err := cli.DescribeIndex(ctx, descOpts) + if err == nil { + log.Printf("[NewIndexer] sparse index for field %s already exists, skipping creation", sparseConf.VectorField) + return nil + } + + createSparseIndexOpt := milvusclient.NewCreateIndexOption(collection, sparseConf.VectorField, sparseIdx) + + createTask, err := cli.CreateIndex(ctx, createSparseIndexOpt) + if err != nil { + return fmt.Errorf("[NewIndexer] failed to create sparse index: %w", err) + } + + if err := createTask.Await(ctx); err != nil { + return fmt.Errorf("[NewIndexer] failed to await sparse index creation: %w", err) + } + return nil +} + +// defaultDocumentConverter returns the default document to column converter. +func defaultDocumentConverter(vector *VectorConfig, sparse *SparseVectorConfig) func(ctx context.Context, docs []*schema.Document, vectors [][]float64) ([]column.Column, error) { + return func(ctx context.Context, docs []*schema.Document, vectors [][]float64) ([]column.Column, error) { + ids := make([]string, 0, len(docs)) + contents := make([]string, 0, len(docs)) + vecs := make([][]float32, 0, len(docs)) + metadatas := make([][]byte, 0, len(docs)) + sparseVecs := make([]entity.SparseEmbedding, 0, len(docs)) + + // Determine if we need to handle sparse vectors + sparseVectorField := "" + if sparse != nil && sparse.Method == SparseMethodPrecomputed { + sparseVectorField = sparse.VectorField + } + + // Determine if we need to handle dense vectors + denseVectorField := "" + if vector != nil { + denseVectorField = vector.VectorField + } + + for idx, doc := range docs { + ids = append(ids, doc.ID) + contents = append(contents, doc.Content) + + var sourceVec []float64 + if len(vectors) == len(docs) { + sourceVec = vectors[idx] + } else { + sourceVec = doc.DenseVector() + } + + // Dense vector is required when vectorField is set (dense-only or hybrid mode). + if denseVectorField != "" { + if len(sourceVec) == 0 { + return nil, fmt.Errorf("vector data missing for document %d (id: %s)", idx, doc.ID) + } + vec := make([]float32, len(sourceVec)) + for i, v := range sourceVec { + vec[i] = float32(v) + } + vecs = append(vecs, vec) + } + + if sparseVectorField != "" { + sv := doc.SparseVector() + se, err := toMilvusSparseEmbedding(sv) + if err != nil { + return nil, fmt.Errorf("failed to convert sparse vector for document %d: %w", idx, err) + } + sparseVecs = append(sparseVecs, se) + } + + metadata, err := sonic.Marshal(doc.MetaData) + if err != nil { + return nil, fmt.Errorf("failed to marshal metadata: %w", err) + } + metadatas = append(metadatas, metadata) + } + + columns := []column.Column{ + column.NewColumnVarChar(defaultIDField, ids), + column.NewColumnVarChar(defaultContentField, contents), + column.NewColumnJSONBytes(defaultMetadataField, metadatas), + } + + if denseVectorField != "" { + dim := 0 + if len(vecs) > 0 { + dim = len(vecs[0]) + } + columns = append(columns, column.NewColumnFloatVector(denseVectorField, dim, vecs)) + } + + if sparseVectorField != "" { + // SparseFloatVector column does not typically require specific dimension argument in insert + columns = append(columns, column.NewColumnSparseVectors(sparseVectorField, sparseVecs)) + } + + return columns, nil + } +} + +func toMilvusSparseEmbedding(sv map[int]float64) (entity.SparseEmbedding, error) { + if len(sv) == 0 { + return entity.NewSliceSparseEmbedding([]uint32{}, []float32{}) + } + + indices := make([]int, 0, len(sv)) + for k := range sv { + indices = append(indices, k) + } + + sort.Ints(indices) + + uint32Indices := make([]uint32, len(indices)) + values := make([]float32, len(indices)) + + for i, idx := range indices { + if idx < 0 { + return nil, fmt.Errorf("negative sparse index: %d", idx) + } + uint32Indices[i] = uint32(idx) + values[i] = float32(sv[idx]) + } + + return entity.NewSliceSparseEmbedding(uint32Indices, values) +} + +// makeEmbeddingCtx creates a context with embedding callback information. +func (i *Indexer) makeEmbeddingCtx(ctx context.Context, emb embedding.Embedder) context.Context { + runInfo := &callbacks.RunInfo{ + Component: components.ComponentOfEmbedding, + } + + if embType, ok := components.GetType(emb); ok { + runInfo.Type = embType + } + + runInfo.Name = runInfo.Type + string(runInfo.Component) + + return callbacks.ReuseHandlers(ctx, runInfo) +} diff --git a/components/indexer/milvus2/indexer_test.go b/components/indexer/milvus2/indexer_test.go new file mode 100644 index 000000000..d46c1d898 --- /dev/null +++ b/components/indexer/milvus2/indexer_test.go @@ -0,0 +1,560 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/column" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" +) + +// mockEmbedding implements embedding.Embedder for testing +type mockEmbedding struct { + err error + dims int +} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + result := make([][]float64, len(texts)) + dims := m.dims + if dims == 0 { + dims = 128 + } + for i := range texts { + result[i] = make([]float64, dims) + for j := 0; j < dims; j++ { + result[i][j] = 0.1 + } + } + return result, nil +} + +func TestIndexerConfig_validate(t *testing.T) { + PatchConvey("test IndexerConfig.validate", t, func() { + mockEmb := &mockEmbedding{} + + PatchConvey("test missing client and client config", func() { + config := &IndexerConfig{ + Client: nil, + ClientConfig: nil, + Collection: "test_collection", + Embedding: mockEmb, + } + err := config.validate() + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "client") + }) + + PatchConvey("test optional embedding", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "test_collection", + Embedding: nil, + Vector: &VectorConfig{ + Dimension: 128, + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + }) + + PatchConvey("test valid config sets defaults", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + // Check defaults are set + convey.So(config.Collection, convey.ShouldEqual, defaultCollection) + convey.So(config.Description, convey.ShouldEqual, defaultDescription) + convey.So(config.Vector.MetricType, convey.ShouldEqual, L2) + convey.So(config.Vector.VectorField, convey.ShouldEqual, defaultVectorField) + convey.So(config.DocumentConverter, convey.ShouldNotBeNil) + }) + + PatchConvey("test valid config sets defaults (sparse)", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "default_sparse", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 256, + }, + Sparse: &SparseVectorConfig{}, // Empty config + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Sparse.VectorField, convey.ShouldEqual, defaultSparseVectorField) + convey.So(config.Sparse.MetricType, convey.ShouldEqual, BM25) + }) + + PatchConvey("test valid config preserves custom sparse vector field", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "my_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 256, + }, + Sparse: &SparseVectorConfig{ + VectorField: "my_sparse_vector", + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Sparse.VectorField, convey.ShouldEqual, "my_sparse_vector") + convey.So(config.Sparse.MetricType, convey.ShouldEqual, BM25) // Default to BM25 + convey.So(len(config.Functions), convey.ShouldEqual, 1) // Auto-generated function + }) + + PatchConvey("test explicit IP metric type", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "my_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 256, + }, + Sparse: &SparseVectorConfig{ + VectorField: "my_sparse_vector", + MetricType: IP, + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Sparse.MetricType, convey.ShouldEqual, IP) + convey.So(len(config.Functions), convey.ShouldEqual, 0) // No auto-generated function + }) + + PatchConvey("test valid config preserves custom values", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "my_collection", + Description: "my description", + Embedding: mockEmb, + PartitionName: "my_partition", + Vector: &VectorConfig{ + Dimension: 256, + MetricType: IP, + VectorField: "my_vector", + IndexBuilder: NewHNSWIndexBuilder(), + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Collection, convey.ShouldEqual, "my_collection") + convey.So(config.Description, convey.ShouldEqual, "my description") + convey.So(config.Vector.VectorField, convey.ShouldEqual, "my_vector") + convey.So(config.Vector.MetricType, convey.ShouldEqual, IP) + }) + + PatchConvey("test sparse-only config", func() { + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "sparse_only", + Embedding: mockEmb, + // Vector is nil implies no dense vector + Sparse: &SparseVectorConfig{ + VectorField: "s_vec", + }, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Vector, convey.ShouldBeNil) + convey.So(config.Sparse.VectorField, convey.ShouldEqual, "s_vec") + }) + + PatchConvey("test auto-generate BM25 function (Default)", func() { + mockEmb := &mockEmbedding{} + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "auto_bm25", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + Sparse: &SparseVectorConfig{ + VectorField: "sparse", + MetricType: BM25, + // Method defaults to BM25 (Auto) + }, + } + + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Sparse.Method, convey.ShouldEqual, SparseMethodAuto) + convey.So(len(config.Functions), convey.ShouldEqual, 1) + fn := config.Functions[0] + convey.So(fn.Name, convey.ShouldEqual, "bm25_auto") + convey.So(fn.Type, convey.ShouldEqual, entity.FunctionTypeBM25) + convey.So(fn.OutputFieldNames[0], convey.ShouldEqual, "sparse") + }) + + PatchConvey("test explicit SparseMethodPrecomputed (No Auto-Gen)", func() { + mockEmb := &mockEmbedding{} + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "client_sparse", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + Sparse: &SparseVectorConfig{ + VectorField: "sparse", + MetricType: BM25, + Method: SparseMethodPrecomputed, + }, + } + + err := config.validate() + convey.So(err, convey.ShouldBeNil) + convey.So(config.Sparse.Method, convey.ShouldEqual, SparseMethodPrecomputed) + // Should NOT generate function + convey.So(len(config.Functions), convey.ShouldEqual, 0) + }) + + PatchConvey("test custom BM25 function (Suppresses Auto-Gen)", func() { + mockEmb := &mockEmbedding{} + customFn := entity.NewFunction(). + WithName("bm25_custom"). + WithType(entity.FunctionTypeBM25). + WithInputFields(defaultContentField). + WithOutputFields("sparse") // Targets the sparse field + + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "custom_bm25", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + Sparse: &SparseVectorConfig{ + VectorField: "sparse", + Method: SparseMethodAuto, + }, + Functions: []*entity.Function{customFn}, + } + + err := config.validate() + convey.So(err, convey.ShouldBeNil) + // Should NOT generate auto function, so count remains 1 + convey.So(len(config.Functions), convey.ShouldEqual, 1) + convey.So(config.Functions[0].Name, convey.ShouldEqual, "bm25_custom") + }) + + PatchConvey("test unrelated function (Allows Auto-Gen)", func() { + mockEmb := &mockEmbedding{} + otherFn := entity.NewFunction(). + WithName("other_fn"). + WithType(entity.FunctionTypeUnknown). + WithInputFields("other_col"). + WithOutputFields("other_out") + + config := &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "mixed_functions", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + Sparse: &SparseVectorConfig{ + VectorField: "sparse", + Method: SparseMethodAuto, + }, + Functions: []*entity.Function{otherFn}, + } + + err := config.validate() + convey.So(err, convey.ShouldBeNil) + // Should generate auto function, so count becomes 2 + convey.So(len(config.Functions), convey.ShouldEqual, 2) + // First is custom, second is auto + convey.So(config.Functions[0].Name, convey.ShouldEqual, "other_fn") + convey.So(config.Functions[1].Name, convey.ShouldEqual, "bm25_auto") + }) + }) +} + +func TestNewIndexer(t *testing.T) { + PatchConvey("test NewIndexer", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockClient := &milvusclient.Client{} + + // Mock milvusclient.New + Mock(milvusclient.New).Return(mockClient, nil).Build() + + PatchConvey("test missing client and client config", func() { + _, err := NewIndexer(ctx, &IndexerConfig{ + Client: nil, + ClientConfig: nil, + Collection: "test_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + }) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "client") + }) + + PatchConvey("test with collection already exists and loaded", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateLoaded}, nil).Build() + + indexer, err := NewIndexer(ctx, &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "test_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + }) + convey.So(err, convey.ShouldBeNil) + convey.So(indexer, convey.ShouldNotBeNil) + }) + + PatchConvey("test with collection not loaded, needs index and load", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateNotLoad}, nil).Build() + Mock(GetMethod(mockClient, "ListIndexes")).Return([]string{}, nil).Build() + Mock(GetMethod(mockClient, "DescribeIndex")).Return(milvusclient.IndexDescription{}, fmt.Errorf("index not found")).Build() + + mockTask := &milvusclient.CreateIndexTask{} + Mock(GetMethod(mockClient, "CreateIndex")).Return(mockTask, nil).Build() + Mock(GetMethod(mockTask, "Await")).Return(nil).Build() + + mockLoadTask := milvusclient.LoadTask{} + Mock(GetMethod(mockClient, "LoadCollection")).Return(mockLoadTask, nil).Build() + Mock(GetMethod(&mockLoadTask, "Await")).Return(nil).Build() + + indexer, err := NewIndexer(ctx, &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "test_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + }) + convey.So(err, convey.ShouldBeNil) + convey.So(indexer, convey.ShouldNotBeNil) + }) + + PatchConvey("test collection does not exist, needs creation", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(false, nil).Build() + Mock(GetMethod(mockClient, "CreateCollection")).Return(nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateNotLoad}, nil).Build() + Mock(GetMethod(mockClient, "ListIndexes")).Return([]string{}, nil).Build() + Mock(GetMethod(mockClient, "DescribeIndex")).Return(milvusclient.IndexDescription{}, fmt.Errorf("index not found")).Build() + + mockTask := &milvusclient.CreateIndexTask{} + Mock(GetMethod(mockClient, "CreateIndex")).Return(mockTask, nil).Build() + Mock(GetMethod(mockTask, "Await")).Return(nil).Build() + + mockLoadTask := milvusclient.LoadTask{} + Mock(GetMethod(mockClient, "LoadCollection")).Return(mockLoadTask, nil).Build() + Mock(GetMethod(&mockLoadTask, "Await")).Return(nil).Build() + + indexer, err := NewIndexer(ctx, &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "test_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 128, + }, + }) + convey.So(err, convey.ShouldBeNil) + convey.So(indexer, convey.ShouldNotBeNil) + }) + + PatchConvey("test collection does not exist but dimension not provided", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(false, nil).Build() + + _, err := NewIndexer(ctx, &IndexerConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: "localhost:19530", + }, + Collection: "test_collection", + Embedding: mockEmb, + Vector: &VectorConfig{ + Dimension: 0, // No dimension + }, + }) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "dimension") + }) + }) +} + +func TestIndexer_GetType(t *testing.T) { + convey.Convey("test Indexer.GetType", t, func() { + indexer := &Indexer{} + convey.So(indexer.GetType(), convey.ShouldNotBeEmpty) + }) +} + +func TestIndexer_IsCallbacksEnabled(t *testing.T) { + convey.Convey("test Indexer.IsCallbacksEnabled", t, func() { + indexer := &Indexer{ + config: &IndexerConfig{}, + } + convey.So(indexer.IsCallbacksEnabled(), convey.ShouldBeTrue) + }) +} + +func TestIndexer_Store(t *testing.T) { + PatchConvey("test Indexer.Store", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockClient := &milvusclient.Client{} + + indexer := &Indexer{ + client: mockClient, + config: &IndexerConfig{ + Collection: "test_collection", + Vector: &VectorConfig{ + Dimension: 128, + }, + Embedding: mockEmb, + DocumentConverter: defaultDocumentConverter(&VectorConfig{ + VectorField: defaultVectorField, + }, nil), + }, + } + + docs := []*schema.Document{ + { + ID: "doc1", + Content: "Test document 1", + MetaData: map[string]interface{}{"key": "value"}, + }, + { + ID: "doc2", + Content: "Test document 2", + MetaData: map[string]interface{}{"key2": "value2"}, + }, + } + + PatchConvey("test store with embedding error", func() { + indexer.config.Embedding = &mockEmbedding{err: fmt.Errorf("embedding error")} + ids, err := indexer.Store(ctx, docs) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "embed") + convey.So(ids, convey.ShouldBeNil) + }) + + PatchConvey("test store with upsert error", func() { + indexer.config.Embedding = mockEmb + Mock(GetMethod(mockClient, "Upsert")).Return(nil, fmt.Errorf("upsert error")).Build() + + ids, err := indexer.Store(ctx, docs) + convey.So(err, convey.ShouldNotBeNil) + convey.So(ids, convey.ShouldBeNil) + }) + + PatchConvey("test store success", func() { + indexer.config.Embedding = mockEmb + + // Create mock ID column + mockIDColumn := column.NewColumnVarChar("id", []string{"doc1", "doc2"}) + mockResult := milvusclient.UpsertResult{ + IDs: mockIDColumn, + } + Mock(GetMethod(mockClient, "Upsert")).Return(mockResult, nil).Build() + + ids, err := indexer.Store(ctx, docs) + convey.So(err, convey.ShouldBeNil) + convey.So(ids, convey.ShouldNotBeNil) + convey.So(len(ids), convey.ShouldEqual, 2) + }) + }) +} + +func TestDefaultDocumentConverter(t *testing.T) { + convey.Convey("test defaultDocumentConverter", t, func() { + convey.Convey("test conversion (dense only)", func() { + converter := defaultDocumentConverter(&VectorConfig{ + VectorField: defaultVectorField, + }, nil) + + ctx := context.Background() + docs := []*schema.Document{ + { + ID: "doc1", + Content: "content1", + MetaData: map[string]interface{}{"key": "value"}, + }, + } + vectors := [][]float64{{0.1, 0.2, 0.3}} + + columns, err := converter(ctx, docs, vectors) + convey.So(err, convey.ShouldBeNil) + convey.So(len(columns), convey.ShouldEqual, 4) // id, content, vector, metadata + convey.So(columns[3].Name(), convey.ShouldEqual, defaultVectorField) + convey.So(columns[2].Name(), convey.ShouldEqual, defaultMetadataField) + }) + + convey.Convey("test conversion with sparse vector", func() { + converter := defaultDocumentConverter(&VectorConfig{ + VectorField: defaultVectorField, + }, &SparseVectorConfig{ + VectorField: "sparse_vector", + Method: SparseMethodPrecomputed, + }) + + ctx := context.Background() + docs := []*schema.Document{ + { + ID: "doc1", + Content: "content1", + MetaData: map[string]interface{}{"key": "value"}, + }, + } + docs[0].WithSparseVector(map[int]float64{1: 0.5}) + vectors := [][]float64{{0.1, 0.2, 0.3}} + + columns, err := converter(ctx, docs, vectors) + convey.So(err, convey.ShouldBeNil) + // Now returns 5 columns: id, content, metadata, dense_vector, sparse_vector + convey.So(len(columns), convey.ShouldEqual, 5) + convey.So(columns[4].Name(), convey.ShouldEqual, "sparse_vector") + }) + + }) +} diff --git a/components/indexer/milvus2/options.go b/components/indexer/milvus2/options.go new file mode 100644 index 000000000..c1f015e63 --- /dev/null +++ b/components/indexer/milvus2/options.go @@ -0,0 +1,33 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import "github.com/cloudwego/eino/components/indexer" + +// ImplOptions contains implementation-specific options for indexing operations. +type ImplOptions struct { + // Partition specifies the target partition for document insertion. + // If empty, documents are inserted into the default partition. + Partition string +} + +// WithPartition returns an option that sets the target partition for insertion. +func WithPartition(partition string) indexer.Option { + return indexer.WrapImplSpecificOptFn(func(o *ImplOptions) { + o.Partition = partition + }) +} diff --git a/components/indexer/milvus2/types.go b/components/indexer/milvus2/types.go new file mode 100644 index 000000000..e83731510 --- /dev/null +++ b/components/indexer/milvus2/types.go @@ -0,0 +1,108 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "github.com/milvus-io/milvus/client/v2/entity" +) + +// ConsistencyLevel represents the consistency level for Milvus operations. +type ConsistencyLevel int + +const ( + // ConsistencyLevelDefault lets Milvus use its built-in default consistency level. + // When creating a collection, if this is set (zero value), no explicit consistency + // level is passed, and Milvus defaults to Bounded Staleness. + ConsistencyLevelDefault ConsistencyLevel = iota + // ConsistencyLevelStrong guarantees that reads return the most recent data. + ConsistencyLevelStrong + // ConsistencyLevelSession ensures read-your-write consistency within the same session. + ConsistencyLevelSession + // ConsistencyLevelBounded allows reads to return data within a certain staleness bound. + ConsistencyLevelBounded + // ConsistencyLevelEventually provides eventual consistency with no staleness guarantees. + ConsistencyLevelEventually + // ConsistencyLevelCustomized allows custom consistency configuration. + ConsistencyLevelCustomized +) + +// ToEntity converts ConsistencyLevel to the Milvus SDK entity.ConsistencyLevel type. +// Note: ConsistencyLevelDefault should be checked before calling this method, +// as it indicates "use default" and should be resolved first. +func (c ConsistencyLevel) ToEntity() entity.ConsistencyLevel { + switch c { + case ConsistencyLevelStrong: + return entity.ClStrong + case ConsistencyLevelSession: + return entity.ClSession + case ConsistencyLevelBounded: + return entity.ClBounded + case ConsistencyLevelEventually: + return entity.ClEventually + case ConsistencyLevelCustomized: + return entity.ClCustomized + default: + // ConsistencyLevelDefault (0) or invalid value - default to Bounded + return entity.ClBounded + } +} + +// MetricType represents the metric type for vector similarity. +type MetricType string + +const ( + // L2 represents Euclidean distance (L2 norm). + // Suitable for dense floating-point vectors. + L2 MetricType = "L2" + + // IP represents Inner Product similarity. + // Higher values indicate greater similarity. Suitable for normalized vectors. + IP MetricType = "IP" + + // COSINE represents Cosine similarity. + // Measures the cosine of the angle between vectors, ignoring magnitude. + COSINE MetricType = "COSINE" + + // HAMMING represents Hamming distance for binary vectors. + // Counts the number of positions where the corresponding bits differ. + HAMMING MetricType = "HAMMING" + + // JACCARD represents Jaccard distance for binary vectors. + // Measures dissimilarity between sample sets. + JACCARD MetricType = "JACCARD" + + // TANIMOTO represents Tanimoto distance for binary vectors. + // Similar to Jaccard, used for molecular fingerprint comparisons. + TANIMOTO MetricType = "TANIMOTO" + + // SUBSTRUCTURE represents substructure search for binary vectors. + // Returns true if A is a subset of B. + SUBSTRUCTURE MetricType = "SUBSTRUCTURE" + + // SUPERSTRUCTURE represents superstructure search for binary vectors. + // Returns true if B is a subset of A. + SUPERSTRUCTURE MetricType = "SUPERSTRUCTURE" + + // BM25 represents BM25 similarity. + // Used for sparse vectors generated by BM25 function. + BM25 MetricType = "BM25" +) + +// toEntity converts MetricType to the Milvus SDK entity.MetricType type. +func (m MetricType) toEntity() entity.MetricType { + return entity.MetricType(m) +} diff --git a/components/indexer/milvus2/types_test.go b/components/indexer/milvus2/types_test.go new file mode 100644 index 000000000..239f8231a --- /dev/null +++ b/components/indexer/milvus2/types_test.go @@ -0,0 +1,122 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "testing" + + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/smartystreets/goconvey/convey" +) + +func TestConsistencyLevel_toEntity(t *testing.T) { + convey.Convey("test ConsistencyLevel.toEntity", t, func() { + convey.Convey("test ConsistencyLevelStrong", func() { + level := ConsistencyLevelStrong + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClStrong) + }) + + convey.Convey("test ConsistencyLevelSession", func() { + level := ConsistencyLevelSession + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClSession) + }) + + convey.Convey("test ConsistencyLevelBounded", func() { + level := ConsistencyLevelBounded + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClBounded) + }) + + convey.Convey("test ConsistencyLevelEventually", func() { + level := ConsistencyLevelEventually + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClEventually) + }) + + convey.Convey("test ConsistencyLevelCustomized", func() { + level := ConsistencyLevelCustomized + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClCustomized) + }) + + convey.Convey("test default/unknown level", func() { + level := ConsistencyLevel(0) + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClBounded) + }) + + convey.Convey("test invalid level", func() { + level := ConsistencyLevel(100) + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClBounded) + }) + }) +} + +func TestMetricType_toEntity(t *testing.T) { + convey.Convey("test MetricType.toEntity", t, func() { + convey.Convey("test L2", func() { + mt := L2 + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("L2")) + }) + + convey.Convey("test IP", func() { + mt := IP + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("IP")) + }) + + convey.Convey("test COSINE", func() { + mt := COSINE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("COSINE")) + }) + + convey.Convey("test HAMMING", func() { + mt := HAMMING + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("HAMMING")) + }) + + convey.Convey("test JACCARD", func() { + mt := JACCARD + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("JACCARD")) + }) + + convey.Convey("test TANIMOTO", func() { + mt := TANIMOTO + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("TANIMOTO")) + }) + + convey.Convey("test SUBSTRUCTURE", func() { + mt := SUBSTRUCTURE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("SUBSTRUCTURE")) + }) + + convey.Convey("test SUPERSTRUCTURE", func() { + mt := SUPERSTRUCTURE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("SUPERSTRUCTURE")) + }) + }) +} diff --git a/components/retriever/milvus/README.md b/components/retriever/milvus/README.md index 19692ca7a..e9edcfba6 100644 --- a/components/retriever/milvus/README.md +++ b/components/retriever/milvus/README.md @@ -6,6 +6,8 @@ An Milvus 2.x retriever implementation for [Eino](https://github.com/cloudwego/e interface. This enables seamless integration with Eino's vector storage and retrieval system for enhanced semantic search capabilities. +> **Note**: This package supports **Milvus 2.4.x**. For Milvus 2.5+ features (BM25, server-side functions, hybrid search), use the [`milvus2`](../milvus2) package instead. + ## Quick Start ### Installation diff --git a/components/retriever/milvus/README_zh.md b/components/retriever/milvus/README_zh.md index dd2effe91..0cfc716ca 100644 --- a/components/retriever/milvus/README_zh.md +++ b/components/retriever/milvus/README_zh.md @@ -5,6 +5,8 @@ 基于 Milvus 2.x 的向量搜索实现，为 [Eino](https://github.com/cloudwego/eino) 提供了符合 `Retriever` 接口的存储方案。该组件可无缝集成 Eino 的向量存储和检索系统，增强语义搜索能力。 +> **注意**: 此包支持 **Milvus 2.4.x**。如需使用 Milvus 2.5+ 的新功能（BM25、服务端函数、混合检索），请使用 [`milvus2`](../milvus2) 包。 + ## 快速开始 ### 安装 diff --git a/components/retriever/milvus2/README.md b/components/retriever/milvus2/README.md new file mode 100644 index 000000000..4125970e2 --- /dev/null +++ b/components/retriever/milvus2/README.md @@ -0,0 +1,262 @@ +# Milvus 2.x Retriever + +English | [中文](./README_zh.md) + +This package provides a Milvus 2.x (V2 SDK) retriever implementation for the EINO framework. It enables vector similarity search with multiple search modes. + +> **Note**: This package requires **Milvus 2.5+** for server-side function support (e.g., BM25). + +## Features + +- **Milvus V2 SDK**: Uses the latest `milvus-io/milvus/client/v2` SDK +- **Multiple Search Modes**: Approximate, Range, Hybrid, Iterator, and Scalar search +- **Dense + Sparse Hybrid Search**: Combine dense and sparse vectors with RRF reranking +- **Custom Result Conversion**: Configurable result-to-document conversion + +## Installation + +```bash +go get github.com/cloudwego/eino-ext/components/retriever/milvus2 +``` + +## Quick Start + +```go +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino-ext/components/embedding/ark" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + // Get the environment variables + addr := os.Getenv("MILVUS_ADDR") + username := os.Getenv("MILVUS_USERNAME") + password := os.Getenv("MILVUS_PASSWORD") + arkApiKey := os.Getenv("ARK_API_KEY") + arkModel := os.Getenv("ARK_MODEL") + + ctx := context.Background() + + // Create an embedding model + emb, err := ark.NewEmbedder(ctx, &ark.EmbeddingConfig{ + APIKey: arkApiKey, + Model: arkModel, + }) + if err != nil { + log.Fatalf("Failed to create embedding: %v", err) + return + } + + // Create a retriever + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: addr, + Username: username, + Password: password, + }, + Collection: "my_collection", + TopK: 10, + SearchMode: search_mode.NewApproximate(milvus2.COSINE), + Embedding: emb, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + return + } + log.Printf("Retriever created successfully") + + // Retrieve documents + documents, err := retriever.Retrieve(ctx, "search query") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + return + } + + // Print the documents + for i, doc := range documents { + fmt.Printf("Document %d:\n", i) + fmt.Printf(" ID: %s\n", doc.ID) + fmt.Printf(" Content: %s\n", doc.Content) + fmt.Printf(" Score: %v\n", doc.Score()) + } +} +``` + +## Configuration + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `Client` | `*milvusclient.Client` | - | Pre-configured Milvus client (optional) | +| `ClientConfig` | `*milvusclient.ClientConfig` | - | Client configuration (required if Client is nil) | +| `Collection` | `string` | `"eino_collection"` | Collection name | +| `TopK` | `int` | `5` | Number of results to return | +| `VectorField` | `string` | `"vector"` | Dense vector field name | +| `SparseVectorField` | `string` | `"sparse_vector"` | Sparse vector field name | +| `OutputFields` | `[]string` | all fields | Fields to return in results | +| `SearchMode` | `SearchMode` | - | Search strategy (required) | +| `Embedding` | `embedding.Embedder` | - | Embedder for query vectorization (optional, required for vector search) | +| `DocumentConverter` | `func` | default converter | Custom result-to-document converter | +| `ConsistencyLevel` | `ConsistencyLevel` | `ConsistencyLevelDefault` | Consistency level (`ConsistencyLevelDefault` uses the collection's level; no per-request override is applied) | +| `Partitions` | `[]string` | - | Partitions to search | + +### VectorType (for Hybrid Search) + +| Value | Description | +|-------|-------------| +| `DenseVector` | Standard dense floating-point vectors (default) | +| `SparseVector` | Sparse vectors (used with BM25 or precomputed sparse embeddings) | + +## Search Modes + +Import search modes from `github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode`. + +### Approximate Search + +Standard approximate nearest neighbor (ANN) search. + +```go +mode := search_mode.NewApproximate(milvus2.COSINE) +``` + +### Range Search + +Search within a distance range (vectors within `Radius`). + +```go +// L2: Distance <= Radius +// IP/Cosine: Score >= Radius +mode := search_mode.NewRange(milvus2.L2, 0.5). + WithRangeFilter(0.1) // Optional: Inner boundary for ring search +``` + +### Sparse Search (BM25) + +Sparse-only search using BM25. Requires Milvus 2.5+ with sparse vector fields and enabled Functions. + +```go +// OutputFields is required to retrieve content for sparse-only search (BM25) +// MetricType: BM25 (default) or IP +mode := search_mode.NewSparse(milvus2.BM25) + +// In config, use "*" or specific fields to ensure content is returned: +// OutputFields: []string{"*"} +``` + +### Hybrid Search (Dense + Sparse) + +Multi-vector search combining dense and sparse vectors with result reranking. Requires a collection with both dense and sparse vector fields (see indexer sparse example). + +```go +import ( + "github.com/milvus-io/milvus/client/v2/milvusclient" + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +// Define hybrid search with Dense + Sparse sub-requests +hybridMode := search_mode.NewHybrid( + milvusclient.NewRRFReranker().WithK(60), // RRF reranker + &search_mode.SubRequest{ + VectorField: "vector", // Dense vector field + VectorType: milvus2.DenseVector, // Default, can be omitted + TopK: 10, + MetricType: milvus2.L2, + }, + // Sparse SubRequest + &search_mode.SubRequest{ + VectorField: "sparse_vector", // Sparse vector field + VectorType: milvus2.SparseVector, // Specify sparse type + TopK: 10, + MetricType: milvus2.BM25, // Use BM25 or IP based on index + }, +) + +// Create retriever (Sparse embedding handled server-side by Milvus Function) +retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "hybrid_collection", + VectorField: "vector", // Default dense field + SparseVectorField: "sparse_vector", // Default sparse field + TopK: 5, + SearchMode: hybridMode, + Embedding: denseEmbedder, // Standard embedder for dense vectors +}) +``` + +### Iterator Search + +Batch-based traversal for large result sets. + +> [!WARNING] +> The `Retrieve` method in `Iterator` mode fetches **all** results until the total limit (`TopK`) or end-of-collection is reached. For extremely large datasets, this may consume significant memory. + +```go +// 100 is the batch size (items per network call) +mode := search_mode.NewIterator(milvus2.COSINE, 100). + WithSearchParams(map[string]string{"nprobe": "10"}) + +// Use RetrieverConfig.TopK to set the total limit (IteratorLimit). +``` + +### Scalar Search + +Metadata-only filtering without vector similarity (uses filter expressions as query). + +```go +mode := search_mode.NewScalar() + +// Query with filter expression +docs, err := retriever.Retrieve(ctx, `category == "electronics" AND year >= 2023`) +``` + +### Dense Vector Metrics +| Metric | Description | +|--------|-------------| +| `L2` | Euclidean distance | +| `IP` | Inner Product | +| `COSINE` | Cosine similarity | + +### Sparse Vector Metrics +| Metric | Description | +|--------|-------------| +| `BM25` | Okapi BM25 (Required for BM25 Search) | +| `IP` | Inner Product (Suitable for precomputed sparse vectors) | + +### Binary Vector Metrics +| Metric | Description | +|--------|-------------| +| `HAMMING` | Hamming distance | +| `JACCARD` | Jaccard distance | +| `TANIMOTO` | Tanimoto distance | +| `SUBSTRUCTURE` | Substructure search | +| `SUPERSTRUCTURE` | Superstructure search | + +> **Important**: The metric type in SearchMode must match the index metric type used when creating the collection. + +## Examples + +See the [examples](./examples) directory for complete working examples: + +- [approximate](./examples/approximate) - Basic ANN search +- [range](./examples/range) - Range search example +- [hybrid](./examples/hybrid) - Hybrid multi-vector search (Dense + BM25) +- [hybrid_chinese](./examples/hybrid_chinese) - Chinese text hybrid search with BM25 +- [iterator](./examples/iterator) - Batch iterator search +- [scalar](./examples/scalar) - Scalar/metadata filtering +- [grouping](./examples/grouping) - Grouping search results +- [filtered](./examples/filtered) - Filtered vector search +- [sparse](./examples/sparse) - Sparse-only search example (BM25) + +## License + +Apache License 2.0 diff --git a/components/retriever/milvus2/README_zh.md b/components/retriever/milvus2/README_zh.md new file mode 100644 index 000000000..076712f82 --- /dev/null +++ b/components/retriever/milvus2/README_zh.md @@ -0,0 +1,255 @@ +# Milvus 2.x Retriever + +[English](./README.md) | 中文 + +本包为 EINO 框架提供 Milvus 2.x (V2 SDK) 检索器实现，支持多种搜索模式的向量相似度搜索。 + +> **注意**: 本包需要 **Milvus 2.5+** 以支持服务器端函数（如 BM25）。 + +## 功能特性 + +- **Milvus V2 SDK**: 使用最新的 `milvus-io/milvus/client/v2` SDK +- **多种搜索模式**: 支持近似搜索、范围搜索、混合搜索、迭代器搜索和标量搜索 +- **稠密 + 稀疏混合搜索**: 结合稠密向量和稀疏向量，使用 RRF 重排序 +- **自定义结果转换**: 可配置的结果到文档转换 + +## 安装 + +```bash +go get github.com/cloudwego/eino-ext/components/retriever/milvus2 +``` + +## 快速开始 + +```go +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino-ext/components/embedding/ark" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + // 获取环境变量 + addr := os.Getenv("MILVUS_ADDR") + username := os.Getenv("MILVUS_USERNAME") + password := os.Getenv("MILVUS_PASSWORD") + arkApiKey := os.Getenv("ARK_API_KEY") + arkModel := os.Getenv("ARK_MODEL") + + ctx := context.Background() + + // 创建 embedding 模型 + emb, err := ark.NewEmbedder(ctx, &ark.EmbeddingConfig{ + APIKey: arkApiKey, + Model: arkModel, + }) + if err != nil { + log.Fatalf("Failed to create embedding: %v", err) + return + } + + // 创建 retriever + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: addr, + Username: username, + Password: password, + }, + Collection: "my_collection", + TopK: 10, + SearchMode: search_mode.NewApproximate(milvus2.COSINE), + Embedding: emb, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + return + } + log.Printf("Retriever created successfully") + + // 检索文档 + documents, err := retriever.Retrieve(ctx, "search query") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + return + } + + // 打印文档 + for i, doc := range documents { + fmt.Printf("Document %d:\n", i) + fmt.Printf(" ID: %s\n", doc.ID) + fmt.Printf(" Content: %s\n", doc.Content) + fmt.Printf(" Score: %v\n", doc.Score()) + } +} +``` + +## 配置选项 + +| 字段 | 类型 | 默认值 | 描述 | +|------|------|--------|------| +| `Client` | `*milvusclient.Client` | - | 预配置的 Milvus 客户端（可选） | +| `ClientConfig` | `*milvusclient.ClientConfig` | - | 客户端配置（Client 为空时必需） | +| `Collection` | `string` | `"eino_collection"` | 集合名称 | +| `TopK` | `int` | `5` | 返回结果数量 | +| `VectorField` | `string` | `"vector"` | 稠密向量字段名 | +| `SparseVectorField` | `string` | `"sparse_vector"` | 稀疏向量字段名 | +| `OutputFields` | `[]string` | 所有字段 | 结果中返回的字段 | +| `SearchMode` | `SearchMode` | - | 搜索策略（必需） | +| `Embedding` | `embedding.Embedder` | - | 用于查询向量化的 Embedder（必需） | +| `DocumentConverter` | `func` | 默认转换器 | 自定义结果到文档转换 | +| `ConsistencyLevel` | `ConsistencyLevel` | `ConsistencyLevelDefault` | 一致性级别 (`ConsistencyLevelDefault` 使用 collection 的级别；不应用按请求覆盖) | +| `Partitions` | `[]string` | - | 要搜索的分区 | + +## 搜索模式 + +从 `github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode` 导入搜索模式。 + +### 近似搜索 (Approximate) + +标准的近似最近邻 (ANN) 搜索。 + +```go +mode := search_mode.NewApproximate(milvus2.COSINE) +``` + +### 范围搜索 (Range) + +在指定距离范围内搜索 (向量在 `Radius` 内)。 + +```go +// L2: 距离 <= Radius +// IP/Cosine: 分数 >= Radius +mode := search_mode.NewRange(milvus2.L2, 0.5). + WithRangeFilter(0.1) // 可选: 环形搜索的内边界 +``` + +### 稀疏搜索 (BM25) + +使用 BM25 进行纯稀疏向量搜索。需要 Milvus 2.5+ 支持稀疏向量字段并启用 Functions。 + +```go +// 纯稀疏搜索 (BM25) 需要指定 OutputFields 以获取内容 +// MetricType: BM25 (默认) 或 IP +mode := search_mode.NewSparse(milvus2.BM25) + +// 在配置中，使用 "*" 或特定字段以确保返回内容: +// OutputFields: []string{"*"} +``` + +### 混合搜索 (Hybrid - 稠密 + 稀疏) + +结合稠密向量和稀疏向量的多向量搜索，支持结果重排序。需要一个同时包含稠密和稀疏向量字段的集合（参见 indexer sparse 示例）。 + +```go +import ( + "github.com/milvus-io/milvus/client/v2/milvusclient" + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +// 定义稠密 + 稀疏子请求的混合搜索 +hybridMode := search_mode.NewHybrid( + milvusclient.NewRRFReranker().WithK(60), // RRF 重排序器 + &search_mode.SubRequest{ + VectorField: "vector", // 稠密向量字段 + VectorType: milvus2.DenseVector, // 默认值，可省略 + TopK: 10, + MetricType: milvus2.L2, + }, + // 稀疏子请求 (Sparse SubRequest) + &search_mode.SubRequest{ + VectorField: "sparse_vector", // 稀疏向量字段 + VectorType: milvus2.SparseVector, // 指定稀疏类型 + TopK: 10, + MetricType: milvus2.BM25, // 使用 BM25 或 IP + }, +) + +// 创建 retriever (稀疏向量生成由 Milvus Function 服务器端处理) +retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "hybrid_collection", + VectorField: "vector", // 默认稠密字段 + SparseVectorField: "sparse_vector", // 默认稀疏字段 + TopK: 5, + SearchMode: hybridMode, + Embedding: denseEmbedder, // 稠密向量的标准 Embedder +}) +``` + +### 迭代器搜索 (Iterator) + +基于批次的遍历，适用于大结果集。 + +> [!WARNING] +> `Iterator` 模式的 `Retrieve` 方法会获取 **所有** 结果，直到达到总限制 (`TopK`) 或集合末尾。对于极大数据集，这可能会消耗大量内存。 + +```go +// 100 是批次大小 (每次网络调用的条目数) +mode := search_mode.NewIterator(milvus2.COSINE, 100). + WithSearchParams(map[string]string{"nprobe": "10"}) + +// 使用 RetrieverConfig.TopK 设置总限制 (IteratorLimit)。 +``` + +### 标量搜索 (Scalar) + +仅基于元数据过滤，不使用向量相似度（将过滤表达式作为查询）。 + +```go +mode := search_mode.NewScalar() + +// 使用过滤表达式查询 +docs, err := retriever.Retrieve(ctx, `category == "electronics" AND year >= 2023`) +``` + +### 稠密向量度量 (Dense) +| 度量类型 | 描述 | +|----------|------| +| `L2` | 欧几里得距离 | +| `IP` | 内积 | +| `COSINE` | 余弦相似度 | + +### 稀疏向量度量 (Sparse) +| 度量类型 | 描述 | +|----------|------| +| `BM25` | Okapi BM25 (BM25 搜索必需) | +| `IP` | 内积 (适用于预计算的稀疏向量) | + +### 二进制向量度量 (Binary) +| 度量类型 | 描述 | +|----------|------| +| `HAMMING` | 汉明距离 | +| `JACCARD` | 杰卡德距离 | +| `TANIMOTO` | Tanimoto 距离 | +| `SUBSTRUCTURE` | 子结构搜索 | +| `SUPERSTRUCTURE` | 超结构搜索 | + +> **重要提示**: SearchMode 中的度量类型必须与创建集合时使用的索引度量类型一致。 + +## 示例 + +查看 [examples](./examples) 目录获取完整的示例代码： + +- [approximate](./examples/approximate) - 基础 ANN 搜索 +- [range](./examples/range) - 范围搜索示例 +- [hybrid](./examples/hybrid) - 混合多向量搜索 (稠密 + BM25) +- [hybrid_chinese](./examples/hybrid_chinese) - 中文混合搜索示例 +- [iterator](./examples/iterator) - 批次迭代器搜索 +- [scalar](./examples/scalar) - 标量/元数据过滤 +- [grouping](./examples/grouping) - 分组搜索结果 +- [filtered](./examples/filtered) - 带过滤的向量搜索 +- [sparse](./examples/sparse) - 纯稀疏搜索示例 (BM25) + +## 许可证 + +Apache License 2.0 diff --git a/components/retriever/milvus2/consts.go b/components/retriever/milvus2/consts.go new file mode 100644 index 000000000..0b68fde42 --- /dev/null +++ b/components/retriever/milvus2/consts.go @@ -0,0 +1,30 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +const ( + typ = "Milvus2" + + defaultCollection = "eino_collection" + defaultIDField = "id" + defaultVectorField = "vector" + defaultSparseVectorField = "sparse_vector" + defaultContentField = "content" + defaultMetadataField = "metadata" + + defaultTopK = 5 +) diff --git a/components/retriever/milvus2/examples/approximate/approximate.go b/components/retriever/milvus2/examples/approximate/approximate.go new file mode 100644 index 000000000..6eb04b828 --- /dev/null +++ b/components/retriever/milvus2/examples/approximate/approximate.go @@ -0,0 +1,94 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates basic vector search using approximate nearest neighbor. +// +// Prerequisites: +// Run the hnsw indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/hnsw && go run . +// +// The collection "demo_hnsw" will be used for search. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create a retriever for approximate nearest neighbor search + // SearchMode encapsulates the entire search execution logic including + // embedding, option building, and result conversion. + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_hnsw", // Uses collection created by indexer/hnsw example + VectorField: "vector", + OutputFields: []string{"id", "content", "metadata"}, + TopK: 5, + SearchMode: search_mode.NewApproximate(milvus2.COSINE), + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Retriever created successfully") + + // Search for similar documents + docs, err := retriever.Retrieve(ctx, "vector search query") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + + // Print results + fmt.Printf("\nFound %d documents:\n", len(docs)) + for i, doc := range docs { + fmt.Printf("\n--- Document %d ---\n", i+1) + fmt.Printf("ID: %s\n", doc.ID) + fmt.Printf("Content: %s\n", doc.Content) + fmt.Printf("Score: %v\n", doc.Score()) + } +} + +// mockEmbedding generates embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/filtered/filtered.go b/components/retriever/milvus2/examples/filtered/filtered.go new file mode 100644 index 000000000..39e0b4c6a --- /dev/null +++ b/components/retriever/milvus2/examples/filtered/filtered.go @@ -0,0 +1,105 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates filtered vector search using Milvus boolean expressions. +// +// Prerequisites: +// Run the hnsw indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/hnsw && go run . +// +// The collection "demo_hnsw" will be used for search. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Create retriever + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_hnsw", + VectorField: "vector", + OutputFields: []string{"id", "content"}, + TopK: 10, + SearchMode: search_mode.NewApproximate(milvus2.COSINE), + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Retriever created successfully") + + // Search without filter + fmt.Println("=== Search without filter ===") + docs, err := retriever.Retrieve(ctx, "vector search") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + fmt.Printf("Found %d documents\n", len(docs)) + for _, doc := range docs { + fmt.Printf("- %s: %s\n", doc.ID, doc.Content) + } + + // Search with filter on ID field + // Milvus filter syntax: https://milvus.io/docs/boolean.md + fmt.Println("\n=== Search with filter (id == 'hnsw-1') ===") + filteredDocs, err := retriever.Retrieve(ctx, "vector search", + milvus2.WithFilter("id == 'hnsw-1'")) + if err != nil { + log.Fatalf("Failed to retrieve with filter: %v", err) + } + fmt.Printf("Found %d documents\n", len(filteredDocs)) + for _, doc := range filteredDocs { + fmt.Printf("- %s: %s\n", doc.ID, doc.Content) + } + + // More filter examples (uncomment to try): + // milvus2.WithFilter("id in ['hnsw-1', 'hnsw-2']") // IN clause + // milvus2.WithFilter("id like 'hnsw%'") // LIKE pattern +} + +// mockEmbedding generates embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/grouping/grouping.go b/components/retriever/milvus2/examples/grouping/grouping.go new file mode 100644 index 000000000..85802dd4f --- /dev/null +++ b/components/retriever/milvus2/examples/grouping/grouping.go @@ -0,0 +1,90 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates Grouping Search (returning top results per group). +// +// Prerequisites: +// Run the demo indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/demo && go run . +// +// The collection "demo_milvus2" has a "category" field for grouping. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + collectionName := "demo_milvus2" + dim := 128 + + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: collectionName, + VectorField: "vector", + OutputFields: []string{"id", "content", "category"}, + TopK: 10, + SearchMode: search_mode.NewApproximate(milvus2.COSINE), + Embedding: &mockEmbedding{dim: dim}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Retriever created successfully") + + // Perform grouping search (group by category, max 2 per group) + results, err := retriever.Retrieve(ctx, "search query", + milvus2.WithGrouping("category", 2, false), + ) + if err != nil { + log.Fatalf("Search failed: %v", err) + } + + log.Printf("Found %d documents (grouped by category, max 2 per group)", len(results)) + for i, doc := range results { + cat := doc.MetaData["category"] + log.Printf(" %d. ID=%s, Category=%v", i+1, doc.ID, cat) + } +} + +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = 0.1 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/hybrid/hybrid.go b/components/retriever/milvus2/examples/hybrid/hybrid.go new file mode 100644 index 000000000..53fbd2cb7 --- /dev/null +++ b/components/retriever/milvus2/examples/hybrid/hybrid.go @@ -0,0 +1,130 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates Hybrid Search combining Dense and Sparse (BM25) vectors. +// Dense vectors capture semantic meaning, while BM25 sparse vectors enable keyword matching. +// Results are fused using RRFReranker for best-of-both-worlds retrieval. +// +// Note on Reranking: +// This example uses Milvus's server-side RRFReranker which fuses results from multiple +// vector searches BEFORE returning. For post-retrieval reranking (e.g., using cross-encoder +// models like Cohere Rerank), use Eino's document.Transformer interface instead. See: +// github.com/cloudwego/eino-ext/components/document/transformer/reranker +// +// Prerequisites: +// Run the indexer hybrid example first to create the collection with BM25 function: +// +// cd ../../../indexer/milvus2/examples/hybrid && go run . +// +// Requires Milvus 2.5+ for server-side BM25 function support. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Collection created by indexer hybrid example with BM25 function + collectionName := "eino_hybrid_test" + denseField := "vector" + sparseField := "sparse_vector" + + // Define Reranker: RRF combines scores from dense and sparse searches + reranker := milvusclient.NewRRFReranker().WithK(60) + + // Define Hybrid Mode with Dense + Sparse (BM25) SubRequests + hybridMode := search_mode.NewHybrid(reranker, + // Dense vector search (semantic similarity) + &search_mode.SubRequest{ + VectorField: denseField, + VectorType: milvus2.DenseVector, + TopK: 10, + MetricType: milvus2.L2, + }, + // Sparse vector search (BM25 keyword matching) + // Uses raw query text - Milvus generates sparse vector server-side + &search_mode.SubRequest{ + VectorField: sparseField, + VectorType: milvus2.SparseVector, + TopK: 10, + MetricType: milvus2.BM25, + }, + ) + + // Create Retriever + // Only dense embedding needed - sparse (BM25) is handled server-side + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: collectionName, + VectorField: denseField, + SparseVectorField: sparseField, + OutputFields: []string{"id", "content", "metadata"}, + TopK: 5, + SearchMode: hybridMode, + Embedding: &mockDenseEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Hybrid (Dense + BM25) Retriever created successfully") + + // Search with raw text + // - Dense: text is embedded via mockDenseEmbedding + // - Sparse: text is passed directly to Milvus for BM25 processing + docs, err := retriever.Retrieve(ctx, "scalable vector database") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + + fmt.Printf("\nFound %d documents (Hybrid Dense + BM25):\n", len(docs)) + for i, doc := range docs { + fmt.Printf("\n--- Document %d ---\n", i+1) + fmt.Printf("ID: %s\n", doc.ID) + fmt.Printf("Content: %s\n", doc.Content) + fmt.Printf("Score: %v\n", doc.Score()) + } +} + +// mockDenseEmbedding for demo purposes +type mockDenseEmbedding struct{ dim int } + +func (m *mockDenseEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/hybrid_chinese/hybrid_chinese.go b/components/retriever/milvus2/examples/hybrid_chinese/hybrid_chinese.go new file mode 100644 index 000000000..ada9e0f12 --- /dev/null +++ b/components/retriever/milvus2/examples/hybrid_chinese/hybrid_chinese.go @@ -0,0 +1,126 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates Chinese text retrieval using BM25 sparse vectors in Hybrid mode. +// Run the indexer hybrid_chinese example first to create the collection: +// +// cd ../../../indexer/milvus2/examples/hybrid_chinese && go run . +// +// Requires Milvus 2.5+ for server-side BM25 function support. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // Collection created by indexer hybrid_chinese example + collectionName := "eino_hybrid_chinese" + denseField := "vector" + sparseField := "sparse_vector" + + // Define Reranker: RRF combines scores from multiple searches + reranker := milvusclient.NewRRFReranker().WithK(60) + + // Define Hybrid Mode with Dense + Sparse (BM25) SubRequests + hybridMode := search_mode.NewHybrid(reranker, + // Dense vector search (semantic similarity) + &search_mode.SubRequest{ + VectorField: denseField, + VectorType: milvus2.DenseVector, + TopK: 10, + MetricType: milvus2.L2, + }, + // Sparse vector search (BM25 keyword matching) + // Uses raw query text - Milvus generates sparse vector server-side + &search_mode.SubRequest{ + VectorField: sparseField, + VectorType: milvus2.SparseVector, + TopK: 10, + MetricType: milvus2.BM25, + }, + ) + + // Create Retriever + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: collectionName, + VectorField: denseField, + SparseVectorField: sparseField, + OutputFields: []string{"id", "content", "metadata"}, + TopK: 5, + SearchMode: hybridMode, + Embedding: &mockDenseEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Chinese Hybrid Retriever created successfully") + + // Search with Chinese query text + // - Dense: text is embedded via mockDenseEmbedding + // - Sparse: text is passed directly to Milvus for BM25 processing with Chinese tokenizer + queries := []string{ + "向量数据库", + "AI 应用", + "框架", + } + + for _, query := range queries { + fmt.Printf("\n=== Query: %s ===\n", query) + + docs, err := retriever.Retrieve(ctx, query) + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + + fmt.Printf("Found %d documents:\n", len(docs)) + for i, doc := range docs { + fmt.Printf("%d. [ID: %s] [Score: %.4f] %s\n", i+1, doc.ID, doc.Score(), doc.Content) + } + } +} + +// mockDenseEmbedding for demo purposes +type mockDenseEmbedding struct{ dim int } + +func (m *mockDenseEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = float64(j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/iterator/iterator.go b/components/retriever/milvus2/examples/iterator/iterator.go new file mode 100644 index 000000000..358cf5004 --- /dev/null +++ b/components/retriever/milvus2/examples/iterator/iterator.go @@ -0,0 +1,101 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates Iterator Search for traversing large result sets. +// +// Prerequisites: +// Run the demo indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/demo && go run . +// +// The collection "demo_milvus2" has 60 documents for testing batch retrieval. +package main + +import ( + "context" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + collectionName := "demo_milvus2" + dim := 128 + + // Fetch 25 items using small batch size to force iteration + totalTopK := 25 + batchSize := 10 + + // Create retriever with Iterator search mode + // Iterator fetches results in batches to handle large result sets efficiently. + // BatchSize controls how many items are fetched per network call. + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: collectionName, + VectorField: "vector", + OutputFields: []string{"id", "content"}, + TopK: totalTopK, + SearchMode: search_mode.NewIterator(milvus2.COSINE, batchSize), + Embedding: &mockEmbedding{dim: dim}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + + log.Printf("Starting Iterator Search (TopK=%d, BatchSize=%d)", totalTopK, batchSize) + results, err := retriever.Retrieve(ctx, "query") + if err != nil { + log.Fatalf("Search failed: %v", err) + } + + log.Printf("Found %d documents (Expected %d)", len(results), totalTopK) + for i, doc := range results { + if i == 0 || i == len(results)-1 { + log.Printf("Result %d: ID=%s, Content=%s, Score=%.4f", i+1, doc.ID, doc.Content, doc.Score()) + } + } + + if len(results) != totalTopK { + log.Fatalf("FAILED: expected %d results, got %d", totalTopK, len(results)) + } else { + log.Println("SUCCESS: Retrieved expected number of documents via iterator") + } +} + +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + vec[j] = 0.1 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/range/range.go b/components/retriever/milvus2/examples/range/range.go new file mode 100644 index 000000000..e79bd9252 --- /dev/null +++ b/components/retriever/milvus2/examples/range/range.go @@ -0,0 +1,95 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates range search (searching within a similarity radius). +// +// Prerequisites: +// Run the hnsw indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/hnsw && go run . +// +// The collection "demo_hnsw" will be used for search. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/components/embedding" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + + // 1. Create Retriever with Range Search + // Range search returns all vectors within a similarity/distance radius. + // For COSINE: radius is minimum similarity (0.99 = very similar). + // For L2: radius is maximum distance. + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: "demo_hnsw", // Using collection from hnsw example + VectorField: "vector", + OutputFields: []string{"id", "content", "metadata"}, + TopK: 100, // TopK acts as a limit for range search results + SearchMode: search_mode.NewRange(milvus2.COSINE, 0.99), + Embedding: &mockEmbedding{dim: 128}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Range Retriever created successfully") + + // 2. Perform Search + // This should return documents that are very close to the query + docs, err := retriever.Retrieve(ctx, "vector search query") + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + + fmt.Printf("\nFound %d documents within radius 0.99:\n", len(docs)) + for i, doc := range docs { + fmt.Printf("\n--- Document %d ---\n", i+1) + fmt.Printf("ID: %s\n", doc.ID) + fmt.Printf("Score: %v\n", doc.Score()) + } +} + +// mockEmbedding generates embeddings for demonstration +type mockEmbedding struct{ dim int } + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + result := make([][]float64, len(texts)) + for i := range texts { + vec := make([]float64, m.dim) + for j := range vec { + // Matches the logic in hnsw example to ensure high similarity + vec[j] = float64(j) * 0.01 + } + result[i] = vec + } + return result, nil +} diff --git a/components/retriever/milvus2/examples/scalar/scalar.go b/components/retriever/milvus2/examples/scalar/scalar.go new file mode 100644 index 000000000..b4bf37754 --- /dev/null +++ b/components/retriever/milvus2/examples/scalar/scalar.go @@ -0,0 +1,89 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This example demonstrates Scalar Search (metadata-only filtering). +// +// Prerequisites: +// Run the demo indexer first to create the collection: +// +// cd ../../../indexer/milvus2/examples/demo && go run . +// +// The collection "demo_milvus2" has documents with tag and year metadata. +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" +) + +func main() { + addr := os.Getenv("MILVUS_ADDR") + if addr == "" { + addr = "localhost:19530" + } + + ctx := context.Background() + collectionName := "demo_milvus2" + + // Create retriever with Scalar search mode + // Scalar uses the Query API (metadata filtering) instead of vector search. + // The query string is treated as a boolean filter expression. + retriever, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: addr}, + Collection: collectionName, + TopK: 10, + VectorField: "vector", + OutputFields: []string{"content", "metadata"}, + SearchMode: search_mode.NewScalar(), + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + log.Println("Retriever created successfully") + + // Case A: Query by ID + fmt.Println("\n--- Query: id == 'doc_1' ---") + res, err := retriever.Retrieve(ctx, "id == 'doc_1'") + printRes(res, err) + + // Case B: Query by JSON Metadata + fmt.Println("\n--- Query: metadata['tag'] == 'coding' ---") + res, err = retriever.Retrieve(ctx, "metadata['tag'] == 'coding'") + printRes(res, err) + + // Case C: Complex Query + fmt.Println("\n--- Query: metadata['year'] == 2024 and metadata['tag'] == 'life' ---") + res, err = retriever.Retrieve(ctx, "metadata['year'] == 2024 && metadata['tag'] == 'life'") + printRes(res, err) +} + +func printRes(docs []*schema.Document, err error) { + if err != nil { + log.Printf("Error: %v\n", err) + return + } + for _, d := range docs { + fmt.Printf("ID: %s, Content: %s\n", d.ID, d.Content) + } +} diff --git a/components/retriever/milvus2/examples/sparse/sparse.go b/components/retriever/milvus2/examples/sparse/sparse.go new file mode 100644 index 000000000..a287f240d --- /dev/null +++ b/components/retriever/milvus2/examples/sparse/sparse.go @@ -0,0 +1,66 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fmt" + "log" + + "github.com/cloudwego/eino-ext/components/retriever/milvus2" + "github.com/cloudwego/eino-ext/components/retriever/milvus2/search_mode" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// This example demonstrates Sparse-only retrieval using BM25. +// It uses the collection created by the indexer sparse example. + +func main() { + ctx := context.Background() + collectionName := "eino_sparse_test" + milvusAddr := "localhost:19530" + + // Create retriever with Sparse search mode (BM25) + // Sparse mode sends raw text to Milvus for server-side BM25 processing. + // No Embedding is needed as the sparse vector is generated server-side. + r, err := milvus2.NewRetriever(ctx, &milvus2.RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{ + Address: milvusAddr, + }, + Collection: collectionName, + SparseVectorField: "sparse_vector", + SearchMode: search_mode.NewSparse(milvus2.BM25), + TopK: 3, + OutputFields: []string{"*"}, + }) + if err != nil { + log.Fatalf("Failed to create retriever: %v", err) + } + + // Retrieve + query := "information retrieval" + docs, err := r.Retrieve(ctx, query) + if err != nil { + log.Fatalf("Failed to retrieve: %v", err) + } + + fmt.Printf("=== Query: %s ===\n", query) + fmt.Printf("Found %d documents:\n", len(docs)) + for _, doc := range docs { + fmt.Printf("ID: %s, Score: %.4f, Content: %s\n", doc.ID, doc.Score(), doc.Content) + } +} diff --git a/components/retriever/milvus2/go.mod b/components/retriever/milvus2/go.mod new file mode 100644 index 000000000..d25026b22 --- /dev/null +++ b/components/retriever/milvus2/go.mod @@ -0,0 +1,142 @@ +module github.com/cloudwego/eino-ext/components/retriever/milvus2 + +go 1.24.6 + +require ( + github.com/bytedance/mockey v1.4.0 + github.com/bytedance/sonic v1.14.1 + github.com/cloudwego/eino v0.6.0 + github.com/milvus-io/milvus/client/v2 v2.6.1 + github.com/smartystreets/goconvey v1.8.1 +) + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.11.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect + github.com/cockroachdb/redact v1.1.3 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.2 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/getsentry/sentry-go v0.12.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3 // indirect + github.com/milvus-io/milvus/pkg/v2 v2.6.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/panjf2000/ants/v2 v2.11.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/samber/lo v1.27.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.etcd.io/bbolt v1.3.8 // indirect + go.etcd.io/etcd/api/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/v2 v2.305.10 // indirect + go.etcd.io/etcd/client/v3 v3.5.10 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/raft/v3 v3.5.10 // indirect + go.etcd.io/etcd/server/v3 v3.5.10 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect + go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e // indirect + google.golang.org/grpc v1.71.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.32.3 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/components/retriever/milvus2/go.sum b/components/retriever/milvus2/go.sum new file mode 100644 index 000000000..1314946b2 --- /dev/null +++ b/components/retriever/milvus2/go.sum @@ -0,0 +1,679 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.4.0 h1:xwuZ3rr4mpbGkkBOYoSM+cO112dvzQ/sY0cVdP9FBSA= +github.com/bytedance/mockey v1.4.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= +github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.6.0 h1:pobGKMOfcQHVNhD9UT/HrvO0eYG6FC2ML/NKY2Eb9+Q= +github.com/cloudwego/eino v0.6.0/go.mod h1:JNapfU+QUrFFpboNDrNOFvmz0m9wjBFHHCr77RH6a50= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eino-contrib/jsonschema v1.0.2 h1:HaxruBMUdnXa7Lg/lX8g0Hk71ZIfdTZXmBQz0e3esr8= +github.com/eino-contrib/jsonschema v1.0.2/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3 h1:w7IBrU25KULWNlHKoKwx6ruTsDAmzrWknotIc6A4ys4= +github.com/milvus-io/milvus-proto/go-api/v2 v2.6.3/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus/client/v2 v2.6.1 h1:JGV+2JoZypc0ORnVj41ZWLdz9EpBGcwXCliIFXFW1f4= +github.com/milvus-io/milvus/client/v2 v2.6.1/go.mod h1:MnickP646pUKhfOS4JQD3uMUukDXhJKpdTXk467MXuU= +github.com/milvus-io/milvus/pkg/v2 v2.6.3 h1:WDf4mXFWL5Sk/V87yLwRKq24MYMkjS2YA6qraXbLbJA= +github.com/milvus-io/milvus/pkg/v2 v2.6.3/go.mod h1:49umaGHK9nKHJNtgBlF/iB24s1sZ/SG5/Q7iLj/Gc14= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= +github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= +github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ= +github.com/samber/lo v1.27.0/go.mod h1:it33p9UtPMS7z72fP4gw/EIfQB2eI8ke7GR2wc6+Rhg= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4= +go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.etcd.io/etcd/pkg/v3 v3.5.10 h1:WPR8K0e9kWl1gAhB5A7gEa5ZBTNkT9NdNWrR8Qpo1CM= +go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs= +go.etcd.io/etcd/raft/v3 v3.5.10 h1:cgNAYe7xrsrn/5kXMSaH8kM/Ky8mAdMqGOxyYwpP0LA= +go.etcd.io/etcd/raft/v3 v3.5.10/go.mod h1:odD6kr8XQXTy9oQnyMPBOr0TVe+gT0neQhElQ6jbGRc= +go.etcd.io/etcd/server/v3 v3.5.10 h1:4NOGyOwD5sUZ22PiWYKmfxqoeh72z6EhYjNosKGLmZg= +go.etcd.io/etcd/server/v3 v3.5.10/go.mod h1:gBplPHfs6YI0L+RpGkTQO7buDbHv5HJGG/Bst0/zIPo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= +golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e h1:YA5lmSs3zc/5w+xsRcHqpETkaYyK63ivEPzNTcUUlSA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250227231956-55c901821b1e/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/components/retriever/milvus2/options.go b/components/retriever/milvus2/options.go new file mode 100644 index 000000000..e81759941 --- /dev/null +++ b/components/retriever/milvus2/options.go @@ -0,0 +1,59 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import "github.com/cloudwego/eino/components/retriever" + +// ImplOptions contains implementation-specific options for retrieval operations. +type ImplOptions struct { + // Filter is a boolean expression for filtering search results. + // Refer to https://milvus.io/docs/boolean.md for filter syntax. + Filter string + + // Grouping configuration for grouping search. + Grouping *GroupingConfig +} + +// WithFilter returns an option that sets a boolean filter expression for search results. +// See https://milvus.io/docs/boolean.md for filter syntax. +func WithFilter(filter string) retriever.Option { + return retriever.WrapImplSpecificOptFn(func(o *ImplOptions) { + o.Filter = filter + }) +} + +// GroupingConfig contains configuration for grouping search results by a specific field. +type GroupingConfig struct { + // GroupByField specifies the field name to group results by. + GroupByField string + // GroupSize specifies the number of results to return per group. + GroupSize int + // StrictGroupSize enforces exact group size even if it degrades performance. + StrictGroupSize bool +} + +// WithGrouping returns an option that enables grouping search by a specified field. +// It configures the group field, number of items per group, and whether to enforce strict sizing. +func WithGrouping(groupByField string, groupSize int, strict bool) retriever.Option { + return retriever.WrapImplSpecificOptFn(func(o *ImplOptions) { + o.Grouping = &GroupingConfig{ + GroupByField: groupByField, + GroupSize: groupSize, + StrictGroupSize: strict, + } + }) +} diff --git a/components/retriever/milvus2/retriever.go b/components/retriever/milvus2/retriever.go new file mode 100644 index 000000000..abd0cf030 --- /dev/null +++ b/components/retriever/milvus2/retriever.go @@ -0,0 +1,267 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "context" + "fmt" + + "github.com/bytedance/sonic" + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// RetrieverConfig contains configuration for the Milvus2 retriever. +type RetrieverConfig struct { + // Client is an optional pre-configured Milvus client. + // If not provided, the component will create one using ClientConfig. + Client *milvusclient.Client + + // ClientConfig for creating Milvus client if Client is not provided. + ClientConfig *milvusclient.ClientConfig + + // Collection is the collection name in Milvus. + // Default: "eino_collection" + Collection string + + // Partitions to search. Empty means search all partitions. + Partitions []string + + // VectorField is the name of the vector field in the collection. + // Default: "vector" + VectorField string + + // SparseVectorField is the field name for sparse vectors. + // Default: "sparse_vector" + SparseVectorField string + + // OutputFields specifies which fields to return in search results. + // Default: all fields + OutputFields []string + + // TopK is the number of results to return. + // Default: 5 + TopK int + + // ConsistencyLevel for Milvus operations. + // Default: ConsistencyLevelBounded + ConsistencyLevel ConsistencyLevel + + // SearchMode defines the search strategy. + // Required. + SearchMode SearchMode + + // DocumentConverter converts Milvus search results to EINO documents. + // If nil, uses default conversion. + DocumentConverter func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) + + // Embedding is the embedder for query vectorization. + // Optional. Required if SearchMode uses vector search. + Embedding embedding.Embedder +} + +// Retriever implements the retriever.Retriever interface for Milvus 2.x using the V2 SDK. +type Retriever struct { + client *milvusclient.Client + config *RetrieverConfig +} + +// NewRetriever creates a new Milvus2 retriever with the provided configuration. +// It returns an error if the configuration is invalid. +func NewRetriever(ctx context.Context, conf *RetrieverConfig) (*Retriever, error) { + if err := conf.validate(); err != nil { + return nil, err + } + + cli, err := initClient(ctx, conf) + if err != nil { + return nil, err + } + + if err := loadCollection(ctx, cli, conf); err != nil { + return nil, err + } + + return &Retriever{ + client: cli, + config: conf, + }, nil +} + +func initClient(ctx context.Context, conf *RetrieverConfig) (*milvusclient.Client, error) { + if conf.Client != nil { + return conf.Client, nil + } + + if conf.ClientConfig == nil { + return nil, fmt.Errorf("[NewRetriever] either Client or ClientConfig must be provided") + } + + cli, err := milvusclient.New(ctx, conf.ClientConfig) + if err != nil { + return nil, fmt.Errorf("[NewRetriever] failed to create milvus client: %w", err) + } + + return cli, nil +} + +func loadCollection(ctx context.Context, cli *milvusclient.Client, conf *RetrieverConfig) error { + hasCollection, err := cli.HasCollection(ctx, milvusclient.NewHasCollectionOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewRetriever] failed to check collection: %w", err) + } + if !hasCollection { + return fmt.Errorf("[NewRetriever] collection %q not found", conf.Collection) + } + + loadState, err := cli.GetLoadState(ctx, milvusclient.NewGetLoadStateOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewRetriever] failed to get load state: %w", err) + } + if loadState.State != entity.LoadStateLoaded { + loadTask, err := cli.LoadCollection(ctx, milvusclient.NewLoadCollectionOption(conf.Collection)) + if err != nil { + return fmt.Errorf("[NewRetriever] failed to load collection: %w", err) + } + if err := loadTask.Await(ctx); err != nil { + return fmt.Errorf("[NewRetriever] failed to await collection load: %w", err) + } + } + return nil +} + +// Retrieve searches for documents matching the given query. +// It returns the matching documents or an error. +func (r *Retriever) Retrieve(ctx context.Context, query string, opts ...retriever.Option) (docs []*schema.Document, err error) { + ctx = callbacks.EnsureRunInfo(ctx, r.GetType(), components.ComponentOfRetriever) + ctx = callbacks.OnStart(ctx, &retriever.CallbackInput{ + Query: query, + TopK: r.config.TopK, + }) + defer func() { + if err != nil { + callbacks.OnError(ctx, err) + } + }() + + docs, err = r.config.SearchMode.Retrieve(ctx, r.client, r.config, query, opts...) + if err != nil { + return nil, err + } + + callbacks.OnEnd(ctx, &retriever.CallbackOutput{Docs: docs}) + return docs, nil +} + +// GetType returns the type of the retriever. +func (r *Retriever) GetType() string { + return typ +} + +// IsCallbacksEnabled checks if callbacks are enabled for this retriever. +func (r *Retriever) IsCallbacksEnabled() bool { + return true +} + +// validate checks the configuration and sets default values. +func (c *RetrieverConfig) validate() error { + if c.Client == nil && c.ClientConfig == nil { + return fmt.Errorf("[NewRetriever] milvus client or client config not provided") + } + if c.SearchMode == nil { + return fmt.Errorf("[NewRetriever] search mode not provided") + } + // Embedding validation is delegated to the specific SearchMode implementation. + if c.Collection == "" { + c.Collection = defaultCollection + } + if c.VectorField == "" { + c.VectorField = defaultVectorField + } + if c.SparseVectorField == "" { + c.SparseVectorField = defaultSparseVectorField + } + if len(c.OutputFields) == 0 { + c.OutputFields = []string{"*"} + } + if c.TopK <= 0 { + c.TopK = defaultTopK + } + if c.DocumentConverter == nil { + c.DocumentConverter = defaultDocumentConverter() + } + return nil +} + +// defaultDocumentConverter returns the default result to document converter. +func defaultDocumentConverter() func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + docs := make([]*schema.Document, 0, result.ResultCount) + + for i := 0; i < result.ResultCount; i++ { + doc := &schema.Document{ + MetaData: make(map[string]any), + } + + if i < len(result.Scores) { + doc = doc.WithScore(float64(result.Scores[i])) + } + + for _, field := range result.Fields { + val, err := field.Get(i) + if err != nil { + continue + } + + switch field.Name() { + case "id": + if id, ok := val.(string); ok { + doc.ID = id + } else if idStr, err := field.GetAsString(i); err == nil { + doc.ID = idStr + } + case defaultContentField: + if content, ok := val.(string); ok { + doc.Content = content + } else if contentStr, err := field.GetAsString(i); err == nil { + doc.Content = contentStr + } + case defaultMetadataField: + if metaBytes, ok := val.([]byte); ok { + var meta map[string]any + if err := sonic.Unmarshal(metaBytes, &meta); err == nil { + for k, v := range meta { + doc.MetaData[k] = v + } + } + } + default: + doc.MetaData[field.Name()] = val + } + } + + docs = append(docs, doc) + } + + return docs, nil + } +} diff --git a/components/retriever/milvus2/retriever_test.go b/components/retriever/milvus2/retriever_test.go new file mode 100644 index 000000000..6c8b29004 --- /dev/null +++ b/components/retriever/milvus2/retriever_test.go @@ -0,0 +1,488 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/column" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" +) + +// mockEmbedding implements embedding.Embedder for testing +type mockEmbedding struct { + err error + dims int +} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + result := make([][]float64, len(texts)) + dims := m.dims + if dims == 0 { + dims = 128 + } + for i := range texts { + result[i] = make([]float64, dims) + for j := 0; j < dims; j++ { + result[i][j] = 0.1 + } + } + return result, nil +} + +// mockSearchMode implements SearchMode for testing (avoids import cycle) +type mockSearchMode struct { + retrieveFunc func(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) +} + +func (m *mockSearchMode) Retrieve(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + if m.retrieveFunc != nil { + return m.retrieveFunc(ctx, client, conf, query, opts...) + } + return []*schema.Document{{ID: "1", Content: "doc1"}}, nil +} + +func TestRetrieverConfig_validate(t *testing.T) { + convey.Convey("test RetrieverConfig.validate", t, func() { + mockEmb := &mockEmbedding{} + mockSM := &mockSearchMode{} + + convey.Convey("test missing client and client config", func() { + config := &RetrieverConfig{ + Client: nil, + ClientConfig: nil, + Collection: "test_collection", + Embedding: mockEmb, + SearchMode: mockSM, + } + err := config.validate() + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "client") + }) + + // QuerySearchMode and SparseSearchMode specific validations are removed as SearchMode is now polymorphic. + // The requirement for Embedding is now enforced by specific implementations inside Retrieve/BuildOptions if needed, not strictly by Config.validate for generic SearchMode. + // However, if the code still checks for embedding availability generally: + + convey.Convey("test valid config", func() { + config := &RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "test_collection", + Embedding: mockEmb, + SearchMode: mockSM, + } + err := config.validate() + convey.So(err, convey.ShouldBeNil) + }) + + convey.Convey("test missing search mode", func() { + config := &RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "test_collection", + Embedding: mockEmb, + SearchMode: nil, + } + err := config.validate() + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "search mode") + }) + }) +} + +func TestNewRetriever(t *testing.T) { + PatchConvey("test NewRetriever", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockSM := &mockSearchMode{} + + PatchConvey("test missing client and client config", func() { + _, err := NewRetriever(ctx, &RetrieverConfig{ + Client: nil, + ClientConfig: nil, + Collection: "test_collection", + Embedding: mockEmb, + SearchMode: mockSM, + }) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "client") + }) + + PatchConvey("test missing search mode", func() { + _, err := NewRetriever(ctx, &RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost:19530"}, + Collection: "test_collection", + Embedding: mockEmb, + SearchMode: nil, + }) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "search mode") + }) + }) +} + +func TestRetriever_GetType(t *testing.T) { + convey.Convey("test Retriever.GetType", t, func() { + r := &Retriever{} + result := r.GetType() + convey.So(result, convey.ShouldEqual, "Milvus2") + }) +} + +func TestRetriever_IsCallbacksEnabled(t *testing.T) { + convey.Convey("test Retriever.IsCallbacksEnabled", t, func() { + r := &Retriever{ + config: &RetrieverConfig{}, + } + result := r.IsCallbacksEnabled() + convey.So(result, convey.ShouldBeTrue) + }) +} + +func TestRetriever_Retrieve(t *testing.T) { + PatchConvey("test Retriever.Retrieve", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockClient := &milvusclient.Client{} + mockSM := &mockSearchMode{} + + r := &Retriever{ + client: mockClient, + config: &RetrieverConfig{ + Collection: "test_collection", + Embedding: mockEmb, + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + SearchMode: mockSM, + }, + } + + PatchConvey("test retrieve success", func() { + expectedDocs := []*schema.Document{{ID: "1", Content: "success"}} + mockSM.retrieveFunc = func(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + return expectedDocs, nil + } + docs, err := r.Retrieve(ctx, "test query") + convey.So(err, convey.ShouldBeNil) + convey.So(docs, convey.ShouldResemble, expectedDocs) + }) + + PatchConvey("test retrieve error", func() { + mockSM.retrieveFunc = func(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + return nil, fmt.Errorf("search error") + } + docs, err := r.Retrieve(ctx, "test query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "search error") + convey.So(docs, convey.ShouldBeNil) + }) + }) +} + +func TestWithFilter(t *testing.T) { + convey.Convey("test WithFilter option", t, func() { + opt := WithFilter("id > 10") + convey.So(opt, convey.ShouldNotBeNil) + }) +} + +func TestWithGrouping(t *testing.T) { + convey.Convey("test WithGrouping option", t, func() { + opt := WithGrouping("category", 3, true) + convey.So(opt, convey.ShouldNotBeNil) + }) +} + +func TestDocumentConverter(t *testing.T) { + convey.Convey("test defaultDocumentConverter", t, func() { + ctx := context.Background() + converter := defaultDocumentConverter() + + convey.Convey("convert standard results with scores and metadata", func() { + ids := []string{"1", "2"} + contents := []string{"doc1", "doc2"} + scores := []float32{0.9, 0.8} + // Metadata: {"key": "val1"}, {"key": "val2"} + metas := [][]byte{ + []byte(`{"key": "val1"}`), + []byte(`{"key": "val2"}`), + } + + resultSet := createMockResultSet(ids, contents, scores, metas) + docs, err := converter(ctx, resultSet) + + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 2) + + // Check first doc + convey.So(docs[0].ID, convey.ShouldEqual, "1") + convey.So(docs[0].Content, convey.ShouldEqual, "doc1") + convey.So(docs[0].Score(), convey.ShouldAlmostEqual, 0.9, 0.0001) + _, ok := docs[0].MetaData["score"] + convey.So(ok, convey.ShouldBeFalse) + convey.So(docs[0].MetaData["key"], convey.ShouldEqual, "val1") + + // Check second doc + convey.So(docs[1].ID, convey.ShouldEqual, "2") + convey.So(docs[1].Content, convey.ShouldEqual, "doc2") + convey.So(docs[1].Score(), convey.ShouldAlmostEqual, 0.8, 0.0001) + _, ok = docs[1].MetaData["score"] + convey.So(ok, convey.ShouldBeFalse) + convey.So(docs[1].MetaData["key"], convey.ShouldEqual, "val2") + }) + + convey.Convey("convert results without scores (e.g. Query)", func() { + ids := []string{"3"} + contents := []string{"doc3"} + metas := [][]byte{[]byte(`{}`)} + + resultSet := createMockQueryResult(ids, contents, metas) + docs, err := converter(ctx, resultSet) + + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + convey.So(docs[0].ID, convey.ShouldEqual, "3") + _, hasScore := docs[0].MetaData["score"] + convey.So(hasScore, convey.ShouldBeFalse) + }) + }) +} + +// Helper to create a mock ResultSet using real column implementations +func createMockResultSet(ids []string, contents []string, scores []float32, metadatas [][]byte) milvusclient.ResultSet { + count := len(ids) + if len(contents) != count { + count = 0 + } + + // Create columns using official SDK constructors + idCol := column.NewColumnVarChar("id", ids) + contentCol := column.NewColumnVarChar("content", contents) + metaCol := column.NewColumnJSONBytes("metadata", metadatas) + + // Cast to column.Column interface + fields := []column.Column{idCol, contentCol, metaCol} + + return milvusclient.ResultSet{ + ResultCount: count, + IDs: idCol, + Scores: scores, + Fields: fields, + } +} + +// For Query results, which are just ResultSet in v2 +func createMockQueryResult(ids []string, contents []string, metadatas [][]byte) milvusclient.ResultSet { + return createMockResultSet(ids, contents, nil, metadatas) +} + +func TestInitClient(t *testing.T) { + PatchConvey("test initClient", t, func() { + ctx := context.Background() + + PatchConvey("test with provided client", func() { + mockClient := &milvusclient.Client{} + conf := &RetrieverConfig{Client: mockClient} + cli, err := initClient(ctx, conf) + convey.So(err, convey.ShouldBeNil) + convey.So(cli, convey.ShouldEqual, mockClient) + }) + + PatchConvey("test with missing client and config", func() { + conf := &RetrieverConfig{Client: nil, ClientConfig: nil} + cli, err := initClient(ctx, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(cli, convey.ShouldBeNil) + }) + + PatchConvey("test new client creation success", func() { + Mock(milvusclient.New).Return(&milvusclient.Client{}, nil).Build() + conf := &RetrieverConfig{ClientConfig: &milvusclient.ClientConfig{Address: "localhost"}} + cli, err := initClient(ctx, conf) + convey.So(err, convey.ShouldBeNil) + convey.So(cli, convey.ShouldNotBeNil) + }) + + PatchConvey("test new client creation failure", func() { + Mock(milvusclient.New).Return(nil, fmt.Errorf("connection error")).Build() + conf := &RetrieverConfig{ClientConfig: &milvusclient.ClientConfig{Address: "localhost"}} + cli, err := initClient(ctx, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(cli, convey.ShouldBeNil) + }) + }) +} + +func TestLoadCollection(t *testing.T) { + PatchConvey("test loadCollection", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + conf := &RetrieverConfig{Collection: "test_coll"} + + PatchConvey("HasCollection error", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(false, fmt.Errorf("network error")).Build() + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to check collection") + }) + + PatchConvey("Collection not found", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(false, nil).Build() + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "not found") + }) + + PatchConvey("GetLoadState error", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{}, fmt.Errorf("state error")).Build() + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to get load state") + }) + + PatchConvey("Already loaded", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateLoaded}, nil).Build() + // Should return nil directly + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldBeNil) + }) + + PatchConvey("Not loaded, load success", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateCode(0)}, nil).Build() + + Mock(GetMethod(mockClient, "LoadCollection")).Return(milvusclient.LoadTask{}, nil).Build() + Mock(GetMethod(&milvusclient.LoadTask{}, "Await")).Return(nil).Build() + + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldBeNil) + }) + + PatchConvey("LoadCollection error", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateCode(0)}, nil).Build() + Mock(GetMethod(mockClient, "LoadCollection")).Return(milvusclient.LoadTask{}, fmt.Errorf("load error")).Build() + + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to load collection") + }) + + PatchConvey("Await error", func() { + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateCode(0)}, nil).Build() + + Mock(GetMethod(mockClient, "LoadCollection")).Return(milvusclient.LoadTask{}, nil).Build() + Mock(GetMethod(&milvusclient.LoadTask{}, "Await")).Return(fmt.Errorf("await error")).Build() + + err := loadCollection(ctx, mockClient, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "failed to await collection load") + }) + }) +} + +func TestNewRetriever_FullFlow(t *testing.T) { + PatchConvey("test NewRetriever full flow", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockSM := &mockSearchMode{} + conf := &RetrieverConfig{ + ClientConfig: &milvusclient.ClientConfig{Address: "localhost"}, + Collection: "test", + Embedding: mockEmb, + SearchMode: mockSM, + } + + PatchConvey("validation error", func() { + badConf := &RetrieverConfig{} // missing everything + r, err := NewRetriever(ctx, badConf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("initClient error", func() { + Mock(milvusclient.New).Return(nil, fmt.Errorf("init error")).Build() + r, err := NewRetriever(ctx, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("loadCollection error", func() { + Mock(milvusclient.New).Return(&milvusclient.Client{}, nil).Build() + Mock(GetMethod(&milvusclient.Client{}, "HasCollection")).Return(false, fmt.Errorf("check error")).Build() + r, err := NewRetriever(ctx, conf) + convey.So(err, convey.ShouldNotBeNil) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("success", func() { + mockClient := &milvusclient.Client{} + Mock(milvusclient.New).Return(mockClient, nil).Build() + Mock(GetMethod(mockClient, "HasCollection")).Return(true, nil).Build() + Mock(GetMethod(mockClient, "GetLoadState")).Return(entity.LoadState{State: entity.LoadStateLoaded}, nil).Build() + + r, err := NewRetriever(ctx, conf) + convey.So(err, convey.ShouldBeNil) + convey.So(r, convey.ShouldNotBeNil) + }) + }) +} + +func TestRetrieve_Callbacks(t *testing.T) { + PatchConvey("test Retrieve callbacks", t, func() { + ctx := context.Background() + mockEmb := &mockEmbedding{dims: 128} + mockClient := &milvusclient.Client{} + mockSM := &mockSearchMode{} + + r := &Retriever{ + client: mockClient, + config: &RetrieverConfig{ + Collection: "test_collection", + Embedding: mockEmb, + VectorField: "vector", + TopK: 10, + SearchMode: mockSM, + }, + } + + // We just want to ensure Retrieve runs through without panic and hits the SearchMode.Retrieve + // Detailed callback verification is complex without mocking callbacks pkg, but we cover the lines + // by executing Retrieve. + mockSM.retrieveFunc = func(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + return []*schema.Document{}, nil + } + + docs, err := r.Retrieve(ctx, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(docs, convey.ShouldNotBeNil) + }) +} diff --git a/components/retriever/milvus2/search_mode.go b/components/retriever/milvus2/search_mode.go new file mode 100644 index 000000000..1b9b52fe6 --- /dev/null +++ b/components/retriever/milvus2/search_mode.go @@ -0,0 +1,32 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "context" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" +) + +// SearchMode defines the interface for executing Milvus search operations. +// It encapsulates the entire search logic, including option building and client execution. +type SearchMode interface { + // Retrieve performs the search operation using the provided client and configuration. + Retrieve(ctx context.Context, client *milvusclient.Client, conf *RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) +} diff --git a/components/retriever/milvus2/search_mode/approximate.go b/components/retriever/milvus2/search_mode/approximate.go new file mode 100644 index 000000000..4758d13d9 --- /dev/null +++ b/components/retriever/milvus2/search_mode/approximate.go @@ -0,0 +1,121 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package search_mode provides search mode implementations for the milvus2 retriever. +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Approximate implements approximate nearest neighbor (ANN) search. +// It provides efficient vector similarity search using Milvus indexes. +type Approximate struct { + // MetricType specifies the metric type for vector similarity. + // Default: L2. + MetricType milvus2.MetricType +} + +// NewApproximate creates a new Approximate search mode with the specified metric type. +func NewApproximate(metricType milvus2.MetricType) *Approximate { + if metricType == "" { + metricType = milvus2.L2 + } + return &Approximate{ + MetricType: metricType, + } +} + +// Retrieve performs the approximate vector search. +func (a *Approximate) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + if conf.Embedding == nil { + return nil, fmt.Errorf("embedding is required for approximate search") + } + + queryVector, err := EmbedQuery(ctx, conf.Embedding, query) + if err != nil { + return nil, err + } + + searchOpt, err := a.BuildSearchOption(ctx, conf, queryVector, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build search option: %w", err) + } + + result, err := client.Search(ctx, searchOpt) + if err != nil { + return nil, fmt.Errorf("failed to search: %w", err) + } + + if len(result) == 0 { + return []*schema.Document{}, nil + } + + return conf.DocumentConverter(ctx, result[0]) +} + +// BuildSearchOption creates a SearchOption for ANN search with the configured metric type. +func (a *Approximate) BuildSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, opts ...retriever.Option) (milvusclient.SearchOption, error) { + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + // Determine final topK + topK := conf.TopK + if co.TopK != nil { + topK = *co.TopK + } + + searchOpt := milvusclient.NewSearchOption(conf.Collection, topK, []entity.Vector{entity.FloatVector(queryVector)}). + WithANNSField(conf.VectorField). + WithOutputFields(conf.OutputFields...) + + // Apply metric type + if a.MetricType != "" { + searchOpt.WithSearchParam("metric_type", string(a.MetricType)) + } + + if len(conf.Partitions) > 0 { + searchOpt = searchOpt.WithPartitions(conf.Partitions...) + } + + if io.Filter != "" { + searchOpt = searchOpt.WithFilter(io.Filter) + } + + if io.Grouping != nil { + searchOpt = searchOpt.WithGroupByField(io.Grouping.GroupByField). + WithGroupSize(io.Grouping.GroupSize) + if io.Grouping.StrictGroupSize { + searchOpt = searchOpt.WithStrictGroupSize(true) + } + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + searchOpt = searchOpt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return searchOpt, nil +} diff --git a/components/retriever/milvus2/search_mode/approximate_test.go b/components/retriever/milvus2/search_mode/approximate_test.go new file mode 100644 index 000000000..1fae87210 --- /dev/null +++ b/components/retriever/milvus2/search_mode/approximate_test.go @@ -0,0 +1,207 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + . "github.com/bytedance/mockey" + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewApproximate(t *testing.T) { + convey.Convey("test NewApproximate", t, func() { + convey.Convey("test with default metric type", func() { + approx := NewApproximate("") + convey.So(approx, convey.ShouldNotBeNil) + convey.So(approx.MetricType, convey.ShouldEqual, milvus2.L2) + }) + + convey.Convey("test with L2 metric type", func() { + approx := NewApproximate(milvus2.L2) + convey.So(approx, convey.ShouldNotBeNil) + convey.So(approx.MetricType, convey.ShouldEqual, milvus2.L2) + }) + + convey.Convey("test with IP metric type", func() { + approx := NewApproximate(milvus2.IP) + convey.So(approx, convey.ShouldNotBeNil) + convey.So(approx.MetricType, convey.ShouldEqual, milvus2.IP) + }) + + convey.Convey("test with COSINE metric type", func() { + approx := NewApproximate(milvus2.COSINE) + convey.So(approx, convey.ShouldNotBeNil) + convey.So(approx.MetricType, convey.ShouldEqual, milvus2.COSINE) + }) + }) +} + +func TestApproximate_BuildSearchOption(t *testing.T) { + convey.Convey("test Approximate.BuildSearchOption", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + convey.Convey("test basic search option", func() { + approx := NewApproximate(milvus2.L2) + opt, err := approx.BuildSearchOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with partitions", func() { + configWithPartitions := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Partitions: []string{"partition1", "partition2"}, + } + approx := NewApproximate(milvus2.IP) + opt, err := approx.BuildSearchOption(ctx, configWithPartitions, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with filter option", func() { + approx := NewApproximate(milvus2.L2) + opt, err := approx.BuildSearchOption(ctx, config, queryVector, + milvus2.WithFilter("id > 10")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with grouping option", func() { + approx := NewApproximate(milvus2.L2) + opt, err := approx.BuildSearchOption(ctx, config, queryVector, + milvus2.WithGrouping("category", 3, true)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with custom TopK", func() { + approx := NewApproximate(milvus2.L2) + customTopK := 20 + opt, err := approx.BuildSearchOption(ctx, config, queryVector, + retriever.WithTopK(customTopK)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + }) +} + +// Verify interface implementation +func TestApproximate_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Approximate implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Approximate)(nil) + }) +} + +func TestApproximate_Retrieve(t *testing.T) { + PatchConvey("test Approximate.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + mockEmb := &mockApproxEmbedding{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Embedding: mockEmb, + } + + approx := NewApproximate(milvus2.L2) + + PatchConvey("success", func() { + // Mock EmbedQuery implicitly by mocking EmbedStrings in mockEmbedding + // Or we assume EmbedQuery works (it's tested elsewhere) + + // Mock Client.Search + Mock(GetMethod(mockClient, "Search")).Return([]milvusclient.ResultSet{ + { + ResultCount: 1, + IDs: nil, // Simplified + }, + }, nil).Build() + + // Mock DocumentConverter to avoid complex ResultSet construction + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := approx.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("embedding error", func() { + mockEmb.err = fmt.Errorf("embed error") + docs, err := approx.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + + PatchConvey("search error", func() { + // Restore embedding + mockEmb.err = nil + + Mock(GetMethod(mockClient, "Search")).Return(nil, fmt.Errorf("search error")).Build() + + docs, err := approx.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + + PatchConvey("empty result", func() { + mockEmb.err = nil + Mock(GetMethod(mockClient, "Search")).Return([]milvusclient.ResultSet{}, nil).Build() + + docs, err := approx.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 0) + }) + }) +} + +// mockApproxEmbedding implements embedding.Embedder for testing +type mockApproxEmbedding struct { + err error +} + +func (m *mockApproxEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + return [][]float64{make([]float64, 128)}, nil +} diff --git a/components/retriever/milvus2/search_mode/hybrid.go b/components/retriever/milvus2/search_mode/hybrid.go new file mode 100644 index 000000000..f1143625b --- /dev/null +++ b/components/retriever/milvus2/search_mode/hybrid.go @@ -0,0 +1,201 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Hybrid implements hybrid search with reranking. +// It allows defining multiple ANN requests that share the same query vector +// (e.g., searching against different vector fields with different parameters), +// and then fusing the results using a Reranker. +type Hybrid struct { + // SubRequests defines the configuration for each sub-search. + SubRequests []*SubRequest + + // Reranker is the ranker used to fuse results from sub-requests. + // Supported types: RRFReranker, WeightedReranker. + Reranker milvusclient.Reranker + + // TopK overrides the final number of results to return. + // If 0, uses RetrieverConfig.TopK. + TopK int +} + +// SubRequest defines a single ANN search request within a hybrid search. +type SubRequest struct { + // VectorField to search in. + // If empty, uses RetrieverConfig.VectorField. + VectorField string + + // MetricType for this request. + // Default: L2 + MetricType milvus2.MetricType + + // TopK for this specific sub-request. + // Default: 10 + TopK int + + // SearchParams are extra parameters (e.g. "nprobe", "ef"). + SearchParams map[string]string + + // VectorType specifies the type of vector field (e.g., DenseVector, SparseVector). + // Default: DenseVector + VectorType milvus2.VectorType +} + +// NewHybrid creates a new Hybrid search mode with the given reranker and sub-requests. +func NewHybrid(reranker milvusclient.Reranker, subRequests ...*SubRequest) *Hybrid { + return &Hybrid{ + Reranker: reranker, + SubRequests: subRequests, + } +} + +// BuildSearchOption returns an error because Hybrid search mode requires BuildHybridSearchOption. +func (h *Hybrid) BuildSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, opts ...retriever.Option) (milvusclient.SearchOption, error) { + return nil, fmt.Errorf("Hybrid search mode requires BuildHybridSearchOption") +} + +// Retrieve performs the hybrid search operation. +func (h *Hybrid) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + if conf.Embedding == nil { + return nil, fmt.Errorf("embedding is required for hybrid search") + } + + queryVector, err := EmbedQuery(ctx, conf.Embedding, query) + if err != nil { + return nil, err + } + + searchOpt, err := h.BuildHybridSearchOption(ctx, conf, queryVector, query, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build hybrid search option: %w", err) + } + + result, err := client.HybridSearch(ctx, searchOpt) + if err != nil { + return nil, fmt.Errorf("failed to hybrid search: %w", err) + } + + if len(result) == 0 { + return []*schema.Document{}, nil + } + + return conf.DocumentConverter(ctx, result[0]) +} + +// BuildHybridSearchOption creates a HybridSearchOption for multi-vector search with reranking. +// It is internal to the hybrid implementation now/helper but kept as method for cleaner code. +func (h *Hybrid) BuildHybridSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, query string, opts ...retriever.Option) (milvusclient.HybridSearchOption, error) { + // Validate SubRequests - hybrid requires at least 2 SubRequests for meaningful fusion + if len(h.SubRequests) < 2 { + return nil, fmt.Errorf("hybrid search requires at least 2 SubRequests; use Approximate or Sparse search mode for single-vector search") + } + + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + finalTopK := conf.TopK + if h.TopK > 0 { + finalTopK = h.TopK + } + if co.TopK != nil { + finalTopK = *co.TopK + } + + annRequests := make([]*milvusclient.AnnRequest, 0, len(h.SubRequests)) + for _, req := range h.SubRequests { + // Determine vector field + field := req.VectorField + if field == "" { + if req.VectorType == milvus2.SparseVector { + field = conf.SparseVectorField + } else { + field = conf.VectorField + } + } + + // Determine Limit + limit := req.TopK + if limit <= 0 { + limit = conf.TopK // Default to global TopK + } + + // Create ANN request based on VectorType + var annReq *milvusclient.AnnRequest + if req.VectorType == milvus2.SparseVector { + // Sparse vector: use raw text for BM25 function + annReq = milvusclient.NewAnnRequest(field, limit, entity.Text(query)) + } else { + // Dense vector: require query vector + if len(queryVector) == 0 { + return nil, fmt.Errorf("dense vector SubRequest requires embedding, but query vector is empty") + } + annReq = milvusclient.NewAnnRequest(field, limit, entity.FloatVector(queryVector)) + } + + // Apply search params + for k, v := range req.SearchParams { + annReq.WithSearchParam(k, v) + } + + if req.MetricType != "" { + annReq.WithSearchParam("metric_type", string(req.MetricType)) + } + + if io.Filter != "" { + annReq.WithFilter(io.Filter) + } + + if io.Grouping != nil { + annReq.WithGroupByField(io.Grouping.GroupByField). + WithGroupSize(io.Grouping.GroupSize) + if io.Grouping.StrictGroupSize { + annReq.WithStrictGroupSize(true) + } + } + + annRequests = append(annRequests, annReq) + } + + hybridOpt := milvusclient.NewHybridSearchOption(conf.Collection, finalTopK, annRequests...). + WithReranker(h.Reranker). + WithOutputFields(conf.OutputFields...) + + // Apply partitions + if len(conf.Partitions) > 0 { + hybridOpt = hybridOpt.WithPartitions(conf.Partitions...) + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + hybridOpt = hybridOpt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return hybridOpt, nil +} diff --git a/components/retriever/milvus2/search_mode/hybrid_test.go b/components/retriever/milvus2/search_mode/hybrid_test.go new file mode 100644 index 000000000..e61834685 --- /dev/null +++ b/components/retriever/milvus2/search_mode/hybrid_test.go @@ -0,0 +1,449 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewHybrid(t *testing.T) { + convey.Convey("test NewHybrid", t, func() { + convey.Convey("test with reranker and no sub-requests", func() { + reranker := milvusclient.NewRRFReranker() + hybrid := NewHybrid(reranker) + convey.So(hybrid, convey.ShouldNotBeNil) + convey.So(hybrid.Reranker, convey.ShouldNotBeNil) + convey.So(len(hybrid.SubRequests), convey.ShouldEqual, 0) + }) + + convey.Convey("test with reranker and sub-requests", func() { + reranker := milvusclient.NewRRFReranker() + subReq1 := &SubRequest{ + VectorField: "vector1", + MetricType: milvus2.L2, + TopK: 10, + } + subReq2 := &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + } + hybrid := NewHybrid(reranker, subReq1, subReq2) + convey.So(hybrid, convey.ShouldNotBeNil) + convey.So(len(hybrid.SubRequests), convey.ShouldEqual, 2) + }) + }) +} + +func TestHybrid_BuildSearchOption(t *testing.T) { + convey.Convey("test Hybrid.BuildSearchOption returns error", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + } + + reranker := milvusclient.NewRRFReranker() + hybrid := NewHybrid(reranker) + + opt, err := hybrid.BuildSearchOption(ctx, config, queryVector) + convey.So(opt, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "BuildHybridSearchOption") + }) +} + +func TestHybrid_BuildHybridSearchOption(t *testing.T) { + convey.Convey("test Hybrid.BuildHybridSearchOption", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + convey.Convey("test with single sub-request", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with multiple sub-requests", func() { + reranker := milvusclient.NewRRFReranker() + subReq1 := &SubRequest{ + VectorField: "vector1", + MetricType: milvus2.L2, + TopK: 10, + } + subReq2 := &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + } + hybrid := NewHybrid(reranker, subReq1, subReq2) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with sub-request using default vector field", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "", // Should use config.VectorField + MetricType: milvus2.L2, + TopK: 0, // Should default to 10 + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with partitions", func() { + configWithPartitions := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Partitions: []string{"partition1"}, + } + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, configWithPartitions, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with filter option", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "", + milvus2.WithFilter("id > 10")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with custom TopK override", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + hybrid.TopK = 50 + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with search params", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + SearchParams: map[string]string{"nprobe": "16", "ef": "64"}, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with grouping", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "", + milvus2.WithGrouping("category", 3, true)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with common TopK option", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + }) + + opt, err := hybrid.BuildHybridSearchOption(ctx, config, queryVector, "", + retriever.WithTopK(100)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with sparse sub-request", func() { + reranker := milvusclient.NewRRFReranker() + subReq := &SubRequest{ + VectorField: "sparse_vector", + VectorType: milvus2.SparseVector, + TopK: 10, + } + hybrid := NewHybrid(reranker, subReq, &SubRequest{ + VectorField: "vector2_sparse", + VectorType: milvus2.SparseVector, + TopK: 5, + }) + + query := "test query" + opt, err := hybrid.BuildHybridSearchOption(ctx, config, nil, query) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + }) +} + +// Verify interface implementation +func TestHybrid_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Hybrid implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Hybrid)(nil) + }) +} + +func TestHybridSearchTopK(t *testing.T) { + PatchConvey("Test Hybrid Search TopK Default", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + + // Mock Reranker + mockReranker := milvusclient.NewRRFReranker() + + // Hybrid mode with no explicit TopK in SubRequest + hybridMode := NewHybrid( + mockReranker, + &SubRequest{ + VectorField: "vector", + MetricType: milvus2.L2, + // TopK is unset (0) + }, + &SubRequest{ + VectorField: "vector_sparse", + MetricType: milvus2.IP, + // TopK is unset (0) + }, + ) + + // We just need a config object to pass to BuildHybridSearchOption + // No need for a full Retriever instance since we are testing the SearchMode method directly + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 50, // Global TopK > 10 + SearchMode: hybridMode, + } + + convey.Convey("should use global TopK when SubRequest TopK is missing", func() { + Mock(GetMethod(mockClient, "HybridSearch")).Return([]milvusclient.ResultSet{}, nil).Build() + + queryVector := make([]float32, 128) + opt, err := hybridMode.BuildHybridSearchOption(ctx, config, queryVector, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + + var capturedLimits []int + Mock(milvusclient.NewAnnRequest).To(func(fieldName string, limit int, vectors ...entity.Vector) *milvusclient.AnnRequest { + capturedLimits = append(capturedLimits, limit) + return &milvusclient.AnnRequest{} + }).Build() + + Mock((*milvusclient.AnnRequest).WithSearchParam).To(func(r *milvusclient.AnnRequest, key string, value string) *milvusclient.AnnRequest { + return r + }).Build() + + _, _ = hybridMode.BuildHybridSearchOption(ctx, config, queryVector, "query") + + convey.So(capturedLimits, convey.ShouldContain, 50) + }) + + convey.Convey("should use explicit TopK when SubRequest TopK is set", func() { + hybridMode.SubRequests[0].TopK = 5 + + var capturedLimits []int + Mock(milvusclient.NewAnnRequest).To(func(fieldName string, limit int, vectors ...entity.Vector) *milvusclient.AnnRequest { + capturedLimits = append(capturedLimits, limit) + return &milvusclient.AnnRequest{} + }).Build() + + Mock((*milvusclient.AnnRequest).WithSearchParam).To(func(r *milvusclient.AnnRequest, key string, value string) *milvusclient.AnnRequest { + return r + }).Build() + + queryVector := make([]float32, 128) + _, _ = hybridMode.BuildHybridSearchOption(ctx, config, queryVector, "query") + + convey.So(capturedLimits, convey.ShouldContain, 5) + }) + }) +} + +func TestHybrid_Retrieve(t *testing.T) { + PatchConvey("test Hybrid.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + mockEmb := &mockHybridEmbedding{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Embedding: mockEmb, + } + + reranker := milvusclient.NewRRFReranker() + subReq1 := &SubRequest{ + VectorField: "vector1", + MetricType: milvus2.L2, + TopK: 10, + } + subReq2 := &SubRequest{ + VectorField: "vector2", + MetricType: milvus2.IP, + TopK: 5, + } + hybrid := NewHybrid(reranker, subReq1, subReq2) + + PatchConvey("success", func() { + // Mock Client.HybridSearch + Mock(GetMethod(mockClient, "HybridSearch")).Return([]milvusclient.ResultSet{ + { + ResultCount: 1, + IDs: nil, + }, + }, nil).Build() + + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := hybrid.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("embedding error", func() { + mockEmb.err = fmt.Errorf("embed error") + docs, err := hybrid.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + + PatchConvey("search error", func() { + mockEmb.err = nil + Mock(GetMethod(mockClient, "HybridSearch")).Return(nil, fmt.Errorf("search error")).Build() + + docs, err := hybrid.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + }) +} + +// mockHybridEmbedding implements embedding.Embedder for testing +type mockHybridEmbedding struct { + err error +} + +func (m *mockHybridEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + return [][]float64{make([]float64, 128)}, nil +} diff --git a/components/retriever/milvus2/search_mode/iterator.go b/components/retriever/milvus2/search_mode/iterator.go new file mode 100644 index 000000000..1751ec4fd --- /dev/null +++ b/components/retriever/milvus2/search_mode/iterator.go @@ -0,0 +1,162 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Iterator implements search iterator mode for traversing large result sets. +// It fetches results in batches to manage memory usage efficiently. +type Iterator struct { + // MetricType specifies the metric type for vector similarity. + // Default: L2. + MetricType milvus2.MetricType + + // BatchSize controls how many items are fetched per network call. + // Default: 100. + BatchSize int + + // SearchParams contains extra search parameters (e.g., "nprobe", "ef"). + SearchParams map[string]string +} + +// NewIterator creates a new Iterator search mode. +func NewIterator(metricType milvus2.MetricType, batchSize int) *Iterator { + if metricType == "" { + metricType = milvus2.L2 + } + if batchSize <= 0 { + batchSize = 100 + } + return &Iterator{ + MetricType: metricType, + BatchSize: batchSize, + } +} + +// WithSearchParams sets additional search parameters such as "nprobe" or "ef". +func (i *Iterator) WithSearchParams(params map[string]string) *Iterator { + i.SearchParams = params + return i +} + +// BuildSearchOption returns an error because Iterator search mode requires BuildSearchIteratorOption. +func (i *Iterator) BuildSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, opts ...retriever.Option) (milvusclient.SearchOption, error) { + return nil, fmt.Errorf("Iterator search mode requires BuildSearchIteratorOption") +} + +// Retrieve performs the search iterator operation, fetching all results. +func (i *Iterator) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + if conf.Embedding == nil { + return nil, fmt.Errorf("embedding is required for iterator search") + } + + queryVector, err := EmbedQuery(ctx, conf.Embedding, query) + if err != nil { + return nil, err + } + + iterOpt, err := i.BuildSearchIteratorOption(ctx, conf, queryVector, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build search iterator option: %w", err) + } + + iterator, err := client.SearchIterator(ctx, iterOpt) + if err != nil { + return nil, fmt.Errorf("failed to create search iterator: %w", err) + } + + var allDocs []*schema.Document + for { + res, err := iterator.Next(ctx) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("iterator next failed: %w", err) + } + if res.ResultCount == 0 { + break + } + + batchDocs, err := conf.DocumentConverter(ctx, res) + if err != nil { + return nil, fmt.Errorf("failed to convert batch results: %w", err) + } + allDocs = append(allDocs, batchDocs...) + } + + return allDocs, nil +} + +// BuildSearchIteratorOption creates a SearchIteratorOption for batch-based result traversal. +func (i *Iterator) BuildSearchIteratorOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, opts ...retriever.Option) (milvusclient.SearchIteratorOption, error) { + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + finalLimit := conf.TopK + if co.TopK != nil { + finalLimit = *co.TopK + } + opt := milvusclient.NewSearchIteratorOption(conf.Collection, entity.FloatVector(queryVector)). + WithANNSField(conf.VectorField). + WithBatchSize(i.BatchSize). + WithOutputFields(conf.OutputFields...). + WithIteratorLimit(int64(finalLimit)) // Set total limit + + if i.MetricType != "" { + opt.WithSearchParam("metric_type", string(i.MetricType)) + } + for k, v := range i.SearchParams { + opt.WithSearchParam(k, v) + } + + if len(conf.Partitions) > 0 { + opt.WithPartitions(conf.Partitions...) + } + + if io.Filter != "" { + opt.WithFilter(io.Filter) + } + + if io.Grouping != nil { + opt.WithGroupByField(io.Grouping.GroupByField). + WithGroupSize(io.Grouping.GroupSize) + if io.Grouping.StrictGroupSize { + opt.WithStrictGroupSize(true) + } + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + opt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return opt, nil +} diff --git a/components/retriever/milvus2/search_mode/iterator_test.go b/components/retriever/milvus2/search_mode/iterator_test.go new file mode 100644 index 000000000..b6586dd09 --- /dev/null +++ b/components/retriever/milvus2/search_mode/iterator_test.go @@ -0,0 +1,257 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "io" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewIterator(t *testing.T) { + convey.Convey("test NewIterator", t, func() { + convey.Convey("test with default values", func() { + iter := NewIterator("", 0) + convey.So(iter, convey.ShouldNotBeNil) + convey.So(iter.MetricType, convey.ShouldEqual, milvus2.L2) + convey.So(iter.BatchSize, convey.ShouldEqual, 100) + }) + + convey.Convey("test with custom metric type", func() { + iter := NewIterator(milvus2.IP, 0) + convey.So(iter, convey.ShouldNotBeNil) + convey.So(iter.MetricType, convey.ShouldEqual, milvus2.IP) + }) + + convey.Convey("test with custom batch size", func() { + iter := NewIterator(milvus2.L2, 50) + convey.So(iter, convey.ShouldNotBeNil) + convey.So(iter.BatchSize, convey.ShouldEqual, 50) + }) + + convey.Convey("test with both custom values", func() { + iter := NewIterator(milvus2.COSINE, 200) + convey.So(iter, convey.ShouldNotBeNil) + convey.So(iter.MetricType, convey.ShouldEqual, milvus2.COSINE) + convey.So(iter.BatchSize, convey.ShouldEqual, 200) + }) + }) +} + +func TestIterator_WithSearchParams(t *testing.T) { + convey.Convey("test Iterator.WithSearchParams", t, func() { + iter := NewIterator(milvus2.L2, 100) + params := map[string]string{ + "nprobe": "16", + "ef": "64", + } + result := iter.WithSearchParams(params) + convey.So(result, convey.ShouldEqual, iter) + convey.So(iter.SearchParams, convey.ShouldResemble, params) + }) +} + +func TestIterator_BuildSearchOption(t *testing.T) { + convey.Convey("test Iterator.BuildSearchOption returns error", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + } + + iter := NewIterator(milvus2.L2, 100) + + opt, err := iter.BuildSearchOption(ctx, config, queryVector) + convey.So(opt, convey.ShouldBeNil) + convey.So(err, convey.ShouldNotBeNil) + convey.So(err.Error(), convey.ShouldContainSubstring, "BuildSearchIteratorOption") + }) +} + +func TestIterator_BuildSearchIteratorOption(t *testing.T) { + convey.Convey("test Iterator.BuildSearchIteratorOption", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + convey.Convey("test basic iterator option", func() { + iter := NewIterator(milvus2.L2, 100) + opt, err := iter.BuildSearchIteratorOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with partitions", func() { + configWithPartitions := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Partitions: []string{"partition1", "partition2"}, + } + iter := NewIterator(milvus2.L2, 100) + opt, err := iter.BuildSearchIteratorOption(ctx, configWithPartitions, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with search params", func() { + iter := NewIterator(milvus2.L2, 100).WithSearchParams(map[string]string{ + "nprobe": "16", + "ef": "64", + }) + opt, err := iter.BuildSearchIteratorOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with filter option", func() { + iter := NewIterator(milvus2.L2, 100) + opt, err := iter.BuildSearchIteratorOption(ctx, config, queryVector, + milvus2.WithFilter("id > 10")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with grouping option", func() { + iter := NewIterator(milvus2.L2, 100) + opt, err := iter.BuildSearchIteratorOption(ctx, config, queryVector, + milvus2.WithGrouping("category", 3, true)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with custom TopK from options", func() { + iter := NewIterator(milvus2.L2, 100) + customTopK := 50 + opt, err := iter.BuildSearchIteratorOption(ctx, config, queryVector, + retriever.WithTopK(customTopK)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + }) +} + +// Verify interface implementation +func TestIterator_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Iterator implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Iterator)(nil) + }) +} + +func TestIterator_Retrieve(t *testing.T) { + PatchConvey("test Iterator.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + mockEmb := &mockIterEmbedding{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Embedding: mockEmb, + } + + iter := NewIterator(milvus2.L2, 10) + + PatchConvey("success", func() { + mockIt := &mockSearchIterator{ + nextFunc: func(ctx context.Context) (milvusclient.ResultSet, error) { + return milvusclient.ResultSet{ResultCount: 1}, nil + }, + } + // Use counter to return result then EOF + called := 0 + mockIt.nextFunc = func(ctx context.Context) (milvusclient.ResultSet, error) { + if called == 0 { + called++ + return milvusclient.ResultSet{ResultCount: 1}, nil + } + return milvusclient.ResultSet{}, io.EOF + } + + Mock(GetMethod(mockClient, "SearchIterator")).Return(mockIt, nil).Build() + + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := iter.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("embedding error", func() { + mockEmb.err = fmt.Errorf("embed error") + docs, err := iter.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + + PatchConvey("SearchIterator creation error", func() { + mockEmb.err = nil + Mock(GetMethod(mockClient, "SearchIterator")).Return(nil, fmt.Errorf("create iter error")).Build() + + docs, err := iter.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + }) +} + +type mockSearchIterator struct { + nextFunc func(ctx context.Context) (milvusclient.ResultSet, error) +} + +func (m *mockSearchIterator) Next(ctx context.Context) (milvusclient.ResultSet, error) { + if m.nextFunc != nil { + return m.nextFunc(ctx) + } + return milvusclient.ResultSet{}, io.EOF +} + +// mockIterEmbedding implements embedding.Embedder for testing +type mockIterEmbedding struct { + err error +} + +func (m *mockIterEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + return [][]float64{make([]float64, 128)}, nil +} diff --git a/components/retriever/milvus2/search_mode/range.go b/components/retriever/milvus2/search_mode/range.go new file mode 100644 index 000000000..0b005a2c9 --- /dev/null +++ b/components/retriever/milvus2/search_mode/range.go @@ -0,0 +1,142 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Range implements range search to find vectors within a specified distance or similarity radius. +// It returns all vectors whose distance or similarity to the query falls within the radius boundary. +type Range struct { + // MetricType specifies the metric type for vector similarity. + MetricType milvus2.MetricType + + // Radius specifies the search radius boundary. + // For distance metrics (L2/Hamming/Jaccard): finds vectors where distance <= Radius. + // For similarity metrics (IP/Cosine): finds vectors where score >= Radius. + Radius float64 + + // RangeFilter excludes vectors that are too close or too similar. + // For L2: excludes vectors where distance < RangeFilter (use with Radius for ring search). + // For IP: excludes vectors where score > RangeFilter. + // Optional; leave nil unless performing ring searches. + RangeFilter *float64 +} + +// NewRange creates a new Range search mode. +func NewRange(metricType milvus2.MetricType, radius float64) *Range { + if metricType == "" { + metricType = milvus2.L2 + } + return &Range{ + MetricType: metricType, + Radius: radius, + } +} + +// WithRangeFilter sets the inner boundary for ring searches. +func (r *Range) WithRangeFilter(rangeFilter float64) *Range { + r.RangeFilter = &rangeFilter + return r +} + +// Retrieve performs the range search operation. +func (r *Range) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + if conf.Embedding == nil { + return nil, fmt.Errorf("embedding is required for range search") + } + + queryVector, err := EmbedQuery(ctx, conf.Embedding, query) + if err != nil { + return nil, err + } + + searchOpt, err := r.BuildSearchOption(ctx, conf, queryVector, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build search option: %w", err) + } + + result, err := client.Search(ctx, searchOpt) + if err != nil { + return nil, fmt.Errorf("failed to search: %w", err) + } + + if len(result) == 0 { + return []*schema.Document{}, nil + } + + return conf.DocumentConverter(ctx, result[0]) +} + +// BuildSearchOption creates a SearchOption for range search with the configured radius and range filter. +func (r *Range) BuildSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, queryVector []float32, opts ...retriever.Option) (milvusclient.SearchOption, error) { + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + // Determine final topK + topK := conf.TopK + if co.TopK != nil { + topK = *co.TopK + } + + searchOpt := milvusclient.NewSearchOption(conf.Collection, topK, []entity.Vector{entity.FloatVector(queryVector)}). + WithANNSField(conf.VectorField). + WithOutputFields(conf.OutputFields...). + WithSearchParam("radius", fmt.Sprintf("%v", r.Radius)) + + // Apply metric type + if r.MetricType != "" { + searchOpt.WithSearchParam("metric_type", string(r.MetricType)) + } + + if r.RangeFilter != nil { + searchOpt = searchOpt.WithSearchParam("range_filter", fmt.Sprintf("%v", *r.RangeFilter)) + } + + if len(conf.Partitions) > 0 { + searchOpt = searchOpt.WithPartitions(conf.Partitions...) + } + + if io.Filter != "" { + searchOpt = searchOpt.WithFilter(io.Filter) + } + + if io.Grouping != nil { + searchOpt = searchOpt.WithGroupByField(io.Grouping.GroupByField). + WithGroupSize(io.Grouping.GroupSize) + if io.Grouping.StrictGroupSize { + searchOpt = searchOpt.WithStrictGroupSize(true) + } + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + searchOpt = searchOpt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return searchOpt, nil +} diff --git a/components/retriever/milvus2/search_mode/range_test.go b/components/retriever/milvus2/search_mode/range_test.go new file mode 100644 index 000000000..6e32212bb --- /dev/null +++ b/components/retriever/milvus2/search_mode/range_test.go @@ -0,0 +1,211 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewRange(t *testing.T) { + convey.Convey("test NewRange", t, func() { + convey.Convey("test with default metric type", func() { + r := NewRange("", 0.5) + convey.So(r, convey.ShouldNotBeNil) + convey.So(r.MetricType, convey.ShouldEqual, milvus2.L2) + convey.So(r.Radius, convey.ShouldEqual, 0.5) + }) + + convey.Convey("test with L2 metric type", func() { + r := NewRange(milvus2.L2, 1.0) + convey.So(r, convey.ShouldNotBeNil) + convey.So(r.MetricType, convey.ShouldEqual, milvus2.L2) + convey.So(r.Radius, convey.ShouldEqual, 1.0) + }) + + convey.Convey("test with IP metric type", func() { + r := NewRange(milvus2.IP, 0.8) + convey.So(r, convey.ShouldNotBeNil) + convey.So(r.MetricType, convey.ShouldEqual, milvus2.IP) + convey.So(r.Radius, convey.ShouldEqual, 0.8) + }) + }) +} + +func TestRange_WithRangeFilter(t *testing.T) { + convey.Convey("test Range.WithRangeFilter", t, func() { + r := NewRange(milvus2.L2, 1.0) + result := r.WithRangeFilter(0.5) + convey.So(result, convey.ShouldEqual, r) + convey.So(r.RangeFilter, convey.ShouldNotBeNil) + convey.So(*r.RangeFilter, convey.ShouldEqual, 0.5) + }) +} + +func TestRange_BuildSearchOption(t *testing.T) { + convey.Convey("test Range.BuildSearchOption", t, func() { + ctx := context.Background() + queryVector := []float32{0.1, 0.2, 0.3} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + convey.Convey("test basic range search option", func() { + r := NewRange(milvus2.L2, 1.0) + opt, err := r.BuildSearchOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with range filter (ring search)", func() { + r := NewRange(milvus2.L2, 1.0).WithRangeFilter(0.5) + opt, err := r.BuildSearchOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with partitions", func() { + configWithPartitions := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Partitions: []string{"partition1"}, + } + r := NewRange(milvus2.L2, 1.0) + opt, err := r.BuildSearchOption(ctx, configWithPartitions, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with filter option", func() { + r := NewRange(milvus2.L2, 1.0) + opt, err := r.BuildSearchOption(ctx, config, queryVector, + milvus2.WithFilter("id > 10")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with grouping option", func() { + r := NewRange(milvus2.L2, 1.0) + opt, err := r.BuildSearchOption(ctx, config, queryVector, + milvus2.WithGrouping("category", 3, true)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with custom TopK", func() { + r := NewRange(milvus2.L2, 1.0) + customTopK := 20 + opt, err := r.BuildSearchOption(ctx, config, queryVector, + retriever.WithTopK(customTopK)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with IP metric type for similarity search", func() { + r := NewRange(milvus2.IP, 0.8) + opt, err := r.BuildSearchOption(ctx, config, queryVector) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + }) +} + +// Verify interface implementation +func TestRange_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Range implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Range)(nil) + }) +} + +func TestRange_Retrieve(t *testing.T) { + PatchConvey("test Range.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + mockEmb := &mockRangeEmbedding{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Embedding: mockEmb, + } + + r := NewRange(milvus2.L2, 1.0) + + PatchConvey("success", func() { + Mock(GetMethod(mockClient, "Search")).Return([]milvusclient.ResultSet{ + { + ResultCount: 1, + }, + }, nil).Build() + + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := r.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("embedding error", func() { + mockEmb.err = fmt.Errorf("embed error") + docs, err := r.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + + PatchConvey("search error", func() { + mockEmb.err = nil + Mock(GetMethod(mockClient, "Search")).Return(nil, fmt.Errorf("search error")).Build() + + docs, err := r.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + }) +} + +// mockRangeEmbedding implements embedding.Embedder for testing +type mockRangeEmbedding struct { + err error +} + +func (m *mockRangeEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + return [][]float64{make([]float64, 128)}, nil +} diff --git a/components/retriever/milvus2/search_mode/scalar.go b/components/retriever/milvus2/search_mode/scalar.go new file mode 100644 index 000000000..31ca37080 --- /dev/null +++ b/components/retriever/milvus2/search_mode/scalar.go @@ -0,0 +1,91 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Scalar implements scalar/metadata search using the Milvus Query API. +// It treats the query string as a boolean filter expression (e.g., "id > 10"). +type Scalar struct{} + +// NewScalar creates a new Scalar search mode. +func NewScalar() *Scalar { + return &Scalar{} +} + +// Retrieve performs the scalar query search. +func (s *Scalar) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + queryOpt, err := s.BuildQueryOption(ctx, conf, query, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build query option: %w", err) + } + + result, err := client.Query(ctx, queryOpt) + if err != nil { + return nil, fmt.Errorf("failed to query: %w", err) + } + + return conf.DocumentConverter(ctx, result) +} + +// BuildQueryOption creates a QueryOption for scalar/metadata-based document retrieval. +func (s *Scalar) BuildQueryOption(ctx context.Context, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) (milvusclient.QueryOption, error) { + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + finalTopK := conf.TopK + if co.TopK != nil { + finalTopK = *co.TopK + } + + // Combine query and filter with AND logic + expr := query + if io.Filter != "" { + if expr != "" { + expr = "(" + expr + ") and (" + io.Filter + ")" + } else { + expr = io.Filter + } + } + + opt := milvusclient.NewQueryOption(conf.Collection). + WithFilter(expr). + WithOutputFields(conf.OutputFields...). + WithLimit(int(finalTopK)) + + // Partitions + if len(conf.Partitions) > 0 { + opt = opt.WithPartitions(conf.Partitions...) + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + opt = opt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return opt, nil +} diff --git a/components/retriever/milvus2/search_mode/scalar_test.go b/components/retriever/milvus2/search_mode/scalar_test.go new file mode 100644 index 000000000..c6903fcb0 --- /dev/null +++ b/components/retriever/milvus2/search_mode/scalar_test.go @@ -0,0 +1,151 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewScalar(t *testing.T) { + convey.Convey("test NewScalar", t, func() { + scalar := NewScalar() + convey.So(scalar, convey.ShouldNotBeNil) + }) +} + +func TestScalar_BuildQueryOption(t *testing.T) { + convey.Convey("test Scalar.BuildQueryOption", t, func() { + ctx := context.Background() + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + convey.Convey("test with query expression", func() { + scalar := NewScalar() + opt, err := scalar.BuildQueryOption(ctx, config, "id > 10") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with empty query", func() { + scalar := NewScalar() + opt, err := scalar.BuildQueryOption(ctx, config, "") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with query and additional filter", func() { + scalar := NewScalar() + opt, err := scalar.BuildQueryOption(ctx, config, "id > 10", + milvus2.WithFilter("category == 'A'")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with only filter (empty query)", func() { + scalar := NewScalar() + opt, err := scalar.BuildQueryOption(ctx, config, "", + milvus2.WithFilter("status == 'active'")) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with partitions", func() { + configWithPartitions := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + Partitions: []string{"partition1", "partition2"}, + } + scalar := NewScalar() + opt, err := scalar.BuildQueryOption(ctx, configWithPartitions, "id > 10") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with custom TopK", func() { + scalar := NewScalar() + customTopK := 50 + opt, err := scalar.BuildQueryOption(ctx, config, "id > 10", + retriever.WithTopK(customTopK)) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + }) +} + +// Verify interface implementation +func TestScalar_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Scalar implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Scalar)(nil) + }) +} + +func TestScalar_Retrieve(t *testing.T) { + PatchConvey("test Scalar.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + scalar := NewScalar() + + PatchConvey("success", func() { + Mock(GetMethod(mockClient, "Query")).Return(milvusclient.ResultSet{ + ResultCount: 1, + }, nil).Build() + + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := scalar.Retrieve(ctx, mockClient, config, "id > 10") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("query error", func() { + Mock(GetMethod(mockClient, "Query")).Return(nil, fmt.Errorf("query error")).Build() + + docs, err := scalar.Retrieve(ctx, mockClient, config, "id > 10") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + }) +} diff --git a/components/retriever/milvus2/search_mode/sparse.go b/components/retriever/milvus2/search_mode/sparse.go new file mode 100644 index 000000000..659c0f9ff --- /dev/null +++ b/components/retriever/milvus2/search_mode/sparse.go @@ -0,0 +1,111 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/milvus-io/milvus/client/v2/milvusclient" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +// Sparse implements sparse vector search (e.g. BM25). +// It uses raw text input allowing Milvus to generate sparse vectors server-side via functions. +type Sparse struct { + // MetricType specifies the metric type for sparse similarity. + // Default: BM25 (or IP if not specified but typically BM25 for text). + MetricType milvus2.MetricType +} + +// NewSparse creates a new Sparse search mode. +func NewSparse(metricType milvus2.MetricType) *Sparse { + if metricType == "" { + metricType = milvus2.BM25 + } + return &Sparse{ + MetricType: metricType, + } +} + +// Retrieve performs the sparse search operation (text search via function). +func (s *Sparse) Retrieve(ctx context.Context, client *milvusclient.Client, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) ([]*schema.Document, error) { + searchOpt, err := s.BuildSparseSearchOption(ctx, conf, query, opts...) + if err != nil { + return nil, fmt.Errorf("failed to build sparse search option: %w", err) + } + + result, err := client.Search(ctx, searchOpt) + if err != nil { + return nil, fmt.Errorf("failed to search: %w", err) + } + + if len(result) == 0 { + return []*schema.Document{}, nil + } + + return conf.DocumentConverter(ctx, result[0]) +} + +// BuildSparseSearchOption creates a SearchOption configured for sparse vector search using text query. +func (s *Sparse) BuildSparseSearchOption(ctx context.Context, conf *milvus2.RetrieverConfig, query string, opts ...retriever.Option) (milvusclient.SearchOption, error) { + io := retriever.GetImplSpecificOptions(&milvus2.ImplOptions{}, opts...) + co := retriever.GetCommonOptions(&retriever.Options{ + TopK: &conf.TopK, + }, opts...) + + // Determine final topK + topK := conf.TopK + if co.TopK != nil { + topK = *co.TopK + } + + searchOpt := milvusclient.NewSearchOption(conf.Collection, topK, []entity.Vector{entity.Text(query)}). + WithANNSField(conf.SparseVectorField). + WithOutputFields(conf.OutputFields...) + + // Apply metric type + if s.MetricType != "" { + searchOpt.WithSearchParam("metric_type", string(s.MetricType)) + } + + if len(conf.Partitions) > 0 { + searchOpt = searchOpt.WithPartitions(conf.Partitions...) + } + + if io.Filter != "" { + searchOpt = searchOpt.WithFilter(io.Filter) + } + + if io.Grouping != nil { + searchOpt = searchOpt.WithGroupByField(io.Grouping.GroupByField). + WithGroupSize(io.Grouping.GroupSize) + if io.Grouping.StrictGroupSize { + searchOpt = searchOpt.WithStrictGroupSize(true) + } + } + + if conf.ConsistencyLevel != milvus2.ConsistencyLevelDefault { + searchOpt = searchOpt.WithConsistencyLevel(conf.ConsistencyLevel.ToEntity()) + } + + return searchOpt, nil +} diff --git a/components/retriever/milvus2/search_mode/sparse_test.go b/components/retriever/milvus2/search_mode/sparse_test.go new file mode 100644 index 000000000..30f4773a4 --- /dev/null +++ b/components/retriever/milvus2/search_mode/sparse_test.go @@ -0,0 +1,134 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/milvus-io/milvus/client/v2/milvusclient" + "github.com/smartystreets/goconvey/convey" + + milvus2 "github.com/cloudwego/eino-ext/components/retriever/milvus2" +) + +func TestNewSparse(t *testing.T) { + convey.Convey("test NewSparse", t, func() { + sparse := NewSparse(milvus2.BM25) + convey.So(sparse, convey.ShouldNotBeNil) + convey.So(sparse.MetricType, convey.ShouldEqual, milvus2.BM25) + + sparseDefault := NewSparse("") + convey.So(sparseDefault, convey.ShouldNotBeNil) + convey.So(sparseDefault.MetricType, convey.ShouldEqual, milvus2.BM25) + }) +} + +func TestSparse_BuildSparseSearchOption(t *testing.T) { + convey.Convey("test Sparse.BuildSparseSearchOption", t, func() { + ctx := context.Background() + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + SparseVectorField: "sparse_vector", + TopK: 10, + } + + convey.Convey("test with default options", func() { + sparse := NewSparse(milvus2.BM25) + opt, err := sparse.BuildSparseSearchOption(ctx, config, "test query") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with options", func() { + sparse := NewSparse(milvus2.BM25) + opt, err := sparse.BuildSparseSearchOption(ctx, config, "test query", + retriever.WithTopK(20), + milvus2.WithFilter("id > 0"), + milvus2.WithGrouping("group", 5, true), + ) + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + + convey.Convey("test with ConsistencyLevel", func() { + configWithConsistency := &milvus2.RetrieverConfig{ + Collection: "test_collection", + VectorField: "vector", + SparseVectorField: "sparse_vector", + TopK: 10, + ConsistencyLevel: milvus2.ConsistencyLevelStrong, + } + sparse := NewSparse(milvus2.BM25) + opt, err := sparse.BuildSparseSearchOption(ctx, configWithConsistency, "test query") + convey.So(err, convey.ShouldBeNil) + convey.So(opt, convey.ShouldNotBeNil) + }) + }) +} + +func TestSparse_ImplementsSearchMode(t *testing.T) { + convey.Convey("test Sparse implements SearchMode", t, func() { + var _ milvus2.SearchMode = (*Sparse)(nil) + }) +} + +func TestSparse_Retrieve(t *testing.T) { + PatchConvey("test Sparse.Retrieve", t, func() { + ctx := context.Background() + mockClient := &milvusclient.Client{} + + config := &milvus2.RetrieverConfig{ + Collection: "test_collection", + SparseVectorField: "sparse", + TopK: 10, + OutputFields: []string{"id", "content"}, + } + + sparse := NewSparse(milvus2.BM25) + + PatchConvey("success", func() { + Mock(GetMethod(mockClient, "Search")).Return([]milvusclient.ResultSet{ + { + ResultCount: 1, + }, + }, nil).Build() + + mockConverter := func(ctx context.Context, result milvusclient.ResultSet) ([]*schema.Document, error) { + return []*schema.Document{{ID: "1"}}, nil + } + config.DocumentConverter = mockConverter + + docs, err := sparse.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldBeNil) + convey.So(len(docs), convey.ShouldEqual, 1) + }) + + PatchConvey("search error", func() { + Mock(GetMethod(mockClient, "Search")).Return(nil, fmt.Errorf("search error")).Build() + + docs, err := sparse.Retrieve(ctx, mockClient, config, "query") + convey.So(err, convey.ShouldNotBeNil) + convey.So(docs, convey.ShouldBeNil) + }) + }) +} diff --git a/components/retriever/milvus2/search_mode/utils.go b/components/retriever/milvus2/search_mode/utils.go new file mode 100644 index 000000000..cc7dc248f --- /dev/null +++ b/components/retriever/milvus2/search_mode/utils.go @@ -0,0 +1,61 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" +) + +// EmbedQuery embeds the query string into a vector. +// It handles the callback context for the embedding operation. +func EmbedQuery(ctx context.Context, emb embedding.Embedder, query string) ([]float32, error) { + if emb == nil { + return nil, fmt.Errorf("[Retriever] embedding not provided") + } + + runInfo := &callbacks.RunInfo{ + Component: components.ComponentOfEmbedding, + } + + if embType, ok := components.GetType(emb); ok { + runInfo.Type = embType + } + + runInfo.Name = runInfo.Type + string(runInfo.Component) + + // Wrap the context with callback info + ctx = callbacks.ReuseHandlers(ctx, runInfo) + + vectors, err := emb.EmbedStrings(ctx, []string{query}) + if err != nil { + return nil, fmt.Errorf("[Retriever] failed to embed query: %w", err) + } + if len(vectors) != 1 { + return nil, fmt.Errorf("[Retriever] invalid embedding result: expected 1, got %d", len(vectors)) + } + + queryVector := make([]float32, len(vectors[0])) + for i, v := range vectors[0] { + queryVector[i] = float32(v) + } + return queryVector, nil +} diff --git a/components/retriever/milvus2/search_mode/utils_test.go b/components/retriever/milvus2/search_mode/utils_test.go new file mode 100644 index 000000000..a189ac393 --- /dev/null +++ b/components/retriever/milvus2/search_mode/utils_test.go @@ -0,0 +1,80 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package search_mode + +import ( + "context" + "fmt" + "testing" + + "github.com/cloudwego/eino/components/embedding" + . "github.com/smartystreets/goconvey/convey" +) + +// mockEmbedding implements embedding.Embedder for testing +type mockEmbedding struct { + err error + dims int +} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.err != nil { + return nil, m.err + } + result := make([][]float64, len(texts)) + dims := m.dims + if dims == 0 { + dims = 128 + } + for i := range texts { + result[i] = make([]float64, dims) + for j := 0; j < dims; j++ { + result[i][j] = 0.1 + } + } + return result, nil +} + +func TestEmbedQuery(t *testing.T) { + Convey("test EmbedQuery", t, func() { + ctx := context.Background() + + Convey("test embedding success returns float32 vector", func() { + mockEmb := &mockEmbedding{dims: 128} + vector, err := EmbedQuery(ctx, mockEmb, "test query") + So(err, ShouldBeNil) + So(len(vector), ShouldEqual, 128) + // First element should be 0.1 converted to float32 + So(vector[0], ShouldEqual, float32(0.1)) + }) + + Convey("test embedding error", func() { + mockEmb := &mockEmbedding{err: fmt.Errorf("embedding failed")} + vector, err := EmbedQuery(ctx, mockEmb, "test query") + So(err, ShouldNotBeNil) + So(vector, ShouldBeNil) + }) + + Convey("test embedding empty result", func() { + mockEmb := &mockEmbedding{dims: 0} + // Even with dims=0, the mock returns 128 (default) + vector, err := EmbedQuery(ctx, mockEmb, "test query") + So(err, ShouldBeNil) + So(len(vector), ShouldBeGreaterThan, 0) + }) + }) +} diff --git a/components/retriever/milvus2/types.go b/components/retriever/milvus2/types.go new file mode 100644 index 000000000..29c6e91e3 --- /dev/null +++ b/components/retriever/milvus2/types.go @@ -0,0 +1,119 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "github.com/milvus-io/milvus/client/v2/entity" +) + +// MetricType represents the metric type for vector similarity. +type MetricType string + +const ( + // L2 represents Euclidean distance (L2 norm). + // Suitable for dense floating-point vectors. + L2 MetricType = "L2" + + // IP represents Inner Product similarity. + // Higher values indicate greater similarity. Suitable for normalized vectors. + IP MetricType = "IP" + + // COSINE represents Cosine similarity. + // Measures the cosine of the angle between vectors, ignoring magnitude. + COSINE MetricType = "COSINE" + + // HAMMING represents Hamming distance for binary vectors. + // Counts the number of positions where the corresponding bits differ. + HAMMING MetricType = "HAMMING" + + // JACCARD represents Jaccard distance for binary vectors. + // Measures dissimilarity between sample sets. + JACCARD MetricType = "JACCARD" + + // TANIMOTO represents Tanimoto distance for binary vectors. + // Similar to Jaccard, used for molecular fingerprint comparisons. + TANIMOTO MetricType = "TANIMOTO" + + // SUBSTRUCTURE represents substructure search for binary vectors. + // Returns true if A is a subset of B. + SUBSTRUCTURE MetricType = "SUBSTRUCTURE" + + // SUPERSTRUCTURE represents superstructure search for binary vectors. + // Returns true if B is a subset of A. + SUPERSTRUCTURE MetricType = "SUPERSTRUCTURE" + + // BM25 represents BM25 similarity. + // Used for sparse vectors generated by BM25 function. + BM25 MetricType = "BM25" +) + +// toEntity converts MetricType to the Milvus SDK entity.MetricType. +func (m MetricType) toEntity() entity.MetricType { + return entity.MetricType(m) +} + +// ConsistencyLevel represents the consistency level for Milvus operations. +type ConsistencyLevel int + +const ( + // ConsistencyLevelDefault uses the collection's default consistency level. + // For the retriever, this is the zero value: no per-request override is applied, + // and the collection-level consistency (set during indexer creation) is used. + ConsistencyLevelDefault ConsistencyLevel = iota + // ConsistencyLevelStrong guarantees that reads return the most recent data. + ConsistencyLevelStrong + // ConsistencyLevelSession ensures read-your-write consistency within the same session. + ConsistencyLevelSession + // ConsistencyLevelBounded allows reads to return data within a certain staleness bound. + ConsistencyLevelBounded + // ConsistencyLevelEventually provides eventual consistency with no staleness guarantees. + ConsistencyLevelEventually + // ConsistencyLevelCustomized allows custom consistency configuration. + ConsistencyLevelCustomized +) + +// ToEntity converts ConsistencyLevel to the Milvus SDK entity.ConsistencyLevel type. +// Note: ConsistencyLevelDefault should be checked before calling this method, +// as it indicates "use collection default" and should not be converted. +func (c ConsistencyLevel) ToEntity() entity.ConsistencyLevel { + switch c { + case ConsistencyLevelStrong: + return entity.ClStrong + case ConsistencyLevelSession: + return entity.ClSession + case ConsistencyLevelBounded: + return entity.ClBounded + case ConsistencyLevelEventually: + return entity.ClEventually + case ConsistencyLevelCustomized: + return entity.ClCustomized + default: + // ConsistencyLevelDefault (0) or invalid value - default to Bounded + return entity.ClBounded + } +} + +// VectorType represents the type of vector field. +type VectorType string + +const ( + // DenseVector represents standard dense floating-point vectors. + DenseVector VectorType = "dense" + + // SparseVector represents sparse vectors (map of index to weight). + SparseVector VectorType = "sparse" +) diff --git a/components/retriever/milvus2/types_test.go b/components/retriever/milvus2/types_test.go new file mode 100644 index 000000000..50498442b --- /dev/null +++ b/components/retriever/milvus2/types_test.go @@ -0,0 +1,104 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package milvus2 + +import ( + "testing" + + "github.com/milvus-io/milvus/client/v2/entity" + "github.com/smartystreets/goconvey/convey" +) + +func TestRetrieverMetricType_toEntity(t *testing.T) { + convey.Convey("test MetricType.toEntity", t, func() { + convey.Convey("test L2", func() { + mt := L2 + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("L2")) + }) + + convey.Convey("test IP", func() { + mt := IP + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("IP")) + }) + + convey.Convey("test COSINE", func() { + mt := COSINE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("COSINE")) + }) + + convey.Convey("test HAMMING", func() { + mt := HAMMING + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("HAMMING")) + }) + + convey.Convey("test JACCARD", func() { + mt := JACCARD + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("JACCARD")) + }) + + convey.Convey("test TANIMOTO", func() { + mt := TANIMOTO + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("TANIMOTO")) + }) + + convey.Convey("test SUBSTRUCTURE", func() { + mt := SUBSTRUCTURE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("SUBSTRUCTURE")) + }) + + convey.Convey("test SUPERSTRUCTURE", func() { + mt := SUPERSTRUCTURE + result := mt.toEntity() + convey.So(result, convey.ShouldEqual, entity.MetricType("SUPERSTRUCTURE")) + }) + }) +} + +func TestRetrieverConsistencyLevel_ToEntity(t *testing.T) { + convey.Convey("test ConsistencyLevel.ToEntity", t, func() { + convey.Convey("test ConsistencyLevelStrong", func() { + level := ConsistencyLevelStrong + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClStrong) + }) + + convey.Convey("test ConsistencyLevelSession", func() { + level := ConsistencyLevelSession + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClSession) + }) + + convey.Convey("test ConsistencyLevelBounded", func() { + level := ConsistencyLevelBounded + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClBounded) + }) + + convey.Convey("test ConsistencyLevelEventually", func() { + level := ConsistencyLevelEventually + result := level.ToEntity() + convey.So(result, convey.ShouldEqual, entity.ClEventually) + }) + }) +}