feat(graph_db): introduce LanceDB graph backend with native compaction and FTS auto-indexing

.gitignore

@@ -237,3 +237,8 @@ outputs
 evaluation/data/
 test_add_pipeline.py
 test_file_pipeline.py
+
+# LanceDB local storage and scripts
+data/
+inspect_lancedb.py
+memos_server.log

pyproject.toml

@@ -71,6 +71,7 @@ memos = "memos.cli:main"
 tree-mem = [
     "neo4j (>=5.28.1,<6.0.0)",  # Graph database
     "schedule (>=1.2.2,<2.0.0)",  # Task scheduling
+    "lancedb (>=0.30.1,<1.0.0)", # LanceDB
 ]
 
 # MemScheduler
@@ -102,6 +103,13 @@ skill-mem = [
     "alibabacloud-oss-v2 (>=1.2.2,<1.2.3)",
 ]
 
+# Lance Vector DB
+lance-mem = [
+    "lancedb (>=0.17.0,<1.0.0)",  # Lance vector database
+    "pyarrow (>=18.0.0,<20.0.0)",  # Arrow format support for Lance
+    "tantivy (>=0.22.0,<1.0.0)",   # FTS engine for LanceDB
+]
+
 # Tavily Search
 tavily = [
     "tavily-python (>=0.5.0,<1.0.0)",
@@ -135,6 +143,9 @@ all = [
     "rake-nltk (>=1.0.6,<1.1.0)",
     "alibabacloud-oss-v2 (>=1.2.2,<1.2.3)",
     "tavily-python (>=0.5.0,<1.0.0)",
+    "lancedb (>=0.17.0,<1.0.0)",
+    "pyarrow (>=18.0.0,<20.0.0)",
+    "tantivy (>=0.22.0,<1.0.0)",
 
     # Uncategorized dependencies
 ]
@@ -199,8 +210,8 @@ langgraph = "^0.5.1"
 pymysql = "^1.1.2"
 
 [[tool.poetry.source]]
-name = "mirrors"
-url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple/"
+name = "volces"
+url = "https://mirrors.volces.com/pypi/simple/"
 priority = "supplemental"
 
 

src/memos/api/config.py

@@ -1049,6 +1049,16 @@ def get_start_default_config() -> dict[str, Any]:
 
         return config
 
+    @staticmethod
+    def get_lance_graph_config(user_id: str | None = None) -> dict[str, Any]:
+        """Get LanceDB graph configuration."""
+        base_uri = os.getenv("LANCE_URI", "./data/lance_db")
+        return {
+            "uri": base_uri,
+            "user_name": user_id,
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 2048)),
+        }
+
     @staticmethod
     def create_user_config(user_name: str, user_id: str) -> tuple["MOSConfig", "GeneralMemCube"]:
         """Create configuration for a specific user."""
@@ -1126,6 +1136,7 @@ def create_user_config(user_name: str, user_id: str) -> tuple["MOSConfig", "Gene
         neo4j_community_config = APIConfig.get_neo4j_community_config(user_id)
         neo4j_config = APIConfig.get_neo4j_config(user_id)
         polardb_config = APIConfig.get_polardb_config(user_id)
+        lance_config = APIConfig.get_lance_graph_config(user_id)
         internet_config = (
             APIConfig.get_internet_config()
             if os.getenv("ENABLE_INTERNET", "false").lower() == "true"
@@ -1137,6 +1148,7 @@ def create_user_config(user_name: str, user_id: str) -> tuple["MOSConfig", "Gene
             "neo4j": neo4j_config,
             "polardb": polardb_config,
             "postgres": postgres_config,
+            "lance": lance_config,
         }
         # Support both GRAPH_DB_BACKEND and legacy NEO4J_BACKEND env vars
         graph_db_backend = os.getenv(
@@ -1210,11 +1222,13 @@ def get_default_cube_config() -> "GeneralMemCubeConfig | None":
         neo4j_config = APIConfig.get_neo4j_config(user_id="default")
         polardb_config = APIConfig.get_polardb_config(user_id="default")
         postgres_config = APIConfig.get_postgres_config(user_id="default")
+        lance_config = APIConfig.get_lance_graph_config(user_id="default")
         graph_db_backend_map = {
             "neo4j-community": neo4j_community_config,
             "neo4j": neo4j_config,
             "polardb": polardb_config,
             "postgres": postgres_config,
+            "lance": lance_config,
         }
         internet_config = (
             APIConfig.get_internet_config()

src/memos/api/handlers/config_builders.py

@@ -41,6 +41,7 @@ def build_graph_db_config(user_id: str = "default") -> dict[str, Any]:
         "neo4j": APIConfig.get_neo4j_config(user_id=user_id),
         "polardb": APIConfig.get_polardb_config(user_id=user_id),
         "postgres": APIConfig.get_postgres_config(user_id=user_id),
+        "lance": APIConfig.get_lance_graph_config(user_id=user_id),
     }
 
     # Support both GRAPH_DB_BACKEND and legacy NEO4J_BACKEND env vars
@@ -62,10 +63,27 @@ def build_vec_db_config() -> dict[str, Any]:
     Returns:
         Validated vector database configuration dictionary
     """
+    vec_db_backend = os.getenv("MOS_VEC_DB_BACKEND", "milvus").lower()
+
+    config = {}
+    if vec_db_backend == "milvus":
+        config = APIConfig.get_milvus_config()
+    elif vec_db_backend == "lance":
+        base_uri = os.getenv("LANCE_URI", "./data/lance_db")
+        config = {
+            "uri": base_uri,
+            "collection_name": ["memories"],
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 2048)),
+        }
+    elif vec_db_backend == "qdrant":
+        config = APIConfig.get_qdrant_config()
+    else:
+        raise ValueError(f"Unsupported vector DB backend: {vec_db_backend}")
+
     return VectorDBConfigFactory.model_validate(
         {
-            "backend": "milvus",
-            "config": APIConfig.get_milvus_config(),
+            "backend": vec_db_backend,
+            "config": config,
         }
     )
 

src/memos/api/handlers/search_handler.py

@@ -6,6 +6,7 @@
 """
 
 import copy
+import logging
 import math
 
 from typing import Any
@@ -71,7 +72,40 @@ def handle_search_memories(self, search_req: APISearchRequest) -> SearchResponse
         results = cube_view.search_memories(search_req_local)
         if not search_req_local.relativity:
             search_req_local.relativity = 0
+
         self.logger.info(f"[SearchHandler] Relativity filter: {search_req_local.relativity}")
+
+        # Extract and log scores for visibility before filtering
+        if self.logger.isEnabledFor(logging.DEBUG):
+            score_details = []
+            for key in ("text_mem", "pref_mem"):
+                buckets = results.get(key)
+                if not isinstance(buckets, list):
+                    continue
+                for bucket in buckets:
+                    memories = bucket.get("memories")
+                    if not isinstance(memories, list):
+                        continue
+                    for mem in memories:
+                        if not isinstance(mem, dict):
+                            continue
+                        mem_text = mem.get("memory", "").replace("\n", "  ")
+                        # Truncate to 100 chars to avoid log flooding
+                        if len(mem_text) > 100:
+                            mem_text = mem_text[:100] + "..."
+                        meta = mem.get("metadata", {})
+                        score = meta.get("relativity", 1.0) if isinstance(meta, dict) else 1.0
+                        try:
+                            score_val = float(score) if score is not None else 1.0
+                        except (TypeError, ValueError):
+                            score_val = 1.0
+                        score_details.append(f"[{score_val:.4f}] {mem_text}")
+
+            if score_details:
+                self.logger.debug(
+                    f"[SearchHandler] Reranker scores before threshold ({search_req_local.relativity}): \n"
+                    + "\n".join(score_details)
+                )
         results = self._apply_relativity_threshold(results, search_req_local.relativity)
 
         if search_req_local.dedup == "sim":

src/memos/configs/graph_db.py

@@ -241,6 +241,28 @@ def validate_config(self):
         return self
 
 
+class LanceGraphDBConfig(BaseConfig):
+    """
+    LanceDB-specific configuration.
+    """
+
+    uri: str = Field(..., description="The URI/path to the LanceDB dataset")
+    user_name: str | None = Field(
+        default=None,
+        description="Logical user or tenant ID for data isolation",
+    )
+    embedding_dimension: int = Field(default=768, description="Dimension of vector embedding")
+    compaction_version_threshold: int = Field(
+        default=500, description="Number of new versions to accumulate before triggering compaction"
+    )
+    compaction_interval_mins: int = Field(
+        default=30, description="Fallback interval in minutes to check and run compaction"
+    )
+    cleanup_older_than_days: int = Field(
+        default=7, description="Number of days to keep old versions before pruning"
+    )
+
+
 class GraphDBConfigFactory(BaseModel):
     backend: str = Field(..., description="Backend for graph database")
     config: dict[str, Any] = Field(..., description="Configuration for the graph database backend")
@@ -250,6 +272,7 @@ class GraphDBConfigFactory(BaseModel):
         "neo4j-community": Neo4jCommunityGraphDBConfig,
         "polardb": PolarDBGraphDBConfig,
         "postgres": PostgresGraphDBConfig,
+        "lance": LanceGraphDBConfig,
     }
 
     @field_validator("backend")

src/memos/graph_dbs/base.py

@@ -302,3 +302,7 @@ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = N
                 - metadata: dict[str, Any] - Node metadata
             user_name: Optional user name (will use config default if not provided)
         """
+
+    @abstractmethod
+    def node_not_exist(self, scope: str, user_name: str | None = None) -> bool:
+        pass

src/memos/graph_dbs/factory.py

@@ -2,6 +2,7 @@
 
 from memos.configs.graph_db import GraphDBConfigFactory
 from memos.graph_dbs.base import BaseGraphDB
+from memos.graph_dbs.lance import LanceGraphDB
 from memos.graph_dbs.neo4j import Neo4jGraphDB
 from memos.graph_dbs.neo4j_community import Neo4jCommunityGraphDB
 from memos.graph_dbs.polardb import PolarDBGraphDB
@@ -16,6 +17,7 @@ class GraphStoreFactory(BaseGraphDB):
         "neo4j-community": Neo4jCommunityGraphDB,
         "polardb": PolarDBGraphDB,
         "postgres": PostgresGraphDB,
+        "lance": LanceGraphDB,
     }
 
     @classmethod