Merge pull request #5 from Ontos-AI/staging

Staging

Author: suguanYang

.github/codeql/codeql-config.yml

@@ -0,0 +1,5 @@
+name: "Knowhere CodeQL config"
+
+paths-ignore:
+  - apps/api/scripts/**
+  - packages/shared-python/shared/utils/api_keys.py

.github/workflows/codeql.yml

@@ -0,0 +1,46 @@
+name: CodeQL
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+
+permissions:
+  contents: read
+  security-events: write
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: python
+          queries: security-extended,security-and-quality
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v4
+
+      - name: Perform CodeQL analysis
+        uses: github/codeql-action/analyze@v4

.github/workflows/pr-ci.yml

@@ -5,11 +5,6 @@ on:
     branches:
       - main
       - staging
-  push:
-    branches:
-      - main
-      - staging
-  workflow_dispatch:
 
 permissions:
   contents: read
@@ -20,7 +15,7 @@ concurrency:
 
 env:
   PYTHON_VERSION: "3.11"
-  UV_VERSION: "0.10.0"
+  UV_VERSION: "0.11.7"
   UV_LINK_MODE: copy
 
 jobs:
@@ -47,8 +42,6 @@ jobs:
           enable-cache: true
           cache-dependency-glob: |
             uv.lock
-            apps/*/uv.lock
-            packages/*/uv.lock
 
       - name: Install dependencies
         run: uv sync --locked --all-packages --all-groups
@@ -79,8 +72,6 @@ jobs:
           enable-cache: true
           cache-dependency-glob: |
             uv.lock
-            apps/*/uv.lock
-            packages/*/uv.lock
 
       - name: Install dependencies
         run: uv sync --locked --all-packages --all-groups
@@ -111,8 +102,6 @@ jobs:
           enable-cache: true
           cache-dependency-glob: |
             uv.lock
-            apps/*/uv.lock
-            packages/*/uv.lock
 
       - name: Install dependencies
         run: uv sync --locked --all-packages --all-groups

.github/workflows/secret-scan.yml

@@ -5,13 +5,6 @@ on:
     branches:
       - main
       - staging
-  push:
-    branches:
-      - main
-      - staging
-  workflow_dispatch:
-  schedule:
-    - cron: "17 3 * * *"
 
 permissions:
   contents: read

.gitignore

@@ -12,6 +12,7 @@ build/
 __pycache__/
 *.pyc
 *.pyo
+.coverage
 .venv/
 venv/
 venv_pptx2md/
@@ -46,6 +47,7 @@ logs/
 *.log
 
 # Generated files
+.build/
 apps/api/openapi.json
 packages/shared-types/generated/
 

README.md

@@ -37,8 +37,6 @@ knowhere-api/
 - Python 3.11+
 - `uv`
 - Docker with `docker compose`
-- a local Chrome or Chromium driver if you plan to run document layout parsing
-  flows
 
 ## Configuration
 
@@ -61,51 +59,35 @@ cp apps/worker/.env.example apps/worker/.env
 
 - database and Redis connection settings
 - S3-compatible storage credentials
-- `SECRET_KEY`
-- `USERS_DATA_PATH`
 - `DS_KEY`
 - any optional LLM, billing, or webhook providers you want to enable
 
-The example files default to the open-source/self-hosted behavior:
-
-- `API_STANDALONE_MODE_ENABLED=false` for the combined dashboard + API flow, where
-  the dashboard initializes Better Auth tables before API migrations.
-- `BILLING_ENABLED=false`, so Stripe and credit deduction are not required.
-- `RATE_LIMIT_ENABLED=false` for local/self-hosted convenience; set it to
-  `true` when you want API rate limits enforced.
-
-For API-only development without the dashboard, set `API_STANDALONE_MODE_ENABLED=true`,
-run API migrations, then create an API-only user/key:
-
-```bash
-cd apps/api
-uv run --python 3.11 python -m alembic upgrade heads
-uv run --python 3.11 python scripts/init_user.py --email you@example.com
-```
-
-If you plan to use the dashboard, start the combined self-hosted stack and
-register through the dashboard instead of using `scripts/init_user.py`.
-
 4. Start the local infrastructure stack:
 
 ```bash
 ./deploy/local-dev/start-dev.sh
 ```
 
-If you also want the helper to initialize the local API user state, rerun it
-with `--init-user`:
+5. Start the API and worker in separate terminals:
 
 ```bash
-./deploy/local-dev/start-dev.sh --init-user
+cd apps/api && uv run main.py
+cd apps/worker && uv run worker.py
 ```
 
-5. Start the API and worker in separate terminals:
+The API runs migrations during startup.
+
+For API-only development without the dashboard, create an API-only user/key
+after the API service starts:
 
 ```bash
-cd apps/api && uv run main.py
-cd apps/worker && uv run worker.py
+cd apps/api
+uv run scripts/init_user.py --email you@example.com
 ```
 
+If you plan to use the dashboard, register through the dashboard instead of
+using `scripts/init_user.py`.
+
 ## Quality Checks
 
 Run lint checks from the repository root:

apps/api/.env.example

@@ -10,7 +10,7 @@
 # Required for specific features:
 # - alternate object storage callbacks
 # - webhooks and async callback delivery
-# - billing, email, analytics, and dashboard auth providers
+# - billing and analytics
 # - alternate parsing providers
 #
 # Optional or development-only values can stay empty unless you need the
@@ -19,20 +19,13 @@
 # Required for local startup: application runtime
 ENVIRONMENT=development
 APP_ENV=
-DEBUG=true
 LOG_LEVEL=INFO
 APP_TITLE=Knowhere API
 APP_VERSION=1.0.0
 APP_DESCRIPTION=Document ingestion, retrieval, and MCP backend
-SECRET_KEY=replace-with-a-long-random-secret
-ALGORITHM=HS256
-ACCESS_TOKEN_EXPIRE_MINUTES=10080
 INTERNAL_DASHBOARD_ENDPOINT=http://localhost:3000
-API_STANDALONE_MODE_ENABLED=false
+API_STANDALONE_MODE_ENABLED=true
 TMP_PATH=/tmp/knowhere
-FONT_PATH=/usr/share/fonts
-CHROMEDRIVER_PATH=/usr/bin/chromedriver
-USERS_DATA_PATH=/tmp/knowhere-users
 
 # Optional or development-only: observability and local dashboard wiring
 LOGFIRE_TOKEN=
@@ -46,7 +39,7 @@ DB_SSL_MODE=disable
 # DB_SSL_ROOT_CERT=/path/to/ca-cert.pem
 
 # Required for local startup: Redis / Celery
-RATE_LIMIT_ENABLED=false
+RATE_LIMIT_ENABLED=true
 REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_PASSWORD=
@@ -57,7 +50,6 @@ BROKER_POOL_LIMIT=10
 # Required for local startup: S3-compatible storage
 S3_TYPE=s3
 S3_BUCKET_NAME=knowhere-uploads
-S3_UPLOADS_BUCKET=knowhere-uploads
 S3_RESULTS_BUCKET=knowhere-results
 S3_ACCESS_KEY_ID=test
 S3_SECRET_ACCESS_KEY=test
@@ -68,7 +60,6 @@ S3_REGION=us-west-1
 S3_USE_SSL=false
 S3_ADDRESSING_STYLE=path
 S3_WEBHOOK_AUTH_TOKEN=replace-with-a-shared-secret
-SNS_SIGNATURE_VERIFICATION=true
 
 # Required for specific features: OSS settings
 OSS_ENDPOINT=
@@ -90,7 +81,6 @@ ALI_SDK_MAX_RETRIES=3
 ALI_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
 ARK_API_KEY=
 ARK_URL=https://ark.cn-beijing.volces.com/api/v3/chat/completions
-EMBEDDING_MODEL=text-embedding-v4
 NORMOL_MODEL=deepseek-chat
 HIERARCHY_LLM_MODEL=qwen3.6-flash
 IMAGE_MODEL=qwen3.5-flash
@@ -99,58 +89,23 @@ IMAGE_MODEL_MAX=qwen3.5-flash
 # File handling defaults
 SUPPORTED_EXTENSIONS=.doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx,.jpg,.jpeg,.png,.md
 MAX_FILE_SIZE=104857600
-MAX_IMAGE_SIZE=10485760
-MIN_CONFIDENCE_THRESHOLD=0.05
-HIGH_IOU_THRESHOLD=0.9
-DEFAULT_EMBEDDING_DIM=1024
-DEFAULT_TOP_K=5
-DEFAULT_BATCH_SIZE=32
-DEFAULT_EPOCHS=3
-DEFAULT_THRESHOLD=0.5
 
 # Required for specific features: webhooks and callbacks
-WEBHOOK_SIGNING_SECRET=
 WEBHOOK_MASTER_KEY=
 QSTASH_TOKEN=
 QSTASH_CALLBACK_BASE_URL=https://api.example.com/api/v1
 QSTASH_MAX_RETRIES=5
 # QSTASH_CURRENT_SIGNING_KEY=
 # QSTASH_NEXT_SIGNING_KEY=
 
-# Required for specific features: billing and notifications
-BILLING_ENABLED=false
+# Required for specific features: billing and analytics
+BILLING_ENABLED=true
 STRIPE_SECRET_KEY=
-STRIPE_PUBLISHABLE_KEY=
 STRIPE_WEBHOOK_SECRET=
-RESEND_API_KEY=
-RESEND_FROM_EMAIL=noreply@example.com
-RESEND_FROM_NAME=Knowhere
-RESEND_MAX_RETRIES=3
-RESEND_RETRY_DELAY=1.0
-RESEND_TEMPLATE_WELCOME=
-RESEND_TEMPLATE_PURCHASE_CONFIRMATION=
-RESEND_TEMPLATE_JOB_COMPLETION=
-RESEND_TEMPLATE_JOB_FAILURE=
-RESEND_TEMPLATE_WELCOME_ENABLED=false
-RESEND_TEMPLATE_PURCHASE_CONFIRMATION_ENABLED=false
-RESEND_TEMPLATE_JOB_COMPLETION_ENABLED=false
-RESEND_TEMPLATE_JOB_FAILURE_ENABLED=false
 MOESIF_APPLICATION_ID=
-NEXT_PUBLIC_POSTHOG_KEY=
-NEXT_PUBLIC_POSTHOG_HOST=https://app.posthog.com
 FREE_PLAN_INITIAL_CREDITS=5
 FRONTEND_URL=http://localhost:3000
 
-# Required for specific features: dashboard and auth providers
-USERS_VERIFY_TOKEN_SECRET=
-USERS_RESET_PASSWORD_TOKEN_SECRET=
-GOOGLE_CLIENT_ID=
-GOOGLE_CLIENT_SECRET=
-GITHUB_CLIENT_ID=
-GITHUB_CLIENT_SECRET=
-APPLE_CLIENT_ID=
-APPLE_CLIENT_SECRET=
-
 # Required for specific features: parsing providers
 MINERU_API_KEYS=
 MINERU_URL=https://mineru.net/api/v4
@@ -163,13 +118,6 @@ ILOVEAPI_SECRET_KEY=
 ILOVEAPI_BASE_URL=https://api.ilovepdf.com/v1
 ILOVEAPI_TIMEOUT=120
 
-# Optional or development-only: compatibility fields kept for retained legacy code paths
+# Legacy parser compatibility fields.
 ALL_DF_COLS=content,path,type,length,keywords,summary,know_id,tokens,connectto,addtime,page_nums
-DEFAULT_FOLDERS=Supplementary_Files,Temporary_Files,templates,images,fragments
-KB_TERM=KB_DATA
-KB_VEC_TERM=KB_VECS
-META_PATH=app/core/config/Meta_setting.csv
-CONFIG_PATH=app/core/config/config.txt
-PATH_IMAGE_PATTERN=.*\.(png|jpe?g|gif)$
-IMG_TBL_PATTERN=\[(?:images|tables)/[^\]]+\]
-SPLIT_CHAR=/
+SPLIT_CHAR=-->

apps/api/alembic/env.py

@@ -136,7 +136,7 @@ def run_with_connection(connection: Connection) -> None:
         return
 
     if isinstance(configured_connection, Engine):
-        with configured_connection.connect() as connection:
+        with configured_connection.begin() as connection:
             run_with_connection(connection)
         return
 
@@ -149,7 +149,7 @@ def run_with_connection(connection: Connection) -> None:
         connect_args=ssl_connect_args,
     )
 
-    with connectable.connect() as connection:
+    with connectable.begin() as connection:
         run_with_connection(connection)
 
 

apps/api/app/api/v1/api_v1.py

@@ -2,7 +2,6 @@
 API v1 route registry.
 """
 
-from app.api.v1 import health
 from app.api.v1.routes import (
     api_key,
     documents,
@@ -59,9 +58,6 @@
     qstash_callbacks.router, prefix="/webhooks", tags=["QStash Callbacks"]
 )
 
-# Health check
-api_router.include_router(health.router, prefix="/health", tags=["Health"])
-
 # Version info
 api_router.include_router(version.router, tags=["Version"])