diff --git a/website/docs/getstarted/build_from_source.mdx b/website/docs/getstarted/build_from_source.mdx index 6da508b..c744ba6 100644 --- a/website/docs/getstarted/build_from_source.mdx +++ b/website/docs/getstarted/build_from_source.mdx @@ -295,7 +295,7 @@ cmake --build . -t test_main 2. Install Python sdk of infinity: ```bash - pip install infinity-sdk==0.6.15 + pip install infinity-sdk==0.7.0.dev1 ``` 3. Run the functional tests: @@ -317,7 +317,7 @@ cmake --build . -t test_main 2. Install Python sdk of infinity: ```bash - pip install infinity-sdk==0.6.15 + pip install infinity-sdk==0.7.0.dev1 ``` 3. Run the functional tests: @@ -342,7 +342,7 @@ cmake --build . -t test_main 2. Install Python sdk of infinity: ```bash - pip install infinity-sdk==0.6.15 + pip install infinity-sdk==0.7.0.dev1 ``` 3. Run the functional tests: diff --git a/website/docs/getstarted/deploy_infinity_server.mdx b/website/docs/getstarted/deploy_infinity_server.mdx index 35c192e..61c80a3 100644 --- a/website/docs/getstarted/deploy_infinity_server.mdx +++ b/website/docs/getstarted/deploy_infinity_server.mdx @@ -35,7 +35,7 @@ This section provides instructions on deploying Infinity using binary package on Fedora/RHEL/CentOS/OpenSUSE ```bash -sudo rpm -i infinity-0.6.15-x86_64.rpm +sudo rpm -i infinity-0.7.0.dev1-x86_64.rpm ``` ```bash @@ -46,7 +46,7 @@ sudo systemctl start infinity ```bash -sudo dpkg -i infinity-0.6.15-x86_64.deb +sudo dpkg -i infinity-0.7.0.dev1-x86_64.deb ``` ```bash @@ -59,7 +59,7 @@ sudo systemctl start infinity ## Install Infinity client ``` -pip install infinity-sdk==0.6.15 +pip install infinity-sdk==0.7.0.dev1 ``` ## Run a vector search diff --git a/website/docs/getstarted/quickstart.mdx b/website/docs/getstarted/quickstart.mdx index 190222b..bc2d953 100644 --- a/website/docs/getstarted/quickstart.mdx +++ b/website/docs/getstarted/quickstart.mdx @@ -59,7 +59,7 @@ If you are on Windows 10+, you must enable WSL or WSL2 to deploy Infinity using ### Install Infinity client ``` -pip install infinity-sdk==0.6.15 +pip install infinity-sdk==0.7.0.dev1 ``` ### Run a vector search diff --git a/website/docs/references/configurations.mdx b/website/docs/references/configurations.mdx index de6642b..44cf754 100644 --- a/website/docs/references/configurations.mdx +++ b/website/docs/references/configurations.mdx @@ -48,7 +48,7 @@ If you start up Infinity from DEB or RPM, the configuration file will be in the ```toml [general] # Infinity server version, the minor version of this parameter should match server minor version. -version = "0.6.15" +version = "0.7.0" # Timezone time_zone = "utc-8" # The number of worker threads. Defaults to the number of CPU cores. diff --git a/website/docs/references/http_api_reference.mdx b/website/docs/references/http_api_reference.mdx index 5983cb2..8884acf 100644 --- a/website/docs/references/http_api_reference.mdx +++ b/website/docs/references/http_api_reference.mdx @@ -3135,7 +3135,7 @@ The response includes a JSON object like the following: "server_address":"0.0.0.0", "temp_dir":"/var/infinity/tmp", "time_zone":"UTC+8", - "version":"0.6.15", + "version":"0.7.0", "wal_compact_threshold":"1073741824", "wal_dir":"/var/infinity/wal", "wal_flush":"FlushAtOnce" @@ -3184,7 +3184,7 @@ The response includes a JSON object like the following: ```shell { "error_code": 0, - "version":"0.6.15" + "version":"0.7.0" } ``` @@ -4435,7 +4435,7 @@ The response includes a JSON object like the following: "server_address":"0.0.0.0", "temp_dir":"/var/infinity/leader/tmp", "time_zone":"UTC-8", - "version":"0.6.15", + "version":"0.7.0", "wal_compact_threshold":"1073741824", "wal_dir":"/var/infinity/leader/wal", "wal_flush":"FlushAtOnce" diff --git a/website/versioned_docs/version-v0.7.0-dev1/getstarted/_category_.json b/website/versioned_docs/version-v0.7.0-dev1/getstarted/_category_.json new file mode 100644 index 0000000..437d33f --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/getstarted/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Get started", + "position": 0, + "link": { + "type": "generated-index", + "description": "quickstart and more" + } +} diff --git a/website/versioned_docs/version-v0.7.0-dev1/getstarted/build_from_source.mdx b/website/versioned_docs/version-v0.7.0-dev1/getstarted/build_from_source.mdx new file mode 100644 index 0000000..c744ba6 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/getstarted/build_from_source.mdx @@ -0,0 +1,368 @@ +--- +sidebar_position: 2 +slug: /build_from_source +--- + +# Build from Source +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Build Infinity from source, build and run unit/functional tests. + +--- + +This document provides instructions for building Infinity from source, as well as building and running unit and functional tests. + +:::tip NOTE +Infinity can be natively compiled on Linux only. For non-Linux operating systems, build it using Docker. +::: + +- [Prerequisites](#prerequisites) +- [Build from source](#build-the-source-code) +- [Build and run unit tests](#build-and-run-unit-tests) +- [Build and run functional tests](#build-and-run-functional-tests) + +## Prerequisites + +### 1. Replace the native Out-Of-Memory killer + +Building the entire project, particularly during the link stage, requires considerable RAM, which can cause the host machine to become unresponsive due to the sluggishness of the kernel's Out-Of-Memory (OOM) killer. To mitigate this, we recommend installing [earlyoom](https://github.com/rfjakob/earlyoom) to improve the host's responsiveness. + +### 2. Limit the number of concurrent link processes + +Configure `cmake` to limit the number of concurrent link processes. For example: + + `-DCMAKE_JOB_POOLS:STRING='link=4'` + +:::tip NOTE +Recommended link pool size settings are as follows: + +- `1`: 6 GB of RAM +- `2`: 16 GB of RAM +- `3`: 32 GB of RAM +::: + +### 3. Set the `cmake` build type + +The `cmake` build type (`CMAKE_BUILD_TYPE`) can be one of the following: + +- `Debug`: Suitable for daily development. + No inline; with symbol info; with address sanitizer. + Typically ~10x slower than `RelWithDebInfo` or `Release`. +- `RelWithDebInfo`: Suitable for performance analysis. + Optimizes with `-O2`; with symbol information. +- `Release`: Suitable for [project releases](https://github.com/infiniflow/infinity/releases). + Optimizes with `-O3`; without symbol information. + The built executables are significantly smaller than those of `RelWithDebInfo`. + +:::tip NOTE +The following procedures set `CMAKE_BUILD_TYPE` to `Debug`. Change it as you see necessary. +::: + +## Build the source code + + + + +This section provides instructions for building Infinity from source on Linux using Docker. + +1. Download the source code: + +```shell +git clone https://github.com/infiniflow/infinity.git +``` + +2. Build the source code using Docker: + +```shell +cd infinity && mkdir cmake-build-debug +TZ=$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}') +docker run -d --name infinity_build -e TZ=$TZ -v $PWD:/infinity -v /boot:/boot infiniflow/infinity_builder:ubuntu20_clang20 +docker exec infinity_build bash -c "cd /infinity/cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. && cmake --build . -t infinity" +``` + +3. Start up the Infinity server: + +```shell +sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity +ulimit -n 500000 +./cmake-build-debug/src/infinity +``` + + + + +This section provides instructions for building Infinity from source on Ubuntu. + +1. Install necessary dependencies: + + + + +```shell +sudo apt update && sudo apt install git wget unzip software-properties-common pkg-config bison +wget https://github.com/Kitware/CMake/releases/download/v4.0.3/cmake-4.0.3-linux-x86_64.tar.gz +tar zxvf cmake-4.0.3-linux-x86_64.tar.gz +sudo cp -rf cmake-4.0.3-linux-x86_64/bin/* /usr/local/bin && sudo cp -rf cmake-4.0.3-linux-x86_64/share/* /usr/local/share && rm -rf cmake-4.0.3-linux-x86_64 +wget https://github.com/ninja-build/ninja/archive/refs/tags/v1.13.1.zip +unzip v1.13.1.zip && cd ninja-1.13.1 && cmake -Bbuild-cmake -DBUILD_TESTING=OFF && cmake --build build-cmake --parallel $(nproc) && cd build-cmake && sudo cp ninja /usr/local/bin +cd ../.. && rm -rf v1.13.1.zip ninja-1.13.1 +wget https://github.com/rui314/mold/archive/refs/tags/v2.40.3.zip +./install-build-deps.sh +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-15 -B build +cmake --build build -j$(nproc) +sudo cmake --build build --target install +# compile gcc-15 and install it +cd ~/Downloads # Of course, you can specify a different path. +git clone https://github.com/microsoft/vcpkg.git +cd vcpkg && ./bootstrap-vcpkg.sh -disableMetrics +sh -c 'cat <> ~/.bashrc +export VCPKG_ROOT=${PWD} +export PATH=${VCPKG_ROOT}:${PATH} +EOF' +source ~/.bashrc +wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh 20 && rm llvm.sh +sudo add-apt-repository -P ppa:ubuntu-toolchain-r/test +sudo apt update && sudo apt install clang-tools-20 flex zlib1g-dev python3-dev autoconf +wget https://github.com/jemalloc/jemalloc/archive/refs/tags/5.3.0.tar.gz +tar zxvf 5.3.0.tar.gz && cd jemalloc-5.3.0 && ./autogen.sh && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-libdl --enable-prof --enable-prof-libunwind --disable-initial-exec-tls && sudo make -j install && cd .. +sudo ldconfig && sudo rm -rf jemalloc-5.3.0 5.3.0.tar.gz +sudo ln -s /usr/lib/llvm-20/bin/clang-scan-deps /usr/bin/clang-scan-deps +sudo ln -s /usr/bin/clang-format-20 /usr/bin/clang-format +sudo ln -s /usr/bin/clang-tidy-20 /usr/bin/clang-tidy +sudo ln -s /usr/bin/llvm-symbolizer-20 /usr/bin/llvm-symbolizer +sudo ln -s /usr/lib/llvm-20/include/x86_64-pc-linux-gnu/c++/v1/__config_site /usr/lib/llvm-20/include/c++/v1/__config_site +``` + + + + +```shell +sudo apt update && sudo apt install -y git wget lsb-release software-properties-common pkg-config bison mold +wget https://github.com/Kitware/CMake/releases/download/v4.0.3/cmake-4.0.3-linux-x86_64.tar.gz +tar zxvf cmake-4.0.3-linux-x86_64.tar.gz +sudo cp -rf cmake-4.0.3-linux-x86_64/bin/* /usr/local/bin && sudo cp -rf cmake-4.0.3-linux-x86_64/share/* /usr/local/share && rm -rf cmake-4.0.3-linux-x86_64 +wget https://github.com/ninja-build/ninja/archive/refs/tags/v1.13.1.zip +unzip v1.13.1.zip && cd ninja-1.13.1 && cmake -Bbuild-cmake -DBUILD_TESTING=OFF && cmake --build build-cmake --parallel $(nproc) && cd build-cmake && sudo cp ninja /usr/local/bin +cd ../.. && rm -rf v1.13.1.zip ninja-1.13.1 +sudo add-apt-repository ppa:puni070/gcc-noble +sudo apt update +sudo apt install gcc-15 g++-15 +# Ensure that no other gcc&g++ versions have a priority set, or that if they do, their priority is below 100. +sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-15 100 +sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-15 100 +cd ~/Downloads # Of course, you can specify a different path. +git clone https://github.com/microsoft/vcpkg.git +cd vcpkg && ./bootstrap-vcpkg.sh -disableMetrics +sh -c 'cat <> ~/.bashrc +export VCPKG_ROOT=${PWD} +export PATH=${VCPKG_ROOT}:${PATH} +EOF' +source ~/.bashrc +wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh 20 && rm llvm.sh +sudo apt install -y clang-tools-20 flex zlib1g-dev python3-dev autoconf libjemalloc-dev +# Ensure that no other Clang versions have a priority set, or that if they do, their priority is below 100. +sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 +sudo ln -s /usr/lib/llvm-20/bin/clang-scan-deps /usr/bin/clang-scan-deps +sudo ln -s /usr/bin/clang-format-20 /usr/bin/clang-format +sudo ln -s /usr/bin/clang-tidy-20 /usr/bin/clang-tidy +sudo ln -s /usr/bin/llvm-symbolizer-20 /usr/bin/llvm-symbolizer +sudo ln -s /usr/lib/llvm-20/include/x86_64-pc-linux-gnu/c++/v1/__config_site /usr/lib/llvm-20/include/c++/v1/__config_site +``` + + + + +2. Download the source code: + +```shell +git clone https://github.com/infiniflow/infinity.git +``` + +3. Build the source code: + +:::tip NOTE +You must *also* install `simde` v0.7.4+ if using Ubuntu 22.04 on an ARM architecture. + +```shell +sudo apt install libsimde-dev +``` + +If your installed version is below v0.7.4, download the include files directly from github and replace them. +::: + +```shell +git config --global --add safe.directory infinity +cd infinity && mkdir cmake-build-debug && cd cmake-build-debug +export CC=/usr/bin/clang-20 +export CXX=/usr/bin/clang++-20 +cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. +cmake --build . -t infinity +``` + +4. Start up the Infinity server: + +```shell +sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity +ulimit -n 500000 +./cmake-build-debug/src/infinity +``` + + + + +## Build and run unit tests + + + + +```shell +docker exec infinity_build bash -c "cd /infinity/cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. && cmake --build . -t test_main" +./cmake-build-debug/src/test_main +``` + + + + +```shell +cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. +cmake --build . -t test_main +./cmake-build-debug/src/test_main +``` + + + + +1. Install the `llvm-20` dependency: + + ```shell + sudo apt install llvm-20 + ``` + +2. Build and run the unit tests with code coverage enabled: + + ```shell + cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON -DCODE_COVERAGE=ON .. + cmake --build . -t test_main + find . -name "infinity*.profraw" | xargs rm -f + export LLVM_PROFILE_FILE="infinity-%p.profraw" + ./cmake-build-debug/src/test_main + ``` + +3. Use llvm-cov-20 to generate summarized code coverage results: + + ```shell + llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata + llvm-cov-20 export cmake-build-debug/src/test_main -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov + ``` + + + + +## Build and run functional tests + + + + +1. Build and start up the Infinity server: + + ```shell + docker exec infinity_build bash -c "cd /infinity/cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. && cmake --build . -t infinity" + ./cmake-build-debug/src/infinity + ``` + +2. Install Python sdk of infinity: + ```bash + pip install infinity-sdk==0.7.0.dev1 + ``` + +3. Run the functional tests: + + ```shell + python3 tools/run_pytest_parallel.py + ``` + + + + +1. Build and start up the Infinity server: + + ```shell + cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON .. + cmake --build . -t infinity + ./cmake-build-debug/src/infinity + ``` + +2. Install Python sdk of infinity: + ```bash + pip install infinity-sdk==0.7.0.dev1 + ``` + +3. Run the functional tests: + + ```shell + python3 tools/run_pytest_parallel.py + ``` + + + + +1. Build and start up the Infinity server with code coverage enabled: + + ```shell + cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_VERBOSE_MAKEFILE=ON -DCODE_COVERAGE=ON .. + cmake --build . -t infinity + find . -name "infinity*.profraw" | xargs rm -f + export LLVM_PROFILE_FILE="infinity-%p.profraw" + export LLVM_PROFILE_DUMP_ON_EXIT=1 + ./cmake-build-debug/src/infinity + ``` + +2. Install Python sdk of infinity: + ```bash + pip install infinity-sdk==0.7.0.dev1 + ``` + +3. Run the functional tests: + + ```shell + python3 tools/run_pytest_parallel.py + ``` + +4. Shut down the Infinity server: + + ```shell + kill `pidof infinity` + ``` + +5. Use llvm-cov-20 to generate summarized code coverage results: + + ```shell + llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata + llvm-cov-20 export cmake-build-debug/src/infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov + ``` + + + diff --git a/website/versioned_docs/version-v0.7.0-dev1/getstarted/deploy_infinity_server.mdx b/website/versioned_docs/version-v0.7.0-dev1/getstarted/deploy_infinity_server.mdx new file mode 100644 index 0000000..61c80a3 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/getstarted/deploy_infinity_server.mdx @@ -0,0 +1,84 @@ +--- +sidebar_position: 3 +slug: /deploy_infinity_server +--- + +# Deploy Infinity using binary +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Deploy the Infinity database using binary. + +--- + +This document provides guidance on deploying the Infinity database using binary, with the client and server as separate processes. + +## Prerequisites + +- CPU: x86_64 with AVX2 support. +- OS: + - Linux with glibc 2.17+. + - Windows 10+ with WSL/WSL2. + +## Install Infinity server + +This section provides instructions on deploying Infinity using binary package on Linux x86_64. You can download the binary package (deb, rpm, or tgz) for your Linux system from https://github.com/infiniflow/infinity/releases. The prebuilt packages are compatible with Linux distributions based on glibc 2.17 or later, for example, RHEL 7, Debian 8, Ubuntu 14.04. + + + + + +Fedora/RHEL/CentOS/OpenSUSE +```bash +sudo rpm -i infinity-0.7.0.dev1-x86_64.rpm +``` + +```bash +sudo systemctl start infinity +``` + + + + +```bash +sudo dpkg -i infinity-0.7.0.dev1-x86_64.deb +``` + +```bash +sudo systemctl start infinity +``` + + + + +## Install Infinity client + +``` +pip install infinity-sdk==0.7.0.dev1 +``` + +## Run a vector search + +```python +import infinity + +infinity_object = infinity.connect(infinity.NetworkAddress("", 23817)) +db_object = infinity_object.get_database("default_db") +table_object = db_object.create_table("my_table", {"num": {"type": "integer"}, "body": {"type": "varchar"}, "vec": {"type": "vector, 4, float"}}) +table_object.insert([{"num": 1, "body": "unnecessary and harmful", "vec": [1.0, 1.2, 0.8, 0.9]}]) +table_object.insert([{"num": 2, "body": "Office for Harmful Blooms", "vec": [4.0, 4.2, 4.3, 4.5]}]) +res = table_object.output(["*"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "ip", 2) + .to_pl() +print(res) +``` + +:::tip NOTE +For detailed information about the capabilities and usage of Infinity's Python API, see the [Python API Reference](../references/pysdk_api_reference.md). +::: + diff --git a/website/versioned_docs/version-v0.7.0-dev1/getstarted/quickstart.mdx b/website/versioned_docs/version-v0.7.0-dev1/getstarted/quickstart.mdx new file mode 100644 index 0000000..bc2d953 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/getstarted/quickstart.mdx @@ -0,0 +1,106 @@ +--- +sidebar_position: 1 +slug: / +--- + +# Quickstart +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +A quickstart guide. + +## Prerequisites + +- CPU: x86_64 with AVX2 support. +- OS: + - Linux with glibc 2.17+. + - Windows 10+ with WSL/WSL2. + - MacOS +- Python: Python 3.10+. + +## Deploy Infinity using Docker + +This section provides guidance on deploying the Infinity database using Docker, with the client and server as separate processes. + +### Install Infinity server + + + + +```bash +sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity +docker pull infiniflow/infinity:nightly +docker run -d --name infinity -v /var/infinity/:/var/infinity --ulimit nofile=500000:500000 --network=host infiniflow/infinity:nightly +``` + + + +If you are on Windows 10+, you must enable WSL or WSL2 to deploy Infinity using Docker. Suppose you've installed Ubuntu in WSL2: + +1. Follow [this](https://learn.microsoft.com/en-us/windows/wsl/systemd) to enable systemd inside WSL2. +2. Install docker-ce according to the [instructions here](https://docs.docker.com/engine/install/ubuntu). +3. If you have installed Docker Desktop version 4.29+ for Windows: **Settings** **>** **Features in development**, then select **Enable host networking**. +4. Pull the Docker image and start Infinity: + + ```bash + sudo mkdir -p /var/infinity && sudo chown -R $USER /var/infinity + docker pull infiniflow/infinity:nightly + docker run -d --name infinity -v /var/infinity/:/var/infinity --ulimit nofile=500000:500000 --network=host infiniflow/infinity:nightly + ``` + + + + +### Install Infinity client + +``` +pip install infinity-sdk==0.7.0.dev1 +``` + +### Run a vector search + +```python +import infinity + +infinity_obj = infinity.connect(infinity.NetworkAddress("", 23817)) +db_object = infinity_object.get_database("default_db") +table_object = db_object.create_table("my_table", {"num": {"type": "integer"}, "body": {"type": "varchar"}, "vec": {"type": "vector, 4, float"}}) +table_object.insert([{"num": 1, "body": "unnecessary and harmful", "vec": [1.0, 1.2, 0.8, 0.9]}]) +table_object.insert([{"num": 2, "body": "Office for Harmful Blooms", "vec": [4.0, 4.2, 4.3, 4.5]}]) +res = table_object.output(["*"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "ip", 2) + .to_pl() +print(res) +``` + +:::tip NOTE +For detailed information about the capabilities and usage of Infinity's Python API, see the [Python API Reference](../references/pysdk_api_reference.md). +::: + +:::info NOTE +If you wish to deploy Infinity *using binary* with the server and client as separate processes, see the [Deploy infinity using binary](https://infiniflow.org/docs/dev/deploy_infinity_server) guide. +::: + +## Build from Source + +If you wish to build Infinity from source, see the [Build from Source](https://infiniflow.org/docs/dev/build_from_source) guide. + +### Try our Python examples + +Try the following links to explore practical examples of using Infinity in Python: + +- [Create table, insert data, and search](https://github.com/infiniflow/infinity/blob/main/example/simple_example.py) +- [Import file and export data](https://github.com/infiniflow/infinity/blob/main/example/import_data.py) +- [Delete or update data](https://github.com/infiniflow/infinity/blob/main/example/delete_update_data.py) +- [Conduct a vector search](https://github.com/infiniflow/infinity/blob/main/example/vector_search.py) +- [Conduct a full-text search](https://github.com/infiniflow/infinity/blob/main/example/fulltext_search.py) +- [Conduct a hybrid search](https://github.com/infiniflow/infinity/blob/main/example/hybrid_search.py) + +## Python API reference + +For detailed information about the capabilities and usage of Infinity's Python API, see the [Python API Reference](https://infiniflow.org/docs/dev/pysdk_api_reference). diff --git a/website/versioned_docs/version-v0.7.0-dev1/guides/_category_.json b/website/versioned_docs/version-v0.7.0-dev1/guides/_category_.json new file mode 100644 index 0000000..b883c9f --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/guides/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Guides", + "position": 2, + "link": { + "type": "generated-index", + "description": "Infinity usage guide" + } +} diff --git a/website/versioned_docs/version-v0.7.0-dev1/guides/search_guide.md b/website/versioned_docs/version-v0.7.0-dev1/guides/search_guide.md new file mode 100644 index 0000000..8091646 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/guides/search_guide.md @@ -0,0 +1,214 @@ +--- +sidebar_position: 1 +slug: /search_guide +--- +# Conduct a search + +Full-text, vector, sparse vector, tensor, hybrid search. + +--- + +## Overview + +This document offers guidance on conducting a search within Infinity. + +## Full-text search + +### Work modes for full-text index + +A full-text index must be built to perform a full-text search, and this index operates in two modes: + +- **Real-time mode** - If created immediately after a table is created, a full-text index will be built on ingested data in real time, accumulating posting data within memory and flushing it to disk when a specified quota is reached. +- **Offline mode** - For data inserted before the creation of a full-text index, index will be built in offline mode using external sorting. + +### Tokenizer + +When creating a full-text index, you are required to specify a tokenizer/analyzer, which will be used for future full-text searches on the same column(s). Infinity has many built-in tokenizers. Except for the Ngram analyzer and the default standard analyzer, all other analyzers require dedicated resource files. Please download the appropriate files for your chosen analyzer from [this link](https://github.com/infiniflow/resource) and save it to the directory specified by `resource_dir` in the configuration file. + +```yaml +[resource] +# Directory for Infinity's resource files, including dictionaries to be used by analyzers +resource_dir = "/usr/share/infinity/resource" +``` + +The following are Infinity's built-in analyzers/tokenizers. + +#### Standard analyzer + +The standard analyzer is the default tokenizer and works best with Latin characters. It uses stemmer before outputting tokens segmented by white space, and `English` is the default stemmer. To specify a stemmer for a different language, use `"standard-xxx"`, where `xxx` is the language to use. + +Supported language stemmers include: `Danish`, `Dutch`, `English`, `Finnish`, `French`, `German`, `Hungarian`, `Italian`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`, `Russian`, `Spanish`, `Swedish`, and `Turkish`. + +#### Ngram analyzer + +A definition of N-gram can be found on [wikipedia](https://en.wikipedia.org/wiki/N-gram). Use `"ngram-x"` to select the Ngram analyzer, where `x` represents the value of `N`. For example, a common choice for full-text searches in code is `"ngram-3"`. + +#### Simplified Chinese analyzer + +Use `"chinese"` to select the simplified Chinese analyzer, which is a wrapper of [Jieba](https://github.com/yanyiwu/cppjieba) analyzer. Use `"chinese-fine"` to output fine-grained analyzer results. + +#### Traditional Chinese analyzer + +Use `"traditional"` to select the traditional Chinese analyzer, which essentially converts simplified Chinese into traditional Chinese based on the outputs of the [Jieba](https://github.com/yanyiwu/cppjieba) analyzer. + +#### Japanese analyzer + +Use `"japanese"` to select the Japanese analyzer, which is a wrapper of [mecab](http://taku910.github.io/mecab/). + +#### Korean analyzer + +Use `"korean"` to select the Korean tokenizer, which is a wrapper of [mecab](http://taku910.github.io/mecab/) but uses a different Korean dictionary. + +#### RAG analyzer + +The RAG analyzer is a bilingual tokenizer that supports Chinese (simplified and traditional) and English. It is a C++ adaptation of [RAGFlow's tokenizer](https://github.com/infiniflow/ragflow/blob/main/rag/nlp/rag_tokenizer.py), and its tokenization of Latin characters derives from [NLTK](https://www.nltk.org/api/nltk.tokenize.punkt.html). + +This analyzer offers better recall for Chinese than the [Jieba](https://github.com/yanyiwu/cppjieba) analyzer, but lower tokenization throughput due to higher computational costs. Its English language processing involves an additional lemmatization step before stemming, different from that of the standard analyzer. + +Use `"rag"` to select the RAG analyzer or `"rag-fine"` for fine-grained mode, which outputs tokenization results with the second highest score. + +:::note +Both RAG tokenization and fine-grained RAG tokenization are used in RAGFlow to ensure high recall. +::: + +#### IK analyzer + +The IK analyzer is a bilingual tokenizer that supports Chinese (simplified and traditional) and English. It is a C++ adaptation of the [IK Analyzer](https://github.com/infinilabs/analysis-ik), which is widely used as a tokenizer by Chinese Elasticsearch users. + +Use `"ik"` to select this analyzer, which is equivalent to the `ik_smart` option in the [IK Analyzer](https://github.com/infinilabs/analysis-ik), or `"ik-fine"` for fine-grained mode, which is equivalent to the `ik_max_word` option in the [IK Analyzer](https://github.com/infinilabs/analysis-ik). + +#### Keyword analyzer + +The keyword analyzer is a "noop" analyzer used for columns containing keywords only, where traditional scoring methods like `BM25` do not apply. It scores `0` or `1`, depending on whether any keywords are matched. + +Use `"keyword"` to select this analyzer. + +### Search and ranking syntax + +Infinity supports the following syntax or full-text search expressions: + +- Single term +- AND multiple terms +- OR multiple terms +- Phrase search +- CARAT operator +- Sloppy phrase search +- Field-specific search +- Escape character + +#### Single term + +Example: `"blooms"` + +#### AND multiple terms + +- `"space AND efficient"` + +#### OR multiple terms + +- `"Bloom OR filter"` +- `"Bloom filter"` + +:::tip NOTE +`OR` is the default semantic in a multi-term full-text search unless explicitly specified otherwise. +::: + +#### Phrase search + +- `"Bloom filter"` +- `'Bloom filter'` + +#### CARAT operator + +Use `^` to boost the importance of a specific term. For example: `quick^2 brown` boosts the importance of `quick` by a factor of 2, making it twice as important as `brown`. + +#### Sloppy phrase search + +Example: `'"harmful chemical"~10'` + +#### Field-specific search + +Example: `"title:(quick OR brown) AND body:foobar"` + +#### Escape character + +Use `\` to escape reserved characters like ` ` `(` `)` `^` `"` `'` `~` `*` `?` `:` `\`. For example: `"space\:efficient"`. + +### Scoring + +Infinity offers `BM25` scoring and block-max `WAND` for dynamic pruning to accelerate multi-term searches. To *not* use `BM25` scoring, do either of the following: + +- Set `"analyzer"` to `"keyword"` when creating index (to select the keyword analyzer). + *The returned score will then be based on whether keywords are matched.* +- Add `{"similarity": "boolean"}` as a search option. + *The scoring will then depend on the number of matched keywords.* + +## Dense vector search + +Infinity's dense vector search is similar to the 'vector search' feature in many other vector databases. It supports two types of vector indexes: HNSW, which offers faster search performance, and IVF, which consumes less memory. + +- Infinity optimizes its implementation of the HNSW index by incorporating locally-adaptive quantization, delivering much higher performance and a much smaller memory footprint than other HNSW implementations. +- Infinity's IVF index supports multiple data types, including `float32`, `float16`, `bfloat16`, `int8` and `uint8`. By implementing scalar quantization and product quantization, Infinity's IVF index achieves a significantly lower memory footprint than its peers. + +For incoming streaming vectors, Infinity executes a brute-force search rather than building index first, ensuring a real-time search experience. The search results are from both the incoming streaming vectors and those indexed. This search strategy is applied autonomously, without the need for human intervention. + +Infinity also has built-in support for multi-vector search. Each row in a table represents a document, comprising a collection of vectors that represent chunks of the document. Infinity builds a vector index on these vectors to retrieve the top K documents (rows). This model is applicable to many scenarios. + +## Sparse vector search + +Infinity supports real-time sparse vector search, which is based on a sparse vector index implemented using [Block-Max Pruning(BMP)](https://arxiv.org/abs/2405.01117). + +## Tensor search + +### Tensor vs. multi-vector + +Tensors work in scenarios where embeddings of all tokens of each document need to be preserved. Tensor is another form of multi-vector, but *not* the multi-vector mentioned in dense vector search. They share similarities but are different in several aspects: + +Multi-vectors are mainly used for dense vector search, while tensors are mainly used for tensor reranking. Tensor search is *not* recommended due to the potential for significant storage overhead when converting raw documents to tensors; tensors are more appropriate for reranking in a [hybrid search](#hybrid-search). + +A multi-vector search remains a KNN search, where search results are aggregated using vector identifiers. In contrast, a tensor search uses 'Colxxx' models, such as ColBERT, ColPali, and ColQwen2. In this context, 'Col' denotes 'contextualized late interaction' and 'xxx' refers to specific models that can be added using a 'Col adapter'. Additionally, a tensor search uses the MaxSim method for similarity operations, which accumulates similarities of all embedding pairs between a query tensor and a document tensor. + +### Tensor array + +Tensor arrays are specifically used with late interaction models with a token limit. For example, the ColBERT model has a token limit of 128, meaning that it cannot generate tensors for documents exceeding this token limit. In such cases, a document is split into multiple chunks, with each chunk represented by a tensor and all tensors for that document stored as a tensor array. During search and reranking, MaxSim scores of all embedding pairs (between a query tensor and a chunk tensor) are calculated and aggregated to produce the final MaxSim score for each document. + +### Performance tuning + +Infinity offers two approaches for improving the performance of a tensor search: + +- Binary quantization: Each float in a tensor is represented by a bit, occupying only 1/32 of the original tensor's space. Infinity supports direct insertion of tensor data resulting from binary quantization. +- Hamming distance: The MaxSim scores of tensor data inserted after binary quantization are calculated using the Hamming distance instead of the inner product, thereby accelerating retrieval. + +## Hybrid search + +Infinity allows you to integrate various search methods, including full-text search, dense vector search, and sparse vector search, into a hybrid search pipeline. For any search combination in a hybrid search, a fusion method is required to determine the final reranking results. The results can be further combined with other search paths. For instance, you can integrate dense vector search with sparse vector search to build the first-stage retrieval, apply a fusion reranker to this path, and integrate the reranking results with a full-text search path to build a second-stage retrieval. + +Infinity offers four types of rerankers for fusion: + +- `rrf` (Reciprocal rank fusion) + RRF is a method that combines multiple result sets with varying relevance indicators into a single result set. This method requires no tuning, and the relevance indicators need not be correlated to achieve high-quality results. It is particularly useful when you are uncertain of each retrieval path's relative importance. +- `weighted_sum` + The weighted sum approach assigns different weights to different retrieval paths, allowing you to emphasize specific paths. It is particularly useful when you are certain of each path's relative importance. It supports `normalize` parameter to normalize the scores before summation. Supported normalization methods are `minmax`, `atan`, `l2`, and `none`. Defaults to `minmax`. +- `match_tensor` + The tensor-based rank fusion, where the final results are determined by the MaxSim operator between the query tensor and each document tensor. +- `max` + The max method returns the maximum score from the different retrieval paths. It supports `normalize` parameter to normalize the scores before comparison. Supported normalization methods are `minmax`, `atan`, `l2`, and `none`. Defaults to `minmax`. + +## Conditional filters + +Conditional filters in Infinity must work through an index to facilitate search. The following two types of indexes in Infinity support conditional filters: + +- **Secondary index**: Built on numeric or string columns. This index does not apply any tokenization to a string column when using conditional filters. +- **Full-text index**: Built on full-text columns. This index applies tokenization to the full-text column but does not trigger any relevance scoring procedure. + +Infinity offers flexible filter syntax. Filters can be applied to either individual search paths, allowing each path to have different filter conditions, or to all search paths, allowing them to share the same filtering conditions. + +### Secondary index filters + +Infinity's secondary index supports real-time construction and search. Optimized for columns with high cardinality, this index can be built and searched rapidly even on numeric columns. + +Filters based on secondary index can have arbitrary logical combinations. Supported expressions include `<`, `<=`, `>`, `>=`, `=`, `==`, `!=`, `in`, and `not in`. + +### Full-text index filters + +Infinity's full-text index supports conditional filtering through the `filter_fulltext` parameter. Full-text index filters are keyword-based and do not support the expressions available for secondary index filters. They use the `minimum_should_match` parameter to specify the minimum number of keywords that must be satisfied during filtering. diff --git a/website/versioned_docs/version-v0.7.0-dev1/guides/set_up_cluster.md b/website/versioned_docs/version-v0.7.0-dev1/guides/set_up_cluster.md new file mode 100644 index 0000000..abfeb8a --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/guides/set_up_cluster.md @@ -0,0 +1,192 @@ +--- +sidebar_position: 2 +slug: /set_up_cluster +--- +# Set up an Infinity cluster + +Architecture overview and user guide for Infinity cluster. + +--- + +## Overview + +An Infinity cluster consists of one leader node, up to four follower nodes, and several learner nodes: + +- **Leader node**: The read node and the only write node. +- **Follower node**: Read node. +- **Learner node**: Read node. + +As of v0.5.0, the supported shared storage is MinIO. + +![infinity_cluster](https://github.com/user-attachments/assets/3e9abeed-1698-4741-8bdb-ba3b05c1d7a3) + +### Architecture + +Infinity employs a distributed architecture comprising one leader node, *N* follower nodes (0 ≤ *N* ≤ 4), and a number of learner nodes. As illustrated in the diagram above, all nodes in the cluster use MinIO for persistent storage. + +- **Leader node**: The node responsible for processing transactions and managing connection status of other nodes in the cluster. When a transaction occurs, the leader node transmits the logs to both follower and learner nodes. The leader node confirms the completion of the transaction only upon receiving messages confirming completion of log persistence from *all* follower nodes. +- **Follower node**: Receives log/WAL from the leader synchronously. It acts as a backup for the leader node, maintaining strong consistency with the leader's data state. +- **Learner node**: Receives log/WAL from the leader *asynchronously*. A learner also serves as a backup for the leader node. However, its state may be behind that of the leader, because it is not required to maintain strong consistency with the leader, and neither does the leader need to confirm whether all learner nodes have completed log persistence. + +From the user's perspective, the leader is the only write node, and all write operations must go through the leader node; all nodes in the cluster serve as read nodes, allowing you to send read operations to any of the leader, follower, or learner nodes, thereby alleviating the write burden on the leader. + +### Startup and communication processes + +When started up as a cluster node (see [Customize configuration files for cluster](#customize-configuration-files-for-cluster)), a node enters `ADMIN` mode, but is not automatically assigned a role like leader, follower, or learner. You must call `ADMIN SET NODE ROLE` to assign it a role. Once a leader node starts, it reads logs from the local disk to determine the metadata and data to read from shared storage. + +Once you set a node to follower or learner using `ADMIN SET NODE ROLE`, it registers with the leader node. Upon receiving the registration request, the leader node sends back its current log for the followers and learners to construct their data state from shared storage. + +### Keep-alive mechanism + +Once successfully registered with the leader node, a follower or learner starts sending periodic heartbeats to it. The leader node relies on these heartbeats to manage the connection status of each node. For example, if it does not receive heartbeats from a particular node for a specified time period, it sets that node's connection status to `timeout`. + +### Log synchronization + +When a transaction occurs, the leader node sends its log to both follower and learner nodes. The leader confirms the transaction's completion only after receiving confirmation that all its follower nodes have successfully persisted the log. While the leader also sends logs to learner nodes, it does not require confirmation from them. + +### Mode and role transition + +![mode_transition](https://github.com/user-attachments/assets/932072a3-9ffb-4aad-89f1-7eef0fff931c) + +## Set up an Infinity cluster + +### Customize configuration files for cluster + +For *each* cluster node, you are required to prepare a customized configuration file to start it. Ensure that you properly set `server_mode`, `peer_ip`, `peer_port`, `storage_type`, and other related parameters. + +1. Set `server_mode` to `"admin"`. +2. Set `storage_type` to `"minio"`. +3. Set `peer_ip` and `peer_port`. +4. Update object storage-specific settings. +5. Save your changes and start up Infinity using the customized configuration file. + *When a cluster node starts, it automatically operates in `ADMIN` mode.* + +For further instructions on specifying a configuration file or setting parameters, see the [Configurations](https://infiniflow.org/docs/dev/configurations). + +### Set the leader node + + A cluster can have only one leader node. If the cluster you start does not have a leader node, call `ADMIN SET NODE ROLE` to promote the node you just started, which currently operates in `ADMIN` mode, to leader. Below is an example code: + +```shell +curl --request POST \ + --url http://localhost:23821/admin/node/current \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "role" : "leader", + "name" : "Harry", + } ' +``` + +*When the method call succeeds, the node switches to leader and operates in `CLUSTER` mode.* + +:::tip NOTE + +A node in `ADMIN` mode with `storage_type = "minio"` or in `CLUSTER` mode (as a follower or learner node) can be promoted to leader. + +::: + +You can also use `ADMIN SHOW CURRENT NODE` to verify the node's role and connection status: + +```shell +curl --request GET \ + --url http://localhost:23821/admin/node/current \ + --header 'accept: application/json' +``` + +### Set a follower node + +If the current node operates in `ADMIN` mode and the number of follower nodes in your cluster is less than four, call `ADMIN SET NODE ROLE` to promote this node to follower node: + +```shell +curl --request POST \ + --url http://localhost:23822/admin/node/current \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "role" : "follower", + "name" : "Hermione", + "address" : "0.0.0.0:23851" + } ' +``` + +*When the method call succeeds, the node is promoted to follower and registered with the leader node, which listens on `0.0.0.0:23851`.* + +:::tip NOTE + +A node in `ADMIN` mode with `storage_type = "minio"` can be promoted to follower node. + +::: + +### Set a learner node + +If the current node operates in `ADMIN` mode, call `ADMIN SET NODE ROLE` to promote this new node to learner node. + +```shell +curl --request POST \ + --url http://localhost:23823/admin/node/current \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "role" : "learner", + "name" : "Ron", + "address" : "0.0.0.0:23851" + } ' +``` + +*When the method call succeeds, the node is promoted to learner and registered with the leader node, which listens on `0.0.0.0:23851`.* + +:::tip NOTE + +Only a node in `ADMIN` mode with `storage_type = "minio"` can be promoted to learner node. + +::: + +### Check cluster health status + +You can send an HTTP request `ADMIN LIST NODES` to any node in the cluster to display the health status of all nodes. In the following code example, a follower node is called: + +```shell +curl --request GET \ + --url http://localhost:23822/admin/nodes \ + --header 'accept: application/json' +``` + +*When the method call succeeds, you get the following information of each node:* + +- *The HTTP address of the node.* +- *The number of heartbeats received from the leader node.* +- *The name of the node.* +- *The role of the node: leader, follower, or learner.* +- *The connection status of the node.* +- *The last time that the node was updated.* + +:::tip NOTE + +See `ADMIN LIST NODES` for further details. + +::: + +### Remove a node from the cluster + +Call `ADMIN REMOVE NODE` to remove a node from the cluster. Note that you must send your HTTP request to the leader node for this action. In the following code example, learner Ron will be removed: + +```shell +curl --request DELETE \ + --url http://localhost:23821/admin/node/ron \ + --header 'accept: application/json' \ + --header 'content-type: application/json' +``` + +*When the method call succeeds, the node operates in `ADMIN` mode and is unregistered.* + +## Distributed APIs + +- [ADMIN SET NODE ROLE](https://infiniflow.org/docs/dev/http_api_reference#admin-set-node-role) +- [ADMIN SHOW NODE VARIABLES](https://infiniflow.org/docs/dev/http_api_reference#admin-show-node-variables) +- [ADMIN SHOW NODE CONFIGS](https://infiniflow.org/docs/dev/http_api_reference#admin-show-node-configs) +- [ADMIN SHOW NODE VARIABLE](https://infiniflow.org/docs/dev/http_api_reference#admin-show-node-variable) +- [ADMIN SHOW CURRENT NODE](https://infiniflow.org/docs/dev/http_api_reference#admin-show-current-node) +- [ADMIN SHOW NODE](https://infiniflow.org/docs/dev/http_api_reference#admin-show-node) +- [ADMIN LIST NODES](https://infiniflow.org/docs/dev/http_api_reference#admin-list-nodes) +- [ADMIN REMOVE NODE](https://infiniflow.org/docs/dev/http_api_reference#admin-remove-node) \ No newline at end of file diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/_category_.json b/website/versioned_docs/version-v0.7.0-dev1/references/_category_.json new file mode 100644 index 0000000..d9b8336 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "References", + "position": 3, + "link": { + "type": "generated-index", + "description": "miscellaneous references" + } +} diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/benchmark.md b/website/versioned_docs/version-v0.7.0-dev1/references/benchmark.md new file mode 100644 index 0000000..5d1764e --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/benchmark.md @@ -0,0 +1,253 @@ +--- +sidebar_position: 3 +slug: /benchmark +--- +# Benchmark + +This document compares the following key specifications of Elasticsearch, Qdrant, Quickwit and Infinity: + +- Time to insert & build index +- Time to import & build index +- query latency +- QPS + +You need to watch resource (persisted index size, peak memory, peak cpu, system load etc.) manually. + +Keep the environment clean to ensure that the database under test is able to use up all resources of the system. + +Avoid running multiple databases at the same time, as each one is a significant resource consumer. + +Test environment: + +- OS: OpenSUSE Tumbleweed x86_64 +- CPU: Intel CORE i5-13500H 16vCPU +- RAM: 32GB +- Disk: 1TB + +## Versions +| | Version | +| ----------------- |---------| +| **Elasticsearch** | v8.13.4 | +| **Qdrant** | v1.9.2 | +| **Quickwit** | v0.8.1 | +| **Infinity** | v0.2.0 | + +## Run Benchmark + +1. Install necessary dependencies. + +```bash +uv sync --all-extras + + +``` + +2. Download the required Benchmark datasets to your **/datasets** folder： + - [SIFT1M](http://ann-benchmarks.com/sift-128-euclidean.hdf5) + - [GIST1M](http://ann-benchmarks.com/gist-960-euclidean.hdf5) + - [Enwiki](https://home.apache.org/~mikemccand/enwiki-20120502-lines-1k.txt.lzma) + +Preprocess dataset: + +```bash +sed '1d' datasets/enwiki/enwiki-20120502-lines-1k.txt > datasets/enwiki/enwiki.csv +``` + +3. Start up the databases to compare: + +```bash +mkdir -p $HOME/elasticsearch/data +docker run -d --name elasticsearch --network host -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms16384m -Xmx32000m" -e "xpack.security.enabled=false" -v $HOME/elasticsearch/data:/usr/share/elasticsearch/data elasticsearch:8.13.4 + +mkdir -p $HOME/qdrant/storage +docker run -d --name qdrant --network host -v $HOME/qdrant/storage:/qdrant/storage qdrant/qdrant:v1.9.2 + +mkdir -p $HOME/quickwit +docker run -d --name quickwit --network=host -v $HOME/quickwit/qwdata:/quickwit/qwdata quickwit/quickwit:0.8.1 run + +mkdir -p $HOME/infinity +docker run -d --name infinity --network=host -v $HOME/infinity:/var/infinity --ulimit nofile=500000:500000 infiniflow/infinity:nightly +``` + +4. Run Benchmark: + +Drop file cache before benchmark. + +```bash +echo 3 | sudo tee /proc/sys/vm/drop_caches +``` + +Tasks of the Python script `run.py` include: + - Generate fulltext query set. + - Measure the time to import data and build index. + - Measure the query latency. + - Measure the QPS. + +```bash +$ python run.py -h +usage: run.py [-h] [--generate] [--import] [--query QUERY] [--query-express QUERY_EXPRESS] [--concurrency CONCURRENCY] [--engine ENGINE] [--dataset DATASET] + +RAG Database Benchmark + +options: + -h, --help show this help message and exit + --generate Generate fulltext query set based on the dataset (default: False) + --import Import dataset into database engine (default: False) + --query QUERY Run the query set only once using given number of clients with recording the result and latency. This is for result validation and latency analysis (default: 0) + --query-express QUERY_EXPRESS + Run the query set randomly using given number of clients without recording the result and latency. This is for QPS measurement. (default: 0) + --concurrency CONCURRENCY + Choose concurrency mechanism, one of: mp - multiprocessing(recommended), mt - multithreading. (default: mp) + --engine ENGINE Choose database engine to benchmark, one of: infinity, qdrant, elasticsearch, quickwit (default: infinity) + --dataset DATASET Choose dataset to benchmark, one of: gist, sift, geonames, enwiki, tantivy (default: enwiki) +``` + +Following are commands for engine `infinity` and dataset `enwiki`: + +```bash +python run.py --generate --engine infinity --dataset enwiki +python run.py --import --engine infinity --dataset enwiki +python run.py --query=16 --engine infinity --dataset enwiki +python run.py --query-express=16 --engine infinity --dataset enwiki +``` + +Following are commands to issue a single query so that you can compare results among several engines. + +```base +curl -X GET "http://localhost:9200/elasticsearch_enwiki/_search" -H 'Content-Type: application/json' -d'{"size":10,"_source":"doctitle","query": {"match": { "body": "wraysbury istorijos" }}}' + +curl -X GET "http://localhost:7280/api/v1/_elastic/qucikwit_enwiki/_search" -H 'Content-Type: application/json' -d'{"query": {"query_string": {"query": "wraysbury istorijos", "fields": [ "body" ] } },"sort": ["_score"],"size":10}' + +psql -h 0.0.0.0 -p 5432 -c "SELECT doctitle, ROW_ID(), SCORE() FROM infinity_enwiki SEARCH MATCH TEXT ('body', 'wraysbury istorijos', 'topn=10');" +``` + +## Benchmark Results +### SIFT1M + +> - Metric: L2 +> - 10000 queries + +| | QPS | Recall | Time to insert & build index | Time to import & build index | Disk | Peak memory | +| ----------------- | ----- | -------------- | ---------------------------- | ---------------------------- | ------ | ----------- | +| **Elasticsearch** | 934 | 0.992 | 131 s | N/A | 874 MB | 1.463 GB | +| **Qdrant** | 1303 | 0.979 | 46 s | N/A | 418 MB | 1.6 GB | +| **Infinity** | 16320 | 0.973 | 74 s | 28 s | 792 MB | 0.95 GB | + + + +### GIST1M + +> - Metric: L2 +> - 1000 queries + +| | QPS | Recall | Time to insert & build index | Time to import & build index | Disk | Peak memory | +| ----------------- | ---- | -------------- | ---------------------------- | ---------------------------- | ------ | ----------- | +| **Elasticsearch** | 305 | 0.885 | 872 s | N/A | 13 GB | 6.9 GB | +| **Qdrant** | 339 | 0.947 | 366 s | N/A | 4.4 GB | 7.3 GB | +| **Infinity** | 2200 | 0.946 | 463 s | 112 s | 4.7 GB | 6.0 GB | + + + +### Enwiki + +> - 33000000 documents +> - 100000 `OR` queries generated based on the dataset. All terms are extracted from the dataset and very rare(occurrence < 100) terms are excluded. The number of terms of each query match the weight `[0.03, 0.15, 0.25, 0.25, 0.15, 0.08, 0.04, 0.03, 0.02]`. + +| | Time to insert & build index | Time to import & build index | P95 Latency(ms)| QPS (16 python clients) | Memory | vCPU | +| ----------------- | ---------------------------- | ---------------------------- | ---------------| ------------------------| --------| ----- | +| **Elasticsearch** | 2289 s | N/A | 14.75 | 1340 | 21.0GB | 10.6 | +| **Quickwit** | 3962 s | N/A | 65.55 | 179 | 1.2GB | 11.3 | +| **Infinity** | 1562 s | 2244 s | 1.37 | 13731 | 10.0GB | 11.0 | + +--- + +## Deprecated Benchmark + +Infinity provides a Python script for benchmarking the SIFT1M and GIST1M datasets. + +### Build and start Infinity + +You have two options for building Infinity. Choose the option that best fits your needs: + +- [Build Infinity using Docker](https://github.com/infiniflow/infinity/blob/main/README.md) +- [Build from source](../getstarted/build_from_source.mdx) + +### Download the Benchmark datasets + +To obtain the benchmark datasets, you have the option to download them using the wget command. + +```sh +#download sift benchmark +wget ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz +#download gist benchmark +wget ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz + +``` + +Alternatively, you can manually download the benchmark datasets by visiting [http://corpus-texmex.irisa.fr/](http://corpus-texmex.irisa.fr/). + +```sh +# Unzip and move the SIFT1M benchmark file. +tar -zxvf sift.tar.gz +mv sift/sift_base.fvecs test/data/benchmark/sift_1m/sift_base.fvecs +mv sift/sift_query.fvecs test/data/benchmark/sift_1m/sift_query.fvecs +mv sift/sift_groundtruth.ivecs test/data/benchmark/sift_1m/sift_groundtruth.ivecs + +# Unzip and move the GIST1M benchmark file. +tar -zxvf gist.tar.gz +mv gist/gist_base.fvecs test/data/benchmark/gist_1m/gist_base.fvecs +mv gist/gist_query.fvecs test/data/benchmark/gist_1m/gist_query.fvecs +mv gist/gist_groundtruth.ivecs test/data/benchmark/gist_1m/gist_groundtruth.ivecs + +``` + +### Benchmark dependencies + +```sh +uv sync --all-extras + + +``` + +### Import the Benchmark datasets + +```sh +cd benchmark + +# options: +# -h, --help show this help message and exit +# -d DATA_SET, --data DATA_SET + +python remote_benchmark_knn_import.py -d sift_1m +python remote_benchmark_knn_import.py -d gist_1m +``` + +### Run Benchmark + +```sh +# options: +# -h, --help show this help message and exit +# -t THREADS, --threads THREADS +# -r ROUNDS, --rounds ROUNDS +# -d DATA_SET, --data DATA_SET + +# ROUNDS indicates the number of times Python executes the benchmark, and the result represents the average duration for each run. + +# Perform a latency benchmark on the SIFT1M dataset using a single thread, running it only once. +python remote_benchmark_knn.py -t 1 -r 1 -d sift_1m +# Perform a latency benchmark on the GIST1M dataset using a single thread, running it only once. +python remote_benchmark_knn.py -t 1 -r 1 -d gist_1m + +# Perform a QPS benchmark on the SIFT1M dataset using a single thread, running it only once. +python remote_benchmark_knn.py -t 16 -r 1 -d sift_1m +# Perform a latency benchmark on the GIST1M dataset using a single thread, running it only once. +python remote_benchmark_knn.py -t 16 -r 1 -d gist_1m +``` +### A SIFT1M Benchmark report + +- **Hardware**: Intel i5-12500H, 16C, 16GB +- **Operating system**: Ubuntu 22.04 +- **Dataset**: SIFT1M; **topk**: 100; **recall**: 97%+ +- **P99 QPS**: 15,688 (16 clients) +- **P99 Latency**: 0.36 ms +- **Memory usage**: 408 MB diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/configurations.mdx b/website/versioned_docs/version-v0.7.0-dev1/references/configurations.mdx new file mode 100644 index 0000000..44cf754 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/configurations.mdx @@ -0,0 +1,217 @@ +--- +sidebar_position: 0 +slug: /configurations +--- + +# Configurations +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +How to set and load configuration file when starting Infinity. + +--- + +This document provides instructions for loading configuration file for Infinity and descriptions of each configuration entry. + + +## Load configuration file + + + + + +To load a configuration file when you start up Infinity using Docker: + +1. Mount the directory for the configuration file to inside the Docker container. +2. When running Docker, specify the configuration file path using the `-f` option. For example: + + ```bash + docker run -d --name infinity -v /var/infinity/:/var/infinity --ulimit nofile=500000:500000 --network=host infiniflow/infinity:nightly -f /var/infinity/infinity_conf.toml + ``` + + + + +If you start up Infinity from DEB or RPM, the configuration file will be in the `/usr/etc` directory. When the Infinity service starts, it will load this configuration file. + + + + +## A configuration file example + + +```toml +[general] +# Infinity server version, the minor version of this parameter should match server minor version. +version = "0.7.0" +# Timezone +time_zone = "utc-8" +# The number of worker threads. Defaults to the number of CPU cores. +# Range: [1, 16384] +cpu_limit = 8 +# The mode in which the server starts. Available options: +# - `"standalone"`: Start Infinity as a standalone server. +# - `"admin"`: +# - Start Infinity either as a standalone server in `ADMIN` mode (when `storage_type` is set to `"local"`) +# - Start Infinity as a cluster node in `ADMIN` mode (when `storage_type` is set to `"minio"`) +server_mode = "standalone" + +# Network configuration +[network] +# IP address of the Infinity server +server_address = "0.0.0.0" + +# Port of the Postgres server +postgres_port = 5432 +# Port of the HTTP server +http_port = 23820 +# Port of the Python SDK server +client_port = 23817 +# The maximum number of connections. Defaults to 256. +# Range: [1, 65536] +connection_pool_size = 128 +# The IP address on which the current node listens. Used for registration and inter-node communication +peer_ip = "0.0.0.0" +# The port number on which the current node listens. Used for registration and inter-node communication +peer_port = 23850 + +# The delay time for reconnecting to the Infinity peer server after a failed connection +peer_retry_delay = 0 +# The maximum number of reconnection attempts to the Infinity peer server after a failed connection +peer_retry_count = 0 +# Timeout for connecting to the Infinity peer server +peer_connect_timeout = 2000 +# Timeout for the Infinity peer server to receive data +peer_recv_timeout = 0 +# Timeout for the Infinity peer server to send data +peer_send_timeout = 0 + +# Log configuration +[log] +# Filename of the log files. +log_filename = "infinity.log" +# Directory for the log files. +log_dir = "/var/infinity/log" +# Controls whether to print the log to the standard output. Defaults to true +# Range: {true|false} +log_to_stdout = true +# The maximum size of each log file. Defaults to "1GB". +# Range: at least 1GB +log_file_max_size = "10GB" +# The number of rotating log files. Defaults to 8 +# Range: [1, 655636] +log_file_rotate_count = 10 +# The log output level. Defaults to "info". +# Range: {"trace"/"debug"/"info"/"warning"/"error"/"critical"} +log_level = "info" + +# Storage configuration +[storage] +# The working directory of the persistence manager: +# - Holds the data that Infinity saves when using the local disk for storage. +# - Caches the accessed object files when using MinIO/S3 for object storage. +persistence_dir = "/var/infinity/persistence" + +# Garbage collection period +# Defined as "number + unit of time": +# - s for seconds (e.g., "60s" for 60 seconds), +# - m for minutes (e.g., "60m" for 60 minutes), +# - h for hours (e.g., "1h" for one hour). +# To disable a specific periodic task, set the corresponding variable to "0s" + +# Interval between index optimization tasks +# Range: ["10s", "720h"] +optimize_interval = "10s" +# Interval between cleanup tasks +# Range: ["10s", "720h"] +cleanup_interval = "60s" +# Interval between compaction tasks +# Range: ["10s", "720h"] +compact_interval = "120s" +# The threshold for flushing an in-memory index to disk. +# When the number of rows in the index exceeds this threshold, +# the system performs a flush operation on that index. +# Range: [8192, 8388608] +mem_index_capacity = 65536 +# The type of storage to use. Available options: +# - `"local"`: (default) +# - `"minio"`: If you set `server_mode` to `"admin"` and `storage_type` to `"minio"`, the node will start as a cluster node in `ADMIN` mode. +# Range: {"local"|"minio"} +storage_type = "local" + +# The number of dense vector index building worker threads. Defaults to the half number of CPU cores. +# Range: [1, number of CPU cores] +dense_index_building_worker = 2 + +# The number of sparse vector index building worker threads. Defaults to the half number of CPU cores. +# Range: [1, number of CPU cores] +sparse_index_building_worker = 2 + +# The number of fulltext index building worker threads. Defaults to the half number of CPU cores. +# Range: [1, number of CPU cores] +fulltext_index_building_worker = 2 + +# Object storage configuration +[storage.object_storage] +# URL of the object storage server +url = "127.0.0.1:9000" +# The name of bucket where the data is stored +bucket_name = "infinity" +# Access key of object storage server +access_key = "minioadmin" +# Secret key of object storage server +secret_key = "minioadmin" +# Whether to enable HTTP or HTTPS protocol +enable_https = false + +# Buffer manager configuration +[buffer] +# The amount of memory occupied by the buffer manager. Defaults to "8GB". +buffer_manager_size = "4GB" +# The number of LRU caches in the buffer manager. +# Range: [1, 100] +lru_num = 7 +# When the required memory size exceeds the free memory in the buffer manager, +# the buffer manager dumps some of the its in-use memory to a temporary storage. +# `temp_dir` specifies the path to this temporary storage. +temp_dir = "/var/infinity/tmp" +# The global memory index threshold for triggering a flush to disk +# When the memory used by all existing in-memory indices in the system exceeds this threshold, +# the system will perform a flush operation on all in-memory indices. +memindex_memory_quota = "1GB" + +# If cache the query result. +# If same query is sent to Infinity, Infinity will check and return the cached result. +result_cache = "on" + +# Cache query capacity +cache_result_capacity = 100 + +# WAL configuration +[wal] +# The directory containing the WAL files +wal_dir = "/var/infinity/wal" +# Interval between checkpoints +# Range: ["0s", "720h"] +checkpoint_interval = "86400s" + +# Size threshold for triggering a compaction on a WAL file +# When the size of a WAL file exceeds this threshold, the system will perform compaction. +# Range: ["1KB", "1024GB"] +wal_compact_threshold = "1GB" + +# Controls how the WAL manager flushes its logs to disk +# - "only_write": (Default) Writes logs but leaves it to the OS to determine when to flush them. +# - "flush_at_once": Writes and flushes logs upon each commit. +# - "flush_per_second": Writes logs after each commit and flushes them to disk once per second. +wal_flush = "only_write" + +[resource] +# Directory for Infinity's resource files, including the dictionary files used by the analyzer +resource_dir = "/usr/share/infinity/resource" +``` \ No newline at end of file diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/faq.md b/website/versioned_docs/version-v0.7.0-dev1/references/faq.md new file mode 100644 index 0000000..4fff176 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/faq.md @@ -0,0 +1,26 @@ +--- +sidebar_position: 4 +slug: /FAQ +--- + +# Frequently asked questions + +FAQs to be developed. + +## What is Retrieval-Augmented Generation? + +Retrieval Augmented Generation (RAG) is a technique used to improve the accuracy and reliability of responses from foundation models, specifically Large Language Models (LLMs). It works by supplementing the existing LLMs with external sources of knowledge. + +The benefits of using RAG include enhanced response quality and relevance, access to the most current and reliable facts, and the ability to verify the model's responses. It reduces the need for continuous training and updating of the LLM, thereby lowering costs. RAG relies on the use of vectors, which are mathematical representations of data, to enrich prompts with relevant external information. + +RAG enables the retrieval of external data from a variety of sources, including document repositories, databases, and APIs. This data is then converted into a compatible format to facilitate relevancy searches. The retrieved information is added to the original user prompt, empowering the LLM to provide responses based on more relevant or up-to-date knowledge. + +## What is an AI-native database? Is it just a paraphrase of vector database? + +An AI-native database is designed specifically to address the challenges of retrieval-augmented generation (RAG), which is currently an industry standard for enhancing the accuracy and relevance of responses generated by foundation models. + +In addition to basic vector search, an AI-vector database also offers advanced capabilities such as more full-text search, multi-vector retrieval, mixed data type query, refined data analytics, and hybrid search. + +## Where can I find a benchmark report of your database? + +You can find a benchmark report on Infinity, the AI-native database, [here](../references/benchmark.md). diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/http_api_reference.mdx b/website/versioned_docs/version-v0.7.0-dev1/references/http_api_reference.mdx new file mode 100644 index 0000000..8884acf --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/http_api_reference.mdx @@ -0,0 +1,4696 @@ +--- +sidebar_position: 1 +slug: /http_api_reference +--- + +# HTTP API Reference +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +A complete reference for Infinity's Python APIs. + +--- + +## SCHEMA MANAGEMENT + +--- + +### Create database + +**POST** `/databases/{database_name}` + +Creates a database by its name. If the database already exists, the action taken depends on the `"create_option"` parameter. + +#### Request + +- Method: POST +- URL: `/database/{database_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"create_option"`: `enum` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"create_option": "ignore_if_exists"} ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. +- `"create_option"`: (*Body parameter*), `enum`, *Optional* + - `"error"`: (Default) Raise an error if a database with the same name exists. + - `"ignore_if_exists"`: Ignore the database creation request and keep the existing database. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3016, + "error_msg": "Duplicate database: {database_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Drop database + +**DELETE** `/databases/{database_name}` + +Deletes a database by its name. If the database does not exist, the action taken depends on the `"drop_option"` parameter. + +#### Request + +- Method: DELETE +- URL: `/database/{database_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"drop_option"`: `enum` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23820/databases/{database_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"drop_option": "error"} ' +``` + +##### Request parameter + +- `database_name`: (*Path parameter*) + The name of the database to delete. +- `"drop_option"`: (*Body parameter*), `enum`, *Optional* + - `"error"`: (Default) Raise an error if the specified database does not exist. + - `"ignore_if_not_exists"`: Ignore the operation and proceed regardless, if the specified database does not exist. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3021, + "error_msg": "{database_name} doesn't exist." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### List databases + +**GET** `/databases` + +Retrieves a list of all available databases within the Infinity system. + +#### Request + +- Method: GET +- URL: `/databases` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "databases": [ + "default_db", + "my_db" + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `databases`: `string[]` + An array of strings representing the names of the databases in the system. + +--- + +### Show database + +**GET** `/databases/{database_name}` + +Shows detailed information about a specified database. + +#### Request + +- Method: GET +- URL: `/database/{database_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name} \ + --header 'accept: application/json' +``` + +##### Request parameter + +- `database_name`: (*Path parameter*) + The name of the database to retrieve. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "database_name": "default_db" + "storage_directory": "/var/infinity/data/nIHniKeHIB_db_default" + "table_count": "4" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `database_name`: `string` + The name of the database. +- `storage_directory`: `string` + The directory path where the database is stored. +- `table_count`: `integer` + The number of tables present in the database. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3021, + "error_msg": "{database_name} doesn't exist." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Create table + +**POST** `/databases/{database_name}/tables/{table_name}` + +Creates a table with a specified name and defined fields (columns) within a given database. If the table already exists, the action taken depends on the `"create_option"` parameter. + +#### Request + +- Method: POST +- URL: `/databases/{database_name}/tables/{table_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"create_option"`: `enum` + - `"fields"`: `object[]` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "create_option": "ignore_if_exists", + "fields": [ + { + "name": "name", + "type": "varchar", + "comment": "name column" + }, + { + "name": "score", + "type": "float", + "default": 3.0 + }, + { + "name": "dense_column", + "type": "vector,8,float", + "default": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + }, + { + "name": "fulltext_column", + "type": "varchar", + "default": "" + }, + { + "name": "sparse_column", + "type": "sparse,128,float,int", + "default": {"10":1.1, "20":2.2, "30": 3.3} + }, + { + "name": "tensor_column", + "type": "tensor,4,float", + "default": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]] + }, + { + "name": "multivector_column", + "type": "multivector,4,float", + "default": [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] + }, + { + "name": "tensorarray_column", + "type": "tensorarray,2,float", + "default": [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0]]] + } + ] + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table to create, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. +- `"create_option"`: (*Body parameter*), `enum`, *Optional* + - `"error"`: (Default) Raise an error if a table with the same name exists. + - `"ignore_if_exists"`: Ignore the table creation request and keep the existing table. +- `field`: (*Body parameter*), `object[]`, *Required* + - `"name"`: `string`, *Required* + A non-empty string indicating the name of the column to create, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. + - `"type"`: `string`, *Required* + The data type of the column. + - Numeric: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - String: `"varchar"` + - Dense vector: e.g., `"vector,128,float"` + - `vector`: The column is a dense vector column. + - The second item in the string: The dimension of dense vectors. + - The third item in the string: The element type of dense vectors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Sparse vector: e.g., `"sparse,128,float,int"` + - `sparse`: The column is a sparse vector column. + - The second item in the string: The dimension of sparse vectors. + - The third item in the string: The element type of sparse vectors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - The fourth item in the string: The data type of the sparse vector indices. Can be: + - `int8` + - `int16` + - `int`/`int32`/`integer` + - `int64` + - Tensor vector: e.g., `"tensor,4,float"` + - `tensor`: The column is a tensor column. + - The second item in the string: The dimension of each vector unit in a tensor. + - The third item in the string: The element type of the tensors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Tensor array: e.g., `"tensorarray,6,float"` + - `tensorarray`: The column is a tensor-array column. + - The second item in the string: The dimension of each vector unit in a tensor array. + - The third item in the string: The element type of tensor arrays. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Multivector: e.g., `"multivector,128,float"` + - `multivector`: The column is a multi-vector column. + - The second item in the string: The dimension of each vector unit in a multi-vector. + - The third item in the string: The element type of multi-vectors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Array: e.g., `"array,varchar"`, `"array,array,varchar"` + - `array`: The column is an array column. + - followed by the element type of the array. Can be recursive. + - for example, `"array,varchar"` is a one-dimensional array of strings, and `"array,array,varchar"` is a two-dimensional array of strings. + - `"default"`: `Any`, *Optional* + The default value for unspecified cells in that column. + - `"comment"`: `string`, *Optional* + User provided text to describe the column. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3017, + "error_msg": "Duplicate table: {table_name} in {database_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Drop table + +**DELETE** `/databases/{database_name}/tables/{table_name}` + +Deletes a table from a specified database. If the table does not exist, the action taken depends on the `"drop_option"` parameter. + +#### Request + +- Method: DELETE +- URL: `/databases/{database_name}/tables/{table_name}` +- Headers: +- `accept: application/json` +- `content-Type: application/json` +- Body: +- `"drop_option"`: `enum` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"drop_option": "ignore_if_not_exists"} ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) +The name of the database. +- `table_name`: (*Path parameter*) +The name of the table to delete. +- `"drop_option"`: (*Body parameter*), `enum`, *Optional* +- `"error"`: (Default) Raise an error if the specified table does not exist. +- `"ignore_if_not_exists"`: Ignore the operation and proceed regardless, if the specified database does not exist. + +#### Response + + + + + The response includes a JSON object like the following: + + ```shell + { + "error_code": 0 + } + ``` + + - `"error_code"`: `integer` + `0`: The operation succeeds. + + + + + The response includes a JSON object like the following: + + ```shell + { + "error_code": 3022, + "error_msg": "Table {table_name} doesn't exist in {database_name}." + } + ``` + + - `"error_code"`: `integer` + A non-zero value indicates a specific error condition. + - `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### List tables + +**GET** `/databases/{database_name}/tables` + +Retrieves a list of all available tables in a specified database. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) +The name of the database. + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "tables": + [ + "table_1", + "table_2", + "table_n" + ] +} +``` + +- `"error_code"`: `integer` +`0`: The operation succeeds. + +--- + +### create table snapshot + +**POST** `/snapshots` + +Creates a snapshot of the specified table with the given snapshot name. + +#### Request + +- Method: POST +- URL: `/snapshots` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"db_name"`: `string` + - `"table_name"`: `string` + - `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "db_name" : "default_db", + "table_name" : "my_table", + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `db_name`: (*Body parameter*) + The name of the database. +- `table_name`: (*Body parameter*) + The name of the table. +- `snapshot_name`: (*Body parameter*) + The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3022, + "error_msg": "{table_name} doesn't exist." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### create database snapshot + +**POST** `/snapshots/database` + +Creates a snapshot of the specified database with the given snapshot name. + +#### Request + +- Method: POST +- URL: `/snapshots/database` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"db_name"`: `string` + - `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots/database \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "db_name" : "default_db", + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `db_name`: (*Body parameter*) + The name of the database. +- `snapshot_name`: (*Body parameter*) + The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3021, + "error_msg": "{db_name} doesn't exist." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### create system snapshot + +**POST** `/snapshots/system` + +Creates a snapshot of the system with the given snapshot name. + +#### Request + +- Method: POST +- URL: `/snapshots/system` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots/system \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `snapshot_name`: (*Body parameter*) + The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3100, + "error_msg": "Snapshot: {snapshot_name} already exists." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### restore table snapshot + +**POST** `/snapshots/restore` + +Restores a table from the specified snapshot. + +#### Request + +- Method: POST +- URL: `/snapshots/restore` +- Headers: +- `accept: application/json` +- `content-Type: application/json` +- Body: +- `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots/restore \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `snapshot_name`: (*Body parameter*) +The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 7008, + "error_msg": "File: /var/infinity/snapshot/{snapshot_name}/{snapshot_name}.json isn't found." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### restore database snapshot + +**POST** `/snapshots/database/restore` + +Restores a database from the specified snapshot. + +#### Request + +- Method: POST +- URL: `/snapshots/database/restore` +- Headers: +- `accept: application/json` +- `content-Type: application/json` +- Body: +- `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots/database/restore \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `snapshot_name`: (*Body parameter*) +The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 7008, + "error_msg": "File: /var/infinity/snapshot/{snapshot_name}/{snapshot_name}.json isn't found." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### restore system snapshot + +**POST** `/snapshots/system/restore` + +Restores the system from the specified snapshot. + +#### Request + +- Method: POST +- URL: `/snapshots/system/restore` +- Headers: +- `accept: application/json` +- `content-Type: application/json` +- Body: +- `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/snapshots/system/restore \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "snapshot_name" : "my_snapshot" + } ' +``` + +##### Request parameters +- `snapshot_name`: (*Body parameter*) +The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 7008, + "error_msg": "File: /var/infinity/snapshot/{snapshot_name}/{snapshot_name}.json isn't found." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### drop snapshot + +**DELETE** `/snapshots/{snapshot_name}` + +Deletes a snapshot with the specified name. + +#### Request + +- Method: POST +- URL: `/snapshots/{snapshot_name}` +- Headers: +- `accept: application/json` +- Body: +- `"snapshot_name"`: `string` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23820/snapshots/{snapshot_name} \ + --header 'accept: application/json' +``` + +##### Request parameters +- `snapshot_name`: (*Path parameter*) +The name of the snapshot. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3088, + "error_msg": "Snapshot: {snapshot_name} not found." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show table + +**GET** `/databases/{database_name}/tables/{table_name}` + +Shows detailed information about a specified table within a given database. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "database_name": "default_db", + "table_name": "my_table", + "storage_directory": "/var/infinity/data/nIHniKeHIB_db_default/h1abZcWuBs_table_my_table", + "column_count" : 3, + "segment_count" : 1, + "row_count" : 5 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 3022, + "error_msg": "Table {table_name} doesn't exist in {database_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show table columns + +**GET** `/databases/{database_name}/tables/{table_name}/columns` + +Shows the column information about a specified table. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}/columns` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/columns \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "columns": + [ + { + "column_name": "name", + "column_type": "Varchar", + "constraints": "", + "default": "Null" + }, + { + "column_name": "score", + "column_type": "Float", + "constraints": "", + "default": "Null", + } + { + "column_name": "vector_column", + "column_type": "Embedding(float,8)", + "constraints": "", + "default": "Null" + }, + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3022, + "error_msg": "Table {table_name} doesn't exist in {database_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Create index + +**POST** `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` + +Creates an index on a specified table. If an index with the same name exists, the action taken depends on the `"create_option"` parameter. + +#### Request + +- Method: POST +- URL: `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"fields"`: `string[]` + - `"index"`: `object` + - `"create_option"`: `enum` + +##### Request example + +- Creates an HNSW index on a dense vector or a multivector column: + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "fields": + [ + "dense_column" + ], + "index": + { + "type": "Hnsw", + "M": "16", + "ef_construction": "50", + "metric": "l2" + }, + "create_option": "ignore_if_exists" + } ' +``` + +- Creates a full-text index on a full-text column: + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "fields": + [ + "fulltext_column" + ], + "index": + { + "type": "fulltext", + "analyzer": "chinese" + }, + "create_option": "ignore_if_exists" + } ' +``` + +- Creates a BMP index on a sparse column + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "fields": + [ + "sparse_column" + ], + "index": + { + "type": "BMP", + "block_size": "16", + "compress_type": "raw" + }, + "create_option": "ignore_if_exists" + } ' +``` + +- Creates a secondary index on a varchar column + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "fields": + [ + "varchar_column" + ], + "index": + { + "type": "Secondary" + }, + "create_option": "ignore_if_exists" + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `index_name`: (*Path parameter*) + The name of the index to create, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. +- `"fields"`: (*Body parameter*), `string[]`, *Required* + A non-empty list of strings indicating the names of the columns to build index on. For now, you are only allowed to create an index on one column. +- `"index"`: (*Body parameter*), `dict[string, string]` + - Parameter settings for an HNSW index: + - `"type"`: `"hnsw"` + - `"M"`: *Optional* - Defaults to`"16"`. + - `"ef_construction"`: *Optional* - Defaults to`"50"`. + - `"metric"` *Required* - The distance metric to use in similarity search. + - `"ip"`: Inner product. + - `"l2"`: Euclidean distance. + - `"cosine"`: Cosine similarity. + - `"encode"`: *Optional* + - `"plain"`: (Default) Plain encoding. + - `"lvq"`: Locally-adaptive vector quantization. Works with float vector element only. + - Parameter settings for an IVF index: + - `"metric"` *Required* - The distance metric to use in similarity search. + - `"ip"`: Inner product. + - `"l2"`: Euclidean distance. + - `"cosine"`: Cosine similarity. + - `"storage_type"`: *Optional* + - `"plain"`: (Default) Plain storage. + - `"scalar_quantization"`: Scalar quantization. + - `"product_quantization"`: Product quantization. + - `"plain_storage_data_type"`: *Optional* for plain storage. + - `"int8"`: default value for `int8` embeddings. + - `"uint8"`: default value for `uint8` embeddings. + - `"float32"`: default value for floating-point embeddings. + - `"float16"`: for floating-point embeddings. + - `"bfloat16"`: for floating-point embeddings. + - `"scalar_quantization_bits"`: *Required* for scalar quantization. Must be either `4` or `8`. + - `"product_quantization_subspace_num"`: *Required* for product quantization. Must be divisor of the embedding dimension. + - `"product_quantization_subspace_bits"`: *Required* for product quantization. Must be in the range `[4, 16]`. + - Parameter settings for a full-text index: + - `"type"`: `"fulltext"` + - `"ANALYZER"`: *Optional* + - `"standard"`: (Default) Standard analyzer, segmented by tokens, lowercase processing, provides stemming outputs. Use `-` to specify stemmer for languages, `English` is the default stemmer: `"standard-english"` and `"standard"` have the same stemmer setting. Supported language stemmer includes: `Danish`, `Dutch`, `English`, `Finnish`, `French`, `German`, `Hungarian`, `Italian`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`,`Russian`,`Spanish`,`Swedish`,`Turkish`. + - `"rag"`: Multilingual RAG analyzer imported from [RAGFlow](https://github.com/infiniflow/ragflow/blob/main/rag/nlp/rag_tokenizer.py), supporting `Chinese` and `English`. Use `-fine` to output the fine-grained analyzer results. + - `"chinese"`: Simplified Chinese. Use `-fine` to output the fine-grained analyzer results. + - `"ik"`: Bilingual analyzer imported from [ik-analyzer](https://github.com/infinilabs/analysis-ik), supporting `Chinese` and `English`. Use `-fine` to output the fine-grained analyzer results. + - `"traditional"`: Traditional Chinese. + - `"japanese"`: Japanese. + - `"korean"`: Korean. + - `"ngram"`: [N-gram](https://en.wikipedia.org/wiki/N-gram). + - `"keyword"`: "noop" analyzer used for columns containing keywords only. + - Parameter settings for a secondary index: + - `"type"`: `"secondary"` + - Parameter settings for a BMP index: + - `"type"`: `"bmp"` + - `block_size`: *Optional* - The size of the block in a BMP index. Range: `"1"` ~ `"256"`. Defaults to `"16"`. + - `"compress_type"`: *Optional* + - `"compress"`: (Default) Store the block-max index in sparse format. Works best with small block size situations. + - `"raw"`: Store the block-max index without compression. +- `"create_option"`: (*Body parameter*), `enum`, *Optional* + - `"error"`: (Default) Raise an error if a table with the same name exists. + - `"ignore_if_exists"`: Ignore the table creation request and keep the existing table. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3018, + "error_msg": "Duplicate index: {index} in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Drop index + +**DELETE** `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` + +Deletes an index by its name. + +#### Request + +- Method: DELETE +- URL: `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"drop_option"`: `enum` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"drop_option": "ignore_if_not_exists"} ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `index_name`: (*Path parameter*) + The name of the index to delete. +- `"drop_option"`: (*Body parameter*), `enum`, *Optional* + - `"error"`: (Default) Raise an error if the specified index does not exist. + - `"ignore_if_not_exists"`: Ignore the operation and proceed regardless, if the specified index does not exist. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3018, + "error_msg": "Index {index_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Alter index + +**PUT** `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` + +Alter index parameters on a specified index of table. + +#### Request + +- Method: PUT +- URL: `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `parameter_name`: `string` + +##### Request example + +- Alter a BMP index + +```shell +curl --request PUT \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data '{ "topk": "3", "bp_reorder": "" }' +``` + +##### Request parameters + +- `database_name`: Required (*Path parameter*) + The name of the database. +- `table_name`: Required (*Path parameter*) + The name of the table. +- `index_name`: Required (*Path parameter*) + The name of the index. +- `json object containing index parameter key-value pairs`: Required (*Body parameter*) + A list of index parameter key-value pairs to alter + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3018, + "error_msg": "Duplicate index: {index} in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### List indexes + +**GET** `/databases/{database_name}/tables/{table_name}/indexes` + +Retrieves a list of all indexes created on a given table. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}/indexes` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "indexes": + [ + { + "columns": "vector_column_1", + "index_name": "idx1", + "index_type": "HNSW" + }, + { + "columns": "vector_column_2", + "index_name": "idx2", + "index_type": "HNSW" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show index + +**GET** `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` + +Shows detailed information about a specified index. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}/indexes/{index_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/indexes/{index_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `index_name`: (*Path parameter*) + The name of the index. + + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "database_name": "default_db", + "table_name": "test_vector_table", + "index_name": "idx1", + "index_column_ids": "2", + "index_column_names": "vector_column", + "index_type": "HNSW", + "other_parameters": "metric = l2, encode_type = plain, M = 16, ef_construction = 50", + "segment_index_count": "0", + "storage_directory": "/var/infinity/data/3C1tizeluV_db_default_db/O0wSJ88IrJ_table_test_vector_table/RYurmCbD5h_index_idx1" + "storage_size": "0B" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3018, + "error_msg": "Index {index_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +## DATA MANAGEMENT + +--- + +### Import data + +**PUT** `/databases/{database_name}/tables/{table_name}` + +Imports data from a selected file into a specified table. + +#### Request + +- Method: PUT +- URL: `/databases/{database_name}/tables/{table_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"file_path"`: `string` + - `"file_type"`: `string` + - `"header"`: `boolean` + - `"delimiter"`: `string` + +##### Request example + +```shell +curl --request PUT \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "file_path":"./filename.csv", + "file_type":"csv", + "header": true, + "delimiter": `\t" + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `"file_path"`: (*Body parameter*) + Absolute path to the file to import +- `"file_type"`: (*Body parameter*) + The type of the file to import. Supported file types include: + - `csv` + - `json` + - `jsonl` + - `parquet` +- `"header"`: (*Body parameter*), `boolean`, *Optional* + Whether to display table header or not. Works with **.csv** files only: + - `True`: Display table header. + - `False`: (Default) Do not display table header. +- `"delimiter"`: (*Body parameter*), `string`, *Optional*, Defaults to "," + Delimiter to separate columns. Works with **.csv** files only. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3032, + "error_msg": "Not supported file type: docx" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Export data + +**GET** `/databases/{database_name}/tables/{table_name}` + +Exports data from a specified table to a specified file. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"file_path"`: `string` + - `"file_type"`: `string` + - `"header"`: `boolean` + - `"delimiter"`: `string` + - `"offset"`: `int` + - `"limit"`: `int` + - `"row_limit"`: `int` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/table/{table_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "file_path": "/var/infinity/filename.csv", + "file_type": "csv", + "header": false, + "delimiter": "\t", + "offset": 2, + "limit": 6, + "row_limit": 2 + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `"file_path"`: (*Body parameter*) + Absolute path to the file for export. +- `"file_type"`: (*Body parameter*) + The type of the file for export. Supported file types include: + - `csv` + - `jsonl` + - `parquet` +- `"header"`: (*Body parameter*), `boolean`, *Optional* + Whether to display table header or not. Works with **.csv** files only: + - `True`: Display table header. + - `False`: (Default) Do not display table header. +- `"delimiter"`: (*Body parameter*), `string`, *Optional*, Defaults to "," + Delimiter to separate columns. Works with **.csv** files only. +- `"offset"`: (*Body parameter*), `int`, *Optional* + Index specifying the starting row for export. Usually used in conjunction with `limit`. If not specified, the file export starts from the first row. +- `"limit"`: `int`, *Optional* + The maximum number of rows to export. Usually used in conjunction with `offset`. If the table's row count exceeds `offset` + `limit`, the excess rows are excluded from the export. +- `"row_limit"`: (*Body parameter*), `int`, *Optional* + Used when you have a large table and need to break the output file into multiple parts. This argument sets the row limit for each part. If you specify **test_export_file.csv** as the file name, the exported files will be named **test_export_file.csv**, **test_export_file.csv.part1**, **test_export_file.csv.part2**, and so on. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 7002, + "error_msg": "File already existed: /var/infinity/filename.csv" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Insert data + +**POST** `/databases/{database_name}/tables/{table_name}/docs` + +Inserts rows of data into a specified table. + +#### Request + +- Method: POST +- URL: `/databases/{database_name}/tables/{table_name}/docs` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `object[]` + +##### Request example + +- **Standard insert with pre-computed vectors:** + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + [ + { + "name": "Jennifer", + "score": 5.0, + "fulltext_column": "You have a heart like gold.", + "dense_column": [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "sparse_column": {"10":1.1, "20":2.2, "30": 3.3}, + "tensor_column": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]], + "tensorarray_column": [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0]]], + "multivector_column": [[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]], + "array_array_varchar_column": {"array": [{"array": ["hello", "world"]}, {"array": ["!"]}, {"array": [""]}, {"array": []}]} + }, + { + "name": "Neil", + "score": 2.0, + "fulltext_column": "You're absolutely lucky.", + "dense_column": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "sparse_column": {"10":1.1, "20":2.2, "30": 3.3}, + "tensor_column": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]], + "tensorarray_column": [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0]]], + "multivector_column": [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]], + "array_array_varchar_column": {"array": []} + } + ] ' +``` + +- **Insert with FDE (Feature Dimension Expansion) function:** + +```shell +curl --request POST \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + [ + { + "name": "Document with FDE", + "score": 7.5, + "dense_column": { + "function": "fde", + "tensor_data": [ + [1.0, 2.0, 3.0, 4.0], + [0.5, 1.5, 2.5, 3.5], + [2.0, 1.0, 4.0, 3.0] + ], + "target_dimension": 8 + } + } + ] ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `"file_path"`: (*Body parameter*), `string`, *Required* +- `Body`: (*Body parameter*), `object[]`, *Required* + Data to insert. Infinity supports inserting multiple rows to a table at one time in the form of `object[]`. Each JSON object corresponds to a row to insert, and each key-value pair it corresponds to a column name and the table cell value. + +#### FDE (Feature Dimension Expansion) Function + +For vector columns, you can use the FDE function to automatically convert 2D tensor data into embeddings instead of providing pre-computed vectors. This is useful when you have feature matrices that need to be converted to fixed-dimension embeddings. + +**FDE Function Format:** +```json +{ + "vector_column_name": { + "function": "fde", + "tensor_data": [[...], [...], ...], + "target_dimension": + } +} +``` + +**Parameters:** +- `"function"`: Must be `"fde"`. +- `"tensor_data"`: A 2D array of numbers representing the input feature matrix. +- `"target_dimension"`: The desired output embedding dimension. + +**Example:** +```json +{ + "vector_col": { + "function": "fde", + "tensor_data": [ + [1.0, 2.0, 3.0, 4.0], + [0.5, 1.5, 2.5, 3.5], + [2.0, 1.0, 4.0, 3.0] + ], + "target_dimension": 64 + } +} +``` + +This will automatically convert the 3x4 feature matrix above into a 64-dimensional embedding vector. + +:::tip NOTE +- When inserting incomplete rows of data, ensure that all un-inserted columns have default values when creating the table. Otherwise, an error will occur. +- You are not allowed to insert both complete and incomplete rows of data in one request. +::: + +:::tip NOTE +Batch row limit: 8,192. You are allowed to insert a maximum of 8,192 rows at once. +::: + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3005, + "error_msg": "Column {column_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Delete data + +**DELETE** `/databases/{database_name}/tables/{table_name}/docs` + +Deletes rows from a table based on the specified condition. + +#### Request + +- Method: DELETE +- URL: `/databases/{database_name}/tables/{table_name}/docs` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"filter"`: `string` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"filter": "score >= 4.0 and score <= 5.0"} ' +``` + +```shell +curl --request DELETE \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' {"filter": "name = '"'Toby'"'"} ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `"filter"`: (*Body parameter*), `string`, *Optional* + A string that defines the condition for selecting rows to delete. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `True` for the rows that should be deleted. If `"filter"` is not specified or set to `Null`, the method will delete all rows in the table. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "deleted_rows": 10 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3005, + "error_msg": "Column {column_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Update data + +**PUT** `/databases/{database_name}/tables/{table_name}/docs` + +Searches for rows that match the specified condition and updates them accordingly. + +#### Request + +- Method: PUT +- URL: `/databases/{database_name}/tables/{table_name}/docs` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"update"`: `object` + - `"filter"`: `string` + +##### Request example + +The following code updates the score to `1.0` for rows where the `name` field is `Neil`. + +```shell +curl --request PUT \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "update": {"score": 1.0}, + "filter": "name = '"'Neil'"'" + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. +- `"filter"`: (*Body parameter*), `string`, *Optional* + A string that defines the condition for selecting rows to update. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `true` for the rows that should be updated. If `"filter"` is not specified or set to `"Null"`, the method will update all rows in the table. + + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "update_row_count": 10 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3005, + "error_msg": "Column {column_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +## SEARCH + +--- + +### Search data + +**GET** `/databases/{database_name}/tables/{table_name}/docs` + +Searches for data in a specified table. The search can range from a simple vector search, sparse vector search, or full-text search to complex hybrid searches involving reranking methods. + +#### Request + +- Method: GET +- URL: `/databases/{database_name}/tables/{table_name}/docs` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"output"`: `string[]` + - `"highlight"`: `string[]` + - `"filter"`: `string` + - `"fusion"`: `object` + - `"sort"` : `object[]` + - `"group_by"`: `string[]` + - `"having"`: `string` + - `"limit"` : `string` + - `"offset"` : `string` + - `"option"` : `object` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "output": + [ + "name", + "score", + "body" + ], + "highlight": + [ + "body" + ], + "filter": "score >= 4.0 and name = '"'Toby'"'", + "search": + [ + { + "match_method": "dense", + "fields": "dense_column", + "query_vector": [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "element_type": "float", + "metric_type": "l2", + "topn": 2, + "params": {"ef": "10"} + }, + { + "match_method": "text", + "fields": "body", + "matching_text": "bloom", + "topn": 1, + "params": + { + "default_fields": "body", + "operator": "or" + } + }, + { + "match_method": "sparse", + "fields": "sparse_column", + "query_vector": {"10":1.1, "20":2.2, "30": 3.3}, + "metric_type": "ip", + "topn": 3, + "params": { + "alpha": "1.0", + "beta": "1.0" + } + }, + { + "fusion_method": "rrf", + "topn": 2, + "params":{"rank_constant": "60"} + } + ] + } ' + ``` + +#### Example: FDE-based dense vector search + +```shell +curl --request GET \ + --url http://localhost:23820/databases/default_db/tables/my_table/docs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data '{ + "output": ["id", "name", "_similarity"], + "search": [ + { + "match_method": "dense", + "fields": "vector_column", + "fde": { + "query_tensor": [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + "target_dimension": 64 + }, + "element_type": "float", + "metric_type": "cosine", + "topn": 5 + } + ] + } ' +``` + +##### Request parameters + +- `database_name`: (*Path parameter*) + The name of the database. +- `table_name`: (*Path parameter*) + The name of the table. + +- `"output"`: (*Body parameter*), `string[]`, *Required* + A non-empty list of strings specifying the columns to include in the output. Each string in the list can represent: + - A user-defined column name: The name of the column to include in the output, e.g., `"body"`. + - All user-defined columns: Use a wildcard `"*"` to select all columns. + - A special system column: system-generated columns include: + - `_row_id`: An automatically generated, unique identifier for each row in the table. It serves as a unique key for each row but does not necessarily correspond to the actual row number. When the data in a row is updated, the `_row_id` for that row is also changed to reflect the update. + - `_score`: A BM25 score used in full-text search. + - `_similarity`: Used by IP and cosine metrics in dense or sparse vector search. + - `_distance`: Used by L2 metric in dense vector search. + - An aggregation function: Apply an aggregation operation on specified columns. Supported aggregation functions include: + - `count` + - `min` + - `max` + - `sum` + - `avg` + - An arithmetic function: Apply an arithmetic operation on specified columns (e.g., `c1+5`). + > The list must contain at least one element. Empty lists are not allowed. + +- `"highlight"`: (*Body parameter*), `string`, *Optional* + A list of strings specifying the columns to include words match in match_text clause. The matched words will be embraced by and + +- `"filter"`: (*Body parameter*), `string`, *Optional* + A string that defines the condition for selecting rows. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `true` for the rows that should be output. + Supported filter expressions: + - Logical combination expressions: 'and', 'or', 'not' + - Comparison expressions: `<`, `<=`, `>`, `>=`, `=`, `==`, `!=` + - `in` and `not in` expression + - `filter_fulltext` expression: + - Similar to `match_text()`. Usage: 'filter_fulltext(fields, matching_text)' or 'filter_fulltext(fields, matching_text, extra_options)' + - 'extra_options' is in the format of 'K1=V1;K2=V2;...;KN=VN', + where each 'K' represents a parameter name and 'V' represents its value + - Available parameters in 'extra_options': + - **'minimum_should_match'**: specifies how many clauses in the 'matching_text' should be satisfied at least. + It can be in the following formats: + - Positive integer `N`: at least `N` clauses should be satisfied. + - Negative integer `-N`: at least (total clause count - `N`) clauses should be satisfied. + - Positive Percentage `N%`: at least `⌊total clause count * N%⌋` clauses should be satisfied. + - Negative Percentage `-N%`: at least `total clause count - ⌊total clause count * N%⌋` clauses should be satisfied. + - Combination `K K`, the requirement `V` applies, otherwise all the clauses should be satisfied + - Multiple combinations `K1= total clause count`, all the clauses should be satisfied. + Otherwise, we find the biggest `V` which is less than the total clause count and apply the correspondent `V`. + - **'default_field'** + - If `"fields"` is an empty string, this parameter specifies the default field to search on. + + +- `"search"`: (*Body parameter*), `object[]`, *Required* + A list of JSON objects, each representing a matching or fusion method. Note that if you have specified multiple matching methods, you must specify a fusion method. + - `"match_method"`: (*Body parameter*), `enum`, *Required* + The matching method for a specified column/field: + - `"dense"`: Creates a dense vector search expression to identify the top n most similar rows to the given dense vector. Suitable for working with dense vectors (dense embeddings) or multi-vectors (multiple dense embeddings in one row). + - `"sparse"`: Creates a sparse vector search expression to identify the top n most similar rows to the given sparse vector. Suitable for working with sparse vectors (sparse embeddings). + - `"text"`: Creates a full-text search expression on the specified field(s)/column(s) to identify the top n most similar rows. + > Ensure that a full-text index has been successfully built on each column involved before executing a full-text search; otherwise, an error will occur. + + - `"fields"`: `string`, *Required* + A non-empty string indicating the name of the column/field to search on. + - `"query_vector"`: `[]/object`, *Optional* + The query vector data to compare with. This should be provided as a list or a JSON object. + Used *only* when `"match_method"` is set to `"dense"` or `"sparse"`. + - Dense query vector example: `[1.0, 2.2, 3.4, 1.5]` + - Sparse query vector example: `{"10":1.1, "20":2.2, "30": 3.3}` + > **Note**: You must provide either `"query_vector"` or `"fde"` for a dense vector search, but *not* both. + - `"fde"`: `object`, *Optional* + Feature Dimension Expansion (FDE) function parameters for generating embeddings from tensor input. + Used *only* when `"match_method"` is set to `"dense"`. + - `"query_tensor"`: `number[][]`, *Required* - A 2D array (matrix) of numeric values representing the input tensor data. + Example: `[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]` + - `"target_dimension"`: `integer`, *Required* - The target dimension for the output embedding vector. Must be a positive integer. + Example: `64` + > **Note**: You must provide either `"query_vector"` or `"fde"` for a dense vector search, but *not* both. + - `"matching_text"`: `string`, *Optional* + A non-empty text string to search for. Used *only* when `"match_method"` is set to `"text"`. + You can use various search options within the matching text, including: + - Single terms: `"blooms"` + - OR multiple terms: `"Bloom OR filter"` or just `"Bloom filter"` + - Phrase search: `'"Bloom filter"'` + - AND multiple terms: `"space AND efficient"` + - Escaping reserved characters: `"space\:efficient"` + - Sloppy phrase search: `'"harmful chemical"~10'` + - Field-specific search: `"title:(quick OR brown) AND body:foobar"` + - `element_type`: `str`, *Required* + Specifies the data type of the embedding vector. Used *only* when `"match_method"` is set to `"dense"`. Commonly used types (values) include: + - `"float"` + - `"uint8"`. + - `metric_type`: `string`, *Required* + The distance metric to use in similarity search. Used *only* when `"match_method"` is set to `"dense"` or `"sparse"`. + - When `"match_method"` is set to `"dense"`: + - `"ip"`: Inner product. + - `"l2"`: Euclidean distance. + - `"cosine"`: Cosine similarity. + - When `"match_method"` is set to `"sparse"`, only `"ip"` can be used. + - `"fusion_method"`: (*Body parameter*), `enum`, *Required when you have specified multiple matching methods* + - `"rrf"`: [Reciprocal rank fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) + RRF is a method for combining multiple result sets with varying relevance indicators into a single result set. It requires no tuning, and the relevance indicators need not be related to achieve high-quality results. RRF is particularly useful when you are uncertain of the relative importance of each retrieval way. + RRF uses the following formula to calculate the score for matching each document: + ```python + score = 0.0 + for q in queries: + if d in result(q): + score += 1.0 / ( k + rank( result(q), d ) ) + return score + + # Where + # k is the ranking constant, + # q is a query in a set of queries, + # d is a document in the result set of q, + # result(q) is the result set of q, and + # rank( result(q), d ) is the rank of d within the result(q), starting from 1. + ``` + + - `"weighted_sum"` + The weighted sum approach assigns different weights to different retrieval ways, allowing you to emphasize specific ways. This is particularly useful when you are certain of each path's relative importance. + - `"match_tensor"` + Infinity's tensor-based late interaction reranking approach. + - `"max"` + The max approach returns the maximum score from multiple retrieval ways. + + - `"topn"`: `int`, *Required* + An integer indicating the number of nearest neighbours (vector search) or most relevant rows (full-text search) to return. + - `"params"`: `object`, *Optional* + Additional matching or reranking parameters. + - For all matching methods, you can provide a index hint when searching, so that the query will run using specified index. + - For `"match_method" : "dense"`, `"match_method" : "sparse"`, `"match_method" : "tensor"`, use `"index_name" : "idx1"` to specify index, note that if the index specified is not present, an error would occur. + - For `"match_method" : "text"`, since the query can run on multiple columns, you should specify multiple indexes using `"index_names" : "idx1,idx2..."`, there will be no error when some of specified indexes are not present. + - If you set `"match_method"` to `"dense"`: + - `"ef"`: `str`, Recommended value: one to ten times the value of `topn`. + - For example, if you set `topn` to `10`, you can set `"ef"` to `"50"`. + - If you set `"ef"` too high, search performance may worsen. + - If you do not set `"ef"` or set it to a value lower than `topn`, the search uses the `topn` value as the value for `"ef"`. + - `"threshold"`: `str`, *Optional* A threshold value for the search. + - For example, if you use the `"cosine"` distance metric and set `"threshold"` to `"0.5"`, the search will return only those rows with a cosine similarity greater than `0.5`. + - `"nprobe"`: `str`, *Optional* The number of cells to search in the IVF index. The default value is `"1"`. + - If you set `"match_method"` to `"sparse"`: + - `"alpha"`: `str` + `"0.0"` ~ `"1.0"` (default: `"1.0"`) - A "Termination Conditions" parameter. The smaller the value, the more aggressive the pruning. + - `"beta"`: `str` + `"0.0"` ~ `"1.0"` (default: `"1.0"`) - A "Query Term Pruning" parameter. The smaller the value, the more aggressive the pruning. + + - If you set `"match_method"` to `"text"`: + - **"default_field"**: `str`, *Optional* + - If `"fields"` is an empty string, this parameter specifies the default field to search on. + - **"operator"**: `str`, *Optional* + - If not specified, the search follows Infinity's full-text search syntax, meaning that logical and arithmetic operators, quotation marks and escape characters will function as full-text search operators, such as: + - AND operator: `AND` + - OR operator: `OR` + - NOT operator: `NOT` + - PAREN operator: `(`, `)`, need to appear in pairs, and can be nested. + - COLON operator: `:`: Used to specify field-specific search, e.g., `body:foobar` searches for `foobar` in the `body` field. + - CARAT operator: `^`: Used to boost the importance of a term, e.g., `quick^2 brown` boosts the importance of `quick` by a factor of 2, making it twice as important as `brown`. + - TILDE operator: `~`: Used to specify sloppy phrase search, e.g., `"harmful chemical"~10` searches for the phrase `"harmful chemical"` within a tolerable distance of 10 words. + - SINGLE_QUOTED_STRING: Used to search for a phrase, e.g., `'Bloom filter'`. + - DOUBLE_QUOTED_STRING: Used to search for a phrase, e.g., `"Bloom filter"`. + - Escape characters: Used to escape reserved characters, e.g., `space\:efficient`. Starting with a backslash `\` will escape the following characters: + `' '`, `'('`, `')'`, `'^'`, `'"'`, `'\''`, `'~'`, `'*'`, `'?'`, `':'`, `'\\'` + - If specified, Infinity's full-text search syntax will not take effect, and the specified operator will be interpolated into `matching_text`. + Useful for searching text including code numbers like `"A01-233:BC"`. + - `{"operator": "or"}`: Interpolates the `OR` operator between words in `matching_text` to create a new search text. + For example, reinterprets `"A01-233:BC"` as `'"A01" OR "-233" OR "BC"'`. + - `{"operator": "and"}`: Interpolates the `AND` operator between words in `matching_text` to create a new search text. + For example, reinterprets `"A01-233:BC"` as `'"A01" AND "-233" AND "BC"'`. + + - If you set `"fusion_method"` to `"rrf"`, use a separate JSON to set the following parameter: + - `"rank_constant"`: The smoothing constant for RRF reranking, e.g., `{"rank_constant": 60}`. Defaults to `60`. + - If you set `"fusion_method"` to `"weighted_sum"`, use a separate JSON to set the following parameter: + - `"weights"`: Specifies the weight for each retrieval way. For example, `{"weights": "1,2,0.5"}` sets weights of `1`, `2`, and `0.5` for the first, second, and third retrieval ways, respectively. The default weight of each retrieval way is `1.0`. If `"weight"` is not specified, all retrieval ways will be assigned the default weight of `1.0`. + - `"normalize"`: Specifies the normalization method for each retrieval way. Options: `"minmax"`, `"atan"`, `"l2"`, `"none"`. Defaults to `"minmax"`. + - If you set `"fusion_method"` to `"max"`, use a separate JSON to set the following parameter: + - `"normalize"`: Specifies the normalization method for each retrieval way. Options: `"minmax"`, `"atan"`, `"l2"`, `"none"`. Defaults to `"minmax"`. + - If you set `"fusion_method"` to `"match_tensor"`, use a separate JSON to set the following parameters: + - `"fields"`: The name of the tensor column for reranking. + - `"query_tensor"`: The tensor data to compare against. This should be provided as a list of lists of numerical values. + - `"element_type"`: The element data type of the query tensor. Usually `"float"`. + +- `"sort"` : `object[]` + Defines how to sort the results. + +- `"group_by"`: `string[]` + Indicates the expression to group by. + +- `"having"`: `string` + Indicates the having condition. + +- `"limit"` : `string` + Indicates the limit row count. + +- `"offset"` : `string` + Indicates the offset position of the limit expression. You must use this parameter together with `limit`. + +- `"option"` : `object` + Indicates some search options. This parameter must be used in conjunction with `limit`. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "output": [ + { + "name": "Tom", + "age": 16 + } + ] + "total_hits_count": 3 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `"total_hits_count"`: `integer`, Optional + Available if you set a search option with `"total_hits_count": "true"` + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3005, + "error_msg": "Column {column_name} doesn't exist in {table_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +## ADMINISTRATION + +--- + +### Set variable + +**POST** `/variables/{variable_name}` + +Assigns a value to a global variable. + +#### Request + +- Method: POST +- URL: `/variables/{variable_name}` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: + - `"profile_record_capacity"`: `integer` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/variables/{variable_name} \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { "profile_record_capacity" : 120 } ' +``` + +##### Request parameter + +- `variable_name`: (*Path parameter*) + The name of the variable, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3076, + "error_msg": "Invalid command: unknown global variable {variable_name}" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show variables + +**GET** `/variables` + +Retrieves all global variables in the Infinity system. + +#### Request + +- Method: GET +- URL: `/variables` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/variables \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "active_txn_count":"1", + "active_wal_filename":"/var/infinity/wal/wal.log", + "buffer_object_count":"6", + "buffer_usage":"0B/4.00GB", + "current_timestamp":"16774", + "delta_log_count":"1", + "next_transaction_id":"6", + "profile_record_capacity":"128", + "query_count":"0", + "schedule_policy":"round robin", + "session_count":"1", + "total_commit_count":"0", + "total_rollback_count":"0", + "unused_buffer_object":"0" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show variable + +**GET** `/variables/{variable_name}` + +Retrieves the value of a global variable. + +#### Request + +- Method: GET +- URL: `/variables/{variable_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/variables/{variable_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `variable_name`: (*Path parameter*) + The name of the variable. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "buffer_object_count":"6" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3027, + "error_msg": "No such system variable {variable_name}." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Set config + +**PUT** `/configs` + +Assigns a value to a configuration parameter. + +#### Request + +- Method: PUT +- URL: `/configs` +- Headers: + - `accept: application/json` + - `content-Type: application/json` +- Body: `object` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/configs \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { "log_level": "trace" } ' +``` + +##### Request parameter + +- `Body`: (*Body parameter*), `object`, *Required* + The configuration parameters to update, where each key (`string`) in the JSON object represents an entry, and the corresponding value (`string`) represents its value. + + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3028, + "error_msg": "log level value range is trace, debug, info, warning, error, critical" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show configs + +**GET** `/configs` + +Retrieves the values of all configuration parameters. + +#### Request + +- Method: GET +- URL: `/configs` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/configs \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "buffer_manager_size":"4294967296", + "cleanup_interval":"10", + "client_port":"23817", + "compact_interval":"10", + "connection_pool_size":"256", + "cpu_limit":"16", + "data_dir":"/var/infinity/data", + "error_code":0, + "checkpoint_interval":"30", + "http_port":"23820", + "log_dir":"/var/infinity/log", + "log_file_max_size":"1073741824", + "log_file_rotate_count":"8", + "log_filename":"infinity.log", + "log_level":"Info", + "log_to_stdout":"False", + "mem_index_capacity":"65536", + "optimize_interval":"10", + "postgres_port":"5432", + "resource_dir":"/usr/share/infinity/resource", + "server_address":"0.0.0.0", + "temp_dir":"/var/infinity/tmp", + "time_zone":"UTC+8", + "version":"0.7.0", + "wal_compact_threshold":"1073741824", + "wal_dir":"/var/infinity/wal", + "wal_flush":"FlushAtOnce" +} +``` + +--- + +### Show config + +**GET** `/configs/{config_name}` + +Retrieves the value of a configuration parameter. + +#### Request + +- Method: GET +- URL: `/configs/{config_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/configs/{config_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `config_name`: (*Path parameter*) + The name of the configuration parameter. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "version":"0.7.0" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 1008, + "error_msg": "Attempt to get option: {config_name} which doesn't exist." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show buffer + +**GET** `/instance/buffer` + +List the cached buffer objects in database. + +#### Request + +- Method: GET +- URL: `/instance/buffer` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/buffer \ + --header 'accept: application/json' +``` + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "buffer": [ + { + "buffered_type": "Persistent", + "path": "/var/infinity/data/nlAn5spku9_db_default_db/sNzUhKy3er_table_table1/seg_0/blk_0/0.col", + "size": "32768", + "status": "Freed", + "type": "data" + }, + { + "buffered_type": "Persistent", + "path": "/var/infinity/data/nlAn5spku9_db_default_db/sNzUhKy3er_table_table1/seg_0/blk_0/2.col", + "size": "131072", + "status": "Freed", + "type": "data" + }, + { + "buffered_type": "Persistent", + "path": "/var/infinity/data/nlAn5spku9_db_default_db/sNzUhKy3er_table_table1/seg_0/blk_0/version", + "size": "65536", + "status": "Freed", + "type": "version data" + } + ], + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 2005, + "error_msg": "Not support in maintenance mode" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show profiles + +**GET** `/instance/profiles` + +When set session variable 'profile', Infinity will record query profile information. This command is to list recorded queries profile. + +#### Request + +- Method: GET +- URL: `/instance/profiles` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/profiles \ + --header 'accept: application/json' +``` + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "profiles": [ + { + "command parsing": "250us", + "commit": "0ns", + "execution": "380us", + "logical plan building": "133us", + "physical plan building": "44us", + "pipeline building": "38us", + "plan optimizing": "1000ns", + "record_no": "0", + "rollback": "35us", + "task building": "70us", + "total_cost": "951us" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 2005, + "error_msg": "Not support in maintenance mode" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show queries + +**GET** `/instance/queries` + +Show running queries, only valid when config: record_running_query is set. + +#### Request + +- Method: GET +- URL: `/instance/queries` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/queries \ + --header 'accept: application/json' +``` + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "queries": [ + { + "query_id": "1928", + "query_kind": "COPY", + "session_id": "63", + "start_time": "21:22:49:708:989:0", + "time_consumption": "2s" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 2005, + "error_msg": "Not support in maintenance mode" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show transactions + +**GET** `/instance/transactions` + +Shows running transactions on database. + +#### Request + +- Method: GET +- URL: `/instance/transactions` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/transactions \ + --header 'accept: application/json' +``` + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "transactions": [ + { + "transaction_id":"27275", + "transaction_text":"" + }, + { + "transaction_id":"27274", + "transaction_text":"" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 2005, + "error_msg": "Not support in maintenance mode" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show transaction + +**GET** `/instance/transactions/{trasaction_id}` + +Shows running transactions on database by transaction_id. + +#### Request + +- Method: GET +- URL: `/instance/transactions` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/transactions/{transaction_id} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `transaction_id`: (*Path parameter*) + The id of the running transaction. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "transaction": { + "transaction_id":"27275", + "transaction_text":"" + } +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 2005, + "error_msg": "Not support in maintenance mode" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show logs + +**GET** `/instance/logs` + +Shows logs. + +#### Request + +- Method: GET +- URL: `/instance/logs` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/logs \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "logs":[ + { + "command_type":"CHECKPOINT", + "commit_ts":"1", + "text":"catalog path: catalog/FULL.82192.json\nmax commit ts: 82192\nis checkpoint: 1\n", + "transaction_id":"82195" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + + +### Show checkpoints + +**GET** `/instance/global_checkpoint` + +Shows delta logs. + +#### Request + +- Method: GET +- URL: `/instance/global_checkpoint` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/global_checkpoint \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "checkpoint":[ + { + "file_path":"/var/infinity/data/catalog/FULL.82192.json", + "max_commit_timestamp":"82192" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show objects + +**GET** `/instance/objects` + +Shows persistence objects. + +#### Request + +- Method: GET +- URL: `/instance/objects` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/objects \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "objects": [ + { + "deleted_ranges": "", + "name": "01927642-6491-79d8-8891-5f91d7960cdf", + "parts": "4", + "reference_count": "0", + "size": "262236" + }, + { + "deleted_ranges": "[0, 426120) ", + "name": "019275ce-9eaf-7950-8bf0-b451cb15e6e1", + "parts": "0", + "reference_count": "0", + "size": "0" + }, + { + "deleted_ranges": "", + "name": "01927641-793c-7a30-b20c-b9089b6c0a9b", + "parts": "9", + "reference_count": "0", + "size": "426188" + }, + { + "deleted_ranges": "[0, 65722552) ", + "name": "019275cd-7310-7b40-a46a-cb12db4a391b", + "parts": "0", + "reference_count": "0", + "size": "0" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show object + +**GET** `/instance/objects/{object_name}` + +Shows persistence object by its name. + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/instance/objects/{object_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `object_name`: Required (*Path parameter*) + The object name. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "object": [ + { + "end": "426120", + "start": "0" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code": 7008, + "error_msg": "File: {object_name} isn't found@src/executor/operator/physical_show.cpp:5631@src/scheduler/fragment_context.cpp:1417" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Show files + +**GET** `/instance/files` + +Shows files in persistent manager. + +#### Request + +- Method: GET +- URL: `/instance/files` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/files \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "files": [ + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_4/0.col", + "object_name": "01927a51-068a-7ba8-8d80-4d89bd6391de", + "offset": "78660", + "size": "131096" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_0/col_1_out_0", + "object_name": "01927a4f-c82c-7ba8-b135-80dd3cfb2d87", + "offset": "340852", + "size": "11435869" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_3/1.col", + "object_name": "01927a50-dd6d-7138-9665-9e21502390a6", + "offset": "209756", + "size": "131096" + }, + { + "file_name": "FC5WLAHAam_db_default_db/o3qQcKBZuI_table_hr_data_mix_remote/seg_0/blk_0/0.col", + "object_name": "01927a4f-c82c-7ba8-b135-80dd3cfb2d87", + "offset": "78660", + "size": "131096" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show memory + +**GET** `/instance/memory` + +Shows the memory usage information of the instance. + +#### Request + +- Method: GET +- URL: `/instance/memory` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/memory \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "memory_allocation":"Not activate", + "memory_objects":"Not activate" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show memory object + +**GET** `/instance/memory/objects` + +Shows the memory object usage information of the instance. + +#### Request + +- Method: GET +- URL: `/instance/memory/objects` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/memory/objects \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Show memory allocation + +**GET** `/instance/memory/allocation` + +Shows the memory allocation information of the instance. + +#### Request + +- Method: GET +- URL: `/instance/memory/allocation` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/instance/memory/allocation \ + --header 'accept: application/json' +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Global Checkpoint + +**POST** `/instance/flush` + +Performs a checkpoint on the instance. + +#### Request + +- Method: POST +- URL: `/instance/flush` +- Headers: + - `accept: application/json` + - `content-type: application/json` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23822/instance/flush\ + --header 'accept: application/json' \ +``` + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Compact Table + +**POST** `/instance/table/compact` + +Performs a compaction operation on a specified table. + +#### Request + +- Method: POST +- URL: `/instance/table/compact` +- Headers: + - `accept: application/json` + - `content-type: application/json` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23820/instance/table/compact \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' + { + "db_name" : "default_db", + "table_name" : "my_table", + } ' +``` + +##### Request parameters + +- `db_name`: Required (*Body parameter*) +- `table_name`: Required (*Body parameter*) +`db_name` and `table_name` specifies the table to perform compaction. + +#### Response + + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code":3022, + "error_msg":"Table id_timep doesn't exist@src/planner/query_binder.cpp:423" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Optimize Index + +**PUT** `/databases/{database_name}/tables/{table_name}/optimize` + +Performs an index optimization operation on a specified table. + +#### Request + +- Method: PUT +- URL: `/databases/{database_name}/tables/{table_name}/optimize` +- Headers: + - `accept: application/json` + - `content-type: application/json` + +##### Request example + +```shell +curl --request PUT \ + --url http://localhost:23820/databases/{database_name}/tables/{table_name}/optimize \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ +``` + +##### Request parameters + +- `db_name`: Required (*Path parameter*) +- `table_name`: Required (*Path parameter*) +`db_name` and `table_name` specifies the table to perform index optimization. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code":3022, + "error_msg":"Table id_timep doesn't exist@src/planner/query_binder.cpp:423" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +## DISTRIBUTED + +--- + +### Admin set node role + +**POST** `/admin/node/current` + +Sets role for a node. + +#### Request + +- Method: POST +- URL: `/admin/node/current` +- Headers: + - `accept: application/json` + - `content-type: application/json` + +##### Request example + +```shell +curl --request POST \ + --url http://localhost:23822/admin/node/current \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data ' { + "role" : "follower", + "name" : "following", + "address" : "0.0.0.0:23851" + } ' +``` + +##### Request parameters + +- `role`: Required (*Body parameter*) + The role of the node. It must be set to `admin`, `standalone`, `leader` or `follower`. + - Parameter settings for setting a node to `admin` and `standalone` are both empty, no extra parameters are required. + - Parameter settings for setting a node to `leader`: + - "name" Required : Name of the node in the cluster. + - Parameter settings for setting a node to `follower` or `learner`: + - "name" Required : Name of the node in the cluster. + - "address" Required : Peer server address of the leader in the cluster. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code":7020 + "error_msg" : "Duplicate node: following" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + + +### Admin remove node + +**POST** `/admin/node/{node_name}` + +Removes a node from the cluster. This command can only be executed by the leader node. + +#### Request + +- Method: DELETE +- URL: `/admin/node/node_name` +- Headers: + - `accept: application/json` + - `content-type: application/json` + +##### Request example + +```shell +curl --request DELETE \ + --url http://localhost:23821/admin/node/follower1 \ + --header 'accept: application/json' \ + --header 'content-type: application/json' +``` + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code":7020 + "error_msg" : "Duplicate node: following" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Admin show node variables + +**GET** `/admin/variables` + +Shows all node variables in admin mode. + +#### Request + +- Method: GET +- URL: `/admin/variables` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/admin/variables \ + --header 'accept: application/json' +``` + +##### Request parameters + +None. + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "role":"admin" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + +--- + +### Admin show node configs + +**GET** `/admin/configs` + +Shows all node configs in admin mode. + +#### Request + +- Method: GET +- URL: `/admin/configs` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/admin/configs \ + --header 'accept: application/json' +``` + +##### Request parameters + +None. + +#### Response + +##### Status Code 200 + +The response includes a JSON object like the following: + +```shell +{ + "buffer_manager_size":"4294967296", + "cleanup_interval":"60", + "client_port":"23818", + "compact_interval":"120", + "connection_pool_size":"128", + "cpu_limit":"16", + "data_dir":"/var/infinity/leader/data", + "error_code":0, + "checkpoint_interval":"86400", + "http_port":"23821", + "log_dir":"/var/infinity/leader/log", + "log_file_max_size":"10737418240", + "log_file_rotate_count":"10", + "log_filename":"infinity.log", + "log_level":"Debug", + "log_to_stdout":"true", + "mem_index_capacity":"65536", + "optimize_interval":"10", + "peer_server_connection_pool_size":"64", + "postgres_port":"5433", + "record_running_query":"false", + "resource_dir":"/var/infinity/leader/resource", + "server_address":"0.0.0.0", + "temp_dir":"/var/infinity/leader/tmp", + "time_zone":"UTC-8", + "version":"0.7.0", + "wal_compact_threshold":"1073741824", + "wal_dir":"/var/infinity/leader/wal", + "wal_flush":"FlushAtOnce" +} +``` + +--- + +### Admin show node variable + +**GET** `/admin/variables/{variable_name}` + +Retrieves the value of a global variable in admin mode. + +#### Request + +- Method: GET +- URL: `/admin/variables/{variable_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23820/admin/variables/{variable_name} \ + --header 'accept: application/json' +``` + +##### Request parameters + +- `variable_name`: (*Path parameter*) + The name of the variable. + +#### Response + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "server_role":"admin" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code" : 3027, + "error_msg":"variable does not exist : role." +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Admin show current node + +**GET** `/admin/node/current` + +Gets information about the currently connected node. + +#### Request + +- Method: GET +- URL: `/admin/node/current` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/admin/node/current \ + --header 'accept: application/json' +``` + +##### Request parameters + +None. + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "node_role":"learner" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `"node_role": string` + The role of querying node. + +--- + +### Admin show node + +**GET** `/admin/node/{node_name}` + +Retrieves information about a node in the cluster. + +#### Request + +- Method: GET +- URL: `/admin/node/{node_name}` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23821/admin/node/learning \ + --header 'accept: application/json' +``` + +##### Request parameters + +None. + +#### Response + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code":0, + "node_role":"learner" +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `"node_role": string` + The role of querying node. + + + + +A `500` HTTP status code indicates an error condition. The response includes a JSON object like the following: + +```shell +{ + "error_code":7019, + "error_msg":"Node doesn't exist: notlearning" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_msg"`: `string` + When `error_code` is non-zero, `"error_msg"` provides additional details about the error. + + + + +--- + +### Admin list nodes + +**GET** `/admin/nodes` + +Lists all nodes in the cluster + +#### Request + +- Method: GET +- URL: `/admin/nodes` +- Headers: `accept: application/json` + +##### Request example + +```shell +curl --request GET \ + --url http://localhost:23822/admin/nodes \ + --header 'accept: application/json' +``` + +##### Request parameters + +None. + +#### Response + +##### Status code 200 + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0, + "nodes": [ + { + "address": "0.0.0.0:23852", + "heartbeat": "91", + "last_update": "Mon Dec 2 14:48:34 2024\n", + "name": "follower", + "role": "follower", + "status": "alive" + }, + { + "address": "0.0.0.0:23851", + "heartbeat": "0", + "last_update": "Mon Dec 2 14:48:34 2024\n", + "name": "boss", + "role": "leader", + "status": "alive" + } + ] +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. +- `"nodes" : array` : + Each element is an object with following information: + - `"address"` : The IP address of the node. + - `"heartbeat"` : The number of heartbeat messages received from the leader. For non-leader roles only. + - `"name"` : The node's name. + - `"role"` : The node's role. + - `"status"` : The node's status, this is one of `"invalid"`, `"alive"`, `"lostconnection"`, `"removed"`, `"timeout"`. + - `"last_update"` : The last time that this node is updated. + +--- + + diff --git a/website/versioned_docs/version-v0.7.0-dev1/references/pysdk_api_reference.md b/website/versioned_docs/version-v0.7.0-dev1/references/pysdk_api_reference.md new file mode 100644 index 0000000..65d9566 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/references/pysdk_api_reference.md @@ -0,0 +1,3051 @@ +--- +sidebar_position: 2 +slug: /pysdk_api_reference +--- +# Python API Reference + +A complete reference for Infinity's Python APIs. + +--- + +## SCHEMA MANAGEMENT + +--- + +### connect + +```python +# Connect to the Infinity server and get an Infinity object +infinity.connect(uri) +``` + +Connects to the Infinity server, and gets an Infinity object. + +:::tip NOTE +You must have an Infinity object ready to perform database-specific operations. +::: + +#### Parameters + +##### uri: *Required* + +The `uri` here is a `NetworkAddress` object: + +`NetworkAddress`: Used in client-server mode, when you have deployed Infinity as a separate server and wish to connect to it remotely. A `NetworkAddress` object comprises two fields: + +- `""`: `str` - The IP address of the Infinity server. +- ``: `int` - The SDK port number on which the Infinity server listens. Defaults to `23817`. + +:::tip IMPORTANT +When setting `uri` as `NetworkAddress`, ensure you: +- Install the Infinity SDK: `pip install infinity==` +- Import the `infinity` module: `import infinity`. +::: + +:::caution IMPORTANT +When connecting to Infinity in client-server mode, ensure that the client version *exactly* matches the server version. For example: + +| **Client version** | **Server version** | +|--------------------|--------------------| +| v0.1.0 | v0.1.0 | +| v0.1.1 | v0.1.1 | +| v0.2.0 | v0.2.0 | +| v0.2.1 | v0.2.1 | +| v0.3.0 | v0.3.0 | +| v0.4.0 | v0.4.0 | +| v0.5.0-dev2 | v0.5.0-dev2 | + +If the versions do not match, please update your client or server to ensure compatibility. + +In client-server mode, also ensure that your server version matches the version specified in your configuration file. Here, the matching rule is less strict than an exact match: + +- The major and minor versions *must* be identical. +- The patch version may differ. + +This allows for bug fixes without requiring changes to the configuration file. + +| **Configuration version** | **Compatible server version** | +|---------------------------|-------------------------------| +| v0.1.0 | v0.1.0, v0.1.1 | +| v0.2.0 | v0.2.0, v0.2.1 | + +::: + +#### Returns + +- Success: An `infinity.remote_thrift.infinity.RemoteThriftInfinityConnection` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` - A non-zero value indicating a specific error condition. + - `error_msg`: `str` - A message providing additional details about the error. + +#### Examples + +##### Connect to Infinity in client-server mode + +If you have deployed Infinity as a separate server and installed the Infinity client via `pip install infinity==`, you can connect to it via its IP address. If your Infinity is running on your local machine, you can also use `infinity.common.LOCAL_HOST` to replace `""` in the following code snippet. + +```python +import infinity +# If Infinity is deployed on the local machine, use infinity.LOCAL_HOST to replace +infinity_object = infinity.connect(infinity.NetworkAddress("192.168.1.101", 23817)) +``` + +--- + +### disconnect + +```python +infinity_object.disconnect() +``` + +Disconnects the client from the Infinity server in client-server mode, or destructs the Infinity object and releases all associated resources when Infinity is imported as a Python module. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +infinity_object.disconnect() +``` + +--- + +### create_database + +```python +infinity_object.create_database(db_name, conflict_type = ConflictType.Error, comment = None) +``` + +Creates a database with a specified name. + +#### Parameters + +##### db_name: `str`, *Required* + +A non-empty string indicating the name of the database, which must adhere to the following requirements: + +- Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) +- Must begin with an English letter or underscore. +- Maximum 65,535 characters. +- Case-insensitive. + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if a database with the same name exists. +- `Ignore`: Ignore the database creation request and keep the existing database with the same name. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +##### comment: `str`, *Optional* + +Additional comment for the database to create. + +#### Returns + +- Success: An `infinity.remote_thrift.db.RemoteDatabase` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` - A non-zero value indicating a specific error condition. + - `error_msg`: `str` - A message providing additional details about the error. + +#### Examples + +```python +# Create a database named 'my_database': +# If the specified database already exists, raise an error. +infinity_object.create_database("my_database") +``` + +```python +# Create a database named 'my_database': +# If the specified database already exists, raise an error (same as above). +infinity_object.create_database("my_database", infinity.common.ConflictType.Error, comment="Database One") +``` + +```python +from infinity.common import ConflictType +# Create a database named 'my_database': +# If the specified database already exists, silently ignore the operation and proceed. +infinity_object.create_database("my_database", ConflictType.Ignore, comment="Database One") +``` + +--- + +### drop_database + +```python +infinity_object.drop_database(db_name, conflict_type = ConflictType.Error) +``` + +Deletes a database by its name. + +#### Parameters + +##### db_name: `str`, *Required* + +A non-empty string indicating the name of the database to delete. + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if the specified database does not exist. +- `Ignore`: Ignore the operation and proceed regardless, if the specified database does not exist. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` + +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Delete a database named 'my_database': +# If the specified database does not exist, raise an error. +infinity_object.drop_database("my_database") +``` + +```python +# Delete a database named 'my_database': +# If the specified database does not exist, raise an error (same as above). +infinity_object.drop_database("my_database", infinity.common.ConflictType.Error) +``` + +```python +from infinity.common import ConflictType +# Delete a database named 'my_database': +# If the specified database does not exist, silently ignore the operation and proceed. +infinity_object.drop_database("my_database", ConflictType.Ignore) +``` + +--- + +### list_databases + +```python +infinity_object.list_databases() +``` + +Retrieves a list of all available databases within the Infinity system. + +#### Returns + +A structure containing the following attributes: + +- `db_names`: `list[str]` A list of strings indicating the names of all available databases. +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +res = infinity_object.list_databases() +print(res.db_names) # ['my_database', 'database_1'] +``` + +--- + +### show_database + +```python +infinity_object.show_database(database_name) +``` + +Shows detailed information about a database. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +res = infinity_object.show_database('my_database') +``` + +--- + +### get_database + +```python +infinity_object.get_database(database_name) +``` + +Retrieves a database object by its name. + +#### Parameters + +##### db_name: `str`, *Required* + +A non-empty string indicating the name of the database to retrieve. + +#### Returns + +- Success: An `infinity.remote_thrift.db.RemoteDatabase` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` - A non-zero value indicating a specific error condition. + - `error_msg`: `str` - A message providing additional details about the error. + +#### Examples + +```python +db_object = infinity_object.get_database("my_database") +``` + +--- + +### create_table + +```python +db_object.create_table(table_name, columns_definition, conflict_type = ConflictType.Error) +``` + +Creates a table with a specified name and defined columns. + +:::tip NOTE +Call `create_database()` or `get_database()` to get a database object for all table-specific operations. +::: + +#### Parameters + +##### table_name: `str`, *Required* + +A non-empty string indicating the name of the table, which must adhere to the following requirements: + +- Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) +- Must begin with an English letter or underscore. +- Maximum 65,535 characters. +- Case-insensitive. + +##### columns_definition: `dict[str, dict[str, Any]]`, *Required* + +Definitions for all table columns as a dictionary. Each key in the dictionary is a column name (a `str` following the same rule as `table_name`), with a corresponding 'value' dictionary defining the column's data type and default value information in key-value pairs: + +- **Data type** (`"type"`) + The data type of the column. + - Numeric: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - String: `"varchar"` + - Dense vector: e.g., `"vector,128,float"` + - `vector`: The column is a dense vector column. + - The second item in the string: The dimension of dense vector. + - The third item in the string: The element type of dense vector. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Sparse vector: e.g., `"sparse,128,float,int"` + - `sparse`: The column is a sparse vector column. + - The second item in the string: The dimension of sparse vector. + - The third item in the string: The element type of sparse vector. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - The fourth item in the string: The data type of sparse vector indices. Can be: + - `int8` + - `int16` + - `int`/`int32`/`integer` + - `int64` + - Tensor vector: e.g., `"tensor,4,float"` + - `tensor`: The column is a tensor column. + - The second item in the string: The dimension of each vector unit in a tensor. + - The third item in the string: The element type of tensors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Tensor array: e.g., `"tensorarray,6,float"` + - `tensorarray`: The column is a tensor-array column. + - The second item in the string: The dimension of each vector unit in a tensor array. + - The third item in the string: The element type of tensor arrays. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Multivector: e.g., `"multivector,128,float"` + - `multivector`: The column is a multi-vector column. + - The second item in the string: The dimension of each vector unit in a multivector. + - The third item in the string: The element type of multi-vectors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - Array: e.g., `"array,varchar"`, `"array,array,varchar"` + - `array`: The column is an array column. + - followed by the element type of the array. Can be recursive. + - for example, `"array,varchar"` is a one-dimensional array of strings, and `"array,array,varchar"` is a two-dimensional array of strings. +- **Default value** (`"default"`) + The default value for unspecified cells in that column. + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if a table with the same name exists. +- `Ignore`: Ignore the table creation request and keep the existing table with the same name. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` + +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +#### Returns + +- Success: An `infinity.remote_infinity.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException`: + - `error_code`: `int` - A non-zero value indicating a specific error condition. + - `error_msg`: `str` - A message providing additional details about the error. + +#### Examples + +##### Create a table with an integer column only + +```python +# The `create_table`method supports creating integer columns in the following data types: +# - int8 +# - int16 +# - int/int32/integer +# - int64 +db_object.create_table("my_table", {"c1": {"type": "int", "default": 1}}) +``` + +##### Create a table with a float column only + +```python +# The `create_table`method supports creating float columns in the following data types: +# - float/float32 +# - double/float64 +# - float16 +# - bfloat16 +db_object.create_table("my_table", {"c1": {"type": "float64"}}) +``` + +##### Create a table with a string column only + +```python +db_object.create_table("my_table", {"c1": {"type": "varchar"}}) +``` + +##### Create a table with a bool column only + +```python +db_object.create_table("my_table", {"c1": {"type": "bool"}}) +``` + +##### Create a table with a vector column only + +:::tip NOTE +You can build a HNSW index on the vector column to speed up the match_dense search. +::: + +```python +# Create a table with a vector column only: +# - `vector`: The column is a vector column +# - `128`: The vector dimension +# - `float`: The primitive data type of the vectors. Can be `float`/`float32`, `float16`, `bfloat16`, `uint8` or `int8` +db_object.create_table("my_table", {"c1": {"type": "vector,128,float"}}) + +``` + +##### Create a table with a multi-vector column only + +:::tip NOTE +You can build an HNSW index on the multi-vector column to accelerate match_dense search. +::: + +```python +# Create a table with a multi-vector column only: +# - `multivector`: The column is a multi-vector column +# - `128`: The basic vector dimension +# - `float`: The primitive data type of the basic vectors. Can be `float`/`float32`, `float16`, `bfloat16`, `uint8` or `int8` +db_object.create_table("my_table", {"c1": {"type": "multivector,128,float"}}) + +``` + +##### Create a table with a sparse vector column only + +:::tip NOTE +You can build a BMP index on the sparse vector column to speed up the match_sparse search. +::: + +```python +from infinity.common import ConflictType +# Create a table with a vector column only: +# - `sparse`: The column is a sparse vector column +# - `128`: The sparse vector dimension +# - `float`: The primitive data type of the sparse vectors. Can be `float`/`float32` or `double`/`float64` +# - `int`: The data type of the sparse vector indices. Can be `int8`, `int16`, `int`/`int32`/`integer`, or `int64` +db_object.create_table("my_table", {"c1": {"type": "sparse,128,float,int"}}, ConflictType.Error) +``` + +##### Create a table with a tensor column only + +```python +from infinity.common import ConflictType +# Create a table with a tensor column only: +# - `tensor`: The column is a tensor column +# - `4`: Dimension of each vector unit in the tensor +# - `float`: The primitive data type of the tensors. Can be `float`/`float32`, `float16`, `bfloat16` or `bit` +db_object.create_table("my_table", {"c1": {"type": "tensor,4,float"}}, ConflictType.Ignore) +``` + +##### Create a table with a tensor array column only + +```python +from infinity.common import ConflictType +# Create a table with a tensor array column only: +# - `tensorarray`: The column is a tensor array column +# - `6`: Dimension of each vector unit in the tensor arrays +# - `float`: The primitive data type of the tensor arrays. Can be `float`/`float32`, `float16`, `bfloat16` or `bit` +db_object.create_table("my_table", {"c1": {"type": "tensorarray,6,float"}}, ConflictType.Ignore) +``` + +--- + +### drop_table + +```python +db_object.drop_table(table_name, conflict_type = ConflictType.Error) +``` + +Deletes a table from the database by its name. + +#### Parameters + +##### table_name: `str`, *Required* + +A non-empty string indicating the name of the table to delete. + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if the specified table does not exist. +- `Ignore`: Ignore the operation and proceed regardless, if the specified table does not exist. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` + +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Delete a table named 'my_table': +# If the specified table does not exist, raise an error. +db_object.drop_table("my_table") +``` + +```python +# Delete a table named 'my_table': +# If the specified table does not exist, raise an error (same as above). +db_object.drop_table("my_table", infinity.common.ConflictType.Error) +``` + +```python +from infinity.common import ConflictType +# Delete a table named 'my_table': +# If the specified table does not exist, silently ignore the operation and proceed. +db_object.drop_table("my_table", ConflictType.Ignore) +``` + +--- + +### create_table_snapshot + +```python +db_object.create_table_snapshot(snapshot_name, table_name) +``` + +Creates a snapshot of the specified table with the given snapshot name. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the table snapshot. + +##### table_name: `str`, *Required* + +A non-empty string indicating the name of the table. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Create a snapshot named 'my_table_snapshot' of the table 'my_table'. +# If the specified snapshot already exists or the specified table does not exist, raise an error. +db_object.create_table_snapshot("my_table_snapshot", "my_table") +``` + +--- + +### create_database_snapshot + +```python +infinity_obj.create_database_snapshot(snapshot_name, database_name) +``` + +Creates a snapshot of the specified database with the given snapshot name. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the database snapshot. + +##### database_name: `str`, *Required* + +A non-empty string indicating the name of the database. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Create a snapshot named 'my_database_snapshot' of the database 'my_database'. +# If the specified snapshot already exists or the specified database does not exist, raise an error. +infinity_obj.create_database_snapshot("my_database_snapshot", "my_database") +``` + +--- + +### create_system_snapshot + +```python +infinity_obj.create_system_snapshot(snapshot_name) +``` + +Creates a snapshot of the system with the given snapshot name. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the system snapshot. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Create a snapshot named 'my_system_snapshot'. +# If the specified snapshot already exists, raise an error. +infinity_obj.create_system_snapshot("my_system_snapshot") +``` + +--- + +### restore_table_snapshot + +```python +db_object.restore_table_snapshot(snapshot_name) +``` + +Restores a table from the specified snapshot. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the table snapshot. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Restore a table from the snapshot 'my_table_snapshot'. +# If the specified snapshot does not exist, raise an error. +db_object.restore_table_snapshot("my_table_snapshot") +``` + +--- + +### restore_database_snapshot + +```python +infinity_obj.restore_database_snapshot(snapshot_name) +``` + +Restores a database from the specified snapshot. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the database snapshot. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Restore a database from the snapshot 'my_database_snapshot'. +# If the specified snapshot does not exist, raise an error. +infinity_obj.restore_database_snapshot("my_database_snapshot") +``` + +--- + +### restore_system_snapshot + +```python +infinity_obj.restore_system_snapshot(snapshot_name) +``` + +Restores a system from the specified snapshot. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the system snapshot. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples +```python +# Restore a system from the snapshot 'my_system_snapshot'. +# If the specified snapshot does not exist, raise an error. +infinity_obj.restore_system_snapshot("my_system_snapshot") +``` + +--- + +### drop_snapshot + +```python +infinity_obj.drop_snapshot(snapshot_name) +``` + +Deletes a snapshot with the specified name. + +#### Parameters + +##### snapshot_name: `str`, *Required* + +A non-empty string indicating the name of the snapshot. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Delete a snapshot named 'my_snapshot': +# If the specified snapshot does not exist, raise an error. +infinity_obj.drop_snapshot("my_snapshot") +``` + +--- + +### add_columns + +```python +table_object.add_columns(column_defs) +``` + +#### Parameters + +##### column_defs: `dict[str, dict[str, Any]]`, *Required* + +A dictionary defining the columns to add. Each key in the dictionary is a column name (`str`), with a corresponding 'value' dictionary defining the column's data type and default value. See the description of `create_table()`'s `columns_definition` for all available settings. + +:::caution NOTE +You must specify a default value each time you add a column. +::: + +#### Examples + +##### Add an Integer column, a float column, and a varchar/string column at once + +```python +table_obj.add_columns({"column_name1": {"type": "integer", "default": 0}, "column_name2": {"type": "float", "default": 0.0}, "column_name3": {"type": "varchar", "default": ""}}) +``` + +##### Add a dense vector column + +```python +table_obj.add_columns({"column_name1": {"type": "vector,4,float", "default": [1.0, 1.2, 2.4, 4.6]}}) +``` + +##### Add a multivector column + +```python +table_obj.add_columns({"column_name1": {"type": "multivector,4,float", "default": [[1.0, 0.0, 0.0, 0.0], [1.2, 0.0, 0.0, 0.0]]}}) +``` + +##### Add a tensor column + +```python +table_obj.add_columns({"column_name1": {"type": "tensor,4,float", "default": [[1.0, 0.0, 0.0, 0.0], [1.2, 0.0, 0.0, 0.0]]}}) +``` + +##### Add a sparse column + +```python +from infinity.common import SparseVector +table_obj.add_columns({"column_name1": {"type": "sparse,128,float,int", "default": SparseVector([10, 20, 30], [1.1, 2.2, 3.3])}}) +``` + +Or, you can set the default value in a different format: + +```python +table_obj.add_columns({"column_name1": {"type": "sparse,128,float,int", "default": {"10":1.1, "20":2.2, "30": 3.3}}}) +``` + +--- + +### drop_columns + +```python +table_object.drop_columns(column_names) +``` + +#### Parameters + +##### column_names: `list[str]`, *Required* + +A list of strings representing the names of the columns to drop. + +#### Examples + +##### Remove one column from the table + +```python +table_object.drop_columns(["column_name"]) +``` + +##### Remove two columns at once + +```python +table_object.drop_columns(["column_name1", "column_name2"]) +``` + +--- + +### get_table + +```python +db_object.get_table(table_name) +``` + +Retrieves a table object by its name. + +#### Parameters + +##### `table_name`: `str`, *Required* + +A non-empty string indicating the name of the table to retrieve. + +#### Returns + +- Success: An `infinity.remote_infinity.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException`: + - `error_code`: `int` - A non-zero value indicating a specific error condition. + - `error_msg`: `str` - A message providing additional details about the error. + +#### Examples + +```python +table_object = db_object.get_table("my_table") +``` + +--- + +### show_columns + +```python +table_object.show_columns() +``` + +Show the column definition of the current table. + +#### Returns + +An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. + +:::tip NOTE +This method specifies the projection columns for the current table but does not directly produce displayable data. To display the query results, use `output()` in conjunction with methods like `to_result()`, `to_df()`, `to_pl()`, or `to_arrow()` to materialize the data. +::: + +#### Examples + +```python +res = table_object.show_columns() +print(res) +``` + +--- + +### list_tables + +```python +db_object.list_tables() +``` + +Retrieves a list of all available tables within the current database. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. +- `table_names`: `list[str]` - A list of strings indicating the names of all available tables in the current database. + +#### Examples + +```python +res = db_object.list_tables() +res.table_names # ['my_table, 'tensor_table', 'sparse_table'] +``` + +--- + +### show_table + +```python +db_object.show_table(table_name) +``` + +Shows detailed information about a table. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +res = db_object.show_table('my_table') +``` + +--- + +### create_index + +```python +table_object.create_index(index_name, index_info, conflict_type = ConflictType.Error, comment = None) +``` + +Creates an index on a specified column. + +:::tip NOTE +Call `create_table()` or `get_table()` to get a database object for all index-specific operations. +::: + +#### Parameters + +##### index_name: `str` *Required* + +A non-empty string indicating the name of the index, which must adhere to the following requirements: + +- Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) +- Must begin with an English letter or underscore. +- Maximum 65,535 characters. +- Case-insensitive. + +##### index_info: `IndexInfo()`, *Required* + +An `IndexInfo` structure contains three fields,`column_name`, `index_type`, and `index_param_list`. + +- **column_name**: `str`, *Required* + The name of the column to build index on. Must not be empty. +- **index_type**: `IndexType`, *Required* + Index type. You may want to import `infinity.index` to set `IndexType`: `from infinity.index import IndexType` + - `Hnsw`: An HNSW index. Works with dense vectors, and multivectors only. + - `IVF`: An IVF index. Works with dense vectors and multivectors only. + - `FullText`: A full-text index. + - `Secondary`: A secondary index. Works with structured data only. + - `BMP`: A Block-Max Pruning index. Works with sparse vectors only. +- **index_param_list**: `dict[str, str]` + A dictionary specifying the parameter settings for the selected index type. Each key-value pair in the dictionary corresponds to a parameter and its value: + - Parameter settings for an HNSW index: + - `"M"`: *Optional* - Defaults to`"16"`. + - `"ef_construction"`: *Optional* - Defaults to`"50"`. + - `"metric"` *Required* - The distance metric to use in similarity search. + - `"ip"`: Inner product. + - `"l2"`: Euclidean distance. + - `"cosine"`: Cosine similarity. + - `"encode"`: *Optional* + - `"plain"`: (Default) Plain encoding. + - `"lvq"`: Locally-adaptive vector quantization. Works with float vector element only. + - `"build_type"`: *Optional* + - `"plain"`: (Default) Plain build. + - `"lsg"`: Local scaling graph. + - `"lsg_config"`: *Optional* - A dictionary with each key-value pair corresponding to a parameter setting of the local scaling graph: + - `"sample_raito"`: *Optional* - Defaults to `"0.01"`. The ratio of the number of samples in the dataset (base). + - `"ls_k"`: *Optional* - Defaults to `"10"`. The number of nearest neighbors of the selected knn mean in local scaling. + - `"alpha"`: *Optional* - Defaults to `"1.0"`. The smoothing factor in local scaling. + - Parameter settings for an IVF index: + - `"metric"` *Required* - The distance metric to use in a similarity search. + - `"ip"`: Inner product. + - `"l2"`: Euclidean distance. + - `"cosine"`: Cosine similarity. + - `"storage_type"`: *Optional* + - `"plain"`: (Default) Plain storage. + - `"scalar_quantization"`: Scalar quantization. + - `"product_quantization"`: Product quantization. + - `"plain_storage_data_type"`: *Optional* for plain storage. + - `"int8"`: default value for `int8` embeddings. + - `"uint8"`: default value for `uint8` embeddings. + - `"float32"`: default value for floating-point embeddings. + - `"float16"`: for floating-point embeddings. + - `"bfloat16"`: for floating-point embeddings. + - `"scalar_quantization_bits"`: *Required* for scalar quantization. Must be either `4` or `8`. + - `"product_quantization_subspace_num"`: *Required* for product quantization. Must be divisor of the embedding dimension. + - `"product_quantization_subspace_bits"`: *Required* for product quantization. Must be in the range `[4, 16]`. + - Parameter settings for a full-text index: + - `"ANALYZER"`: *Optional* + - `"standard"`: (Default) The standard analyzer, segmented by token, lowercase processing, and provides stemming output. Use `-` to specify the languages' stemmer. `English` is the default stemmer: `"standard-english"` and `"standard"` are the same stemmer setting. Supported language stemmers include: `Danish`, `Dutch`, `English`, `Finnish`, `French`, `German`, `Hungarian`, `Italian`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`, `Russian`, `Spanish`, `Swedish`, and `Turkish`. + - `"rag"`: Multilingual RAG analyzer imported from [RAGFlow](https://github.com/infiniflow/ragflow/blob/main/rag/nlp/rag_tokenizer.py), supporting `Chinese` and `English`. Use `-fine` to output the fine-grained analyzer results. + - `"ik"`: Bilingual analyzer imported from [ik-analyzer](https://github.com/infinilabs/analysis-ik), supporting `Chinese` and `English`. Use `-fine` to output the fine-grained analyzer results. + - `"chinese"`: Simplified Chinese. Use `-fine` to output the fine-grained analyzer results. + - `"traditional"`: Traditional Chinese. + - `"japanese"`: Japanese. + - `"korean"`: Korean. + - `"ngram"`: [N-gram](https://en.wikipedia.org/wiki/N-gram). + - `"keyword"`: "noop" analyzer used for columns containing keywords only. + - Parameter settings for a secondary index: + No parameters are required. For now, use an empty list `[]`. + - Parameter settings for a BMP index: + - `block_size`: *Optional* - The size of the block in a BMP index. Range: `"1"` ~ `"256"`. Defaults to `"16"`. + - `"compress_type"`: *Optional* + - `"compress"`: (Default) Store the block-max index in sparse format. Works best with small block size situations. + - `"raw"`: Store the block-max index without compression. + +:::tip NOTE +Import the `infinity.index` package to set `IndexInfo`, and `IndexType`. + +```python +from infinity.index import IndexInfo, IndexType +``` + +::: + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if an index with the same name exists. +- `Ignore`: Ignore the index creation request and keep the existing table with the same name. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` + +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +##### comment: `str`, *Optional* + +Additional comment for the index to create. + +#### Returns + +A structure containing these attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +##### Create an HNSW index on a dense vector column + +```python {1} +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_hnsw" with a 1024-dimensional float vector column "c1" +table_object = db_object.create_table("test_index_hnsw", {"c1": {"type": "vector,1024,float"}}) +# Create an HNSW index named "my_index" on column "c1" with default parameter settings: +# - "M": "16", +# - "ef_construction": "50", +# - "encode": "plain" +# Only the "metric" parameter (required) is explicitly set to L2 distance. +table_object.create_index("my_index",IndexInfo("c1", IndexType.Hnsw, {"metric": "l2"}), comment="Index One") +``` + +```python {1} +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_hnsw" with a 1024-dimensional float vector column "c1" +table_object = db_object.create_table("test_index_hnsw", {"c1": {"type": "vector,1024,float"}}) +# Create an HNSW index named "my_index" on column "c1" +# Settings for "M", "ef_construction", and "metric" are the same as above, except: +# "encoding" is set to "lvq" +table_object.create_index( + "my_index", + IndexInfo( + "c1", + IndexType.Hnsw, + { + "M": "16", + "ef_construction": "50", + "metric": "l2", + "encode": "lvq" + } + ) +) +``` + +##### Create an HNSW index on a multi-vector column + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_hnsw" with a 4-dimensional float multivector column "c1" +table_object = db_object.create_table("test_index_hnsw", {"c1": {"type": "multivector,4,float"}}) +# Create an HNSW index named "my_index" on column "c1" with default parameter settings: +# - "M": "16", +# - "ef_construction": "50", +# - "encode": "plain" +# Only the "metric" parameter (required) is explicitly set to L2 distance. +table_object.create_index("my_index", IndexInfo("c1", IndexType.Hnsw, {"metric": "l2"})) +``` + +##### Create a full-text index + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_fulltext" with a varchar column "body" +table_object = db_object.create_table("test_index_fulltext", {"body": {"type": "varchar"}}) +# Create a full-text index named "my_index" on column "body" with default parameter settings: +# - "ANALYZER": "standard" +table_object.create_index( + "my_index", + IndexInfo( + "body", + IndexType.FullText, + ) +) +``` + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_fulltext" with a varchar column "body" +table_object = db_object.create_table("test_index_fulltext", {"body": {"type": "varchar"}}) +# Create a full-text index named "my_index" on column "body" +# Setting "ANALYZER" to "standard" (same as the above) +table_object.create_index( + "my_index", + IndexInfo( + "body", + IndexType.FullText, + { + "ANALYZER": "standard" + } + ) +) +``` + +##### Create a secondary index + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_secondary" with a varchar column "body" +table_object = db_object.create_table("test_index_secondary", {"c1": {"type": "varchar"}}) +# Create a secondary index named "my_index" on column "c1" +table_object.create_index( + "my_index", + IndexInfo( + "c1", + IndexType.Secondary + ) +) +``` + +##### Create a BMP index + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_bmp" with a sparse vector column "c1" +table_object = db_object.create_table("test_index_bmp", {"c1": {"type": "sparse,30000,float,int16"}}) +# Create a BMP index named "my_index" on column "c1" with default parameter settings: +# - "block_size": "16" +# - "compress_type": "compress" +table_object.create_index( + "my_index", + IndexInfo( + "c1", + IndexType.BMP + ) +) +``` + +```python +from infinity.index import IndexInfo, IndexType +# Create a table named "test_index_bmp" with a sparse vector column "c1" +table_object = db_object.create_table("test_index_bmp", {"c1": {"type": "sparse,30000,float,int16"}}) +# Create a BMP index named "my_index" on column "c1" +# Settings for "block_size" and "compress_type" are the same as above +table_object.create_index( + "my_index", + IndexInfo( + "c1", + IndexType.BMP, + { + "block_size": "16", + "compress_type": "compress" + } + ) +) +``` + +--- + +### optimize_index + +```python +table_obj.optimize() +``` + +Optimize all indexes in a table. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Optimize all indexes in table +res = table_object.optimize() +``` + +--- + +--- + +### alter_index + +```python +table_obj.alter_index(index_name, opt_params: dict[str, str]) +``` + +alter an index by its name. + +#### Parameters + +##### index_name: `str`, *Required* + +A non-empty string indicating the name of the index to alter. + +##### opt_params: `dict[str, str]`, *Required* + +A dictionary specifying the alter index parameters for the selected index type. Each key-value pair in the dictionary corresponds to a parameter and its value: + +- Parameters settings for an HNSW index: + - `"compress_to_lvq"`: *Optional* - Defaults to `"false"`. Compress existing plain HNSW index to LVQ index. + - `"compress_to_rabitq"`: *Optional* - Defaults to `"false"`. Compress existing plain HNSW index to RaBitQ index. + - `"lvq_avg"`: *Optional* - Defaults to `"false"`. Calculate the average of the LVQ index. +- Parameters settings for an BMP index: + - `"topk"`: *Optional* - Alter bmp index for topk search. + - `"bp_reorder"`: *Optional* - Defaults to `"false"`. Reorder internal sequence of the bmp index by heuristic rules(binary partition). + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Alter BMP index my_index +res = table_object.alter_index("my_index", {"topk": "3", "bp_reorder" : ""}) +``` +--- + +--- + +### drop_index + +```python +table_object.drop_index(index_name, conflict_type = ConflictType.Error) +``` + +Deletes an index by its name. + +#### Parameters + +##### index_name: `str`, *Required* + +A non-empty string indicating the name of the index to delete. + +##### conflict_type: `ConflictType`, *Optional* + +- `Error`: Raise an error if an index with the specified name does not exist. +- `Ignore`: Ignore the index creation request if the index does not exist. + +:::tip NOTE +You may want to import the `infinity.common` package to set `ConflictType`: + +```python +from infinity.common import ConflictType +``` +::: + +:::tip NOTE +If `ConflictType` is not set, it defaults to `Error`. +::: + +#### Returns + +A structure containing these attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +table_object.drop_index("my_index") +``` + +--- + +### list_indexes + +```python +table_object.list_indexes() +``` + +Retrieves a list of all available indexes built on the current table. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. +- `table_names`: `list[str]` - A list of strings indicating the names of all available indexes. + +#### Examples + +```python +res = table_object.list_indexes() +res.index_names # ['my_index', 'tensor_index', 'sparse_index'] +``` + +--- + +### show_index + +```python +table_object.show_index(index_name) +``` + +Shows detailed information about an index. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +res = table_object.show_index('my_index') +``` + +--- + +### show_current_node + +```python +infinity.show_current_node() +``` + +Shows the role of the currently connected node. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. +- `role`: `str` + - `"admin"` + - `"standalone"` + - `"leader"` + - `"follower"` + - `"learner"`. + +#### Examples + +```python +res = infinity_object.show_current_row() +``` + +--- + +## DATA MANAGEMENT + +--- + +### insert + +```python +table_object.insert(data) +``` + +Inserts rows of data into the current table. + +#### Parameters + +##### data: `dict[str, Any]`, *Required* + +Data to insert. Infinity supports inserting multiple rows to a table at one time in the form of `dict[str, Any]` (one row) or `list[dict[str, Any]]` (multiple rows), with each key-value pair corresponding to a column name and a table cell value. + +:::tip NOTE +- When inserting incomplete rows of data, ensure that all un-inserted columns have default values when calling `create_table()`. Otherwise, an error will occur. +- You are not allowed to insert both complete and incomplete rows of data in one request. +For information about setting default column values, see `create_table()`. +::: + +:::tip NOTE +Batch row limit: 8,192. You are allowed to insert a maximum of 8,192 rows at once. +::: + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +##### Insert primitives + +```python +# Create a table with four primitive columns: +table_instance = db_instance.create_table("primitive_table", { + "c1": {"type": "int8", "default": 0}, + "c2": {"type": "int16", "default": 0}, + "c3": {"type": "int", "default": 0}, + "c4": {"type": "int32", "default": 0}, # Same as int + "c5": {"type": "integer", "default": 0}, # Same as int + "c6": {"type": "int64", "default": 0}, + "c7": {"type": "varchar"}, + "c8": {"type": "float", "default": 1.0}, + "c9": {"type": "float32", "default": 1.0}, # Same as float + "c10": {"type": "double", "default": 1.0}, + "c11": {"type": "float64", "default": 1.0}, # Same as double + "c12": {"type": "bool", "default": False}, +}) + +# Insert an incomplete row, with remaining cells defaulting to their column defaults: +table_instance.insert({"c1": 1, "c7": "Tom", "c12": True}) +``` + +##### Insert dense vectors + +```python +# Create a table with an integer column and a 3-d vector column: +table_object = db_object.create_table("vector_table", {"c1": {"type": "integer", "default": 2024}, "vector_column": {"type": "vector,3,float"}}) + +# Insert one incomplete row into the table: +# Note that the 'c1' cell defaults to 0. +table_object.insert({"vector_column": [1.1, 2.2, 3.3]}) + +# Insert two incomplete rows into the table: +# Note that the 'c1' cells default to 0. +table_object.insert([{"vector_column": [1.1, 2.2, 3.3]}, {"vector_column": [4.4, 5.5, 6.6]}]) +``` + +##### Insert vectors using FDE (Feature Dimension Expansion) + +```python +from infinity.common import FDE + +# Create a table with a 64-dimensional vector column: +table_object = db_object.create_table("fde_table", {"id": {"type": "integer"}, "embedding_col": {"type": "vector,64,float"}}) + +# Insert data using FDE function to generate embeddings from 2D tensor data: +table_object.insert([ + { + "id": 1, + "embedding_col": FDE( + tensor_data=[ + [1.0, 2.0, 3.0, 4.0], # Feature vector 1 + [0.5, 1.5, 2.5, 3.5], # Feature vector 2 + [2.0, 1.0, 4.0, 3.0] # Feature vector 3 + ], + target_dimension=64 # Output embedding dimension + ) + }, + { + "id": 2, + "vector_col": FDE( + tensor_data=[ + [2.5, 1.5, 3.5, 2.5], + [1.0, 3.0, 2.0, 4.0], + [0.8, 1.2, 2.8, 3.2], + [1.5, 2.5, 1.5, 2.5] + ], + target_dimension=64 + ) + } +]) +``` + +**FDE Parameters:** +- `tensor_data`: A 2D list (matrix) of numeric values representing input features. +- `target_dimension`: The desired output embedding dimension (must match the vector column dimension). + +**Benefits of FDE:** +- Dynamically generates embeddings from feature matrices. +- Eliminates the need to compute embeddings beforehand. +- Flexible input tensor dimensions. +- Automatic conversion to fixed-dimension vectors. + +##### Insert sparse vectors + +```python +from infinity.common import SparseVector +# Create a table with an integer column and a 100-d sparse vector column: +table_object = db_object.create_table("sparse_vector_table", {"c1": {"type": "integer"}, "sparse_column": {"type": "sparse,100,float,int"}}) + +# Insert one row into the table: +# `indices` specifies the corresponding indices to the values in `values`. +# Note that the second row sets "c1" as 2024 by default. +table_object.insert([{"c1": 2022, "sparse_column": SparseVector([10, 20, 30], [1.1, 2.2, 3.3])}, {"sparse_column": SparseVector([70, 80, 90], [7.7, 8.8, 9.9])}]) +``` + +##### Insert tensors + +```python +# Create a table with a tensor column: +table_object = db_object.create_table("tensor_table", {"c1": {"type": "integer", "default": 2024}, "tensor_column": {"type": "tensor,4,float"}}) + +# Insert one row into the table: +table_instance.insert([{"tensor_column": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]]}]) +``` + +##### Insert tensor arrays + +```python +# Creat a table with only one tensor array column: +table_object = db_object.create_table("tensor_array_table", {"tensor_array_column": {"type": "tensorarray,2,float"}}) +table_object.insert([{"tensor_array_column": [[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0]]]}]) +``` + +##### Insert arrays + +```python +from infinity.common import Array +# Creat a table with only one varchar array array column: +table_object = db_object.create_table("varchar_array_array_table", + {"varchar_array_array_column": {"type": "array,array,varchar"}}) +# Insert one row into the table: +table_object.insert([{"varchar_array_array_column": Array(Array("hello", "world"), Array("!"), Array())}]) +``` + +--- + +### import_data + +```python +table_object.import_data(filepath, import_options) +``` + +Imports data from a specified file into the current table. + +#### Parameters + +##### file_path: `str`, *Required* + +Absolute path to the file for export. + +##### import_options: `dict[str, bool | str]` + +Example: `{"header":True, "delimiter": "\t", file_type}` + +- **header**: `bool` + Whether to display table header or not. Works with **.csv** files only: + - `True`: Display table header. + - `False`: (Default) Do not display table header. + +- **delimiter**: `str`, *Optional*, Defaults to "," + Delimiter to separate columns. Works with **.csv** files only. + +- **file_type**: `str`, *Required* + The type of the imported file. Supported file types include: + - `csv` + - `json` + - `jsonl` + - `parquet` + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +##### Import a csv file + +```python +table_object.import_data(os.getcwd() + "/your_file.csv", {"header": False, "file_type": "csv", "delimiter": "\t"}) +``` + +##### Import a jsonl file + +```python +table_object.import_data(os.getcwd() + "/your_file.jsonl", {"file_type": "jsonl"}) +``` + +##### Import a parquet file + +```python +table_object.import_data(os.getcwd() + "/your_file.parquet", {"file_type": "parquet"}) +``` + +--- + +### export_data + +```python +table_object.export_data(filepath, export_options, columns = None) +``` + +Exports data in the current table to a specified file. + +#### Parameters + +##### file_path: `str`, *Required* + +Absolute path to the file for export. + +##### export_options: `dict[str, Any]`, *Required* + +Example: `{"header": False, "delimiter": "\t", "file_type": "jsonl", "offset": 2, "limit": 5}` + +- **header**: `bool`, *Optional* + Whether to display table header or not. Works with **.csv** files only: + - `True`: Display table header. + - `False`: (Default) Do not display table header. + +- **delimiter**: `str`, *Optional*, Defaults to "," + Delimiter to separate columns. Works with **.csv** files only. + +- **file_type**: `str`, *Required* + The type of the exported file. Supported file types include: + - `csv` + - `jsonl` + - `parquet` + +- **offset**: `int`, *Optional* + Index specifying the starting row for export. Usually used in conjunction with `limit`. If not specified, the file export starts from the first row. + +- **limit**: `int`, *Optional* + The maximum number of rows to export. Usually used in conjunction with `offset`. If the table's row count exceeds `offset` + `limit`, the excess rows are excluded from the export. + +- **row_limit**: `int`, *Optional* + Used when you have a large table and need to break the output file into multiple parts. This argument sets the row limit for each part. If you specify **test_export_file.csv** as the file name, the exported files will be named **test_export_file.csv**, **test_export_file.csv.part1**, **test_export_file.csv.part2**, and so on. + +##### columns: `[str]`, *Optional* + +Columns to export to the output file, for example, `["num", "name", "score"]`. If not specified, the entire table is exported. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +##### Export your table to a csv file + +```python +table_object.export_data(os.getcwd() + "/export_data.csv", {"header": True, "file_type": "csv", "delimiter": ",", "offset": 2, "limit": 7, "row_limit": 3}, ["num", "name", "score"]) +``` + +##### Export your table to a jsonl file + +```python +table_object.export_data(os.getcwd() + "/export_data.jsonl", {"file_type": "jsonl", "offset": 1, "limit": 8, "row_limit": 2}, ["num", "name", "score"]) +``` + +##### Export your table to a parquet file + +```python +table_object.export_data(os.getcwd() + "/export_data.parquet", {"file_type": "parquet") +``` + +--- + +### delete + +```python +table_object.delete(cond = None) +``` + +Deletes rows from the table based on the specified condition. + +#### Parameters + +##### cond: `str`, *Optional* + +A string that defines the condition for selecting rows to delete. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `True` for the rows that should be deleted. If `cond` is not specified or set to `None`, the method will delete all rows in the table. + +:::tip NOTE + +- The `cond` parameter currently supports 'and' and 'or' logical expressions only. +- `cond` must not be an empty string. +::: + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. +- `deleted_rows`: `int` + The number of deleted rows. + +#### Examples + +##### Remove all rows in the table + +```python +# Clear all data in the current table +table_object.delete() +``` + +```python +# Clear all data in the current table +table_object.delete(None) + +``` + +##### Conditional row deletion + +```python +# Create a table named "my_table" with two columns: +# - Integer column "c1" +# - Vector column "vec" +table_object = db_instance.create_table("my_table", {"c1": {"type": "integer"}, "vec": {"type": "vector,4,float"},}) +# Insert two rows of data into the "my_table" +table_object.insert([{"c1": 90, "vec": [1.0, 1.2, 0.8, 0.9],}, {"c1": 80, "vec": [4.0, 4.2, 4.3, 4.5],},]) +# Delete rows where "c1" equals 1 +table_object.delete("c1 = 90") +``` + +```python +# Create a table named "my_table" with one integer column "c1" +table_object = db_instance.create_table("my_table", {"c1": {"type": "integer"}}) +# Insert three rows of data into the "my_table" +table_object.insert([{"c1": 90}, {"c1": 80}, {"c1": 95}]) +# Delete rows where "c1" is between 70 and 90 (inclusive) +table_object.delete("c1 >= 70 and c1 <= 90") +``` + +--- + +### update + +```python +table_object.update(cond, data) +``` + +Searches for rows that match the specified condition and updates them accordingly. + +#### Parameters + +##### cond: `str`, *Required* + +A string that defines the condition for selecting rows to update. It represents a logical expression, a function, or any other form of conditional logic that evaluates to `True` for the rows that should be updated. If `cond` is not specified or set to `Null`, the method will update all rows in the table. + +##### data: `dict[str, Any]`, *Required* + +A non-empty dictionary where each key indicates a column name and each value indicates the new value for the corresponding cell. + +#### Returns + +A structure containing the following attributes: + +- `error_code`: `int` + - `0`: The operation succeeds. + - A non-zero value indicates a specific error condition. +- `error_msg`: `str` + When `error_code` is non-zero, `error_msg` provides additional details about the error. + +#### Examples + +```python +# Update rows where column "c1" equals 1, setting "c2" to 90 and "c3" to 900 +table_object.update("c1 = 1", {"c2": 90, "c3": 900}) +``` + +```python +# Update rows where column "c1" is greater than 2, setting "c2" to 100 and "c3" to 1,000 +table_object.update("c1 > 2", {"c2": 100, "c3": 1000}) +``` + +--- + +## SEARCH + +--- + +### output + +```python +table_object.output(columns) +``` + +This method allows you to customize the output of your query by selecting specific columns, applying aggregation functions, or performing arithmetic operations. + +#### Parameters + +##### columns: `list[str]`, *Required* + +A non-empty list of strings specifying the columns to include in the output. Each string in the list can represent: + +- A user-defined column name: The name of the column to include in the output, e.g., `"body"`. +- All user-defined columns: Use a wildcard `"*"` to select all columns. +- A special system column: system-generated columns include: + - `_row_id`: An automatically generated, unique identifier for each row in the table. It serves as a unique key for each row but does not necessarily correspond to the actual row number. When the data in a row is updated, the `_row_id` for that row is also changed to reflect the update. + - `_score`: A BM25 score used in full-text search. + - `_similarity`: Used by IP and cosine metrics in dense or sparse vector search. + - `_distance`: Used by L2 metric in dense vector search. +- An aggregation function: Apply an aggregation operation on specified columns. Supported aggregation functions include: + - `count` + - `min` + - `max` + - `sum` + - `avg` +- An arithmetic function: Apply an arithmetic operation on specified columns (e.g., `c1+5`). +- An unnest function: Unnest an array column to multiple rows (e.g., `unnest(c1)`). + +:::tip NOTE +The list must contain at least one element. Empty lists are not allowed. +::: + +#### Returns + +An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. + +:::tip NOTE +This method specifies the projection columns for the current table but does not directly produce displayable data. To display the query results, use `output()` in conjunction with methods like `to_result()`, `to_df()`, `to_pl()`, or `to_arrow()` to materialize the data. +::: + +#### Examples + +##### Select columns to display + +```python +# Select all columns +table_object.output(["*"]).to_pl() +``` + +```python +# Select columns "num" and "body" +table_object.output(["num", "body"]).to_df() +``` + +```python +# Select a system-generated column "_row_id" +table_object.output(["_row_id"]).to_pl() +``` + +##### Select unnest columns to + +```python +# Select column "c1" and unnest its cells +table_object.output(["unnest(c1)"]).to_pl() +``` + +##### Perform aggregation or arithmetic operations on selected columns + +```python +# Specify that the output should display the average value of all cells in column "c2" +table_object.output(["avg(c2)"]).to_pl() +``` + +```python +# Select column "c1" and request all cells in this column to be displayed with their original values increased by 5 +table_object.output(["c1+5"]).to_pl() +``` + +```python +# Specify that the output should display the result of an arithmetic operation combining two aggregation functions +table_object.output(["min(c1) + max(c2)"]).to_pl() +``` + +```python +# Specify that the output should display the row number of the current table +table_object.output(["count(*)"]).to_pl() +``` + +```python +# Select column "num" and request all cells in this column to be displayed with their original values divided by 10 +table_object.output(["num / 10"]).to_pl() +``` + +```python +# Select column "num" and display all its cells as absolute values +table_object.output(["abs(num)"]).to_pl() +``` + +```python +# Specify that the output should display the result of three multiplied by five +# Note that no columns are involved in this example! +# Either of the following works: +table_object.output(["3 * 5"]).to_pl() +``` + +--- + +### filter + +```python +table_object.filter(cond) +``` + +Creates a filtering condition expression for the current table. + +:::tip NOTE +This method creates a filtering condition for your query. To display the results, you must chain it with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### cond: `str`, *Required* + +A non-empty string representing the filter condition. It comprises one or multiple expressions combined by 'and', 'or' or 'not' logical operators, where each expression uses comparison operators to set criteria for keeping or removing rows. + +##### filter expressions in cond + +* `<`, `<=`, `>`, `>=`, `=`, `==`, `!=` expression +* `in` and `not in` expression +* `filter_fulltext` expression + - Similar to `match_text()`. + Usage: 'filter_fulltext(fields, matching_text)' or 'filter_fulltext(fields, matching_text, extra_options)' + - 'extra_options' is in the format of 'K1=V1;K2=V2;...;KN=VN', + where each 'K' represents a parameter name and 'V' represents its value + - Available parameters in 'extra_options': + - **'minimum_should_match'**: specifies how many clauses in the 'matching_text' should be satisfied at least. + It can be in the following format: + - Positive integer `N`: at least `N` clauses should be satisfied. + - Negative integer `-N`: at least (total clause count - `N`) clauses should be satisfied. + - Positive Percentage `N%`: at least `⌊total clause count * N%⌋` clauses should be satisfied. + - Negative Percentage `-N%`: at least `total clause count - ⌊total clause count * N%⌋` clauses should be satisfied. + - Combination `K K`, the requirement `V` applies, otherwise all the clauses should be satisfied + - Multiple combinations `K1= total clause count`, all the clauses should be satisfied. + Otherwise, we find the biggest `V` which is less than the total clause count and apply the correspondent `V`. + - **'default_field'** + - If `"fields"` is an empty string, this parameter specifies the default field to search on. + +##### Returns + +An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. + +:::tip NOTE +This method specifies a filtering condition for the rows in the current table but does not directly produce displayable data. To display the query results, use `filter()` in conjunction with methods like `to_result()`, `to_df()`, `to_pl()`, or `to_arrow()` to materialize the data. +::: + +#### Examples + +```python +table_object.output(["c1", "c2"]).filter("(-7 < c1 or 9 >= c1) and (c2 = 3)").to_pl() +``` + +```python +table_object.output(["*"]).filter("c2 = 3").to_pl() +``` + +```python +table_object.output(["*"]).filter("c1 not in (1, 2, 3) and (c2 + 1) in (1, 2, 3)").to_df() +``` + +```python +table_object.output(["*"]).filter("filter_fulltext('doc', 'first second', 'minimum_should_match=99%') and not num = 2").to_pl() +``` + +--- + +### sort + +```python +table_object.sort(sort_expression_list) +``` + +Creates a sort expression using `sort_expression_list`. + +#### Parameters + +##### sort_expression_list: `list`, *Required* + +An expression list defining how to sort the results. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +# Output results sorted by the `c2` expression in ascending order and the `c1` expression in descending order +table_obj.output(["c1", "c2"]).sort([["c2", SortType.Asc], ["c1", SortType.Desc]]).to_df() +``` + +--- + +### group_by + +```python +table_object.group_by(group_by_columns) +``` + +Creates a group-by expression using `group_by_columns`. + +#### Parameters + +##### group_by_columns: `list[str] | str`, *Required* + +A list of strings specifying the expression to group by. Each string in the list represents a column name or an expression. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +table_obj.output(["c1", "sum(c2)"]).group_by(["c1"]).to_df() +table_obj.output(["c1", "avg(c1)", "count(c2)", "min(c3)", "max(c4)"]).group_by(["c1", "c1+c2"]).to_df() +``` + +--- + +### having + +```python +table_object.having(expr) +``` + +Creates a filtering condition expression for the group-by result. + +#### Parameters + +##### having_expr: `str`, *Required* + +A string specifying the having expression. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +table_obj.output(["c1", "sum(c2)"]).group_by(["c1"]).having("sum(c2) > 10").to_df() +``` + +--- + +### limit + +```python +table_object.limit(limit_num) +``` + +Creates an expression to limit the number of the output rows to a maximum of `limit_num`. + +#### Parameters + +##### limit_num: `int`, *Required* + +An integer specifying the maximum number of output rows. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +# Limit the output row count to a maximum of two +table_instance.output(["num", "vec"]).limit(2).to_pl() +``` + +--- + +### offset + +```python +table_object.limit(limit_num).offset(offset_value) +``` + +Creates a limit expression with an offset value, setting the output to start from `offset_value` and limiting the row count to a maximum of `limit_num`. This method must be used in conjunction with `limit()`. + +#### Parameters + +##### offset_value: `int`, *Required* + +An integer specifying the offset position of the limit expression. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +# Limit the output row count not more than 2, start from position 1 +table_instance.output(["num", "vec"]).offset(1).limit(2).to_pl() +``` + +### option + +```python +table_object.option(option_dict) +``` + +Indicates some search options. + +#### Parameters + +##### option_dict: `dict`, *Required* + +A dictionary specifying the following search options: + +- **"total_hits_count"**: `bool`, *Optional* + - Must combine with limit expression. If `"total_hits_count"` is `True`, the query will output an extra result including total hits row count of the query. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python +# Limit the output row count not more than 2, start from position 1, output an extra result to indicate total hits row count +table_instance.output(["num", "vec"]).limit(2).offset(1).option({"total_hits_count": True}).to_pl() +``` + +--- + +### match_dense + +```python +table_object.match_dense(vector_column_name, embedding_data, embedding_data_type, distance_type, topn, knn_params = None) +``` + +Creates a dense vector search expression to identify the closest top n rows to the given dense vector. Suitable for working with dense vectors (dense embeddings) or multi-vectors (multiple dense embeddings in one row). + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### vector_column_name: `str`, *Required* + +A non-empty string indicating the name of the vector multi-vector column to search on. + +##### embedding_data: `list/np.ndarray`, *Required* + +The query vector data to compare against. This should be provided as a list or a one-dimensional NumPy array of numerical values. + +##### embedding_data_type: `str`, *Required* + +Specifies the data type of the embedding vector. Commonly used types (values) include: + +- `"float"` +- `"uint8"`. + +##### distance_type: `str`, *Required* + +The distance metric to use in similarity search. + +- `"ip"`: Inner product. +- `"l2"`: Euclidean distance. +- `"cosine"`: Cosine similarity. + +##### topn: `int`, *Required* + +An integer indicating the number of nearest neighbours to return. + +##### knn_params: `dict[str, str]`, *Optional* + +A dictionary representing additional KNN or ANN search parameters. + +- `"ef"`: `str`, Recommended value: one to ten times the value of `topn`. + - For example, if you set `topn` to `10`, you can set `"ef"` to `"50"`. + - If you set `"ef"` too high, search performance may worsen. + - If you do not set `"ef"` or set it to a value lower than `topn`, the search uses the `topn` value as the value for `"ef"`. +- `"threshold"`: `str`, *Optional* A threshold value for the search. + - For example, if you use the `"cosine"` distance metric and set `"threshold"` to `"0.5"`, the search will return only those rows with a cosine similarity greater than `0.5`. +- `"nprobe"`: `str`, *Optional* The number of cells to search for the IVF index. The default value is `"1"`. +- `"index_name"` : `str`, *Optional* The name of index on which you would like the database to perform query on. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +##### Perform a brute-force vector search + +```python +# Find the 100 nearest neighbors using Euclidean distance +# If no vector index is created on the column being queried, then the vector search defaults to a brute-force search. +# In such case, set `knn_params` to `None` or leave it blank. +table_object.output(["*"]).match_dense("vec", [0.1,0.2,0.3], "float", "l2", 100).to_pl() +``` + +:::caution NOTE +`knn_params` settings will not take effect here because no index has been created. +::: + +##### Perform a vector search in HNSW + +1. Ensure that you have successfully built an HNSW index. If uncertain, you can rebuild the index, setting `ConflictType` to `Ignore`. +2. Set the `ef` value as follows: + +```python +from infinity.index import IndexInfo, IndexType +table_object.create_index("my_index", IndexInfo("vec", IndexType.Hnsw, {"ef_construction": "50"})) +# Find the 2 nearest neighbors using cosine distance +# If an HNSW index is successfully built on the column being queried, then the vector search uses this index, +# regardless of whether `knn_params` is set. +# If you leave `knn_params` blank, the search uses the `topn` value as the value for `"ef"`. +table_object.output(["*"]).match_dense("vec", [1, 2, 3], "uint8", "cosine", 2).to_pl() +``` + +```python +from infinity.index import IndexInfo, IndexType +table_object.create_index("my_index", IndexInfo("vec", IndexType.Hnsw, {"ef_construction": "50"})) +# Find the 2 nearest neighbors using inner product distance +# If an HNSW index is successfully built on the column being queried, then the vector search uses this index, +# regardless of whether `knn_params` is set. +table_object.output(["*"]).match_dense("vec", [0.1,0.2,0.3], "float", "ip", 2, {"ef": "100"}).to_pl() +``` + +:::tip NOTE +If the HNSW index is not created successfully, the search will fall back to a brute-force search. +::: + +--- + +### FDE (Feature Dimension Expansion) with match_dense + +The `match_dense` method supports FDE (Feature Dimension Expansion) functionality by accepting `FDE` objects as the `embedding_data` parameter. This allows you to use 2D tensor input instead of pre-computed embeddings. + +```python +from infinity.common import FDE + +# Create an FDE object with tensor data and target dimension +fde_query = FDE(tensor_data, target_dimension) + +# Use the FDE object in match_dense +table_object.match_dense(vector_column_name, fde_query, embedding_data_type, distance_type, topn, knn_params) +``` + +Creates a dense vector search expression using FDE (Feature Dimension Expansion) function to identify the closest top n rows. The FDE function takes a 2D tensor as input and expands it to the target dimension for a vector search. + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### vector_column_name: `str`, *Required* + +A non-empty string indicating the name of the vector column to search on. + +##### query_tensor: `list[list[float]]`, *Required* + +A 2D array (matrix) of numeric values representing the input tensor data. Each inner list represents a feature vector. + +**Example**: `[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]` + +##### target_dimension: `int`, *Required* + +The target dimension for the output embedding vector. Must be a positive integer that matches the dimension of the vector column being searched. + +**Example**: `64` + +##### embedding_data_type: `str`, *Required* + +The data type of the embedding vector. Commonly used types include: + +- `"float"` or `"float32"` +- `"float16"` +- `"bfloat16"` +- `"uint8"` +- `"int8"` + +##### distance_type: `str`, *Required* + +The distance metric to use in a similarity search: + +- `"ip"`: Inner product +- `"l2"`: Euclidean distance +- `"cosine"`: Cosine similarity + +##### topn: `int`, *Required* + +An integer specifying the number of nearest neighbors to return. + +##### knn_params: `dict`, *Optional* + +Additional parameters for the KNN search. Defaults to `None`. + +- `"ef"`: `str` - Recommended value: one to ten times the `topn` value. +- `"threshold"`: `str` - A threshold value for the search. +- `"nprobe"`: `str` - The number of cells to search in the IVF index. + +#### Returns + +- `Table` object: You can chain this method with other methods to form a query. + +#### Examples + +##### Basic FDE search + +```python +# Perform FDE-based vector search with a 2D tensor input +query_tensor = [ + [1.0, 2.0, 3.0, 4.0], # Feature vector 1 + [0.5, 1.5, 2.5, 3.5], # Feature vector 2 + [2.0, 1.0, 4.0, 3.0], # Feature vector 3 +] + +table_object.output(["id", "name", "_similarity"]).match_dense("embedding_col", query_tensor, 64, "float", "cosine", 5).to_pl() +``` + +##### FDE search with different configurations + +```python +# Small tensor with different target dimension +small_tensor = [ + [1.0, 0.5], + [0.8, 1.2] +] + +table_object.output(["*"]).match_dense("vec_col", small_tensor, 128, "float", "ip", 10).to_pl() +``` + +##### FDE search with additional parameters + +```python +# FDE search with HNSW parameters +query_tensor = [[1.0, 2.0], [3.0, 4.0]] + +table_object.output(["*"]).match_dense("vec_col", query_tensor, 64, "float", "cosine", 5, {"ef": "50"}).to_pl() +``` + +:::tip NOTE +- The FDE function allows you to use 2D tensor input instead of pre-computed embeddings. +- The target dimension can be specified dynamically and must match the vector column dimension. +- All existing distance metrics and KNN parameters are supported. +- FDE is particularly useful when you have feature matrices that need to be transformed into embeddings for search. +::: + +--- + +### match_sparse + +```python +table_object.match_sparse(vector_column_name, sparse_data, distance_type, topn, opt_params) +``` + +Creates a sparse vector search expression to identify the top n closest rows to the given sparse vector. Suitable for working with sparse vectors (sparse embeddings). + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### vector_column_name: `str`, *Required* + +A non-empty string indicating the name of the column to query on. + +##### sparse_data: `SparseVector(list[int], list[int] | list[float])`, *Required* + +The query sparse vector data to compare against. The `sparse_data` parameter should be provided as a SparseVector object, which has two members: + +- `indices`: A list of the indices, each corresponding to a non-zero value in the sparse vector. +- `values`: A list of the corresponding values for each index in the `indices` list. + +:::tip NOTE +If you have a dictionary of indices and values, you can create a SparseVector object using the `SparseVector` class. For example: + +```python +from infinity.common import SparseVector +dic_sparse_vector = {"indices": [0, 10, 20], "values": [0.1, 0.2, 0.3]} +sparse_vector = SparseVector(**dic_sparse_vector) +``` + +::: + +##### distance_type: `str`, *Required* + + A non-empty string indicating the distance type for the search. Currently, only `"ip"` (inner product) is supported. + +##### topn: `int`, *Required* + +An integer indicating the number of nearest neighbours to return. + +##### opt_params: `dict[str, str]`, *Optional* + +A dictionary representing additional parameters for the sparse vector search. Following are parameters for the BMP index: + +- `"alpha"`: `str` + `"0.0"` ~ `"1.0"` (default: `"1.0"`) - A "Termination Conditions" parameter. The smaller the value, the more aggressive the pruning. +- `"beta"`: `str` + `"0.0"` ~ `"1.0"` (default: `"1.0"`) - A "Query Term Pruning" parameter. The smaller the value, the more aggressive the pruning. +- `"index_name"` : `str`, *Optional* The name of index on which you would like the database to perform query on. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +##### Perform a brute-force sparse vector search + +```python +# As demonstrated in the following example: +# The sparse vector search is performed on column "sparse_column" to find the 100 nearest neighbors using inner product +# SparseVector([0, 10, 20], [0.1, 0.2, 0.3]) represents the sparse vector to compare against: +# - 0: the index of 0.1 +# - 10: the index of 0.2 +# - 20: the index of 0.3 +# If no sparse vector index is created on the column being queried, then the search defaults to a brute-force search. +# In such case, set `opt_params` to `None` or leave it blank. +from infinity.common import SparseVector +table_object.output(["*"]).match_sparse('sparse_column', SparseVector([0, 10, 20], [0.1, 0.2, 0.3]), 'ip', 100).to_df() +``` + +:::caution NOTE +`opt_params` settings will not take effect here because no index has been created. +::: + +##### Perform a sparse vector search in BMP + +```python +from infinity.index import IndexInfo, IndexType +table_object.create_index("my_index", [IndexInfo("sparse_column", IndexType.BMP)]) +# Find the 100 nearest neighbors using inner product +# If a BMP index is successfully built on the column being queried, then the sparse vector search uses this index, +# regardless of whether `opt_params` is set. +# If you leave `opt_params` blank, the search takes the default settings for `"alpha"` and `"beta"`. +from infinity.common import SparseVector +table_object.output(["*"]).match_sparse('sparse_column', SparseVector([0, 10, 20], [0.1, 0.2, 0.3]), 'ip', 100, {"alpha": "1.0", "beta": "1.0"}).to_df() +``` + +```python +from infinity.index import IndexInfo, IndexType +table_object.create_index("my_index", IndexInfo("sparse_column", IndexType.BMP)) +# Find the 100 nearest neighbors using inner product +# If a BMP index is successfully built on the column being queried, then the sparse vector search uses this index, +# regardless of whether `opt_params` is set. +# You can set the values of `"alpha"` or `"beta"` in `opt_params`, which overrides the default settings. +from infinity.common import SparseVector +table_object.output(["*"]).match_sparse('sparse_column', SparseVector([0, 10, 20], [8, 10, 66]), 'ip', 100, {"alpha": "1.0", "beta": "1.0"}).to_df() +``` + +--- + +### match_text + +```python +table_object.match_text(fields, matching_text, topn, extra_options) +``` + +Creates a full-text search expression on the specified field(s)/column(s) to identify the most relevant rows. + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### fields: `str`, *Required* + +A non-empty, comma-separated string of column names on which the full-text search will be performed. + +:::danger NOTE +Ensure that a full-text index has been successfully built on each column involved before executing a full-text search; otherwise, an error will occur. +::: + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +##### matching_text: `str`, *Required* + +A non-empty text string to search for. You can use various search options within the matching text, including: + +- Single terms: `"blooms"` +- OR multiple terms: `"Bloom OR filter"` or just `"Bloom filter"` +- Phrase search: `'"Bloom filter"'` +- AND multiple terms: `"space AND efficient"` +- Escaping reserved characters: `"space\:efficient"` +- Sloppy phrase search: `'"harmful chemical"~10'` +- Field-specific search: `"title:(quick OR brown) AND body:foobar"` + +##### topn: `int`, *Required* + +Specifies the number of the most relevant rows to retrieve, e.g., assign `10` to obtain the ten most relevant rows. + +##### extra_options: `dict`, *Optional* + +An optional dictionary specifying the following search options: + +- **"default_field"**: `str`, *Optional* + - If `"fields"` is an empty string, this parameter specifies the default field to search on. +- **"operator"**: `str`, *Optional* + - If not specified, the search follows Infinity's full-text search syntax, meaning that logical and arithmetic operators, quotation marks and escape characters will function as full-text search operators, such as: + - AND operator: `AND` + - OR operator: `OR` + - NOT operator: `NOT` + - PAREN operator: `(`, `)`, need to appear in pairs, and can be nested. + - COLON operator: `:`: Used to specify field-specific search, e.g., `body:foobar` searches for `foobar` in the `body` field. + - CARAT operator: `^`: Used to boost the importance of a term, e.g., `quick^2 brown` boosts the importance of `quick` by a factor of 2, making it twice as important as `brown`. + - TILDE operator: `~`: Used for sloppy phrase search, e.g., `"harmful chemical"~10` searches for the phrase `"harmful chemical"` within a tolerable distance of 10 words. + - SINGLE_QUOTED_STRING: Used to search for a phrase, e.g., `'Bloom filter'`. + - DOUBLE_QUOTED_STRING: Used to search for a phrase, e.g., `"Bloom filter"`. + - Escape characters: Used to escape reserved characters, e.g., `space\:efficient`. Starting with a backslash `\` will escape the following characters: + `' '`, `'('`, `')'`, `'^'`, `'"'`, `'\''`, `'~'`, `'*'`, `'?'`, `':'`, `'\\'` + - If specified, Infinity's full-text search syntax will not take effect, and the specified operator will be interpolated into `matching_text`. + Useful for searching text including code numbers like `"A01-233:BC"`. + - `{"operator": "or"}`: Interpolates the `OR` operator between words in `matching_text` to create a new search text. + For example, reinterprets `"A01-233:BC"` as `'"A01" OR "-233" OR "BC"'`. + - `{"operator": "and"}`: Interpolates the `AND` operator between words in `matching_text` to create a new search text. + For example, reinterprets `"A01-233:BC"` as `'"A01" AND "-233" AND "BC"'`. + - **`"index_name"`** : `str`, *Optional* The names of indexes on which you would like the database to perform query on. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +```python + +questions = [ + r"blooms", # single term + r"Bloom filter", # OR multiple terms + r'"Bloom filter"', # phrase: adjacent multiple terms + r"space efficient", # OR multiple terms + r"space\-efficient", # Escape reserved character '-', equivalent to: `space efficient` + r'"space\-efficient"', # phrase and escape reserved character, equivalent to: `"space efficient"` + r'"harmful chemical"~10', # sloppy phrase, refers to https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query-phrase.html + r'title:(quick OR brown) AND body:foobar', # search `(quick OR brown)` in the `title` field. keep fields empty. +] +for question in questions: + table_object.output(["*"]).match_text('body', question, 2).to_df() + table_object.output(["*"]).match_text('', question, 2, {'default_field': 'body'}).to_df() +``` + +--- + +### highlight + +```python +table_object.highlight(columns) +``` + +This method allows you to highlight the words of the output of the query, when the word text are matched. + +#### Parameters + +##### columns: `list[str]`, *Required* + +A list of strings specifying the columns to include words match in match_text clause. Each string in the list can represent: + +- A user-defined column name: The name of the column to include in the output, e.g., `"body"`. + +:::tip NOTE +The columns must also be the `output` clause output and also be the match_text clause column. +::: + +#### Returns + +An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. + +:::tip NOTE +This method specifies the projection columns for the current table but does not directly produce displayable data. To display the query results, use `output()` in conjunction with methods like `to_result()`, `to_df()`, `to_pl()`, or `to_arrow()` to materialize the data. +::: + +#### Examples + +##### Highlight the matched column to display + +```python +table_obj.output(["doctitle", "docdate", "body"]).highlight(["body"]).match_text("body^5", "harmful chemical", 3).to_pl() +# The matched words will be enclosed in and +``` + +--- + +### fusion + +```python +table_object.fusion(method, topn, fusion_params = None) +``` + +Creates a reranking expression for multiple retrieval ways to identify the top n closest rows. + +:::tip NOTE +To display your query results, you must chain this method with `output(columns)`, which specifies the columns to output, and a method such as `to_pl()`, `to_df()`, or `to_arrow()` to format the query results. +::: + +#### Parameters + +##### method: `str`, *Required* + +A non-empty string indicating the reranking methods to use: + +- `"rrf"`: [Reciprocal rank fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) + RRF is a method for combining multiple result sets with varying relevance indicators into a single result set. It requires no tuning, and the relevance indicators need not be related to achieve high-quality results. RRF is particularly useful when you are uncertain of the relative importance of each retrieval way. + RRF uses the following formula to calculate the score for ranking each document: + + ```python + score = 0.0 + for q in queries: + if d in result(q): + score += 1.0 / ( k + rank( result(q), d ) ) + return score + + # Where + # k is the ranking constant, + # q is a query in a set of queries, + # d is a document in the result set of q, + # result(q) is the result set of q, and + # rank( result(q), d ) is the rank of d within the result(q), starting from 1. + ``` + +- `"weighted_sum"` + The weighted sum approach assigns different weights to different retrieval ways, allowing you to emphasize specific ways. This is particularly useful when you are certain of each path's relative importance. +- `"match_tensor"` + Infinity's tensor-based late interaction reranking approach. +- `"max"` + The max approach returns the maximum score from multiple retrieval ways. + +##### topn: `int`, *Required* + +An integer indicating the number of the most relevant rows to retrieve. + +##### fusion_params: `dict[str, Any]`, *Optional* + +A dictionary representing additional options for the selected reranking method: + +- **RRF-specific options**: *Optional* + Settings when employing RRF for reranking. + - `"rank_constant"`: The smoothing constant for RRF reranking, e.g., `{"rank_constant": 60}`. Defaults to `60`. + +- **weighted_sum-specific options**: *Optional* + Settings when employing Weighted Sum for reranking. + - `"weights"`: Specifies the weight for each retrieval way. For example, `{"weights": "1,2,0.5"}` sets weights of `1`, `2`, and `0.5` for the first, second, and third retrieval ways, respectively. The default weight of each retrieval way is `1.0`. If `"weight"` is not specified, all retrieval ways will be assigned the default weight of `1.0`. + - `"normalize"`: Specifies the normalization method for each retrieval way. Options: `"minmax"`, `"atan"`, `"l2"`, `"none"`. Defaults to `"minmax"`. + +- **max-specific options**: *Optional* + Settings when employing Max for reranking. + - `"normalize"`: Specifies the normalization method for each retrieval way. Options: `"minmax"`, `"atan"`, `"l2"`, `"none"`. Defaults to `"minmax"`. + +- **match_tensor-specific options**: *Optional* + Settings when employing match_tensor for reranking. + - `"field"`: The name of the tensor column for reranking. + - `"query_tensor"`: The tensor data to compare against. This should be provided as a list of lists or a two-dimensional NumPy + array of numerical values. + - `"element_type"`: The element data type of the query tensor. Usually `"float"`. + +#### Returns + +- Success: An `infinity.remote_thrift.table.RemoteTable` object in client-server mode. +- Failure: `InfinityException` + - `error_code`: `int` A non-zero value indicating a specific error condition. + - `error_msg`: `str` A message providing additional details about the error. + +#### Examples + +The following code snippets illustrate the use of fused reranking in a three-way retrieval. + +##### Use RRF for reranking + +```python {6} +from infinity.common import SparseVector +table_object.output(["num", "body", "vec", "sparse_column", "year", "tensor", "_score"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "cosine", 3) + .match_sparse("sparse_column", SparseVector([0, 20, 80], [1.0, 2.0, 3.0]), "ip", 3) + .match_text("body", "blooms", 10) + .filter("year < 2024") + .fusion("rrf", 2) + .to_pl() +``` + +```python {6} +from infinity.common import SparseVector +table_object.output(["num", "body", "vec", "sparse_column", "year", "tensor", "_score"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "cosine", 3) + .match_sparse("sparse_column", SparseVector([0, 20, 80], [1.0, 2.0, 3.0]), "ip", 3) + .match_text("body", "blooms", 10) + .filter("year < 2024") + .fusion("rrf", 2, {"rank_constant": 30}) + .to_pl() +``` + +##### Use Weighted Sum for reranking + +```python {6} +from infinity.common import SparseVector +table_object.output(["num", "body", "vec", "sparse_column", "year", "tensor", "_score"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "cosine", 3) + .match_sparse("sparse_column", SparseVector([0, 20, 80], [1.0, 2.0, 3.0]), "ip", 3) + .match_text("body", "blooms", 10) + .filter("year < 2024") + .fusion("weighted_sum", 2, {"weights": "1,2,0.5"}) + .to_pl() +``` + +##### Use tensor reranking + +```python {8} +from infinity.common import SparseVector +table_object.output(["num", "body", "vec", "sparse_column", "year", "tensor", "_score"]) + .match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "cosine", 3) + .match_sparse("sparse_column", SparseVector([0, 20, 80], [1.0, 2.0, 3.0]), "ip", 3) + .match_text("body", "blooms", 10) + .filter("year < 2024") + .fusion("match_tensor", 2, {"field": "tensor", "element_type": "float", "query_tensor": [[0.0, -10.0, 0.0, 0.7], [9.2, 45.6, -55.8, 3.5]]}) + .to_pl() +``` + +--- + +### to_result + + +```python +table_object.to_result() +``` + +Returns the query result as a tuple. + +:::tip NOTE +Call `to_result()` in a chain after (not necessarily "immediately after") `output(columns)` on the same table object. +::: + +:::caution NOTE +We recommend calling `to_df()`, `to_pl()`, or `to_arrow()` to format your results. +::: + +#### Returns + +A `tuple[dict[str, list[Any]], dict[str, Any]], {}` object + +### to_df + +```python +table_object.to_df() +``` + +Returns the query result as a tuple consisting of a pandas DataFrame and a dict. + +:::tip NOTE +Call `to_df()` in a chain after (not necessarily "immediately after") `output(columns)` on the same table object. +::: + +#### Returns + +A `tuple[pandas.DataFrame, {}]` object + +#### Examples + +```python +# Format columns "c1" and C2" of the current table into a pandas DataFrame +res, extra_res = table_object.output(["c1", "c2"]).to_df() +``` + +### to_pl + +```python +table_object.to_pl() +``` + +Returns the query result as a tuple consisting of a Polars DataFrame and a dict. + +:::tip NOTE +Call `to_pl()` in a chain after (not necessarily "immediately after") `output(columns)` on the same table object. +::: + +#### Returns + +A `tuple[polas.DataFrame, {}]` object. + +#### Examples + +```python +# Format a vector search result into a Polars DataFrame. +res, extra_res = table_object.output(["*"]).match_dense("vec", [3.0, 2.8, 2.7, 3.1], "float", "ip", 10).to_pl() +``` + +### to_arrow + +```python +table_object.to_arrow() +``` + +Returns the query result as a tuple consisting of an Apache Arrow Table and a dict. + +:::tip NOTE +Call `to_arrow()` in a chain after (not necessarily "immediately after") `output(columns)` on the same table object. +::: + +#### Returns + +A `tuple[pyarrow.Table, {}]` object. + +#### Examples + +```python +# Format the current table object into an Apache Arrow Table. +res, extra_result = table_object.output(["*"]).filter("score >= 90").to_arrow() +``` + +--- diff --git a/website/versioned_docs/version-v0.7.0-dev1/release_notes.md b/website/versioned_docs/version-v0.7.0-dev1/release_notes.md new file mode 100644 index 0000000..3e627b6 --- /dev/null +++ b/website/versioned_docs/version-v0.7.0-dev1/release_notes.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +slug: /release_notes +--- + +# Release notes + +Key features, improvements and bug fixes in the latest releases. + +## v0.5.0 + +Released on December 18, 2024. + +### New features + +- Implements a highly available architecture based on shared storage, featuring one write node and multiple read nodes. +- Implements 8-bit scalar quantization and product quantization in the IVF index. +- Supports using Hamming distance as metric when matching/searching binary vectors. +- Implements query result caching and pagination. +- Supports specifying comments when creating database or index. +- Supports regular expressions (regex), including md5，substring, lower, upper, ltrim, rtrim, trim, char_posiition, sqrt, round, ceil, floor, IN, isnan, isinf, and isfinite +- Enhances the search performance for full-text searches with multiple terms/clauses. +- Supports using `MinimumShouldMatch` as a full-text filter. +- When using BM25 for full-text search or using L2/IP/Cosine distance metric for dense vector, sparse vector, or tensor search, you can use the `threshold` option to filter results. +- Supports ARM64 CPUs. +- Tokenizers: Adds an IK analyzer and enhances the performance of the RAG analyzer. +- Integrated into RAGFlow. + +### Documentation + +#### Added documents + +- [Conduct a search](https://infiniflow.org/docs/dev/search_guide) +- [Set up an Infinity cluster](https://infiniflow.org/docs/dev/set_up_cluster) \ No newline at end of file diff --git a/website/versioned_sidebars/version-v0.7.0-dev1-sidebars.json b/website/versioned_sidebars/version-v0.7.0-dev1-sidebars.json new file mode 100644 index 0000000..ab07bdc --- /dev/null +++ b/website/versioned_sidebars/version-v0.7.0-dev1-sidebars.json @@ -0,0 +1,3 @@ +{ + "tutorialSidebar": [{"type": "autogenerated", "dirName": "."}] +} diff --git a/website/versions.json b/website/versions.json index e2ec52b..62857a3 100644 --- a/website/versions.json +++ b/website/versions.json @@ -1,4 +1,4 @@ [ - "v0.6.15", - "v0.5.2" + "v0.7.0-dev1", + "v0.6.15" ] \ No newline at end of file