Skip to content

Commit a14c19d

Browse files
Guikingonechr-hertel
Guikingone
authored and
chr-hertel
committed
[Store] Add SurrealDB
1 parent d022fa1 commit a14c19d

File tree

7 files changed

+614
-1
lines changed

7 files changed

+614
-1
lines changed

‎examples/.env

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,5 +83,10 @@ MEILISEARCH_API_KEY=changeMe
8383
LMSTUDIO_HOST_URL=http://127.0.0.1:1234
8484

8585
# Qdrant
86-
QDRANT_HOST='http://127.0.0.1:6333'
86+
QDRANT_HOST=http://127.0.0.1:6333
8787
QDRANT_SERVICE_API_KEY=changeMe
88+
89+
# SurrealDB
90+
SURREALDB_HOST=http://127.0.0.1:8000
91+
SURREALDB_USER=symfony
92+
SURREALDB_PASS=symfony

‎examples/compose.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,11 @@ services:
2929
QDRANT__SERVICE__API_KEY: '${QDRAT_SERVICE_API_KEY:-changeMe}'
3030
ports:
3131
- '6333:6333'
32+
33+
surrealdb:
34+
image: surrealdb/surrealdb:v2
35+
command: ['start', '--user', 'symfony', '--pass', 'symfony']
36+
environment:
37+
SURREAL_HTTP_MAX_KEY_BODY_SIZE: 49152
38+
ports:
39+
- '8000:8000'

‎examples/store/surrealdb-similarity-search.php

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Agent\Toolbox\AgentProcessor;
14+
use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
15+
use Symfony\AI\Agent\Toolbox\Toolbox;
16+
use Symfony\AI\Fixtures\Movies;
17+
use Symfony\AI\Platform\Bridge\OpenAI\Embeddings;
18+
use Symfony\AI\Platform\Bridge\OpenAI\GPT;
19+
use Symfony\AI\Platform\Bridge\OpenAI\PlatformFactory;
20+
use Symfony\AI\Platform\Message\Message;
21+
use Symfony\AI\Platform\Message\MessageBag;
22+
use Symfony\AI\Store\Bridge\SurrealDB\Store;
23+
use Symfony\AI\Store\Document\Metadata;
24+
use Symfony\AI\Store\Document\TextDocument;
25+
use Symfony\AI\Store\Document\Vectorizer;
26+
use Symfony\AI\Store\Indexer;
27+
use Symfony\Component\HttpClient\HttpClient;
28+
use Symfony\Component\Uid\Uuid;
29+
30+
require_once dirname(__DIR__).'/bootstrap.php';
31+
32+
// initialize the store
33+
$store = new Store(
34+
httpClient: HttpClient::create(),
35+
endpointUrl: env('SURREALDB_HOST'),
36+
user: env('SURREALDB_USER'),
37+
password: env('SURREALDB_PASS'),
38+
namespace: 'default',
39+
database: 'movies',
40+
table: 'movies',
41+
);
42+
43+
// initialize the table
44+
$store->initialize();
45+
46+
// create embeddings and documents
47+
$documents = [];
48+
foreach (Movies::all() as $i => $movie) {
49+
$documents[] = new TextDocument(
50+
id: Uuid::v4(),
51+
content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
52+
metadata: new Metadata($movie),
53+
);
54+
}
55+
56+
// create embeddings for documents
57+
$platform = PlatformFactory::create($_SERVER['OPENAI_API_KEY']);
58+
$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
59+
$indexer = new Indexer($vectorizer, $store);
60+
$indexer->index($documents);
61+
62+
$model = new GPT(GPT::GPT_4O_MINI);
63+
64+
$similaritySearch = new SimilaritySearch($platform, $embeddings, $store);
65+
$toolbox = new Toolbox([$similaritySearch], logger: logger());
66+
$processor = new AgentProcessor($toolbox);
67+
$agent = new Agent($platform, $model, [$processor], [$processor]);
68+
69+
$messages = new MessageBag(
70+
Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
71+
Message::ofUser('Which movie fits the theme of technology?')
72+
);
73+
$response = $agent->call($messages);
74+
75+
echo $response->getContent().\PHP_EOL;

‎src/store/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ CHANGELOG
4343
- ChromaDB
4444
- Pinecone
4545
- Qdrant
46+
- SurrealDB
4647
* Add Retrieval Augmented Generation (RAG) support:
4748
- Document embedding storage
4849
- Similarity search for relevant documents

‎src/store/doc/index.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ You can find more advanced usage in combination with an Agent using the store fo
4141
* `Similarity Search with MongoDB (RAG)`_
4242
* `Similarity Search with Pinecone (RAG)`_
4343
* `Similarity Search with Meilisearch (RAG)`_
44+
* `Similarity Search with SurrealDB (RAG)`_
4445
* `Similarity Search with memory storage (RAG)`_
4546
* `Similarity Search with Qdrant (RAG)`_
4647

@@ -54,6 +55,7 @@ Supported Stores
5455
* `Pinecone`_ (requires `probots-io/pinecone-php` as additional dependency)
5556
* `Postgres`_ (requires `ext-pdo`)
5657
* `Meilisearch`_
58+
* `SurrealDB`_
5759
* `InMemory`_
5860
* `Qdrant`_
5961

@@ -93,6 +95,7 @@ This leads to a store implementing two methods::
9395
.. _`Similarity Search with MongoDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/mongodb-similarity-search.php
9496
.. _`Similarity Search with Pinecone (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/pinecone-similarity-search.php
9597
.. _`Similarity Search with Meilisearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/meilisearch-similarity-search.php
98+
.. _`Similarity Search with SurrealDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/surrealdb-similarity-search.php
9699
.. _`Similarity Search with memory storage (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/memory-similarity-search.php
97100
.. _`Similarity Search with Qdrant (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/qdrant-similarity-search.php
98101
.. _`Azure AI Search`: https://azure.microsoft.com/products/ai-services/ai-search
@@ -102,6 +105,7 @@ This leads to a store implementing two methods::
102105
.. _`Pinecone`: https://www.pinecone.io/
103106
.. _`Postgres`: https://www.postgresql.org/about/news/pgvector-070-released-2852/
104107
.. _`Meilisearch`: https://www.meilisearch.com/
108+
.. _`SurrealDB`: https://surrealdb.com/
105109
.. _`InMemory`: https://www.php.net/manual/en/language.types.array.php
106110
.. _`Qdrant`: https://qdrant.tech/
107111
.. _`GitHub`: https://github.com/symfony/ai/issues/16

‎src/store/src/Bridge/SurrealDB/Store.php

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Store\Bridge\SurrealDB;
13+
14+
use Symfony\AI\Platform\Vector\NullVector;
15+
use Symfony\AI\Platform\Vector\Vector;
16+
use Symfony\AI\Store\Document\Metadata;
17+
use Symfony\AI\Store\Document\VectorDocument;
18+
use Symfony\AI\Store\Exception\InvalidArgumentException;
19+
use Symfony\AI\Store\Exception\RuntimeException;
20+
use Symfony\AI\Store\InitializableStoreInterface;
21+
use Symfony\AI\Store\VectorStoreInterface;
22+
use Symfony\Component\Uid\Uuid;
23+
use Symfony\Contracts\HttpClient\HttpClientInterface;
24+
25+
/**
26+
* @author Guillaume Loulier <[email protected]>
27+
*/
28+
final class Store implements InitializableStoreInterface, VectorStoreInterface
29+
{
30+
private string $authenticationToken = '';
31+
32+
public function __construct(
33+
private readonly HttpClientInterface $httpClient,
34+
private readonly string $endpointUrl,
35+
#[\SensitiveParameter] private readonly string $user,
36+
#[\SensitiveParameter] private readonly string $password,
37+
#[\SensitiveParameter] private readonly string $namespace,
38+
#[\SensitiveParameter] private readonly string $database,
39+
private readonly string $table = 'vectors',
40+
private readonly string $vectorFieldName = '_vectors',
41+
private readonly string $strategy = 'cosine',
42+
private readonly int $embeddingsDimension = 1536,
43+
private readonly bool $isNamespacedUser = false,
44+
) {
45+
}
46+
47+
public function add(VectorDocument ...$documents): void
48+
{
49+
foreach ($documents as $document) {
50+
$this->request('POST', \sprintf('key/%s', $this->table), $this->convertToIndexableArray($document));
51+
}
52+
}
53+
54+
public function query(Vector $vector, array $options = [], ?float $minScore = null): array
55+
{
56+
$vectors = json_encode($vector->getData());
57+
58+
$results = $this->request('POST', 'sql', \sprintf(
59+
'SELECT id, %s, _metadata, vector::similarity::%s(%s, %s) AS distance FROM %s WHERE %s <|2|> %s;',
60+
$this->vectorFieldName, $this->strategy, $this->vectorFieldName, $vectors, $this->table, $this->vectorFieldName, $vectors,
61+
));
62+
63+
return array_map($this->convertToVectorDocument(...), $results[0]['result']);
64+
}
65+
66+
public function initialize(array $options = []): void
67+
{
68+
$this->authenticate();
69+
70+
$this->request('POST', 'sql', \sprintf(
71+
'DEFINE INDEX %s_vectors ON %s FIELDS %s MTREE DIMENSION %d DIST %s TYPE F32',
72+
$this->table, $this->table, $this->vectorFieldName, $this->embeddingsDimension, $this->strategy
73+
));
74+
}
75+
76+
/**
77+
* @param array<string, mixed>|string $payload
78+
*
79+
* @return array<string|int, mixed>
80+
*/
81+
private function request(string $method, string $endpoint, array|string $payload): array
82+
{
83+
$url = \sprintf('%s/%s', $this->endpointUrl, $endpoint);
84+
85+
$finalPayload = [
86+
'json' => $payload,
87+
];
88+
89+
if (\is_string($payload)) {
90+
$finalPayload = [
91+
'body' => $payload,
92+
];
93+
}
94+
95+
$response = $this->httpClient->request($method, $url, array_merge($finalPayload, [
96+
'headers' => [
97+
'Accept' => 'application/json',
98+
'Content-Type' => 'application/json',
99+
'Surreal-NS' => $this->namespace,
100+
'Surreal-DB' => $this->database,
101+
'Authorization' => \sprintf('Bearer %s', $this->authenticationToken),
102+
],
103+
]));
104+
105+
return $response->toArray();
106+
}
107+
108+
/**
109+
* @return array<string, mixed>
110+
*/
111+
private function convertToIndexableArray(VectorDocument $document): array
112+
{
113+
return [
114+
'id' => $document->id->toRfc4122(),
115+
$this->vectorFieldName => $document->vector->getData(),
116+
'_metadata' => array_merge($document->metadata->getArrayCopy(), [
117+
'_id' => $document->id->toRfc4122(),
118+
]),
119+
];
120+
}
121+
122+
/**
123+
* @param array<string, mixed> $data
124+
*/
125+
private function convertToVectorDocument(array $data): VectorDocument
126+
{
127+
$id = $data['_metadata']['_id'] ?? throw new InvalidArgumentException('Missing "id" field in the document data');
128+
129+
$vector = !\array_key_exists($this->vectorFieldName, $data) || null === $data[$this->vectorFieldName]
130+
? new NullVector()
131+
: new Vector($data[$this->vectorFieldName]);
132+
133+
unset($data['_metadata']['_id']);
134+
135+
return new VectorDocument(
136+
id: Uuid::fromString($id),
137+
vector: $vector,
138+
metadata: new Metadata($data['_metadata']),
139+
);
140+
}
141+
142+
private function authenticate(): void
143+
{
144+
if ('' !== $this->authenticationToken) {
145+
return;
146+
}
147+
148+
$authenticationPayload = [
149+
'user' => $this->user,
150+
'pass' => $this->password,
151+
];
152+
153+
if ($this->isNamespacedUser) {
154+
$authenticationPayload['ns'] = $this->namespace;
155+
$authenticationPayload['db'] = $this->database;
156+
}
157+
158+
$authenticationResponse = $this->httpClient->request('POST', \sprintf('%s/signin', $this->endpointUrl), [
159+
'headers' => [
160+
'Accept' => 'application/json',
161+
],
162+
'json' => $authenticationPayload,
163+
]);
164+
165+
$payload = $authenticationResponse->toArray();
166+
167+
if (!\array_key_exists('token', $payload)) {
168+
throw new RuntimeException('The SurrealDB authentication response does not contain a token.');
169+
}
170+
171+
$this->authenticationToken = $payload['token'];
172+
}
173+
}

0 commit comments

Comments
 (0)