Skip to content

Commit c14093b

Browse files
author
Sam Partee
authored
Introduce Pydantic schema for index (#15)
Change from manual parsing to Pydantic for schema validation for the fields of the index. This makes it much easier to support having new additions that come with future releases of RediSearch.
1 parent bff17f6 commit c14093b

File tree

17 files changed

+484
-333
lines changed

17 files changed

+484
-333
lines changed

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
__pycache__/
22
redisvl.egg-info/
3-
.coverage
3+
.coverage
4+
scratch

conftest.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,10 @@
66
get_async_redis_connection,
77
get_redis_connection
88
)
9+
REDIS_ADDRESS = os.getenv("REDIS_ADDRESS", "redis://localhost:6379")
910

10-
HOST = os.environ.get("REDIS_HOST", "localhost")
11-
PORT = os.environ.get("REDIS_PORT", 6379)
12-
USER = os.environ.get("REDIS_USER", "default")
13-
PASS = os.environ.get("REDIS_PASSWORD", "")
14-
15-
aredis = get_async_redis_connection(HOST, PORT, PASS)
16-
redis = get_redis_connection(HOST, PORT, PASS)
11+
aredis = get_async_redis_connection(REDIS_ADDRESS)
12+
redis = get_redis_connection(REDIS_ADDRESS)
1713

1814

1915
@pytest.fixture

examples/01-getting-started.ipynb

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"cells": [
33
{
4+
"attachments": {},
45
"cell_type": "markdown",
56
"metadata": {},
67
"source": [
@@ -21,6 +22,7 @@
2122
]
2223
},
2324
{
25+
"attachments": {},
2426
"cell_type": "markdown",
2527
"metadata": {},
2628
"source": [
@@ -31,7 +33,7 @@
3133
},
3234
{
3335
"cell_type": "code",
34-
"execution_count": 2,
36+
"execution_count": 1,
3537
"metadata": {},
3638
"outputs": [],
3739
"source": [
@@ -49,6 +51,7 @@
4951
]
5052
},
5153
{
54+
"attachments": {},
5255
"cell_type": "markdown",
5356
"metadata": {},
5457
"source": [
@@ -61,7 +64,7 @@
6164
},
6265
{
6366
"cell_type": "code",
64-
"execution_count": 3,
67+
"execution_count": 2,
6568
"metadata": {},
6669
"outputs": [
6770
{
@@ -133,7 +136,7 @@
133136
"2 b'fff?fff?\\xcd\\xcc\\xcc=' "
134137
]
135138
},
136-
"execution_count": 3,
139+
"execution_count": 2,
137140
"metadata": {},
138141
"output_type": "execute_result"
139142
}
@@ -148,6 +151,7 @@
148151
]
149152
},
150153
{
154+
"attachments": {},
151155
"cell_type": "markdown",
152156
"metadata": {},
153157
"source": [
@@ -157,6 +161,7 @@
157161
]
158162
},
159163
{
164+
"attachments": {},
160165
"cell_type": "markdown",
161166
"metadata": {},
162167
"source": [
@@ -179,7 +184,7 @@
179184
},
180185
{
181186
"cell_type": "code",
182-
"execution_count": 4,
187+
"execution_count": 3,
183188
"metadata": {},
184189
"outputs": [],
185190
"source": [
@@ -191,24 +196,22 @@
191196
" \"storage_type\": \"hash\",\n",
192197
" },\n",
193198
" \"fields\": {\n",
194-
" # key is the field type\n",
195-
" # value is the name of the column in the dataset(frame)\n",
196-
" \"tag\": {\"credit_score\": {}},\n",
197-
" \"text\": {\"job\": {}},\n",
198-
" \"numeric\": {\"age\": {}},\n",
199-
" \"vector\": {\n",
200-
" \"user_embedding\": {\n",
199+
" \"tag\": [{\"name\": \"credit_score\"}],\n",
200+
" \"text\": [{\"name\": \"job\"}],\n",
201+
" \"numeric\": [{\"name\": \"age\"}],\n",
202+
" \"vector\": [{\n",
203+
" \"name\": \"user_embedding\",\n",
201204
" \"dims\": 3,\n",
202205
" \"distance_metric\": \"cosine\",\n",
203206
" \"algorithm\": \"flat\",\n",
204-
" \"datatype\": \"float32\",\n",
205-
" }\n",
206-
" },\n",
207+
" \"datatype\": \"float32\"}\n",
208+
" ]\n",
207209
" },\n",
208-
"}"
210+
"}\n"
209211
]
210212
},
211213
{
214+
"attachments": {},
212215
"cell_type": "markdown",
213216
"metadata": {},
214217
"source": [
@@ -221,7 +224,7 @@
221224
},
222225
{
223226
"cell_type": "code",
224-
"execution_count": 5,
227+
"execution_count": 4,
225228
"metadata": {},
226229
"outputs": [],
227230
"source": [
@@ -231,13 +234,33 @@
231234
"index = SearchIndex.from_dict(schema)\n",
232235
"\n",
233236
"# connect to local redis instance\n",
234-
"index.connect(\"localhost\", 6379)\n",
237+
"index.connect(\"redis://localhost:6379\")\n",
235238
"\n",
236239
"# create the index (no data yet)\n",
237240
"index.create()"
238241
]
239242
},
240243
{
244+
"cell_type": "code",
245+
"execution_count": 5,
246+
"metadata": {},
247+
"outputs": [
248+
{
249+
"name": "stdout",
250+
"output_type": "stream",
251+
"text": [
252+
"\u001b[32m17:14:25\u001b[0m \u001b[35msam.partee-NW9MQX5Y74\u001b[0m \u001b[34mredisvl.cli.index[58677]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
253+
"\u001b[32m17:14:25\u001b[0m \u001b[35msam.partee-NW9MQX5Y74\u001b[0m \u001b[34mredisvl.cli.index[58677]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. user_index\n"
254+
]
255+
}
256+
],
257+
"source": [
258+
"# use the CLI to see the created index\n",
259+
"!redisvl index listall"
260+
]
261+
},
262+
{
263+
"attachments": {},
241264
"cell_type": "markdown",
242265
"metadata": {},
243266
"source": [
@@ -262,6 +285,7 @@
262285
]
263286
},
264287
{
288+
"attachments": {},
265289
"cell_type": "markdown",
266290
"metadata": {},
267291
"source": [
@@ -272,7 +296,7 @@
272296
},
273297
{
274298
"cell_type": "code",
275-
"execution_count": 8,
299+
"execution_count": 7,
276300
"metadata": {},
277301
"outputs": [],
278302
"source": [
@@ -297,19 +321,19 @@
297321
},
298322
{
299323
"cell_type": "code",
300-
"execution_count": 9,
324+
"execution_count": 8,
301325
"metadata": {},
302326
"outputs": [
303327
{
304328
"name": "stdout",
305329
"output_type": "stream",
306330
"text": [
307331
"Score: 0\n",
308-
"Document {'id': 'user:john', 'payload': None, 'vector_score': '0', 'users': 'john', 'age': '1', 'job': 'engineer', 'credit_score': 'high'}\n",
332+
"Document {'id': 'user::john', 'payload': None, 'vector_score': '0', 'users': 'john', 'age': '1', 'job': 'engineer', 'credit_score': 'high'}\n",
309333
"Score: 0\n",
310-
"Document {'id': 'user:mary', 'payload': None, 'vector_score': '0', 'users': 'mary', 'age': '2', 'job': 'doctor', 'credit_score': 'low'}\n",
334+
"Document {'id': 'user::mary', 'payload': None, 'vector_score': '0', 'users': 'mary', 'age': '2', 'job': 'doctor', 'credit_score': 'low'}\n",
311335
"Score: 0.653301358223\n",
312-
"Document {'id': 'user:joe', 'payload': None, 'vector_score': '0.653301358223', 'users': 'joe', 'age': '3', 'job': 'dentist', 'credit_score': 'medium'}\n"
336+
"Document {'id': 'user::joe', 'payload': None, 'vector_score': '0.653301358223', 'users': 'joe', 'age': '3', 'job': 'dentist', 'credit_score': 'medium'}\n"
313337
]
314338
}
315339
],

examples/cli/sample.yml

Lines changed: 0 additions & 21 deletions
This file was deleted.

examples/cli/schema.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
index:
2+
name: sample
3+
storage_type: hash
4+
prefix: "vector:"
5+
key_field: "id"
6+
7+
fields:
8+
tag:
9+
- name: categories
10+
separator: "|"
11+
- name: year
12+
separator: "|"
13+
vector:
14+
- initial_cap: 20000
15+
datatype: float32
16+
algorithm: flat
17+
dims: 768
18+
distance_metric: "cosine"
19+
name: vector

0 commit comments

Comments
 (0)