Introduce Pydantic schema for index (#15)

Change from manual parsing to Pydantic for schema validation for the
fields of the index.

This makes it much easier to support having new additions that come with
future releases of RediSearch.

Author: Sam Partee

.gitignore

@@ -1,3 +1,4 @@
 __pycache__/
 redisvl.egg-info/
-.coverage
+.coverage
+scratch

conftest.py

@@ -6,14 +6,10 @@
     get_async_redis_connection,
     get_redis_connection
 )
+REDIS_ADDRESS = os.getenv("REDIS_ADDRESS", "redis://localhost:6379")
 
-HOST = os.environ.get("REDIS_HOST", "localhost")
-PORT = os.environ.get("REDIS_PORT", 6379)
-USER = os.environ.get("REDIS_USER", "default")
-PASS = os.environ.get("REDIS_PASSWORD", "")
-
-aredis = get_async_redis_connection(HOST, PORT, PASS)
-redis = get_redis_connection(HOST, PORT, PASS)
+aredis = get_async_redis_connection(REDIS_ADDRESS)
+redis = get_redis_connection(REDIS_ADDRESS)
 
 
 @pytest.fixture

examples/01-getting-started.ipynb

@@ -1,6 +1,7 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -21,6 +22,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -31,7 +33,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -49,6 +51,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -61,7 +64,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -133,7 +136,7 @@
        "2                    b'fff?fff?\\xcd\\xcc\\xcc='  "
       ]
      },
-     "execution_count": 3,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -148,6 +151,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -157,6 +161,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -179,7 +184,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -191,24 +196,22 @@
     "        \"storage_type\": \"hash\",\n",
     "    },\n",
     "    \"fields\": {\n",
-    "        # key is the field type\n",
-    "        # value is the name of the column in the dataset(frame)\n",
-    "        \"tag\": {\"credit_score\": {}},\n",
-    "        \"text\": {\"job\": {}},\n",
-    "        \"numeric\": {\"age\": {}},\n",
-    "        \"vector\": {\n",
-    "            \"user_embedding\": {\n",
+    "        \"tag\": [{\"name\": \"credit_score\"}],\n",
+    "        \"text\": [{\"name\": \"job\"}],\n",
+    "        \"numeric\": [{\"name\": \"age\"}],\n",
+    "        \"vector\": [{\n",
+    "                \"name\": \"user_embedding\",\n",
     "                \"dims\": 3,\n",
     "                \"distance_metric\": \"cosine\",\n",
     "                \"algorithm\": \"flat\",\n",
-    "                \"datatype\": \"float32\",\n",
-    "            }\n",
-    "        },\n",
+    "                \"datatype\": \"float32\"}\n",
+    "        ]\n",
     "    },\n",
-    "}"
+    "}\n"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -221,7 +224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,13 +234,33 @@
     "index = SearchIndex.from_dict(schema)\n",
     "\n",
     "# connect to local redis instance\n",
-    "index.connect(\"localhost\", 6379)\n",
+    "index.connect(\"redis://localhost:6379\")\n",
     "\n",
     "# create the index (no data yet)\n",
     "index.create()"
    ]
   },
   {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m17:14:25\u001b[0m \u001b[35msam.partee-NW9MQX5Y74\u001b[0m \u001b[34mredisvl.cli.index[58677]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n",
+      "\u001b[32m17:14:25\u001b[0m \u001b[35msam.partee-NW9MQX5Y74\u001b[0m \u001b[34mredisvl.cli.index[58677]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. user_index\n"
+     ]
+    }
+   ],
+   "source": [
+    "# use the CLI to see the created index\n",
+    "!redisvl index listall"
+   ]
+  },
+  {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -262,6 +285,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -272,7 +296,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -297,19 +321,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "Score: 0\n",
-      "Document {'id': 'user:john', 'payload': None, 'vector_score': '0', 'users': 'john', 'age': '1', 'job': 'engineer', 'credit_score': 'high'}\n",
+      "Document {'id': 'user::john', 'payload': None, 'vector_score': '0', 'users': 'john', 'age': '1', 'job': 'engineer', 'credit_score': 'high'}\n",
       "Score: 0\n",
-      "Document {'id': 'user:mary', 'payload': None, 'vector_score': '0', 'users': 'mary', 'age': '2', 'job': 'doctor', 'credit_score': 'low'}\n",
+      "Document {'id': 'user::mary', 'payload': None, 'vector_score': '0', 'users': 'mary', 'age': '2', 'job': 'doctor', 'credit_score': 'low'}\n",
       "Score: 0.653301358223\n",
-      "Document {'id': 'user:joe', 'payload': None, 'vector_score': '0.653301358223', 'users': 'joe', 'age': '3', 'job': 'dentist', 'credit_score': 'medium'}\n"
+      "Document {'id': 'user::joe', 'payload': None, 'vector_score': '0.653301358223', 'users': 'joe', 'age': '3', 'job': 'dentist', 'credit_score': 'medium'}\n"
      ]
     }
    ],

examples/cli/sample.yml

@@ -0,0 +1,19 @@
+index:
+  name: sample
+  storage_type: hash
+  prefix: "vector:"
+  key_field: "id"
+
+fields:
+  tag:
+  - name: categories
+    separator: "|"
+  - name: year
+    separator: "|"
+  vector:
+  - initial_cap: 20000
+    datatype: float32
+    algorithm: flat
+    dims: 768
+    distance_metric: "cosine"
+    name: vector