From a83e81f31e2ae8e051cfde62e095392b5794c0c0 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Thu, 23 Mar 2023 20:44:57 -0400 Subject: [PATCH 1/6] update vss docs --- .../search_vector_similarity_examples.ipynb | 225 ++++++++++++++++-- 1 file changed, 209 insertions(+), 16 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index 2b0261097c..905a0c0912 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -1,11 +1,15 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Vector Similarity\n", - "## Adding Vector Fields" + "Vector Similarity Search (VSS) is the process of finding data points that are similar to a given query vector in a vector database. Popular VSS uses include recommendation systems, image and video search, document retrieval, and question answering.\n", + "\n", + "## Index Creation\n", + "Before doing vector search, define the schema and create an index." ] }, { @@ -26,32 +30,105 @@ ], "source": [ "import redis\n", - "from redis.commands.search.field import VectorField\n", + "\n", + "from redis.commands.search.indexDefinition import (\n", + " IndexDefinition,\n", + " IndexType\n", + ")\n", "from redis.commands.search.query import Query\n", + "from redis.commands.search.field import (\n", + " TagField,\n", + " VectorField\n", + ")\n", + "\n", + "r = redis.Redis(host=\"localhost\", port=6379)\n", + "\n", + "INDEX_NAME = \"index\"\n", + "VECTOR_DIMENSIONS = 1536\n", + "DOC_PREFIX = \"doc:\"\n", + "\n", + "# Schema\n", + "schema = (\n", + " TagField(\"tag\"),\n", + " VectorField(\"vector\",\n", + " \"FLAT\", {\n", + " \"TYPE\": \"FLOAT32\",\n", + " \"DIM\": VECTOR_DIMENSIONS,\n", + " \"DISTANCE_METRIC\": \"COSINE\",\n", + " }\n", + " ),\n", + ")\n", + "\n", + "# Index Definition\n", + "definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)\n", "\n", - "r = redis.Redis(host='localhost', port=36379)\n", + "# Create Index\n", + "r.ft(INDEX_NAME).create_index(fields=schema, definition=definition)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding Vectors to Redis\n", "\n", - "schema = (VectorField(\"v\", \"HNSW\", {\"TYPE\": \"FLOAT32\", \"DIM\": 2, \"DISTANCE_METRIC\": \"L2\"}),)\n", - "r.ft().create_index(schema)" + "Next, we add vectors (dummy data) to Redis using `hset`. The search index listens to keyspace notifications and will include any written HASH objects prefixed by `DOC_PREFIX`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#pip install numpy\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r.hset(f\"{DOC_PREFIX}a\", mapping={\n", + " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", + " \"tag\": \"foo\"\n", + "})\n", + "r.hset(f\"{DOC_PREFIX}b\", mapping={\n", + " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", + " \"tag\": \"foo\"\n", + "})\n", + "r.hset(f\"{DOC_PREFIX}c\", mapping={\n", + " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", + " \"tag\": \"bar\"\n", + "})" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## Searching" + "## Searching\n", + "You can use VSS queries with the `.ft(...).search(...)` query command. To use a VSS query, you must specify the option `.dialect(2)`.\n", + "\n", + "There are two supported types of vector queries in Redis: `KNN` and `Range`. `Hybrid` queries can work in both settings and combine elements of traditional search and VSS." ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Querying vector fields" + "### KNN Queries\n", + "KNN queries are for finding the topK most similar vectors given a query vector." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": { "pycharm": { "name": "#%%\n" @@ -61,21 +138,137 @@ { "data": { "text/plain": [ - "Result{2 total, docs: [Document {'id': 'a', 'payload': None, '__v_score': '0'}, Document {'id': 'b', 'payload': None, '__v_score': '3.09485009821e+26'}]}" + "[Document {'id': 'doc:c', 'payload': None, 'score': '0.244824767113'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.25022560358'}]" ] }, - "execution_count": 2, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "r.hset(\"a\", \"v\", \"aaaaaaaa\")\n", - "r.hset(\"b\", \"v\", \"aaaabaaa\")\n", - "r.hset(\"c\", \"v\", \"aaaaabaa\")\n", + "query = (\n", + " Query(\"*=>[KNN 2 @vector $vec as score]\")\n", + " .sort_by(\"score\")\n", + " .return_fields(\"id\", \"score\")\n", + " .paging(0, 2)\n", + " .dialect(2)\n", + ")\n", "\n", - "q = Query(\"*=>[KNN 2 @v $vec]\").return_field(\"__v_score\").dialect(2)\n", - "r.ft().search(q, query_params={\"vec\": \"aaaaaaaa\"})" + "query_params = {\n", + " \"vec\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes()\n", + "}\n", + "r.ft(INDEX_NAME).search(query, query_params).docs" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Range Queries\n", + "Range queries provide a way to filter results by the distance between a vector field in Redis and a query vector based on some pre-defined threshold (radius)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document {'id': 'doc:a', 'payload': None, 'score': '0.245782673359'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.25076341629'},\n", + " Document {'id': 'doc:c', 'payload': None, 'score': '0.2565549016'}]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query = (\n", + " Query(\"@vector:[VECTOR_RANGE $radius $vec]=>{$YIELD_DISTANCE_AS: score}\")\n", + " .sort_by(\"score\")\n", + " .return_fields(\"id\", \"score\")\n", + " .paging(0, 3)\n", + " .dialect(2)\n", + ")\n", + "\n", + "# Find all vectors within 0.8 of the query vector\n", + "query_params = {\n", + " \"radius\": 0.8,\n", + " \"vec\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes()\n", + "}\n", + "r.ft(INDEX_NAME).search(query, query_params).docs" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See additional Range Query examples in [this Jupyter notebook](https://github.com/RediSearch/RediSearch/blob/master/docs/docs/vecsim-range_queries_examples.ipynb)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hybrid Queries\n", + "Hybrid queries contain both traditional filters (numeric, tags, text) and VSS in one single Redis command." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document {'id': 'doc:b', 'payload': None, 'score': '0.24173271656', 'tag': 'foo'},\n", + " Document {'id': 'doc:a', 'payload': None, 'score': '0.25843077898', 'tag': 'foo'}]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query = (\n", + " Query(\"(@tag:{ foo })=>[KNN 2 @vector $vec as score]\")\n", + " .sort_by(\"score\")\n", + " .return_fields(\"id\", \"tag\", \"score\")\n", + " .paging(0, 2)\n", + " .dialect(2)\n", + ")\n", + "\n", + "query_params = {\n", + " \"vec\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes()\n", + "}\n", + "r.ft(INDEX_NAME).search(query, query_params).docs" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See additional Hybrid Query examples in [this Jupyter notebook](https://github.com/RediSearch/RediSearch/blob/master/docs/docs/vecsim-hybrid_queries_examples.ipynb)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Find several example apps, tutorials, and projects [in this GitHub organization](https://github.com/RedisVentures)." ] } ], @@ -98,7 +291,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.2" + "version": "3.9.12" }, "orig_nbformat": 4 }, From 389bd42f1d910a1648ec0d010768874a47419603 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Fri, 24 Mar 2023 15:45:05 -0400 Subject: [PATCH 2/6] add embeddings creation and storage examples --- .../search_vector_similarity_examples.ipynb | 226 +++++++++++++++--- 1 file changed, 194 insertions(+), 32 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index 905a0c0912..ad806f4fe9 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -6,15 +6,15 @@ "metadata": {}, "source": [ "# Vector Similarity\n", - "Vector Similarity Search (VSS) is the process of finding data points that are similar to a given query vector in a vector database. Popular VSS uses include recommendation systems, image and video search, document retrieval, and question answering.\n", + "**Vectors** (also called \"Embeddings\"), represent an AI model's impression (or understanding) of a piece of unstructured data like text, images, audio, videos, etc. Vector Similarity Search (VSS) is the process of finding vectors in the vector database that are similar to a given query vector. Popular VSS uses include recommendation systems, image and video search, document retrieval, and question answering.\n", "\n", "## Index Creation\n", - "Before doing vector search, define the schema and create an index." + "Before doing vector search, first define the schema and create an index." ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -23,7 +23,7 @@ "b'OK'" ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -43,21 +43,24 @@ "\n", "r = redis.Redis(host=\"localhost\", port=6379)\n", "\n", - "INDEX_NAME = \"index\"\n", - "VECTOR_DIMENSIONS = 1536\n", - "DOC_PREFIX = \"doc:\"\n", + "INDEX_NAME = \"index\" # Vector Index Name\n", + "VECTOR_DIMENSIONS = 1536 # Number of Vector Dimensions\n", + "DOC_PREFIX = \"doc:\" # RediSearch Key Prefix for the Index\n", "\n", "# Schema\n", "schema = (\n", " TagField(\"tag\"),\n", - " VectorField(\"vector\",\n", - " \"FLAT\", {\n", - " \"TYPE\": \"FLOAT32\",\n", - " \"DIM\": VECTOR_DIMENSIONS,\n", - " \"DISTANCE_METRIC\": \"COSINE\",\n", + " VectorField(\"vector\", # Vector Field Name\n", + " \"FLAT\", { # Vector Index Type: FLAT or HNSW\n", + " \"TYPE\": \"FLOAT32\", # FLOAT32 or FLOAT64\n", + " \"DIM\": VECTOR_DIMENSIONS, # Number of Vector Dimensions\n", + " \"DISTANCE_METRIC\": \"COSINE\", # Vector Search Distance Metric\n", + " \"INITIAL_CAP\": 1000 # Helps with memory planning and allocation\n", " }\n", " ),\n", ")\n", + "# NOTE: sometimes users experience issues with the INITIAL_CAP param.\n", + "# Best tip is to start with a smaller INITIAL_CAP value than required, especially if Redis throws an error.\n", "\n", "# Index Definition\n", "definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)\n", @@ -78,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -88,19 +91,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "r.hset(f\"{DOC_PREFIX}a\", mapping={\n", + "r.hset(f\"doc:a\", mapping={\n", " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", " \"tag\": \"foo\"\n", "})\n", - "r.hset(f\"{DOC_PREFIX}b\", mapping={\n", + "r.hset(f\"doc:b\", mapping={\n", " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", " \"tag\": \"foo\"\n", "})\n", - "r.hset(f\"{DOC_PREFIX}c\", mapping={\n", + "r.hset(f\"doc:c\", mapping={\n", " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", " \"tag\": \"bar\"\n", "})" @@ -128,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "pycharm": { "name": "#%%\n" @@ -138,11 +152,11 @@ { "data": { "text/plain": [ - "[Document {'id': 'doc:c', 'payload': None, 'score': '0.244824767113'},\n", - " Document {'id': 'doc:b', 'payload': None, 'score': '0.25022560358'}]" + "[Document {'id': 'doc:c', 'payload': None, 'score': '0.240548729897'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.249360978603'}]" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -173,18 +187,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[Document {'id': 'doc:a', 'payload': None, 'score': '0.245782673359'},\n", - " Document {'id': 'doc:b', 'payload': None, 'score': '0.25076341629'},\n", - " Document {'id': 'doc:c', 'payload': None, 'score': '0.2565549016'}]" + "[Document {'id': 'doc:a', 'payload': None, 'score': '0.247005105019'},\n", + " Document {'id': 'doc:c', 'payload': None, 'score': '0.247780561447'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.255188882351'}]" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -225,17 +239,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[Document {'id': 'doc:b', 'payload': None, 'score': '0.24173271656', 'tag': 'foo'},\n", - " Document {'id': 'doc:a', 'payload': None, 'score': '0.25843077898', 'tag': 'foo'}]" + "[Document {'id': 'doc:a', 'payload': None, 'score': '0.244236528873', 'tag': 'foo'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.249226748943', 'tag': 'foo'}]" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -268,7 +282,155 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Find several example apps, tutorials, and projects [in this GitHub organization](https://github.com/RedisVentures)." + "## Vector Creation and Storage Examples\n", + "The above examples use dummy data as vectors. However, in reality, most use cases leverage production-grade AI models for creating embeddings. Below we will take some sample text data, pass it to the OpenAI and Cohere API's respectively, and then write them to Redis." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "texts = [\n", + " \"Today is a really great day!\",\n", + " \"The dog next door barks really loudly.\",\n", + " \"My cat escaped and got out before I could close the door.\"\n", + "]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install openai\n", + "import openai\n", + "\n", + "openai.api_key = \"YOUR OPENAI API KEY\"" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# Create Embeddings with OpenAI\n", + "response = openai.Embedding.create(input=texts, engine=\"text-embedding-ada-002\")\n", + "embeddings = np.array([r[\"embedding\"] for r in response[\"data\"]], dtype=np.float32)\n", + "\n", + "# Write to Redis\n", + "for i, embedding in enumerate(embeddings):\n", + " r.hset(f\"doc:{i}\", mapping = {\n", + " \"vector\": embedding.tobytes(),\n", + " \"tag\": \"openai\"\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 0.00509819, 0.0010873 , -0.00228475, ..., -0.00457579,\n", + " 0.01329307, -0.03167175],\n", + " [-0.00352492, -0.00551083, -0.01318067, ..., -0.02915609,\n", + " 0.01472911, -0.01369681],\n", + " [-0.01286718, 0.00351361, -0.01723753, ..., -0.01536361,\n", + " 0.01949651, -0.05041625]], dtype=float32)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embeddings" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cohere Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install cohere\n", + "import cohere\n", + "\n", + "co = cohere.Client(\"YOUR COHERE API KEY\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# Create Embeddings with OpenAI\n", + "response = co.embed(texts=texts, model=\"small\")\n", + "embeddings = np.array(response.embeddings, dtype=np.float32)\n", + "\n", + "# Write to Redis\n", + "for i, embedding in enumerate(embeddings):\n", + " r.hset(f\"doc:{i}\", mapping = {\n", + " \"vector\": embedding.tobytes(),\n", + " \"tag\": \"cohere\"\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[-0.3010254 , -0.7158203 , -0.28515625, ..., 0.8125 ,\n", + " 1.0292969 , -0.8095703 ],\n", + " [-0.02745056, -1.4892578 , 0.23937988, ..., -0.8930664 ,\n", + " 0.15991211, -3.2050781 ],\n", + " [ 0.09777832, 0.7270508 , -0.296875 , ..., -1.9638672 ,\n", + " 1.6650391 , -0.23693848]], dtype=float32)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embeddings" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Find more example apps, tutorials, and projects [in this GitHub organization](https://github.com/RedisVentures)." ] } ], From 70e93889bf23a1fd9587039bd3fed897026651f5 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 29 Mar 2023 00:07:13 -0400 Subject: [PATCH 3/6] update based on feedback --- .../search_vector_similarity_examples.ipynb | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index ad806f4fe9..a7d44f960f 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -30,16 +30,9 @@ ], "source": [ "import redis\n", - "\n", - "from redis.commands.search.indexDefinition import (\n", - " IndexDefinition,\n", - " IndexType\n", - ")\n", + "from redis.commands.search.field import TagField, VectorField\n", + "from redis.commands.search.indexDefinition import IndexDefinition, IndexType\n", "from redis.commands.search.query import Query\n", - "from redis.commands.search.field import (\n", - " TagField,\n", - " VectorField\n", - ")\n", "\n", "r = redis.Redis(host=\"localhost\", port=6379)\n", "\n", @@ -59,8 +52,6 @@ " }\n", " ),\n", ")\n", - "# NOTE: sometimes users experience issues with the INITIAL_CAP param.\n", - "# Best tip is to start with a smaller INITIAL_CAP value than required, especially if Redis throws an error.\n", "\n", "# Index Definition\n", "definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)\n", @@ -79,13 +70,21 @@ "Next, we add vectors (dummy data) to Redis using `hset`. The search index listens to keyspace notifications and will include any written HASH objects prefixed by `DOC_PREFIX`." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install numpy" + ] + }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "#pip install numpy\n", "import numpy as np" ] }, @@ -304,7 +303,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### OpenAI Embeddings" + "### OpenAI Embeddings [text-embedding-ada-002](https://openai.com/blog/new-and-improved-embedding-model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install openai" ] }, { @@ -313,7 +321,6 @@ "metadata": {}, "outputs": [], "source": [ - "#!pip install openai\n", "import openai\n", "\n", "openai.api_key = \"YOUR OPENAI API KEY\"" @@ -370,13 +377,21 @@ "### Cohere Embeddings" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install cohere" + ] + }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "#!pip install cohere\n", "import cohere\n", "\n", "co = cohere.Client(\"YOUR COHERE API KEY\")" From 173c8c0161215a753df2162796c1e793a482df0a Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 29 Mar 2023 00:14:29 -0400 Subject: [PATCH 4/6] fix version and link --- docs/examples/search_vector_similarity_examples.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index a7d44f960f..ce39087b09 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -303,7 +303,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### OpenAI Embeddings [text-embedding-ada-002](https://openai.com/blog/new-and-improved-embedding-model)" + "### OpenAI Embeddings" ] }, { @@ -332,7 +332,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Create Embeddings with OpenAI\n", + "# Create Embeddings with OpenAI text-embedding-ada-002\n", + "# https://openai.com/blog/new-and-improved-embedding-model\n", "response = openai.Embedding.create(input=texts, engine=\"text-embedding-ada-002\")\n", "embeddings = np.array([r[\"embedding\"] for r in response[\"data\"]], dtype=np.float32)\n", "\n", @@ -468,7 +469,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.2" }, "orig_nbformat": 4 }, From 19c986655c8f0df779e0b6dceceaa34a753146cf Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Fri, 31 Mar 2023 13:59:38 -0400 Subject: [PATCH 5/6] include more realistic search examples and clean up indices --- .../search_vector_similarity_examples.ipynb | 366 +++++++++++++----- 1 file changed, 279 insertions(+), 87 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index ce39087b09..c378bdb7b2 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -14,20 +14,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "b'OK'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import redis\n", "from redis.commands.search.field import TagField, VectorField\n", @@ -36,28 +25,54 @@ "\n", "r = redis.Redis(host=\"localhost\", port=6379)\n", "\n", - "INDEX_NAME = \"index\" # Vector Index Name\n", - "VECTOR_DIMENSIONS = 1536 # Number of Vector Dimensions\n", - "DOC_PREFIX = \"doc:\" # RediSearch Key Prefix for the Index\n", + "INDEX_NAME = \"index\" # Vector Index Name\n", + "DOC_PREFIX = \"doc:\" # RediSearch Key Prefix for the Index\n", "\n", - "# Schema\n", - "schema = (\n", - " TagField(\"tag\"),\n", - " VectorField(\"vector\", # Vector Field Name\n", - " \"FLAT\", { # Vector Index Type: FLAT or HNSW\n", - " \"TYPE\": \"FLOAT32\", # FLOAT32 or FLOAT64\n", - " \"DIM\": VECTOR_DIMENSIONS, # Number of Vector Dimensions\n", - " \"DISTANCE_METRIC\": \"COSINE\", # Vector Search Distance Metric\n", - " \"INITIAL_CAP\": 1000 # Helps with memory planning and allocation\n", - " }\n", - " ),\n", - ")\n", + "def create_index(vector_dimensions: int):\n", + " try:\n", + " # check to see if index exists\n", + " r.ft(INDEX_NAME).info()\n", + " print(\"Index already exists!\")\n", + " except:\n", + " # schema\n", + " schema = (\n", + " TagField(\"tag\"), # Tag Field Name\n", + " VectorField(\"vector\", # Vector Field Name\n", + " \"FLAT\", { # Vector Index Type: FLAT or HNSW\n", + " \"TYPE\": \"FLOAT32\", # FLOAT32 or FLOAT64\n", + " \"DIM\": vector_dimensions, # Number of Vector Dimensions\n", + " \"DISTANCE_METRIC\": \"COSINE\", # Vector Search Distance Metric\n", + " \"INITIAL_CAP\": 1000 # Helps with memory planning and allocation\n", + " }\n", + " ),\n", + " )\n", + "\n", + " # index Definition\n", + " definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)\n", "\n", - "# Index Definition\n", - "definition = IndexDefinition(prefix=[DOC_PREFIX], index_type=IndexType.HASH)\n", + " # create Index\n", + " r.ft(INDEX_NAME).create_index(fields=schema, definition=definition)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll start by working with vectors that have 1536 dimensions." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# define vector dimensions\n", + "VECTOR_DIMENSIONS = 1536\n", "\n", - "# Create Index\n", - "r.ft(INDEX_NAME).create_index(fields=schema, definition=definition)" + "# create the index\n", + "create_index(vector_dimensions=VECTOR_DIMENSIONS)" ] }, { @@ -92,31 +107,28 @@ "cell_type": "code", "execution_count": 4, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "r.hset(f\"doc:a\", mapping={\n", - " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", - " \"tag\": \"foo\"\n", - "})\n", - "r.hset(f\"doc:b\", mapping={\n", - " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", - " \"tag\": \"foo\"\n", - "})\n", - "r.hset(f\"doc:c\", mapping={\n", - " \"vector\": np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes(),\n", - " \"tag\": \"bar\"\n", - "})" + "# instantiate a redis pipeline\n", + "pipe = r.pipeline()\n", + "\n", + "# define some dummy data\n", + "objects = [\n", + " {\"name\": \"a\", \"tag\": \"foo\"},\n", + " {\"name\": \"b\", \"tag\": \"foo\"},\n", + " {\"name\": \"c\", \"tag\": \"bar\"},\n", + "]\n", + "\n", + "# write data\n", + "for obj in objects:\n", + " # define key\n", + " key = f\"doc:{obj['name']}\"\n", + " # create a random \"dummy\" vector\n", + " obj[\"vector\"] = np.random.rand(VECTOR_DIMENSIONS).astype(np.float32).tobytes()\n", + " # HSET\n", + " pipe.hset(key, mapping=obj)\n", + "\n", + "res = pipe.execute()" ] }, { @@ -151,8 +163,8 @@ { "data": { "text/plain": [ - "[Document {'id': 'doc:c', 'payload': None, 'score': '0.240548729897'},\n", - " Document {'id': 'doc:b', 'payload': None, 'score': '0.249360978603'}]" + "[Document {'id': 'doc:b', 'payload': None, 'score': '0.2376562953'},\n", + " Document {'id': 'doc:c', 'payload': None, 'score': '0.240063905716'}]" ] }, "execution_count": 5, @@ -192,9 +204,9 @@ { "data": { "text/plain": [ - "[Document {'id': 'doc:a', 'payload': None, 'score': '0.247005105019'},\n", - " Document {'id': 'doc:c', 'payload': None, 'score': '0.247780561447'},\n", - " Document {'id': 'doc:b', 'payload': None, 'score': '0.255188882351'}]" + "[Document {'id': 'doc:a', 'payload': None, 'score': '0.243115246296'},\n", + " Document {'id': 'doc:c', 'payload': None, 'score': '0.24981123209'},\n", + " Document {'id': 'doc:b', 'payload': None, 'score': '0.251443207264'}]" ] }, "execution_count": 6, @@ -244,8 +256,8 @@ { "data": { "text/plain": [ - "[Document {'id': 'doc:a', 'payload': None, 'score': '0.244236528873', 'tag': 'foo'},\n", - " Document {'id': 'doc:b', 'payload': None, 'score': '0.249226748943', 'tag': 'foo'}]" + "[Document {'id': 'doc:b', 'payload': None, 'score': '0.24422544241', 'tag': 'foo'},\n", + " Document {'id': 'doc:a', 'payload': None, 'score': '0.259926855564', 'tag': 'foo'}]" ] }, "execution_count": 7, @@ -294,7 +306,8 @@ "texts = [\n", " \"Today is a really great day!\",\n", " \"The dog next door barks really loudly.\",\n", - " \"My cat escaped and got out before I could close the door.\"\n", + " \"My cat escaped and got out before I could close the door.\",\n", + " \"It's supposed to rain and thunder tomorrow.\"\n", "]" ] }, @@ -303,7 +316,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### OpenAI Embeddings" + "### OpenAI Embeddings\n", + "Before working with OpenAI Embeddings, we clean up our existing search index and create a new one." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# delete index\n", + "r.ft(INDEX_NAME).dropindex(delete_documents=True)\n", + "\n", + "# make a new one\n", + "create_index(vector_dimensions=VECTOR_DIMENSIONS)" ] }, { @@ -317,18 +344,19 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "import openai\n", "\n", + "# set your OpenAI API key - get one at https://platform.openai.com\n", "openai.api_key = \"YOUR OPENAI API KEY\"" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -338,16 +366,19 @@ "embeddings = np.array([r[\"embedding\"] for r in response[\"data\"]], dtype=np.float32)\n", "\n", "# Write to Redis\n", + "pipe = r.pipeline()\n", "for i, embedding in enumerate(embeddings):\n", - " r.hset(f\"doc:{i}\", mapping = {\n", + " pipe.hset(f\"doc:{i}\", mapping = {\n", " \"vector\": embedding.tobytes(),\n", + " \"content\": texts[i],\n", " \"tag\": \"openai\"\n", - " })" + " })\n", + "res = pipe.execute()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -355,13 +386,15 @@ "text/plain": [ "array([[ 0.00509819, 0.0010873 , -0.00228475, ..., -0.00457579,\n", " 0.01329307, -0.03167175],\n", - " [-0.00352492, -0.00551083, -0.01318067, ..., -0.02915609,\n", - " 0.01472911, -0.01369681],\n", - " [-0.01286718, 0.00351361, -0.01723753, ..., -0.01536361,\n", - " 0.01949651, -0.05041625]], dtype=float32)" + " [-0.00357223, -0.00550784, -0.01314328, ..., -0.02915693,\n", + " 0.01470436, -0.01367203],\n", + " [-0.01284631, 0.0034875 , -0.01719686, ..., -0.01537451,\n", + " 0.01953256, -0.05048691],\n", + " [-0.01145045, -0.00785481, 0.00206323, ..., -0.02070181,\n", + " -0.01629098, -0.00300795]], dtype=float32)" ] }, - "execution_count": 14, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -375,7 +408,92 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Cohere Embeddings" + "### Search with OpenAI Embeddings\n", + "\n", + "Now that we've created embeddings with OpenAI, we can also perform a search to find relevant documents to some input text.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 0.00062901, -0.0070723 , -0.00148926, ..., -0.01904645,\n", + " -0.00436092, -0.01117944], dtype=float32)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text = \"animals\"\n", + "\n", + "# create query embedding\n", + "response = openai.Embedding.create(input=[text], engine=\"text-embedding-ada-002\")\n", + "query_embedding = np.array([r[\"embedding\"] for r in response[\"data\"]], dtype=np.float32)[0]\n", + "\n", + "query_embedding" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document {'id': 'doc:1', 'payload': None, 'score': '0.214349985123', 'content': 'The dog next door barks really loudly.', 'tag': 'openai'},\n", + " Document {'id': 'doc:2', 'payload': None, 'score': '0.237052619457', 'content': 'My cat escaped and got out before I could close the door.', 'tag': 'openai'}]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# query for similar documents that have the openai tag\n", + "query = (\n", + " Query(\"(@tag:{ openai })=>[KNN 2 @vector $vec as score]\")\n", + " .sort_by(\"score\")\n", + " .return_fields(\"content\", \"tag\", \"score\")\n", + " .paging(0, 2)\n", + " .dialect(2)\n", + ")\n", + "\n", + "query_params = {\"vec\": query_embedding.tobytes()}\n", + "r.ft(INDEX_NAME).search(query, query_params).docs\n", + "\n", + "# the two pieces of content related to animals are returned" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cohere Embeddings\n", + "Before working with Cohere Embeddings, we clean up our existing search index and create a new one." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# delete index\n", + "r.ft(INDEX_NAME).dropindex(delete_documents=True)\n", + "\n", + "# make a new one for cohere embeddings (1024 dimensions)\n", + "VECTOR_DIMENSIONS = 1024\n", + "create_index(vector_dimensions=VECTOR_DIMENSIONS)" ] }, { @@ -389,7 +507,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -400,11 +518,12 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ - "# Create Embeddings with OpenAI\n", + "# Create Embeddings with Cohere\n", + "# https://docs.cohere.ai/docs/embeddings\n", "response = co.embed(texts=texts, model=\"small\")\n", "embeddings = np.array(response.embeddings, dtype=np.float32)\n", "\n", @@ -412,10 +531,75 @@ "for i, embedding in enumerate(embeddings):\n", " r.hset(f\"doc:{i}\", mapping = {\n", " \"vector\": embedding.tobytes(),\n", + " \"content\": texts[i],\n", " \"tag\": \"cohere\"\n", " })" ] }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[-0.3034668 , -0.71533203, -0.2836914 , ..., 0.81152344,\n", + " 1.0253906 , -0.8095703 ],\n", + " [-0.02560425, -1.4912109 , 0.24267578, ..., -0.89746094,\n", + " 0.15625 , -3.203125 ],\n", + " [ 0.10125732, 0.7246094 , -0.29516602, ..., -1.9638672 ,\n", + " 1.6630859 , -0.23291016],\n", + " [-2.09375 , 0.8588867 , -0.23352051, ..., -0.01541138,\n", + " 0.17053223, -3.4042969 ]], dtype=float32)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "embeddings" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Search with Cohere Embeddings\n", + "\n", + "Now that we've created embeddings with Cohere, we can also perform a search to find relevant documents to some input text." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-0.49682617, 1.7070312 , 0.3466797 , ..., 0.58984375,\n", + " 0.1060791 , -2.9023438 ], dtype=float32)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text = \"animals\"\n", + "\n", + "# create query embedding\n", + "response = co.embed(texts=[text], model=\"small\")\n", + "query_embedding = np.array(response.embeddings[0], dtype=np.float32)\n", + "\n", + "query_embedding" + ] + }, { "cell_type": "code", "execution_count": 20, @@ -424,12 +608,8 @@ { "data": { "text/plain": [ - "array([[-0.3010254 , -0.7158203 , -0.28515625, ..., 0.8125 ,\n", - " 1.0292969 , -0.8095703 ],\n", - " [-0.02745056, -1.4892578 , 0.23937988, ..., -0.8930664 ,\n", - " 0.15991211, -3.2050781 ],\n", - " [ 0.09777832, 0.7270508 , -0.296875 , ..., -1.9638672 ,\n", - " 1.6650391 , -0.23693848]], dtype=float32)" + "[Document {'id': 'doc:1', 'payload': None, 'score': '0.658673524857', 'content': 'The dog next door barks really loudly.', 'tag': 'cohere'},\n", + " Document {'id': 'doc:2', 'payload': None, 'score': '0.662699103355', 'content': 'My cat escaped and got out before I could close the door.', 'tag': 'cohere'}]" ] }, "execution_count": 20, @@ -438,7 +618,19 @@ } ], "source": [ - "embeddings" + "# query for similar documents that have the cohere tag\n", + "query = (\n", + " Query(\"(@tag:{ cohere })=>[KNN 2 @vector $vec as score]\")\n", + " .sort_by(\"score\")\n", + " .return_fields(\"content\", \"tag\", \"score\")\n", + " .paging(0, 2)\n", + " .dialect(2)\n", + ")\n", + "\n", + "query_params = {\"vec\": query_embedding.tobytes()}\n", + "r.ft(INDEX_NAME).search(query, query_params).docs\n", + "\n", + "# the two pieces of content related to animals are returned" ] }, { @@ -446,7 +638,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Find more example apps, tutorials, and projects [in this GitHub organization](https://github.com/RedisVentures)." + "Find more example apps, tutorials, and projects using Redis Vector Similarity Search [in this GitHub organization](https://github.com/RedisVentures)." ] } ], From 42dc33487e7f7b2e56677302cf45796873c8eb37 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Fri, 31 Mar 2023 16:19:56 -0400 Subject: [PATCH 6/6] completely remove initial cap reference --- docs/examples/search_vector_similarity_examples.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/examples/search_vector_similarity_examples.ipynb b/docs/examples/search_vector_similarity_examples.ipynb index c378bdb7b2..bd1df3c1ef 100644 --- a/docs/examples/search_vector_similarity_examples.ipynb +++ b/docs/examples/search_vector_similarity_examples.ipynb @@ -42,7 +42,6 @@ " \"TYPE\": \"FLOAT32\", # FLOAT32 or FLOAT64\n", " \"DIM\": vector_dimensions, # Number of Vector Dimensions\n", " \"DISTANCE_METRIC\": \"COSINE\", # Vector Search Distance Metric\n", - " \"INITIAL_CAP\": 1000 # Helps with memory planning and allocation\n", " }\n", " ),\n", " )\n", @@ -661,7 +660,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.2" + "version": "3.9.12" }, "orig_nbformat": 4 },