Merge pull request #33 from merefield/semantic_search

FEATURE: Semantic Search

Author: merefield

.github/workflows/plugin-tests.yml

@@ -59,6 +59,15 @@ jobs:
           sudo -E -u postgres script/start_test_db.rb
           sudo -u postgres psql -c "CREATE ROLE $PGUSER LOGIN SUPERUSER PASSWORD '$PGPASSWORD';"
 
+      - name: Install pg_embeddings
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install -y postgresql-server-dev-13
+          git clone https://github.com/neondatabase/pg_embedding.git
+          cd pg_embedding
+          make PG_CONFIG=/usr/lib/postgresql/13/bin/pg_config
+          make PG_CONFIG=/usr/lib/postgresql/13/bin/pg_config install
+
       - name: Bundler cache
         uses: actions/cache@v3
         with:

app/jobs/regular/chatbot_post_embedding_delete_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Job is triggered on a Post destruction.
+class ::Jobs::ChatbotPostEmbeddingDeleteJob < Jobs::Base
+  sidekiq_options retry: false
+
+  def execute(opts)
+    begin
+      post_id = opts[:id]
+
+      ::DiscourseChatbot.progress_debug_message("101. Deleting a Post Embedding for Post id: #{post_id}")
+
+      ::DiscourseChatbot::PostEmbedding.find_by(post_id: post_id).destroy!
+    rescue => e
+      Rails.logger.error ("OpenAIBot Post Embedding: There was a problem, but will retry til limit: #{e}")
+    end
+  end
+end

app/jobs/regular/chatbot_post_embedding_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Job is triggered on an update to a Post.
+class ::Jobs::ChatbotPostEmbeddingJob < Jobs::Base
+  sidekiq_options retry: 5, dead: false
+
+  def execute(opts)
+    begin
+      post_id = opts[:id]
+
+      ::DiscourseChatbot.progress_debug_message("100. Creating/updating a Post Embedding for Post id: #{post_id}")
+
+      post_embedding = ::DiscourseChatbot::PostEmbeddingProcess.new
+
+      post_embedding.upsert_embedding(post_id)
+    rescue => e
+      Rails.logger.error ("OpenAIBot Post Embedding: There was a problem, but will retry til limit: #{e}")
+    end
+  end
+end

app/models/embedding.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class ::DiscourseChatbot::PostEmbedding < ActiveRecord::Base
+  self.table_name = 'chatbot_post_embeddings'
+
+  validates :post_id, presence: true, uniqueness: true
+end

config/locales/server.en.yml

@@ -47,6 +47,95 @@ en:
       title: "The subject of this conversation is %{topic_title}"
       first_post: "The first thing someone said was %{username} who said %{raw}"
       post: "%{username} said %{raw}"
+      function:
+        calculator:
+          description: |
+            Useful for getting the result of a math expression.  It is a general purpose calculator.  It works with Ruby expressions.
+
+            You can retrieve the current date from it too and using the core Ruby Time method to calculate dates.
+
+            The input to this tool should be a valid mathematical expression that could be executed by the base Ruby programming language with no extensions.
+
+            Be certain to prefix any functions with 'Math.'
+
+            Usage:
+              Action Input: 1 + 1
+              Action Input: 3 * 2 / 4
+              Action Input: 9 - 7
+              Action Input: Time.now - 2 * 24 * 60 * 60
+              Action Input: Math.cbrt(13) + Math.cbrt(12)
+              Action Input: Math.sqrt(8)
+              Action Input: (4.1 + 2.3) / (2.0 - 5.6) * 3
+          parameters:
+            input: the mathematical expression you need to process and get the answer to. Make sure it is Ruby compatible.
+          error: "'%{parameter}' is an invalid mathematical expression, make sure if you are trying to calculate dates use Ruby Time class"
+        forum_search:
+          description: |
+            Search the local forum for information that may help you answer the question.  Especially useful when the forum specialises in the subject matter of the query.
+            Searching the local forum is preferable to searching google or the internet and should be considered higher priority.  It is quicker and cheaper.
+
+            Input should be a search query.  You can optionally also specify the number of posts you wish returned from your query.
+
+            Outputs text from the Post and a url link to it you can provide the user.  When presenting the url in your reply, do not embed in an anchor, just write the straight link.
+          parameters:
+            query: "search query for looking up information on the forum"
+            number_of_posts: "specify the number of posts you want returned from your query"
+          answer_summary: "The top %{number_of_posts} posts on the forum related to this query are, best match first:\n\n"
+          answer: "Number %{rank}: the post is at this web address: %{url}, it was written by '%{username}' on %{date} and the text is '%{raw}'.\n\n"
+          error:  "'%{query}': my search for this on the forum failed."
+        google_search:
+          description: |
+            A wrapper around Google Search.
+
+            Useful for when you need to answer questions about current events.
+            Always one of the first options when you need to find information on internet.
+
+            Input should be a search query.
+          parameters:
+            query: "search query for looking up information on the internet"
+          error: "%{query}: my search for this on the internet failed."
+        news:
+          description: |
+            A wrapper around the News API.
+
+            Useful for when you need to answer questions about current events in the news, current events or affairs.
+
+            Input should be a search query and a date from which to search news, so if the request is today, the search should be for todays date
+          parameters:
+            query: "query string for searching current news and events"
+            start_date: "start date from which to search for news in format YYYY-MM-DD"
+          answer: "The latest news about this is: "
+          error: "ERROR: Had trouble retrieving the news!"
+        stock_data:
+          description: |
+            An API for MarketStack stock data.  You need to call it using the stock ticker.  You can optionally also provide a specific date.
+          parameters:
+            ticker: "ticker for share or stock query"
+            date: "date for data in format YYYY-MM-DD"
+          answer: "Ticker %{ticker} had a day close of %{close} on %{date}, with a high of %{high} and a low of %{low}"
+          error: "ERROR: Had trouble retrieving information from Market Stack for stock market information!"
+        wikipedia:
+          description: |
+            A wrapper around Wikipedia.
+
+            Useful for when you need to answer general questions about
+            people, places, companies, facts, historical events, or other subjects.
+
+            Input should be a search query
+          parameters:
+            query: "query string for wikipedia search"
+          answer: "The relevant wikipedia page has the following summary: '%{summary}' and the article can be found at this url link: %{url}"
+          error: "ERROR: Had trouble retrieving information from Wikipedia!"
+      agent:
+        handle_function_call:
+          answer: "The answer is %{result}."
+        call_function:
+          error: "There was something wrong with your function arguments"
+        final_thought_answer:
+          opener: "To answer the question I will use these step by step instructions.\n\n"
+          thought_declaration: "I will use the %{function_name} function to calculate the answer with arguments %{arguments}.\n\n"
+          final_thought: "%{thoughts} Based on the above, I will now answer the question, this message will only be seen by me so answer with the assumption with that the user has not seen this message."
+        
     errors:
       general: "Sorry, I'm not well right now. Lets talk some other time. Meanwhile, please ask the admin to check the logs, thank you!"
       retries: "I've tried working out a response for you several times, but ultimately failed.  Please contact the admin if this persists, thank you!"

config/settings.yml

@@ -66,11 +66,13 @@ plugins:
     default: gpt-3.5-turbo
     choices:
       - gpt-3.5-turbo
-      - gpt-3.5-turbo-16k
       - gpt-3.5-turbo-0613
+      - gpt-3.5-turbo-16k
+      - gpt-3.5-turbo-16k-0613
       - gpt-4
-      - gpt-4-32k
       - gpt-4-0613
+      - gpt-4-32k
+      - gpt-4-32k-0613
   chatbot_reply_job_time_delay:
     client: false
     default: 3

db/migrate/20230820010101_enable_embedding_extension.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class EnableEmbeddingExtension < ActiveRecord::Migration[7.0]
+  def change
+    begin
+      enable_extension :embedding
+    rescue Exception => e
+      if DB.query_single("SELECT 1 FROM pg_available_extensions WHERE name = 'embedding';").empty?
+        STDERR.puts "----------------------------DISCOURSE CHATBOT ERROR----------------------------------"
+        STDERR.puts " Discourse Chatbot now requires the embedding extension on the PostgreSQL database."
+        STDERR.puts "         Run a `./launcher rebuild app` to fix it on a standard install."
+        STDERR.puts "            Alternatively, you can remove Discourse Chatbot to rebuild."
+        STDERR.puts "----------------------------DISCOURSE CHATBOT ERROR----------------------------------"
+      end
+      raise e
+    end
+  end
+end

db/migrate/20230820010103_create_chatbot_embeddings_table.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class CreateChatbotEmbeddingsTable < ActiveRecord::Migration[7.0]
+  def change
+    create_table :chatbot_embeddings do |t|
+      t.integer :post_id, null: false, index: { unique: true }, foreign_key: true
+        t.column :embedding, "real[]", null: false
+        t.timestamps
+    end
+  end
+end

db/migrate/20230820010105_create_chatbot_embeddings_index.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class CreateChatbotEmbeddingsIndex < ActiveRecord::Migration[7.0]
+  def up
+    execute <<-SQL
+      CREATE INDEX hnsw_index_on_chatbot_embeddings ON chatbot_embeddings USING hnsw(embedding)
+      WITH (dims=1536, m=64, efconstruction=64, efsearch=64);
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      DROP INDEX hnsw_index_on_chatbot_embeddings;
+    SQL
+  end
+end

db/migrate/20230826010101_rename_chatbot_embeddings_table.rb

@@ -0,0 +1,13 @@
+
+# frozen_string_literal: true
+
+class RenameChatbotEmbeddingsTable < ActiveRecord::Migration[7.0]
+  def change
+    begin
+      Migration::SafeMigrate.disable!
+      rename_table :chatbot_embeddings, :chatbot_post_embeddings
+    ensure
+      Migration::SafeMigrate.enable!
+    end
+  end
+end