Advanced query parser

Gemfile

@@ -19,7 +19,7 @@ gem "connection_pool"
 # gem "sequel"
 # gem "sqlite3"
 
-# gem "aws-sdk-s3", "~> 1"
+gem "aws-sdk-s3", require: false
 
 # Caching - Rails 5.2 shipped with a redis cache for fragments, but doesn't
 # provide session storage via redis too, which redis-actionpack does.
@@ -34,7 +34,7 @@ gem "rack-attack"
 gem "bcrypt"
 
 # Server
-gem "puma"#, "~> 3.11"
+gem "puma"
 
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem
 gem "tzinfo-data"
@@ -63,11 +63,10 @@ gem "kaminari"
 gem "jbuilder"
 
 # ActiveJob Worker, Cron Schedulers
-# gem "sidekiq"
 gem "good_job"
 
 # Logs
-gem "logster", github: "discourse/logster", branch: "redis_4_6"
+gem "logster"
 
 # Auth
 # Provider
@@ -92,9 +91,7 @@ gem "nokogiri"
 # gem "ruby-readability"
 # gem "stopwords-filter"
 gem "marcel", "~> 1.0"
-# gem "parslet"
-
-gem "aws-sdk-s3", require: false
+gem "parslet"
 
 gem "exception_notification"
 gem "slack-notifier"

Gemfile.lock

@@ -1,10 +1,3 @@
-GIT
-  remote: https://github.com/discourse/logster.git
-  revision: fbc7cc43e58305d5f46d6fc243091158c57ee814
-  branch: redis_4_6
-  specs:
-    logster (2.11.0)
-
 GIT
   remote: https://github.com/elabs/pundit.git
   revision: 856d74d55d79a87102cbaa2df64d87c94dc7e3bc
@@ -135,9 +128,9 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    debug (1.4.0)
+    debug (1.6.2)
       irb (>= 1.3.6)
-      reline (>= 0.2.7)
+      reline (>= 0.3.1)
     diff-lcs (1.5.0)
     docile (1.4.0)
     domain_name (0.5.20190701)
@@ -227,7 +220,7 @@ GEM
       activesupport (>= 3.0)
       nokogiri (>= 1.6)
     io-console (0.5.11)
-    irb (1.4.1)
+    irb (1.4.2)
       reline (>= 0.3.0)
     jbuilder (2.11.5)
       actionview (>= 5.0.0)
@@ -257,6 +250,7 @@ GEM
     llhttp-ffi (0.4.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
+    logster (2.11.3)
     loofah (2.19.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -302,6 +296,7 @@ GEM
     parallel (1.21.0)
     parser (3.1.0.0)
       ast (~> 2.4.1)
+    parslet (2.0.0)
     pg (1.3.1)
     propshaft (0.6.1)
       actionpack (>= 7.0.0)
@@ -532,10 +527,11 @@ DEPENDENCIES
   json_matchers
   kaminari
   listen
-  logster!
+  logster
   mailcatcher
   marcel (~> 1.0)
   nokogiri
+  parslet
   pg
   propshaft
   puma

app/models/bookmark.rb

@@ -49,28 +49,32 @@ class Bookmark < ApplicationRecord
     where(Arel::Nodes.build_quoted(query).eq(any_host))
   }
 
+  # Searches the title for the query OR where the query is similar to
+  # words in an existing bookmark's title
   scope :title_search, ->(query) { where(<<~SQL.squish, { query: }) }
     search_title @@ websearch_to_tsquery(:query)
     OR search_title @@ to_tsquery(
-      (
-        select
-          word
-        from ts_stat('select search_title from bookmarks')
-        where similarity(:query, word) > 0.5
-        order by similarity(:query, word) > 0.5 desc
-        limit 1
-      )
+      SELECT
+        word
+      FROM ts_stat('SELECT search_title FROM bookmarks')
+      WHERE similarity(:query, word) > 0.5
+      ORDER BY similarity(:query, word) DESC
+      LIMIT 1
     )
   SQL
 
   scope :search, lambda { |query|
-    left_joins(:tags)
+    joins(:tags)
       .uri_search(query)
       .or(breakdown_search(query))
       .or(title_search(query))
       .or(tag_search(query))
   }
 
+  scope :advanced_search, lambda { |query|
+    BookmarksSearcher.new.search query
+  }
+
   # This has potential performance costs if we start retrying lots of times
   def self.for(user, uri)= find_or_initialize_by(user:, uri:)
 

app/searchers/application_searcher.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ApplicationSearcher
+  class << self
+    attr_reader :index, :compiler, :search
+
+    def define_index(&)
+      @index = QueryGrammar::Index.build(&)
+    end
+
+    def use_compiler val
+      @compiler = val
+    end
+
+    def execute_query &block
+      @search = block
+    end
+  end
+
+  def search query
+    query
+      .then { QueryGrammar.parse _1, index: self.class.index }
+      .then { self.class.compiler.compile _1 }
+      .then { QueryGrammar::Cloaker.new(bind: self).cloak _1, &self.class.search }
+  end
+end

app/searchers/bookmarks_searcher.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+class BookmarksSearcher < ApplicationSearcher
+  define_index do
+    # Defines the available types of specialized prefix searches
+    # in the query. The symbol name gets mapped to a function that
+    # registers a prefix
+    type :text do |clause|
+      clause.values.map do |value|
+        QueryGrammar::Ast::MatchClause.new(
+          field: clause.prefix,
+          value:,
+          origin: clause
+        )
+      end
+    end
+
+    type :keyword do |clause|
+      clause.values.map do |value|
+        QueryGrammar::Ast::EqualClause.new(
+          field: clause.prefix,
+          value:,
+          origin: clause
+        )
+      end
+    end
+
+    type :number do |clause|
+      clause.values.map do |value|
+        QueryGrammar::Ast::EqualClause.new(
+          field: clause.prefix,
+          value:,
+          origin: clause
+        )
+      end
+    end
+
+    type :date do |clause|
+      clause.values.map do |value|
+        QueryGrammar::Ast::EqualClause.new(
+          field: clause.prefix,
+          value:,
+          origin: clause
+        )
+      end
+    end
+
+    # This sets up the prefixes that are available in the query
+    # as well as what "type" they parse as as defined by the types
+    # above
+    field :uri,
+          type: :keyword,
+          name: "URI",
+          description: ""
+
+    field :host,
+          type: :keyword,
+          name: "Host",
+          description: "",
+          sortable: true
+
+    field :title,
+          type: :text,
+          name: "Title",
+          description: "",
+          sortable: true
+
+    field :description,
+          type: :text,
+          name: "Description/Notes",
+          description: "",
+          existable: true
+
+    field :tags,
+          type: :keyword,
+          name: "Tags",
+          description: "",
+          aliases: [ "tag" ],
+          existable: true
+
+    field :created_at,
+          type: :date,
+          name: "Created at Date",
+          description: "",
+          aliases: [ "created_date" ],
+          sortable: true
+
+    # Handles custom prefixes for various other operations such as breaking apart
+    # the created_date field into two pseudo fields "after" and "before" or an
+    # existence "has" or sort helpers
+    operator :after do
+      name "Created After Date"
+      description <<~DESC
+      DESC
+
+      arity 1
+      types :date
+
+      parse do |clause|
+        QueryGrammar::Ast::GtRangeClause.new(
+          field: :created_at,
+          value: clause.values.first,
+          origin: clause
+        )
+      end
+    end
+
+    operator :before do
+      name "Created Before Date"
+      description <<~DESC
+      DESC
+
+      arity 1
+      types :date
+
+      parse do |clause|
+        QueryGrammar::Ast::LtRangeClause.new(
+          field: :created_at,
+          value: clause.values.first,
+          origin: clause
+        )
+      end
+    end
+
+    operator :between do
+      name "Created Between Dates"
+      description <<~DESC
+      DESC
+
+      arity 2
+      types :date
+
+      parse do |clause|
+        QueryGrammar::Ast::RangeClause.new(
+          field: :created_at,
+          low: clause.values.first,
+          high: clause.values.second,
+          origin: clause
+        )
+      end
+    end
+
+    operator :has do
+      name "Has property"
+      description <<~DESC
+      DESC
+
+      parse do |clause|
+        clause.values.map do |value|
+          field = resolve_field value
+
+          QueryGrammar::Ast::ExistClause.new(
+            field:,
+            origin: clause
+          )
+        end
+      end
+    end
+
+    operator :sort do
+      name "Sort Field and Direction"
+      description <<~DESC
+      DESC
+
+      parse do |clause|
+        clause.values.map do |value|
+          field = resolve_field value
+
+          QueryGrammar::Ast::SortClause.new(
+            field:,
+            direction: (clause.unary == "+" ? :asc : :desc),
+            origin: clause
+          )
+        end
+      end
+    end
+
+    fallback do |clause|
+      QueryGrammar::Ast::MatchClause.new(
+        field: :title,
+        value: "#{ clause.unary }#{ clause.prefix }:#{ clause.values.join ' ' }",
+        origin: clause
+      )
+    end
+
+    # Default fields to search on
+    default :title, :tags, :uri
+  end
+
+  use_compiler QueryGrammar::Compiler::Arel.new(Bookmark)
+
+  execute_query do |compiled_query|
+    # pp compiled_query
+    Bookmark.left_joins(:tags)
+      .where(compiled_query[:query])
+      .order(compiled_query[:order])
+  end
+end