From 3a7869cd18ee723abeeb4775a02ef7cdbe5b054b Mon Sep 17 00:00:00 2001 From: Nikita Prokopov Date: Fri, 17 Jan 2025 20:56:10 +0100 Subject: [PATCH] Use attrs in query --- server/src/instant/db/model/triple.clj | 6 +- .../instant/db/permissioned_transaction.clj | 2 +- server/src/instant/db/transaction.clj | 66 +++++++++++-------- server/src/instant/jdbc/sql.clj | 7 ++ server/test/instant/db/transaction_test.clj | 9 +-- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/server/src/instant/db/model/triple.clj b/server/src/instant/db/model/triple.clj index eb4e7e4c..d5debce5 100644 --- a/server/src/instant/db/model/triple.clj +++ b/server/src/instant/db/model/triple.clj @@ -1,7 +1,6 @@ (ns instant.db.model.triple (:require [clojure.spec.alpha :as s] - [clojure.string :as string] [honey.sql :as hsql] [instant.data.constants :refer [empty-app-id]] [instant.db.model.attr :as attr-model] @@ -12,7 +11,8 @@ [instant.util.json :refer [->json <-json]] [instant.util.spec :as uspec] [instant.util.string :refer [multiline->single-line]] - [instant.util.tracer :as tracer]) + [instant.util.tracer :as tracer] + [tool]) (:import (java.util UUID) (java.time Instant LocalDate LocalDateTime ZonedDateTime ZoneOffset) @@ -515,7 +515,7 @@ :returning :*} params {:app-id app-id :system-catalog-app-id system-catalog-app-id}] - (tool/time* "delete-entity!" + (tool/time* (str "delete-entity-multi! " (count id+etypes) " entities") (sql/do-execute! conn (hsql/format query {:params params}))))) (defn delete-multi! diff --git a/server/src/instant/db/permissioned_transaction.clj b/server/src/instant/db/permissioned_transaction.clj index 81ea9512..b80824fd 100644 --- a/server/src/instant/db/permissioned_transaction.clj +++ b/server/src/instant/db/permissioned_transaction.clj @@ -613,7 +613,7 @@ (let [tx-steps (->> tx-steps (tx/resolve-lookups-for-delete-entity tx-conn app-id) (tx/resolve-etypes-for-delete-entity tx-conn app-id) - (tx/expand-delete-entity-cascade tx-conn app-id))] + (tx/expand-delete-entity-cascade tx-conn app-id attrs))] ;; transact does read and then a write. ;; We need to protect against a case where a different diff --git a/server/src/instant/db/transaction.clj b/server/src/instant/db/transaction.clj index b88b0d61..a4472d0e 100644 --- a/server/src/instant/db/transaction.clj +++ b/server/src/instant/db/transaction.clj @@ -8,6 +8,7 @@ [instant.db.model.attr :as attr-model] [instant.db.model.transaction :as transaction-model] [instant.db.model.triple :as triple-model] + [instant.jdbc.aurora :as aurora] [instant.jdbc.sql :as sql] [instant.system-catalog :refer [system-catalog-app-id]] [instant.util.coll :as coll] @@ -15,7 +16,8 @@ [instant.util.e2e-tracer :as e2e-tracer] [instant.util.json :refer [->json]] [instant.util.tracer :as tracer] - [next.jdbc :as next-jdbc])) + [next.jdbc :as next-jdbc] + [tool])) (s/def ::add-triple-step (s/cat :op #{:add-triple} :triple ::triple-model/triple)) @@ -165,16 +167,15 @@ [:= :triples.attr_id :lookups.attr_id] [:= :triples.value [:cast :lookups.value :jsonb]]]] :select [:triples.attr_id :triples.value :triples.entity_id]} - results (tool/time* "resolve execute" + results (tool/time* "resolve-lookups" (sql/do-execute! conn (hsql/format query)))] - (into {} (for [{:triples/keys [attr_id value entity_id]} results] [[attr_id value] entity_id]))))) (comment (resolve-lookups - (instant.jdbc.aurora/conn-pool) + (aurora/conn-pool) #uuid "92cb730c-8b4f-46ef-9925-4fab953694c6" [[#uuid "20b65ea3-faad-4e80-863e-87468ff7792f" "joe@instantdb.com"] [#uuid "6a089759-2a2f-4898-9bb8-a7bc9f6f791a" "stopa"]])) @@ -184,20 +185,20 @@ [conn app-id entity-ids] (if (empty? entity-ids) {} - (->> {:from :triples - :join [:attrs [:= :triples.attr_id :attrs.id] - :idents [:= :attrs.forward_ident :idents.id]] - :where [:and - [:= :triples.app_id app-id] - [:in :triples.entity_id entity-ids]] - :select-distinct [:triples.entity_id :idents.etype]} - hsql/format - (sql/do-execute! conn) - (coll/group-by-to :triples/entity_id :idents/etype)))) + (let [query {:from :triples + :join [:attrs [:= :triples.attr_id :attrs.id] + :idents [:= :attrs.forward_ident :idents.id]] + :where [:and + [:= :triples.app_id app-id] + [:in :triples.entity_id entity-ids]] + :select-distinct [:triples.entity_id :idents.etype]} + res (tool/time* "resolve-etypes" + (sql/do-execute! conn (hsql/format query)))] + (coll/group-by-to :triples/entity_id :idents/etype res)))) (comment (resolve-etypes - (instant.jdbc.aurora/conn-pool) + (aurora/conn-pool) #uuid "0e563ace-d25f-44f6-ae00-4e6b9b6d1b2e" [#uuid "4d39508b-9ee2-48a3-b70d-8192d9c5a059" #uuid "005a8767-c0e7-4158-bb9a-62ce1a5858ed" @@ -228,28 +229,35 @@ [op eid etype'])] (concat delete-entity-steps' tx-steps'))) -(defn expand-delete-entity-cascade [conn app-id tx-steps] +(defn expand-delete-entity-cascade [conn app-id attrs tx-steps] (let [[delete-entity-steps tx-steps'] (coll/split-by #(= :delete-entity (first %)) tx-steps)] (if (empty? delete-entity-steps) tx-steps (let [ids+etypes (map next delete-entity-steps) + attrs+etypes (->> attrs + (filter #(= :ref (:value-type %))) + (filter #(= :cascade (:on-delete %))) + (mapv #(vector (:id %) (-> % :forward-identity second) (-> % :reverse-identity second)))) query+args (hsql/format {:with-recursive [[[:entids {:columns [:entity_id :etype]}] {:union [{:values ids+etypes} - {:from :triples - :join [:attrs [:= :triples.attr_id :attrs.id] - [:idents :forward_ident] [:= :forward_ident.id :attrs.forward_ident] - [:idents :reverse_ident] [:= :reverse_ident.id :attrs.reverse_ident] - :entids [:= [:to_jsonb :entids.entity_id] :triples.value]] - :where [:and - [:= :triples.app_id app-id] - [:= :attrs.on_delete [:cast "cascade" :attr_on_delete]] - [:= :attrs.value_type "ref"] - :triples.vae - [:= :entids.etype :reverse_ident.etype]] - :select [:triples.entity_id :forward_ident.etype]}]}]] + {:from :entids + :join [:refs [:and + ;; TODO entity_id/value_ref join instead + [:= [:to_jsonb :entids.entity_id] :refs.value] + [:= :entids.etype :refs.reverse_etype]]] + :select [:refs.entity_id :refs.forward_etype]}]}] + [[:cascade {:columns [:id :forward_etype :reverse_etype]}] + {:values attrs+etypes}] + [:refs {:select [:triples.entity_id :triples.value :cascade.forward_etype :cascade.reverse_etype] + :from :triples + :join [:cascade [:= :triples.attr_id :cascade.id]] + :where [:and + :triples.vae + [:= :triples.app_id app-id]]}]] :from :entids :select :*}) + #__ #_(println (sql/analyze conn query+args)) res (tool/time* "expand-delete-entity-cascade" (sql/execute! conn query+args)) ids+etypes' (map (juxt :entity_id :etype) res) @@ -270,7 +278,7 @@ (->> tx-steps (resolve-lookups-for-delete-entity conn app-id) (resolve-etypes-for-delete-entity conn app-id) - (expand-delete-entity-cascade conn app-id)) + (expand-delete-entity-cascade conn app-id attrs)) tx-steps) results diff --git a/server/src/instant/jdbc/sql.clj b/server/src/instant/jdbc/sql.clj index a9a4061c..180dbb3c 100644 --- a/server/src/instant/jdbc/sql.clj +++ b/server/src/instant/jdbc/sql.clj @@ -262,6 +262,13 @@ :return-keys true}) (defsql do-execute! next-jdbc/execute! :write {:return-keys false}) +(defn analyze [conn query] + (-> query + (update 0 #(str "EXPLAIN ANALYZE " %)) + (->> (execute! conn) + (mapcat vals) + (string/join "\n")))) + (defn patch-hikari [] ;; Hikari will send an extra query to ensure the connection is valid ;; if it has been idle for half a second. This raises the limit so diff --git a/server/test/instant/db/transaction_test.clj b/server/test/instant/db/transaction_test.clj index 6dbcc66d..4d20f655 100644 --- a/server/test/instant/db/transaction_test.clj +++ b/server/test/instant/db/transaction_test.clj @@ -19,7 +19,8 @@ [instant.model.rule :as rule-model] [instant.util.instaql :refer [instaql-nodes->object-tree]] [instant.util.exception :as ex] - [instant.util.test :refer [instant-ex-data pretty-perm-q]]) + [instant.util.test :refer [instant-ex-data pretty-perm-q]] + [tool]) (:import (java.util UUID))) @@ -2565,11 +2566,11 @@ :current-user nil} tx-steps [[:delete-entity root-user-id "users"]] #_#_res (tx/transact! (aurora/conn-pool) (attr-model/get-by-app-id app-id) app-id tx-steps) - res (permissioned-tx/transact! ctx tx-steps) + res (tool/time* "on-delete-cascade-perf" + (permissioned-tx/transact! ctx tx-steps)) dt (-> (System/nanoTime) (- t0) (/ 1000000.0)) deleted-triples (count (:delete-entity (:results res)))] (is (= (-> @children (* 2) (+ 1)) deleted-triples)) - (is (< dt 500))))))) - + #_(is (< dt 500))))))) (comment (test/run-tests *ns*))