Skip to content

Commit

Permalink
Use attrs in query
Browse files Browse the repository at this point in the history
  • Loading branch information
@tonsky
tonsky committed Jan 17, 2025
1 parent a3682f1 commit 3a7869c
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 37 deletions.
6 changes: 3 additions & 3 deletions server/src/instant/db/model/triple.clj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
(ns instant.db.model.triple
(:require
[clojure.spec.alpha :as s]
[clojure.string :as string]
[honey.sql :as hsql]
[instant.data.constants :refer [empty-app-id]]
[instant.db.model.attr :as attr-model]
Expand All @@ -12,7 +11,8 @@
[instant.util.json :refer [->json <-json]]
[instant.util.spec :as uspec]
[instant.util.string :refer [multiline->single-line]]
[instant.util.tracer :as tracer])
[instant.util.tracer :as tracer]
[tool])
(:import
(java.util UUID)
(java.time Instant LocalDate LocalDateTime ZonedDateTime ZoneOffset)
Expand Down Expand Up @@ -515,7 +515,7 @@
:returning :*}
params {:app-id app-id
:system-catalog-app-id system-catalog-app-id}]
(tool/time* "delete-entity!"
(tool/time* (str "delete-entity-multi! " (count id+etypes) " entities")
(sql/do-execute! conn (hsql/format query {:params params})))))

(defn delete-multi!
Expand Down
2 changes: 1 addition & 1 deletion server/src/instant/db/permissioned_transaction.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@
(let [tx-steps (->> tx-steps
(tx/resolve-lookups-for-delete-entity tx-conn app-id)
(tx/resolve-etypes-for-delete-entity tx-conn app-id)
(tx/expand-delete-entity-cascade tx-conn app-id))]
(tx/expand-delete-entity-cascade tx-conn app-id attrs))]

;; transact does read and then a write.
;; We need to protect against a case where a different
Expand Down
66 changes: 37 additions & 29 deletions server/src/instant/db/transaction.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,16 @@
[instant.db.model.attr :as attr-model]
[instant.db.model.transaction :as transaction-model]
[instant.db.model.triple :as triple-model]
[instant.jdbc.aurora :as aurora]
[instant.jdbc.sql :as sql]
[instant.system-catalog :refer [system-catalog-app-id]]
[instant.util.coll :as coll]
[instant.util.exception :as ex]
[instant.util.e2e-tracer :as e2e-tracer]
[instant.util.json :refer [->json]]
[instant.util.tracer :as tracer]
[next.jdbc :as next-jdbc]))
[next.jdbc :as next-jdbc]
[tool]))

(s/def ::add-triple-step
(s/cat :op #{:add-triple} :triple ::triple-model/triple))
Expand Down Expand Up @@ -165,16 +167,15 @@
[:= :triples.attr_id :lookups.attr_id]
[:= :triples.value [:cast :lookups.value :jsonb]]]]
:select [:triples.attr_id :triples.value :triples.entity_id]}
results (tool/time* "resolve execute"
results (tool/time* "resolve-lookups"
(sql/do-execute! conn (hsql/format query)))]

(into {}
(for [{:triples/keys [attr_id value entity_id]} results]
[[attr_id value] entity_id])))))

(comment
(resolve-lookups
(instant.jdbc.aurora/conn-pool)
(aurora/conn-pool)
#uuid "92cb730c-8b4f-46ef-9925-4fab953694c6"
[[#uuid "20b65ea3-faad-4e80-863e-87468ff7792f" "[email protected]"]
[#uuid "6a089759-2a2f-4898-9bb8-a7bc9f6f791a" "stopa"]]))
Expand All @@ -184,20 +185,20 @@
[conn app-id entity-ids]
(if (empty? entity-ids)
{}
(->> {:from :triples
:join [:attrs [:= :triples.attr_id :attrs.id]
:idents [:= :attrs.forward_ident :idents.id]]
:where [:and
[:= :triples.app_id app-id]
[:in :triples.entity_id entity-ids]]
:select-distinct [:triples.entity_id :idents.etype]}
hsql/format
(sql/do-execute! conn)
(coll/group-by-to :triples/entity_id :idents/etype))))
(let [query {:from :triples
:join [:attrs [:= :triples.attr_id :attrs.id]
:idents [:= :attrs.forward_ident :idents.id]]
:where [:and
[:= :triples.app_id app-id]
[:in :triples.entity_id entity-ids]]
:select-distinct [:triples.entity_id :idents.etype]}
res (tool/time* "resolve-etypes"
(sql/do-execute! conn (hsql/format query)))]
(coll/group-by-to :triples/entity_id :idents/etype res))))

(comment
(resolve-etypes
(instant.jdbc.aurora/conn-pool)
(aurora/conn-pool)
#uuid "0e563ace-d25f-44f6-ae00-4e6b9b6d1b2e"
[#uuid "4d39508b-9ee2-48a3-b70d-8192d9c5a059"
#uuid "005a8767-c0e7-4158-bb9a-62ce1a5858ed"
Expand Down Expand Up @@ -228,28 +229,35 @@
[op eid etype'])]
(concat delete-entity-steps' tx-steps')))

(defn expand-delete-entity-cascade [conn app-id tx-steps]
(defn expand-delete-entity-cascade [conn app-id attrs tx-steps]
(let [[delete-entity-steps tx-steps'] (coll/split-by #(= :delete-entity (first %)) tx-steps)]
(if (empty? delete-entity-steps)
tx-steps
(let [ids+etypes (map next delete-entity-steps)
attrs+etypes (->> attrs
(filter #(= :ref (:value-type %)))
(filter #(= :cascade (:on-delete %)))
(mapv #(vector (:id %) (-> % :forward-identity second) (-> % :reverse-identity second))))
query+args (hsql/format
{:with-recursive [[[:entids {:columns [:entity_id :etype]}]
{:union [{:values ids+etypes}
{:from :triples
:join [:attrs [:= :triples.attr_id :attrs.id]
[:idents :forward_ident] [:= :forward_ident.id :attrs.forward_ident]
[:idents :reverse_ident] [:= :reverse_ident.id :attrs.reverse_ident]
:entids [:= [:to_jsonb :entids.entity_id] :triples.value]]
:where [:and
[:= :triples.app_id app-id]
[:= :attrs.on_delete [:cast "cascade" :attr_on_delete]]
[:= :attrs.value_type "ref"]
:triples.vae
[:= :entids.etype :reverse_ident.etype]]
:select [:triples.entity_id :forward_ident.etype]}]}]]
{:from :entids
:join [:refs [:and
;; TODO entity_id/value_ref join instead
[:= [:to_jsonb :entids.entity_id] :refs.value]
[:= :entids.etype :refs.reverse_etype]]]
:select [:refs.entity_id :refs.forward_etype]}]}]
[[:cascade {:columns [:id :forward_etype :reverse_etype]}]
{:values attrs+etypes}]
[:refs {:select [:triples.entity_id :triples.value :cascade.forward_etype :cascade.reverse_etype]
:from :triples
:join [:cascade [:= :triples.attr_id :cascade.id]]
:where [:and
:triples.vae
[:= :triples.app_id app-id]]}]]
:from :entids
:select :*})
#__ #_(println (sql/analyze conn query+args))
res (tool/time* "expand-delete-entity-cascade"
(sql/execute! conn query+args))
ids+etypes' (map (juxt :entity_id :etype) res)
Expand All @@ -270,7 +278,7 @@
(->> tx-steps
(resolve-lookups-for-delete-entity conn app-id)
(resolve-etypes-for-delete-entity conn app-id)
(expand-delete-entity-cascade conn app-id))
(expand-delete-entity-cascade conn app-id attrs))
tx-steps)

results
Expand Down
7 changes: 7 additions & 0 deletions server/src/instant/jdbc/sql.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,13 @@
:return-keys true})
(defsql do-execute! next-jdbc/execute! :write {:return-keys false})

(defn analyze [conn query]
(-> query
(update 0 #(str "EXPLAIN ANALYZE " %))
(->> (execute! conn)
(mapcat vals)
(string/join "\n"))))

(defn patch-hikari []
;; Hikari will send an extra query to ensure the connection is valid
;; if it has been idle for half a second. This raises the limit so
Expand Down
9 changes: 5 additions & 4 deletions server/test/instant/db/transaction_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
[instant.model.rule :as rule-model]
[instant.util.instaql :refer [instaql-nodes->object-tree]]
[instant.util.exception :as ex]
[instant.util.test :refer [instant-ex-data pretty-perm-q]])
[instant.util.test :refer [instant-ex-data pretty-perm-q]]
[tool])
(:import
(java.util UUID)))

Expand Down Expand Up @@ -2565,11 +2566,11 @@
:current-user nil}
tx-steps [[:delete-entity root-user-id "users"]]
#_#_res (tx/transact! (aurora/conn-pool) (attr-model/get-by-app-id app-id) app-id tx-steps)
res (permissioned-tx/transact! ctx tx-steps)
res (tool/time* "on-delete-cascade-perf"
(permissioned-tx/transact! ctx tx-steps))
dt (-> (System/nanoTime) (- t0) (/ 1000000.0))
deleted-triples (count (:delete-entity (:results res)))]
(is (= (-> @children (* 2) (+ 1)) deleted-triples))
(is (< dt 500)))))))

#_(is (< dt 500)))))))
(comment
(test/run-tests *ns*))

0 comments on commit 3a7869c

Please sign in to comment.