(cloud) big performance fixes
This commit is contained in:
@@ -636,7 +636,7 @@
|
||||
true (update :tempids merge (:tempids tx-result)))))
|
||||
|
||||
{}
|
||||
(partition-all 50 txes))))
|
||||
(partition-all 200 txes))))
|
||||
|
||||
(defn audit-transact [txes id]
|
||||
(dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx"
|
||||
@@ -664,10 +664,13 @@
|
||||
(str (UUID/randomUUID)))
|
||||
|
||||
(defn pull-id [db id]
|
||||
(ffirst (dc/q '[:find ?i
|
||||
:in $ ?i]
|
||||
db
|
||||
id)))
|
||||
(if (sequential? id)
|
||||
(ffirst (dc/q '[:find ?i
|
||||
:in $ [?a ?v]
|
||||
:where [?i ?a ?v]]
|
||||
db
|
||||
id))
|
||||
id))
|
||||
|
||||
(defn pull-attr [db k id]
|
||||
(get (dc/pull db [k] id) k))
|
||||
|
||||
@@ -54,8 +54,9 @@
|
||||
(not (str/blank? (:name-like args)))
|
||||
(merge-query {:query {:in ['?name-like]
|
||||
:where ['[?e :vendor/name ?n]
|
||||
'[(re-find ?name-like ?n)]]}
|
||||
:args [(re-pattern (str "(?i)" (:name-like args)))]})
|
||||
'[(re-pattern ?name-like) ?name-like-2]
|
||||
'[(re-find ?name-like-2 ?n)]]}
|
||||
:args [(str "(?i)" (:name-like args))]})
|
||||
|
||||
true
|
||||
(merge-query {:query {:find ['?e]
|
||||
|
||||
@@ -573,70 +573,74 @@
|
||||
:starting-at starting-at
|
||||
:location location})))))
|
||||
|
||||
(defn find-running-balance-start [{:keys [client account location starting-at]} db ]
|
||||
(let [client (pull-id db client)
|
||||
account (pull-id db account)]
|
||||
(or
|
||||
(->> (dc/index-pull db
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
|
||||
:start [:journal-entry-line/client+account+location+date [client account location starting-at]]
|
||||
:reverse true
|
||||
:limit 500})
|
||||
(take-while (fn [result]
|
||||
(= [client
|
||||
account
|
||||
location]
|
||||
(take 3 (:journal-entry-line/client+account+location+date result)))))
|
||||
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
|
||||
(>= (compare date starting-at) 0)))
|
||||
first
|
||||
:journal-entry-line/running-balance)
|
||||
0.0)))
|
||||
|
||||
(defn find-running-balance-start [account-needing-rebuild db ]
|
||||
(or
|
||||
(->> (dc/index-pull db
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
|
||||
:start [:journal-entry-line/client+account+location+date
|
||||
[(:client account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild)
|
||||
(:starting-at account-needing-rebuild)]]
|
||||
|
||||
:reverse true
|
||||
:limit 500})
|
||||
(take-while (fn [result]
|
||||
(= [(:client account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild)]
|
||||
(take 3 (:journal-entry-line/client+account+location+date result)))))
|
||||
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
|
||||
(>= (compare date (:starting-at account-needing-rebuild)) 0)))
|
||||
first
|
||||
:journal-entry-line/running-balance
|
||||
)
|
||||
0.0))
|
||||
|
||||
(defn get-dirty-entries [account-needing-rebuild db ]
|
||||
(->> (dc/index-pull db
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
|
||||
:start [:journal-entry-line/client+account+location+date
|
||||
[(:client account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild)
|
||||
(:starting-at account-needing-rebuild)]]
|
||||
})
|
||||
(take-while (fn [result]
|
||||
(= [(:client account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild)]
|
||||
(take 3 (:journal-entry-line/client+account+location+date result)))))
|
||||
(map (fn [result]
|
||||
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))))
|
||||
(defn get-dirty-entries [{:keys [client account location starting-at]} db ]
|
||||
(mu/trace ::get-dirty-entries
|
||||
[]
|
||||
(let [client (pull-id db client)
|
||||
account (pull-id db account)]
|
||||
(into []
|
||||
(comp
|
||||
(mapcat (fn [i]
|
||||
(dc/index-pull db
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
|
||||
:start [:journal-entry-line/client+account+location+date
|
||||
[client account location starting-at]]
|
||||
:offset (* i 1000)
|
||||
:limit 1000}
|
||||
))
|
||||
)
|
||||
|
||||
(take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}]
|
||||
(and
|
||||
(= client result-client)
|
||||
(= account result-account)
|
||||
(= location result-location))))
|
||||
(map (fn [result]
|
||||
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ])))
|
||||
(range)))))
|
||||
|
||||
(defn compute-running-balance [account-needing-refresh]
|
||||
(mu/log ::compute
|
||||
:dirty-count (count (:dirty-entries account-needing-refresh)))
|
||||
(second
|
||||
(reduce
|
||||
(fn [[running-balance rows] [id debit credit] ]
|
||||
(let [new-running-balance (+ running-balance
|
||||
(if (#{:account-type/asset
|
||||
:account-type/dividend
|
||||
:account-type/expense} (:account-type account-needing-refresh))
|
||||
(- debit credit)
|
||||
(- credit debit)))]
|
||||
[new-running-balance
|
||||
(conj rows
|
||||
{:db/id id
|
||||
:journal-entry-line/running-balance new-running-balance
|
||||
:journal-entry-line/dirty false})]))
|
||||
(mu/trace ::compute
|
||||
[:dirty-count (count (:dirty-entries account-needing-refresh))]
|
||||
(second
|
||||
(reduce
|
||||
(fn [[running-balance rows] [id debit credit] ]
|
||||
(let [new-running-balance (+ running-balance
|
||||
(if (#{:account-type/asset
|
||||
:account-type/dividend
|
||||
:account-type/expense} (:account-type account-needing-refresh))
|
||||
(- debit credit)
|
||||
(- credit debit)))]
|
||||
[new-running-balance
|
||||
(conj rows
|
||||
{:db/id id
|
||||
:journal-entry-line/running-balance new-running-balance
|
||||
:journal-entry-line/dirty false})]))
|
||||
|
||||
[(:build-from account-needing-refresh) []]
|
||||
(:dirty-entries account-needing-refresh))))
|
||||
[(:build-from account-needing-refresh) []]
|
||||
(:dirty-entries account-needing-refresh)))))
|
||||
|
||||
|
||||
(defn refresh-running-balance-cache
|
||||
@@ -653,7 +657,7 @@
|
||||
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
|
||||
(when (seq accounts-needing-rebuild)
|
||||
(mu/log ::found-accounts-needing-rebuild
|
||||
:accounts accounts-needing-rebuild)
|
||||
:accounts (count accounts-needing-rebuild))
|
||||
(audit-transact-batch
|
||||
(->> accounts-needing-rebuild
|
||||
(mapcat (fn [account-needing-rebuild]
|
||||
@@ -666,6 +670,7 @@
|
||||
{:user/name "running-balance-cache"}))))))))
|
||||
|
||||
|
||||
;; TODO only enable once IOL is set up in clod
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
#_(mount/defstate running-balance-cache-worker
|
||||
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
|
||||
|
||||
Reference in New Issue
Block a user