notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) big performance fixes

Browse Source
This commit is contained in:
Bryce Covert
2023-03-23 20:53:54 -07:00
parent e810612fbb
commit 7fd3528d0f
5 changed files with 100 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
iol_ion/src/iol_ion/tx.clj
View File

@@ -164,8 +164,7 @@
(assert (:journal-entry/client ledger-entry) "Must at least provide client when updating ledger")
(assert (every? :journal-entry-line/account (:journal-entry/line-items ledger-entry)) "must at least provide account when updating ledger")
(assert (every? :journal-entry-line/location (:journal-entry/line-items ledger-entry)) "Must at least provide location when updating ledger")
(let [
extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)]
(let [extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)]
(dc/pull db extant-read [:journal-entry/original-entity original-entity]))
(when-let [external-id (:journal-entry/external-id ledger-entry)]
(dc/pull db extant-read [:journal-entry/external-id external-id])))

13
src/clj/auto_ap/datomic.clj
View File

@@ -636,7 +636,7 @@
true (update :tempids merge (:tempids tx-result)))))
{}
(partition-all 50 txes))))
(partition-all 200 txes))))
(defn audit-transact [txes id]
(dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx"
@@ -664,10 +664,13 @@
(str (UUID/randomUUID)))
(defn pull-id [db id]
(ffirst (dc/q '[:find ?i
:in $ ?i]
db
id)))
(if (sequential? id)
(ffirst (dc/q '[:find ?i
:in $ [?a ?v]
:where [?i ?a ?v]]
db
id))
id))
(defn pull-attr [db k id]
(get (dc/pull db [k] id) k))

5
src/clj/auto_ap/datomic/vendors.clj
View File

@@ -54,8 +54,9 @@
(not (str/blank? (:name-like args)))
(merge-query {:query {:in ['?name-like]
:where ['[?e :vendor/name ?n]
'[(re-find ?name-like ?n)]]}
:args [(re-pattern (str "(?i)" (:name-like args)))]})
'[(re-pattern ?name-like) ?name-like-2]
'[(re-find ?name-like-2 ?n)]]}
:args [(str "(?i)" (:name-like args))]})
true
(merge-query {:query {:find ['?e]

127
src/clj/auto_ap/ledger.clj
View File

@@ -573,70 +573,74 @@
:starting-at starting-at
:location location})))))
(defn find-running-balance-start [{:keys [client account location starting-at]} db ]
(let [client (pull-id db client)
account (pull-id db account)]
(or
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date [client account location starting-at]]
:reverse true
:limit 500})
(take-while (fn [result]
(= [client
account
location]
(take 3 (:journal-entry-line/client+account+location+date result)))))
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
(>= (compare date starting-at) 0)))
first
:journal-entry-line/running-balance)
0.0)))
(defn find-running-balance-start [account-needing-rebuild db ]
(or
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
[(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)
(:starting-at account-needing-rebuild)]]
:reverse true
:limit 500})
(take-while (fn [result]
(= [(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)]
(take 3 (:journal-entry-line/client+account+location+date result)))))
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
(>= (compare date (:starting-at account-needing-rebuild)) 0)))
first
:journal-entry-line/running-balance
)
0.0))
(defn get-dirty-entries [account-needing-rebuild db ]
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
[(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)
(:starting-at account-needing-rebuild)]]
})
(take-while (fn [result]
(= [(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)]
(take 3 (:journal-entry-line/client+account+location+date result)))))
(map (fn [result]
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))))
(defn get-dirty-entries [{:keys [client account location starting-at]} db ]
(mu/trace ::get-dirty-entries
[]
(let [client (pull-id db client)
account (pull-id db account)]
(into []
(comp
(mapcat (fn [i]
(dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
[client account location starting-at]]
:offset (* i 1000)
:limit 1000}
))
)
(take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}]
(and
(= client result-client)
(= account result-account)
(= location result-location))))
(map (fn [result]
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ])))
(range)))))
(defn compute-running-balance [account-needing-refresh]
(mu/log ::compute
:dirty-count (count (:dirty-entries account-needing-refresh)))
(second
(reduce
(fn [[running-balance rows] [id debit credit] ]
(let [new-running-balance (+ running-balance
(if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account-type account-needing-refresh))
(- debit credit)
(- credit debit)))]
[new-running-balance
(conj rows
{:db/id id
:journal-entry-line/running-balance new-running-balance
:journal-entry-line/dirty false})]))
(mu/trace ::compute
[:dirty-count (count (:dirty-entries account-needing-refresh))]
(second
(reduce
(fn [[running-balance rows] [id debit credit] ]
(let [new-running-balance (+ running-balance
(if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account-type account-needing-refresh))
(- debit credit)
(- credit debit)))]
[new-running-balance
(conj rows
{:db/id id
:journal-entry-line/running-balance new-running-balance
:journal-entry-line/dirty false})]))
[(:build-from account-needing-refresh) []]
(:dirty-entries account-needing-refresh))))
[(:build-from account-needing-refresh) []]
(:dirty-entries account-needing-refresh)))))
(defn refresh-running-balance-cache
@@ -653,7 +657,7 @@
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
(when (seq accounts-needing-rebuild)
(mu/log ::found-accounts-needing-rebuild
:accounts accounts-needing-rebuild)
:accounts (count accounts-needing-rebuild))
(audit-transact-batch
(->> accounts-needing-rebuild
(mapcat (fn [account-needing-rebuild]
@@ -666,6 +670,7 @@
{:user/name "running-balance-cache"}))))))))
;; TODO only enable once IOL is set up in clod
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
#_(mount/defstate running-balance-cache-worker
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))

32
things-to-search-for.txt
View File

@@ -1,15 +1,9 @@
or-join syntax changed?
Make sure no history on ledger
regex no longer supported as argument for query. Check vendor datomic for how to do it right.
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity
Wrap tests around every api call
Fix tests
Make a different solution for running balance cache
* Store the running balance on the line
* Make ledger changes mark as dirty
* Recompute the cache periodically, somewhat randomly to avoid races
* Have a background job recompute all at night.
Fix searching
* indexing should happen more regularly, and just look for changes since last time it was run
@@ -17,9 +11,27 @@ Fix searching
Running Balance Cache
* Add tests for upsert-ledger
* try again to see if we can get upsert-ledger into the same transaction, making it all or nothing
* make rest of rebuilding the cache use new index
* ensure somehow that the index is always right
* Make a new way to reset the entire cache for a client
Address memory
* JVM settings now and in prod
Release steps:
Stop prod
Make database snapshot
Create new database for prod-cloud (just called prod)
Restore database
Transact new schema
(reset-client+account+location+date)
(force-rebuild-running-balance-cache)
Merge branch into master
Rename prod-cloud to prod everywhere
Release again
Sanity checks later:
* Run query
Future improvements:
Make reports just be based on running-balances