(cloud) Added running balance cache for ledger
This commit is contained in:
@@ -8,7 +8,7 @@
|
||||
[auto-ap.time :as atime]
|
||||
[auto-ap.ledger.reports :as l-reports]
|
||||
[auto-ap.graphql.utils
|
||||
:refer [->graphql <-graphql assert-admin assert-can-see-client result->page]]
|
||||
:refer [->graphql <-graphql assert-admin assert-can-see-client result->page attach-tracing-resolvers]]
|
||||
[auto-ap.parse.util :as parse]
|
||||
[auto-ap.pdf.ledger :refer [print-balance-sheet print-pnl print-journal-detail-report]]
|
||||
[auto-ap.utils :refer [by dollars= heartbeat]]
|
||||
@@ -19,13 +19,9 @@
|
||||
[datomic.client.api :as dc]
|
||||
[mount.core :as mount]
|
||||
[com.brunobonacci.mulog :as mu]
|
||||
[yang.scheduler :as scheduler]
|
||||
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]])
|
||||
[yang.scheduler :as scheduler])
|
||||
(:import [org.apache.commons.codec.binary Base64]))
|
||||
|
||||
(mount/defstate running-balance-cache
|
||||
:start (atom {}))
|
||||
|
||||
(defn get-ledger-page [context args _]
|
||||
(let [args (assoc args :id (:id context))
|
||||
[journal-entries journal-entries-count] (l/get-graphql (assoc (<-graphql (:filters args))
|
||||
@@ -34,15 +30,7 @@
|
||||
journal-entries (mapv
|
||||
(fn [je]
|
||||
(-> je
|
||||
(update :journal-entry/original-entity :db/id)
|
||||
(update :journal-entry/line-items
|
||||
(fn [jels]
|
||||
(mapv
|
||||
(fn [jel]
|
||||
(assoc jel :running-balance (get-in @running-balance-cache [(:db/id (:journal-entry/client je))
|
||||
(:db/id jel)])))
|
||||
|
||||
jels)))))
|
||||
(update :journal-entry/original-entity :db/id)))
|
||||
journal-entries)]
|
||||
(result->page journal-entries journal-entries-count :journal_entries (:filters args))))
|
||||
|
||||
@@ -485,95 +473,120 @@
|
||||
:errors (map (fn [x] {:external_id (:external_id x)
|
||||
:error (:error x)}) errors)}))
|
||||
|
||||
(defn build-running-balance
|
||||
([lookup-account all-ledger-entries]
|
||||
(->> all-ledger-entries
|
||||
(reduce
|
||||
(fn [[rollup cache] [_ _ jel account location debit credit]]
|
||||
(let [rollup (-> rollup
|
||||
(update-in [[location account] :debit] (fnil + 0.0) debit)
|
||||
(update-in [[location account] :credit] (fnil + 0.0) credit)
|
||||
(update-in [[location account] :count] (fnil + 0) 1))]
|
||||
[rollup
|
||||
(assoc cache jel (assoc (get rollup [location account]) :account-id account))]))
|
||||
[{} {}])
|
||||
(second)
|
||||
(reduce-kv
|
||||
(fn [acc jel {:keys [debit credit account-id]}]
|
||||
(let [account (lookup-account account-id)
|
||||
account-type (:account_type account)]
|
||||
(assoc acc jel
|
||||
(if account-type (if (#{:account-type/asset
|
||||
(defn accounts-needing-rebuild [ db client]
|
||||
(->> (dc/qseq '[:find ?c ?a ?l (min ?d)
|
||||
:in $ ?c
|
||||
:where [?je :journal-entry/client ?c]
|
||||
[?je :journal-entry/line-items ?jel]
|
||||
(or (not [?jel :journal-entry-line/running-balance])
|
||||
[?jel :journal-entry-line/dirty true])
|
||||
[?jel :journal-entry-line/account ?a]
|
||||
[?jel :journal-entry-line/location ?l]
|
||||
[?je :journal-entry/date ?d]]
|
||||
db
|
||||
client)
|
||||
(map (fn [[client account location starting-at ]]
|
||||
{:client client
|
||||
:account account
|
||||
:starting-at starting-at
|
||||
:location location}))))
|
||||
|
||||
(defn find-running-balance-start [account-needing-rebuild db ]
|
||||
(let [starting-from (or (->> (dc/q '[:find ?d ?je ?jel ?rbs
|
||||
:in $ ?c ?starting-at ?a ?l
|
||||
:where
|
||||
[?je :journal-entry/client ?c]
|
||||
[?je :journal-entry/date ?d]
|
||||
[(< ?d ?starting-at)]
|
||||
[?je :journal-entry/line-items ?jel]
|
||||
[?jel :journal-entry-line/account ?a]
|
||||
[?jel :journal-entry-line/location ?l]
|
||||
[?jel :journal-entry-line/running-balance ?rbs]
|
||||
]
|
||||
db
|
||||
(:client account-needing-rebuild)
|
||||
(:starting-at account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild))
|
||||
(sort)
|
||||
(last)
|
||||
(last))
|
||||
0.0)]
|
||||
(mu/log ::starting-rebuild-at
|
||||
:at starting-from)
|
||||
starting-from))
|
||||
|
||||
(defn get-dirty-entries [account-needing-rebuild db ]
|
||||
(->> (dc/q
|
||||
'[:find ?d ?jel ?debit ?credit
|
||||
:in $ ?c ?starting-at ?a ?l
|
||||
:where
|
||||
[?e :journal-entry/client ?c]
|
||||
[?e :journal-entry/date ?d]
|
||||
[(>= ?d ?starting-at)]
|
||||
[?e :journal-entry/line-items ?jel]
|
||||
[?jel :journal-entry-line/account ?a]
|
||||
[?jel :journal-entry-line/location ?l]
|
||||
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit ]
|
||||
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]]
|
||||
db
|
||||
(:client account-needing-rebuild)
|
||||
(:starting-at account-needing-rebuild)
|
||||
(:account account-needing-rebuild)
|
||||
(:location account-needing-rebuild))
|
||||
sort
|
||||
(map #(drop 1 %))))
|
||||
|
||||
(defn compute-running-balance [account-needing-refresh]
|
||||
(mu/log ::compute
|
||||
:dirty-count (count (:dirty-entries account-needing-refresh)))
|
||||
(second
|
||||
(reduce
|
||||
(fn [[running-balance rows] [id debit credit] ]
|
||||
(let [new-running-balance (+ running-balance
|
||||
(if (#{:account-type/asset
|
||||
:account-type/dividend
|
||||
:account-type/expense} account-type)
|
||||
:account-type/expense} (:account-type account-needing-refresh))
|
||||
(- debit credit)
|
||||
(- credit debit))
|
||||
0.0))))
|
||||
|
||||
{}))))
|
||||
|
||||
(defn running-balance-for [client-id]
|
||||
(let [lookup-account (build-account-lookup client-id)]
|
||||
(->> (dc/q
|
||||
{:query {:find ['?d '?e '?jel '?account '?location '?debit '?credit]
|
||||
:in ['$ '?client-id]
|
||||
:where '[[?e :journal-entry/client ?client-id]
|
||||
[?e :journal-entry/date ?d]
|
||||
[?e :journal-entry/line-items ?jel]
|
||||
(or-join [?e]
|
||||
(and [?e :journal-entry/original-entity ?i]
|
||||
(or-join [?e ?i]
|
||||
(and
|
||||
[?i :transaction/bank-account ?b]
|
||||
(or [?b :bank-account/include-in-reports true]
|
||||
(not [?b :bank-account/include-in-reports])))
|
||||
(not [?i :transaction/bank-account])))
|
||||
(not [?e :journal-entry/original-entity ]))
|
||||
[(get-else $ ?jel :journal-entry-line/account :account/unknown) ?account]
|
||||
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit ]
|
||||
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
|
||||
[(get-else $ ?jel :journal-entry-line/location "") ?location]]
|
||||
}
|
||||
:args [(dc/db conn) client-id]})
|
||||
(sort-by (juxt first second))
|
||||
(build-running-balance lookup-account))))
|
||||
|
||||
(def last-run-running-balance (atom nil))
|
||||
|
||||
(defn build-running-balance-cache []
|
||||
(let [clients-needing-refresh (if-let [last-run @last-run-running-balance]
|
||||
(->> (dc/q
|
||||
{:query {:find ['?v]
|
||||
:in ['$ '[?tx ...]]
|
||||
:where ['[$ _ :journal-entry/client ?v ?tx]]}
|
||||
:args [(dc/history (dc/db conn))
|
||||
(map :t (mapcat :data (dc/tx-range conn {:start last-run
|
||||
:end (java.util.Date.)})))]})
|
||||
(map first)
|
||||
(into #{}))
|
||||
(into #{} (map :db/id (d-clients/get-all))))
|
||||
starting (java.util.Date.)]
|
||||
(log/info (count clients-needing-refresh) "Clients need their balance cache refreshed.")
|
||||
(swap! running-balance-cache
|
||||
merge
|
||||
(reduce
|
||||
(fn [acc client]
|
||||
(log/info "Computing running balance cache for " (pull-attr (dc/db conn) :client/code client ))
|
||||
(assoc acc client (running-balance-for client)))
|
||||
{}
|
||||
clients-needing-refresh))
|
||||
(log/info "Done refreshing " (count clients-needing-refresh) " client caches")
|
||||
(reset! last-run-running-balance starting)))
|
||||
(- credit debit)))]
|
||||
[new-running-balance
|
||||
(conj rows
|
||||
{:db/id id
|
||||
:journal-entry-line/running-balance new-running-balance
|
||||
:journal-entry-line/dirty false})]))
|
||||
|
||||
[(:build-from account-needing-refresh) []]
|
||||
(:dirty-entries account-needing-refresh))))
|
||||
|
||||
|
||||
(defn refresh-running-balance-cache []
|
||||
(build-running-balance-cache))
|
||||
(doseq [c (shuffle (map first
|
||||
(dc/q '[:find (pull ?c [:client/code :db/id])
|
||||
:where [?c :client/code]]
|
||||
(dc/db conn))))]
|
||||
(mu/trace ::building-running-balance
|
||||
[:client c]
|
||||
(mu/with-context {:client c}
|
||||
(let [db (dc/db conn)
|
||||
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
|
||||
(when (seq accounts-needing-rebuild)
|
||||
(mu/log ::found-accounts-needing-rebuild
|
||||
:accounts accounts-needing-rebuild)
|
||||
(audit-transact-batch
|
||||
(->> accounts-needing-rebuild
|
||||
(mapcat (fn [account-needing-rebuild]
|
||||
(mu/with-context {:account account-needing-rebuild}
|
||||
(-> account-needing-rebuild
|
||||
(assoc :build-from (find-running-balance-start account-needing-rebuild db))
|
||||
(assoc :dirty-entries (get-dirty-entries account-needing-rebuild db))
|
||||
(assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild))))
|
||||
(compute-running-balance))))))
|
||||
{:user/name "running-balance-cache"})))))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(mount/defstate running-balance-cache-worker
|
||||
:start (scheduler/every (* 15 60 1000) (heartbeat refresh-running-balance-cache "running-balance-cache"))
|
||||
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
|
||||
:stop (scheduler/stop running-balance-cache-worker))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user