notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) makes ledger running balances fast and smooth

Browse Source
This commit is contained in:
Bryce Covert
2023-03-23 12:18:35 -07:00
parent cfc5c561c6
commit e810612fbb
12 changed files with 679 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
iol_ion/src/iol_ion/tx.clj
View File

@@ -60,11 +60,13 @@
;; TODO unit test this ;; TODO unit test this
(defn upsert-entity [db entity] (defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id") (assert (or (:db/id entity)
(let [e (:db/id entity) (:db/ident entity))
(str "Cannot upsert without :db/id or :db/ident, " entity))
(let [e (or (:db/id entity) (:db/ident entity))
is-new? (string? e) is-new? (string? e)
extant-entity (when-not is-new? extant-entity (when-not is-new?
(dc/pull db (keys entity) (:db/id entity))) (dc/pull db (keys entity) (or (:db/id entity) (:db/ident entity))))
ident->value-type (by :db/ident (comp :db/ident ident->value-type (by :db/ident (comp :db/ident
:db/valueType) :db/valueType)
(pull-many (pull-many
@@ -78,6 +80,9 @@
(= :db/id a) (= :db/id a)
ops ops
(= :db/ident a)
ops
(or (= v (a extant-entity)) (or (= v (a extant-entity))
(= v (:db/ident (a extant-entity) :nope)) (= v (:db/ident (a extant-entity) :nope))
(= v (:db/id (a extant-entity)) :nope)) (= v (:db/id (a extant-entity)) :nope))
@@ -92,6 +97,10 @@
ops ops
;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq) ;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq)
(and (sequential? v) (= :db.type/tuple (ident->value-type a)))
(conj ops [:db/add e a v])
(and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v))
(into ops (reset-rels db e a v)) (into ops (reset-rels db e a v))
@@ -111,3 +120,101 @@
[]))] []))]
ops)) ops))
(defn min-by [sorter]
(->> sorter
sort
last
last))
(defn get-line-items-after [db journal-entry]
(for [jel (:journal-entry/line-items journal-entry)
:let [next-jel (->> (dc/index-pull db {:index :avet
:selector [:db/id :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
(:journal-entry-line/client+account+location+date jel)
(:db/id jel)]
:limit 3
})
(filter (fn line-must-match-client-account-location [result]
(and
(= (take 3 (:journal-entry-line/client+account+location+date result))
(take 3 (:journal-entry-line/client+account+location+date jel)))
(not= (:db/id jel)
(:db/id result)))
))
first
:db/id)]
:when next-jel]
next-jel))
(def extant-read '[:db/id :journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id :journal-entry-line/client+account+location+date]}])
(defn calc-client+account+location+date [je jel]
[(or
(:db/id (:journal-entry/client je))
(:journal-entry/client je))
(or (:db/id (:journal-entry-line/account jel))
(:journal-entry-line/account jel))
(-> jel :journal-entry-line/location)
(-> je :journal-entry/date)])
(defn upsert-ledger [db ledger-entry]
(assert (:journal-entry/date ledger-entry) "Must at least provide date when updating ledger")
(assert (:journal-entry/client ledger-entry) "Must at least provide client when updating ledger")
(assert (every? :journal-entry-line/account (:journal-entry/line-items ledger-entry)) "must at least provide account when updating ledger")
(assert (every? :journal-entry-line/location (:journal-entry/line-items ledger-entry)) "Must at least provide location when updating ledger")
(let [
extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)]
(dc/pull db extant-read [:journal-entry/original-entity original-entity]))
(when-let [external-id (:journal-entry/external-id ledger-entry)]
(dc/pull db extant-read [:journal-entry/external-id external-id])))
extant-entry-exists? (:db/id extant-entry)]
(cond->
(upsert-entity db (into (-> ledger-entry
(assoc :db/id (or
(:db/id ledger-entry)
(:db/id extant-entry)
(random-tempid)))
(update :journal-entry/line-items
(fn [lis]
(mapv #(-> %
(assoc :journal-entry-line/dirty true)
(assoc :journal-entry-line/client+account+location+date
(calc-client+account+location+date ledger-entry %)))
lis))))
))
extant-entry-exists? (into (map (fn [li]
{:journal-entry-line/dirty true
:db/id li})
(get-line-items-after db extant-entry))))))
(defn remove-nils [m]
(let [result (reduce-kv
(fn [m k v]
(if (not (nil? v))
(assoc m k v)
m
))
{}
m)]
(if (seq result)
result
nil)))
(defn propose-invoice [db invoice]
(let [existing? (boolean (seq (dc/q '[:find ?i
:in $ ?invoice-number ?client ?vendor
:where
[?i :invoice/invoice-number ?invoice-number]
[?i :invoice/client ?client]
[?i :invoice/vendor ?vendor]
(not [?i :invoice/status :invoice-status/voided])]
db
(:invoice/invoice-number invoice)
(:invoice/client invoice)
(:invoice/vendor invoice))))]
(if existing?
[]
[(remove-nils invoice)])))

2
project.clj
View File

@@ -116,7 +116,7 @@
[lein-ancient "0.6.15"]] [lein-ancient "0.6.15"]]
:clean-targets ^{:protect false} ["resources/public/js/compiled" "target"] :clean-targets ^{:protect false} ["resources/public/js/compiled" "target"]
:ring {:handler auto-ap.handler/app} :ring {:handler auto-ap.handler/app}
:source-paths ["src/clj" "src/cljc" "src/cljs" "iol_ion/src"] :source-paths ["iol_ion/src" "src/clj" "src/cljc" "src/cljs" ]
:resource-paths ["resources"] :resource-paths ["resources"]
:aliases {"build" ["do" "clean" ["uberjar"]] :aliases {"build" ["do" "clean" ["uberjar"]]
"fig:dev" ["run" "-m" "figwheel.main" "-b" "dev" "-r"] "fig:dev" ["run" "-m" "figwheel.main" "-b" "dev" "-r"]

7
resources/cloud-migration-schema.edn
View File

@@ -10,4 +10,11 @@
:db/doc "Whether or not this journal entry line is dirty and needs to recalculate balances", :db/doc "Whether or not this journal entry line is dirty and needs to recalculate balances",
:db/ident :journal-entry-line/dirty, :db/ident :journal-entry-line/dirty,
} }
{:db/valueType :db.type/tuple
:db/tupleTypes [:db.type/ref :db.type/ref :db.type/string :db.type/instant]
:db/cardinality :db.cardinality/one,
:db/ident :journal-entry-line/client+account+location+date
:db/doc "Used to find accounts and locations quickly",
:db/noHistory true
}
] ]

2
resources/datomic/ion-config.edn
View File

@@ -1,5 +1,7 @@
{ :allow [iol-ion.tx/upsert-entity { :allow [iol-ion.tx/upsert-entity
iol-ion.tx/reset-scalars iol-ion.tx/reset-scalars
iol-ion.tx/reset-rels iol-ion.tx/reset-rels
iol-ion.tx/upsert-ledger
iol-ion.tx/min-by
iol-ion.tx/propose-invoice] iol-ion.tx/propose-invoice]
:app-name "iol-cloud"} :app-name "iol-cloud"}

60
scratch-sessions/test_solr.clj Normal file
View File

@@ -0,0 +1,60 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(require '[clj-http.client :as client])
(require ' [clojure.data.json :as json])
(defn upsert-schema []
(client/post
"http://localhost:8983/solr/gettingstarted/schema"
{:headers {"Content-Type" "application/json"}
:method "POST"
:body (json/write-str {"add-field" [{"name" "client-id"
"type" "string"}
{"name" "order-date"
"type" "pdate"}]})
}))
(defn load-sales-orders []
(clojure.pprint/pprint
(doseq [[client code] (dc/q '[:find ?c ?code :where [?c :client/code ?code]] (dc/db conn))
:let [_ (println "loading" code)]
batch (->> (dc/qseq '[:find ?so ?date ?client-id
:in $ ?client-id
:where
[?so :sales-order/client ?client-id]
[?so :sales-order/date ?date]
]
(dc/db conn)
client)
(map (fn [[so date client-id]]
{"id" so
"order-date" (str date)
"client-id" (str client-id)}))
(partition-all 1000)
)]
(print ".")
(flush)
(client/post
"http://localhost:8983/solr/gettingstarted/update?commitWithin=60000"
{:headers {"Content-Type" "application/json"}
:method "POST"
:body (json/write-str batch)}))))
(comment
(try
(upsert-schema)
(catch Exception e
(println e)))
(load-sales-orders)
(dc/pull (dc/db conn) '[:client/code :db/id] [:client/code "NGOP"])
)

6
src/clj/auto_ap/datomic.clj
View File

@@ -663,6 +663,12 @@
(defn random-tempid [] (defn random-tempid []
(str (UUID/randomUUID))) (str (UUID/randomUUID)))
(defn pull-id [db id]
(ffirst (dc/q '[:find ?i
:in $ ?i]
db
id)))
(defn pull-attr [db k id] (defn pull-attr [db k id]
(get (dc/pull db [k] id) k)) (get (dc/pull db [k] id) k))

364
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -1,26 +1,33 @@
(ns auto-ap.graphql.ledger (ns auto-ap.graphql.ledger
(:require (:require
[auto-ap.datomic [auto-ap.datomic
:refer [audit-transact-batch conn pull-attr pull-many remove-nils]] :refer [audit-transact-batch conn pull-many remove-nils]]
[auto-ap.datomic.accounts :as a] [auto-ap.datomic.accounts :as a]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.ledger :as l] [auto-ap.datomic.ledger :as l]
[auto-ap.time :as atime]
[auto-ap.ledger.reports :as l-reports]
[auto-ap.graphql.utils [auto-ap.graphql.utils
:refer [->graphql <-graphql assert-admin assert-can-see-client result->page attach-tracing-resolvers]] :refer [->graphql
<-graphql
assert-admin
assert-can-see-client
attach-tracing-resolvers
result->page]]
[auto-ap.ledger :refer [build-account-lookup]]
[auto-ap.ledger.reports :as l-reports]
[auto-ap.parse.util :as parse] [auto-ap.parse.util :as parse]
[auto-ap.pdf.ledger :refer [print-balance-sheet print-pnl print-journal-detail-report]] [auto-ap.pdf.ledger
[auto-ap.utils :refer [by dollars= heartbeat]] :refer [print-balance-sheet print-journal-detail-report print-pnl]]
[auto-ap.time :as atime]
[auto-ap.utils :refer [by dollars=]]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clj-time.core :as t] [clj-time.core :as t]
[clojure.tools.logging :as log]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[datomic.client.api :as dc] [clojure.tools.logging :as log]
[mount.core :as mount]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
[yang.scheduler :as scheduler]) [datomic.client.api :as dc]
(:import [org.apache.commons.codec.binary Base64])) [iol-ion.tx :refer [upsert-ledger]])
(:import
(org.apache.commons.codec.binary Base64)))
(defn get-ledger-page [context args _] (defn get-ledger-page [context args _]
(let [args (assoc args :id (:id context)) (let [args (assoc args :id (:id context))
@@ -88,73 +95,35 @@
(filter (fn [[d]] (filter (fn [[d]]
(if start-date (if start-date
(and (and
(>= (compare d start-date) 0) (>= (compare d start-date) 0)
(<= (compare d end-date) 0)) (<= (compare d end-date) 0))
(<= (compare d end-date) 0)))) (<= (compare d end-date) 0))))
(reduce (reduce
(fn [acc [_ _ account location debit credit]] (fn [acc [_ _ account location debit credit]]
(-> acc (-> acc
(update-in [[location account] :debit] (fnil + 0.0) debit) (update-in [[location account] :debit] (fnil + 0.0) debit)
(update-in [[location account] :credit] (fnil + 0.0) credit) (update-in [[location account] :credit] (fnil + 0.0) credit)
(update-in [[location account] :count] (fnil + 0) 1)) (update-in [[location account] :count] (fnil + 0) 1))
) )
{}) {})
(reduce-kv (reduce-kv
(fn [acc [location account-id] {:keys [debit credit count]}] (fn [acc [location account-id] {:keys [debit credit count]}]
(let [account (lookup-account account-id) (let [account (lookup-account account-id)
account-type (:account_type account)] account-type (:account_type account)]
(conj acc (merge {:id (str account-id "-" location) (conj acc (merge {:id (str account-id "-" location)
:location (or location "") :location (or location "")
:count count :count count
:debits debit :debits debit
:credits credit :credits credit
:amount (if account-type (if (#{:account-type/asset :amount (if account-type (if (#{:account-type/asset
:account-type/dividend :account-type/dividend
:account-type/expense} account-type) :account-type/expense} account-type)
(- debit credit) (- debit credit)
(- credit debit)) (- credit debit))
0.0)} 0.0)}
account))) account))))
) []))))
[]))))
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(dc/db conn )]})))
bank-accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(dc/db conn)]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
(map (fn [o]
[[(:db/id a) (:db/id (:account-client-override/client o))]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
:account_type (or (:db/ident (:account/type (accounts a)))
({:bank-account-type/check :account-type/asset
:bank-account-type/cash :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (bank-accounts a)))))
:numeric_code (or (:account/numeric-code (accounts a))
(:bank-account/numeric-code (bank-accounts a)))
:client_id client-id})))
(defn full-ledger-for-client [client-id] (defn full-ledger-for-client [client-id]
(->> (dc/q (->> (dc/q
@@ -371,69 +340,70 @@
(assoc entry (assoc entry
:status :success :status :success
:tx :tx
(remove-nils `(upsert-ledger
{:journal-entry/source (:source entry) ~(remove-nils
:journal-entry/client [:client/code (:client_code entry)] {:journal-entry/source (:source entry)
:journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry))) :journal-entry/client [:client/code (:client_code entry)]
:journal-entry/external-id (:external_id entry) :journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
:journal-entry/vendor (:db/id (all-vendors (:vendor_name entry))) :journal-entry/external-id (:external_id entry)
:journal-entry/amount (:amount entry) :journal-entry/vendor (:db/id (all-vendors (:vendor_name entry)))
:journal-entry/note (:note entry) :journal-entry/amount (:amount entry)
:journal-entry/cleared-against (:cleared_against entry) :journal-entry/note (:note entry)
:journal-entry/cleared-against (:cleared_against entry)
:journal-entry/line-items :journal-entry/line-items
(mapv (fn [ea] (mapv (fn [ea]
(let [debit (or (:debit ea) 0.0) (let [debit (or (:debit ea) 0.0)
credit (or (:credit ea) 0.0)] credit (or (:credit ea) 0.0)]
(when (and (not (get (when (and (not (get
(get all-client-locations (:client_code entry)) (get all-client-locations (:client_code entry))
(:location ea))) (:location ea)))
(not= "A" (:location ea))) (not= "A" (:location ea)))
(throw (ex-info (str "Location '" (:location ea) "' not found.") (throw (ex-info (str "Location '" (:location ea) "' not found.")
{:status :error}))) {:status :error})))
(when (and (<= debit 0.0) (when (and (<= debit 0.0)
(<= credit 0.0)) (<= credit 0.0))
(throw (ex-info (str "Line item amount " (or debit credit) " must be greater than 0.") (throw (ex-info (str "Line item amount " (or debit credit) " must be greater than 0.")
{:status :error}))) {:status :error})))
(when (and (not (all-accounts (:account_identifier ea))) (when (and (not (all-accounts (:account_identifier ea)))
(not (get (not (get
(get all-client-bank-accounts (:client_code entry)) (get all-client-bank-accounts (:client_code entry))
(:account_identifier ea)))) (:account_identifier ea))))
(throw (ex-info (str "Account '" (:account_identifier ea) "' not found.") (throw (ex-info (str "Account '" (:account_identifier ea) "' not found.")
{:status :error}))) {:status :error})))
(let [matching-account (when (re-matches #"^[0-9]+$" (:account_identifier ea)) (let [matching-account (when (re-matches #"^[0-9]+$" (:account_identifier ea))
(a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))] (a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))]
(when (and matching-account (when (and matching-account
(:account/location matching-account) (:account/location matching-account)
(not= (:account/location matching-account) (not= (:account/location matching-account)
(:location ea))) (:location ea)))
(throw (ex-info (str "Account '" (throw (ex-info (str "Account '"
(:account/numeric-code matching-account) (:account/numeric-code matching-account)
"' requires location '" "' requires location '"
(:account/location matching-account) (:account/location matching-account)
"' but got '" "' but got '"
(:location ea) (:location ea)
"'") "'")
{:status :error}))) {:status :error})))
(when (and matching-account (when (and matching-account
(not (:account/location matching-account)) (not (:account/location matching-account))
(= "A" (:location ea))) (= "A" (:location ea)))
(throw (ex-info (str "Account '" (throw (ex-info (str "Account '"
(:account/numeric-code matching-account) (:account/numeric-code matching-account)
"' cannot use location '" "' cannot use location '"
(:location ea) (:location ea)
"'") "'")
{:status :error}))) {:status :error})))
(remove-nils (cond-> {:journal-entry-line/location (:location ea) (remove-nils (cond-> {:journal-entry-line/location (:location ea)
:journal-entry-line/debit (when (> debit 0) :journal-entry-line/debit (when (> debit 0)
debit) debit)
:journal-entry-line/credit (when (> credit 0) :journal-entry-line/credit (when (> credit 0)
credit)} credit)}
matching-account (assoc :journal-entry-line/account (:db/id matching-account)) matching-account (assoc :journal-entry-line/account (:db/id matching-account))
(not matching-account) (assoc :journal-entry-line/account [:bank-account/code (:account_identifier ea)])))))) (not matching-account) (assoc :journal-entry-line/account [:bank-account/code (:account_identifier ea)]))))))
(:line_items entry)) (:line_items entry))
:journal-entry/cleared true}))))) :journal-entry/cleared true}))))))
(:entries args)))) (:entries args))))
errors (filter #(= (:status %) :error) transaction) errors (filter #(= (:status %) :error) transaction)
ignored (filter #(= (:status %) :ignored) transaction) ignored (filter #(= (:status %) :ignored) transaction)
@@ -473,122 +443,6 @@
:errors (map (fn [x] {:external_id (:external_id x) :errors (map (fn [x] {:external_id (:external_id x)
:error (:error x)}) errors)})) :error (:error x)}) errors)}))
(defn accounts-needing-rebuild [ db client]
(->> (dc/qseq '[:find ?c ?a ?l (min ?d)
:in $ ?c
:where [?je :journal-entry/client ?c]
[?je :journal-entry/line-items ?jel]
(or (not [?jel :journal-entry-line/running-balance])
[?jel :journal-entry-line/dirty true])
[?jel :journal-entry-line/account ?a]
[?jel :journal-entry-line/location ?l]
[?je :journal-entry/date ?d]]
db
client)
(map (fn [[client account location starting-at ]]
{:client client
:account account
:starting-at starting-at
:location location}))))
(defn find-running-balance-start [account-needing-rebuild db ]
(let [starting-from (or (->> (dc/q '[:find ?d ?je ?jel ?rbs
:in $ ?c ?starting-at ?a ?l
:where
[?je :journal-entry/client ?c]
[?je :journal-entry/date ?d]
[(< ?d ?starting-at)]
[?je :journal-entry/line-items ?jel]
[?jel :journal-entry-line/account ?a]
[?jel :journal-entry-line/location ?l]
[?jel :journal-entry-line/running-balance ?rbs]
]
db
(:client account-needing-rebuild)
(:starting-at account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild))
(sort)
(last)
(last))
0.0)]
(mu/log ::starting-rebuild-at
:at starting-from)
starting-from))
(defn get-dirty-entries [account-needing-rebuild db ]
(->> (dc/q
'[:find ?d ?jel ?debit ?credit
:in $ ?c ?starting-at ?a ?l
:where
[?e :journal-entry/client ?c]
[?e :journal-entry/date ?d]
[(>= ?d ?starting-at)]
[?e :journal-entry/line-items ?jel]
[?jel :journal-entry-line/account ?a]
[?jel :journal-entry-line/location ?l]
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit ]
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]]
db
(:client account-needing-rebuild)
(:starting-at account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild))
sort
(map #(drop 1 %))))
(defn compute-running-balance [account-needing-refresh]
(mu/log ::compute
:dirty-count (count (:dirty-entries account-needing-refresh)))
(second
(reduce
(fn [[running-balance rows] [id debit credit] ]
(let [new-running-balance (+ running-balance
(if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account-type account-needing-refresh))
(- debit credit)
(- credit debit)))]
[new-running-balance
(conj rows
{:db/id id
:journal-entry-line/running-balance new-running-balance
:journal-entry-line/dirty false})]))
[(:build-from account-needing-refresh) []]
(:dirty-entries account-needing-refresh))))
(defn refresh-running-balance-cache []
(doseq [c (shuffle (map first
(dc/q '[:find (pull ?c [:client/code :db/id])
:where [?c :client/code]]
(dc/db conn))))]
(mu/trace ::building-running-balance
[:client c]
(mu/with-context {:client c}
(let [db (dc/db conn)
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
(when (seq accounts-needing-rebuild)
(mu/log ::found-accounts-needing-rebuild
:accounts accounts-needing-rebuild)
(audit-transact-batch
(->> accounts-needing-rebuild
(mapcat (fn [account-needing-rebuild]
(mu/with-context {:account account-needing-rebuild}
(-> account-needing-rebuild
(assoc :build-from (find-running-balance-start account-needing-rebuild db))
(assoc :dirty-entries (get-dirty-entries account-needing-rebuild db))
(assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild))))
(compute-running-balance))))))
{:user/name "running-balance-cache"})))))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(mount/defstate running-balance-cache-worker
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
:stop (scheduler/stop running-balance-cache-worker))
(defn get-journal-detail-report [context input _] (defn get-journal-detail-report [context input _]
(let [category-totals (atom {}) (let [category-totals (atom {})
@@ -652,8 +506,8 @@
(into base-categories (into base-categories
(for [client-id (:client_ids input) (for [client-id (:client_ids input)
:let [_ (assert-can-see-client (:id context) client-id) :let [_ (assert-can-see-client (:id context) client-id)
account-lookup (build-account-lookup client-id) account-lookup (build-account-lookup client-id)
c (dc/pull (dc/db conn) '[:client/locations] client-id)] c (dc/pull (dc/db conn) '[:client/locations] client-id)]
location (:client/locations c) location (:client/locations c)
line [{:client_id client-id line [{:client_id client-id
:location location :location location

17
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -1,14 +1,6 @@
(ns auto-ap.graphql.transactions (ns auto-ap.graphql.transactions
(:require (:require
[auto-ap.datomic [auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils]]
:refer [audit-transact
audit-transact-batch
conn
pull-attr
pull-many
pull-ref
remove-nils]]
[iol-ion.tx :refer [upsert-entity]]
[auto-ap.datomic.accounts :as a] [auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.invoices :as d-invoices]
@@ -19,10 +11,10 @@
:refer [->graphql :refer [->graphql
<-graphql <-graphql
assert-admin assert-admin
attach-tracing-resolvers
assert-can-see-client assert-can-see-client
assert-not-locked assert-not-locked
assert-power-user assert-power-user
attach-tracing-resolvers
enum->keyword enum->keyword
ident->enum-f ident->enum-f
snake->kebab]] snake->kebab]]
@@ -35,7 +27,8 @@
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[datomic.client.api :as dc])) [datomic.client.api :as dc]
[iol-ion.tx :refer [random-tempid upsert-entity]]))
(def approval-status->graphql (ident->enum-f :transaction/approval-status)) (def approval-status->graphql (ident->enum-f :transaction/approval-status))
@@ -327,7 +320,7 @@
(defn transaction-account->entity [{:keys [id account_id amount location]}] (defn transaction-account->entity [{:keys [id account_id amount location]}]
#:transaction-account {:amount amount #:transaction-account {:amount amount
:db/id id :db/id (or id (random-tempid))
:account account_id :account account_id
:location location}) :location location})

255
src/clj/auto_ap/ledger.clj
View File

@@ -1,12 +1,20 @@
(ns auto-ap.ledger (ns auto-ap.ledger
(:require (:require
[auto-ap.datomic :refer [conn remove-nils pull-ref audit-transact]] [auto-ap.datomic
[auto-ap.utils :refer [dollars-0? dollars=]] :refer [audit-transact
audit-transact-batch
conn
pull-id
pull-ref
remove-nils]]
[auto-ap.utils :refer [by dollars-0? dollars=]]
[clj-time.coerce :as c] [clj-time.coerce :as c]
[clj-time.core :as t] [clj-time.core :as t]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[com.brunobonacci.mulog :as mu]
[com.unbounce.dogstatsd.core :as statsd] [com.unbounce.dogstatsd.core :as statsd]
[datomic.client.api :as dc])) [datomic.client.api :as dc]
[iol-ion.tx :refer [upsert-ledger]]))
(defn datums->impacted-entity [db [e changes]] (defn datums->impacted-entity [db [e changes]]
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) (let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
@@ -47,16 +55,16 @@
:journal-entry/vendor (:db/id (:invoice/vendor entity)) :journal-entry/vendor (:db/id (:invoice/vendor entity))
:journal-entry/amount (Math/abs (:invoice/total entity)) :journal-entry/amount (Math/abs (:invoice/total entity))
:journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable :journal-entry/line-items (into [(cond-> {:db/id (str (:db/id entity) "-" 0)
:journal-entry-line/dirty true :journal-entry-line/account :account/accounts-payable
:journal-entry-line/location "A" :journal-entry-line/location "A"
} }
credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity))) credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity)))
(not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))] (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
(map (fn [ea] (map-indexed (fn [i ea]
(cond-> (cond->
{:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)) {:db/id (str (:db/id entity) "-" (inc i))
:journal-entry-line/dirty true :journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
:journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ") :journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
} }
credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea))) credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea)))
@@ -93,18 +101,19 @@
:journal-entry/cleared-against (:transaction/cleared-against entity) :journal-entry/cleared-against (:transaction/cleared-against entity)
:journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity)) :journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity))
:journal-entry-line/dirty true :db/id (str (:db/id entity) "-" 0)
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/credit (when credit-from-bank? :journal-entry-line/credit (when credit-from-bank?
(Math/abs (:transaction/amount entity))) (Math/abs (:transaction/amount entity)))
:journal-entry-line/debit (when debit-from-bank? :journal-entry-line/debit (when debit-from-bank?
(Math/abs (:transaction/amount entity)))}) (Math/abs (:transaction/amount entity)))})
] ]
(map (map-indexed
(fn [a] (fn [i a]
(remove-nils{:journal-entry-line/account (:db/id (:transaction-account/account a)) (remove-nils{
:db/id (str (:db/id entity) "-" (inc i))
:journal-entry-line/account (:db/id (:transaction-account/account a))
:journal-entry-line/location (:transaction-account/location a) :journal-entry-line/location (:transaction-account/location a)
:journal-entry-line/dirty true
:journal-entry-line/debit (when credit-from-bank? :journal-entry-line/debit (when credit-from-bank?
(Math/abs (:transaction-account/amount a))) (Math/abs (:transaction-account/amount a)))
:journal-entry-line/credit (when debit-from-bank? :journal-entry-line/credit (when debit-from-bank?
@@ -186,20 +195,19 @@
(defn touch-transaction [e] (defn touch-transaction [e]
(dc/transact conn {:tx-data [[:db/retractEntity [:journal-entry/original-entity e]]]})
(when-let [change (entity-change->ledger (dc/db conn) (when-let [change (entity-change->ledger (dc/db conn)
[:transaction e])] [:transaction e])]
(dc/transact conn {:tx-data [{:db/id "datomic.tx" (dc/transact conn {:tx-data [{:db/id "datomic.tx"
:db/doc "touching transaction to update ledger"} :db/doc "touching transaction to update ledger"}
change]}))) `(upsert-ledger ~change)]})))
(defn touch-invoice [e] (defn touch-invoice [e]
(dc/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
(when-let [change (entity-change->ledger (dc/db conn) (when-let [change (entity-change->ledger (dc/db conn)
[:invoice e])] [:invoice e])]
(dc/transact conn [{:db/id "datomic.tx" (dc/transact conn [{:db/id "datomic.tx"
:db/doc "touching invoice to update ledger"} :db/doc "touching invoice to update ledger"}
change]))) `(upsert-ledger ~change)])))
(defn lazy-tx-range (defn lazy-tx-range
([start end xf] (lazy-tx-range start end xf 0)) ([start end xf] (lazy-tx-range start end xf 0))
([start end xf o] ([start end xf o]
@@ -432,11 +440,9 @@
(set)) (set))
ledger-txs (->> affected-entities ledger-txs (->> affected-entities
(map #(entity-change->ledger (:db-after tx) %)) (map #(entity-change->ledger (:db-after tx) %))
(filter seq)) (filter seq)
;; TODO mark deleted journal-entry-line accounts as dirty, needing refresh (map (fn [l]
retractions (map (fn [[_ e]] [:db/retractEntity [:journal-entry/original-entity e]]) affected-entities)] `(upsert-ledger ~l))))]
(when (seq retractions)
(audit-transact retractions id))
(when (seq ledger-txs) (when (seq ledger-txs)
(audit-transact ledger-txs id)) (audit-transact ledger-txs id))
tx)) tx))
@@ -459,3 +465,208 @@
{} {}
(partition-all 50 txes)))) (partition-all 50 txes))))
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(dc/db conn )]})))
bank-accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(dc/db conn)]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
(map (fn [o]
[[(:db/id a) (:db/id (:account-client-override/client o))]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
:account_type (or (:db/ident (:account/type (accounts a)))
({:bank-account-type/check :account-type/asset
:bank-account-type/cash :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (bank-accounts a)))))
:numeric_code (or (:account/numeric-code (accounts a))
(:bank-account/numeric-code (bank-accounts a)))
:client_id client-id})))
(defn reset-client+account+location+date
([] (reset-client+account+location+date (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn)))))
([clients]
(doseq [client clients
:let [_ (mu/log ::reseting-index-for :client client)]
batch
(->> (dc/qseq '[:find (pull ?je [:journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id]}])
:in $ ?c
:where [?je :journal-entry/client ?c]]
(dc/db conn)
client
)
(map first)
(mapcat (fn [je]
(map (fn [jel]
{:db/id (:db/id jel)
:journal-entry-line/client+account+location+date
[(-> je :journal-entry/client :db/id)
(-> jel :journal-entry-line/account :db/id)
(-> jel :journal-entry-line/location)
(-> je :journal-entry/date)]})
(:journal-entry/line-items je))))
(partition-all 500)
)]
(mu/log ::batch-completed)
(dc/transact conn {:tx-data batch}))))
(defn find-mismatch-index []
(reduce + 0
(for [c (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn)))
:let [_ (println "searching for" c)
a (->> (dc/index-pull (dc/db conn)
{:index :avet
:selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}]
:start [:journal-entry-line/client+account+location+date [c]]})
(take-while (fn [result]
(= c (first (:journal-entry-line/client+account+location+date result)))
))
(filter (fn [{index :journal-entry-line/client+account+location+date :as result}]
(not= index
[(-> result :journal-entry/_line-items :journal-entry/client :db/id)
(-> result :journal-entry-line/account :db/id)
(-> result :journal-entry-line/location)
(-> result :journal-entry/_line-items :journal-entry/date)]))))]]
(do (println (count a))
(count a)))))
(defn accounts-needing-rebuild [ db client]
(let [client (pull-id db client)]
(->> (dc/qseq '[:find ?c ?a ?l (min ?d)
:in $ ?c
:where
[?jel :journal-entry-line/dirty true]
[?jel :journal-entry-line/account ?a]
[?jel :journal-entry-line/location ?l]
[?je :journal-entry/line-items ?jel]
[?je :journal-entry/client ?c]
[?je :journal-entry/date ?d]]
db
client)
(map (fn [[client account location starting-at ]]
{:client client
:account account
:starting-at starting-at
:location location})))))
(defn find-running-balance-start [account-needing-rebuild db ]
(or
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
[(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)
(:starting-at account-needing-rebuild)]]
:reverse true
:limit 500})
(take-while (fn [result]
(= [(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)]
(take 3 (:journal-entry-line/client+account+location+date result)))))
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
(>= (compare date (:starting-at account-needing-rebuild)) 0)))
first
:journal-entry-line/running-balance
)
0.0))
(defn get-dirty-entries [account-needing-rebuild db ]
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
:start [:journal-entry-line/client+account+location+date
[(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)
(:starting-at account-needing-rebuild)]]
})
(take-while (fn [result]
(= [(:client account-needing-rebuild)
(:account account-needing-rebuild)
(:location account-needing-rebuild)]
(take 3 (:journal-entry-line/client+account+location+date result)))))
(map (fn [result]
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))))
(defn compute-running-balance [account-needing-refresh]
(mu/log ::compute
:dirty-count (count (:dirty-entries account-needing-refresh)))
(second
(reduce
(fn [[running-balance rows] [id debit credit] ]
(let [new-running-balance (+ running-balance
(if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account-type account-needing-refresh))
(- debit credit)
(- credit debit)))]
[new-running-balance
(conj rows
{:db/id id
:journal-entry-line/running-balance new-running-balance
:journal-entry-line/dirty false})]))
[(:build-from account-needing-refresh) []]
(:dirty-entries account-needing-refresh))))
(defn refresh-running-balance-cache
([] (refresh-running-balance-cache (shuffle (map first
(dc/q '[:find (pull ?c [:client/code :db/id])
:where [?c :client/code]]
(dc/db conn))))))
([clients]
(doseq [c clients]
(mu/trace ::building-running-balance
[:client c]
(mu/with-context {:client c}
(let [db (dc/db conn)
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
(when (seq accounts-needing-rebuild)
(mu/log ::found-accounts-needing-rebuild
:accounts accounts-needing-rebuild)
(audit-transact-batch
(->> accounts-needing-rebuild
(mapcat (fn [account-needing-rebuild]
(mu/with-context {:account account-needing-rebuild}
(-> account-needing-rebuild
(assoc :build-from (find-running-balance-start account-needing-rebuild db))
(assoc :dirty-entries (get-dirty-entries account-needing-rebuild db))
(assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild))))
(compute-running-balance))))))
{:user/name "running-balance-cache"}))))))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
#_(mount/defstate running-balance-cache-worker
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
:stop (scheduler/stop running-balance-cache-worker))

143
src/clj/iol_ion/tx.clj
View File

@@ -1,143 +0,0 @@
(ns iol-ion.tx
(:require [datomic.client.api :as dc])
(:import [java.util UUID]))
(defn random-tempid []
(str (UUID/randomUUID)))
(defn remove-nils [m]
(let [result (reduce-kv
(fn [m k v]
(if (not (nil? v))
(assoc m k v)
m
))
{}
m)]
(if (seq result)
result
nil)))
(defn by
([f xs]
(by f identity xs))
([f fv xs]
(reduce
#(assoc %1 (f %2) (fv %2))
{}
xs)))
(defn pull-many [db read ids ]
(->> (dc/q '[:find (pull ?e r)
:in $ [?e ...] r]
db
ids
read)
(map first)))
(declare upsert-entity)
(defn reset-rels [db e a vs]
(assert (every? :db/id vs) (format "In order to reset attribute %s, every value must have :db/id" a))
(let [ids (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
new-id-set (set (map :db/id vs))
retract-ids (filter (complement new-id-set) ids)
{is-component? :db/isComponent} (dc/pull db [:db/isComponent] a)
new-rels (filter (complement (set ids)) (map :db/id vs))]
(-> []
(into (map (fn [i] (if is-component?
[:db/retractEntity i]
[:db/retract e a i ])) retract-ids))
(into (map (fn [i] [:db/add e a i]) new-rels))
(into (mapcat (fn [i] (upsert-entity db i)) vs)))))
(defn reset-scalars [db e a vs]
(let [extant (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
retracts (filter (complement (set vs)) extant)
new (filter (complement (set extant)) vs)]
(-> []
(into (map (fn [i] [:db/retract e a i ]) retracts))
(into (map (fn [i] [:db/add e a i]) new)))))
;; TODO unit test this
(defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id")
(let [e (:db/id entity)
is-new? (string? e)
extant-entity (when-not is-new?
(dc/pull db (keys entity) (:db/id entity)))
ident->value-type (by :db/ident (comp :db/ident
:db/valueType)
(pull-many
db
[:db/valueType :db/ident]
(keys entity)))
ops (->> entity
(reduce
(fn [ops [a v]]
(cond
(= :db/id a)
ops
(or (= v (a extant-entity))
(= v (:db/ident (a extant-entity) :nope))
(= v (:db/id (a extant-entity)) :nope))
ops
(and (nil? v)
(not (nil? (a extant-entity))))
(conj ops [:db/retract e a (cond-> (a extant-entity)
(:db/id (a extant-entity)) :db/id)])
(nil? v)
ops
;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq)
(and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v))
(into ops (reset-rels db e a v))
(and (sequential? v) (not= :db.type/ref (ident->value-type a)))
(into ops (reset-scalars db e a v))
(and (map? v)
(= :db.type/ref (ident->value-type a)))
(let [id (or (:db/id v) (random-tempid))]
(-> ops
(conj [:db/add e a id])
(into (upsert-entity db (assoc v :db/id id)))))
:else
(conj ops [:db/add e a v])
))
[]))]
ops))
(defn propose-invoice [db invoice]
(let [existing? (boolean (seq (dc/q '[:find ?i
:in $ ?invoice-number ?client ?vendor
:where
[?i :invoice/invoice-number ?invoice-number]
[?i :invoice/client ?client]
[?i :invoice/vendor ?vendor]
(not [?i :invoice/status :invoice-status/voided])]
db
(:invoice/invoice-number invoice)
(:invoice/client invoice)
(:invoice/vendor invoice))))]
(if existing?
[]
[(remove-nils invoice)])))

142
test/clj/auto_ap/integration/graphql/ledger/running_balance.clj Normal file
View File

@@ -0,0 +1,142 @@
(ns auto-ap.integration.graphql.ledger.running-balance
(:require
[auto-ap.datomic :refer [conn pull-attr]]
[auto-ap.graphql.ledger :as sut]
[iol-ion.tx :refer [upsert-entity upsert-ledger]]
[auto-ap.integration.util :refer [wrap-setup]]
[clojure.test :as t :refer [deftest is testing use-fixtures]]
[datomic.client.api :as d]
[datomic.client.api :as dc]))
(use-fixtures :each wrap-setup)
(deftest running-balance
(let [{:strs [test-account-1
test-account-2
test-client
journal-entry-1
journal-entry-2
journal-entry-3
line-1-1
line-1-2
line-2-1
line-2-2
line-3-1
line-3-2]} (:tempids (doto (d/transact conn {:tx-data [{:db/id "test-account-1"
:account/type :account-type/asset}
{:db/id "test-account-2"
:account/type :account-type/equity}
{:db/id "test-client"
:client/code "TEST"}
`(upsert-ledger {:db/id "journal-entry-1"
:journal-entry/external-id "1"
:journal-entry/date #inst "2022-01-01"
:journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-1-1"
:journal-entry-line/account "test-account-1"
:journal-entry-line/location "A"
:journal-entry-line/debit 10.0}
{:db/id "line-1-2"
:journal-entry-line/account "test-account-2"
:journal-entry-line/location "A"
:journal-entry-line/credit 10.0}]})
`(upsert-ledger {:db/id "journal-entry-2"
:journal-entry/date #inst "2022-01-02"
:journal-entry/external-id "2"
:journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-2-1"
:journal-entry-line/account "test-account-1"
:journal-entry-line/location "A"
:journal-entry-line/debit 50.0}
{:db/id "line-2-2"
:journal-entry-line/account "test-account-2"
:journal-entry-line/location "A"
:journal-entry-line/credit 50.0}]})
`(upsert-ledger {:db/id "journal-entry-3"
:journal-entry/date #inst "2022-01-03"
:journal-entry/external-id "3"
:journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-3-1"
:journal-entry-line/account "test-account-1"
:journal-entry-line/location "A"
:journal-entry-line/debit 150.0}
{:db/id "line-3-2"
:journal-entry-line/account "test-account-2"
:journal-entry-line/location "A"
:journal-entry-line/credit 150.0}]})]})
clojure.pprint/pprint))]
(testing "should set running-balance on ledger entries missing them"
(sut/refresh-running-balance-cache)
(println (d/pull (d/db conn) '[*] line-1-1))
(is (= [-10.0 -60.0 -210.0]
(map #(pull-attr (d/db conn) :journal-entry-line/running-balance %) [line-1-1 line-2-1 line-3-1
])))
(is (= [10.0 60.0 210.0]
(map #(pull-attr (d/db conn) :journal-entry-line/running-balance %) [line-1-2 line-2-2 line-3-2]))))
(testing "should recompute if the data is out of date"
(d/transact conn
{:tx-data
[{:db/id line-1-1
:journal-entry-line/dirty true
:journal-entry-line/running-balance 123810.23}]})
(sut/refresh-running-balance-cache)
(is (= [-10.0 -60.0 -210.0]
(map #(pull-attr (d/db conn) :journal-entry-line/running-balance %) [line-1-1 line-2-1 line-3-1]))))
(testing "should recompute every entry after the out of date one"
(d/transact conn
{:tx-data
[{:db/id line-1-1
:journal-entry-line/dirty true
:journal-entry-line/debit 70.0}]})
(sut/refresh-running-balance-cache)
(is (= [-70.0 -120.0 -270.0]
(map #(pull-attr (d/db conn) :journal-entry-line/running-balance %) [line-1-1 line-2-1 line-3-1]))))
(testing "should not recompute entries that aren't dirty"
(d/transact conn
{:tx-data
[{:db/id line-1-1
:journal-entry-line/dirty false
:journal-entry-line/debit 90.0}]})
(sut/refresh-running-balance-cache)
(is (= [-70.0 -120.0 -270.0]
(map #(pull-attr (d/db conn) :journal-entry-line/running-balance %) [line-1-1 line-2-1 line-3-1])))
)
(testing "changing a ledger entry should mark the line items as dirty"
(d/transact conn
{:tx-data
[`(upsert-ledger ~{:db/id journal-entry-2
:journal-entry/date #inst "2022-01-02"
:journal-entry/client test-client
:journal-entry/external-id "2"
:journal-entry/line-items [{:db/id "line-2-1"
:journal-entry-line/account test-account-1
:journal-entry-line/location "A"
:journal-entry-line/debit 50.0}
{:db/id "line-2-2"
:journal-entry-line/account test-account-2
:journal-entry-line/location "A"
:journal-entry-line/credit 50.0}]})]})
(is (= [true true]
(->> (d/pull (d/db conn) '[{:journal-entry/line-items [:journal-entry-line/dirty]}] journal-entry-2)
(:journal-entry/line-items)
(map :journal-entry-line/dirty))))
(testing "should also mark the next entry as dirty, so that if a ledger entry is changed, the old accounts get updated"
(is (= [false false]
(->> (d/pull (d/db conn) '[{:journal-entry/line-items [:journal-entry-line/dirty]}] journal-entry-1)
(:journal-entry/line-items)
(map :journal-entry-line/dirty))))
(is (= [true true]
(->> (d/pull (d/db conn) '[{:journal-entry/line-items [:journal-entry-line/dirty]}] journal-entry-3)
(:journal-entry/line-items)
(map :journal-entry-line/dirty))))))))

6
things-to-search-for.txt
View File

@@ -15,8 +15,10 @@ Fix searching
* indexing should happen more regularly, and just look for changes since last time it was run * indexing should happen more regularly, and just look for changes since last time it was run
Running Balance Cache Running Balance Cache
* much simpler now, just make it handle reverts (see TODO) * Add tests for upsert-ledger
** when a journal entry gets reset, you have to recalculate running balance. Could just make it do this globally by marking the earliest journal entry * try again to see if we can get upsert-ledger into the same transaction, making it all or nothing
* make rest of rebuilding the cache use new index
* ensure somehow that the index is always right
Address memory Address memory