diff --git a/iol_ion/src/iol_ion/tx/upsert_ledger.clj b/iol_ion/src/iol_ion/tx/upsert_ledger.clj index 255bde91..f8af3c66 100644 --- a/iol_ion/src/iol_ion/tx/upsert_ledger.clj +++ b/iol_ion/src/iol_ion/tx/upsert_ledger.clj @@ -1,5 +1,5 @@ (ns iol-ion.tx.upsert-ledger - (:import [java.util UUID]) + (:import [java.util UUID Date]) (:require [datomic.api :as dc])) (defn -random-tempid [] @@ -10,7 +10,7 @@ next-jel (->> (dc/index-pull db {:index :avet :selector [:db/id :journal-entry-line/client+account+location+date] :start [:journal-entry-line/client+account+location+date - (:journal-entry-line/client+account+location+date jel) + (:journal-entry-line/client+account+location+date jel) (:db/id jel)]}) (take-while (fn line-must-match-client-account-location [result] (and @@ -25,6 +25,9 @@ (def extant-read '[:db/id :journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id :journal-entry-line/client+account+location+date]}]) +(defn now [] + (Date.)) + (defn calc-client+account+location+date [je jel] [(or (:db/id (:journal-entry/client je)) @@ -43,8 +46,7 @@ (let [extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)] (dc/pull db extant-read [:journal-entry/original-entity original-entity])) (when-let [external-id (:journal-entry/external-id ledger-entry)] - (dc/pull db extant-read [:journal-entry/external-id external-id]))) - extant-entry-exists? (:db/id extant-entry)] + (dc/pull db extant-read [:journal-entry/external-id external-id]))) ] (cond-> [[:upsert-entity (into (-> ledger-entry @@ -55,12 +57,9 @@ (update :journal-entry/line-items (fn [lis] (mapv #(-> % - (assoc :journal-entry-line/dirty true) - (assoc :journal-entry-line/client+account+location+date - (calc-client+account+location+date ledger-entry %))) - lis)))))]] - extant-entry-exists? (into (map (fn [li] - {:journal-entry-line/dirty true - :db/id li}) - (get-line-items-after db extant-entry)))))) + (assoc :journal-entry-line/date (:journal-entry/date ledger-entry)) + (assoc :journal-entry-line/client (:journal-entry/client ledger-entry))) + lis)))))] + {:db/id (:journal-entry/client ledger-entry) + :client/ledger-last-change (now)}]))) diff --git a/resources/functions.edn b/resources/functions.edn index 9c6497a0..e7d426d9 100644 --- a/resources/functions.edn +++ b/resources/functions.edn @@ -1 +1 @@ -[#:db{:ident :pay, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e amount], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) amount (genfn-coerce-arg amount)] (do (let [current-outstanding-balance (-> (dc/pull db [:invoice/outstanding-balance] e) :invoice/outstanding-balance) new-outstanding-balance (- current-outstanding-balance amount)] [[:upsert-invoice {:invoice/status (if (< -1.0E-4 new-outstanding-balance 1.0E-4) :invoice-status/paid :invoice-status/unpaid), :db/id e, :invoice/outstanding-balance new-outstanding-balance}]]))))))"}} #:db{:ident :plus, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a amount], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) amount (genfn-coerce-arg amount)] (do [[:db/add e a (-> (dc/pull db [a] e) a (+ amount))]])))))"}} #:db{:ident :propose-invoice, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db invoice], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [invoice (genfn-coerce-arg invoice)] (do (let [existing? (boolean (seq (dc/q (quote [:find ?i :in $ ?invoice-number ?client ?vendor :where [?i :invoice/invoice-number ?invoice-number] [?i :invoice/client ?client] [?i :invoice/vendor ?vendor] (not [?i :invoice/status :invoice-status/voided])]) db (:invoice/invoice-number invoice) (:invoice/client invoice) (:invoice/vendor invoice))))] (if existing? [] [[:upsert-invoice invoice]])))))))"}} #:db{:ident :reset-rels, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a vs], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) vs (genfn-coerce-arg vs)] (do (assert (every? :db/id vs) (format \"In order to reset attribute %s, every value must have :db/id\" a)) (let [ids (when-not (string? e) (->> (dc/q (quote [:find ?z :in $ ?e ?a :where [?e ?a ?z]]) db e a) (map first))) new-id-set (set (map :db/id vs)) retract-ids (filter (complement new-id-set) ids) {is-component? :db/isComponent} (dc/pull db [:db/isComponent] a) new-rels (filter (complement (set ids)) (map :db/id vs))] (-> [] (into (map (fn [i] (if is-component? [:db/retractEntity i] [:db/retract e a i])) retract-ids)) (into (map (fn [i] [:db/add e a i]) new-rels)) (into (map (fn [i] [:upsert-entity i]) vs)))))))))"}} #:db{:ident :reset-scalars, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a vs], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) vs (genfn-coerce-arg vs)] (do (let [extant (when-not (string? e) (->> (dc/q (quote [:find ?z :in $ ?e ?a :where [?e ?a ?z]]) db e a) (map first))) retracts (filter (complement (set vs)) extant) new (filter (complement (set extant)) vs)] (-> [] (into (map (fn [i] [:db/retract e a i]) retracts)) (into (map (fn [i] [:db/add e a i]) new)))))))))"}} #:db{:ident :upsert-entity, :fn #db/fn{:lang :clojure, :imports [[java.util UUID]], :requires [[datomic.api :as dc]], :params [db entity], :code "(let [] (letfn [(-random-tempid [] (do (str (UUID/randomUUID)))) (-by [f fv xs] (do (reduce (fn* [p1__703811# p2__703812#] (assoc p1__703811# (f p2__703812#) (fv p2__703812#))) {} xs))) (-pull-many [db read ids] (do (->> (dc/q (quote [:find (pull ?e r) :in $ [?e ...] r]) db ids read) (map first))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [entity (genfn-coerce-arg entity)] (do (when-not (or (:db/id entity) (:db/ident entity)) (datomic.api/cancel #:cognitect.anomalies{:message (str \"Cannot upsert without :db/id or :db/ident, \" entity), :category :cognitect.anomalies/incorrect})) (let [e (or (:db/id entity) (:db/ident entity)) is-new? (string? e) extant-entity (when-not is-new? (dc/pull db (keys entity) (or (:db/id entity) (:db/ident entity)))) ident->value-type (-by :db/ident (comp :db/ident :db/valueType) (-pull-many db [#:db{:valueType [:db/ident]} :db/ident] (keys entity))) ident->cardinality (-by :db/ident (comp :db/ident :db/cardinality) (-pull-many db [#:db{:cardinality [:db/ident]} :db/ident] (keys entity))) ops (->> entity (reduce (fn [ops [a v]] (cond (= :db/id a) ops (= :db/ident a) ops (or (= v (a extant-entity)) (= v (:db/ident (a extant-entity) :nope)) (= v (:db/id (a extant-entity)) :nope)) ops (and (nil? v) (not (nil? (a extant-entity)))) (if (= :db.cardinality/many (ident->cardinality a)) (into ops (map (fn [v] [:db/retract e a (cond-> v (:db/id v) :db/id)]) (a extant-entity))) (conj ops [:db/retract e a (cond-> (a extant-entity) (:db/id (a extant-entity)) :db/id)])) (nil? v) ops (and (sequential? v) (= :db.type/tuple (ident->value-type a)) (not (= :db.cardinality/many (ident->cardinality a)))) (conj ops [:db/add e a v]) (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) (into ops [[:reset-rels e a v]]) (= :db.cardinality/many (ident->cardinality a)) (into ops [[:reset-scalars e a v]]) (and (sequential? v) (not= :db.type/ref (ident->value-type a))) (into ops [[:reset-scalars e a v]]) (and (map? v) (= :db.type/ref (ident->value-type a))) (let [id (or (:db/id v) (-random-tempid))] (-> ops (conj [:db/add e a id]) (into [[:upsert-entity (assoc v :db/id id)]]))) :else (conj ops [:db/add e a v]))) []))] ops))))))"}} #:db{:ident :upsert-invoice, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db invoice], :code "(let [] (letfn [(-remove-nils [m] (do (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m)) {} m)] (if (seq result) result nil)))) (invoice->journal-entry [db invoice-id raw-invoice-id] (do (let [entity (dc/pull db (quote [:invoice/total :invoice/exclude-from-ledger :invoice/outstanding-balance :invoice/date #:invoice{:client [:db/id :client/code], :status [:db/ident], :import-status [:db/ident], :vendor [:db/id :vendor/name], :payment [:db/id #:payment{:status [:db/ident]}], :expense-accounts [:invoice-expense-account/account :invoice-expense-account/amount :invoice-expense-account/location]}]) invoice-id) credit-invoice? (< (:invoice/total entity 0.0) 0.0)] (when-not (or (not (:invoice/total entity)) (= true (:invoice/exclude-from-ledger entity)) (= :import-status/pending (:db/ident (:invoice/import-status entity))) (= :invoice-status/voided (:db/ident (:invoice/status entity))) (< -0.001 (:invoice/total entity) 0.001)) (-remove-nils #:journal-entry{:date (:invoice/date entity), :original-entity raw-invoice-id, :client (:db/id (:invoice/client entity)), :line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable, :db/id (str raw-invoice-id \"-\" 0), :journal-entry-line/location \"A\"} credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity))) (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))] (map-indexed (fn [i ea] (cond-> {:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)), :db/id (str raw-invoice-id \"-\" (inc i)), :journal-entry-line/location (or (:invoice-expense-account/location ea) \"HQ\")} credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea))) (not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea))))) (:invoice/expense-accounts entity))), :source \"invoice\", :cleared (and (< (:invoice/outstanding-balance entity) 0.01) (every? (fn* [p1__703814#] (= :payment-status/cleared (:payment/status p1__703814#))) (:invoice/payments entity))), :amount (Math/abs (:invoice/total entity)), :vendor (:db/id (:invoice/vendor entity))})))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [invoice (genfn-coerce-arg invoice)] (do (let [upserted-entity [[:upsert-entity invoice]] with-invoice (dc/with db upserted-entity) invoice-id (or (-> with-invoice :tempids (get (:db/id invoice))) (:db/id invoice)) journal-entry (invoice->journal-entry (:db-after with-invoice) invoice-id (:db/id invoice))] (into upserted-entity (if journal-entry [[:upsert-ledger journal-entry]] [[:db/retractEntity [:journal-entry/original-entity (:db/id invoice)]]]))))))))"}} #:db{:ident :upsert-ledger, :fn #db/fn{:lang :clojure, :imports [[java.util UUID]], :requires [[datomic.api :as dc]], :params [db ledger-entry], :code "(let [extant-read (quote [:db/id :journal-entry/date :journal-entry/client #:journal-entry{:line-items [:journal-entry-line/account :journal-entry-line/location :db/id :journal-entry-line/client+account+location+date]}])] (letfn [(-random-tempid [] (do (dc/tempid :db.part/user))) (get-line-items-after [db journal-entry] (do (for [jel (:journal-entry/line-items journal-entry) next-jel (->> (dc/index-pull db {:selector [:db/id :journal-entry-line/client+account+location+date], :index :avet, :start [:journal-entry-line/client+account+location+date (:journal-entry-line/client+account+location+date jel) (:db/id jel)]}) (take-while (fn line-must-match-client-account-location [result] (and (= (take 3 (:journal-entry-line/client+account+location+date result)) (take 3 (:journal-entry-line/client+account+location+date jel))) (not= (:db/id jel) (:db/id result))))) (take 2)) :when next-jel] (:db/id next-jel)))) (calc-client+account+location+date [je jel] (do [(or (:db/id (:journal-entry/client je)) (:journal-entry/client je)) (or (:db/id (:journal-entry-line/account jel)) (:journal-entry-line/account jel)) (-> jel :journal-entry-line/location) (-> je :journal-entry/date)]))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [ledger-entry (genfn-coerce-arg ledger-entry)] (do (assert (:journal-entry/date ledger-entry) (format \"Must at least provide date when updating ledger: %s\" (pr-str ledger-entry))) (assert (:journal-entry/client ledger-entry) \"Must at least provide client when updating ledger\") (let [extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)] (dc/pull db extant-read [:journal-entry/original-entity original-entity])) (when-let [external-id (:journal-entry/external-id ledger-entry)] (dc/pull db extant-read [:journal-entry/external-id external-id]))) extant-entry-exists? (:db/id extant-entry)] (cond-> [[:upsert-entity (into (-> ledger-entry (assoc :db/id (or (:db/id ledger-entry) (:db/id extant-entry) (-random-tempid))) (update :journal-entry/line-items (fn [lis] (mapv (fn* [p1__703818#] (-> p1__703818# (assoc :journal-entry-line/dirty true) (assoc :journal-entry-line/client+account+location+date (calc-client+account+location+date ledger-entry p1__703818#)))) lis)))))]] extant-entry-exists? (into (map (fn [li] {:db/id li, :journal-entry-line/dirty true}) (get-line-items-after db extant-entry))))))))))"}} #:db{:ident :upsert-transaction, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db transaction], :code "(let [my-transaction {:transaction/bank-account 17592232681223, :transaction/date (read-string \"#inst \\\"2024-02-24T08:00:00.000-00:00\\\"\"), :transaction/matched-rule 17592233159891, :transaction/client 17592232577980, :transaction/status \"POSTED\", :transaction/plaid-merchant {:plaid-merchant/name \"Rotten Robbie\", :db/id \"b2776792-9e2b-46e8-a9c8-bf80abea359e\"}, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc\", :transaction/id \"11a4a13e713d63f476009027e9a53e217e13d0192a37df8ab96c0eed4bdbe996\", :transaction/description-original \"Rotten Robbie #03\", :transaction/approval-status #:db{:id 17592231963877, :ident :transaction-approval-status/approved}, :transaction/amount -84.43, :transaction/accounts [{:transaction-account/account 17592231963549, :db/id \"c402c7b3-c11b-484b-b670-bd48f79a3e5f\", :transaction-account/location \"CB\", :transaction-account/amount 84.43}], :transaction/raw-id \"gQypbv5946F08op74wZmidDg8qD8Q1fM6gEBP\", :transaction/vendor 17592232627053} my-journal #:journal-entry{:vendor 17592232627053, :amount 84.43, :date (read-string \"#inst \\\"2024-02-24T08:00:00.000-00:00\\\"\"), :alternate-description \"Rotten Robbie #03\", :original-entity \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc\", :client 17592232577980, :source \"transaction\", :line-items [{:journal-entry-line/credit 84.43, :journal-entry-line/account 17592232681223, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-0\", :journal-entry-line/location \"A\"} {:journal-entry-line/account 17592231963549, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-1\", :journal-entry-line/debit 84.43, :journal-entry-line/location \"CB\"} {:journal-entry-line/account 17592231963549, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-2\", :journal-entry-line/debit 84.43, :journal-entry-line/location \"CB\"}], :cleared true}] (letfn [(-remove-nils [m] (do (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m)) {} m)] (if (seq result) result nil)))) (transaction->journal-entry [db transaction-id raw-transaction-id] (do (let [entity (dc/pull db [:transaction/client :transaction/date :transaction/description-original :db/id :transaction/vendor :transaction/amount :transaction/cleared-against #:transaction{:bank-account [:db/id #:bank-account{:type [:db/ident]}], :approval-status [:db/ident], :accounts [:transaction-account/account :transaction-account/location :transaction-account/amount]}] transaction-id) decreasing? (< (or (:transaction/amount entity) 0.0) 0.0) credit-from-bank? decreasing? debit-from-bank? (not decreasing?)] (when (and (not (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))) (not (= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))) (:transaction/amount entity) (not (< -0.001 (:transaction/amount entity) 0.001))) (-remove-nils #:journal-entry{:vendor (:db/id (:transaction/vendor entity)), :amount (Math/abs (:transaction/amount entity)), :date (:transaction/date entity), :alternate-description (:transaction/description-original entity), :original-entity raw-transaction-id, :client (:db/id (:transaction/client entity)), :cleared-against (:transaction/cleared-against entity), :source \"transaction\", :line-items (into [(-remove-nils {:journal-entry-line/credit (when credit-from-bank? (Math/abs (:transaction/amount entity))), :journal-entry-line/account (:db/id (:transaction/bank-account entity)), :db/id (str raw-transaction-id \"-\" 0), :journal-entry-line/debit (when debit-from-bank? (Math/abs (:transaction/amount entity))), :journal-entry-line/location \"A\"})] (map-indexed (fn [i a] (-remove-nils {:journal-entry-line/credit (when debit-from-bank? (Math/abs (:transaction-account/amount a))), :journal-entry-line/account (:db/id (:transaction-account/account a)), :db/id (str raw-transaction-id \"-\" (inc i)), :journal-entry-line/debit (when credit-from-bank? (Math/abs (:transaction-account/amount a))), :journal-entry-line/location (:transaction-account/location a)})) (if (seq (:transaction/accounts entity)) (:transaction/accounts entity) [#:transaction-account{:amount (:transaction/amount entity)}]))), :cleared true})))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [transaction (genfn-coerce-arg transaction)] (do (let [upserted-entity [[:upsert-entity (dissoc transaction :transaction/payment :import-batch/_entry)]] with-transaction (dc/with db upserted-entity) transaction-id (or (-> with-transaction :tempids (get (:db/id transaction))) (:db/id transaction)) journal-entry (transaction->journal-entry (:db-after with-transaction) transaction-id (:db/id transaction))] (into [[:upsert-entity transaction]] (if journal-entry [[:upsert-ledger journal-entry]] [[:db/retractEntity [:journal-entry/original-entity (:db/id transaction)]]]))))))))"}}] \ No newline at end of file +[#:db{:ident :pay, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e amount], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) amount (genfn-coerce-arg amount)] (do (let [current-outstanding-balance (-> (dc/pull db [:invoice/outstanding-balance] e) :invoice/outstanding-balance) new-outstanding-balance (- current-outstanding-balance amount)] [[:upsert-invoice {:invoice/status (if (< -1.0E-4 new-outstanding-balance 1.0E-4) :invoice-status/paid :invoice-status/unpaid), :db/id e, :invoice/outstanding-balance new-outstanding-balance}]]))))))"}} #:db{:ident :plus, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a amount], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) amount (genfn-coerce-arg amount)] (do [[:db/add e a (-> (dc/pull db [a] e) a (+ amount))]])))))"}} #:db{:ident :propose-invoice, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db invoice], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [invoice (genfn-coerce-arg invoice)] (do (let [existing? (boolean (seq (dc/q (quote [:find ?i :in $ ?invoice-number ?client ?vendor :where [?i :invoice/invoice-number ?invoice-number] [?i :invoice/client ?client] [?i :invoice/vendor ?vendor] (not [?i :invoice/status :invoice-status/voided])]) db (:invoice/invoice-number invoice) (:invoice/client invoice) (:invoice/vendor invoice))))] (if existing? [] [[:upsert-invoice invoice]])))))))"}} #:db{:ident :reset-rels, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a vs], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) vs (genfn-coerce-arg vs)] (do (assert (every? :db/id vs) (format \"In order to reset attribute %s, every value must have :db/id\" a)) (let [ids (when-not (string? e) (->> (dc/q (quote [:find ?z :in $ ?e ?a :where [?e ?a ?z]]) db e a) (map first))) new-id-set (set (map :db/id vs)) retract-ids (filter (complement new-id-set) ids) {is-component? :db/isComponent} (dc/pull db [:db/isComponent] a) new-rels (filter (complement (set ids)) (map :db/id vs))] (-> [] (into (map (fn [i] (if is-component? [:db/retractEntity i] [:db/retract e a i])) retract-ids)) (into (map (fn [i] [:db/add e a i]) new-rels)) (into (map (fn [i] [:upsert-entity i]) vs)))))))))"}} #:db{:ident :reset-scalars, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db e a vs], :code "(let [] (letfn [] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [e (genfn-coerce-arg e) a (genfn-coerce-arg a) vs (genfn-coerce-arg vs)] (do (let [extant (when-not (string? e) (->> (dc/q (quote [:find ?z :in $ ?e ?a :where [?e ?a ?z]]) db e a) (map first))) retracts (filter (complement (set vs)) extant) new (filter (complement (set extant)) vs)] (-> [] (into (map (fn [i] [:db/retract e a i]) retracts)) (into (map (fn [i] [:db/add e a i]) new)))))))))"}} #:db{:ident :upsert-entity, :fn #db/fn{:lang :clojure, :imports [[java.util UUID]], :requires [[datomic.api :as dc]], :params [db entity], :code "(let [] (letfn [(-random-tempid [] (do (str (UUID/randomUUID)))) (-by [f fv xs] (do (reduce (fn* [p1__83547# p2__83548#] (assoc p1__83547# (f p2__83548#) (fv p2__83548#))) {} xs))) (-pull-many [db read ids] (do (->> (dc/q (quote [:find (pull ?e r) :in $ [?e ...] r]) db ids read) (map first))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [entity (genfn-coerce-arg entity)] (do (when-not (or (:db/id entity) (:db/ident entity)) (datomic.api/cancel #:cognitect.anomalies{:message (str \"Cannot upsert without :db/id or :db/ident, \" entity), :category :cognitect.anomalies/incorrect})) (let [e (or (:db/id entity) (:db/ident entity)) is-new? (string? e) extant-entity (when-not is-new? (dc/pull db (keys entity) (or (:db/id entity) (:db/ident entity)))) ident->value-type (-by :db/ident (comp :db/ident :db/valueType) (-pull-many db [#:db{:valueType [:db/ident]} :db/ident] (keys entity))) ident->cardinality (-by :db/ident (comp :db/ident :db/cardinality) (-pull-many db [#:db{:cardinality [:db/ident]} :db/ident] (keys entity))) ops (->> entity (reduce (fn [ops [a v]] (cond (= :db/id a) ops (= :db/ident a) ops (or (= v (a extant-entity)) (= v (:db/ident (a extant-entity) :nope)) (= v (:db/id (a extant-entity)) :nope)) ops (and (nil? v) (not (nil? (a extant-entity)))) (if (= :db.cardinality/many (ident->cardinality a)) (into ops (map (fn [v] [:db/retract e a (cond-> v (:db/id v) :db/id)]) (a extant-entity))) (conj ops [:db/retract e a (cond-> (a extant-entity) (:db/id (a extant-entity)) :db/id)])) (nil? v) ops (and (sequential? v) (= :db.type/tuple (ident->value-type a)) (not (= :db.cardinality/many (ident->cardinality a)))) (conj ops [:db/add e a v]) (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) (into ops [[:reset-rels e a v]]) (= :db.cardinality/many (ident->cardinality a)) (into ops [[:reset-scalars e a v]]) (and (sequential? v) (not= :db.type/ref (ident->value-type a))) (into ops [[:reset-scalars e a v]]) (and (map? v) (= :db.type/ref (ident->value-type a))) (let [id (or (:db/id v) (-random-tempid))] (-> ops (conj [:db/add e a id]) (into [[:upsert-entity (assoc v :db/id id)]]))) :else (conj ops [:db/add e a v]))) []))] ops))))))"}} #:db{:ident :upsert-invoice, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db invoice], :code "(let [] (letfn [(-remove-nils [m] (do (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m)) {} m)] (if (seq result) result nil)))) (invoice->journal-entry [db invoice-id raw-invoice-id] (do (let [entity (dc/pull db (quote [:invoice/total :invoice/exclude-from-ledger :invoice/outstanding-balance :invoice/date #:invoice{:client [:db/id :client/code], :status [:db/ident], :import-status [:db/ident], :vendor [:db/id :vendor/name], :payment [:db/id #:payment{:status [:db/ident]}], :expense-accounts [:invoice-expense-account/account :invoice-expense-account/amount :invoice-expense-account/location]}]) invoice-id) credit-invoice? (< (:invoice/total entity 0.0) 0.0)] (when-not (or (not (:invoice/total entity)) (= true (:invoice/exclude-from-ledger entity)) (= :import-status/pending (:db/ident (:invoice/import-status entity))) (= :invoice-status/voided (:db/ident (:invoice/status entity))) (< -0.001 (:invoice/total entity) 0.001)) (-remove-nils #:journal-entry{:date (:invoice/date entity), :original-entity raw-invoice-id, :client (:db/id (:invoice/client entity)), :line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable, :db/id (str raw-invoice-id \"-\" 0), :journal-entry-line/location \"A\"} credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity))) (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))] (map-indexed (fn [i ea] (cond-> {:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)), :db/id (str raw-invoice-id \"-\" (inc i)), :journal-entry-line/location (or (:invoice-expense-account/location ea) \"HQ\")} credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea))) (not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea))))) (:invoice/expense-accounts entity))), :source \"invoice\", :cleared (and (< (:invoice/outstanding-balance entity) 0.01) (every? (fn* [p1__83550#] (= :payment-status/cleared (:payment/status p1__83550#))) (:invoice/payments entity))), :amount (Math/abs (:invoice/total entity)), :vendor (:db/id (:invoice/vendor entity))})))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [invoice (genfn-coerce-arg invoice)] (do (let [upserted-entity [[:upsert-entity invoice]] with-invoice (dc/with db upserted-entity) invoice-id (or (-> with-invoice :tempids (get (:db/id invoice))) (:db/id invoice)) journal-entry (invoice->journal-entry (:db-after with-invoice) invoice-id (:db/id invoice))] (into upserted-entity (if journal-entry [[:upsert-ledger journal-entry]] [[:db/retractEntity [:journal-entry/original-entity (:db/id invoice)]]]))))))))"}} #:db{:ident :upsert-ledger, :fn #db/fn{:lang :clojure, :imports [[java.util UUID Date]], :requires [[datomic.api :as dc]], :params [db ledger-entry], :code "(let [extant-read (quote [:db/id :journal-entry/date :journal-entry/client #:journal-entry{:line-items [:journal-entry-line/account :journal-entry-line/location :db/id :journal-entry-line/client+account+location+date]}])] (letfn [(-random-tempid [] (do (dc/tempid :db.part/user))) (get-line-items-after [db journal-entry] (do (for [jel (:journal-entry/line-items journal-entry) next-jel (->> (dc/index-pull db {:selector [:db/id :journal-entry-line/client+account+location+date], :index :avet, :start [:journal-entry-line/client+account+location+date (:journal-entry-line/client+account+location+date jel) (:db/id jel)]}) (take-while (fn line-must-match-client-account-location [result] (and (= (take 3 (:journal-entry-line/client+account+location+date result)) (take 3 (:journal-entry-line/client+account+location+date jel))) (not= (:db/id jel) (:db/id result))))) (take 2)) :when next-jel] (:db/id next-jel)))) (now [] (do (Date.))) (calc-client+account+location+date [je jel] (do [(or (:db/id (:journal-entry/client je)) (:journal-entry/client je)) (or (:db/id (:journal-entry-line/account jel)) (:journal-entry-line/account jel)) (-> jel :journal-entry-line/location) (-> je :journal-entry/date)]))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [ledger-entry (genfn-coerce-arg ledger-entry)] (do (assert (:journal-entry/date ledger-entry) (format \"Must at least provide date when updating ledger: %s\" (pr-str ledger-entry))) (assert (:journal-entry/client ledger-entry) \"Must at least provide client when updating ledger\") (let [extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)] (dc/pull db extant-read [:journal-entry/original-entity original-entity])) (when-let [external-id (:journal-entry/external-id ledger-entry)] (dc/pull db extant-read [:journal-entry/external-id external-id])))] (cond-> [[:upsert-entity (into (-> ledger-entry (assoc :db/id (or (:db/id ledger-entry) (:db/id extant-entry) (-random-tempid))) (update :journal-entry/line-items (fn [lis] (mapv (fn* [p1__83554#] (-> p1__83554# (assoc :journal-entry-line/date (:journal-entry/date ledger-entry)) (assoc :journal-entry-line/client (:journal-entry/client ledger-entry)))) lis)))))] {:client/ledger-last-change (now), :db/id (:journal-entry/client ledger-entry)}])))))))"}} #:db{:ident :upsert-transaction, :fn #db/fn{:lang :clojure, :imports [], :requires [[datomic.api :as dc]], :params [db transaction], :code "(let [my-transaction {:transaction/bank-account 17592232681223, :transaction/date (read-string \"#inst \\\"2024-02-24T08:00:00.000-00:00\\\"\"), :transaction/matched-rule 17592233159891, :transaction/client 17592232577980, :transaction/status \"POSTED\", :transaction/plaid-merchant {:plaid-merchant/name \"Rotten Robbie\", :db/id \"b2776792-9e2b-46e8-a9c8-bf80abea359e\"}, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc\", :transaction/id \"11a4a13e713d63f476009027e9a53e217e13d0192a37df8ab96c0eed4bdbe996\", :transaction/description-original \"Rotten Robbie #03\", :transaction/approval-status #:db{:id 17592231963877, :ident :transaction-approval-status/approved}, :transaction/amount -84.43, :transaction/accounts [{:transaction-account/account 17592231963549, :db/id \"c402c7b3-c11b-484b-b670-bd48f79a3e5f\", :transaction-account/location \"CB\", :transaction-account/amount 84.43}], :transaction/raw-id \"gQypbv5946F08op74wZmidDg8qD8Q1fM6gEBP\", :transaction/vendor 17592232627053} my-journal #:journal-entry{:vendor 17592232627053, :amount 84.43, :date (read-string \"#inst \\\"2024-02-24T08:00:00.000-00:00\\\"\"), :alternate-description \"Rotten Robbie #03\", :original-entity \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc\", :client 17592232577980, :source \"transaction\", :line-items [{:journal-entry-line/credit 84.43, :journal-entry-line/account 17592232681223, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-0\", :journal-entry-line/location \"A\"} {:journal-entry-line/account 17592231963549, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-1\", :journal-entry-line/debit 84.43, :journal-entry-line/location \"CB\"} {:journal-entry-line/account 17592231963549, :db/id \"ac2efd80-bb03-48b2-b0d0-6b47a5c119dc-2\", :journal-entry-line/debit 84.43, :journal-entry-line/location \"CB\"}], :cleared true}] (letfn [(-remove-nils [m] (do (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m)) {} m)] (if (seq result) result nil)))) (transaction->journal-entry [db transaction-id raw-transaction-id] (do (let [entity (dc/pull db [:transaction/client :transaction/date :transaction/description-original :db/id :transaction/vendor :transaction/amount :transaction/cleared-against #:transaction{:bank-account [:db/id #:bank-account{:type [:db/ident]}], :approval-status [:db/ident], :accounts [:transaction-account/account :transaction-account/location :transaction-account/amount]}] transaction-id) decreasing? (< (or (:transaction/amount entity) 0.0) 0.0) credit-from-bank? decreasing? debit-from-bank? (not decreasing?)] (when (and (not (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))) (not (= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))) (:transaction/amount entity) (not (< -0.001 (:transaction/amount entity) 0.001))) (-remove-nils #:journal-entry{:vendor (:db/id (:transaction/vendor entity)), :amount (Math/abs (:transaction/amount entity)), :date (:transaction/date entity), :alternate-description (:transaction/description-original entity), :original-entity raw-transaction-id, :client (:db/id (:transaction/client entity)), :cleared-against (:transaction/cleared-against entity), :source \"transaction\", :line-items (into [(-remove-nils {:journal-entry-line/credit (when credit-from-bank? (Math/abs (:transaction/amount entity))), :journal-entry-line/account (:db/id (:transaction/bank-account entity)), :db/id (str raw-transaction-id \"-\" 0), :journal-entry-line/debit (when debit-from-bank? (Math/abs (:transaction/amount entity))), :journal-entry-line/location \"A\"})] (map-indexed (fn [i a] (-remove-nils {:journal-entry-line/credit (when debit-from-bank? (Math/abs (:transaction-account/amount a))), :journal-entry-line/account (:db/id (:transaction-account/account a)), :db/id (str raw-transaction-id \"-\" (inc i)), :journal-entry-line/debit (when credit-from-bank? (Math/abs (:transaction-account/amount a))), :journal-entry-line/location (:transaction-account/location a)})) (if (seq (:transaction/accounts entity)) (:transaction/accounts entity) [#:transaction-account{:amount (:transaction/amount entity)}]))), :cleared true})))))] (let [genfn-coerce-arg (clojure.core/fn [x] (clojure.walk/prewalk (clojure.core/fn [e] (if (clojure.core/some? e) (do (clojure.core/when (clojure.core/instance? clojure.lang.PersistentTreeMap e) (throw (clojure.core/ex-info \"Using sorted-map will cause different types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/var? e) (throw (clojure.core/ex-info \"Using var does not work for remote transactor\" {:val e}))) (clojure.core/when (clojure.core/or (clojure.core/= clojure.lang.PersistentList$EmptyList (.getClass e)) (clojure.core/instance? clojure.lang.PersistentList e)) (throw (clojure.core/ex-info \"Using list will cause indistinguishable types in transactor for in-mem and remote\" {:val e}))) (clojure.core/when (clojure.core/instance? clojure.lang.PersistentQueue e) (throw (clojure.core/ex-info \"Using clojure.lang.PersistentQueue does not work for remote transactor\" {:val e}))) (clojure.core/cond (clojure.core/instance? java.util.HashSet e) (clojure.core/into #{} e) (clojure.core/and (clojure.core/instance? java.util.List e) (clojure.core/not (clojure.core/vector? e))) (clojure.core/vec e) :else e)) e)) x))] (let [transaction (genfn-coerce-arg transaction)] (do (let [upserted-entity [[:upsert-entity (dissoc transaction :transaction/payment :import-batch/_entry)]] with-transaction (dc/with db upserted-entity) transaction-id (or (-> with-transaction :tempids (get (:db/id transaction))) (:db/id transaction)) journal-entry (transaction->journal-entry (:db-after with-transaction) transaction-id (:db/id transaction))] (into [[:upsert-entity transaction]] (if journal-entry [[:upsert-ledger journal-entry]] [[:db/retractEntity [:journal-entry/original-entity (:db/id transaction)]]]))))))))"}}] \ No newline at end of file diff --git a/resources/schema.edn b/resources/schema.edn index a48f1f53..246cb476 100644 --- a/resources/schema.edn +++ b/resources/schema.edn @@ -325,6 +325,16 @@ :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "A client's human-friendly name", :db/ident :client/name} +{:db/valueType :db.type/instant + :db/cardinality :db.cardinality/one, + :db/doc "When was the ledger last updated?", + :db/ident :client/ledger-last-change + :db/noHistory true} +{:db/valueType :db.type/instant + :db/cardinality :db.cardinality/one, + :db/doc "When were the running balances last updated?", + :db/ident :client/last-running-balance + :db/noHistory true} {:db/valueType #:db{:ident :db.type/string}, :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "hello@example.com", @@ -878,10 +888,32 @@ :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "The amount to credit", :db/ident :journal-entry-line/credit} +{:db/valueType #:db{:ident :db.type/instant}, + :db/cardinality #:db{:ident :db.cardinality/one}, + :db/doc "The time for this entry", + :db/ident :journal-entry-line/date} {:db/valueType #:db{:ident :db.type/string}, :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "Location of the entry", :db/ident :journal-entry-line/location} +{:db/valueType :db.type/ref, + :db/cardinality #:db{:ident :db.cardinality/one}, + :db/doc "The client for the journal entry line", + :db/ident :journal-entry-line/client} + {:db/valueType :db.type/tuple + :db/tupleAttrs [:journal-entry-line/client + :journal-entry-line/account + :journal-entry-line/location + :journal-entry-line/date + :journal-entry-line/debit + :journal-entry-line/credit + :journal-entry-line/running-balance] + :db/index true + :db/cardinality :db.cardinality/one, + :db/ident :journal-entry-line/running-balance-tuple + :db/doc "[:journal-entry-line/client :journal-entry-line/account :journal-entry-line/location :journal-entry-line/date :db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/running-balance]", + :db/noHistory true + } {:db/ident :legal-entity-1099-type/none} {:db/ident :legal-entity-1099-type/landlord} diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 97e6afae..ae08abe1 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -390,7 +390,8 @@ (get-stats [_] @stats) (get-pending-balance [_ bank-account] - (get @pending-balance bank-account)) + (or (get @pending-balance bank-account) + 0.0)) (fail! [_ error] (alog/error ::cant-complete-import diff --git a/src/clj/auto_ap/jobs/restore_from_backup.clj b/src/clj/auto_ap/jobs/restore_from_backup.clj index 92b13bad..50fbc645 100644 --- a/src/clj/auto_ap/jobs/restore_from_backup.clj +++ b/src/clj/auto_ap/jobs/restore_from_backup.clj @@ -130,10 +130,9 @@ (mu/log ::restore-complete) (mu/log ::beginning-index-build) (auto-ap.datomic/transact-schema auto-ap.datomic/conn) - (auto-ap.ledger/reset-client+account+location+date) (mu/log ::index-build-complete) (mu/log ::refresh-running-balance-cache) - (auto-ap.ledger/rebuild-running-balance-cache) + #_(auto-ap.ledger/rebuild-running-balance-cache) (mu/log ::refresh-running-balance-cache-complete) (mu/log ::done)) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index a37b768e..88863dbe 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -332,51 +332,6 @@ (:bank-account/numeric-code (bank-accounts a))) :client_id client-id}))) - -(defn reset-client+account+location+date - ([] (reset-client+account+location+date (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn))))) - ([clients] - (doseq [[client i] (map vector clients (range))] - (mu/trace ::reset-index - [:client client] - (mu/with-context {:client client - :client-index i - :client-count (count clients)} - (mu/log ::reseting-index) - (let [so-far (atom 0)] - @(->> (dc/qseq {:query '[:find (pull ?je [:journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id]}]) - :in $ ?c - :where [?je :journal-entry/client ?c]] - :args [(dc/db conn) client]}) - (map first) - (mapcat (fn [je] - (map (fn [jel] - {:db/id (:db/id jel) - :journal-entry-line/client+account+location+date - [(-> je :journal-entry/client :db/id) - (-> jel :journal-entry-line/account :db/id) - - (-> jel :journal-entry-line/location) - - (-> je :journal-entry/date)]}) - (:journal-entry/line-items je)))) - (partition-all 500) - (s/->source) - (s/map (fn [batch] - (de/future - (transact-with-backoff batch) - (count batch)))) - (s/buffer 50) - (s/realize-each) - (s/consume (fn [batch-count] - (swap! so-far #(+ % batch-count)) - (mu/log ::reset :count batch-count - :so-far @so-far - :client client - :client-index i - :client-count (count clients)))))) - (mu/log ::client-completed)))))) - (defn find-mismatch-index [] (reduce + 0 (for [c (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn))) @@ -397,214 +352,18 @@ (do (println (count a)) (count a))))) -(defn accounts-needing-rebuild [ db client] - (let [client (pull-id db client)] - (->> (dc/qseq {:query '[:find ?c ?a ?l - :in $ $recent ?c - :where - [$recent ?jel :journal-entry-line/dirty true] - [?je :journal-entry/line-items ?jel] - [?je :journal-entry/client ?c] - [(get-else $ ?jel :journal-entry-line/account :unknown-account) ?a] - [(get-else $ ?jel :journal-entry-line/location "Unknown") ?l]] - :args [db (dc/since db (c/to-date (t/plus (t/now) (t/hours -8)))) client]}) - (map (fn [[client account location]] - {:client client - :account account - :location location}))))) - -(defn all-accounts-needing-rebuild [ db client] - (let [client (pull-id db client)] - (->> (dc/qseq {:query '[:find ?c ?a ?l - :in $ ?c - :where - [?je :journal-entry/client ?c] - [?je :journal-entry/line-items ?jel] - [(get-else $ ?jel :journal-entry-line/account :unknown-account) ?a] - [(get-else $ ?jel :journal-entry-line/location "Unknown") ?l] ] - :args [db client]}) - (map (fn [[client account location ]] - {:client client - :account account - :location location}))))) - -(defn get-entries-to-refresh [{:keys [client account location]} db ] - (let [client (pull-id db client) - account-id (if (= :unknown-account account) - nil - (pull-id db account)) - matching-location (if (= "Unknown" location) - nil - location)] - - (into [] - (comp - (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}] - (and - (= client result-client) - (= account-id result-account) - (= matching-location result-location)))) - (map (fn [result] - [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) (:journal-entry-line/running-balance result 0.0) (:journal-entry-line/dirty result)]))) - (dc/index-pull db - {:index :avet - :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date :journal-entry-line/running-balance :journal-entry-line/dirty] - :start [:journal-entry-line/client+account+location+date - [client - account-id - matching-location - #inst "0001-01-01" ]] })))) - -(defn compute-running-balance [account-needing-refresh] - (mu/trace ::compute - [:dirty-count (count (:dirty-entries account-needing-refresh))] - (when (= 0 (count (:dirty-entries account-needing-refresh))) - (alog/warn ::no-entries-to-compute - :message "This typically means that an account is determined to be dirty, but no entries are found, meaning bad query")) - (second - (reduce - (fn [[running-balance changed-rows] [id debit credit extant-running-balance dirty] ] - (let [new-running-balance (+ running-balance - (if (#{:account-type/asset - :account-type/dividend - :account-type/expense} (:account-type account-needing-refresh)) - (- debit credit) - (- credit debit))) - row-changed? (or (not (dollars= new-running-balance extant-running-balance)) - dirty)] - [new-running-balance - (if row-changed? - (conj changed-rows - {:db/id id - :journal-entry-line/running-balance new-running-balance - :journal-entry-line/debit debit - :journal-entry-line/credit credit - :journal-entry-line/dirty false}) - changed-rows)])) - [0.0 []] - (:dirty-entries account-needing-refresh))))) - -(defn refresh-running-balance-accounts [accounts-needing-rebuild clients c i db] - - (mu/log ::found-accounts-needing-rebuild - :accounts (count accounts-needing-rebuild)) - (let [so-far (atom 0)] - @(->> accounts-needing-rebuild - (s/->source) - (s/map (fn [account-needing-rebuild] - (-> - (de/future - (mu/with-context {:account account-needing-rebuild} - (let [result (-> account-needing-rebuild - (assoc :dirty-entries (get-entries-to-refresh account-needing-rebuild db)) - (assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild)))) - (compute-running-balance))] - (alog/info ::recomputed-entries-with-new-sum :count (count result) ) - result) - )) - (de/catch (fn [error] - (alog/error ::cant-rebuild - :error error) - nil))))) - (s/realize-each) - (s/mapcat (fn [x] - x)) - (s/buffer 50) - (s/transform (partition-all 500)) - (s/map (fn [batch] - (de/future - (transact-with-backoff batch) - (count batch)))) - (s/buffer 50) - (s/realize-each) - (s/consume (fn [batch-count] - (swap! so-far #(+ % batch-count)) - (mu/log ::reset - :count batch-count - :so-far @so-far - :client c - :client-index i - :client-count (count clients))))))) - - -(defn clients-needing-refresh [] +(defn clients-needing-refresh [db] (->> - (dc/q '[:find (pull ?c [:client/code :db/id]) (count ?jel) - :in $ $recent - :where [$recent ?jel :journal-entry-line/dirty true] - [$ ?je :journal-entry/line-items ?jel] - [$ ?je :journal-entry/client ?c] - [$ ?jel :journal-entry-line/dirty true]] - (dc/db conn) - (dc/since - (dc/db conn) - (c/to-date (t/plus (t/now) (t/hours -8))))) - (map (fn [[client outdated]] - (assoc client :dirty-count outdated))))) - -(defn refresh-running-balance-cache - ([] (refresh-running-balance-cache (shuffle (clients-needing-refresh)))) - ([clients] - (doseq [[c i] (map vector clients (range))] - (mu/trace ::building-running-balance - [:client c] - (mu/with-context {:client c - :client-index i - :client-count (count clients)} - (mu/log ::searching-for-accounts) - (let [db (dc/db conn) - accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))] - (when (seq accounts-needing-rebuild) - (refresh-running-balance-accounts accounts-needing-rebuild clients c i db) - (mu/log ::client-completed)))))))) - -(defn rebuild-running-balance-cache - ([] (rebuild-running-balance-cache (shuffle (map first - (dc/q '[:find (pull ?c [:client/code :db/id]) - :where [?c :client/code]] - (dc/db conn)))))) - ([clients] - (doseq [[c i] (map vector clients (range))] - (mu/trace ::building-running-balance - [:client c] - (mu/with-context {:client c - :client-index i - :client-count (count clients)} - (mu/log ::searching-for-accounts) - (let [db (dc/db conn) - accounts-needing-rebuild (all-accounts-needing-rebuild db (:db/id c))] - (when (seq accounts-needing-rebuild) - (refresh-running-balance-accounts accounts-needing-rebuild clients c i db) - (mu/log ::client-completed)))))))) - -#_(seq (dc/q '[:find ?le ?d - :in $ - :where [?le :journal-entry/date ?d] - [(<= ?d #inst "2000-01-01")]] - (dc/db conn))) - -#_( - (refresh-running-balance-cache) - - (accounts-needing-rebuild (dc/db conn) [ :client/code "SCCB"]) - - (user/init-repl) - - (get-entries-to-refresh {:client (pull-id (dc/db conn) [:client/code "SCCB"]) - :account (pull-id (dc/db conn) [:bank-account/code "SCCB-USB9598" ]) - :location "A"} - - (dc/db conn)) - (rebuild-running-balance-cache [{:client/code "SCCB" :db/id (pull-id (dc/db conn) [:client/code "SCCB"])}]) - - (clients-needing-refresh) - (dc/q '[:find (count ?d) - :in $ - :where [?d :journal-entry-line/dirty true] - ] - (dc/db conn)) - -) + (dc/q '[:find (pull ?c [:client/code :db/id :client/ledger-last-change :client/last-running-balance]) + :in $ + :where + [?c :client/code] + [(get-else $ ?c :client/ledger-last-change #inst "2040-01-01") ?last-change] + [(get-else $ ?c :client/last-running-balance #inst "2000-01-01") ?last-running-balance] + [(> ?last-change ?last-running-balance)] ] + db) + (map (fn [[client]] + client)))) #_(comment [17592334354011 #inst "0024-08-03T07:52:58.000-00:00"] [17592302554688 #inst "0023-07-20T07:52:58.000-00:00"] @@ -617,15 +376,107 @@ [17592302554685 #inst "0023-07-16T07:52:58.000-00:00"] [17592334353244 #inst "0024-07-14T07:52:58.000-00:00"]) -;; TODO -;; 1. X Having an uncategorized running balance -;; 1a. X dirty should always be 0 -;; 2. X rebuild from beginning of history always, only update entry if it currently doesn't match -;; 3. X deterministic order (date + entityid) -;; 4. Check for errors +(defn mark-all-clients-dirty [] + (auto-ap.datomic/audit-transact-batch + (for [[c] (dc/q '[:find ?c + :in $ + :where [?c :client/code]] + (dc/db conn))] + {:db/id c :client/ledger-last-change (c/to-date (t/now))}) + {:user/name "backfill-client and dates"})) + +(defn upsert-running-balance [] + (let [db (dc/db conn) + starting-at (c/to-date (t/now)) + clients (clients-needing-refresh db) + _ (alog/info ::clients-needing-update :clients clients :count (count clients)) + client-change-stats (atom {}) + changes (for [c clients + :let [client-id (:db/id c) + account-lookup (build-account-lookup client-id)] + running-balance-set (->> (dc/index-range db :journal-entry-line/running-balance-tuple [client-id] [(inc client-id)]) + (seq) + (partition-by (fn [{[current-client current-account current-location current-date debit credit running-balance] + :v}] + [current-client current-account current-location]))) + running-balance-change (->> running-balance-set + (sort-by (fn [{id :e [_ _ _ current-date] :v}] + [current-date id])) + (reduce + (fn [{:keys [changes last-running-balance]} + {id :e + [current-client current-account current-location current-date debit credit running-balance] + :v}] + (let [delta (if (#{:account-type/asset + :account-type/dividend + :account-type/expense} (:account_type (account-lookup current-account))) + (- (or debit 0.0) (or credit 0.0)) + (- (or credit 0.0) (or debit 0.0))) + correct-running-balance (+ last-running-balance delta) + running-balance-changed? (not (dollars= correct-running-balance (or running-balance 0.0)))] + (when running-balance-changed? + (swap! client-change-stats update (:client/code c) (fnil inc 0))) + (cond-> {:last-account-lookup account-lookup + :last-running-balance correct-running-balance + :changes changes} + + running-balance-changed? + (update :changes conj {:db/id id + :journal-entry-line/running-balance correct-running-balance})))) + {:last-running-balance 0.0}) + :changes)] + running-balance-change)] + (alog/info ::change-stats :stats @client-change-stats) + (mu/trace ::update-running-balance [] + (auto-ap.datomic/audit-transact-batch changes + {:user/name "running balance updater"})) + (auto-ap.datomic/audit-transact (mapv (fn [c] + {:db/id (:db/id c) + :client/last-running-balance starting-at}) + clients) + {:user/name "running balance updater"}) + (count changes))) + + + +(comment +(mark-all-clients-dirty) + + (upsert-running-balance) + + ;; SETUP running-balance-tuple + (doseq [[c] (dc/q '[:find ?c + :in $ + :where [?c :client/code]] + (dc/db conn))] + (println "CLIENT " c) + (auto-ap.datomic/audit-transact-batch + (for [[date client line] (->> (dc/q '[:find ?jed ?jec (pull ?jel [:journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/running-balance :db/id]) + :in $ ?jec + :where [?je :journal-entry/client ?jec] + [?je :journal-entry/date ?jed] + [?je :journal-entry/line-items ?jel]] + (dc/db conn) + c))] + {:db/id (:db/id line) + :journal-entry-line/date date + :journal-entry-line/client client}) + + + {:user/name "backfill-client and dates"}) + (println "done.")) + + + + + #_(dc/q '[:find (pull ?je [*]) (pull ?jel [*]) + :where [?je :journal-entry/line-items ?jel] + (not [?jel :journal-entry-line/running-balance-tuple])] + (dc/db conn))) + ;; TODO only enable once IOL is set up in clod #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (mount/defstate running-balance-cache-worker - :start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache")) + :start (scheduler/every (* 5 60 (+ 500 (rand-int 500))) (heartbeat upsert-running-balance "running-balance-cache")) :stop (scheduler/stop running-balance-cache-worker)) diff --git a/terraform/deploy.tf b/terraform/deploy.tf index 7662e878..6e6f95c9 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -293,19 +293,6 @@ module "reconcile_ledger_job" { cpu = 2048 } -module "current_balance_cache" { - count = var.enable_schedules ? 1 : 0 - source = "./background-job/" - ecs_cluster = var.ecs_cluster - task_role_arn = var.task_role_arn - stage = var.stage - schedule = "rate(30 minutes)" - job_name = "current-balance-cache" - execution_role_arn = var.execution_role_arn - memory = 2048 - cpu = 512 -} - module "yodlee2_job" { count = var.enable_schedules ? 1 : 0 source = "./background-job/"