diff --git a/iol_ion/src/iol_ion/tx.clj b/iol_ion/src/iol_ion/tx.clj index ae6b0edd..2f08346a 100644 --- a/iol_ion/src/iol_ion/tx.clj +++ b/iol_ion/src/iol_ion/tx.clj @@ -164,8 +164,7 @@ (assert (:journal-entry/client ledger-entry) "Must at least provide client when updating ledger") (assert (every? :journal-entry-line/account (:journal-entry/line-items ledger-entry)) "must at least provide account when updating ledger") (assert (every? :journal-entry-line/location (:journal-entry/line-items ledger-entry)) "Must at least provide location when updating ledger") - (let [ - extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)] + (let [extant-entry (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)] (dc/pull db extant-read [:journal-entry/original-entity original-entity])) (when-let [external-id (:journal-entry/external-id ledger-entry)] (dc/pull db extant-read [:journal-entry/external-id external-id]))) diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index 6ebec88b..da7d70cf 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -636,7 +636,7 @@ true (update :tempids merge (:tempids tx-result))))) {} - (partition-all 50 txes)))) + (partition-all 200 txes)))) (defn audit-transact [txes id] (dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx" @@ -664,10 +664,13 @@ (str (UUID/randomUUID))) (defn pull-id [db id] - (ffirst (dc/q '[:find ?i - :in $ ?i] - db - id))) + (if (sequential? id) + (ffirst (dc/q '[:find ?i + :in $ [?a ?v] + :where [?i ?a ?v]] + db + id)) + id)) (defn pull-attr [db k id] (get (dc/pull db [k] id) k)) diff --git a/src/clj/auto_ap/datomic/vendors.clj b/src/clj/auto_ap/datomic/vendors.clj index 7a267de5..1ba7233c 100644 --- a/src/clj/auto_ap/datomic/vendors.clj +++ b/src/clj/auto_ap/datomic/vendors.clj @@ -54,8 +54,9 @@ (not (str/blank? (:name-like args))) (merge-query {:query {:in ['?name-like] :where ['[?e :vendor/name ?n] - '[(re-find ?name-like ?n)]]} - :args [(re-pattern (str "(?i)" (:name-like args)))]}) + '[(re-pattern ?name-like) ?name-like-2] + '[(re-find ?name-like-2 ?n)]]} + :args [(str "(?i)" (:name-like args))]}) true (merge-query {:query {:find ['?e] diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 7b32bba7..ce638696 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -573,70 +573,74 @@ :starting-at starting-at :location location}))))) +(defn find-running-balance-start [{:keys [client account location starting-at]} db ] + (let [client (pull-id db client) + account (pull-id db account)] + (or + (->> (dc/index-pull db + {:index :avet + :selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date] + :start [:journal-entry-line/client+account+location+date [client account location starting-at]] + :reverse true + :limit 500}) + (take-while (fn [result] + (= [client + account + location] + (take 3 (:journal-entry-line/client+account+location+date result))))) + (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}] + (>= (compare date starting-at) 0))) + first + :journal-entry-line/running-balance) + 0.0))) -(defn find-running-balance-start [account-needing-rebuild db ] - (or - (->> (dc/index-pull db - {:index :avet - :selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date] - :start [:journal-entry-line/client+account+location+date - [(:client account-needing-rebuild) - (:account account-needing-rebuild) - (:location account-needing-rebuild) - (:starting-at account-needing-rebuild)]] - - :reverse true - :limit 500}) - (take-while (fn [result] - (= [(:client account-needing-rebuild) - (:account account-needing-rebuild) - (:location account-needing-rebuild)] - (take 3 (:journal-entry-line/client+account+location+date result))))) - (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}] - (>= (compare date (:starting-at account-needing-rebuild)) 0))) - first - :journal-entry-line/running-balance - ) - 0.0)) - -(defn get-dirty-entries [account-needing-rebuild db ] - (->> (dc/index-pull db - {:index :avet - :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date] - :start [:journal-entry-line/client+account+location+date - [(:client account-needing-rebuild) - (:account account-needing-rebuild) - (:location account-needing-rebuild) - (:starting-at account-needing-rebuild)]] - }) - (take-while (fn [result] - (= [(:client account-needing-rebuild) - (:account account-needing-rebuild) - (:location account-needing-rebuild)] - (take 3 (:journal-entry-line/client+account+location+date result))))) - (map (fn [result] - [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ])))) +(defn get-dirty-entries [{:keys [client account location starting-at]} db ] + (mu/trace ::get-dirty-entries + [] + (let [client (pull-id db client) + account (pull-id db account)] + (into [] + (comp + (mapcat (fn [i] + (dc/index-pull db + {:index :avet + :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date] + :start [:journal-entry-line/client+account+location+date + [client account location starting-at]] + :offset (* i 1000) + :limit 1000} + )) + ) + + (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}] + (and + (= client result-client) + (= account result-account) + (= location result-location)))) + (map (fn [result] + [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))) + (range))))) (defn compute-running-balance [account-needing-refresh] - (mu/log ::compute - :dirty-count (count (:dirty-entries account-needing-refresh))) - (second - (reduce - (fn [[running-balance rows] [id debit credit] ] - (let [new-running-balance (+ running-balance - (if (#{:account-type/asset - :account-type/dividend - :account-type/expense} (:account-type account-needing-refresh)) - (- debit credit) - (- credit debit)))] - [new-running-balance - (conj rows - {:db/id id - :journal-entry-line/running-balance new-running-balance - :journal-entry-line/dirty false})])) + (mu/trace ::compute + [:dirty-count (count (:dirty-entries account-needing-refresh))] + (second + (reduce + (fn [[running-balance rows] [id debit credit] ] + (let [new-running-balance (+ running-balance + (if (#{:account-type/asset + :account-type/dividend + :account-type/expense} (:account-type account-needing-refresh)) + (- debit credit) + (- credit debit)))] + [new-running-balance + (conj rows + {:db/id id + :journal-entry-line/running-balance new-running-balance + :journal-entry-line/dirty false})])) - [(:build-from account-needing-refresh) []] - (:dirty-entries account-needing-refresh)))) + [(:build-from account-needing-refresh) []] + (:dirty-entries account-needing-refresh))))) (defn refresh-running-balance-cache @@ -653,7 +657,7 @@ accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))] (when (seq accounts-needing-rebuild) (mu/log ::found-accounts-needing-rebuild - :accounts accounts-needing-rebuild) + :accounts (count accounts-needing-rebuild)) (audit-transact-batch (->> accounts-needing-rebuild (mapcat (fn [account-needing-rebuild] @@ -666,6 +670,7 @@ {:user/name "running-balance-cache"})))))))) +;; TODO only enable once IOL is set up in clod #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_(mount/defstate running-balance-cache-worker :start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache")) diff --git a/things-to-search-for.txt b/things-to-search-for.txt index 425b5f8c..b5f80b64 100644 --- a/things-to-search-for.txt +++ b/things-to-search-for.txt @@ -1,15 +1,9 @@ or-join syntax changed? -Make sure no history on ledger +regex no longer supported as argument for query. Check vendor datomic for how to do it right. it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes ezcater graphql needs search index too make sure that temporary ids are set on all new things when using upsert-entity Wrap tests around every api call -Fix tests -Make a different solution for running balance cache -* Store the running balance on the line -* Make ledger changes mark as dirty -* Recompute the cache periodically, somewhat randomly to avoid races -* Have a background job recompute all at night. Fix searching * indexing should happen more regularly, and just look for changes since last time it was run @@ -17,9 +11,27 @@ Fix searching Running Balance Cache * Add tests for upsert-ledger * try again to see if we can get upsert-ledger into the same transaction, making it all or nothing -* make rest of rebuilding the cache use new index -* ensure somehow that the index is always right - +* Make a new way to reset the entire cache for a client Address memory * JVM settings now and in prod + +Release steps: +Stop prod +Make database snapshot +Create new database for prod-cloud (just called prod) +Restore database +Transact new schema +(reset-client+account+location+date) +(force-rebuild-running-balance-cache) +Merge branch into master +Rename prod-cloud to prod everywhere +Release again + +Sanity checks later: +* Run query + + +Future improvements: +Make reports just be based on running-balances +