notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) running balance update way more streamlined now

Browse Source
This commit is contained in:
Bryce Covert
2023-03-30 16:44:25 -07:00
parent f7d21e2bbf
commit 5df47c8837
5 changed files with 78 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/clj/auto_ap/graphql/clients.clj
View File

@@ -170,77 +170,34 @@
lms))) lms)))
->graphql))) ->graphql)))
(defn refresh-bank-account-current-balance [bank-account-id]
(let [all-transactions (dc/q
{:query {:find ['?e '?debit '?credit]
:in ['$ '?bank-account-id]
:where '[[?e :journal-entry-line/account ?bank-account-id]
[(get-else $ ?e :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?e :journal-entry-line/credit 0.0) ?credit]]}
:args [(dc/db conn) bank-account-id]})
debits (->> all-transactions
(map (fn [[_ debit _]]
debit))
(reduce + 0.0))
credits (->> all-transactions
(map (fn [[_ _ credit]]
credit))
(reduce + 0.0))
current-balance (if (= :bank-account-type/check (:db/ident (:bank-account/type (dc/pull (dc/db conn) [:bank-account/type] bank-account-id))))
(- debits credits)
(- credits debits))]
(dc/transact conn {:tx-data [{:db/id bank-account-id
:bank-account/current-balance current-balance}]})))
(defn bank-accounts-needing-refresh []
(let [last-refreshed (->> (dc/q {:query {:find ['?ba '?tx]
:in ['$ ]
:where ['[?ba :bank-account/current-balance _ ?tx true]]}
:args [(dc/history (dc/db conn))]})
(group-by first)
(map (fn [[ba balance-txes]]
[ba (->> balance-txes
(map second)
(sort-by #(- %))
first)])))
has-newer-transaction (->> (dc/q
{:query {:find ['?ba]
:in '[$ [[?ba ?last-refreshed] ...] ]
:where ['[_ :journal-entry-line/account ?ba ?tx]
'[(>= ?tx ?last-refreshed)]]}
:args [(dc/history (dc/db conn))
last-refreshed]})
(map first)
(set))
no-current-balance (->> (dc/q {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]
'(not [?ba :bank-account/current-balance])
]}
:args [(dc/db conn)]})
(map first))]
(into has-newer-transaction no-current-balance)))
(defn build-current-balance [bank-accounts]
(doseq [bank-account bank-accounts]
(log/info "Refreshing bank account" (-> (dc/db conn) (dc/pull [:bank-account/code] bank-account)))
(try
(refresh-bank-account-current-balance bank-account)
(catch Exception e
(log/error "Can't refresh current balance" e)))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn refresh-all-current-balance [] (defn refresh-all-current-balance []
(lc/with-context {:source "current-balance-cache"} (lc/with-context {:source "current-balance-refresh"}
(build-current-balance (->> (dc/q {:query {:find ['?ba] (let [db (dc/db conn)
:in '[$] clients (dc/q '[:find (pull ?c [:db/id :client/code {:client/bank-accounts [:db/id :bank-account/code]}])
:where ['[?ba :bank-account/code]]} :where [?c :client/code]]
:args [(dc/db conn)]}) db )]
(map first))))) (dc/transact conn
{:tx-data
(defn refresh-current-balance [] (for [[{client :db/id code :client/code bank-accounts :client/bank-accounts}] clients
(build-current-balance (bank-accounts-needing-refresh))) {bank-account :db/id bac :bank-account/code} bank-accounts]
{:db/id bank-account
:bank-account/current-balance
(or
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/running-balance :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}]
:start [:journal-entry-line/client+account+location+date [client bank-account "A" #inst "2030-01-01"]]
:limit 1
:reverse true
})
(filter (fn [{[c b] :journal-entry-line/client+account+location+date}]
(and (= c client)
(= b bank-account))))
(map :journal-entry-line/running-balance)
(first))
0.0)})})))
)
(defn get-client [context _ _] (defn get-client [context _ _]
(->graphql (->graphql

2
src/clj/auto_ap/jobs/current_balance_cache.clj
View File

@@ -5,7 +5,7 @@
[auto-ap.jobs.core :refer [execute]])) [auto-ap.jobs.core :refer [execute]]))
(defn -main [& _] (defn -main [& _]
(execute "current-balance-cache" clients/refresh-current-balance)) (execute "current-balance-cache" clients/refresh-all-current-balance))

13
src/clj/auto_ap/ledger.clj
View File

@@ -2,13 +2,12 @@
(:require (:require
[auto-ap.datomic [auto-ap.datomic
:refer [audit-transact :refer [audit-transact
audit-transact-batch
transact-with-backoff
conn conn
pull-id pull-id
pull-ref pull-ref
remove-nils]] remove-nils
[auto-ap.utils :refer [by dollars-0? dollars=]] transact-with-backoff]]
[auto-ap.utils :refer [by dollars-0? dollars= heartbeat]]
[clj-time.coerce :as c] [clj-time.coerce :as c]
[clj-time.core :as t] [clj-time.core :as t]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
@@ -17,7 +16,9 @@
[datomic.client.api :as dc] [datomic.client.api :as dc]
[iol-ion.tx :refer [upsert-ledger]] [iol-ion.tx :refer [upsert-ledger]]
[manifold.deferred :as de] [manifold.deferred :as de]
[manifold.stream :as s])) [manifold.stream :as s]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
(defn datums->impacted-entity [db [e changes]] (defn datums->impacted-entity [db [e changes]]
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) (let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
@@ -754,6 +755,6 @@
;; TODO only enable once IOL is set up in clod ;; TODO only enable once IOL is set up in clod
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
#_(mount/defstate running-balance-cache-worker (mount/defstate running-balance-cache-worker
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache")) :start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
:stop (scheduler/stop running-balance-cache-worker)) :stop (scheduler/stop running-balance-cache-worker))

57
test/clj/auto_ap/integration/graphql/clients.clj
View File

@@ -9,63 +9,6 @@
(use-fixtures :each wrap-setup) (use-fixtures :each wrap-setup)
(deftest current-balance
(testing "it should start with a balance of 0"
(dc/transact conn
{:tx-data
[{:client/code "TEST"
:client/bank-accounts [{:bank-account/code "TEST-1"}]}]})
(sut/refresh-current-balance)
(is (= 0.00
(pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"] ))))
(testing "it should consider a single transaction"
(dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-1"
:db/id "TEST-1"}]}
{:journal-entry-line/account "TEST-1"
:journal-entry-line/debit 13.50}
{:journal-entry-line/account "TEST-1"
:journal-entry-line/debit 19.50}]})
(sut/refresh-current-balance)
(is (= -33.0
(pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"]))))
(testing "bank accounts should start in a needing refresh state"
(let [bank-account-id (-> (dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-2" :db/id "TEST-2"}]}
]})
:tempids
(get "TEST-2"))]
(is ((sut/bank-accounts-needing-refresh) bank-account-id ))))
(testing "bank accounts should not need a refresh if balance is up-to-date"
(let [bank-account-id (-> (dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-3" :db/id "TEST-3"}]}
]})
:tempids
(get "TEST-3"))]
(sut/refresh-bank-account-current-balance bank-account-id)
(is (not ((sut/bank-accounts-needing-refresh) bank-account-id )))
(dc/transact conn
{:tx-data
[{:journal-entry-line/account [:bank-account/code "TEST-3"]
:journal-entry-line/debit -10.50}
]})
(is ((sut/bank-accounts-needing-refresh) bank-account-id ))
(sut/refresh-bank-account-current-balance bank-account-id)
(is (not ((sut/bank-accounts-needing-refresh) bank-account-id ))))))
(deftest upsert-client (deftest upsert-client
(testing "Should create a new client" (testing "Should create a new client"

58
things-to-search-for.txt
View File

@@ -1,8 +1,9 @@
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling
Double check each job still functions in the new system
Fix searching Reconcile ledger. Does it work? What are the downsides? Can it be made faster now?
* indexing should happen more regularly, and just look for changes since last time it was run Make reports just be based on running-balances
When you add a vendor, it should be searchable immediately
Running Balance Cache Running Balance Cache
* Add tests for upsert-ledger * Add tests for upsert-ledger
@@ -12,17 +13,6 @@ Running Balance Cache
Address memory Address memory
* JVM settings now and in prod * JVM settings now and in prod
Release steps:
Stop prod
Make database snapshot
Create new database for prod-cloud (just called prod)
Restore database
Transact new schema
(reset-client+account+location+date)
(force-rebuild-running-balance-cache)
Merge branch into master
Rename prod-cloud to prod everywhere
Release again
Sanity checks later: Sanity checks later:
* Run query * Run query
@@ -30,6 +20,46 @@ Sanity checks later:
Future improvements: Future improvements:
Make reports just be based on running-balances Make reports just be based on running-balances
Just use a periodic request or event instead of a job for running balance cache, and perhaps others too
get rid of account-groups get rid of account-groups
move to solr move to solr
upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null
Release steps:
Set prod web workers to 0
Make database snapshot (run export-job)
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition "XXX"
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d"
:starting-at "sales-order"})}]}])))
Create new database for prod-cloud (just called prod)
(dc/create-database conn {:db-name "prod"})
Set this in the prod-cloud config file and prod-cloud-background-worker
Restore database
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition "restore_from_backup_prod_cloud:3"
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d"
:starting-at "sales-order"})}]}])))
Merge branch into master
Rename prod-cloud to prod everywhere
Release again
git push deploy master