notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) running balance update way more streamlined now

Browse Source
This commit is contained in:
Bryce Covert
2023-03-30 16:44:25 -07:00
parent f7d21e2bbf
commit 5df47c8837
5 changed files with 78 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/clj/auto_ap/graphql/clients.clj
View File

@@ -170,77 +170,34 @@
lms)))
->graphql)))
(defn refresh-bank-account-current-balance [bank-account-id]
(let [all-transactions (dc/q
{:query {:find ['?e '?debit '?credit]
:in ['$ '?bank-account-id]
:where '[[?e :journal-entry-line/account ?bank-account-id]
[(get-else $ ?e :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?e :journal-entry-line/credit 0.0) ?credit]]}
:args [(dc/db conn) bank-account-id]})
debits (->> all-transactions
(map (fn [[_ debit _]]
debit))
(reduce + 0.0))
credits (->> all-transactions
(map (fn [[_ _ credit]]
credit))
(reduce + 0.0))
current-balance (if (= :bank-account-type/check (:db/ident (:bank-account/type (dc/pull (dc/db conn) [:bank-account/type] bank-account-id))))
(- debits credits)
(- credits debits))]
(dc/transact conn {:tx-data [{:db/id bank-account-id
:bank-account/current-balance current-balance}]})))
(defn bank-accounts-needing-refresh []
(let [last-refreshed (->> (dc/q {:query {:find ['?ba '?tx]
:in ['$ ]
:where ['[?ba :bank-account/current-balance _ ?tx true]]}
:args [(dc/history (dc/db conn))]})
(group-by first)
(map (fn [[ba balance-txes]]
[ba (->> balance-txes
(map second)
(sort-by #(- %))
first)])))
has-newer-transaction (->> (dc/q
{:query {:find ['?ba]
:in '[$ [[?ba ?last-refreshed] ...] ]
:where ['[_ :journal-entry-line/account ?ba ?tx]
'[(>= ?tx ?last-refreshed)]]}
:args [(dc/history (dc/db conn))
last-refreshed]})
(map first)
(set))
no-current-balance (->> (dc/q {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]
'(not [?ba :bank-account/current-balance])
]}
:args [(dc/db conn)]})
(map first))]
(into has-newer-transaction no-current-balance)))
(defn build-current-balance [bank-accounts]
(doseq [bank-account bank-accounts]
(log/info "Refreshing bank account" (-> (dc/db conn) (dc/pull [:bank-account/code] bank-account)))
(try
(refresh-bank-account-current-balance bank-account)
(catch Exception e
(log/error "Can't refresh current balance" e)))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn refresh-all-current-balance []
(lc/with-context {:source "current-balance-cache"}
(build-current-balance (->> (dc/q {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]]}
:args [(dc/db conn)]})
(map first)))))
(defn refresh-current-balance []
(build-current-balance (bank-accounts-needing-refresh)))
(lc/with-context {:source "current-balance-refresh"}
(let [db (dc/db conn)
clients (dc/q '[:find (pull ?c [:db/id :client/code {:client/bank-accounts [:db/id :bank-account/code]}])
:where [?c :client/code]]
db )]
(dc/transact conn
{:tx-data
(for [[{client :db/id code :client/code bank-accounts :client/bank-accounts}] clients
{bank-account :db/id bac :bank-account/code} bank-accounts]
{:db/id bank-account
:bank-account/current-balance
(or
(->> (dc/index-pull db
{:index :avet
:selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/running-balance :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}]
:start [:journal-entry-line/client+account+location+date [client bank-account "A" #inst "2030-01-01"]]
:limit 1
:reverse true
})
(filter (fn [{[c b] :journal-entry-line/client+account+location+date}]
(and (= c client)
(= b bank-account))))
(map :journal-entry-line/running-balance)
(first))
0.0)})})))
)
(defn get-client [context _ _]
(->graphql

2
src/clj/auto_ap/jobs/current_balance_cache.clj
View File

@@ -5,7 +5,7 @@
[auto-ap.jobs.core :refer [execute]]))
(defn -main [& _]
(execute "current-balance-cache" clients/refresh-current-balance))
(execute "current-balance-cache" clients/refresh-all-current-balance))

13
src/clj/auto_ap/ledger.clj
View File

@@ -2,13 +2,12 @@
(:require
[auto-ap.datomic
:refer [audit-transact
audit-transact-batch
transact-with-backoff
conn
pull-id
pull-ref
remove-nils]]
[auto-ap.utils :refer [by dollars-0? dollars=]]
remove-nils
transact-with-backoff]]
[auto-ap.utils :refer [by dollars-0? dollars= heartbeat]]
[clj-time.coerce :as c]
[clj-time.core :as t]
[clojure.tools.logging :as log]
@@ -17,7 +16,9 @@
[datomic.client.api :as dc]
[iol-ion.tx :refer [upsert-ledger]]
[manifold.deferred :as de]
[manifold.stream :as s]))
[manifold.stream :as s]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
(defn datums->impacted-entity [db [e changes]]
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
@@ -754,6 +755,6 @@
;; TODO only enable once IOL is set up in clod
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
#_(mount/defstate running-balance-cache-worker
(mount/defstate running-balance-cache-worker
:start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))
:stop (scheduler/stop running-balance-cache-worker))

57
test/clj/auto_ap/integration/graphql/clients.clj
View File

@@ -9,63 +9,6 @@
(use-fixtures :each wrap-setup)
(deftest current-balance
(testing "it should start with a balance of 0"
(dc/transact conn
{:tx-data
[{:client/code "TEST"
:client/bank-accounts [{:bank-account/code "TEST-1"}]}]})
(sut/refresh-current-balance)
(is (= 0.00
(pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"] ))))
(testing "it should consider a single transaction"
(dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-1"
:db/id "TEST-1"}]}
{:journal-entry-line/account "TEST-1"
:journal-entry-line/debit 13.50}
{:journal-entry-line/account "TEST-1"
:journal-entry-line/debit 19.50}]})
(sut/refresh-current-balance)
(is (= -33.0
(pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"]))))
(testing "bank accounts should start in a needing refresh state"
(let [bank-account-id (-> (dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-2" :db/id "TEST-2"}]}
]})
:tempids
(get "TEST-2"))]
(is ((sut/bank-accounts-needing-refresh) bank-account-id ))))
(testing "bank accounts should not need a refresh if balance is up-to-date"
(let [bank-account-id (-> (dc/transact conn
{:tx-data
[{:client/code "TEST"
:db/id "TEST"
:client/bank-accounts [{:bank-account/code "TEST-3" :db/id "TEST-3"}]}
]})
:tempids
(get "TEST-3"))]
(sut/refresh-bank-account-current-balance bank-account-id)
(is (not ((sut/bank-accounts-needing-refresh) bank-account-id )))
(dc/transact conn
{:tx-data
[{:journal-entry-line/account [:bank-account/code "TEST-3"]
:journal-entry-line/debit -10.50}
]})
(is ((sut/bank-accounts-needing-refresh) bank-account-id ))
(sut/refresh-bank-account-current-balance bank-account-id)
(is (not ((sut/bank-accounts-needing-refresh) bank-account-id ))))))
(deftest upsert-client
(testing "Should create a new client"

58
things-to-search-for.txt
View File

@@ -1,8 +1,9 @@
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling
Fix searching
* indexing should happen more regularly, and just look for changes since last time it was run
Double check each job still functions in the new system
Reconcile ledger. Does it work? What are the downsides? Can it be made faster now?
Make reports just be based on running-balances
When you add a vendor, it should be searchable immediately
Running Balance Cache
* Add tests for upsert-ledger
@@ -12,17 +13,6 @@ Running Balance Cache
Address memory
* JVM settings now and in prod
Release steps:
Stop prod
Make database snapshot
Create new database for prod-cloud (just called prod)
Restore database
Transact new schema
(reset-client+account+location+date)
(force-rebuild-running-balance-cache)
Merge branch into master
Rename prod-cloud to prod everywhere
Release again
Sanity checks later:
* Run query
@@ -30,6 +20,46 @@ Sanity checks later:
Future improvements:
Make reports just be based on running-balances
Just use a periodic request or event instead of a job for running balance cache, and perhaps others too
get rid of account-groups
move to solr
upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null
Release steps:
Set prod web workers to 0
Make database snapshot (run export-job)
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition "XXX"
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d"
:starting-at "sales-order"})}]}])))
Create new database for prod-cloud (just called prod)
(dc/create-database conn {:db-name "prod"})
Set this in the prod-cloud config file and prod-cloud-background-worker
Restore database
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition "restore_from_backup_prod_cloud:3"
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d"
:starting-at "sales-order"})}]}])))
Merge branch into master
Rename prod-cloud to prod everywhere
Release again
git push deploy master