From 5df47c88377b7289bdef0cc4b4fbf570a8a4cb8e Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 30 Mar 2023 16:44:25 -0700 Subject: [PATCH] (cloud) running balance update way more streamlined now --- src/clj/auto_ap/graphql/clients.clj | 95 +++++-------------- .../auto_ap/jobs/current_balance_cache.clj | 2 +- src/clj/auto_ap/ledger.clj | 13 +-- .../auto_ap/integration/graphql/clients.clj | 57 ----------- things-to-search-for.txt | 58 ++++++++--- 5 files changed, 78 insertions(+), 147 deletions(-) diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index a8cef96e..0b307060 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -170,77 +170,34 @@ lms))) ->graphql))) -(defn refresh-bank-account-current-balance [bank-account-id] - (let [all-transactions (dc/q - {:query {:find ['?e '?debit '?credit] - :in ['$ '?bank-account-id] - :where '[[?e :journal-entry-line/account ?bank-account-id] - [(get-else $ ?e :journal-entry-line/debit 0.0) ?debit] - [(get-else $ ?e :journal-entry-line/credit 0.0) ?credit]]} - :args [(dc/db conn) bank-account-id]}) - debits (->> all-transactions - (map (fn [[_ debit _]] - debit)) - (reduce + 0.0)) - credits (->> all-transactions - (map (fn [[_ _ credit]] - credit)) - (reduce + 0.0)) - current-balance (if (= :bank-account-type/check (:db/ident (:bank-account/type (dc/pull (dc/db conn) [:bank-account/type] bank-account-id)))) - (- debits credits) - (- credits debits))] - (dc/transact conn {:tx-data [{:db/id bank-account-id - :bank-account/current-balance current-balance}]}))) -(defn bank-accounts-needing-refresh [] - (let [last-refreshed (->> (dc/q {:query {:find ['?ba '?tx] - :in ['$ ] - :where ['[?ba :bank-account/current-balance _ ?tx true]]} - :args [(dc/history (dc/db conn))]}) - (group-by first) - (map (fn [[ba balance-txes]] - [ba (->> balance-txes - (map second) - (sort-by #(- %)) - first)]))) - has-newer-transaction (->> (dc/q - {:query {:find ['?ba] - :in '[$ [[?ba ?last-refreshed] ...] ] - :where ['[_ :journal-entry-line/account ?ba ?tx] - '[(>= ?tx ?last-refreshed)]]} - :args [(dc/history (dc/db conn)) - last-refreshed]}) - (map first) - (set)) - - no-current-balance (->> (dc/q {:query {:find ['?ba] - :in '[$] - :where ['[?ba :bank-account/code] - '(not [?ba :bank-account/current-balance]) - ]} - :args [(dc/db conn)]}) - (map first))] - (into has-newer-transaction no-current-balance))) - -(defn build-current-balance [bank-accounts] - (doseq [bank-account bank-accounts] - (log/info "Refreshing bank account" (-> (dc/db conn) (dc/pull [:bank-account/code] bank-account))) - (try - (refresh-bank-account-current-balance bank-account) - (catch Exception e - (log/error "Can't refresh current balance" e))))) - -#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn refresh-all-current-balance [] - (lc/with-context {:source "current-balance-cache"} - (build-current-balance (->> (dc/q {:query {:find ['?ba] - :in '[$] - :where ['[?ba :bank-account/code]]} - :args [(dc/db conn)]}) - (map first))))) - -(defn refresh-current-balance [] - (build-current-balance (bank-accounts-needing-refresh))) + (lc/with-context {:source "current-balance-refresh"} + (let [db (dc/db conn) + clients (dc/q '[:find (pull ?c [:db/id :client/code {:client/bank-accounts [:db/id :bank-account/code]}]) + :where [?c :client/code]] + db )] + (dc/transact conn + {:tx-data + (for [[{client :db/id code :client/code bank-accounts :client/bank-accounts}] clients + {bank-account :db/id bac :bank-account/code} bank-accounts] + {:db/id bank-account + :bank-account/current-balance + (or + (->> (dc/index-pull db + {:index :avet + :selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/running-balance :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}] + :start [:journal-entry-line/client+account+location+date [client bank-account "A" #inst "2030-01-01"]] + :limit 1 + :reverse true + }) + (filter (fn [{[c b] :journal-entry-line/client+account+location+date}] + (and (= c client) + (= b bank-account)))) + (map :journal-entry-line/running-balance) + (first)) + 0.0)})}))) + ) (defn get-client [context _ _] (->graphql diff --git a/src/clj/auto_ap/jobs/current_balance_cache.clj b/src/clj/auto_ap/jobs/current_balance_cache.clj index a1301aee..8f9d447a 100644 --- a/src/clj/auto_ap/jobs/current_balance_cache.clj +++ b/src/clj/auto_ap/jobs/current_balance_cache.clj @@ -5,7 +5,7 @@ [auto-ap.jobs.core :refer [execute]])) (defn -main [& _] - (execute "current-balance-cache" clients/refresh-current-balance)) + (execute "current-balance-cache" clients/refresh-all-current-balance)) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index d6b96054..cc083feb 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -2,13 +2,12 @@ (:require [auto-ap.datomic :refer [audit-transact - audit-transact-batch - transact-with-backoff conn pull-id pull-ref - remove-nils]] - [auto-ap.utils :refer [by dollars-0? dollars=]] + remove-nils + transact-with-backoff]] + [auto-ap.utils :refer [by dollars-0? dollars= heartbeat]] [clj-time.coerce :as c] [clj-time.core :as t] [clojure.tools.logging :as log] @@ -17,7 +16,9 @@ [datomic.client.api :as dc] [iol-ion.tx :refer [upsert-ledger]] [manifold.deferred :as de] - [manifold.stream :as s])) + [manifold.stream :as s] + [mount.core :as mount] + [yang.scheduler :as scheduler])) (defn datums->impacted-entity [db [e changes]] (let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) @@ -754,6 +755,6 @@ ;; TODO only enable once IOL is set up in clod #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -#_(mount/defstate running-balance-cache-worker +(mount/defstate running-balance-cache-worker :start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache")) :stop (scheduler/stop running-balance-cache-worker)) diff --git a/test/clj/auto_ap/integration/graphql/clients.clj b/test/clj/auto_ap/integration/graphql/clients.clj index 9b3d9db4..9604e9a8 100644 --- a/test/clj/auto_ap/integration/graphql/clients.clj +++ b/test/clj/auto_ap/integration/graphql/clients.clj @@ -9,63 +9,6 @@ (use-fixtures :each wrap-setup) -(deftest current-balance - (testing "it should start with a balance of 0" - (dc/transact conn - {:tx-data - [{:client/code "TEST" - :client/bank-accounts [{:bank-account/code "TEST-1"}]}]}) - (sut/refresh-current-balance) - (is (= 0.00 - (pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"] )))) - - (testing "it should consider a single transaction" - (dc/transact conn - {:tx-data - [{:client/code "TEST" - :db/id "TEST" - :client/bank-accounts [{:bank-account/code "TEST-1" - :db/id "TEST-1"}]} - {:journal-entry-line/account "TEST-1" - :journal-entry-line/debit 13.50} - - {:journal-entry-line/account "TEST-1" - :journal-entry-line/debit 19.50}]}) - (sut/refresh-current-balance) - (is (= -33.0 - (pull-attr (dc/db conn) :bank-account/current-balance [:bank-account/code "TEST-1"])))) - - (testing "bank accounts should start in a needing refresh state" - (let [bank-account-id (-> (dc/transact conn - {:tx-data - [{:client/code "TEST" - :db/id "TEST" - :client/bank-accounts [{:bank-account/code "TEST-2" :db/id "TEST-2"}]} - ]}) - :tempids - (get "TEST-2"))] - - (is ((sut/bank-accounts-needing-refresh) bank-account-id )))) - - (testing "bank accounts should not need a refresh if balance is up-to-date" - (let [bank-account-id (-> (dc/transact conn - {:tx-data - [{:client/code "TEST" - :db/id "TEST" - :client/bank-accounts [{:bank-account/code "TEST-3" :db/id "TEST-3"}]} - ]}) - :tempids - (get "TEST-3"))] - (sut/refresh-bank-account-current-balance bank-account-id) - (is (not ((sut/bank-accounts-needing-refresh) bank-account-id ))) - (dc/transact conn - {:tx-data - [{:journal-entry-line/account [:bank-account/code "TEST-3"] - :journal-entry-line/debit -10.50} - ]}) - (is ((sut/bank-accounts-needing-refresh) bank-account-id )) - (sut/refresh-bank-account-current-balance bank-account-id) - (is (not ((sut/bank-accounts-needing-refresh) bank-account-id )))))) (deftest upsert-client (testing "Should create a new client" diff --git a/things-to-search-for.txt b/things-to-search-for.txt index 0a3cb56e..cb24a436 100644 --- a/things-to-search-for.txt +++ b/things-to-search-for.txt @@ -1,8 +1,9 @@ it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling - -Fix searching -* indexing should happen more regularly, and just look for changes since last time it was run +Double check each job still functions in the new system +Reconcile ledger. Does it work? What are the downsides? Can it be made faster now? +Make reports just be based on running-balances +When you add a vendor, it should be searchable immediately Running Balance Cache * Add tests for upsert-ledger @@ -12,17 +13,6 @@ Running Balance Cache Address memory * JVM settings now and in prod -Release steps: -Stop prod -Make database snapshot -Create new database for prod-cloud (just called prod) -Restore database -Transact new schema -(reset-client+account+location+date) -(force-rebuild-running-balance-cache) -Merge branch into master -Rename prod-cloud to prod everywhere -Release again Sanity checks later: * Run query @@ -30,6 +20,46 @@ Sanity checks later: Future improvements: Make reports just be based on running-balances +Just use a periodic request or event instead of a job for running balance cache, and perhaps others too get rid of account-groups move to solr upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null + + + +Release steps: +Set prod web workers to 0 + +Make database snapshot (run export-job) +(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}] + :count 1 + :cluster "default" + :enable-ecs-managed-tags true + :task-definition "XXX" + :network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"] + :security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"] + :assign-public-ip AssignPublicIp/ENABLED}}} + true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d" + :starting-at "sales-order"})}]}]))) + +Create new database for prod-cloud (just called prod) +(dc/create-database conn {:db-name "prod"}) + +Set this in the prod-cloud config file and prod-cloud-background-worker + +Restore database +(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}] + :count 1 + :cluster "default" + :enable-ecs-managed-tags true + :task-definition "restore_from_backup_prod_cloud:3" + :network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"] + :security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"] + :assign-public-ip AssignPublicIp/ENABLED}}} + true (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str {:backup "63646188-90cd-4cec-a115-feeb7e33d54d" + :starting-at "sales-order"})}]}]))) + +Merge branch into master +Rename prod-cloud to prod everywhere +Release again +git push deploy master