notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) experimental approach to ensure the ledgers do not get out of sync

Browse Source
This commit is contained in:
Bryce Covert
2023-04-02 07:45:12 -07:00
parent baa1c2e001
commit 68e809d8fb
10 changed files with 301 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/clj/auto_ap/graphql/checks.clj
View File

@@ -1,8 +1,9 @@
(ns auto-ap.graphql.checks
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn remove-nils plus pull-many]]
[auto-ap.datomic :refer [conn remove-nils plus pull-many audit-transact]]
[auto-ap.datomic.accounts :as a]
[iol-ion.tx :refer [upsert-invoice]]
[auto-ap.datomic.bank-accounts :as d-bank-accounts]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.clients :as d-clients]
@@ -18,7 +19,6 @@
assert-failure
assert-not-locked
enum->keyword]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.numeric :refer [num->words]]
[auto-ap.time :refer [iso-date local-now parse]]
[auto-ap.utils :refer [by dollars-0?]]
@@ -415,7 +415,7 @@
(make-pdfs (filter #(and (= :payment-type/check (:payment/type %))
(> (:payment/amount %) 0.0))
checks))))
(transact-with-ledger (map #(if (map? %)
(audit-transact (map #(if (map? %)
(dissoc % :payment/pdf-data)
%) checks ) id)
@@ -476,7 +476,7 @@
0
invoice-payment-lookup)]
(transact-with-ledger
(audit-transact
(into [(assoc base-payment
:payment/type :payment-type/check
:payment/status :payment-status/pending
@@ -501,60 +501,60 @@
new-balance (+ (:invoice/outstanding-balance invoice)
(:invoice-payment/amount x))]
[[:db.fn/retractEntity (:db/id x)]
{:db/id (:db/id invoice)
:invoice/outstanding-balance new-balance
:invoice/status (if (dollars-0? new-balance)
(:invoice/status invoice)
:invoice-status/unpaid)}]))
`(upsert-invoice ~{:db/id (:db/id invoice)
:invoice/outstanding-balance new-balance
:invoice/status (if (dollars-0? new-balance)
(:invoice/status invoice)
:invoice-status/unpaid)})]))
(:payment/invoices check))
updated-payment {:db/id id
:payment/amount 0.0
:payment/status :payment-status/voided}]
(transact-with-ledger (conj removing-payments updated-payment)
(audit-transact (conj removing-payments updated-payment)
(:id context)))
(-> (d-checks/get-by-id id)
(->graphql))))
(defn void-payments-internal [all-ids id]
(transact-with-ledger (->> all-ids
(dc/q '[:find (pull ?p [:db/id
{:invoice-payment/_payment [:invoice-payment/amount
:db/id
{:invoice-payment/invoice [:db/id :invoice/outstanding-balance]}]}])
:in $ [?p ...]
:where
(not [_ :transaction/payment ?p])
(not [?p :payment/status :payment-status/voided])
[?p :payment/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?p :payment/date ?d]
[(>= ?d ?lu)]
]
(dc/db conn))
(map first)
(mapcat (fn [{:keys [:db/id]
invoices :invoice-payment/_payment}]
(into
[{:db/id id
:payment/amount 0.0
:payment/status :payment-status/voided}]
(->> invoices
(mapcat (fn [{:keys [:invoice-payment/invoice :db/id :invoice-payment/amount]}]
(let [new-balance (+ (:invoice/outstanding-balance invoice)
amount)]
[[:db.fn/retractEntity id]
{:db/id (:db/id invoice)
:invoice/outstanding-balance new-balance
:invoice/status (if (dollars-0? new-balance)
(:invoice/status invoice)
:invoice-status/unpaid)}]))))))))
id))
(audit-transact (->> all-ids
(dc/q '[:find (pull ?p [:db/id
{:invoice-payment/_payment [:invoice-payment/amount
:db/id
{:invoice-payment/invoice [:db/id :invoice/outstanding-balance]}]}])
:in $ [?p ...]
:where
(not [_ :transaction/payment ?p])
(not [?p :payment/status :payment-status/voided])
[?p :payment/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?p :payment/date ?d]
[(>= ?d ?lu)]
]
(dc/db conn))
(map first)
(mapcat (fn [{:keys [:db/id]
invoices :invoice-payment/_payment}]
(into
[{:db/id id
:payment/amount 0.0
:payment/status :payment-status/voided}]
(->> invoices
(mapcat (fn [{:keys [:invoice-payment/invoice :db/id :invoice-payment/amount]}]
(let [new-balance (+ (:invoice/outstanding-balance invoice)
amount)]
[[:db.fn/retractEntity id]
`(upsert-invoice ~{:db/id (:db/id invoice)
:invoice/outstanding-balance new-balance
:invoice/status (if (dollars-0? new-balance)
(:invoice/status invoice)
:invoice-status/unpaid)})]))))))))
id))
(defn void-payments [context args _]
(assert-admin (:id context))
(let [args (assoc args :id (:id context))
ids (some-> args
(let [args (assoc args :id (:id context))
ids (some-> args
:filters
(assoc :id (:id context))
(<-graphql)
@@ -649,10 +649,7 @@
:payment/type :payment-type/balance-credit
:payment/status :payment-status/cleared}]
(transact-with-ledger (-> []
(audit-transact (-> []
(conj payment)
(into (invoice-payments invoices invoice-amounts))) (:id context))
(->graphql {:invoices (d-invoices/get-multi (map :db/id invoices))})))

133
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -1,8 +1,8 @@
(ns auto-ap.graphql.invoices
(:require
[auto-ap.datomic
:refer [conn pull-attr pull-many pull-ref random-tempid ]]
[iol-ion.tx :refer [upsert-entity]]
:refer [conn pull-attr pull-many pull-ref random-tempid audit-transact audit-transact-batch]]
[iol-ion.tx :refer [upsert-entity upsert-invoice]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices]
[auto-ap.datomic.vendors :as d-vendors]
@@ -18,7 +18,7 @@
attach-tracing-resolvers
enum->keyword]]
[auto-ap.ledger
:refer [transact-batch-with-ledger transact-with-ledger]]
:refer [transact-batch-with-ledger]]
[auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
@@ -62,8 +62,10 @@
(assert-power-user (:id context))
(doseq [i invoices]
(assert-can-see-client (:id context) (:db/id (:invoice/client (dc/pull (dc/db conn) [{:invoice/client [:db/id]}] i)))))
(let [transactions (map (fn [i] [:db/retractEntity i]) invoices)]
(transact-with-ledger transactions (:id context))
(let [transactions (mapcat (fn [i] [[:db/retractEntity i]
[:db/retractEntity [:journal-entry/original-invoice i]]])
invoices)]
(audit-transact transactions (:id context))
invoices))
(defn approve-invoices [context {:keys [invoices]} _]
@@ -74,8 +76,8 @@
i)]]
(assert-can-see-client (:id context) (-> invoice :invoice/client :db/id))
(assert-not-locked (-> invoice :invoice/client :db/id) (-> invoice :invoice/date)))
(let [transactions (map (fn [i] {:db/id i :invoice/import-status :import-status/imported}) invoices)]
(transact-with-ledger transactions (:id context))
(let [transactions (map (fn [i] `(upsert-invoice ~{:db/id i :invoice/import-status :import-status/imported})) invoices)]
(audit-transact transactions (:id context))
invoices))
(defn assert-no-conflicting [{:keys [invoice_number client_id vendor_id]}]
@@ -109,19 +111,19 @@
_ (when-not (:db/id account)
(throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))]
`(upsert-entity ~{:db/id "invoice"
:invoice/invoice-number invoice_number
:invoice/client client_id
:invoice/vendor vendor_id
:invoice/import-status :import-status/imported
:invoice/total total
:invoice/outstanding-balance total
:invoice/status :invoice-status/unpaid
:invoice/date (coerce/to-date date)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)
:invoice/due (coerce/to-date due)
:invoice/scheduled-payment (coerce/to-date scheduled_payment)})))
`(upsert-invoice ~{:db/id "invoice"
:invoice/invoice-number invoice_number
:invoice/client client_id
:invoice/vendor vendor_id
:invoice/import-status :import-status/imported
:invoice/total total
:invoice/outstanding-balance total
:invoice/status :invoice-status/unpaid
:invoice/date (coerce/to-date date)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)
:invoice/due (coerce/to-date due)
:invoice/scheduled-payment (coerce/to-date scheduled_payment)})))
(defn assert-valid-expense-accounts [expense_accounts vendor_id]
(doseq [expense-account expense_accounts
@@ -176,7 +178,7 @@
(assert-valid-expense-accounts expense_accounts vendor_id)
(assert-invoice-amounts-add-up in)
(let [transaction-result (transact-with-ledger [(add-invoice-transaction in)] (:id context))]
(let [transaction-result (audit-transact [(add-invoice-transaction in)] (:id context))]
(-> (d-invoices/get-by-id (get-in transaction-result [:tempids "invoice"]))
(->graphql (:id context)))))
@@ -195,7 +197,7 @@
(assert-not-locked client_id (:date in))
(assert-valid-expense-accounts (:expense_accounts in) vendor_id)
(assert-invoice-amounts-add-up in)))
(let [transaction-result (transact-with-ledger [(add-invoice-transaction in)] (:id context))]
(let [transaction-result (audit-transact [(add-invoice-transaction in)] (:id context))]
(mu/trace ::printing-checks
[]
(-> (gq-checks/print-checks-internal [{:invoice-id (get-in transaction-result [:tempids "invoice"])
@@ -230,7 +232,7 @@
expense_accounts)
:invoice/due (coerce/to-date due)
:invoice/scheduled-payment (coerce/to-date scheduled_payment)}]
(transact-with-ledger [`(upsert-entity ~updated-invoice)]
(audit-transact [`(upsert-invoice ~updated-invoice)]
(:id context))
(-> (d-invoices/get-by-id id)
(->graphql (:id context)))))
@@ -239,13 +241,13 @@
(let [invoice (d-invoices/get-by-id id)]
(assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
(assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
(transact-with-ledger [{:db/id id
:invoice/total 0.0
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/voided
:invoice/expense-accounts (map (fn [ea] {:db/id (:db/id ea)
:invoice-expense-account/amount 0.0})
(:invoice/expense-accounts invoice))}]
(audit-transact [`(upsert-invoice ~{:db/id id
:invoice/total 0.0
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/voided
:invoice/expense-accounts (map (fn [ea] {:db/id (:db/id ea)
:invoice-expense-account/amount 0.0})
(:invoice/expense-accounts invoice))})]
(:id context))
(-> (d-invoices/get-by-id id) (->graphql (:id context)))))
@@ -296,7 +298,7 @@
(gq-checks/void-payments-internal voidable-cash-payments (:id context))
(log/info "Voiding " (count all-ids) args)
(transact-with-ledger
(audit-transact
(->> all-ids
(dc/q '[:find (pull ?i [:db/id :invoice/date {:invoice/expense-accounts [:db/id]}])
:in $ [?i ...]
@@ -306,20 +308,17 @@
[?i :invoice/date ?d]
[(>= ?d ?lu)]]
(dc/db conn))
(map first)
(mapcat
(fn [i]
(into
[{:db/id (:db/id i)
:invoice/total 0.0
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/voided
}]
(map
(fn [iea]
{:db/id (:db/id iea)
:invoice-expense-account/amount 0.0})
(:invoice/expense-accounts i))))))
(map
(fn [[i]]
`(upsert-invoice ~{:db/id (:db/id i)
:invoice/total 0.0
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/voided
:invoice/expense-accounts (map
(fn [iea]
{:db/id (:db/id iea)
:invoice-expense-account/amount 0.0})
(:invoice/expense-accounts i))}))))
(:id context))
{:message (str "Succesfully voided " (count all-ids))}))
@@ -337,17 +336,17 @@
:in ['$ '?e]}
:args [history id]})
[last-transaction] (->> txs (sort-by first) (last))]
(mu/log ::here
:txes txs)
(transact-with-ledger [(->> txs
(filter (fn [[tx]] (= tx last-transaction)))
(reduce (fn [new-transaction [_ entity original-status original-outstanding total expense-account expense-account-amount]]
(-> new-transaction
(assoc :db/id entity
:invoice/total total
:invoice/status original-status
:invoice/outstanding-balance original-outstanding)
(update :invoice/expense-accounts conj {:db/id expense-account :invoice-expense-account/amount expense-account-amount}))) {}))]
(audit-transact [`(upsert-invoice
~(->> txs
(filter (fn [[tx]] (= tx last-transaction)))
(reduce (fn [new-transaction [_ entity original-status original-outstanding total expense-account expense-account-amount]]
(-> new-transaction
(assoc :db/id entity
:invoice/total total
:invoice/status original-status
:invoice/outstanding-balance original-outstanding)
(update :invoice/expense-accounts conj {:db/id expense-account :invoice-expense-account/amount expense-account-amount})))
{})))]
(:id context))
(-> (d-invoices/get-by-id id)
@@ -363,9 +362,11 @@
(assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
(assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
(assert (not (seq (:invoice-payment/_invoice invoice))))
(transact-with-ledger [[:db/add id :invoice/status :invoice-status/unpaid]
[:db/add id :invoice/outstanding-balance (:invoice/total invoice)]
[:db/retract id :invoice/scheduled-payment (:invoice/scheduled-payment invoice)]]
(audit-transact [`(upsert-invoice
~{:db/id id
:invoice/status :invoice-status/unpaid
:invoice/outstanding-balance (:invoice/total invoice)
:invoice/scheduled-payment nil})]
(:id context))
(-> (d-invoices/get-by-id id)
@@ -378,11 +379,11 @@
_ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
_ (assert-valid-expense-accounts (:expense_accounts args) (:db/id (:invoice/vendor invoice )))]
(transact-with-ledger [`(upsert-entity ~{:db/id invoice-id
:invoice/expense-accounts (map
expense-account->entity
(:expense_accounts args))})]
(:id context))
(audit-transact [`(upsert-invoice ~{:db/id invoice-id
:invoice/expense-accounts (map
expense-account->entity
(:expense_accounts args))})]
(:id context))
(->graphql
(d-invoices/get-by-id (:invoice_id args))
(:id context))))
@@ -468,9 +469,9 @@
(let [err (str "Account " name " uses location " (:location a) ", but doesn't belong to the client.")]
(throw (ex-info err {:validation-error err}) ))))
(log/info "Bulk coding " (count all-ids) args)
(transact-batch-with-ledger
(audit-transact-batch
(map (fn [i]
`(upsert-entity ~{:db/id (:db/id i)
`(upsert-invoice ~{:db/id (:db/id i)
:invoice/expense-accounts (maybe-code-accounts i (:accounts args) locations)}))
invoices)
(:id context))

10
src/clj/auto_ap/jobs/register_invoice_import.clj
View File

@@ -2,9 +2,8 @@
(:gen-class)
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn pull-attr]]
[auto-ap.datomic :refer [audit-transact conn pull-attr]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.time :as atime]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
@@ -14,7 +13,8 @@
[clojure.tools.logging :as log]
[config.core :refer [env]]
[datomic.client.api :as dc])
(:import [java.util UUID]))
(:import
(java.util UUID)))
(def bucket (:data-bucket env))
@@ -141,8 +141,8 @@
(log/info "contains " (count data) " rows")
(doseq [n (partition-all 50 (register-invoice-import* data))]
(log/info "transacting" n)
(transact-with-ledger n {:user/name "register-invoice-import"
:user/role "admin"}))))
(audit-transact n {:user/name "register-invoice-import"
:user/role "admin"}))))
(defn -main [& _]
(execute "register-invoice-import" #(register-invoice-import (:args env))))

11
src/clj/auto_ap/routes/invoices.clj
View File

@@ -1,8 +1,7 @@
(ns auto-ap.routes.invoices
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn remove-nils uri]]
[iol-ion.tx :refer [propose-invoice]]
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices]
@@ -10,7 +9,6 @@
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
[auto-ap.import.manual :as manual]
[auto-ap.import.manual.common :as c]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.parse :as parse]
[auto-ap.routes.utils :refer [wrap-secure]]
[auto-ap.utils :refer [by]]
@@ -22,6 +20,7 @@
[config.core :refer [env]]
[datomic.client.api :as dc]
[digest]
[iol-ion.tx :refer [propose-invoice]]
[ring.middleware.json :refer [wrap-json-response]]
[unilog.context :as lc])
(:import
@@ -278,7 +277,7 @@
(mapv (fn [i] `(propose-invoice ~i))))]
(log/info "creating invoice" potential-invoices)
(let [tx (transact-with-ledger potential-invoices user)]
(let [tx (audit-transact potential-invoices user)]
(when-not (seq (dc/q '[:find ?i
:in $ [?i ...]
:where [?i :invoice/invoice-number]]
@@ -395,7 +394,7 @@
conj
[]
rows)]
(transact-with-ledger txes nil)))
(audit-transact txes nil)))
(defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}]
(assert-admin user)
@@ -479,7 +478,7 @@
(not= "Cash" (:check %))))
(map :vendor-name)
set)
_ (transact-with-ledger (invoice-rows->transaction (:new grouped-rows)
_ (audit-transact (invoice-rows->transaction (:new grouped-rows)
user)
user)]
{:status 200