notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

no need for transact-with-ledger.

Browse Source
This commit is contained in:
Bryce Covert
2023-04-03 09:04:33 -07:00
parent 68e809d8fb
commit 6622aae589
12 changed files with 213 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/clj/auto_ap/datomic/invoices.clj
View File

@@ -6,8 +6,7 @@
apply-sort-3
conn
merge-query
pull-many
remove-nils]]
pull-many]]
[auto-ap.datomic.accounts :as d-accounts]
[auto-ap.datomic.vendors :as d-vendors]
[auto-ap.graphql.utils :refer [limited-clients]]
@@ -15,7 +14,8 @@
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.set :refer [rename-keys]]
[datomic.client.api :as dc]))
[datomic.client.api :as dc]
[iol-ion.tx :refer [random-tempid]]))
(def default-read '[*
{:invoice/client [:client/name :db/id :client/locations :client/code]}
@@ -323,7 +323,8 @@
(next-dom schedule-payment-dom)
coerce/to-date)
:else nil)
default-expense-account #:invoice-expense-account {:account (d-vendors/account-for-client-id vendor client-id)
default-expense-account #:invoice-expense-account {:db/id (random-tempid)
:account (d-vendors/account-for-client-id vendor client-id)
:location (:invoice/location invoice)
:amount (:invoice/total invoice)}]
(cond-> invoice

2
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -17,8 +17,6 @@
assert-power-user
attach-tracing-resolvers
enum->keyword]]
[auto-ap.ledger
:refer [transact-batch-with-ledger]]
[auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]

151
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.graphql.transactions
(:require
[auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils]]
[auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils audit-transact audit-transact-batch]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.invoices :as d-invoices]
@@ -19,8 +19,6 @@
ident->enum-f
snake->kebab]]
[auto-ap.import.transactions :as i-transactions]
[auto-ap.ledger
:refer [transact-batch-with-ledger transact-with-ledger]]
[auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
@@ -28,7 +26,7 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.client.api :as dc]
[iol-ion.tx :refer [random-tempid upsert-entity]]
[iol-ion.tx :refer [random-tempid upsert-transaction]]
[com.brunobonacci.mulog :as mu]))
(def approval-status->graphql (ident->enum-f :transaction/approval-status))
@@ -91,11 +89,11 @@
all-ids-not-locked)]
(log/info "Unapproving " (count all-ids) args)
(transact-batch-with-ledger
(audit-transact-batch
(->> all-ids
(mapv (fn [t]
{:db/id t
:transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")})))
`(upsert-transaction ~{:db/id t
:transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")}))))
(:id context))
{:message (str "Succesfully changed " (count all-ids) " transactions to be " (name (:status args) ) ".")}))
@@ -172,12 +170,12 @@
(throw (ex-info err {:validation-error err}) ))))
(log/info "Bulk coding " (count all-ids) args)
(transact-batch-with-ledger
(audit-transact-batch
(map (fn [i]
`(upsert-entity ~(cond-> i
(:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status"))
(:vendor args) (assoc :transaction/vendor (:vendor args))
(seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations)))))
`(upsert-transaction ~(cond-> i
(:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status"))
(:vendor args) (assoc :transaction/vendor (:vendor args))
(seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations)))))
transactions)
(:id context))
{:message (str "Successfully coded " (count all-ids) " transactions.")}))
@@ -190,7 +188,7 @@
db (dc/db conn)]
(log/info "Deleting " (count all-ids) args)
(transact-batch-with-ledger
(audit-transact-batch
(mapcat (fn [i]
(let [transaction (dc/pull db [:transaction/payment
:transaction/expected-deposit
@@ -272,46 +270,36 @@
(when (not= :payment-status/cleared (-> payment :payment/status :db/ident))
(throw (ex-info "Payment can't be undone because it isn't cleared." {:validation-error "Payment can't be undone because it isn't cleared."})))
(if is-autopay-payment?
(transact-with-ledger
(cond-> [{:db/id (:db/id payment)
:payment/status :payment-status/pending}
{:db/id transaction-id
:transaction/approval-status :transaction-approval-status/unapproved}
(audit-transact
(-> [{:db/id (:db/id payment)
:payment/status :payment-status/pending}
`(upsert-transaction
~{:db/id transaction-id
:transaction/approval-status :transaction-approval-status/unapproved
:transaction/payment nil
:transaction/vendor nil
:transaction/location nil
:transaction/accounts nil})
[:db/retractEntity (:db/id payment) ]
[:db/retract transaction-id :transaction/payment (:db/id payment)]
[:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))]]
[:db/retractEntity (:db/id payment) ]]
(:transaction/location transaction)
(conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)])
(seq (:transaction/accounts transaction))
(into (map (fn [a]
[:db/retract transaction-id :transaction/accounts (:db/id a)])
(:transaction/accounts transaction)))
true
(into (map (fn [[invoice-payment]]
[:db/retractEntity invoice-payment])
(dc/q {:query {:find ['?ip]
:in ['$ '?p]
:where ['[?ip :invoice-payment/payment ?p]]}
:args [(dc/db conn) (:db/id payment)]} ))))
(into (map (fn [[invoice-payment]]
[:db/retractEntity invoice-payment])
(dc/q {:query {:find ['?ip]
:in ['$ '?p]
:where ['[?ip :invoice-payment/payment ?p]]}
:args [(dc/db conn) (:db/id payment)]} ))))
(:id context))
(transact-with-ledger
(into (cond-> [{:db/id (:db/id payment)
:payment/status :payment-status/pending}
{:db/id transaction-id
:transaction/approval-status :transaction-approval-status/unapproved}
[:db/retract transaction-id :transaction/payment (:db/id payment)]
[:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))]
]
(:transaction/location transaction)
(conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)]))
(map (fn [a]
[:db/retract transaction-id :transaction/accounts (:db/id a)])
(:transaction/accounts transaction)))
(audit-transact
[{:db/id (:db/id payment)
:payment/status :payment-status/pending}
`(upsert-transaction
~{:db/id transaction-id
:transaction/approval-status :transaction-approval-status/unapproved
:transaction/payment nil
:transaction/vendor nil
:transaction/location nil
:transaction/accounts nil})]
(:id context)))
(-> (d-transactions/get-by-id transaction-id)
approval-status->graphql
@@ -372,14 +360,14 @@
(when missing-locations
(throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) )
(transact-with-ledger [`(upsert-entity ~{:db/id id
:transaction/vendor vendor_id
:transaction/approval-status (some->> approval_status
name
snake->kebab
(keyword "transaction-approval-status"))
:transaction/accounts (map transaction-account->entity accounts)
:transaction/forecast-match forecast_match})]
(audit-transact [`(upsert-transaction ~{:db/id id
:transaction/vendor vendor_id
:transaction/approval-status (some->> approval_status
name
snake->kebab
(keyword "transaction-approval-status"))
:transaction/accounts (map transaction-account->entity accounts)
:transaction/forecast-match forecast_match})]
(:id context))
(-> (d-transactions/get-by-id id)
approval-status->graphql
@@ -398,22 +386,22 @@
(when-not (dollars= (- (:transaction/amount transaction))
(:payment/amount payment))
(throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"})))
(transact-with-ledger (into
(audit-transact (into
[{:db/id (:db/id payment)
:payment/status :payment-status/cleared
:payment/date (coerce/to-date (first (sort [(:payment/date payment)
(:transaction/date transaction)])))}
{:db/id (:db/id transaction)
:transaction/payment (:db/id payment)
:transaction/vendor (:db/id (:payment/vendor payment))
:transaction/location "A"
:transaction/approval-status :transaction-approval-status/approved
:transaction/accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:transaction-account/location "A"
:transaction-account/amount (Math/abs (:transaction/amount transaction))}]}]
(map (fn [x] [:db/retractEntity (:db/id x)] )
(:transaction/accounts transaction)))
`(upsert-transaction
~{:db/id (:db/id transaction)
:transaction/payment (:db/id payment)
:transaction/vendor (:db/id (:payment/vendor payment))
:transaction/location "A"
:transaction/approval-status :transaction-approval-status/approved
:transaction/accounts [{:db/id (random-tempid)
:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:transaction-account/location "A"
:transaction-account/amount (Math/abs (:transaction/amount transaction))}]})])
(:id context)))
(-> (d-transactions/get-by-id transaction_id)
approval-status->graphql
@@ -447,7 +435,7 @@
(:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))]
(log/info "Adding a new payment" payment-tx)
(transact-with-ledger payment-tx (:id context)))
(audit-transact payment-tx (:id context)))
(-> (d-transactions/get-by-id transaction_id)
approval-status->graphql
@@ -484,8 +472,7 @@
unpaid_invoice_ids)
(:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))]
(log/info "Adding a new payment" payment-tx)
(transact-with-ledger payment-tx (:id context)))
(audit-transact payment-tx (:id context)))
(-> (d-transactions/get-by-id transaction_id)
approval-status->graphql
@@ -521,19 +508,15 @@
(when (:transaction/payment transaction)
(throw (ex-info "Transaction already associated with a payment" {:validation-error "Transaction already associated with a payment"}))))
(transact-with-ledger (transduce
(map #(into
[(remove-nils (rm/apply-rule {:db/id (:db/id %)
:transaction/amount (:transaction/amount %)}
transaction-rule
(audit-transact (mapv (fn [t]
`(upsert-transaction
~(remove-nils (rm/apply-rule {:db/id (:db/id t)
:transaction/amount (:transaction/amount t)}
transaction-rule
(or (-> % :transaction/bank-account :bank-account/locations)
(-> % :transaction/client :client/locations))))]
(map (fn [x] [:db/retractEntity (:db/id x)] )
(:transaction/accounts %))))
into
[]
transactions)
(or (-> t :transaction/bank-account :bank-account/locations)
(-> t :transaction/client :client/locations))))))
transactions)
(:id context))
)
(transduce

60
src/clj/auto_ap/import/transactions.clj
View File

@@ -1,12 +1,12 @@
(ns auto-ap.import.transactions
(:require
[auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]]
[auto-ap.datomic.accounts :as a]
[iol-ion.tx :refer [upsert-entity]]
[iol-ion.tx :refer [upsert-transaction upsert-invoice]]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.datomic.transactions :as d-transactions]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.rule-matching :as rm]
[auto-ap.time :as atime]
[auto-ap.utils :refer [dollars=]]
@@ -113,17 +113,17 @@
(defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id]
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
(let [payment-id (random-tempid)]
(-> [`(upsert-entity ~(assoc transaction
:transaction/payment payment-id
:transaction/approval-status :transaction-approval-status/approved
:transaction/vendor vendor
:transaction/location "A"
:transaction/accounts
[#:transaction-account
{:db/id (random-tempid)
:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:location "A"
:amount (Math/abs (:transaction/amount transaction))}]))]
(-> [`(upsert-transaction ~(assoc transaction
:transaction/payment payment-id
:transaction/approval-status :transaction-approval-status/approved
:transaction/vendor vendor
:transaction/location "A"
:transaction/accounts
[#:transaction-account
{:db/id (random-tempid)
:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:location "A"
:amount (Math/abs (:transaction/amount transaction))}]))]
(conj {:payment/bank-account bank-account-id
:payment/client client-id
@@ -137,9 +137,9 @@
[{:invoice-payment/invoice invoice-id
:invoice-payment/payment payment-id
:invoice-payment/amount invoice-amount}
{:db/id invoice-id
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}])
`(upsert-invoice ~{:db/id invoice-id
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid})])
invoice-payments)))))
(defn extract-check-number [{:transaction/keys [description-original]}]
@@ -255,25 +255,25 @@
#_maybe-autopay-invoices
#(maybe-code % apply-rules valid-locations)
identity)]
[(-> transaction
(assoc :transaction/client client-id)
(assoc :transaction/bank-account bank-account-id)
(assoc :transaction/approval-status :transaction-approval-status/unapproved)
maybe-assoc-check-number
code-fn
remove-nils)]))
(-> transaction
(assoc :transaction/client client-id)
(assoc :transaction/bank-account bank-account-id)
(assoc :transaction/approval-status :transaction-approval-status/unapproved)
maybe-assoc-check-number
code-fn
remove-nils)))
(defn get-existing [bank-account]
(log/info "looking up bank account data for" bank-account)
(into {}
(dc/q {:query {:find ['?tid '?as2]
:in ['$ '?ba]
:where ['[?e :transaction/bank-account ?ba]
'[?e :transaction/id ?tid]
'[?e :transaction/approval-status ?as]
'[?as :db/ident ?as2]]}
:args [(dc/db conn) bank-account]})))
:in ['$ '?ba]
:where ['[?e :transaction/bank-account ?ba]
'[?e :transaction/id ?tid]
'[?e :transaction/approval-status ?as]
'[?as :db/ident ?as2]]}
:args [(dc/db conn) bank-account]})))
(defprotocol ImportBatch
(import-transaction! [this transaction])
@@ -319,7 +319,7 @@
:error :import-batch/error
:not-ready :import-batch/not-ready) inc))
(when (= :import action)
(transact-with-ledger (transaction->txs transaction bank-account rule-applying-function)
(audit-transact [`(upsert-transaction ~(transaction->txs transaction bank-account rule-applying-function))]
{:user/name user
:user/role ":admin"}))))

9
src/clj/auto_ap/jobs/sysco.clj
View File

@@ -3,10 +3,10 @@
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.datomic :refer [audit-transact]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice]]
[iol-ion.tx :refer [propose-invoice]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.parse :as parse]
[auto-ap.time :as t]
[clj-time.coerce :as coerce]
@@ -145,11 +145,10 @@
(s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key (doto (str "sysco/error/"
(.getName (io/file k)))
println)}))
:destination-key (str "sysco/error/"
(.getName (io/file k)))}))
[])))))
result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})])
result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
(doseq [k keys]
(mark-key k))))

39
src/clj/auto_ap/ledger.clj
View File

@@ -459,45 +459,6 @@
:priority :low}
nil))
(defn transact-with-ledger [transaction id]
(let [db (dc/db conn)
tx (audit-transact transaction id)
affected-entities (->> (:tx-data tx)
(map (fn [x]
{:e (:e x)
:a (:a x)
:v (:v x)
:added (:added x)}))
(group-by :e)
(mapcat #(datums->impacted-entity db %))
(set))
ledger-txs (->> affected-entities
(map #(entity-change->ledger (:db-after tx) %))
(filter seq)
(map (fn [l]
`(upsert-ledger ~l))))]
(when (seq ledger-txs)
(audit-transact ledger-txs id))
tx))
(defn transact-batch-with-ledger [txes id]
(let [batch-id (.toString (java.util.UUID/randomUUID))]
(reduce
(fn [full-tx batch]
(let [batch (conj (vec batch) {:db/id "datomic.tx"
:audit/batch batch-id})
_ (log/info "transacting batch " batch-id " " (count batch))
tx-result (transact-with-ledger batch id)]
(cond-> full-tx
(:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result)))
(not (:tx-data full-tx)) (assoc :tx-data (vec (:tx-data tx-result)))
(not (:db-before full-tx)) (assoc :db-before (:db-before tx-result))
true (assoc :db-after (:db-after tx-result))
true (update :tempids merge (:tempids tx-result)))))
{}
(partition-all 50 txes))))
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name

28
src/clj/auto_ap/routes/invoices.clj
View File

@@ -20,7 +20,7 @@
[config.core :refer [env]]
[datomic.client.api :as dc]
[digest]
[iol-ion.tx :refer [propose-invoice]]
[iol-ion.tx :refer [propose-invoice random-tempid]]
[ring.middleware.json :refer [wrap-json-response]]
[unilog.context :as lc])
(:import
@@ -150,7 +150,8 @@
matching-client)
text
full-text))]
#:invoice {:invoice/client matching-client
#:invoice {:db/id (random-tempid)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
@@ -175,29 +176,6 @@
invoice)
(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}]
(try
(->> (dc/q
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
[?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/status ?status]
[?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]}
:args [(dc/db conn) invoice-number vendor client]}))
first
boolean)
(catch Exception e
(throw (ex-info (str "Failed to find potential matching invoice with"
" invoice " invoice-number
" vendor " vendor
" client " client
". "
(.toString e))
{:invoice-number invoice-number})))))
(defn invoice-rows->transaction [rows user]
(->> rows

9
src/clj/auto_ap/rule_matching.clj
View File

@@ -1,4 +1,5 @@
(ns auto-ap.rule-matching)
(ns auto-ap.rule-matching
(:require [iol-ion.tx :refer [random-tempid]]))
(defn ->pattern [x]
(. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE)))
@@ -110,11 +111,13 @@
(->> valid-locations
(map
(fn [cents location]
{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
{:db/id (random-tempid)
:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (* 0.01 cents)
:transaction-account/location location})
(spread-cents cents-to-distribute (count valid-locations))))
[(cond-> {:transaction-account/account (:db/id (:transaction-rule-account/account tra))
[(cond-> {:db/id (random-tempid)
:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (* 0.01 cents-to-distribute)}
(:transaction-rule-account/location tra) (assoc :transaction-account/location (:transaction-rule-account/location tra)))])))
(filter (comp seq :transaction-rule-account/account) (:transaction-rule/accounts rule))))

47
src/clj/user.clj
View File

@@ -2,7 +2,7 @@
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn pull-attr random-tempid]]
[auto-ap.ledger :as l :refer [transact-with-ledger]]
[auto-ap.ledger :as l ]
[auto-ap.server]
[auto-ap.square.core :as square]
[auto-ap.square.core2 :as square2]
@@ -68,48 +68,6 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn mark-until-date [client end]
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :invoice/client ?c]
'[?c :client/code ?client]
'[?e :invoice/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:invoice/exclude-from-ledger true}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100"))
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :transaction/client ?c]
'[?c :client/code ?client]
'[?e :transaction/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:transaction/approval-status :transaction-approval-status/excluded}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100")))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn load-accounts [conn]
(let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
@@ -443,6 +401,9 @@
(mu/start-publisher! {:type :dev})
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
(defn start-search []
(mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer})))
(defn restart-db []
#_(require 'datomic.dev-local)
#_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})