notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

no need for transact-with-ledger.

Browse Source
This commit is contained in:
Bryce Covert
2023-04-03 09:04:33 -07:00
parent 68e809d8fb
commit 6622aae589
12 changed files with 213 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
iol_ion/src/iol_ion/tx.clj
View File

@@ -259,7 +259,7 @@
:journal-entry/vendor (:db/id (:invoice/vendor entity)) :journal-entry/vendor (:db/id (:invoice/vendor entity))
:journal-entry/amount (Math/abs (:invoice/total entity)) :journal-entry/amount (Math/abs (:invoice/total entity))
:journal-entry/line-items (into [(cond-> {:db/id (str (:db/id entity) "-" 0) :journal-entry/line-items (into [(cond-> {:db/id (str raw-invoice-id "-" 0)
:journal-entry-line/account :account/accounts-payable :journal-entry-line/account :account/accounts-payable
:journal-entry-line/location "A" :journal-entry-line/location "A"
} }
@@ -267,7 +267,7 @@
(not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))] (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
(map-indexed (fn [i ea] (map-indexed (fn [i ea]
(cond-> (cond->
{:db/id (str (:db/id entity) "-" (inc i)) {:db/id (str raw-invoice-id "-" (inc i))
:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)) :journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
:journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ") :journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
} }
@@ -291,6 +291,7 @@
journal-entry (invoice->journal-entry (:db-after with-invoice) journal-entry (invoice->journal-entry (:db-after with-invoice)
invoice-id invoice-id
(:db/id invoice))] (:db/id invoice))]
(into upserted-entity (into upserted-entity
(if journal-entry (if journal-entry
(upsert-ledger db journal-entry) (upsert-ledger db journal-entry)
@@ -311,3 +312,82 @@
(if existing? (if existing?
[] []
(upsert-invoice db invoice)))) (upsert-invoice db invoice))))
(defn transaction->journal-entry
([db transaction-id]
(transaction->journal-entry db transaction-id transaction-id))
;; the 3-arity version allows you to pass a potential tempid in instead of the invoice-id,
;; which would be a temporary value after the transaction
([db transaction-id raw-transaction-id]
(let [entity (dc/pull db [:transaction/client
:transaction/date
:transaction/description-original
:db/id
:transaction/vendor
:transaction/amount
:transaction/cleared-against
{:transaction/accounts [:transaction-account/account
:transaction-account/location
:transaction-account/amount]
:transaction/approval-status [:db/ident]
:transaction/bank-account [:db/id {:bank-account/type [:db/ident]}]}]
transaction-id)
decreasing? (< (or (:transaction/amount entity) 0.0) 0.0)
credit-from-bank? decreasing?
debit-from-bank? (not decreasing?)]
(when (and (not (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity))))
(not (= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity))))
(:transaction/amount entity)
(not (< -0.001 (:transaction/amount entity) 0.001)))
(remove-nils
{:journal-entry/source "transaction"
:journal-entry/client (:db/id (:transaction/client entity))
:journal-entry/date (:transaction/date entity)
:journal-entry/original-entity raw-transaction-id
:journal-entry/alternate-description (:transaction/description-original entity)
:journal-entry/vendor (:db/id (:transaction/vendor entity))
:journal-entry/amount (Math/abs (:transaction/amount entity))
:journal-entry/cleared-against (:transaction/cleared-against entity)
:journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity))
:db/id (str raw-transaction-id "-" 0)
:journal-entry-line/location "A"
:journal-entry-line/credit (when credit-from-bank?
(Math/abs (:transaction/amount entity)))
:journal-entry-line/debit (when debit-from-bank?
(Math/abs (:transaction/amount entity)))})
]
(map-indexed
(fn [i a]
(remove-nils{
:db/id (str raw-transaction-id "-" (inc i))
:journal-entry-line/account (:db/id (:transaction-account/account a))
:journal-entry-line/location (:transaction-account/location a)
:journal-entry-line/debit (when credit-from-bank?
(Math/abs (:transaction-account/amount a)))
:journal-entry-line/credit (when debit-from-bank?
(Math/abs (:transaction-account/amount a)))}))
(if (seq (:transaction/accounts entity))
(:transaction/accounts entity)
[{:transaction-account/amount (:transaction/amount entity)}])))
:journal-entry/cleared true})))))
(defn upsert-transaction [db transaction]
;; because some transactions will reference temp ids, you have to dissoc them, like :transaction/payment
(let [upserted-entity (upsert-entity db (dissoc transaction :transaction/payment))
with-transaction (try (dc/with db {:tx-data upserted-entity})
(catch ClassCastException e
(println "Dev local does not support with in tx functions. :(")
(dc/with (dc/with-db @(resolve 'auto-ap.datomic/conn)) {:tx-data upserted-entity})
))
transaction-id (or (-> with-transaction :tempids (get (:db/id transaction)))
(:db/id transaction))
journal-entry (transaction->journal-entry (:db-after with-transaction)
transaction-id
(:db/id transaction))]
(into (upsert-entity db transaction)
(if journal-entry
(upsert-ledger db journal-entry)
[[:db/retractEntity [:journal-entry/original-entity (:db/id transaction)]]]))))

9
src/clj/auto_ap/datomic/invoices.clj
View File

@@ -6,8 +6,7 @@
apply-sort-3 apply-sort-3
conn conn
merge-query merge-query
pull-many pull-many]]
remove-nils]]
[auto-ap.datomic.accounts :as d-accounts] [auto-ap.datomic.accounts :as d-accounts]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
[auto-ap.graphql.utils :refer [limited-clients]] [auto-ap.graphql.utils :refer [limited-clients]]
@@ -15,7 +14,8 @@
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.set :refer [rename-keys]] [clojure.set :refer [rename-keys]]
[datomic.client.api :as dc])) [datomic.client.api :as dc]
[iol-ion.tx :refer [random-tempid]]))
(def default-read '[* (def default-read '[*
{:invoice/client [:client/name :db/id :client/locations :client/code]} {:invoice/client [:client/name :db/id :client/locations :client/code]}
@@ -323,7 +323,8 @@
(next-dom schedule-payment-dom) (next-dom schedule-payment-dom)
coerce/to-date) coerce/to-date)
:else nil) :else nil)
default-expense-account #:invoice-expense-account {:account (d-vendors/account-for-client-id vendor client-id) default-expense-account #:invoice-expense-account {:db/id (random-tempid)
:account (d-vendors/account-for-client-id vendor client-id)
:location (:invoice/location invoice) :location (:invoice/location invoice)
:amount (:invoice/total invoice)}] :amount (:invoice/total invoice)}]
(cond-> invoice (cond-> invoice

2
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -17,8 +17,6 @@
assert-power-user assert-power-user
attach-tracing-resolvers attach-tracing-resolvers
enum->keyword]] enum->keyword]]
[auto-ap.ledger
:refer [transact-batch-with-ledger]]
[auto-ap.rule-matching :as rm] [auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]] [auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]

151
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.graphql.transactions (ns auto-ap.graphql.transactions
(:require (:require
[auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils]] [auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils audit-transact audit-transact-batch]]
[auto-ap.datomic.accounts :as a] [auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.invoices :as d-invoices]
@@ -19,8 +19,6 @@
ident->enum-f ident->enum-f
snake->kebab]] snake->kebab]]
[auto-ap.import.transactions :as i-transactions] [auto-ap.import.transactions :as i-transactions]
[auto-ap.ledger
:refer [transact-batch-with-ledger transact-with-ledger]]
[auto-ap.rule-matching :as rm] [auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]] [auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
@@ -28,7 +26,7 @@
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[iol-ion.tx :refer [random-tempid upsert-entity]] [iol-ion.tx :refer [random-tempid upsert-transaction]]
[com.brunobonacci.mulog :as mu])) [com.brunobonacci.mulog :as mu]))
(def approval-status->graphql (ident->enum-f :transaction/approval-status)) (def approval-status->graphql (ident->enum-f :transaction/approval-status))
@@ -91,11 +89,11 @@
all-ids-not-locked)] all-ids-not-locked)]
(log/info "Unapproving " (count all-ids) args) (log/info "Unapproving " (count all-ids) args)
(transact-batch-with-ledger (audit-transact-batch
(->> all-ids (->> all-ids
(mapv (fn [t] (mapv (fn [t]
{:db/id t `(upsert-transaction ~{:db/id t
:transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")}))) :transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")}))))
(:id context)) (:id context))
{:message (str "Succesfully changed " (count all-ids) " transactions to be " (name (:status args) ) ".")})) {:message (str "Succesfully changed " (count all-ids) " transactions to be " (name (:status args) ) ".")}))
@@ -172,12 +170,12 @@
(throw (ex-info err {:validation-error err}) )))) (throw (ex-info err {:validation-error err}) ))))
(log/info "Bulk coding " (count all-ids) args) (log/info "Bulk coding " (count all-ids) args)
(transact-batch-with-ledger (audit-transact-batch
(map (fn [i] (map (fn [i]
`(upsert-entity ~(cond-> i `(upsert-transaction ~(cond-> i
(:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status")) (:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status"))
(:vendor args) (assoc :transaction/vendor (:vendor args)) (:vendor args) (assoc :transaction/vendor (:vendor args))
(seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations))))) (seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations)))))
transactions) transactions)
(:id context)) (:id context))
{:message (str "Successfully coded " (count all-ids) " transactions.")})) {:message (str "Successfully coded " (count all-ids) " transactions.")}))
@@ -190,7 +188,7 @@
db (dc/db conn)] db (dc/db conn)]
(log/info "Deleting " (count all-ids) args) (log/info "Deleting " (count all-ids) args)
(transact-batch-with-ledger (audit-transact-batch
(mapcat (fn [i] (mapcat (fn [i]
(let [transaction (dc/pull db [:transaction/payment (let [transaction (dc/pull db [:transaction/payment
:transaction/expected-deposit :transaction/expected-deposit
@@ -272,46 +270,36 @@
(when (not= :payment-status/cleared (-> payment :payment/status :db/ident)) (when (not= :payment-status/cleared (-> payment :payment/status :db/ident))
(throw (ex-info "Payment can't be undone because it isn't cleared." {:validation-error "Payment can't be undone because it isn't cleared."}))) (throw (ex-info "Payment can't be undone because it isn't cleared." {:validation-error "Payment can't be undone because it isn't cleared."})))
(if is-autopay-payment? (if is-autopay-payment?
(transact-with-ledger (audit-transact
(cond-> [{:db/id (:db/id payment) (-> [{:db/id (:db/id payment)
:payment/status :payment-status/pending} :payment/status :payment-status/pending}
{:db/id transaction-id `(upsert-transaction
:transaction/approval-status :transaction-approval-status/unapproved} ~{:db/id transaction-id
:transaction/approval-status :transaction-approval-status/unapproved
:transaction/payment nil
:transaction/vendor nil
:transaction/location nil
:transaction/accounts nil})
[:db/retractEntity (:db/id payment) ] [:db/retractEntity (:db/id payment) ]]
[:db/retract transaction-id :transaction/payment (:db/id payment)]
[:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))]]
(:transaction/location transaction) (into (map (fn [[invoice-payment]]
(conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)]) [:db/retractEntity invoice-payment])
(dc/q {:query {:find ['?ip]
(seq (:transaction/accounts transaction)) :in ['$ '?p]
(into (map (fn [a] :where ['[?ip :invoice-payment/payment ?p]]}
[:db/retract transaction-id :transaction/accounts (:db/id a)]) :args [(dc/db conn) (:db/id payment)]} ))))
(:transaction/accounts transaction)))
true
(into (map (fn [[invoice-payment]]
[:db/retractEntity invoice-payment])
(dc/q {:query {:find ['?ip]
:in ['$ '?p]
:where ['[?ip :invoice-payment/payment ?p]]}
:args [(dc/db conn) (:db/id payment)]} ))))
(:id context)) (:id context))
(transact-with-ledger (audit-transact
(into (cond-> [{:db/id (:db/id payment) [{:db/id (:db/id payment)
:payment/status :payment-status/pending} :payment/status :payment-status/pending}
{:db/id transaction-id `(upsert-transaction
:transaction/approval-status :transaction-approval-status/unapproved} ~{:db/id transaction-id
[:db/retract transaction-id :transaction/payment (:db/id payment)] :transaction/approval-status :transaction-approval-status/unapproved
[:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))] :transaction/payment nil
] :transaction/vendor nil
(:transaction/location transaction) :transaction/location nil
(conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)])) :transaction/accounts nil})]
(map (fn [a]
[:db/retract transaction-id :transaction/accounts (:db/id a)])
(:transaction/accounts transaction)))
(:id context))) (:id context)))
(-> (d-transactions/get-by-id transaction-id) (-> (d-transactions/get-by-id transaction-id)
approval-status->graphql approval-status->graphql
@@ -372,14 +360,14 @@
(when missing-locations (when missing-locations
(throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) ) (throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) )
(transact-with-ledger [`(upsert-entity ~{:db/id id (audit-transact [`(upsert-transaction ~{:db/id id
:transaction/vendor vendor_id :transaction/vendor vendor_id
:transaction/approval-status (some->> approval_status :transaction/approval-status (some->> approval_status
name name
snake->kebab snake->kebab
(keyword "transaction-approval-status")) (keyword "transaction-approval-status"))
:transaction/accounts (map transaction-account->entity accounts) :transaction/accounts (map transaction-account->entity accounts)
:transaction/forecast-match forecast_match})] :transaction/forecast-match forecast_match})]
(:id context)) (:id context))
(-> (d-transactions/get-by-id id) (-> (d-transactions/get-by-id id)
approval-status->graphql approval-status->graphql
@@ -398,22 +386,22 @@
(when-not (dollars= (- (:transaction/amount transaction)) (when-not (dollars= (- (:transaction/amount transaction))
(:payment/amount payment)) (:payment/amount payment))
(throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"}))) (throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"})))
(transact-with-ledger (into (audit-transact (into
[{:db/id (:db/id payment) [{:db/id (:db/id payment)
:payment/status :payment-status/cleared :payment/status :payment-status/cleared
:payment/date (coerce/to-date (first (sort [(:payment/date payment) :payment/date (coerce/to-date (first (sort [(:payment/date payment)
(:transaction/date transaction)])))} (:transaction/date transaction)])))}
{:db/id (:db/id transaction) `(upsert-transaction
:transaction/payment (:db/id payment) ~{:db/id (:db/id transaction)
:transaction/vendor (:db/id (:payment/vendor payment)) :transaction/payment (:db/id payment)
:transaction/location "A" :transaction/vendor (:db/id (:payment/vendor payment))
:transaction/approval-status :transaction-approval-status/approved :transaction/location "A"
:transaction/accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :transaction/approval-status :transaction-approval-status/approved
:transaction-account/location "A" :transaction/accounts [{:db/id (random-tempid)
:transaction-account/amount (Math/abs (:transaction/amount transaction))}]}] :transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
(map (fn [x] [:db/retractEntity (:db/id x)] ) :transaction-account/location "A"
(:transaction/accounts transaction))) :transaction-account/amount (Math/abs (:transaction/amount transaction))}]})])
(:id context))) (:id context)))
(-> (d-transactions/get-by-id transaction_id) (-> (d-transactions/get-by-id transaction_id)
approval-status->graphql approval-status->graphql
@@ -447,7 +435,7 @@
(:db/id (:transaction/bank-account transaction)) (:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))] (:db/id (:transaction/client transaction)))]
(log/info "Adding a new payment" payment-tx) (log/info "Adding a new payment" payment-tx)
(transact-with-ledger payment-tx (:id context))) (audit-transact payment-tx (:id context)))
(-> (d-transactions/get-by-id transaction_id) (-> (d-transactions/get-by-id transaction_id)
approval-status->graphql approval-status->graphql
@@ -484,8 +472,7 @@
unpaid_invoice_ids) unpaid_invoice_ids)
(:db/id (:transaction/bank-account transaction)) (:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))] (:db/id (:transaction/client transaction)))]
(log/info "Adding a new payment" payment-tx) (audit-transact payment-tx (:id context)))
(transact-with-ledger payment-tx (:id context)))
(-> (d-transactions/get-by-id transaction_id) (-> (d-transactions/get-by-id transaction_id)
approval-status->graphql approval-status->graphql
@@ -521,19 +508,15 @@
(when (:transaction/payment transaction) (when (:transaction/payment transaction)
(throw (ex-info "Transaction already associated with a payment" {:validation-error "Transaction already associated with a payment"})))) (throw (ex-info "Transaction already associated with a payment" {:validation-error "Transaction already associated with a payment"}))))
(transact-with-ledger (transduce (audit-transact (mapv (fn [t]
(map #(into `(upsert-transaction
[(remove-nils (rm/apply-rule {:db/id (:db/id %) ~(remove-nils (rm/apply-rule {:db/id (:db/id t)
:transaction/amount (:transaction/amount %)} :transaction/amount (:transaction/amount t)}
transaction-rule transaction-rule
(or (-> % :transaction/bank-account :bank-account/locations) (or (-> t :transaction/bank-account :bank-account/locations)
(-> % :transaction/client :client/locations))))] (-> t :transaction/client :client/locations))))))
(map (fn [x] [:db/retractEntity (:db/id x)] ) transactions)
(:transaction/accounts %))))
into
[]
transactions)
(:id context)) (:id context))
) )
(transduce (transduce

60
src/clj/auto_ap/import/transactions.clj
View File

@@ -1,12 +1,12 @@
(ns auto-ap.import.transactions (ns auto-ap.import.transactions
(:require (:require
[auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]]
[auto-ap.datomic.accounts :as a] [auto-ap.datomic.accounts :as a]
[iol-ion.tx :refer [upsert-entity]] [iol-ion.tx :refer [upsert-transaction upsert-invoice]]
[auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transaction-rules :as tr]
[auto-ap.datomic.transactions :as d-transactions] [auto-ap.datomic.transactions :as d-transactions]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.rule-matching :as rm] [auto-ap.rule-matching :as rm]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[auto-ap.utils :refer [dollars=]] [auto-ap.utils :refer [dollars=]]
@@ -113,17 +113,17 @@
(defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id] (defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id]
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
(let [payment-id (random-tempid)] (let [payment-id (random-tempid)]
(-> [`(upsert-entity ~(assoc transaction (-> [`(upsert-transaction ~(assoc transaction
:transaction/payment payment-id :transaction/payment payment-id
:transaction/approval-status :transaction-approval-status/approved :transaction/approval-status :transaction-approval-status/approved
:transaction/vendor vendor :transaction/vendor vendor
:transaction/location "A" :transaction/location "A"
:transaction/accounts :transaction/accounts
[#:transaction-account [#:transaction-account
{:db/id (random-tempid) {:db/id (random-tempid)
:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:location "A" :location "A"
:amount (Math/abs (:transaction/amount transaction))}]))] :amount (Math/abs (:transaction/amount transaction))}]))]
(conj {:payment/bank-account bank-account-id (conj {:payment/bank-account bank-account-id
:payment/client client-id :payment/client client-id
@@ -137,9 +137,9 @@
[{:invoice-payment/invoice invoice-id [{:invoice-payment/invoice invoice-id
:invoice-payment/payment payment-id :invoice-payment/payment payment-id
:invoice-payment/amount invoice-amount} :invoice-payment/amount invoice-amount}
{:db/id invoice-id `(upsert-invoice ~{:db/id invoice-id
:invoice/outstanding-balance 0.0 :invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}]) :invoice/status :invoice-status/paid})])
invoice-payments))))) invoice-payments)))))
(defn extract-check-number [{:transaction/keys [description-original]}] (defn extract-check-number [{:transaction/keys [description-original]}]
@@ -255,25 +255,25 @@
#_maybe-autopay-invoices #_maybe-autopay-invoices
#(maybe-code % apply-rules valid-locations) #(maybe-code % apply-rules valid-locations)
identity)] identity)]
[(-> transaction (-> transaction
(assoc :transaction/client client-id) (assoc :transaction/client client-id)
(assoc :transaction/bank-account bank-account-id) (assoc :transaction/bank-account bank-account-id)
(assoc :transaction/approval-status :transaction-approval-status/unapproved) (assoc :transaction/approval-status :transaction-approval-status/unapproved)
maybe-assoc-check-number maybe-assoc-check-number
code-fn code-fn
remove-nils)])) remove-nils)))
(defn get-existing [bank-account] (defn get-existing [bank-account]
(log/info "looking up bank account data for" bank-account) (log/info "looking up bank account data for" bank-account)
(into {} (into {}
(dc/q {:query {:find ['?tid '?as2] (dc/q {:query {:find ['?tid '?as2]
:in ['$ '?ba] :in ['$ '?ba]
:where ['[?e :transaction/bank-account ?ba] :where ['[?e :transaction/bank-account ?ba]
'[?e :transaction/id ?tid] '[?e :transaction/id ?tid]
'[?e :transaction/approval-status ?as] '[?e :transaction/approval-status ?as]
'[?as :db/ident ?as2]]} '[?as :db/ident ?as2]]}
:args [(dc/db conn) bank-account]}))) :args [(dc/db conn) bank-account]})))
(defprotocol ImportBatch (defprotocol ImportBatch
(import-transaction! [this transaction]) (import-transaction! [this transaction])
@@ -319,7 +319,7 @@
:error :import-batch/error :error :import-batch/error
:not-ready :import-batch/not-ready) inc)) :not-ready :import-batch/not-ready) inc))
(when (= :import action) (when (= :import action)
(transact-with-ledger (transaction->txs transaction bank-account rule-applying-function) (audit-transact [`(upsert-transaction ~(transaction->txs transaction bank-account rule-applying-function))]
{:user/name user {:user/name user
:user/role ":admin"})))) :user/role ":admin"}))))

9
src/clj/auto_ap/jobs/sysco.clj
View File

@@ -3,10 +3,10 @@
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]] [auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]] [auto-ap.jobs.core :refer [execute]]
[auto-ap.datomic :refer [audit-transact]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice]] [auto-ap.datomic.invoices :refer [code-invoice]]
[iol-ion.tx :refer [propose-invoice]] [iol-ion.tx :refer [propose-invoice]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.parse :as parse] [auto-ap.parse :as parse]
[auto-ap.time :as t] [auto-ap.time :as t]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
@@ -145,11 +145,10 @@
(s3/copy-object {:source-bucket-name (:data-bucket env) (s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env) :destination-bucket-name (:data-bucket env)
:source-key k :source-key k
:destination-key (doto (str "sysco/error/" :destination-key (str "sysco/error/"
(.getName (io/file k))) (.getName (io/file k)))}))
println)}))
[]))))) [])))))
result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]) result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
(doseq [k keys] (doseq [k keys]
(mark-key k)))) (mark-key k))))

39
src/clj/auto_ap/ledger.clj
View File

@@ -459,45 +459,6 @@
:priority :low} :priority :low}
nil)) nil))
(defn transact-with-ledger [transaction id]
(let [db (dc/db conn)
tx (audit-transact transaction id)
affected-entities (->> (:tx-data tx)
(map (fn [x]
{:e (:e x)
:a (:a x)
:v (:v x)
:added (:added x)}))
(group-by :e)
(mapcat #(datums->impacted-entity db %))
(set))
ledger-txs (->> affected-entities
(map #(entity-change->ledger (:db-after tx) %))
(filter seq)
(map (fn [l]
`(upsert-ledger ~l))))]
(when (seq ledger-txs)
(audit-transact ledger-txs id))
tx))
(defn transact-batch-with-ledger [txes id]
(let [batch-id (.toString (java.util.UUID/randomUUID))]
(reduce
(fn [full-tx batch]
(let [batch (conj (vec batch) {:db/id "datomic.tx"
:audit/batch batch-id})
_ (log/info "transacting batch " batch-id " " (count batch))
tx-result (transact-with-ledger batch id)]
(cond-> full-tx
(:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result)))
(not (:tx-data full-tx)) (assoc :tx-data (vec (:tx-data tx-result)))
(not (:db-before full-tx)) (assoc :db-before (:db-before tx-result))
true (assoc :db-after (:db-after tx-result))
true (update :tempids merge (:tempids tx-result)))))
{}
(partition-all 50 txes))))
(defn build-account-lookup [client-id] (defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name (let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name

28
src/clj/auto_ap/routes/invoices.clj
View File

@@ -20,7 +20,7 @@
[config.core :refer [env]] [config.core :refer [env]]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[digest] [digest]
[iol-ion.tx :refer [propose-invoice]] [iol-ion.tx :refer [propose-invoice random-tempid]]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[unilog.context :as lc]) [unilog.context :as lc])
(:import (:import
@@ -150,7 +150,8 @@
matching-client) matching-client)
text text
full-text))] full-text))]
#:invoice {:invoice/client matching-client #:invoice {:db/id (random-tempid)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier) :invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor) :invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url :invoice/source-url source-url
@@ -175,29 +176,6 @@
invoice) invoice)
(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}]
(try
(->> (dc/q
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
[?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/status ?status]
[?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]}
:args [(dc/db conn) invoice-number vendor client]}))
first
boolean)
(catch Exception e
(throw (ex-info (str "Failed to find potential matching invoice with"
" invoice " invoice-number
" vendor " vendor
" client " client
". "
(.toString e))
{:invoice-number invoice-number})))))
(defn invoice-rows->transaction [rows user] (defn invoice-rows->transaction [rows user]
(->> rows (->> rows

9
src/clj/auto_ap/rule_matching.clj
View File

@@ -1,4 +1,5 @@
(ns auto-ap.rule-matching) (ns auto-ap.rule-matching
(:require [iol-ion.tx :refer [random-tempid]]))
(defn ->pattern [x] (defn ->pattern [x]
(. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE))) (. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE)))
@@ -110,11 +111,13 @@
(->> valid-locations (->> valid-locations
(map (map
(fn [cents location] (fn [cents location]
{:transaction-account/account (:db/id (:transaction-rule-account/account tra)) {:db/id (random-tempid)
:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (* 0.01 cents) :transaction-account/amount (* 0.01 cents)
:transaction-account/location location}) :transaction-account/location location})
(spread-cents cents-to-distribute (count valid-locations)))) (spread-cents cents-to-distribute (count valid-locations))))
[(cond-> {:transaction-account/account (:db/id (:transaction-rule-account/account tra)) [(cond-> {:db/id (random-tempid)
:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (* 0.01 cents-to-distribute)} :transaction-account/amount (* 0.01 cents-to-distribute)}
(:transaction-rule-account/location tra) (assoc :transaction-account/location (:transaction-rule-account/location tra)))]))) (:transaction-rule-account/location tra) (assoc :transaction-account/location (:transaction-rule-account/location tra)))])))
(filter (comp seq :transaction-rule-account/account) (:transaction-rule/accounts rule)))) (filter (comp seq :transaction-rule-account/account) (:transaction-rule/accounts rule))))

47
src/clj/user.clj
View File

@@ -2,7 +2,7 @@
(:require (:require
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn pull-attr random-tempid]] [auto-ap.datomic :refer [conn pull-attr random-tempid]]
[auto-ap.ledger :as l :refer [transact-with-ledger]] [auto-ap.ledger :as l ]
[auto-ap.server] [auto-ap.server]
[auto-ap.square.core :as square] [auto-ap.square.core :as square]
[auto-ap.square.core2 :as square2] [auto-ap.square.core2 :as square2]
@@ -68,48 +68,6 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn mark-until-date [client end]
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :invoice/client ?c]
'[?c :client/code ?client]
'[?e :invoice/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:invoice/exclude-from-ledger true}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100"))
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :transaction/client ?c]
'[?c :client/code ?client]
'[?e :transaction/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:transaction/approval-status :transaction-approval-status/excluded}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100")))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn load-accounts [conn] (defn load-accounts [conn]
(let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) (let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
@@ -443,6 +401,9 @@
(mu/start-publisher! {:type :dev}) (mu/start-publisher! {:type :dev})
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
(defn start-search []
(mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer})))
(defn restart-db [] (defn restart-db []
#_(require 'datomic.dev-local) #_(require 'datomic.dev-local)
#_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) #_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})

22
test/clj/auto_ap/import/transactions_test.clj
View File

@@ -3,6 +3,7 @@
[auto-ap.datomic :refer [conn]] [auto-ap.datomic :refer [conn]]
[auto-ap.import.transactions :as sut] [auto-ap.import.transactions :as sut]
[auto-ap.integration.util :refer [wrap-setup]] [auto-ap.integration.util :refer [wrap-setup]]
[iol-ion.tx :refer [upsert-transaction]]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clojure.test :as t] [clojure.test :as t]
@@ -83,14 +84,13 @@
:client/code "TEST" :client/code "TEST"
:client/locations ["Z" "E"] :client/locations ["Z" "E"]
:client/bank-accounts ["bank-account-id"]}]})) :client/bank-accounts ["bank-account-id"]}]}))
_ (println bank-account-id client-id)
result (sut/transaction->txs base-transaction result (sut/transaction->txs base-transaction
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
noop-rule)] noop-rule)]
(t/is (= [(assoc base-transaction (t/is (= (assoc base-transaction
:transaction/approval-status :transaction-approval-status/unapproved :transaction/approval-status :transaction-approval-status/unapproved
:transaction/bank-account bank-account-id :transaction/bank-account bank-account-id
:transaction/client client-id)] :transaction/client client-id)
result)))) result))))
(t/testing "Should match an uncleared check" (t/testing "Should match an uncleared check"
@@ -109,7 +109,7 @@
(dc/transact conn) (dc/transact conn)
:tempids)] :tempids)]
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/description-original "CHECK 10001" :transaction/description-original "CHECK 10001"
:transaction/amount -30.0) :transaction/amount -30.0)
(dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id)
@@ -121,7 +121,7 @@
(t/testing "Should match a check that matches on amount if check number does not match" (t/testing "Should match a check that matches on amount if check number does not match"
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/description-original "CHECK 12301" :transaction/description-original "CHECK 12301"
:transaction/amount -30.0) :transaction/amount -30.0)
(dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id)
@@ -133,7 +133,7 @@
(t/testing "Should not match an already matched check" (t/testing "Should not match an already matched check"
(dc/transact conn {:tx-data [{:db/id payment-id :payment/status :payment-status/cleared}]}) (dc/transact conn {:tx-data [{:db/id payment-id :payment/status :payment-status/cleared}]})
(let [[result] (sut/transaction->txs (assoc base-transaction (let [result (sut/transaction->txs (assoc base-transaction
:transaction/description-original "CHECK 10001" :transaction/description-original "CHECK 10001"
:transaction/amount -30.0) :transaction/amount -30.0)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
@@ -162,7 +162,7 @@
(t/testing "Should match within 10 days" (t/testing "Should match within 10 days"
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/date #inst "2021-07-03T00:00:00-08:00" :transaction/date #inst "2021-07-03T00:00:00-08:00"
:transaction/amount 100.0) :transaction/amount 100.0)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
@@ -175,7 +175,7 @@
(:transaction/expected-deposit transaction-result))))) (:transaction/expected-deposit transaction-result)))))
(t/testing "Should copy vendor from expected-depoisit" (t/testing "Should copy vendor from expected-depoisit"
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/vendor :vendor/ccp-square) :transaction/vendor :vendor/ccp-square)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
noop-rule)] noop-rule)]
@@ -183,7 +183,7 @@
(:transaction/vendor transaction-result))))) (:transaction/vendor transaction-result)))))
(t/testing "Should credit CCP" (t/testing "Should credit CCP"
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/date #inst "2021-07-03T00:00:00-08:00" :transaction/date #inst "2021-07-03T00:00:00-08:00"
:transaction/amount 100.0) :transaction/amount 100.0)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
@@ -194,7 +194,7 @@
(:transaction/accounts transaction-result))))) (:transaction/accounts transaction-result)))))
(t/testing "Should not match old expected deposits" (t/testing "Should not match old expected deposits"
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/date #inst "2021-07-13" :transaction/date #inst "2021-07-13"
:transaction/amount 100.0) :transaction/amount 100.0)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)
@@ -202,7 +202,7 @@
(t/is (not (:transaction/expected-deposit transaction-result))))) (t/is (not (:transaction/expected-deposit transaction-result)))))
(t/testing "Should only match exact." (t/testing "Should only match exact."
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction (let [transaction-result (sut/transaction->txs (assoc base-transaction
:transaction/date "2021-07-03" :transaction/date "2021-07-03"
:transaction/amount 100.01) :transaction/amount 100.01)
(dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id)

2
things-to-search-for.txt
View File

@@ -1,6 +1,5 @@
* try again to see if we can get upsert-ledger into the same transaction, making it all or nothing * try again to see if we can get upsert-ledger into the same transaction, making it all or nothing
* look for all usages of transact-with-ledger * look for all usages of transact-with-ledger
* propose-invoice should use upsert invoice as well
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling
@@ -33,6 +32,7 @@ Just use a periodic request or event instead of a job for running balance cache,
get rid of account-groups get rid of account-groups
move to solr move to solr
upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null
new way to do backups