360 lines
18 KiB
Clojure
360 lines
18 KiB
Clojure
(ns auto-ap.import.transactions
|
|
(:require
|
|
[auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]]
|
|
[auto-ap.datomic.accounts :as a]
|
|
[auto-ap.datomic.checks :as d-checks]
|
|
[auto-ap.datomic.transaction-rules :as tr]
|
|
[auto-ap.datomic.transactions :as d-transactions]
|
|
[auto-ap.ledger :refer [transact-with-ledger]]
|
|
[auto-ap.rule-matching :as rm]
|
|
[auto-ap.time :as atime]
|
|
[auto-ap.utils :refer [dollars=]]
|
|
[clj-time.coerce :as coerce]
|
|
[clj-time.core :as t]
|
|
[clojure.core.cache :as cache]
|
|
[clojure.tools.logging :as log]
|
|
[datomic.client.api :as dc]
|
|
[digest :as di]))
|
|
|
|
(defn rough-match [client-id bank-account-id amount]
|
|
(if (and client-id bank-account-id amount)
|
|
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:amount (- amount)
|
|
:status :payment-status/pending})]
|
|
(if (= 1 (count matching-checks))
|
|
(first matching-checks)
|
|
nil))
|
|
nil))
|
|
|
|
|
|
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
|
(log/info "Searching for a matching check for "
|
|
{:client-id client-id
|
|
:check-number check-number
|
|
:bank-account-id bank-account-id
|
|
:amount amount})
|
|
(cond (not (and client-id bank-account-id))
|
|
nil
|
|
|
|
(:transaction/payment (d-transactions/get-by-id [:transaction/id #_{:clj-kondo/ignore [:unresolved-var]}
|
|
(di/sha-256 (str id))]))
|
|
nil
|
|
|
|
check-number
|
|
(or (-> (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:check-number check-number
|
|
:amount (- amount)
|
|
:status :payment-status/pending})
|
|
first
|
|
first)
|
|
(rough-match client-id bank-account-id amount))
|
|
|
|
:else
|
|
(rough-match client-id bank-account-id amount)))
|
|
|
|
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
|
(log/info "trying to find uncleared autopay invoices")
|
|
(let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?total '?sd]
|
|
:in ['$ '?client-id]
|
|
:where ['[?e :invoice/client ?client-id]
|
|
'[?e :invoice/scheduled-payment ?sd]
|
|
'[?e :invoice/status :invoice-status/paid]
|
|
'(not [_ :invoice-payment/invoice ?e])
|
|
'[?e :invoice/vendor ?vendor-id]
|
|
'[?e :invoice/total ?total]]}
|
|
:args [(dc/db conn) client-id]})
|
|
(sort-by last) ;; sort by scheduled payment date
|
|
(group-by first) ;; group by vendors
|
|
vals)
|
|
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
|
invoice-count (range 1 32)
|
|
consideration (partition invoice-count 1 candidate-invoices)
|
|
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
|
(+ acc amount)) 0.0 consideration)
|
|
(- amount))]
|
|
consideration)]
|
|
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
|
considerations
|
|
))
|
|
|
|
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
|
(log/info "trying to find unpaid invoices for " client-id amount)
|
|
(let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
|
:in ['$ '?client-id]
|
|
:where ['[?e :invoice/client ?client-id]
|
|
'[?e :invoice/status :invoice-status/unpaid]
|
|
'(not [_ :invoice-payment/invoice ?e])
|
|
'[?e :invoice/vendor ?vendor-id]
|
|
'[?e :invoice/outstanding-balance ?outstanding-balance]
|
|
'[?e :invoice/date ?d]]}
|
|
:args [(dc/db conn) client-id]})
|
|
(sort-by last) ;; sort by scheduled payment date
|
|
(group-by first) ;; group by vendors
|
|
vals)
|
|
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
|
invoice-count (range 1 32)
|
|
consideration (partition invoice-count 1 candidate-invoices)
|
|
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
|
(+ acc amount)) 0.0 consideration)
|
|
(- amount))]
|
|
consideration)]
|
|
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
|
considerations))
|
|
|
|
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
|
(let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)]
|
|
(if (= 1 (count considerations))
|
|
(first considerations)
|
|
[])))
|
|
|
|
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
|
|
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
|
(let [payment-id (random-tempid)]
|
|
(-> tx
|
|
|
|
(conj {:payment/bank-account bank-account-id
|
|
:payment/client client-id
|
|
:payment/amount (- (:transaction/amount transaction))
|
|
:payment/vendor vendor
|
|
:payment/date (:transaction/date transaction)
|
|
:payment/type :payment-type/debit
|
|
:payment/status :payment-status/cleared
|
|
:db/id payment-id})
|
|
(into (mapcat (fn [[_ invoice-id invoice-amount]]
|
|
[{:invoice-payment/invoice invoice-id
|
|
:invoice-payment/payment payment-id
|
|
:invoice-payment/amount invoice-amount}
|
|
{:db/id invoice-id
|
|
:invoice/outstanding-balance 0.0
|
|
:invoice/status :invoice-status/paid}])
|
|
invoice-payments))
|
|
(update 0 assoc
|
|
:transaction/payment payment-id
|
|
:transaction/approval-status :transaction-approval-status/approved
|
|
:transaction/vendor vendor
|
|
:transaction/location "A")
|
|
(conj [:reset (:db/id transaction) :transaction/accounts
|
|
[#:transaction-account
|
|
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
|
:location "A"
|
|
:amount (Math/abs (:transaction/amount transaction))}]]))))
|
|
|
|
(defn extract-check-number [{:transaction/keys [description-original]}]
|
|
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
|
(try
|
|
(Integer/parseInt check-number)
|
|
(catch NumberFormatException _
|
|
nil))
|
|
nil))
|
|
|
|
(defn find-expected-deposit [client-id amount date]
|
|
(when date
|
|
(-> (dc/q
|
|
'[:find (pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}])
|
|
:in $ ?c ?a ?d-start
|
|
:where
|
|
[?ed :expected-deposit/client ?c]
|
|
(not [?ed :expected-deposit/status :expected-deposit-status/cleared])
|
|
[?ed :expected-deposit/date ?d]
|
|
[(>= ?d ?d-start)]
|
|
[?ed :expected-deposit/total ?a2]
|
|
[(auto-ap.utils/dollars= ?a2 ?a)]
|
|
]
|
|
(dc/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
|
|
ffirst)))
|
|
|
|
|
|
(defn categorize-transaction [transaction bank-account existing]
|
|
(cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction)))
|
|
:suppressed
|
|
|
|
(existing (:transaction/id transaction))
|
|
:extant
|
|
|
|
(not (:transaction/client transaction))
|
|
:error
|
|
|
|
(not (:transaction/bank-account transaction))
|
|
:error
|
|
|
|
(not (:transaction/id transaction))
|
|
:error
|
|
|
|
(not= "POSTED" (:transaction/status transaction))
|
|
:not-ready
|
|
|
|
(and (:bank-account/start-date bank-account)
|
|
(and (not (t/after? (coerce/to-date-time (:transaction/date transaction))
|
|
(-> bank-account :bank-account/start-date coerce/to-date-time)))
|
|
(not (t/equal? (coerce/to-date-time (:transaction/date transaction))
|
|
(-> bank-account :bank-account/start-date coerce/to-date-time)))))
|
|
:not-ready
|
|
|
|
(and (:client/locked-until (:client/_bank-accounts bank-account))
|
|
(and (not (t/after? (coerce/to-date-time (:transaction/date transaction))
|
|
(coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account)))))
|
|
(not (t/equal? (coerce/to-date-time (:transaction/date transaction))
|
|
(coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account)))))))
|
|
:not-ready
|
|
|
|
:else
|
|
:import))
|
|
|
|
(defn maybe-assoc-check-number [transaction]
|
|
(if-let [check-number (or (:transaction/check-number transaction)
|
|
(extract-check-number transaction))]
|
|
(assoc transaction :transaction/check-number check-number)
|
|
transaction))
|
|
|
|
|
|
(defn maybe-clear-payment [{:transaction/keys [check-number client bank-account amount id] :as transaction}]
|
|
(when-let [existing-payment (transaction->existing-payment transaction check-number client bank-account amount id)]
|
|
(assoc transaction
|
|
:transaction/approval-status :transaction-approval-status/approved
|
|
:transaction/payment {:db/id (:db/id existing-payment)
|
|
:payment/status :payment-status/cleared}
|
|
:transaction/vendor (:db/id (:payment/vendor existing-payment))
|
|
:transaction/location "A"
|
|
:transaction/accounts [#:transaction-account
|
|
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
|
:location "A"
|
|
:amount (Math/abs (double amount))}])))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}]
|
|
(when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))]
|
|
(add-new-payment transaction autopay-invoices-matches bank-account client)))
|
|
|
|
(defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}]
|
|
(when (>= amount 0.0)
|
|
(when-let [expected-deposit (find-expected-deposit client amount (coerce/to-date-time date))]
|
|
(assoc transaction
|
|
:transaction/expected-deposit {:db/id (:db/id expected-deposit)
|
|
:expected-deposit/status :expected-deposit-status/cleared}
|
|
:transaction/accounts [{:transaction-account/account :account/ccp
|
|
:transaction-account/amount amount
|
|
:transaction-account/location "A"}]
|
|
:transaction/approval-status :transaction-approval-status/approved
|
|
:transaction/vendor (:db/id (:expected-deposit/vendor expected-deposit))
|
|
))))
|
|
|
|
(defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations]
|
|
(when (seq (match-transaction-to-unpaid-invoices amount client))
|
|
(apply-rules transaction valid-locations)))
|
|
|
|
(defn transaction->txs [transaction bank-account apply-rules]
|
|
(let [bank-account-id (:db/id bank-account)
|
|
client (:client/_bank-accounts bank-account)
|
|
client-id (:db/id client)
|
|
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
|
code-fn (some-fn maybe-clear-payment
|
|
maybe-clear-expected-deposit
|
|
#_maybe-autopay-invoices
|
|
#(maybe-code % apply-rules valid-locations)
|
|
identity)]
|
|
[(-> transaction
|
|
(assoc :transaction/client client-id)
|
|
(assoc :transaction/bank-account bank-account-id)
|
|
(assoc :transaction/approval-status :transaction-approval-status/unapproved)
|
|
maybe-assoc-check-number
|
|
code-fn
|
|
remove-nils)]))
|
|
|
|
|
|
(defn get-existing [bank-account]
|
|
(log/info "looking up bank account data for" bank-account)
|
|
(into {}
|
|
(dc/q {:query {:find ['?tid '?as2]
|
|
:in ['$ '?ba]
|
|
:where ['[?e :transaction/bank-account ?ba]
|
|
'[?e :transaction/id ?tid]
|
|
'[?e :transaction/approval-status ?as]
|
|
'[?as :db/ident ?as2]]}
|
|
:args [(dc/db conn) bank-account]})))
|
|
|
|
(defprotocol ImportBatch
|
|
(import-transaction! [this transaction])
|
|
(get-stats [this ])
|
|
(finish! [this])
|
|
(fail! [this error]))
|
|
|
|
(defn start-import-batch [source user]
|
|
(let [stats (atom {:import-batch/imported 0
|
|
:import-batch/suppressed 0
|
|
:import-batch/error 0
|
|
:import-batch/not-ready 0
|
|
:import-batch/extant 0})
|
|
extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 ))
|
|
import-id (get (:tempids (dc/transact conn {:tx-data [{:db/id "import-batch"
|
|
:import-batch/date (coerce/to-date (t/now))
|
|
:import-batch/source source
|
|
:import-batch/status :import-status/started
|
|
:import-batch/user-name user}]})) "import-batch")
|
|
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
|
|
(log/info "Importing transactions from " source)
|
|
|
|
(reify ImportBatch
|
|
(import-transaction! [_ transaction]
|
|
(let [bank-account (dc/pull (dc/db conn)
|
|
[:bank-account/code
|
|
:db/id
|
|
:bank-account/locations
|
|
:bank-account/start-date
|
|
{:client/_bank-accounts [:client/code :client/locked-until :client/locations :db/id]} ]
|
|
(:transaction/bank-account transaction))
|
|
extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing)
|
|
(:transaction/bank-account transaction))
|
|
action (categorize-transaction transaction bank-account extant)
|
|
transaction (assoc transaction :import-batch/_entry import-id)]
|
|
(swap! stats
|
|
#(update % (condp = action
|
|
:import :import-batch/imported
|
|
:extant :import-batch/extant
|
|
:suppressed :import-batch/suppressed
|
|
:error :import-batch/error
|
|
:not-ready :import-batch/not-ready) inc))
|
|
(when (= :import action)
|
|
(transact-with-ledger (transaction->txs transaction bank-account rule-applying-function)
|
|
{:user/name user
|
|
:user/role ":admin"}))))
|
|
|
|
(get-stats [_]
|
|
@stats)
|
|
|
|
(fail! [_ error]
|
|
(log/errorf "Couldn't complete import %d with error." import-id)
|
|
(log/error error)
|
|
(dc/transact conn {:tx-data [(merge {:db/id import-id
|
|
:import-batch/status :import-status/completed
|
|
:import-batch/error-message (str error)}
|
|
@stats)]}))
|
|
|
|
(finish! [_]
|
|
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
|
|
(dc/transact conn [(merge {:db/id import-id
|
|
|
|
:import-batch/status :import-status/completed}
|
|
@stats)])))))
|
|
|
|
|
|
|
|
(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index]
|
|
(str (str (some-> date coerce/to-date-time atime/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
|
|
|
|
(defn apply-synthetic-ids [transactions]
|
|
(->> transactions
|
|
(group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client]))
|
|
(vals)
|
|
(mapcat (fn [group]
|
|
(map (fn [index transaction]
|
|
(let [raw-id (synthetic-key transaction index)]
|
|
(assoc transaction
|
|
:transaction/id #_{:clj-kondo/ignore [:unresolved-var]}
|
|
(di/sha-256 raw-id)
|
|
:transaction/raw-id raw-id)))
|
|
(range)
|
|
group)))))
|
|
|