enormous refactor but simplified much!
This commit is contained in:
Bryce Covert
336
src/clj/auto_ap/import/transactions.clj
Normal file
336
src/clj/auto_ap/import/transactions.clj
Normal file
@@ -0,0 +1,336 @@
|
||||
(ns auto-ap.import.transactions
|
||||
(:require [auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
|
||||
[auto-ap.datomic.accounts :as a]
|
||||
[auto-ap.datomic.checks :as d-checks]
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
[auto-ap.utils :refer [dollars=]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as t]
|
||||
[clojure.core.cache :as cache]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]))
|
||||
|
||||
(defn rough-match [client-id bank-account-id amount]
|
||||
(if (and client-id bank-account-id amount)
|
||||
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})]
|
||||
(if (= 1 (count matching-checks))
|
||||
(first matching-checks)
|
||||
nil))
|
||||
nil))
|
||||
|
||||
|
||||
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
||||
(log/info "Searching for a matching check for "
|
||||
{:client-id client-id
|
||||
:check-number check-number
|
||||
:bank-account-id bank-account-id
|
||||
:amount amount})
|
||||
(cond (not (and client-id bank-account-id))
|
||||
nil
|
||||
|
||||
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
||||
nil
|
||||
|
||||
check-number
|
||||
(or (-> (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:check-number check-number
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})
|
||||
first
|
||||
first)
|
||||
(rough-match client-id bank-account-id amount))
|
||||
|
||||
:else
|
||||
(rough-match client-id bank-account-id amount)))
|
||||
|
||||
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
||||
(log/info "trying to find uncleared autopay invoices")
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/scheduled-payment ?sd]
|
||||
'[?e :invoice/status :invoice-status/paid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/total ?total]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
||||
considerations
|
||||
))
|
||||
|
||||
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
||||
(log/info "trying to find unpaid invoices for " client-id amount)
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/status :invoice-status/unpaid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/outstanding-balance ?outstanding-balance]
|
||||
'[?e :invoice/date ?d]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
||||
(let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)]
|
||||
(if (= 1 (count considerations))
|
||||
(first considerations)
|
||||
[])))
|
||||
|
||||
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
|
||||
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
||||
(let [payment-id (d/tempid :db.part/user)]
|
||||
(-> tx
|
||||
|
||||
(conj {:payment/bank-account bank-account-id
|
||||
:payment/client client-id
|
||||
:payment/amount (- (:transaction/amount transaction))
|
||||
:payment/vendor vendor
|
||||
:payment/date (:transaction/date transaction)
|
||||
:payment/type :payment-type/debit
|
||||
:payment/status :payment-status/cleared
|
||||
:db/id payment-id})
|
||||
(into (mapcat (fn [[vendor invoice-id invoice-amount]]
|
||||
[{:invoice-payment/invoice invoice-id
|
||||
:invoice-payment/payment payment-id
|
||||
:invoice-payment/amount invoice-amount}
|
||||
{:db/id invoice-id
|
||||
:invoice/outstanding-balance 0.0
|
||||
:invoice/status :invoice-status/paid}])
|
||||
invoice-payments))
|
||||
(update 0 assoc
|
||||
:transaction/payment payment-id
|
||||
:transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/vendor vendor
|
||||
:transaction/location "A")
|
||||
(conj [:reset (:db/id transaction) :transaction/accounts
|
||||
[#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (:transaction/amount transaction))}]]))))
|
||||
|
||||
(defn extract-check-number [{:transaction/keys [description-original]}]
|
||||
|
||||
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
||||
(try
|
||||
(Integer/parseInt check-number)
|
||||
(catch NumberFormatException e
|
||||
nil))
|
||||
nil))
|
||||
|
||||
(defn find-expected-deposit [client-id amount date]
|
||||
(when date
|
||||
(-> (d/q
|
||||
'[:find ?ed
|
||||
:in $ ?c ?a ?d-start
|
||||
:where
|
||||
[?ed :expected-deposit/client ?c]
|
||||
(not [?ed :expected-deposit/status :expected-deposit-status/cleared])
|
||||
[?ed :expected-deposit/date ?d]
|
||||
[(>= ?d ?d-start)]
|
||||
[?ed :expected-deposit/total ?a2]
|
||||
[(auto-ap.utils/dollars= ?a2 ?a)]
|
||||
]
|
||||
(d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
|
||||
first
|
||||
first)))
|
||||
|
||||
|
||||
(defn categorize-transaction [transaction bank-account existing]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction)))
|
||||
:suppressed
|
||||
|
||||
(existing (:transaction/id transaction))
|
||||
:extant
|
||||
|
||||
(not (:transaction/client transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/bank-account transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/id transaction))
|
||||
:error
|
||||
|
||||
(not= "POSTED" (:transaction/status transaction))
|
||||
:not-posted
|
||||
|
||||
(and (:bank-account/start-date bank-account)
|
||||
(not (t/after? (coerce/to-date-time (:transaction/date transaction))
|
||||
(-> bank-account :bank-account/start-date coerce/to-date-time))))
|
||||
:not-ready
|
||||
|
||||
:else
|
||||
:import)))
|
||||
|
||||
(defn transaction->txs [transaction bank-account apply-rules]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(into []
|
||||
(let [{:transaction/keys [amount id date]} transaction
|
||||
check-number (extract-check-number transaction)
|
||||
existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit (:db/id client) amount (coerce/to-date-time date)))]
|
||||
(cond->
|
||||
[(assoc transaction :transaction/approval-status :transaction-approval-status/unapproved)]
|
||||
check-number (update 0 #(assoc % :transaction/check-number check-number))
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils))))))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
(defn get-existing [bank-account]
|
||||
(log/info "looking up bank account data for" bank-account)
|
||||
(into {}
|
||||
(d/query {:query {:find ['?tid '?as2]
|
||||
:in ['$ '?ba]
|
||||
:where ['[?e :transaction/bank-account ?ba]
|
||||
'[?e :transaction/id ?tid]
|
||||
'[?e :transaction/approval-status ?as]
|
||||
'[?as :db/ident ?as2]]}
|
||||
:args [(d/db (d/connect uri)) bank-account]})))
|
||||
|
||||
(defprotocol ImportBatch
|
||||
(import-transaction! [this transaction])
|
||||
(get-stats [this ])
|
||||
(finish! [this])
|
||||
(fail! [this error]))
|
||||
|
||||
(defn start-import-batch [source user]
|
||||
(let [stats (atom {:import-batch/imported 0
|
||||
:import-batch/suppressed 0
|
||||
:import-batch/error 0
|
||||
:import-batch/not-ready 0
|
||||
:import-batch/extant 0})
|
||||
extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 ))
|
||||
import-id (get (:tempids @(d/transact conn [{:db/id "import-batch"
|
||||
:import-batch/date (coerce/to-date (t/now))
|
||||
:import-batch/source source
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch")
|
||||
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
|
||||
(log/info "Importing transactions from " source)
|
||||
|
||||
(reify ImportBatch
|
||||
(import-transaction! [this transaction]
|
||||
(let [bank-account (d/pull (d/db conn)
|
||||
[:bank-account/code
|
||||
:db/id
|
||||
:bank-account/locations
|
||||
:bank-account/start-date
|
||||
{:client/_bank-accounts [:client/code :client/locations :db/id]} ]
|
||||
(:transaction/bank-account transaction))
|
||||
extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing)
|
||||
(:transaction/bank-account transaction))
|
||||
action (categorize-transaction transaction bank-account extant)
|
||||
transaction (assoc transaction :import-batch/_entry import-id)]
|
||||
(swap! stats
|
||||
#(update % (condp = action
|
||||
:import :import-batch/imported
|
||||
:extant :import-batch/extant
|
||||
:suppressed :import-batch/suppressed
|
||||
:error :import-batch/error
|
||||
:not-read :import-batch/not-ready) inc))
|
||||
(when (= :import action)
|
||||
(audit-transact (transaction->txs transaction bank-account rule-applying-function)
|
||||
{:user/name user
|
||||
:user/role ":admin"}))))
|
||||
|
||||
(get-stats [this]
|
||||
@stats)
|
||||
|
||||
(fail! [this error]
|
||||
(log/errorf "Couldn't complete import %d with error." import-id)
|
||||
(log/error error)
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
:import-batch/status :import-status/completed
|
||||
:import-batch/error-message (str error)}
|
||||
@stats)]))
|
||||
|
||||
(finish! [this]
|
||||
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
|
||||
:import-batch/status :import-status/completed}
|
||||
@stats)])))))
|
||||
|
||||
|
||||
|
||||
(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index]
|
||||
(str (str (some-> date coerce/to-date-time auto-ap.time/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
|
||||
|
||||
(defn apply-synthetic-ids [transactions]
|
||||
(->> transactions
|
||||
(group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client]))
|
||||
(vals)
|
||||
(mapcat (fn [group]
|
||||
(map (fn [index transaction]
|
||||
(let [raw-id (synthetic-key transaction index)]
|
||||
(assoc transaction
|
||||
:transaction/id (digest/sha-256 raw-id)
|
||||
:transaction/raw-id raw-id)))
|
||||
(range)
|
||||
group)))))
|
||||
|
||||
Reference in New Issue
Block a user