(ns auto-ap.import.transactions (:require [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.rule-matching :as rm] [auto-ap.time :as atime] [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] [clj-time.core :as t] [clojure.core.cache :as cache] [clojure.tools.logging :as log] [datomic.client.api :as dc] [digest :as di])) (defn rough-match [client-id bank-account-id amount] (if (and client-id bank-account-id amount) (let [[matching-checks] (d-checks/get-graphql {:client-id client-id :bank-account-id bank-account-id :amount (- amount) :status :payment-status/pending})] (if (= 1 (count matching-checks)) (first matching-checks) nil)) nil)) (defn transaction->existing-payment [_ check-number client-id bank-account-id amount id] (log/info "Searching for a matching check for " {:client-id client-id :check-number check-number :bank-account-id bank-account-id :amount amount}) (cond (not (and client-id bank-account-id)) nil (:transaction/payment (d-transactions/get-by-id [:transaction/id #_{:clj-kondo/ignore [:unresolved-var]} (di/sha-256 (str id))])) nil check-number (or (-> (d-checks/get-graphql {:client-id client-id :bank-account-id bank-account-id :check-number check-number :amount (- amount) :status :payment-status/pending}) first first) (rough-match client-id bank-account-id amount)) :else (rough-match client-id bank-account-id amount))) (defn match-transaction-to-unfulfilled-autopayments [amount client-id] (log/info "trying to find uncleared autopay invoices") (let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?total '?sd] :in ['$ '?client-id] :where ['[?e :invoice/client ?client-id] '[?e :invoice/scheduled-payment ?sd] '[?e :invoice/status :invoice-status/paid] '(not [_ :invoice-payment/invoice ?e]) '[?e :invoice/vendor ?vendor-id] '[?e :invoice/total ?total]]} :args [(dc/db conn) client-id]}) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] (+ acc amount)) 0.0 consideration) (- amount))] consideration)] (log/info "Found " (count considerations) "considerations for transaction of" amount) considerations )) (defn match-transaction-to-unpaid-invoices [amount client-id] (log/info "trying to find unpaid invoices for " client-id amount) (let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?outstanding-balance '?d] :in ['$ '?client-id] :where ['[?e :invoice/client ?client-id] '[?e :invoice/status :invoice-status/unpaid] '(not [_ :invoice-payment/invoice ?e]) '[?e :invoice/vendor ?vendor-id] '[?e :invoice/outstanding-balance ?outstanding-balance] '[?e :invoice/date ?d]]} :args [(dc/db conn) client-id]}) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] (+ acc amount)) 0.0 consideration) (- amount))] consideration)] (log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount) considerations)) (defn match-transaction-to-single-unfulfilled-autopayments [amount client-id] (let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)] (if (= 1 (count considerations)) (first considerations) []))) (defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id] (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) (let [payment-id (random-tempid)] (-> tx (conj {:payment/bank-account bank-account-id :payment/client client-id :payment/amount (- (:transaction/amount transaction)) :payment/vendor vendor :payment/date (:transaction/date transaction) :payment/type :payment-type/debit :payment/status :payment-status/cleared :db/id payment-id}) (into (mapcat (fn [[_ invoice-id invoice-amount]] [{:invoice-payment/invoice invoice-id :invoice-payment/payment payment-id :invoice-payment/amount invoice-amount} {:db/id invoice-id :invoice/outstanding-balance 0.0 :invoice/status :invoice-status/paid}]) invoice-payments)) (update 0 assoc :transaction/payment payment-id :transaction/approval-status :transaction-approval-status/approved :transaction/vendor vendor :transaction/location "A") (conj [:reset (:db/id transaction) :transaction/accounts [#:transaction-account {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :location "A" :amount (Math/abs (:transaction/amount transaction))}]])))) (defn extract-check-number [{:transaction/keys [description-original]}] (if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)] (try (Integer/parseInt check-number) (catch NumberFormatException _ nil)) nil)) (defn find-expected-deposit [client-id amount date] (when date (-> (dc/q '[:find (pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}]) :in $ ?c ?a ?d-start :where [?ed :expected-deposit/client ?c] (not [?ed :expected-deposit/status :expected-deposit-status/cleared]) [?ed :expected-deposit/date ?d] [(>= ?d ?d-start)] [?ed :expected-deposit/total ?a2] [(auto-ap.utils/dollars= ?a2 ?a)] ] (dc/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) ffirst))) (defn categorize-transaction [transaction bank-account existing] (cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction))) :suppressed (existing (:transaction/id transaction)) :extant (not (:transaction/client transaction)) :error (not (:transaction/bank-account transaction)) :error (not (:transaction/id transaction)) :error (not= "POSTED" (:transaction/status transaction)) :not-ready (and (:bank-account/start-date bank-account) (and (not (t/after? (coerce/to-date-time (:transaction/date transaction)) (-> bank-account :bank-account/start-date coerce/to-date-time))) (not (t/equal? (coerce/to-date-time (:transaction/date transaction)) (-> bank-account :bank-account/start-date coerce/to-date-time))))) :not-ready (and (:client/locked-until (:client/_bank-accounts bank-account)) (and (not (t/after? (coerce/to-date-time (:transaction/date transaction)) (coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account))))) (not (t/equal? (coerce/to-date-time (:transaction/date transaction)) (coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account))))))) :not-ready :else :import)) (defn maybe-assoc-check-number [transaction] (if-let [check-number (or (:transaction/check-number transaction) (extract-check-number transaction))] (assoc transaction :transaction/check-number check-number) transaction)) (defn maybe-clear-payment [{:transaction/keys [check-number client bank-account amount id] :as transaction}] (when-let [existing-payment (transaction->existing-payment transaction check-number client bank-account amount id)] (assoc transaction :transaction/approval-status :transaction-approval-status/approved :transaction/payment {:db/id (:db/id existing-payment) :payment/status :payment-status/cleared} :transaction/vendor (:db/id (:payment/vendor existing-payment)) :transaction/location "A" :transaction/accounts [#:transaction-account {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :location "A" :amount (Math/abs (double amount))}]))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}] (when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))] (add-new-payment transaction autopay-invoices-matches bank-account client))) (defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}] (when (>= amount 0.0) (when-let [expected-deposit (find-expected-deposit client amount (coerce/to-date-time date))] (assoc transaction :transaction/expected-deposit {:db/id (:db/id expected-deposit) :expected-deposit/status :expected-deposit-status/cleared} :transaction/accounts [{:transaction-account/account :account/ccp :transaction-account/amount amount :transaction-account/location "A"}] :transaction/approval-status :transaction-approval-status/approved :transaction/vendor (:db/id (:expected-deposit/vendor expected-deposit)) )))) (defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations] (when (seq (match-transaction-to-unpaid-invoices amount client)) (apply-rules transaction valid-locations))) (defn transaction->txs [transaction bank-account apply-rules] (let [bank-account-id (:db/id bank-account) client (:client/_bank-accounts bank-account) client-id (:db/id client) valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) code-fn (some-fn maybe-clear-payment maybe-clear-expected-deposit #_maybe-autopay-invoices #(maybe-code % apply-rules valid-locations) identity)] [(-> transaction (assoc :transaction/client client-id) (assoc :transaction/bank-account bank-account-id) (assoc :transaction/approval-status :transaction-approval-status/unapproved) maybe-assoc-check-number code-fn remove-nils)])) (defn get-existing [bank-account] (log/info "looking up bank account data for" bank-account) (into {} (dc/q {:query {:find ['?tid '?as2] :in ['$ '?ba] :where ['[?e :transaction/bank-account ?ba] '[?e :transaction/id ?tid] '[?e :transaction/approval-status ?as] '[?as :db/ident ?as2]]} :args [(dc/db conn) bank-account]}))) (defprotocol ImportBatch (import-transaction! [this transaction]) (get-stats [this ]) (finish! [this]) (fail! [this error])) (defn start-import-batch [source user] (let [stats (atom {:import-batch/imported 0 :import-batch/suppressed 0 :import-batch/error 0 :import-batch/not-ready 0 :import-batch/extant 0}) extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 )) import-id (get (:tempids (dc/transact conn {:tx-data [{:db/id "import-batch" :import-batch/date (coerce/to-date (t/now)) :import-batch/source source :import-batch/status :import-status/started :import-batch/user-name user}]})) "import-batch") rule-applying-function (rm/rule-applying-fn (tr/get-all))] (log/info "Importing transactions from " source) (reify ImportBatch (import-transaction! [_ transaction] (let [bank-account (dc/pull (dc/db conn) [:bank-account/code :db/id :bank-account/locations :bank-account/start-date {:client/_bank-accounts [:client/code :client/locked-until :client/locations :db/id]} ] (:transaction/bank-account transaction)) extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing) (:transaction/bank-account transaction)) action (categorize-transaction transaction bank-account extant) transaction (assoc transaction :import-batch/_entry import-id)] (swap! stats #(update % (condp = action :import :import-batch/imported :extant :import-batch/extant :suppressed :import-batch/suppressed :error :import-batch/error :not-ready :import-batch/not-ready) inc)) (when (= :import action) (transact-with-ledger (transaction->txs transaction bank-account rule-applying-function) {:user/name user :user/role ":admin"})))) (get-stats [_] @stats) (fail! [_ error] (log/errorf "Couldn't complete import %d with error." import-id) (log/error error) (dc/transact conn {:tx-data [(merge {:db/id import-id :import-batch/status :import-status/completed :import-batch/error-message (str error)} @stats)]})) (finish! [_] (log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats)) (dc/transact conn [(merge {:db/id import-id :import-batch/status :import-status/completed} @stats)]))))) (defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index] (str (str (some-> date coerce/to-date-time atime/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client)) (defn apply-synthetic-ids [transactions] (->> transactions (group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client])) (vals) (mapcat (fn [group] (map (fn [index transaction] (let [raw-id (synthetic-key transaction index)] (assoc transaction :transaction/id #_{:clj-kondo/ignore [:unresolved-var]} (di/sha-256 raw-id) :transaction/raw-id raw-id))) (range) group)))))