(ns auto-ap.import.transactions (:require [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.logging :as alog] [auto-ap.rule-matching :as rm] [auto-ap.solr :as solr] [auto-ap.time :as atime] [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] [clj-time.core :as t] [clojure.core.cache :as cache] [com.brunobonacci.mulog :as mu] [datomic.api :as dc] [digest :as di])) (defn rough-match [client-id bank-account-id amount] (if (and client-id bank-account-id amount) (let [[matching-checks] (d-checks/get-graphql {:client-id client-id :clients [client-id] :bank-account-id bank-account-id :amount (- amount) :status :payment-status/pending})] (if (= 1 (count matching-checks)) (first matching-checks) nil)) nil)) (defn transaction->existing-payment [_ check-number client-id bank-account-id amount id] (alog/info ::searching :client-id client-id :check-number check-number :bank-account-id bank-account-id :amount amount) (cond (not (and client-id bank-account-id)) nil (:transaction/payment (d-transactions/get-by-id [:transaction/id #_{:clj-kondo/ignore [:unresolved-var]} (di/sha-256 (str id))])) nil check-number (-> (d-checks/get-graphql {:client-id client-id :clients [client-id] :bank-account-id bank-account-id :check-number check-number :amount (- amount) :status :payment-status/pending}) first first) :else (rough-match client-id bank-account-id amount))) (defn match-transaction-to-unfulfilled-autopayments [amount client-id] (let [candidate-invoices-vendor-groups (->> (dc/q {:find ['?vendor-id '?e '?total '?sd] :in ['$ '?client-id] :where ['[?e :invoice/client ?client-id] '[?e :invoice/scheduled-payment ?sd] '[?e :invoice/status :invoice-status/paid] '(not [_ :invoice-payment/invoice ?e]) '[?e :invoice/vendor ?vendor-id] '[?e :invoice/total ?total]]} (dc/db conn) client-id) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] (+ acc amount)) 0.0 consideration) (- amount))] consideration)] (alog/info ::unfulfilled-autoapayment-considerations :count (count considerations) :amount amount) considerations)) (defn match-transaction-to-unpaid-invoices [amount client-id] (alog/info ::searching-unpaid-invoice :client-id client-id :amount amount) (try (let [candidate-invoices-vendor-groups (->> (dc/q {:find ['?vendor-id '?e '?outstanding-balance '?d] :in ['$ '?client-id] :where ['[?e :invoice/client ?client-id] '[?e :invoice/status :invoice-status/unpaid] '(not [_ :invoice-payment/invoice ?e]) '[?e :invoice/vendor ?vendor-id] '[?e :invoice/outstanding-balance ?outstanding-balance] '[?e :invoice/date ?d]]} (dc/db conn) client-id) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] (+ acc amount)) 0.0 consideration) (- amount))] consideration)] (alog/info ::unpaid-invoice-considerations-found :client-id client-id :amount amount :count (count considerations)) considerations) (catch Exception e (alog/error ::cant-get-considerations :error e) []))) (defn match-transaction-to-single-unfulfilled-autopayments [amount client-id] (let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)] (if (= 1 (count considerations)) (first considerations) []))) (defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id] (alog/info ::adding-payment :transaction-id (:transaction/id transaction) :invoices (count invoice-payments)) (let [payment-id (random-tempid)] (-> [[:upsert-transaction (assoc transaction :transaction/payment payment-id :transaction/approval-status :transaction-approval-status/approved :transaction/vendor vendor :transaction/location "A" :transaction/accounts [#:transaction-account {:db/id (random-tempid) :account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :location "A" :amount (Math/abs (:transaction/amount transaction))}])]] (conj {:payment/bank-account bank-account-id :payment/client client-id :payment/amount (- (:transaction/amount transaction)) :payment/vendor vendor :payment/date (:transaction/date transaction) :payment/type :payment-type/debit :payment/status :payment-status/cleared :db/id payment-id}) (into (mapcat (fn [[_ invoice-id invoice-amount]] [{:invoice-payment/invoice invoice-id :invoice-payment/payment payment-id :invoice-payment/amount invoice-amount} [:upsert-invoice {:db/id invoice-id :invoice/outstanding-balance 0.0 :invoice/status :invoice-status/paid}]]) invoice-payments))))) (defn extract-check-number [{:transaction/keys [description-original]}] (if-let [[_ _ check-number] (re-find #"(?i)check(card|.*?([0-9]{4,}))" description-original)] (try (Integer/parseInt check-number) (catch NumberFormatException _ nil)) nil)) (comment (= 1234 (extract-check-number {:transaction/description-original "Check 1234"})) (= 1234 (extract-check-number {:transaction/description-original "Check abc 1234"})) (= 1234 (extract-check-number {:transaction/description-original "Check abc 4/10 1234"})) (= 1234 (extract-check-number {:transaction/description-original "Check abc 4/10 1234 12/3"})) (not= 1234 (extract-check-number {:transaction/description-original "Checkcard 4/10 1234"}))) (defn find-expected-deposit [client-id amount date] (when date (-> (dc/q '[:find (pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}]) :in $ ?c ?a ?d-start :where [?ed :expected-deposit/client ?c] (not [?ed :expected-deposit/status :expected-deposit-status/cleared]) [?ed :expected-deposit/date ?d] [(>= ?d ?d-start)] [?ed :expected-deposit/total ?a2] [(auto-ap.utils/dollars= ?a2 ?a)]] (dc/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) ffirst))) (defn categorize-transaction [transaction bank-account existing] (cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction))) :suppressed (existing (:transaction/id transaction)) :extant (not (:transaction/client transaction)) :error (not (:transaction/bank-account transaction)) :error (not (:transaction/id transaction)) :error (not= "POSTED" (:transaction/status transaction)) :not-ready (and (:bank-account/start-date bank-account) (and (not (t/after? (coerce/to-date-time (:transaction/date transaction)) (-> bank-account :bank-account/start-date coerce/to-date-time))) (not (t/equal? (coerce/to-date-time (:transaction/date transaction)) (-> bank-account :bank-account/start-date coerce/to-date-time))))) :not-ready (and (:client/locked-until (:client/_bank-accounts bank-account)) (and (not (t/after? (coerce/to-date-time (:transaction/date transaction)) (coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account))))) (not (t/equal? (coerce/to-date-time (:transaction/date transaction)) (coerce/to-date-time (:client/locked-until (:client/_bank-accounts bank-account))))))) :not-ready :else :import)) (defn maybe-assoc-check-number [transaction] (if-let [check-number (or (:transaction/check-number transaction) (extract-check-number transaction))] (assoc transaction :transaction/check-number check-number) transaction)) (defn maybe-clear-payment [{:transaction/keys [check-number client bank-account amount id] :as transaction}] (when-let [existing-payment (transaction->existing-payment transaction check-number client bank-account amount id)] (assoc transaction :transaction/approval-status :transaction-approval-status/approved :transaction/payment {:db/id (:db/id existing-payment) :payment/status :payment-status/cleared} :transaction/vendor (:db/id (:payment/vendor existing-payment)) :transaction/location "A" :transaction/accounts [#:transaction-account {:db/id (random-tempid) :account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :location "A" :amount (Math/abs (double amount))}]))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}] (when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))] (add-new-payment transaction autopay-invoices-matches bank-account client))) (defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}] (when (>= amount 0.0) (when-let [expected-deposit (find-expected-deposit client amount (coerce/to-date-time date))] (assoc transaction :transaction/expected-deposit {:db/id (:db/id expected-deposit) :expected-deposit/status :expected-deposit-status/cleared} :transaction/accounts [{:db/id (random-tempid) :transaction-account/account :account/ccp :transaction-account/amount amount :transaction-account/location "A"}] :transaction/approval-status :transaction-approval-status/approved :transaction/vendor (:db/id (:expected-deposit/vendor expected-deposit)))))) (defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations] (mu/trace ::maybe-code [] (when-not (seq (match-transaction-to-unpaid-invoices amount client)) (apply-rules transaction valid-locations)))) (defn maybe-apply-default-vendor [t] (cond-> t (and (not (:transaction/vendor t)) (:transaction/default-vendor t)) (assoc :transaction/vendor (:transaction/default-vendor t)) true (dissoc :transaction/default-vendor))) (defn transaction->txs [transaction bank-account apply-rules] (let [bank-account-id (:db/id bank-account) client (:client/_bank-accounts bank-account) client-id (:db/id client) valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) code-fn (some-fn maybe-clear-payment maybe-clear-expected-deposit #_maybe-autopay-invoices #(maybe-code % apply-rules valid-locations) identity)] (-> transaction (assoc :transaction/client client-id) (assoc :transaction/bank-account bank-account-id) (assoc :transaction/approval-status :transaction-approval-status/unapproved) maybe-assoc-check-number code-fn (maybe-apply-default-vendor) remove-nils))) (defn get-existing [bank-account] (into {} (dc/q '[:find ?tid ?as2 :in $ ?ba :where [?e :transaction/bank-account ?ba] [?e :transaction/id ?tid] [?e :transaction/approval-status ?as] [?as :db/ident ?as2]] (dc/db conn) bank-account))) (defprotocol ImportBatch (import-transaction! [this transaction]) (get-stats [this]) (get-pending-balance [this bank-account]) (finish! [this]) (fail! [this error])) (def bank-account-pull [:bank-account/code :db/id :bank-account/locations :bank-account/start-date {:client/_bank-accounts [:client/code :client/locked-until :client/locations :db/id]}]) (defn start-import-batch [source user] (let [stats (atom {:import-batch/imported 0 :import-batch/suppressed 0 :import-batch/error 0 :import-batch/not-ready 0 :import-batch/extant 0}) pending-balance (atom {}) extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000)) import-id (get (:tempids @(dc/transact-async conn [{:db/id "import-batch" :import-batch/date (coerce/to-date (t/now)) :import-batch/source source :import-batch/status :import-status/started :import-batch/user-name user}])) "import-batch") rule-applying-function (rm/rule-applying-fn (tr/get-all))] (alog/info ::starting-transaction-import :source source) (reify ImportBatch (import-transaction! [_ transaction] (let [bank-account (dc/pull (dc/db conn) bank-account-pull (:transaction/bank-account transaction)) extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing) (:transaction/bank-account transaction)) action (categorize-transaction transaction bank-account extant)] (try (when (not= "POSTED" (:transaction/status transaction)) (swap! pending-balance (fn [pb] (update pb (:transaction/bank-account transaction) (fnil + 0.0) (:transaction/amount transaction))))) (catch Exception e (alog/warn ::cant-capture-pending :error e))) (swap! stats #(update % (condp = action :import :import-batch/imported :extant :import-batch/extant :suppressed :import-batch/suppressed :error :import-batch/error :not-ready :import-batch/not-ready) inc)) (when (= :import action) (try (let [result (audit-transact [[:upsert-transaction (transaction->txs transaction bank-account rule-applying-function)] {:db/id import-id :import-batch/entry (:db/id transaction)}] {:user/name user :user/role ":admin"})] (doseq [[_ n] (:tempids result)] (solr/touch-with-ledger n))) (catch Exception e (swap! stats #(update % :import-batch/error inc)) (alog/error ::invalid-transaction :hint "It may be that the same bank account is linked twice" :error e)))))) (get-stats [_] @stats) (get-pending-balance [_ bank-account] (or (get @pending-balance bank-account) 0.0)) (fail! [_ error] (alog/error ::cant-complete-import :import-id import-id :error error) @(dc/transact-async conn [(merge {:db/id import-id :import-batch/status :import-status/completed :import-batch/error-message (str error)} @stats)])) (finish! [_] (alog/info ::finished :import-id import-id :source source :stats (pr-str @stats)) @(dc/transact conn [(merge {:db/id import-id :import-batch/status :import-status/completed} @stats)]))))) (defn synthetic-key [{:transaction/keys [date bank-account description-original amount client]} index] (str (str (some-> date coerce/to-date-time atime/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client)) (defn apply-synthetic-ids [transactions] (->> transactions (group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client])) (vals) (mapcat (fn [group] (map (fn [index transaction] (let [raw-id (synthetic-key transaction index)] (assoc transaction :transaction/id #_{:clj-kondo/ignore [:unresolved-var]} (di/sha-256 raw-id) :transaction/raw-id raw-id :db/id (random-tempid)))) (range) group)))))