enormous refactor but simplified much!
This commit is contained in:
Bryce Covert
@@ -5,9 +5,12 @@
|
||||
[mount.core :as mount]
|
||||
[yang.scheduler :as scheduler]
|
||||
[clojure.tools.logging :as log]
|
||||
[auto-ap.intuit.import :as i]
|
||||
[auto-ap.import.intuit :as i]
|
||||
[auto-ap.import.plaid :as p]
|
||||
[unilog.context :as lc]
|
||||
[auto-ap.yodlee.import :as y]))
|
||||
[auto-ap.import.yodlee :as y]
|
||||
[auto-ap.import.yodlee2 :as y2]
|
||||
))
|
||||
|
||||
(def queue-url (:requests-queue-url env))
|
||||
|
||||
@@ -22,19 +25,25 @@
|
||||
(cond
|
||||
(= ":intuit" body)
|
||||
(try
|
||||
(i/upsert-transactions)
|
||||
(i/import-intuit)
|
||||
(catch Exception e
|
||||
(log/error e)))
|
||||
|
||||
(= ":yodlee" body)
|
||||
(try
|
||||
(y/do-import)
|
||||
(y/import-yodlee)
|
||||
(catch Exception e
|
||||
(log/error e)))
|
||||
|
||||
(= ":yodlee2" body)
|
||||
(try
|
||||
(y/do-import2)
|
||||
(y2/import-yodlee2)
|
||||
(catch Exception e
|
||||
(log/error e)))
|
||||
|
||||
(= ":plaid" body)
|
||||
(try
|
||||
(p/import-plaid)
|
||||
(catch Exception e
|
||||
(log/error e))))
|
||||
(sqs/delete-message {:queue-url queue-url
|
||||
|
||||
@@ -430,7 +430,20 @@
|
||||
:auto-ap/add-suppression {:txes [[{:db/ident :transaction-approval-status/suppressed}
|
||||
{:db/ident :transaction/approval-status
|
||||
:db/index true}]]
|
||||
:requires [:auto-ap/add-transaction-rules]}}
|
||||
:requires [:auto-ap/add-transaction-rules]}
|
||||
:auto-ap/add-other-statuses {:txes [[{:db/ident :import-batch/error
|
||||
:db/doc "How many entries were an error "
|
||||
:db/valueType :db.type/long
|
||||
:db/cardinality :db.cardinality/one}
|
||||
{:db/ident :import-batch/not-ready
|
||||
:db/doc "How many entries were before a start date "
|
||||
:db/valueType :db.type/long
|
||||
:db/cardinality :db.cardinality/one}
|
||||
{:db/ident :import-batch/error-message
|
||||
:db/doc "error message for a failed job"
|
||||
:db/valueType :db.type/string
|
||||
:db/cardinality :db.cardinality/one}]]
|
||||
:requires [:auto-ap/add-transaction-import2]}}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.graphql.transaction-rules :as g-tr]
|
||||
[auto-ap.yodlee.import :as import]
|
||||
[auto-ap.import.transactions :as i-transactions]
|
||||
[auto-ap.graphql.utils
|
||||
:refer
|
||||
[->graphql
|
||||
@@ -101,7 +101,7 @@
|
||||
(let [transaction (d-transactions/get-by-id (:transaction_id args))
|
||||
_ (assert-can-see-client (:id context) (:transaction/client transaction) )]
|
||||
|
||||
(let [matches-set (import/match-transaction-to-unfulfilled-autopayments (:transaction/amount transaction)
|
||||
(let [matches-set (i-transactions/match-transaction-to-unfulfilled-autopayments (:transaction/amount transaction)
|
||||
(:db/id (:transaction/client transaction)))]
|
||||
(->graphql (for [matches matches-set]
|
||||
(for [[_ invoice-id ] matches]
|
||||
@@ -112,7 +112,7 @@
|
||||
(let [transaction (d-transactions/get-by-id (:transaction_id args))
|
||||
_ (assert-can-see-client (:id context) (:transaction/client transaction) )]
|
||||
|
||||
(let [matches-set (import/match-transaction-to-unpaid-invoices (:transaction/amount transaction)
|
||||
(let [matches-set (i-transactions/match-transaction-to-unpaid-invoices (:transaction/amount transaction)
|
||||
(:db/id (:transaction/client transaction)))]
|
||||
(->graphql (for [matches matches-set]
|
||||
(for [[_ invoice-id ] matches]
|
||||
@@ -337,7 +337,7 @@
|
||||
[(-> entity :invoice/vendor :db/id)
|
||||
(-> entity :db/id)
|
||||
(-> entity :invoice/total)])))))
|
||||
(let [payment-tx (import/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
|
||||
(let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
|
||||
(map (fn [id]
|
||||
(let [entity (d/entity db id)]
|
||||
[(-> entity :invoice/vendor :db/id)
|
||||
@@ -373,7 +373,7 @@
|
||||
(when (:transaction/payment transaction)
|
||||
(throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"})))
|
||||
|
||||
(let [payment-tx (import/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
|
||||
(let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
|
||||
(map (fn [id]
|
||||
(let [entity (d/entity db id)]
|
||||
[(-> entity :invoice/vendor :db/id)
|
||||
|
||||
74
src/clj/auto_ap/import/intuit.clj
Normal file
74
src/clj/auto_ap/import/intuit.clj
Normal file
@@ -0,0 +1,74 @@
|
||||
(ns auto-ap.import.intuit
|
||||
(:require [amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic :refer [conn remove-nils]]
|
||||
[auto-ap.intuit.core :as i]
|
||||
[auto-ap.utils :refer [by allow-once]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clj-time.format :as f]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(defn get-intuit-bank-accounts [db]
|
||||
(d/q '[:find ?external-id ?ba ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/intuit-bank-account ?iab]
|
||||
[?iab :intuit-bank-account/external-id ?external-id]]
|
||||
db))
|
||||
|
||||
(defn intuit->transaction [transaction]
|
||||
{:transaction/description-original (:Memo/Description transaction)
|
||||
:transaction/amount (Double/parseDouble (:Amount transaction))
|
||||
:transaction/date (coerce/to-date (auto-ap.time/parse (:Date transaction) auto-ap.time/iso-date))
|
||||
:transaction/status "POSTED"})
|
||||
|
||||
(defn intuits->transactions [transactions bank-account-id client-id]
|
||||
(->> transactions
|
||||
(map intuit->transaction)
|
||||
(map #(assoc %
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))
|
||||
(t/apply-synthetic-ids)))
|
||||
|
||||
(defn import-intuit []
|
||||
(lc/with-context {:source "Import intuit transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
|
||||
db (d/db conn)
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
(try
|
||||
(doseq [[external-id bank-account-id client-id] (get-intuit-bank-accounts db)
|
||||
transaction (-> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
|
||||
(auto-ap.time/unparse end auto-ap.time/iso-date)
|
||||
external-id)
|
||||
(intuits->transactions bank-account-id client-id))]
|
||||
(t/import-transaction! import-batch transaction))
|
||||
(t/finish! import-batch)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def upsert-transactions (allow-once upsert-transactions))
|
||||
|
||||
(defn upsert-accounts []
|
||||
(let [token (i/get-fresh-access-token)
|
||||
bank-accounts (i/get-bank-accounts token)]
|
||||
@(d/transact conn (mapv
|
||||
(fn [ba]
|
||||
{:intuit-bank-account/external-id (:name ba)
|
||||
:intuit-bank-account/name (:name ba)})
|
||||
bank-accounts))))
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) import-intuit)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
(mount/defstate account-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) upsert-accounts)
|
||||
:stop (scheduler/stop account-worker))
|
||||
20
src/clj/auto_ap/import/intuit_test.clj
Normal file
20
src/clj/auto_ap/import/intuit_test.clj
Normal file
@@ -0,0 +1,20 @@
|
||||
(ns auto-ap.import.intuit-test
|
||||
(:require [auto-ap.import.intuit :as sut]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(def base-transaction {:Memo/Description "this is a description"
|
||||
:Amount "45.23"
|
||||
:Date "2021-10-11"})
|
||||
|
||||
(t/deftest intuit->transaction
|
||||
(t/testing "Should parse dates"
|
||||
(t/is (= #inst "2021-01-01T00:00:00-08:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-01-01")))))
|
||||
(t/is (= #inst "2021-06-01T00:00:00-07:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-06-01")))))))
|
||||
|
||||
|
||||
(t/deftest intuits->transactions
|
||||
(t/testing "should give unique ids to duplicates"
|
||||
(t/is (= ["2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-0-345"
|
||||
"2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-1-345"] (map :transaction/raw-id (sut/intuits->transactions [base-transaction base-transaction]
|
||||
123
|
||||
345))))))
|
||||
66
src/clj/auto_ap/import/manual.clj
Normal file
66
src/clj/auto_ap/import/manual.clj
Normal file
@@ -0,0 +1,66 @@
|
||||
(ns auto-ap.import.manual
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.manual.common :as c]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[unilog.context :as lc]))
|
||||
|
||||
(defn manual-import-batch [transactions user]
|
||||
)
|
||||
|
||||
(def columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code])
|
||||
|
||||
(defn tabulate-data [data]
|
||||
(->>
|
||||
(csv/read-csv data :separator \tab)
|
||||
(drop 1)
|
||||
(map (fn [row]
|
||||
(into {} (->> (map vector columns row)
|
||||
(filter (fn [[k v]] k))))))))
|
||||
|
||||
(defn manual->transaction [{:keys [description-original amount bank-account-code date] :as transaction} bank-account-lookup client-lookup]
|
||||
(-> {:transaction/description-original description-original
|
||||
:transaction/status "POSTED"}
|
||||
(c/assoc-or-error :transaction/client #(if-let [client-id (client-lookup bank-account-code)]
|
||||
client-id
|
||||
(throw (Exception. (str "Cannot find client for bank account code " bank-account-code)))))
|
||||
(c/assoc-or-error :transaction/bank-account #(if-let [bank-account-id (bank-account-lookup bank-account-code)]
|
||||
bank-account-id
|
||||
(throw (Exception. (str "Cannot find bank account by code " bank-account-code)))))
|
||||
(c/assoc-or-error :transaction/date #(coerce/to-date (c/parse-date transaction)))
|
||||
(c/assoc-or-error :transaction/amount #(c/parse-amount transaction))))
|
||||
|
||||
(defn import-batch [transactions user]
|
||||
(lc/with-context {:source "Manual import transactions"}
|
||||
(let [bank-account-code->client (into {}
|
||||
(d/q '[:find ?bac ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/code ?bac]]
|
||||
(d/db conn)))
|
||||
bank-account-code->bank-account (into {}
|
||||
(d/q '[:find ?bac ?ba
|
||||
:in $
|
||||
:where [?ba :bank-account/code ?bac]]
|
||||
(d/db conn)))]
|
||||
(let [import-batch (t/start-import-batch :import-source/manual user)
|
||||
transactions (->> transactions
|
||||
(map (fn [t]
|
||||
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
|
||||
(t/apply-synthetic-ids ))]
|
||||
(try
|
||||
(doseq [transaction transactions]
|
||||
(when-not (seq (:errors transaction))
|
||||
(t/import-transaction! import-batch transaction)))
|
||||
|
||||
(t/finish! import-batch)
|
||||
(assoc (t/get-stats import-batch)
|
||||
:failed-validation (count (filter :errors transactions))
|
||||
:sample-error (first (first (map :errors (filter :errors transactions)))))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(t/get-stats import-batch)))))))
|
||||
53
src/clj/auto_ap/import/manual/common.clj
Normal file
53
src/clj/auto_ap/import/manual/common.clj
Normal file
@@ -0,0 +1,53 @@
|
||||
(ns auto-ap.import.manual.common
|
||||
(:require [auto-ap.datomic.accounts :as d-accounts]
|
||||
[auto-ap.parse.util :as parse-u]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn parse-amount [i]
|
||||
(try
|
||||
(Double/parseDouble (str/replace (or (second
|
||||
(re-matches #"[^0-9\.,\\-]*([0-9\.,\\-]+)[^0-9\.,]*" (:amount i)))
|
||||
"0")
|
||||
#"," ""))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e)))))
|
||||
|
||||
(defn parse-account-numeric-code [i]
|
||||
(try
|
||||
(when-let [account-numeric-code (:account-numeric-code i)]
|
||||
(:db/id (d-accounts/get-account-by-numeric-code-and-sets (Integer/parseInt account-numeric-code)
|
||||
["default"])))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse expense account from value '" (:account-numeric-code i) "'") e)))))
|
||||
|
||||
(defn parse-account-id [i]
|
||||
(try
|
||||
(Long/parseLong (second
|
||||
(re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i))))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e)))))
|
||||
|
||||
(defn parse-date [{:keys [raw-date]}]
|
||||
(when-not
|
||||
(re-find #"\d{1,2}/\d{1,2}/\d{4}" raw-date)
|
||||
(throw (Exception. (str "Date " raw-date " must match MM/dd/yyyy"))))
|
||||
(try
|
||||
|
||||
(parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date)
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse date from '" raw-date "'") e)))))
|
||||
|
||||
(defn parse-or-error [key f]
|
||||
(fn [x]
|
||||
(try
|
||||
(assoc x key (f x))
|
||||
(catch Exception e
|
||||
(update x :errors conj {:info (.getMessage e)
|
||||
:details (str e)})))))
|
||||
|
||||
(defn assoc-or-error [m k f]
|
||||
(try
|
||||
(assoc m k (f))
|
||||
(catch Exception e
|
||||
(update m :errors conj {:info (.getMessage e)
|
||||
:details (str e)}))))
|
||||
60
src/clj/auto_ap/import/plaid.clj
Normal file
60
src/clj/auto_ap/import/plaid.clj
Normal file
@@ -0,0 +1,60 @@
|
||||
(ns auto-ap.import.plaid
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.plaid.core :as p]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.core :as time]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]
|
||||
[clj-time.coerce :as coerce]))
|
||||
|
||||
|
||||
(defn get-plaid-accounts [db]
|
||||
(-> (d/q '[:find ?ba ?c ?external-id ?t
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/plaid-account ?pa]
|
||||
[?pa :plaid-account/external-id ?external-id]
|
||||
[?pi :plaid-item/accounts ?pa]
|
||||
[?pi :plaid-item/access-token ?t]]
|
||||
db )))
|
||||
|
||||
|
||||
(defn plaid->transaction [t]
|
||||
#:transaction {:description-original (:name t)
|
||||
:raw-id (:transaction_id t)
|
||||
:id (digest/sha-256 (:transaction_id t))
|
||||
:amount (double (:amount t))
|
||||
:date (coerce/to-date (auto-ap.time/parse (:date t) auto-ap.time/iso-date))
|
||||
:status "POSTED"})
|
||||
|
||||
|
||||
(defn import-plaid []
|
||||
(lc/with-context {:source "Import plaid transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user")
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
(try
|
||||
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn))
|
||||
transaction (:transactions (p/get-transactions access-token external-id start end))]
|
||||
(when (not (:pending transaction))
|
||||
(t/import-transaction! import-batch (assoc (plaid->transaction transaction)
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))))
|
||||
(t/finish! import-batch)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def import-plaid (allow-once import-plaid))
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 3) import-plaid)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
|
||||
|
||||
336
src/clj/auto_ap/import/transactions.clj
Normal file
336
src/clj/auto_ap/import/transactions.clj
Normal file
@@ -0,0 +1,336 @@
|
||||
(ns auto-ap.import.transactions
|
||||
(:require [auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
|
||||
[auto-ap.datomic.accounts :as a]
|
||||
[auto-ap.datomic.checks :as d-checks]
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
[auto-ap.utils :refer [dollars=]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as t]
|
||||
[clojure.core.cache :as cache]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]))
|
||||
|
||||
(defn rough-match [client-id bank-account-id amount]
|
||||
(if (and client-id bank-account-id amount)
|
||||
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})]
|
||||
(if (= 1 (count matching-checks))
|
||||
(first matching-checks)
|
||||
nil))
|
||||
nil))
|
||||
|
||||
|
||||
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
||||
(log/info "Searching for a matching check for "
|
||||
{:client-id client-id
|
||||
:check-number check-number
|
||||
:bank-account-id bank-account-id
|
||||
:amount amount})
|
||||
(cond (not (and client-id bank-account-id))
|
||||
nil
|
||||
|
||||
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
||||
nil
|
||||
|
||||
check-number
|
||||
(or (-> (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:check-number check-number
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})
|
||||
first
|
||||
first)
|
||||
(rough-match client-id bank-account-id amount))
|
||||
|
||||
:else
|
||||
(rough-match client-id bank-account-id amount)))
|
||||
|
||||
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
||||
(log/info "trying to find uncleared autopay invoices")
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/scheduled-payment ?sd]
|
||||
'[?e :invoice/status :invoice-status/paid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/total ?total]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
||||
considerations
|
||||
))
|
||||
|
||||
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
||||
(log/info "trying to find unpaid invoices for " client-id amount)
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/status :invoice-status/unpaid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/outstanding-balance ?outstanding-balance]
|
||||
'[?e :invoice/date ?d]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
||||
(let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)]
|
||||
(if (= 1 (count considerations))
|
||||
(first considerations)
|
||||
[])))
|
||||
|
||||
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
|
||||
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
||||
(let [payment-id (d/tempid :db.part/user)]
|
||||
(-> tx
|
||||
|
||||
(conj {:payment/bank-account bank-account-id
|
||||
:payment/client client-id
|
||||
:payment/amount (- (:transaction/amount transaction))
|
||||
:payment/vendor vendor
|
||||
:payment/date (:transaction/date transaction)
|
||||
:payment/type :payment-type/debit
|
||||
:payment/status :payment-status/cleared
|
||||
:db/id payment-id})
|
||||
(into (mapcat (fn [[vendor invoice-id invoice-amount]]
|
||||
[{:invoice-payment/invoice invoice-id
|
||||
:invoice-payment/payment payment-id
|
||||
:invoice-payment/amount invoice-amount}
|
||||
{:db/id invoice-id
|
||||
:invoice/outstanding-balance 0.0
|
||||
:invoice/status :invoice-status/paid}])
|
||||
invoice-payments))
|
||||
(update 0 assoc
|
||||
:transaction/payment payment-id
|
||||
:transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/vendor vendor
|
||||
:transaction/location "A")
|
||||
(conj [:reset (:db/id transaction) :transaction/accounts
|
||||
[#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (:transaction/amount transaction))}]]))))
|
||||
|
||||
(defn extract-check-number [{:transaction/keys [description-original]}]
|
||||
|
||||
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
||||
(try
|
||||
(Integer/parseInt check-number)
|
||||
(catch NumberFormatException e
|
||||
nil))
|
||||
nil))
|
||||
|
||||
(defn find-expected-deposit [client-id amount date]
|
||||
(when date
|
||||
(-> (d/q
|
||||
'[:find ?ed
|
||||
:in $ ?c ?a ?d-start
|
||||
:where
|
||||
[?ed :expected-deposit/client ?c]
|
||||
(not [?ed :expected-deposit/status :expected-deposit-status/cleared])
|
||||
[?ed :expected-deposit/date ?d]
|
||||
[(>= ?d ?d-start)]
|
||||
[?ed :expected-deposit/total ?a2]
|
||||
[(auto-ap.utils/dollars= ?a2 ?a)]
|
||||
]
|
||||
(d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
|
||||
first
|
||||
first)))
|
||||
|
||||
|
||||
(defn categorize-transaction [transaction bank-account existing]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction)))
|
||||
:suppressed
|
||||
|
||||
(existing (:transaction/id transaction))
|
||||
:extant
|
||||
|
||||
(not (:transaction/client transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/bank-account transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/id transaction))
|
||||
:error
|
||||
|
||||
(not= "POSTED" (:transaction/status transaction))
|
||||
:not-posted
|
||||
|
||||
(and (:bank-account/start-date bank-account)
|
||||
(not (t/after? (coerce/to-date-time (:transaction/date transaction))
|
||||
(-> bank-account :bank-account/start-date coerce/to-date-time))))
|
||||
:not-ready
|
||||
|
||||
:else
|
||||
:import)))
|
||||
|
||||
(defn transaction->txs [transaction bank-account apply-rules]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(into []
|
||||
(let [{:transaction/keys [amount id date]} transaction
|
||||
check-number (extract-check-number transaction)
|
||||
existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit (:db/id client) amount (coerce/to-date-time date)))]
|
||||
(cond->
|
||||
[(assoc transaction :transaction/approval-status :transaction-approval-status/unapproved)]
|
||||
check-number (update 0 #(assoc % :transaction/check-number check-number))
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils))))))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
(defn get-existing [bank-account]
|
||||
(log/info "looking up bank account data for" bank-account)
|
||||
(into {}
|
||||
(d/query {:query {:find ['?tid '?as2]
|
||||
:in ['$ '?ba]
|
||||
:where ['[?e :transaction/bank-account ?ba]
|
||||
'[?e :transaction/id ?tid]
|
||||
'[?e :transaction/approval-status ?as]
|
||||
'[?as :db/ident ?as2]]}
|
||||
:args [(d/db (d/connect uri)) bank-account]})))
|
||||
|
||||
(defprotocol ImportBatch
|
||||
(import-transaction! [this transaction])
|
||||
(get-stats [this ])
|
||||
(finish! [this])
|
||||
(fail! [this error]))
|
||||
|
||||
(defn start-import-batch [source user]
|
||||
(let [stats (atom {:import-batch/imported 0
|
||||
:import-batch/suppressed 0
|
||||
:import-batch/error 0
|
||||
:import-batch/not-ready 0
|
||||
:import-batch/extant 0})
|
||||
extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 ))
|
||||
import-id (get (:tempids @(d/transact conn [{:db/id "import-batch"
|
||||
:import-batch/date (coerce/to-date (t/now))
|
||||
:import-batch/source source
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch")
|
||||
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
|
||||
(log/info "Importing transactions from " source)
|
||||
|
||||
(reify ImportBatch
|
||||
(import-transaction! [this transaction]
|
||||
(let [bank-account (d/pull (d/db conn)
|
||||
[:bank-account/code
|
||||
:db/id
|
||||
:bank-account/locations
|
||||
:bank-account/start-date
|
||||
{:client/_bank-accounts [:client/code :client/locations :db/id]} ]
|
||||
(:transaction/bank-account transaction))
|
||||
extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing)
|
||||
(:transaction/bank-account transaction))
|
||||
action (categorize-transaction transaction bank-account extant)
|
||||
transaction (assoc transaction :import-batch/_entry import-id)]
|
||||
(swap! stats
|
||||
#(update % (condp = action
|
||||
:import :import-batch/imported
|
||||
:extant :import-batch/extant
|
||||
:suppressed :import-batch/suppressed
|
||||
:error :import-batch/error
|
||||
:not-read :import-batch/not-ready) inc))
|
||||
(when (= :import action)
|
||||
(audit-transact (transaction->txs transaction bank-account rule-applying-function)
|
||||
{:user/name user
|
||||
:user/role ":admin"}))))
|
||||
|
||||
(get-stats [this]
|
||||
@stats)
|
||||
|
||||
(fail! [this error]
|
||||
(log/errorf "Couldn't complete import %d with error." import-id)
|
||||
(log/error error)
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
:import-batch/status :import-status/completed
|
||||
:import-batch/error-message (str error)}
|
||||
@stats)]))
|
||||
|
||||
(finish! [this]
|
||||
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
|
||||
:import-batch/status :import-status/completed}
|
||||
@stats)])))))
|
||||
|
||||
|
||||
|
||||
(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index]
|
||||
(str (str (some-> date coerce/to-date-time auto-ap.time/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
|
||||
|
||||
(defn apply-synthetic-ids [transactions]
|
||||
(->> transactions
|
||||
(group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client]))
|
||||
(vals)
|
||||
(mapcat (fn [group]
|
||||
(map (fn [index transaction]
|
||||
(let [raw-id (synthetic-key transaction index)]
|
||||
(assoc transaction
|
||||
:transaction/id (digest/sha-256 raw-id)
|
||||
:transaction/raw-id raw-id)))
|
||||
(range)
|
||||
group)))))
|
||||
|
||||
70
src/clj/auto_ap/import/yodlee.clj
Normal file
70
src/clj/auto_ap/import/yodlee.clj
Normal file
@@ -0,0 +1,70 @@
|
||||
(ns auto-ap.import.yodlee
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.time :as atime]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.string :as str]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]
|
||||
[clojure.tools.logging :as log]))
|
||||
|
||||
(defn yodlee->transaction [transaction]
|
||||
(let [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
{amount :amount} :amount
|
||||
{description-original :original
|
||||
description-simple :simple} :description
|
||||
{merchant-id :id
|
||||
merchant-name :name} :merchant
|
||||
base-type :baseType
|
||||
type :type
|
||||
status :status} transaction
|
||||
amount (if (= "DEBIT" base-type)
|
||||
(- amount)
|
||||
amount)
|
||||
date (atime/parse date "YYYY-MM-dd")]
|
||||
#:transaction
|
||||
{:post-date (coerce/to-date (atime/parse post-date "YYYY-MM-dd"))
|
||||
:id (sha-256 (str id))
|
||||
:raw-id (str id)
|
||||
:account-id account-id
|
||||
:date (coerce/to-date date)
|
||||
:amount (double amount)
|
||||
:description-original (some-> description-original (str/replace #"\s+" " "))
|
||||
:description-simple (some-> description-simple (str/replace #"\s+" " "))
|
||||
:type type
|
||||
:status status}))
|
||||
|
||||
(defn import-yodlee []
|
||||
(lc/with-context {:source "Import yodlee transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")]
|
||||
(try
|
||||
(let [account-lookup (d/q '[:find ?ya ?ba ?c
|
||||
:in $
|
||||
:where [?ba :bank-account/yodlee-account-id ?ya]
|
||||
[?c :client/bank-accounts ?ba]]
|
||||
(d/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-id] account-lookup
|
||||
transaction (client/get-specific-transactions yodlee-account)]
|
||||
(log/info "importing")
|
||||
(t/import-transaction! import-batch (assoc (yodlee->transaction transaction)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client client-id)))
|
||||
|
||||
(t/finish! import-batch))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def import-yodlee (allow-once import-yodlee))
|
||||
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 4) import-yodlee)
|
||||
:stop (scheduler/stop import-worker))
|
||||
45
src/clj/auto_ap/import/yodlee2.clj
Normal file
45
src/clj/auto_ap/import/yodlee2.clj
Normal file
@@ -0,0 +1,45 @@
|
||||
(ns auto-ap.import.yodlee2
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.import.yodlee :as y]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.yodlee.core2 :as client2]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(defn import-yodlee2 []
|
||||
(lc/with-context {:source "Import yodlee2 transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
|
||||
(try
|
||||
(let [account-lookup (d/q '[:find ?ya ?ba ?cd
|
||||
:in $
|
||||
:where
|
||||
[?ba :bank-account/yodlee-account ?y]
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?c :client/code ?cd]
|
||||
[?y :yodlee-account/id ?ya]]
|
||||
(d/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-code] account-lookup
|
||||
transaction (client2/get-specific-transactions client-code yodlee-account)]
|
||||
(t/import-transaction! import-batch (assoc (y/yodlee->transaction transaction)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client [:client/code client-code])))
|
||||
|
||||
(t/finish! import-batch))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
|
||||
|
||||
(def import-yodlee2 (allow-once import-yodlee2))
|
||||
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 4) import-yodlee2)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
(mount/defstate account-worker
|
||||
:start (scheduler/every (* 5 60 1000) client2/upsert-accounts)
|
||||
:stop (scheduler/stop account-worker))
|
||||
@@ -1,95 +0,0 @@
|
||||
(ns auto-ap.intuit.import
|
||||
(:require [amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic :refer [conn remove-nils]]
|
||||
[auto-ap.intuit.core :as i]
|
||||
[auto-ap.utils :refer [by allow-once]]
|
||||
[auto-ap.yodlee.import :as y]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clj-time.format :as f]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(def client-whitelist #{"PPFB"})
|
||||
|
||||
(defn get-intuit-bank-accounts [db]
|
||||
(-> (d/q '[:find ?ba ?external-id
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/intuit-bank-account ?iab]
|
||||
[?iab :intuit-bank-account/external-id ?external-id]]
|
||||
db #_client-whitelist)))
|
||||
|
||||
|
||||
(defn upsert-transactions []
|
||||
(lc/with-context {:source "Importing intuit transactions"}
|
||||
(let [db (d/db conn)]
|
||||
(->>
|
||||
(for [[bank-account external-id] (get-intuit-bank-accounts db)
|
||||
:let [bank-account (d/entity db bank-account)
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))
|
||||
_ (log/infof "importing from %s to %s for %s" start end external-id)]
|
||||
transaction (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
|
||||
(auto-ap.time/unparse end auto-ap.time/iso-date)
|
||||
external-id)]
|
||||
{:client-id (:db/id (:client/_bank-accounts bank-account))
|
||||
:bank-account-id (:db/id bank-account)
|
||||
:description-original (:Memo/Description transaction)
|
||||
:amount (Double/parseDouble (:Amount transaction))
|
||||
:date (auto-ap.time/parse (:Date transaction) auto-ap.time/iso-date)
|
||||
:status "posted"})
|
||||
(y/grouped-import :import-source/intuit "Automated Intuit User"))
|
||||
(log/info "Intuit transactions imported"))))
|
||||
|
||||
(def upsert-transactions (allow-once upsert-transactions))
|
||||
|
||||
(defn dry-run-upsert-transactions []
|
||||
(let [db (d/db conn)]
|
||||
(clojure.data.csv/write-csv
|
||||
*out*
|
||||
(->>
|
||||
(for [[bank-account external-id] (get-intuit-bank-accounts db)
|
||||
:let [bank-account (d/entity db bank-account)
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
|
||||
(auto-ap.time/unparse end auto-ap.time/iso-date)
|
||||
external-id)]
|
||||
{:client-id (:db/id (:client/_bank-accounts bank-account))
|
||||
:client-code (:client/code (:client/_bank-accounts bank-account))
|
||||
:bank-account-id (:db/id bank-account)
|
||||
:bank-account-code (:bank-account/code bank-account)
|
||||
:external-id external-id
|
||||
:description-original (:Memo/Description r)
|
||||
:amount (Double/parseDouble (:Amount r))
|
||||
:date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
|
||||
:status "posted"})
|
||||
(y/grouped-new)
|
||||
:import
|
||||
(mapcat identity)
|
||||
(map (juxt #(:bank-account/code (d/entity db(:transaction/bank-account %))) :transaction/id :transaction/raw-id :transaction/amount :transaction/description-original #(auto-ap.time/unparse-local (coerce/to-date-time (:transaction/date %)) auto-ap.time/normal-date) )))
|
||||
:separator \tab)))
|
||||
|
||||
|
||||
(defn upsert-accounts []
|
||||
(let [token (i/get-fresh-access-token)
|
||||
bank-accounts (i/get-bank-accounts token)]
|
||||
@(d/transact conn (mapv
|
||||
(fn [ba]
|
||||
{:intuit-bank-account/external-id (:name ba)
|
||||
:intuit-bank-account/name (:name ba)})
|
||||
bank-accounts))))
|
||||
|
||||
(mount/defstate upsert-transaction-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) upsert-transactions)
|
||||
:stop (scheduler/stop upsert-transaction-worker))
|
||||
|
||||
(mount/defstate upsert-account-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) upsert-accounts)
|
||||
:stop (scheduler/stop upsert-account-worker))
|
||||
@@ -58,6 +58,7 @@
|
||||
:body))
|
||||
|
||||
(defn get-transactions [access-token account-id start end]
|
||||
(log/infof "looking up transactions from %s to %s for %s" start end account-id)
|
||||
(-> (client/post (str base-url "/transactions/get")
|
||||
{:as :json
|
||||
:headers {"Content-Type" "application/json"}
|
||||
|
||||
@@ -9,10 +9,11 @@
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
[yang.scheduler :as scheduler]
|
||||
[clj-time.coerce :as coerce]))
|
||||
|
||||
(defn get-plaid-accounts [db]
|
||||
(-> (d/q '[:find ?ba ?external-id ?t
|
||||
(-> (d/q '[:find ?ba ?c ?external-id ?t
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
@@ -22,47 +23,36 @@
|
||||
[?pi :plaid-item/access-token ?t]]
|
||||
db )))
|
||||
|
||||
(defn import-plaid
|
||||
[]
|
||||
(lc/with-context {:source "Importing plaid transactions"}
|
||||
(let [db (d/db conn)
|
||||
import-id (y/start-import :import-source/plaid "Automated Plaid User")]
|
||||
|
||||
(defn plaid->transaction [t]
|
||||
#:transaction {:description-original (:name t)
|
||||
:raw-id (:transaction_id t)
|
||||
:id (digest/sha-256 (:transaction_id t))
|
||||
:amount (double (:amount t))
|
||||
:date (coerce/to-date (auto-ap.time/parse (:date t) auto-ap.time/iso-date))
|
||||
:status "POSTED"})
|
||||
|
||||
|
||||
(defn import-plaid []
|
||||
(lc/with-context {:source "Import plaid transactions"}
|
||||
(let [import-batch (y/start-import-batch :import-source/plaid "Automated plaid user")
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
(try
|
||||
(let [result (->>
|
||||
(for [[bank-account external-id access-token] (get-plaid-accounts db)
|
||||
:let [end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))
|
||||
_ (log/infof "importing from %s to %s for %s" start end external-id)
|
||||
transactions (p/get-transactions access-token external-id start end)
|
||||
transactions (->> transactions
|
||||
:transactions
|
||||
(filter (fn [t]
|
||||
(not (:pending t))))
|
||||
(mapv (fn [t]
|
||||
{:description {:original (:name t)
|
||||
:simple (:name t)}
|
||||
:id (:transaction_id t)
|
||||
:amount {:amount (:amount t)}
|
||||
:date (:date t)
|
||||
:status "POSTED"}))
|
||||
)]]
|
||||
|
||||
|
||||
(y/import-for-bank-account transactions bank-account import-id))
|
||||
y/aggregate-results)]
|
||||
(log/info "Plaid transactions imported" result)
|
||||
(y/finish-import (assoc result :db/id import-id)))
|
||||
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn))
|
||||
transaction (:transactions (p/get-transactions access-token external-id start end))]
|
||||
(when (not (:pending transaction))
|
||||
(y/import-transaction! import-batch (assoc (plaid->transaction transaction)
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))))
|
||||
(y/finish! import-batch)
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
(y/finish-import {:db/id import-id}))))))
|
||||
(y/fail! import-batch e))))))
|
||||
|
||||
(defn do-import []
|
||||
(import-plaid))
|
||||
|
||||
(def do-import (allow-once do-import))
|
||||
(def import-plaid (allow-once import-plaid))
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 3) do-import)
|
||||
:start (scheduler/every (* 1000 60 60 3) import-plaid)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
(ns auto-ap.routes.events
|
||||
(:require [auto-ap.routes.utils :refer [wrap-secure]]
|
||||
[auto-ap.yodlee.import :as yodlee-import]
|
||||
[config.core :refer [env]]
|
||||
[clj-http.client :as http]
|
||||
[clj-time.coerce :as c]
|
||||
|
||||
@@ -5,13 +5,14 @@
|
||||
[auto-ap.datomic.invoices :as d-invoices]
|
||||
[auto-ap.datomic.vendors :as d-vendors]
|
||||
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
|
||||
[auto-ap.import.manual :as manual]
|
||||
[auto-ap.import.manual.common :as c]
|
||||
[auto-ap.logging :refer [info-event]]
|
||||
[auto-ap.parse :as parse]
|
||||
[auto-ap.parse.util :as parse-u]
|
||||
[auto-ap.routes.utils :refer [wrap-secure]]
|
||||
[auto-ap.time-utils :refer [next-dom]]
|
||||
[auto-ap.utils :refer [by]]
|
||||
[auto-ap.yodlee.import :refer [grouped-import] :as y]
|
||||
[clj-time.coerce :as coerce :refer [to-date]]
|
||||
[clj-time.core :as time]
|
||||
[clojure.data.csv :as csv]
|
||||
@@ -73,50 +74,6 @@
|
||||
(get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor))
|
||||
client-id))
|
||||
|
||||
(defn parse-amount [i]
|
||||
(try
|
||||
(Double/parseDouble (str/replace (or (second
|
||||
(re-matches #"[^0-9\.,\\-]*([0-9\.,\\-]+)[^0-9\.,]*" (:amount i)))
|
||||
"0")
|
||||
#"," ""))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e)))))
|
||||
|
||||
(defn parse-account-numeric-code [i]
|
||||
(try
|
||||
(when-let [account-numeric-code (:account-numeric-code i)]
|
||||
(:db/id (d-accounts/get-account-by-numeric-code-and-sets (Integer/parseInt account-numeric-code)
|
||||
["default"])))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse expense account from value '" (:account-numeric-code i) "'") e)))))
|
||||
|
||||
(defn parse-account-id [i]
|
||||
(try
|
||||
(Long/parseLong (second
|
||||
(re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i))))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e)))))
|
||||
|
||||
|
||||
|
||||
(defn parse-date [{:keys [raw-date]}]
|
||||
(when-not
|
||||
(re-find #"\d{1,2}/\d{1,2}/\d{4}" raw-date)
|
||||
(throw (Exception. (str "Date " raw-date " must match MM/dd/yyyy"))))
|
||||
(try
|
||||
|
||||
(parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date)
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse date from '" raw-date "'") e)))))
|
||||
|
||||
(defn parse-or-error [key f]
|
||||
(fn [x]
|
||||
(try
|
||||
(assoc x key (f x))
|
||||
(catch Exception e
|
||||
(update x :errors conj {:info (.getMessage e)
|
||||
:details (str e)})))))
|
||||
|
||||
(defn parse-invoice-rows [excel-rows]
|
||||
(let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code]
|
||||
all-vendors (by :vendor/name (d-vendors/get-graphql {}))
|
||||
@@ -127,15 +84,15 @@
|
||||
(map #(into {} (map (fn [c k] [k c] ) % columns)))
|
||||
(map reset-id)
|
||||
(map assoc-client-code)
|
||||
(map (parse-or-error :client-id #(parse-client % all-clients)))
|
||||
(map (parse-or-error :vendor #(parse-vendor % all-vendors)))
|
||||
(map (parse-or-error :vendor-id #(parse-vendor-id %)))
|
||||
(map (parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
|
||||
(map (parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
|
||||
(map (parse-or-error :account-id parse-account-numeric-code))
|
||||
(map (parse-or-error :invoice-number parse-invoice-number))
|
||||
(map (parse-or-error :total parse-amount))
|
||||
(map (parse-or-error :date parse-date)))]
|
||||
(map (c/parse-or-error :client-id #(parse-client % all-clients)))
|
||||
(map (c/parse-or-error :vendor #(parse-vendor % all-vendors)))
|
||||
(map (c/parse-or-error :vendor-id #(parse-vendor-id %)))
|
||||
(map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
|
||||
(map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
|
||||
(map (c/parse-or-error :account-id c/parse-account-numeric-code))
|
||||
(map (c/parse-or-error :invoice-number parse-invoice-number))
|
||||
(map (c/parse-or-error :total c/parse-amount))
|
||||
(map (c/parse-or-error :date c/parse-date)))]
|
||||
|
||||
|
||||
rows))
|
||||
@@ -428,6 +385,9 @@
|
||||
rows)]
|
||||
@(d/transact (d/connect uri) txes)))
|
||||
|
||||
|
||||
|
||||
|
||||
(defroutes routes
|
||||
(wrap-routes
|
||||
(context "/" []
|
||||
@@ -435,55 +395,18 @@
|
||||
(POST "/batch-upload"
|
||||
{{:keys [data]} :edn-params user :identity}
|
||||
(assert-admin user)
|
||||
|
||||
(try
|
||||
#_(throw (Exception. "Unexpected error"))
|
||||
(let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code]
|
||||
all-clients (d-clients/get-all)
|
||||
all-bank-accounts (mapcat :client/bank-accounts all-clients)
|
||||
all-clients (merge (by :client/code all-clients) (by :client/name all-clients))
|
||||
all-bank-accounts (merge (by :bank-account/code all-bank-accounts))
|
||||
rows (->> (str/split data #"\n" )
|
||||
(drop 1)
|
||||
(map #(str/split % #"\t"))
|
||||
(map #(into {} (filter identity (map (fn [c k] [k c] ) % columns))))
|
||||
(map (parse-or-error :amount parse-amount))
|
||||
(map (parse-or-error :date parse-date))
|
||||
(map (fn [{:keys [bank-account-code] :as row}]
|
||||
(if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))]
|
||||
(assoc row :bank-account-id bank-account-id)
|
||||
(update row :errors conj {:info (str "Cannot find bank by code " bank-account-code)
|
||||
:details (str "Cannot find bank by code " bank-account-code)}))))
|
||||
(map (fn [{:keys [client-code] :as row}]
|
||||
(if-let [client-id (:db/id (all-clients client-code))]
|
||||
(assoc row :client-id client-id)
|
||||
(update row :errors conj {:info (str "Cannot find client by code " client-code)
|
||||
:details (str "Cannot find client by code " client-code)})))))
|
||||
error-rows (filter :errors rows)
|
||||
_ (log/info "Importing " (count rows) "raw transactions")
|
||||
raw-transactions (vec (->> rows
|
||||
(filter #(not (seq (:errors %))) )
|
||||
(map (fn [row]
|
||||
(select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id])))))
|
||||
import-id (y/start-import :import-source/manual (:user/name user))
|
||||
result (grouped-import raw-transactions import-id)]
|
||||
(y/finish-import {:db/id import-id
|
||||
:import-batch/imported (count (:import result))
|
||||
:import-batch/suppressed (count (:suppressed result))
|
||||
:import-batch/extant (count (:extant result))})
|
||||
|
||||
{:status 200
|
||||
:body (pr-str {:imported (count (:import result))
|
||||
:errors (map #(dissoc % :date) error-rows)})
|
||||
:headers {"Content-Type" "application/edn"}})
|
||||
(let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))]
|
||||
{:status 200
|
||||
:body (pr-str stats)
|
||||
:headers {"Content-Type" "application/edn"}})
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
{:status 500
|
||||
:body (pr-str {:message (.getMessage e)
|
||||
:error (.toString e)
|
||||
:data (ex-data e)})
|
||||
:headers {"Content-Type" "application/edn"}}))
|
||||
))
|
||||
:headers {"Content-Type" "application/edn"}}))))
|
||||
|
||||
(context "/invoices" []
|
||||
(POST "/upload"
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
(:require [auto-ap.handler :refer [app]]
|
||||
[auto-ap.ledger :as ledger]
|
||||
[auto-ap.yodlee.core]
|
||||
[auto-ap.yodlee.core2 :as yodlee2]
|
||||
[auto-ap.graphql.clients :as gq-clients]
|
||||
[auto-ap.background.invoices]
|
||||
[auto-ap.background.requests :as requests]
|
||||
@@ -10,8 +9,10 @@
|
||||
[auto-ap.background.vendor :as vendor]
|
||||
[auto-ap.square.core :as square]
|
||||
[auto-ap.datomic.migrate :as migrate]
|
||||
[auto-ap.yodlee.import :as yodlee]
|
||||
[auto-ap.intuit.import :as intuit]
|
||||
[auto-ap.import.yodlee :as yodlee]
|
||||
[auto-ap.import.yodlee2 :as yodlee2]
|
||||
[auto-ap.import.intuit :as intuit]
|
||||
[auto-ap.import.plaid :as plaid]
|
||||
[nrepl.server :refer [start-server stop-server]]
|
||||
[config.core :refer [env]]
|
||||
[ring.adapter.jetty :refer [run-jetty]]
|
||||
@@ -37,8 +38,6 @@
|
||||
(not (env :run-web? )) (into [#'jetty])
|
||||
(not (env :run-background?)) (into [#'square/square-loader
|
||||
#'vendor/refresh-vendor-usages-worker
|
||||
#'intuit/upsert-transaction-worker
|
||||
#'intuit/upsert-account-worker
|
||||
#'ledger/touch-broken-ledger-worker
|
||||
#'ledger/process-txes-worker
|
||||
#'ledger/ledger-reconciliation-worker
|
||||
@@ -46,8 +45,14 @@
|
||||
#'sysco/sysco-invoice-importer
|
||||
#'auto-ap.background.invoices/close-auto-invoices-worker
|
||||
#'gq-clients/current-balance-worker
|
||||
#'yodlee/import-transaction-worker
|
||||
#'yodlee2/yodlee-sync-worker
|
||||
|
||||
#'yodlee/import-worker
|
||||
#'yodlee2/import-worker
|
||||
#'yodlee2/account-worker
|
||||
#'intuit/import-worker
|
||||
#'intuit/account-worker
|
||||
#'plaid/import-worker
|
||||
|
||||
#'migrate/migrate-start]))]
|
||||
(log/info "starting without " without)
|
||||
|
||||
|
||||
@@ -209,6 +209,7 @@
|
||||
|
||||
|
||||
(defn get-specific-transactions [account]
|
||||
(log/infof "Getting yodlee transactions for account %s" account)
|
||||
(let [cob-session (login-cobrand)
|
||||
user-session (login-user cob-session)
|
||||
batch-size 100
|
||||
|
||||
@@ -322,9 +322,7 @@
|
||||
(log/info "Current yodlee state is " result)
|
||||
@(d/transact conn result))))
|
||||
|
||||
(mount/defstate yodlee-sync-worker
|
||||
:start (scheduler/every (* 5 60 1000) upsert-accounts)
|
||||
:stop (scheduler/stop upsert-accounts))
|
||||
|
||||
|
||||
(defn update-yodlee [id]
|
||||
(update-provider-account id)
|
||||
|
||||
@@ -1,485 +0,0 @@
|
||||
(ns auto-ap.yodlee.import
|
||||
(:require
|
||||
[auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
|
||||
[auto-ap.datomic.accounts :as a]
|
||||
[auto-ap.datomic.checks :as d-checks]
|
||||
[auto-ap.datomic.clients :as d-clients]
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
[auto-ap.time :as time]
|
||||
[auto-ap.utils :refer [allow-once by dollars=]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[auto-ap.yodlee.core2 :as client2]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as t]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(defn rough-match [client-id bank-account-id amount]
|
||||
(if (and client-id bank-account-id amount)
|
||||
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})]
|
||||
(if (= 1 (count matching-checks))
|
||||
(first matching-checks)
|
||||
nil))
|
||||
nil))
|
||||
|
||||
|
||||
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
||||
(log/info "Searching for a matching check for "
|
||||
{:client-id client-id
|
||||
:check-number check-number
|
||||
:bank-account-id bank-account-id
|
||||
:amount amount})
|
||||
(cond (not (and client-id bank-account-id))
|
||||
nil
|
||||
|
||||
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
||||
nil
|
||||
|
||||
check-number
|
||||
(or (-> (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:check-number check-number
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})
|
||||
first
|
||||
first)
|
||||
(rough-match client-id bank-account-id amount))
|
||||
|
||||
:else
|
||||
(rough-match client-id bank-account-id amount)))
|
||||
|
||||
(defn aggregate-results [results]
|
||||
(reduce
|
||||
(fn [acc result]
|
||||
(merge-with + acc
|
||||
{:import-batch/imported (count (:import result))
|
||||
:import-batch/extant (count (:extant result))
|
||||
:import-batch/suppressed (count (:suppressed result))}))
|
||||
|
||||
{:import-batch/imported 0
|
||||
:import-batch/extant 0
|
||||
:import-batch/suppressed 0}
|
||||
results))
|
||||
|
||||
|
||||
|
||||
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
||||
(log/info "trying to find uncleared autopay invoices")
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/scheduled-payment ?sd]
|
||||
'[?e :invoice/status :invoice-status/paid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/total ?total]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
||||
considerations
|
||||
))
|
||||
|
||||
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
||||
(log/info "trying to find unpaid invoices for " client-id amount)
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/status :invoice-status/unpaid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/outstanding-balance ?outstanding-balance]
|
||||
'[?e :invoice/date ?d]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
||||
(let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)]
|
||||
(if (= 1 (count considerations))
|
||||
(first considerations)
|
||||
[])))
|
||||
|
||||
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
|
||||
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
||||
(let [payment-id (d/tempid :db.part/user)]
|
||||
(-> tx
|
||||
|
||||
(conj {:payment/bank-account bank-account-id
|
||||
:payment/client client-id
|
||||
:payment/amount (- (:transaction/amount transaction))
|
||||
:payment/vendor vendor
|
||||
:payment/date (:transaction/date transaction)
|
||||
:payment/type :payment-type/debit
|
||||
:payment/status :payment-status/cleared
|
||||
:db/id payment-id})
|
||||
(into (mapcat (fn [[vendor invoice-id invoice-amount]]
|
||||
[{:invoice-payment/invoice invoice-id
|
||||
:invoice-payment/payment payment-id
|
||||
:invoice-payment/amount invoice-amount}
|
||||
{:db/id invoice-id
|
||||
:invoice/outstanding-balance 0.0
|
||||
:invoice/status :invoice-status/paid}])
|
||||
invoice-payments))
|
||||
(update 0 assoc
|
||||
:transaction/payment payment-id
|
||||
:transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/vendor vendor
|
||||
:transaction/location "A")
|
||||
(conj [:reset (:db/id transaction) :transaction/accounts
|
||||
[#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (:transaction/amount transaction))}]]))))
|
||||
|
||||
(defn extract-check-number [{{description-original :original} :description}]
|
||||
|
||||
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
||||
(try
|
||||
(Integer/parseInt check-number)
|
||||
(catch NumberFormatException e
|
||||
nil))
|
||||
nil))
|
||||
|
||||
(defn find-expected-deposit [client-id amount date]
|
||||
(when date
|
||||
(-> (d/q
|
||||
'[:find ?ed
|
||||
:in $ ?c ?a ?d-start
|
||||
:where
|
||||
[?ed :expected-deposit/client ?c]
|
||||
(not [?ed :expected-deposit/status :expected-deposit-status/cleared])
|
||||
[?ed :expected-deposit/date ?d]
|
||||
[(>= ?d ?d-start)]
|
||||
[?ed :expected-deposit/total ?a2]
|
||||
[(auto-ap.utils/dollars= ?a2 ?a)]
|
||||
]
|
||||
(d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
|
||||
first
|
||||
first)))
|
||||
|
||||
(defn transactions->txs [transactions bank-account import-id apply-rules existing]
|
||||
(let [client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
bank-account-id (:db/id bank-account)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
||||
grouped-transactions (group-by
|
||||
(fn [{id :id
|
||||
date :date
|
||||
status :status :as transaction}]
|
||||
(let [date (time/parse date "YYYY-MM-dd")
|
||||
id (sha-256 (str id))]
|
||||
(cond (= :transaction-approval-status/suppressed (existing id))
|
||||
:suppressed
|
||||
|
||||
(existing id)
|
||||
:extant
|
||||
|
||||
(not client)
|
||||
:error
|
||||
|
||||
(not bank-account)
|
||||
:error
|
||||
|
||||
(not= "POSTED" status)
|
||||
:not-posted
|
||||
|
||||
(and (:bank-account/start-date bank-account)
|
||||
(not (t/after? date (-> bank-account :bank-account/start-date coerce/to-date-time))))
|
||||
:not-ready
|
||||
|
||||
:else
|
||||
:import)))
|
||||
transactions)]
|
||||
|
||||
(update grouped-transactions :import
|
||||
(fn [transactions]
|
||||
(into []
|
||||
|
||||
(for [transaction transactions
|
||||
:let [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
{amount :amount} :amount
|
||||
{description-original :original
|
||||
description-simple :simple} :description
|
||||
{merchant-id :id
|
||||
merchant-name :name} :merchant
|
||||
base-type :baseType
|
||||
type :type
|
||||
status :status}
|
||||
transaction
|
||||
amount (if (= "DEBIT" base-type)
|
||||
(- amount)
|
||||
amount)
|
||||
check-number (extract-check-number transaction)
|
||||
date (time/parse date "YYYY-MM-dd")]]
|
||||
(let [
|
||||
existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit (:db/id client) amount date))]
|
||||
(cond->
|
||||
[#:transaction
|
||||
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
|
||||
:id (sha-256 (str id))
|
||||
:raw-id (str id)
|
||||
:account-id account-id
|
||||
:date (coerce/to-date date)
|
||||
:amount (double amount)
|
||||
:import-batch/_entry import-id
|
||||
:description-original (some-> description-original (str/replace #"\s+" " "))
|
||||
:description-simple (some-> description-simple (str/replace #"\s+" " "))
|
||||
:approval-status :transaction-approval-status/unapproved
|
||||
:type type
|
||||
:status status
|
||||
:client client-id
|
||||
:check-number check-number
|
||||
:bank-account bank-account-id}]
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils)))))))))
|
||||
|
||||
|
||||
(defn batch-transact [transactions]
|
||||
(reduce (fn [acc batch]
|
||||
(into acc (->> batch
|
||||
(d/transact (d/connect uri) )
|
||||
(deref)
|
||||
:tempids
|
||||
vals)))
|
||||
[]
|
||||
(partition-all 100 transactions)))
|
||||
|
||||
|
||||
|
||||
(defn get-existing [bank-account]
|
||||
(into {}
|
||||
(d/query {:query {:find ['?tid '?as2]
|
||||
:in ['$ '?ba]
|
||||
:where ['[?e :transaction/bank-account ?ba]
|
||||
'[?e :transaction/id ?tid]
|
||||
'[?e :transaction/approval-status ?as]
|
||||
'[?as :db/ident ?as2]]}
|
||||
:args [(d/db (d/connect uri)) bank-account]})))
|
||||
|
||||
(defn get-all-bank-accounts []
|
||||
(->> (d-clients/get-all)
|
||||
(mapcat (fn [client]
|
||||
(->> client
|
||||
:client/bank-accounts
|
||||
#_(filter :bank-account/yodlee-account-id)
|
||||
(filter :db/id)
|
||||
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
|
||||
(assoc bank-account
|
||||
:client/_bank-accounts client))))))))
|
||||
(defn start-import [source user]
|
||||
(get (:tempids @(d/transact conn [{:db/id "import-batch"
|
||||
:import-batch/date (coerce/to-date (t/now))
|
||||
:import-batch/source source
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch"))
|
||||
|
||||
(defn finish-import [i]
|
||||
@(d/transact conn [(assoc i :import-batch/status :import-status/completed)]))
|
||||
|
||||
(defn import-for-bank-account [transactions bank-account-id import-id]
|
||||
(let [all-rules (tr/get-all)
|
||||
bank-account (d/pull (d/db conn)
|
||||
[:bank-account/code
|
||||
:db/id
|
||||
:bank-account/locations
|
||||
:bank-account/start-date
|
||||
{:client/_bank-accounts [:client/code :client/locations :db/id]} ]
|
||||
bank-account-id)
|
||||
_ (log/infof "Importing %d transactions for bank account %s for client %s" (count transactions) (-> bank-account :bank-account/code) (-> bank-account :client/_bank-accounts :client/code))
|
||||
transactions (transactions->txs transactions bank-account import-id (rm/rule-applying-fn all-rules) (get-existing bank-account-id))]
|
||||
(doseq [tx (:import transactions)]
|
||||
(audit-transact tx {:user/name "Yodlee import"
|
||||
:user/role ":admin"}))
|
||||
(log/infof "Completed import for %s" bank-account)
|
||||
transactions))
|
||||
|
||||
(defn grouped-import [source user manual-transactions]
|
||||
(let [import-id (start-import source user)]
|
||||
(lc/with-context {:import-id import-id}
|
||||
(try
|
||||
(let [transactions-by-bank-account (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
||||
:bank-account-id bank-account-id
|
||||
:date (time/unparse date "YYYY-MM-dd")
|
||||
:amount {:amount amount}
|
||||
:description {:original description-original
|
||||
:simple high-level-category}
|
||||
:status "POSTED"})
|
||||
(range)
|
||||
transaction-group)))
|
||||
(group-by :bank-account-id))
|
||||
result (->> transactions-by-bank-account
|
||||
(filter (fn [[bank-account]]
|
||||
bank-account))
|
||||
(map (fn [[bank-account transactions]]
|
||||
(import-for-bank-account transactions bank-account import-id)))
|
||||
aggregate-results)]
|
||||
(finish-import (assoc result :db/id import-id)))
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
(finish-import {:db/id import-id})
|
||||
)))))
|
||||
|
||||
(defn grouped-new [manual-transactions]
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
||||
:bank-account-id bank-account-id
|
||||
:date (time/unparse date "YYYY-MM-dd")
|
||||
:amount {:amount amount}
|
||||
:description {:original description-original
|
||||
:simple high-level-category}
|
||||
:status "POSTED"})
|
||||
(range)
|
||||
transaction-group))))
|
||||
all-rules (tr/get-all)
|
||||
all-bank-accounts (by :db/id (get-all-bank-accounts))
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)
|
||||
transactions (transactions->txs transformed-transactions nil transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
transactions))
|
||||
|
||||
|
||||
|
||||
(defn import-yodlee [transactions]
|
||||
(lc/with-context {:source "Import yodlee transactions"}
|
||||
(let [import-id (start-import :import-source/yodlee "Automated Yodlee User")]
|
||||
(lc/with-context {:import-id import-id}
|
||||
(try
|
||||
(log/info "importing from yodlee")
|
||||
(let [account-lookup (->> (d/q '[:find ?ya ?ba
|
||||
:in $
|
||||
:where [?ba :bank-account/yodlee-account-id ?ya]]
|
||||
(d/db conn))
|
||||
(into {}))
|
||||
transactions-by-bank-account (group-by
|
||||
#(account-lookup (:accountId %))
|
||||
transactions)
|
||||
result (->> transactions-by-bank-account
|
||||
(filter (fn [[bank-account]]
|
||||
bank-account))
|
||||
(map (fn [[bank-account transactions]]
|
||||
(import-for-bank-account transactions bank-account import-id)))
|
||||
aggregate-results)]
|
||||
|
||||
|
||||
(finish-import (assoc result :db/id import-id)))
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
(finish-import {:db/id import-id})))))))
|
||||
|
||||
(defn do-import []
|
||||
(import-yodlee (client/get-transactions)))
|
||||
|
||||
(def do-import (allow-once do-import))
|
||||
|
||||
|
||||
(defn import-yodlee2
|
||||
[transactions]
|
||||
(lc/with-context {:source "Import yodlee2 transactions"}
|
||||
(let [import-id (start-import :import-source/yodlee2 "Automated Yodlee User")]
|
||||
(lc/with-context {:import-id import-id}
|
||||
(try
|
||||
(let [account-lookup (->> (d/q '[:find ?ya ?ba
|
||||
:in $
|
||||
:where
|
||||
[?ba :bank-account/yodlee-account ?y]
|
||||
[?y :yodlee-account/id ?ya]]
|
||||
(d/db conn))
|
||||
(into {}))
|
||||
transactions-by-bank-account (group-by
|
||||
#(account-lookup (:accountId %))
|
||||
transactions)
|
||||
result (->> transactions-by-bank-account
|
||||
(filter (fn [[bank-account]]
|
||||
bank-account))
|
||||
(map (fn [[bank-account transactions]]
|
||||
(import-for-bank-account transactions bank-account import-id)))
|
||||
aggregate-results)]
|
||||
|
||||
|
||||
(finish-import (assoc result :db/id import-id)))
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
(finish-import {:db/id import-id})))))))
|
||||
|
||||
(defn do-import2 []
|
||||
(log/info "starting import")
|
||||
(import-yodlee2 (client2/get-transactions "NGGL")))
|
||||
|
||||
(def do-import2 (allow-once do-import2))
|
||||
|
||||
|
||||
|
||||
|
||||
(mount/defstate import-transaction-worker
|
||||
:start (scheduler/every (* 1000 60 60 4) do-import)
|
||||
:stop (scheduler/stop import-transaction-worker))
|
||||
@@ -727,24 +727,3 @@
|
||||
(mapcat identity)
|
||||
vec))
|
||||
|
||||
(defn manually-add-transaction []
|
||||
(auto-ap.yodlee.import/transactions->txs [{:postDate "2014-01-04"
|
||||
:accountId 1234
|
||||
:date "2021-06-05"
|
||||
:id 1
|
||||
:amount {:amount -1743.25}
|
||||
:description {:original "original-description"
|
||||
:simple "simple-description"}
|
||||
:merchant {:id "123"
|
||||
:name "456"}
|
||||
:baseType "DEBIT"
|
||||
:status "POSTED"
|
||||
|
||||
:bank-account {:db/id [:bank-account/code "NGAK-1"]
|
||||
:client/_bank-accounts {:db/id 17592186045456
|
||||
:client/locations ["MH"]}}}]
|
||||
:bank-account
|
||||
(fn noop-rule [transaction locations]
|
||||
transaction)
|
||||
#{}))
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
[re-frame.core :as re-frame]
|
||||
[reagent.core :as reagent]
|
||||
[vimsical.re-frame.fx.track :as track]
|
||||
[auto-ap.status :as status]))
|
||||
[auto-ap.status :as status]
|
||||
[clojure.string :as str]))
|
||||
|
||||
|
||||
|
||||
@@ -134,12 +135,21 @@
|
||||
{:id ::manual-import
|
||||
:events #{::manual/import-completed}
|
||||
:event-fn (fn [[_ {:keys [imported errors] :as result}]]
|
||||
[::status/info ::manual-import (str "Successfully imported " imported " transactions"
|
||||
|
||||
(when (seq errors)
|
||||
(str
|
||||
", " (count errors) " errors. "
|
||||
"Example: '" (str (:info (first (:errors (first errors))))) "'")))])}]}))
|
||||
(println result)
|
||||
[::status/info ::manual-import
|
||||
(str "Successfully "
|
||||
(str/join ", "
|
||||
[(when-let [imported (:import-batch/imported result)]
|
||||
(str "imported " imported))
|
||||
(when-let [extant (:import-batch/extant result)]
|
||||
(str "extant " extant))
|
||||
(when-let [suppressed (:import-batch/suppressed result)]
|
||||
(str "suppressed " suppressed))
|
||||
(when-let [error (:validation-error result)]
|
||||
(str "errored " error))])
|
||||
" transactions."
|
||||
(when (:sample-error result)
|
||||
(str " Sample error: " (:info (:sample-error result)))))])}]}))
|
||||
|
||||
|
||||
(defn content []
|
||||
|
||||
60
test/clj/auto_ap/import/manual_test.clj
Normal file
60
test/clj/auto_ap/import/manual_test.clj
Normal file
@@ -0,0 +1,60 @@
|
||||
(ns auto-ap.import.manual-test
|
||||
(:require [auto-ap.datomic :refer [conn uri]]
|
||||
[auto-ap.import.manual :as sut]
|
||||
[auto-ap.time :as time]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.test :as t]
|
||||
[datomic.api :as d]
|
||||
[auto-ap.datomic.migrate :as m]))
|
||||
|
||||
(def raw-tsv "Status Date Original Description High Level Category Category-Subcategory End Here Amount Amount Split Type Note Account Name SLO Acct Name SLO F1 Code Comp ID
|
||||
posted 8/23/2021 MOUNTAIN MIKES PIZZA - -24.27 -24.27 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC
|
||||
posted 9/1/2021 Prime Video*258CB55D0 - -3.99 -3.99 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC
|
||||
posted 5/21/2021 Prime Video*2L1I62RT2 - -5.99 -5.99 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC
|
||||
posted 9/6/2021 STARBUCKS STORE 06536 - -38.85 -38.85 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC")
|
||||
|
||||
(def base-transaction {:status "posted"
|
||||
:raw-date "8/23/2021"
|
||||
:description-original "MOUNTAIN MIKES PIZZA"
|
||||
:high-level-category ""
|
||||
:amount "-24.27"
|
||||
:bank-account-code "MVSC-6"
|
||||
:client-code "MVSC"})
|
||||
|
||||
(t/deftest tabulate-data
|
||||
(t/testing "Should tabulate a single row"
|
||||
(t/is (= base-transaction (first (sut/tabulate-data raw-tsv))))))
|
||||
|
||||
(t/deftest manual->transactions
|
||||
(t/testing "Should transform a single transaction"
|
||||
(t/is (= #:transaction{:bank-account 1
|
||||
:client 12
|
||||
:date #inst "2021-08-23T00:00:00.000-07:00"
|
||||
:amount -24.27
|
||||
:description-original "MOUNTAIN MIKES PIZZA"
|
||||
:status "POSTED"}
|
||||
(sut/manual->transaction base-transaction {"MVSC-6" 1} {"MVSC-6" 12}))))
|
||||
|
||||
(t/testing "Should regard a bad date as an error"
|
||||
(let [row (sut/manual->transaction (assoc base-transaction :raw-date "124") {"MVSC-6" 1} {"MVSC-6" 12})]
|
||||
(t/is (=
|
||||
[{:info "Date 124 must match MM/dd/yyyy"
|
||||
:details "java.lang.Exception: Date 124 must match MM/dd/yyyy"}]
|
||||
(:errors row)))))
|
||||
|
||||
(t/testing "Should consider an unparseable amount as an error"
|
||||
(let [row (sut/manual->transaction (assoc base-transaction :amount "212.23,.41") {"MVSC-6" 1} {"MVSC-6" 12})]
|
||||
(t/is (=
|
||||
[{:info "Could not parse total from value '212.23,.41'"
|
||||
:details "java.lang.Exception: Could not parse total from value '212.23,.41'"}]
|
||||
(:errors row)))))
|
||||
|
||||
(t/testing "Should consider a nonexistant bank account as an error"
|
||||
(let [row (sut/manual->transaction (assoc base-transaction :bank-account-code "DUMMY") {"MVSC-6" 1} {"MVSC-6" 12})]
|
||||
(t/is (=
|
||||
[{:info "Cannot find bank account by code DUMMY"
|
||||
:details "java.lang.Exception: Cannot find bank account by code DUMMY"}
|
||||
{:info "Cannot find client for bank account code DUMMY"
|
||||
:details "java.lang.Exception: Cannot find client for bank account code DUMMY"}]
|
||||
(:errors row))))))
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
(ns auto-ap.integration.yodlee.import
|
||||
(:require [auto-ap.yodlee.import :as sut]
|
||||
[auto-ap.yodlee.core :as c]
|
||||
[datomic.api :as d]
|
||||
[auto-ap.datomic :refer [uri conn]]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
(ns auto-ap.import.transactions-test
|
||||
(:require [auto-ap.datomic :refer [conn uri]]
|
||||
[auto-ap.datomic.migrate :as m]
|
||||
[auto-ap.import.transactions :as sut]
|
||||
[clojure.test :as t]
|
||||
[clojure.tools.logging :as log]
|
||||
[clojure.set :as set]))
|
||||
[datomic.api :as d]
|
||||
[clj-time.coerce :as coerce]))
|
||||
|
||||
(defn wrap-setup
|
||||
[f]
|
||||
@@ -24,311 +21,72 @@
|
||||
(defn noop-rule [transaction locations]
|
||||
transaction)
|
||||
|
||||
(def base-transaction {:postDate "2014-01-04"
|
||||
:accountId 1234
|
||||
:date "2014-01-02"
|
||||
:id 1
|
||||
:amount {:amount 12.0}
|
||||
:description {:original "original-description"
|
||||
:simple "simple-description"}
|
||||
:merchant {:id "123"
|
||||
:name "456"}
|
||||
:baseType "DEBIT"
|
||||
:status "POSTED"
|
||||
(def base-transaction #:transaction {:date #inst "2020-01-02T00:00:00-08:00"
|
||||
:raw-id "1"
|
||||
:id (digest/sha-256 "1")
|
||||
:amount 12.0
|
||||
:description-original "original-description"
|
||||
:status "POSTED"
|
||||
:client 123
|
||||
:bank-account 456})
|
||||
|
||||
:bank-account {:db/id 456
|
||||
:client/_bank-accounts {:db/id 123
|
||||
:client/locations ["Z" "E"]}}})
|
||||
(t/deftest categorize-transactions
|
||||
(let [bank-account {:db/id 456
|
||||
:client/_bank-accounts {:db/id 123
|
||||
:client/locations ["MH"]}}]
|
||||
(t/testing "Should exclude a transaction before start date"
|
||||
(t/is (= :not-ready
|
||||
(sut/categorize-transaction (assoc base-transaction :transaction/date #inst "2020-01-01")
|
||||
(assoc bank-account :bank-account/start-date #inst "2030-01-01")
|
||||
{})))
|
||||
(t/is (= :import
|
||||
(sut/categorize-transaction (assoc base-transaction :transaction/date #inst "2020-01-02")
|
||||
(assoc bank-account :bank-account/start-date #inst "2020-01-01")
|
||||
{}))))
|
||||
(t/testing "Should error a transaction without a client"
|
||||
(t/is (= :error
|
||||
(sut/categorize-transaction (dissoc base-transaction :transaction/client)
|
||||
bank-account
|
||||
{}))))
|
||||
|
||||
(t/deftest add-transactions
|
||||
(t/testing "Should error a transaction without a bank-account"
|
||||
(t/is (= :error
|
||||
(sut/categorize-transaction (dissoc base-transaction :transaction/bank-account)
|
||||
bank-account
|
||||
{}))))
|
||||
(t/testing "Should error a transaction without an id"
|
||||
(t/is (= :error
|
||||
(sut/categorize-transaction (dissoc base-transaction :transaction/id)
|
||||
bank-account
|
||||
{}))))
|
||||
(t/testing "Should ignore a transaction that exists"
|
||||
(t/is (= :extant
|
||||
(sut/categorize-transaction (assoc base-transaction :transaction/id "123")
|
||||
bank-account
|
||||
{"123" :transaction-approval-status/unapproved}))))
|
||||
(t/testing "Should import a transaction"
|
||||
(t/is (= :import
|
||||
(sut/categorize-transaction base-transaction
|
||||
bank-account
|
||||
{}))))))
|
||||
|
||||
|
||||
(t/deftest transaction->txs
|
||||
(t/testing "Should import and code transactions"
|
||||
(let [[result] (sut/transactions->txs [base-transaction]
|
||||
:bank-account
|
||||
noop-rule
|
||||
#{})]
|
||||
(t/is (= [#:transaction {:amount -12.0
|
||||
:date #inst "2014-01-02T08:00:00.000-00:00"
|
||||
:bank-account 456
|
||||
:client 123
|
||||
:post-date #inst "2014-01-04T08:00:00.000-00:00"
|
||||
:account-id 1234
|
||||
:description-original "original-description"
|
||||
:status "POSTED"
|
||||
:id "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
|
||||
:raw-id "1"
|
||||
:approval-status :transaction-approval-status/unapproved
|
||||
:description-simple "simple-description"}]
|
||||
result)))))
|
||||
(t/testing "Should import one transaction"
|
||||
(let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id"
|
||||
:bank-account/code "TEST-1"}
|
||||
{:db/id "client-id"
|
||||
:client/code "TEST"
|
||||
:client/locations ["Z" "E"]
|
||||
:client/bank-accounts ["bank-account-id"]}]))
|
||||
result (sut/transaction->txs base-transaction
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
noop-rule)]
|
||||
(t/is (= [(assoc base-transaction :transaction/approval-status :transaction-approval-status/unapproved)]
|
||||
result))))
|
||||
|
||||
(t/deftest do-import
|
||||
(t/testing "Should import single transaction"
|
||||
(println "HER")
|
||||
(let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id"
|
||||
:bank-account/code "TEST-1"}
|
||||
{:db/id "client-id"
|
||||
:client/code "TEST"
|
||||
:client/locations ["Z" "E"]
|
||||
:client/bank-accounts ["bank-account-id"]}]))
|
||||
result (sut/transactions->txs [base-transaction]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
#{})]
|
||||
(t/is (= {:import [[#:transaction {:amount -12.0
|
||||
:date #inst "2014-01-02T08:00:00.000-00:00"
|
||||
:bank-account bank-account-id
|
||||
:client client-id
|
||||
:post-date #inst "2014-01-04T08:00:00.000-00:00"
|
||||
:account-id 1234
|
||||
:description-original "original-description"
|
||||
:status "POSTED"
|
||||
:id "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"
|
||||
:raw-id "1"
|
||||
:approval-status :transaction-approval-status/unapproved
|
||||
:description-simple "simple-description"
|
||||
:import-batch/_entry 1}]]}
|
||||
result))))
|
||||
|
||||
(t/testing "Should exclude a transaction before start date"
|
||||
(let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id"
|
||||
:bank-account/code "TEST-1"
|
||||
:bank-account/start-date #inst "2030-01-01"}
|
||||
{:db/id "client-id"
|
||||
:client/code "TEST"
|
||||
:client/locations ["Z" "E"]
|
||||
:client/bank-accounts ["bank-account-id"]}]))
|
||||
result (sut/transactions->txs [(assoc base-transaction :date "2020-05-01")]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
#{})]
|
||||
(t/is (= 1 (count (:not-ready result))))))
|
||||
|
||||
(t/testing "Should not reimport an existing transaction"
|
||||
(let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id"
|
||||
:bank-account/code "TEST-1"
|
||||
:bank-account/start-date #inst "2030-01-01"}
|
||||
{:db/id "client-id"
|
||||
:client/code "TEST"
|
||||
:client/locations ["Z" "E"]
|
||||
:client/bank-accounts ["bank-account-id"]}]))
|
||||
result (sut/transactions->txs [(assoc base-transaction :id "123")]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{(digest/sha-256 "123") :transaction-approval-status/unapproved})]
|
||||
(t/is (= 1 (count (:extant result))))))
|
||||
|
||||
|
||||
(t/testing "Should skip transaction if no client is found"
|
||||
(let [result (sut/transactions->txs [base-transaction]
|
||||
(d/entity (d/db conn) 1238970123)
|
||||
1
|
||||
noop-rule
|
||||
{})]
|
||||
(t/is (= 1 (count (:error result))))))
|
||||
|
||||
(t/testing "Should match an uncleared check"
|
||||
(let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending
|
||||
:date #inst "2019-01-01"
|
||||
:bank-account "bank-account-id"
|
||||
:client "client-id"
|
||||
:check-number 10001
|
||||
:amount 30.0
|
||||
:db/id "payment-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)]
|
||||
|
||||
|
||||
(println (sut/transactions->txs [(assoc base-transaction
|
||||
:description {:original "CHECK 10001"
|
||||
:simple ""}
|
||||
:amount {:amount 30.0})]
|
||||
(d/entity (d/db conn ) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{}))
|
||||
(let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:description {:original "CHECK 10001"
|
||||
:simple ""}
|
||||
:amount {:amount 30.0})]
|
||||
(d/entity (d/db conn ) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{}))]
|
||||
|
||||
(t/is (= {:db/id payment-id
|
||||
:payment/status :payment-status/cleared}
|
||||
(:transaction/payment transaction-result))))
|
||||
|
||||
(t/testing "Should not match an already matched check"
|
||||
@(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}])
|
||||
(let [[result] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:description {:original "CHECK 10001"
|
||||
:simple ""}
|
||||
:amount {:amount 30.0}
|
||||
:id 789)]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{}))]
|
||||
|
||||
(t/is (= nil
|
||||
(:transaction/payment result)))))))
|
||||
|
||||
(t/testing "Should match expected-deposits"
|
||||
(let [{:strs [bank-account-id client-id expected-deposit-id]} (->> [#:expected-deposit {:client "client-id"
|
||||
:date #inst "2021-07-01T00:00:00-08:00"
|
||||
:total 100.0
|
||||
:location "MF"
|
||||
:status :expected-deposit-status/pending
|
||||
:db/id "expected-deposit-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:locations ["MF"]
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)]
|
||||
|
||||
|
||||
(t/testing "Should match within 10 days"
|
||||
(let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:date "2021-07-03"
|
||||
:amount {:amount -100.0})]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
#{}))]
|
||||
(t/is (= expected-deposit-id
|
||||
(sut/find-expected-deposit client-id 100.0 (clj-time.coerce/to-date-time #inst "2021-07-03T00:00:00-08:00"))))
|
||||
|
||||
(t/is (= {:db/id expected-deposit-id
|
||||
:expected-deposit/status :expected-deposit-status/cleared}
|
||||
(:transaction/expected-deposit transaction-result)))))
|
||||
|
||||
(t/testing "Should not match old expected deposits"
|
||||
(let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:date "2021-07-13"
|
||||
:amount {:amount -100.0})]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{}))]
|
||||
(t/is (not (:transaction/expected-deposit transaction-result)))))
|
||||
|
||||
(t/testing "Should only match exact."
|
||||
(let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:date "2021-07-03"
|
||||
:amount {:amount -100.01})]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
noop-rule
|
||||
{}))]
|
||||
(t/is (not (:transaction/expected-deposit transaction-result)))))))
|
||||
|
||||
#_(t/testing "Auto-pay Invoices"
|
||||
(t/testing "Should match paid invoice that doesn't have a payment yet"
|
||||
(let [{:strs [bank-account-id client-id invoice1-id invoice2-id vendor-id]} (->> [#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:scheduled-payment #inst "2019-01-04"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 20.0
|
||||
:db/id "invoice1-id"}
|
||||
#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:scheduled-payment #inst "2019-01-04"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 10.0
|
||||
:db/id "invoice2-id"}
|
||||
#:vendor {:name "Autopay vendor"
|
||||
:db/id "vendor-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)
|
||||
[[transaction-tx payment-tx invoice-payments1-tx invoice-payments2-tx]] (sut/transactions->txs [(assoc base-transaction
|
||||
:amount {:amount 30.0}
|
||||
:bank-account {:db/id bank-account-id
|
||||
:client/_bank-accounts {:db/id client-id
|
||||
:client/locations ["A"]}})]
|
||||
:bank-account
|
||||
noop-rule
|
||||
#{})]
|
||||
|
||||
(t/is (= :transaction-approval-status/approved
|
||||
(:transaction/approval-status transaction-tx))
|
||||
(str "Should have approved transaction " transaction-tx))
|
||||
(t/is (= #:payment{:status :payment-status/cleared
|
||||
:type :payment-type/debit
|
||||
:date (:transaction/date transaction-tx)
|
||||
:client client-id
|
||||
:bank-account bank-account-id
|
||||
:vendor vendor-id
|
||||
:amount 30.0}
|
||||
|
||||
(dissoc payment-tx :db/id))
|
||||
(str "Should have created payment " payment-tx))
|
||||
(t/is (= #:invoice-payment{:invoice invoice1-id
|
||||
:amount 20.0
|
||||
:payment (:db/id payment-tx)}
|
||||
|
||||
(dissoc invoice-payments1-tx :db/id))
|
||||
(str "Should have paid invoice 1" invoice-payments1-tx))
|
||||
(t/is (= #:invoice-payment{:invoice invoice2-id
|
||||
:amount 10.0
|
||||
:payment (:db/id payment-tx)}
|
||||
|
||||
(dissoc invoice-payments2-tx :db/id))
|
||||
(str "Should have paid invoice 2" invoice-payments2-tx))))
|
||||
|
||||
(t/testing "Should not match paid invoice that isn't a scheduled payment"
|
||||
(let [{:strs [bank-account-id client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 30.0
|
||||
:db/id "invoice-id"}
|
||||
#:vendor {:name "Autopay vendor"
|
||||
:db/id "vendor-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)
|
||||
[[transaction-tx payment-tx]] (sut/transactions->txs [(assoc base-transaction
|
||||
:amount {:amount 30.0}
|
||||
:bank-account {:db/id bank-account-id
|
||||
:client/_bank-accounts {:db/id client-id
|
||||
:client/locations ["A"]}})]
|
||||
:bank-account
|
||||
noop-rule
|
||||
#{})]
|
||||
|
||||
(t/is (= :transaction-approval-status/unapproved
|
||||
(:transaction/approval-status transaction-tx)))
|
||||
(t/is (nil? (:transaction/payment transaction-tx))))))
|
||||
|
||||
|
||||
(t/testing "Rules"
|
||||
(t/testing "Should apply rules to imported transaction"
|
||||
(t/testing "Should match an uncleared check"
|
||||
(let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending
|
||||
:date #inst "2019-01-01"
|
||||
:bank-account "bank-account-id"
|
||||
@@ -343,198 +101,114 @@
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)
|
||||
[[transaction-tx]] (:import (sut/transactions->txs [(assoc base-transaction
|
||||
:description {:original "Hello XXX039"
|
||||
:simple ""}
|
||||
:amount {:amount 31.0}
|
||||
:id 789)]
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
1
|
||||
(rm/rule-applying-fn [{:transaction-rule/description "XXX039"
|
||||
:transaction-rule/transaction-approval-status :transaction-approval-status/approved}])
|
||||
#{}))]
|
||||
|
||||
(t/is (= :transaction-approval-status/approved
|
||||
(:transaction/approval-status transaction-tx)))))
|
||||
:tempids)]
|
||||
|
||||
(t/testing "Should apply vendor and approval status"
|
||||
(let [apply-rules (rm/rule-applying-fn [{:db/id 1
|
||||
:transaction-rule/description "XXX039"
|
||||
:transaction-rule/transaction-approval-status :transaction-approval-status/approved
|
||||
:transaction-rule/vendor {:db/id 123}
|
||||
:transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9}
|
||||
:transaction-rule-account/location "Shared"
|
||||
:transaction-rule-account/percentage 1.0}]}
|
||||
{:db/id 2
|
||||
:transaction-rule/description "OtherMatch"
|
||||
:transaction-rule/transaction-approval-status :transaction-approval-status/requires-feedback
|
||||
:transaction-rule/vendor {:db/id 456}
|
||||
:transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9}
|
||||
:transaction-rule-account/location "Z"
|
||||
:transaction-rule-account/percentage 1.0}]}])]
|
||||
(t/is (= {:transaction/description-original "Hello XXX039",
|
||||
:transaction/vendor 123
|
||||
:transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/accounts [{:transaction-account/account 9
|
||||
:transaction-account/amount 30.0
|
||||
:transaction-account/location "Z"}]
|
||||
:transaction/matched-rule 1
|
||||
:transaction/amount 30.0}
|
||||
(-> {:transaction/description-original "Hello XXX039"
|
||||
:transaction/amount 30.0}
|
||||
(apply-rules ["Z"]))))
|
||||
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction
|
||||
:transaction/description-original "CHECK 10001"
|
||||
:transaction/amount -30.0)
|
||||
(d/entity (d/db conn ) bank-account-id)
|
||||
noop-rule)]
|
||||
|
||||
(t/is (= {:db/id payment-id
|
||||
:payment/status :payment-status/cleared}
|
||||
(:transaction/payment transaction-result))))
|
||||
|
||||
(t/is (= {:transaction/description-original "OtherMatch",
|
||||
:transaction/approval-status :transaction-approval-status/requires-feedback
|
||||
:transaction/vendor 456
|
||||
:transaction/amount 30.0
|
||||
:transaction/matched-rule 2
|
||||
:transaction/accounts [{:transaction-account/account 9
|
||||
:transaction-account/amount 30.0
|
||||
:transaction-account/location "Z"}]}
|
||||
(-> {:transaction/description-original "OtherMatch"
|
||||
:transaction/amount 30.0}
|
||||
(apply-rules ["Z"]))))
|
||||
(t/testing "Should not match an already matched check"
|
||||
@(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}])
|
||||
(let [[result] (sut/transaction->txs (assoc base-transaction
|
||||
:transaction/description-original "CHECK 10001"
|
||||
:transaction/amount -30.0)
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
noop-rule)]
|
||||
|
||||
(t/is (= nil
|
||||
(:transaction/payment result)))))))
|
||||
|
||||
(t/is (= {:transaction/description-original "Hello Not match"}
|
||||
(-> {:transaction/description-original "Hello Not match"}
|
||||
(apply-rules []))))))
|
||||
|
||||
(t/testing "Should match if day of month matches"
|
||||
(let [apply-rules (rm/rule-applying-fn [{:db/id 123
|
||||
:transaction-rule/dom-gte 3
|
||||
:transaction-rule/dom-lte 9
|
||||
:transaction-rule/transaction-approval-status :transaction-approval-status/approved
|
||||
:transaction-rule/vendor {:db/id 123}}])]
|
||||
(t/is (= 123
|
||||
(-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00"
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 123
|
||||
(-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00"
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 123
|
||||
(-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00"
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00"
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00"
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))))
|
||||
(t/testing "Should match expected-deposits"
|
||||
(let [{:strs [bank-account-id client-id expected-deposit-id]} (->> [#:expected-deposit {:client "client-id"
|
||||
:date #inst "2021-07-01T00:00:00-08:00"
|
||||
:total 100.0
|
||||
:location "MF"
|
||||
:status :expected-deposit-status/pending
|
||||
:db/id "expected-deposit-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:locations ["MF"]
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)]
|
||||
|
||||
(t/testing "Should match if amount matches"
|
||||
(let [apply-rules (rm/rule-applying-fn [{:db/id 123
|
||||
:transaction-rule/amount-gte 3.0
|
||||
:transaction-rule/amount-lte 9.0
|
||||
:transaction-rule/transaction-approval-status :transaction-approval-status/approved
|
||||
:transaction-rule/accounts [#:transaction-rule-account {:percentage 1.0
|
||||
:locatoin "HQ"}]
|
||||
:transaction-rule/vendor {:db/id 123}}])]
|
||||
(t/is (= 123
|
||||
(-> {:transaction/amount 4.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 123
|
||||
(-> {:transaction/amount 3.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 123
|
||||
(-> {:transaction/amount 9.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/amount 9.01}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/amount 2.99}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))))
|
||||
|
||||
(t/testing "Should match if client matches"
|
||||
(let [apply-rules (rm/rule-applying-fn [{:db/id 123
|
||||
:transaction-rule/client {:db/id 456}}])]
|
||||
(t/is (= 123
|
||||
(-> {:transaction/client 456
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/client 89
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))))
|
||||
(t/testing "Should match if bank-account matches"
|
||||
(let [apply-rules (rm/rule-applying-fn [{:db/id 123
|
||||
:transaction-rule/bank-account {:db/id 456}}])]
|
||||
(t/is (= 123
|
||||
(-> {:transaction/bank-account 456
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (nil?
|
||||
(-> {:transaction/bank-account 89
|
||||
:transaction/amount 1.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))))
|
||||
(t/testing "Should match within 10 days"
|
||||
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction
|
||||
:transaction/date #inst "2021-07-03T00:00:00-08:00"
|
||||
:transaction/amount 100.0)
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
noop-rule)]
|
||||
(t/is (= expected-deposit-id
|
||||
(sut/find-expected-deposit client-id 100.0 (clj-time.coerce/to-date-time #inst "2021-07-03T00:00:00-08:00"))))
|
||||
|
||||
(t/is (= {:db/id expected-deposit-id
|
||||
:expected-deposit/status :expected-deposit-status/cleared}
|
||||
(:transaction/expected-deposit transaction-result)))))
|
||||
|
||||
(t/testing "Should prioritize rules"
|
||||
(let [apply-rules (rm/rule-applying-fn (shuffle [{:db/id 2
|
||||
:transaction-rule/description "Hello"
|
||||
:transaction-rule/amount-gte 5.0}
|
||||
{:db/id 1
|
||||
:transaction-rule/description "Hello"}
|
||||
{:db/id 0
|
||||
:transaction-rule/description "Hello"}
|
||||
{:db/id 3
|
||||
:transaction-rule/description "Hello"
|
||||
:transaction-rule/client {:db/id 789}}
|
||||
{:db/id 4
|
||||
:transaction-rule/description "Hello"
|
||||
:transaction-rule/client {:db/id 789}
|
||||
:transaction-rule/bank-account {:db/id 456}}]))]
|
||||
(t/testing "Should not match old expected deposits"
|
||||
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction
|
||||
:transaction/date #inst "2021-07-13"
|
||||
:transaction/amount 100.0)
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
noop-rule)]
|
||||
(t/is (not (:transaction/expected-deposit transaction-result)))))
|
||||
|
||||
(t/is (= 4
|
||||
(-> {:transaction/bank-account 456
|
||||
:transaction/client 789
|
||||
:transaction/description-original "Hello"
|
||||
:transaction/amount 6.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 3
|
||||
(-> {:transaction/bank-account 457
|
||||
:transaction/client 789
|
||||
:transaction/amount 6.0
|
||||
:transaction/description-original "Hello"}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/is (= 3
|
||||
(-> {:transaction/bank-account 457
|
||||
:transaction/client 789
|
||||
:transaction/amount 6.0
|
||||
:transaction/description-original "Hello"}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule)))
|
||||
(t/testing "Should only match exact."
|
||||
(let [[transaction-result] (sut/transaction->txs (assoc base-transaction
|
||||
:transaction/date "2021-07-03"
|
||||
:transaction/amount 100.01)
|
||||
(d/entity (d/db conn) bank-account-id)
|
||||
noop-rule)]
|
||||
(t/is (not (:transaction/expected-deposit transaction-result)))))))))
|
||||
|
||||
(t/testing "Should only apply if there is a single rule at that specificity level"
|
||||
(t/is (nil?
|
||||
(-> {:transaction/bank-account 3
|
||||
:transaction/client 1
|
||||
:transaction/description-original "Hello"
|
||||
:transaction/amount 0.0}
|
||||
(apply-rules [])
|
||||
:transaction/matched-rule))))))))
|
||||
(t/deftest apply-synthetic-ids
|
||||
(t/testing "Should increment index for duplicate transactions"
|
||||
(t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
"2020-01-02T00:00:00.000-08:00-456-original-description-12.0-1-123"]
|
||||
(map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction base-transaction])))))
|
||||
|
||||
(t/testing "Should use unique ids if a parameter is different"
|
||||
(t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
"2020-01-02T00:00:00.000-08:00-456-original-description-13.0-0-123"]
|
||||
(map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction (assoc base-transaction :transaction/amount 13.0)])))))
|
||||
|
||||
(t/testing "Should increment index if an unimportant field is set"
|
||||
(t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
"2020-01-02T00:00:00.000-08:00-456-original-description-12.0-1-123"]
|
||||
(map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction (assoc base-transaction :transaction/other-random-value 10.0)])))))
|
||||
|
||||
(t/testing "Should be forgiving if dates come in other formats"
|
||||
(t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
(->> (sut/apply-synthetic-ids [(assoc base-transaction
|
||||
:transaction/date #inst "2020-01-02T00:00:00-08:00")])
|
||||
first
|
||||
:transaction/raw-id)))
|
||||
(t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
(->> (sut/apply-synthetic-ids [(assoc base-transaction
|
||||
:transaction/date #inst "2020-01-02T08:00:00-00:00")])
|
||||
first
|
||||
:transaction/raw-id)))
|
||||
(t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
(->> (sut/apply-synthetic-ids [(assoc base-transaction
|
||||
:transaction/date (coerce/to-date-time #inst "2020-01-02T08:00:00-00:00"))])
|
||||
first
|
||||
:transaction/raw-id)))
|
||||
(t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123"
|
||||
(->> (sut/apply-synthetic-ids [(assoc base-transaction
|
||||
:transaction/date (coerce/to-date-time #inst "2020-01-02T00:00:00-08:00"))])
|
||||
first
|
||||
:transaction/raw-id)))))
|
||||
|
||||
|
||||
(t/deftest match-transaction-to-single-unfulfilled-payments
|
||||
@@ -580,7 +254,7 @@
|
||||
(t/testing "Should not match unpaid invoice"
|
||||
(let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/unpaid
|
||||
:scheduled-payment #inst "2019-01-04"
|
||||
:vendor vendor1-id
|
||||
:vendor vendor1-id
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 30.0
|
||||
@@ -609,7 +283,7 @@
|
||||
deref
|
||||
:tempids)
|
||||
invoices-matches (sut/match-transaction-to-single-unfulfilled-autopayments -30.0
|
||||
client-id)]
|
||||
client-id)]
|
||||
(t/is (= [] invoices-matches))))
|
||||
(t/testing "Should match multiple invoices for same vendor that total to transaction amount"
|
||||
(let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid
|
||||
@@ -709,3 +383,98 @@
|
||||
(str "Expected to match with the chronologically adjacent invoice-1 and invoice-3."))
|
||||
(t/is (= [] (sut/match-transaction-to-single-unfulfilled-autopayments -31.0 client-id))
|
||||
(str "Expected to not match, because there is invoice-3 is between invoice-1 and invoice-2.")))))))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#_(t/testing "Auto-pay Invoices"
|
||||
(t/testing "Should match paid invoice that doesn't have a payment yet"
|
||||
(let [{:strs [bank-account-id client-id invoice1-id invoice2-id vendor-id]} (->> [#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:scheduled-payment #inst "2019-01-04"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 20.0
|
||||
:db/id "invoice1-id"}
|
||||
#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:scheduled-payment #inst "2019-01-04"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 10.0
|
||||
:db/id "invoice2-id"}
|
||||
#:vendor {:name "Autopay vendor"
|
||||
:db/id "vendor-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)
|
||||
[[transaction-tx payment-tx invoice-payments1-tx invoice-payments2-tx]] (sut/yodlees->transactions [(assoc base-yodlee-transaction
|
||||
:amount {:amount 30.0}
|
||||
:bank-account {:db/id bank-account-id
|
||||
:client/_bank-accounts {:db/id client-id
|
||||
:client/locations ["A"]}})]
|
||||
:bank-account
|
||||
noop-rule
|
||||
#{})]
|
||||
|
||||
(t/is (= :transaction-approval-status/approved
|
||||
(:transaction/approval-status transaction-tx))
|
||||
(str "Should have approved transaction " transaction-tx))
|
||||
(t/is (= #:payment{:status :payment-status/cleared
|
||||
:type :payment-type/debit
|
||||
:date (:transaction/date transaction-tx)
|
||||
:client client-id
|
||||
:bank-account bank-account-id
|
||||
:vendor vendor-id
|
||||
:amount 30.0}
|
||||
|
||||
(dissoc payment-tx :db/id))
|
||||
(str "Should have created payment " payment-tx))
|
||||
(t/is (= #:invoice-payment{:invoice invoice1-id
|
||||
:amount 20.0
|
||||
:payment (:db/id payment-tx)}
|
||||
|
||||
(dissoc invoice-payments1-tx :db/id))
|
||||
(str "Should have paid invoice 1" invoice-payments1-tx))
|
||||
(t/is (= #:invoice-payment{:invoice invoice2-id
|
||||
:amount 10.0
|
||||
:payment (:db/id payment-tx)}
|
||||
|
||||
(dissoc invoice-payments2-tx :db/id))
|
||||
(str "Should have paid invoice 2" invoice-payments2-tx))))
|
||||
|
||||
(t/testing "Should not match paid invoice that isn't a scheduled payment"
|
||||
(let [{:strs [bank-account-id client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid
|
||||
:vendor "vendor-id"
|
||||
:date #inst "2019-01-01"
|
||||
:client "client-id"
|
||||
:total 30.0
|
||||
:db/id "invoice-id"}
|
||||
#:vendor {:name "Autopay vendor"
|
||||
:db/id "vendor-id"}
|
||||
#:bank-account {:name "Bank account"
|
||||
:db/id "bank-account-id"}
|
||||
#:client {:name "Client"
|
||||
:db/id "client-id"
|
||||
:bank-accounts ["bank-account-id"]}]
|
||||
(d/transact (d/connect uri))
|
||||
deref
|
||||
:tempids)
|
||||
[[transaction-tx payment-tx]] (sut/yodlees->transactions [(assoc base-yodlee-transaction
|
||||
:amount {:amount 30.0}
|
||||
:bank-account {:db/id bank-account-id
|
||||
:client/_bank-accounts {:db/id client-id
|
||||
:client/locations ["A"]}})]
|
||||
:bank-account
|
||||
noop-rule
|
||||
#{})]
|
||||
|
||||
(t/is (= :transaction-approval-status/unapproved
|
||||
(:transaction/approval-status transaction-tx)))
|
||||
(t/is (nil? (:transaction/payment transaction-tx))))))
|
||||
29
test/clj/auto_ap/import/yodlee_test.clj
Normal file
29
test/clj/auto_ap/import/yodlee_test.clj
Normal file
@@ -0,0 +1,29 @@
|
||||
(ns auto-ap.import.yodlee-test
|
||||
(:require [auto-ap.import.yodlee :as sut]
|
||||
[clojure.test :as t]))
|
||||
|
||||
|
||||
(def base-transaction {:postDate "2014-01-04"
|
||||
:accountId 1234
|
||||
:date "2014-01-02"
|
||||
:id 1
|
||||
:amount {:amount 12.0}
|
||||
:description {:original "original-description"
|
||||
:simple "simple-description"}
|
||||
:merchant {:id "123"
|
||||
:name "456"}
|
||||
:baseType "DEBIT"
|
||||
:status "POSTED"})
|
||||
|
||||
(t/deftest yodlee->transaction
|
||||
(t/testing "Should parse dates"
|
||||
(t/is (= #inst "2021-01-01T00:00:00-08:00" (:transaction/date (sut/yodlee->transaction (assoc base-transaction :date "2021-01-01")))))
|
||||
(t/is (= #inst "2021-06-01T00:00:00-07:00" (:transaction/date (sut/yodlee->transaction (assoc base-transaction :date "2021-06-01"))))))
|
||||
|
||||
(t/testing "Should invert amount for debits"
|
||||
(t/is (= -12.0 (:transaction/amount (sut/yodlee->transaction (assoc base-transaction
|
||||
:amount {:amount 12.0}
|
||||
:baseType "DEBIT")))))
|
||||
(t/is (= 12.0 (:transaction/amount (sut/yodlee->transaction (assoc base-transaction
|
||||
:amount {:amount 12.0}
|
||||
:baseType "CREDIT")))))))
|
||||
Reference in New Issue
Block a user