From 7489426ccb2941e4f61c439ab01cfd71084d63fc Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 22 Dec 2021 18:14:49 -0800 Subject: [PATCH] enormous refactor but simplified much! --- src/clj/auto_ap/background/requests.clj | 19 +- src/clj/auto_ap/datomic/migrate.clj | 15 +- src/clj/auto_ap/graphql/transactions.clj | 10 +- src/clj/auto_ap/import/intuit.clj | 74 ++ src/clj/auto_ap/import/intuit_test.clj | 20 + src/clj/auto_ap/import/manual.clj | 66 ++ src/clj/auto_ap/import/manual/common.clj | 53 ++ src/clj/auto_ap/import/plaid.clj | 60 ++ src/clj/auto_ap/import/transactions.clj | 336 ++++++++ src/clj/auto_ap/import/yodlee.clj | 70 ++ src/clj/auto_ap/import/yodlee2.clj | 45 ++ src/clj/auto_ap/intuit/import.clj | 95 --- src/clj/auto_ap/plaid/core.clj | 1 + src/clj/auto_ap/plaid/import.clj | 66 +- src/clj/auto_ap/routes/events.clj | 1 - src/clj/auto_ap/routes/invoices.clj | 115 +-- src/clj/auto_ap/server.clj | 19 +- src/clj/auto_ap/yodlee/core.clj | 1 + src/clj/auto_ap/yodlee/core2.clj | 4 +- src/clj/auto_ap/yodlee/import.clj | 485 ----------- src/clj/user.clj | 21 - .../auto_ap/views/pages/transactions.cljs | 24 +- test/clj/auto_ap/import/manual_test.clj | 60 ++ .../transactions_test.clj} | 757 ++++++------------ test/clj/auto_ap/import/yodlee_test.clj | 29 + 25 files changed, 1188 insertions(+), 1258 deletions(-) create mode 100644 src/clj/auto_ap/import/intuit.clj create mode 100644 src/clj/auto_ap/import/intuit_test.clj create mode 100644 src/clj/auto_ap/import/manual.clj create mode 100644 src/clj/auto_ap/import/manual/common.clj create mode 100644 src/clj/auto_ap/import/plaid.clj create mode 100644 src/clj/auto_ap/import/transactions.clj create mode 100644 src/clj/auto_ap/import/yodlee.clj create mode 100644 src/clj/auto_ap/import/yodlee2.clj delete mode 100644 src/clj/auto_ap/intuit/import.clj delete mode 100644 src/clj/auto_ap/yodlee/import.clj create mode 100644 test/clj/auto_ap/import/manual_test.clj rename test/clj/auto_ap/{integration/yodlee/import.clj => import/transactions_test.clj} (50%) create mode 100644 test/clj/auto_ap/import/yodlee_test.clj diff --git a/src/clj/auto_ap/background/requests.clj b/src/clj/auto_ap/background/requests.clj index 866d80d5..fad4cc76 100644 --- a/src/clj/auto_ap/background/requests.clj +++ b/src/clj/auto_ap/background/requests.clj @@ -5,9 +5,12 @@ [mount.core :as mount] [yang.scheduler :as scheduler] [clojure.tools.logging :as log] - [auto-ap.intuit.import :as i] + [auto-ap.import.intuit :as i] + [auto-ap.import.plaid :as p] [unilog.context :as lc] - [auto-ap.yodlee.import :as y])) + [auto-ap.import.yodlee :as y] + [auto-ap.import.yodlee2 :as y2] + )) (def queue-url (:requests-queue-url env)) @@ -22,19 +25,25 @@ (cond (= ":intuit" body) (try - (i/upsert-transactions) + (i/import-intuit) (catch Exception e (log/error e))) (= ":yodlee" body) (try - (y/do-import) + (y/import-yodlee) (catch Exception e (log/error e))) (= ":yodlee2" body) (try - (y/do-import2) + (y2/import-yodlee2) + (catch Exception e + (log/error e))) + + (= ":plaid" body) + (try + (p/import-plaid) (catch Exception e (log/error e)))) (sqs/delete-message {:queue-url queue-url diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 9a7ed19a..824f07f7 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -430,7 +430,20 @@ :auto-ap/add-suppression {:txes [[{:db/ident :transaction-approval-status/suppressed} {:db/ident :transaction/approval-status :db/index true}]] - :requires [:auto-ap/add-transaction-rules]}} + :requires [:auto-ap/add-transaction-rules]} + :auto-ap/add-other-statuses {:txes [[{:db/ident :import-batch/error + :db/doc "How many entries were an error " + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/not-ready + :db/doc "How many entries were before a start date " + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/error-message + :db/doc "error message for a failed job" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one}]] + :requires [:auto-ap/add-transaction-import2]}} diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index 3f871c68..02e3840d 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -7,7 +7,7 @@ [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.graphql.transaction-rules :as g-tr] - [auto-ap.yodlee.import :as import] + [auto-ap.import.transactions :as i-transactions] [auto-ap.graphql.utils :refer [->graphql @@ -101,7 +101,7 @@ (let [transaction (d-transactions/get-by-id (:transaction_id args)) _ (assert-can-see-client (:id context) (:transaction/client transaction) )] - (let [matches-set (import/match-transaction-to-unfulfilled-autopayments (:transaction/amount transaction) + (let [matches-set (i-transactions/match-transaction-to-unfulfilled-autopayments (:transaction/amount transaction) (:db/id (:transaction/client transaction)))] (->graphql (for [matches matches-set] (for [[_ invoice-id ] matches] @@ -112,7 +112,7 @@ (let [transaction (d-transactions/get-by-id (:transaction_id args)) _ (assert-can-see-client (:id context) (:transaction/client transaction) )] - (let [matches-set (import/match-transaction-to-unpaid-invoices (:transaction/amount transaction) + (let [matches-set (i-transactions/match-transaction-to-unpaid-invoices (:transaction/amount transaction) (:db/id (:transaction/client transaction)))] (->graphql (for [matches matches-set] (for [[_ invoice-id ] matches] @@ -337,7 +337,7 @@ [(-> entity :invoice/vendor :db/id) (-> entity :db/id) (-> entity :invoice/total)]))))) - (let [payment-tx (import/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] + (let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] (map (fn [id] (let [entity (d/entity db id)] [(-> entity :invoice/vendor :db/id) @@ -373,7 +373,7 @@ (when (:transaction/payment transaction) (throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"}))) - (let [payment-tx (import/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] + (let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] (map (fn [id] (let [entity (d/entity db id)] [(-> entity :invoice/vendor :db/id) diff --git a/src/clj/auto_ap/import/intuit.clj b/src/clj/auto_ap/import/intuit.clj new file mode 100644 index 00000000..03cb9c05 --- /dev/null +++ b/src/clj/auto_ap/import/intuit.clj @@ -0,0 +1,74 @@ +(ns auto-ap.import.intuit + (:require [amazonica.aws.s3 :as s3] + [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.intuit.core :as i] + [auto-ap.utils :refer [by allow-once]] + [auto-ap.import.transactions :as t] + [clj-time.coerce :as coerce] + [clj-time.core :as time] + [clj-time.format :as f] + [clojure.string :as str] + [clojure.tools.logging :as log] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) + +(defn get-intuit-bank-accounts [db] + (d/q '[:find ?external-id ?ba ?c + :in $ + :where + [?c :client/bank-accounts ?ba] + [?ba :bank-account/intuit-bank-account ?iab] + [?iab :intuit-bank-account/external-id ?external-id]] + db)) + +(defn intuit->transaction [transaction] + {:transaction/description-original (:Memo/Description transaction) + :transaction/amount (Double/parseDouble (:Amount transaction)) + :transaction/date (coerce/to-date (auto-ap.time/parse (:Date transaction) auto-ap.time/iso-date)) + :transaction/status "POSTED"}) + +(defn intuits->transactions [transactions bank-account-id client-id] + (->> transactions + (map intuit->transaction) + (map #(assoc % + :transaction/bank-account bank-account-id + :transaction/client client-id)) + (t/apply-synthetic-ids))) + +(defn import-intuit [] + (lc/with-context {:source "Import intuit transactions"} + (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user") + db (d/db conn) + end (auto-ap.time/local-now) + start (time/plus end (time/days -30))] + (try + (doseq [[external-id bank-account-id client-id] (get-intuit-bank-accounts db) + transaction (-> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) + (auto-ap.time/unparse end auto-ap.time/iso-date) + external-id) + (intuits->transactions bank-account-id client-id))] + (t/import-transaction! import-batch transaction)) + (t/finish! import-batch) + (catch Exception e + (t/fail! import-batch e)))))) + +(def upsert-transactions (allow-once upsert-transactions)) + +(defn upsert-accounts [] + (let [token (i/get-fresh-access-token) + bank-accounts (i/get-bank-accounts token)] + @(d/transact conn (mapv + (fn [ba] + {:intuit-bank-account/external-id (:name ba) + :intuit-bank-account/name (:name ba)}) + bank-accounts)))) + +(mount/defstate import-worker + :start (scheduler/every (* 1000 60 60 24) import-intuit) + :stop (scheduler/stop import-worker)) + +(mount/defstate account-worker + :start (scheduler/every (* 1000 60 60 24) upsert-accounts) + :stop (scheduler/stop account-worker)) diff --git a/src/clj/auto_ap/import/intuit_test.clj b/src/clj/auto_ap/import/intuit_test.clj new file mode 100644 index 00000000..4fe46121 --- /dev/null +++ b/src/clj/auto_ap/import/intuit_test.clj @@ -0,0 +1,20 @@ +(ns auto-ap.import.intuit-test + (:require [auto-ap.import.intuit :as sut] + [clojure.test :as t])) + +(def base-transaction {:Memo/Description "this is a description" + :Amount "45.23" + :Date "2021-10-11"}) + +(t/deftest intuit->transaction + (t/testing "Should parse dates" + (t/is (= #inst "2021-01-01T00:00:00-08:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-01-01"))))) + (t/is (= #inst "2021-06-01T00:00:00-07:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-06-01"))))))) + + +(t/deftest intuits->transactions + (t/testing "should give unique ids to duplicates" + (t/is (= ["2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-0-345" + "2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-1-345"] (map :transaction/raw-id (sut/intuits->transactions [base-transaction base-transaction] + 123 + 345)))))) diff --git a/src/clj/auto_ap/import/manual.clj b/src/clj/auto_ap/import/manual.clj new file mode 100644 index 00000000..e70aa623 --- /dev/null +++ b/src/clj/auto_ap/import/manual.clj @@ -0,0 +1,66 @@ +(ns auto-ap.import.manual + (:require [auto-ap.datomic :refer [conn]] + [auto-ap.import.manual.common :as c] + [auto-ap.import.transactions :as t] + [clj-time.coerce :as coerce] + [clojure.data.csv :as csv] + [clojure.tools.logging :as log] + [datomic.api :as d] + [unilog.context :as lc])) + +(defn manual-import-batch [transactions user] + ) + +(def columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code]) + +(defn tabulate-data [data] + (->> + (csv/read-csv data :separator \tab) + (drop 1) + (map (fn [row] + (into {} (->> (map vector columns row) + (filter (fn [[k v]] k)))))))) + +(defn manual->transaction [{:keys [description-original amount bank-account-code date] :as transaction} bank-account-lookup client-lookup] + (-> {:transaction/description-original description-original + :transaction/status "POSTED"} + (c/assoc-or-error :transaction/client #(if-let [client-id (client-lookup bank-account-code)] + client-id + (throw (Exception. (str "Cannot find client for bank account code " bank-account-code))))) + (c/assoc-or-error :transaction/bank-account #(if-let [bank-account-id (bank-account-lookup bank-account-code)] + bank-account-id + (throw (Exception. (str "Cannot find bank account by code " bank-account-code))))) + (c/assoc-or-error :transaction/date #(coerce/to-date (c/parse-date transaction))) + (c/assoc-or-error :transaction/amount #(c/parse-amount transaction)))) + +(defn import-batch [transactions user] + (lc/with-context {:source "Manual import transactions"} + (let [bank-account-code->client (into {} + (d/q '[:find ?bac ?c + :in $ + :where + [?c :client/bank-accounts ?ba] + [?ba :bank-account/code ?bac]] + (d/db conn))) + bank-account-code->bank-account (into {} + (d/q '[:find ?bac ?ba + :in $ + :where [?ba :bank-account/code ?bac]] + (d/db conn)))] + (let [import-batch (t/start-import-batch :import-source/manual user) + transactions (->> transactions + (map (fn [t] + (manual->transaction t bank-account-code->bank-account bank-account-code->client))) + (t/apply-synthetic-ids ))] + (try + (doseq [transaction transactions] + (when-not (seq (:errors transaction)) + (t/import-transaction! import-batch transaction))) + + (t/finish! import-batch) + (assoc (t/get-stats import-batch) + :failed-validation (count (filter :errors transactions)) + :sample-error (first (first (map :errors (filter :errors transactions))))) + (catch Exception e + (t/fail! import-batch e) + (t/get-stats import-batch))))))) diff --git a/src/clj/auto_ap/import/manual/common.clj b/src/clj/auto_ap/import/manual/common.clj new file mode 100644 index 00000000..ff6e205d --- /dev/null +++ b/src/clj/auto_ap/import/manual/common.clj @@ -0,0 +1,53 @@ +(ns auto-ap.import.manual.common + (:require [auto-ap.datomic.accounts :as d-accounts] + [auto-ap.parse.util :as parse-u] + [clojure.string :as str])) + +(defn parse-amount [i] + (try + (Double/parseDouble (str/replace (or (second + (re-matches #"[^0-9\.,\\-]*([0-9\.,\\-]+)[^0-9\.,]*" (:amount i))) + "0") + #"," "")) + (catch Exception e + (throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e))))) + +(defn parse-account-numeric-code [i] + (try + (when-let [account-numeric-code (:account-numeric-code i)] + (:db/id (d-accounts/get-account-by-numeric-code-and-sets (Integer/parseInt account-numeric-code) + ["default"]))) + (catch Exception e + (throw (Exception. (str "Could not parse expense account from value '" (:account-numeric-code i) "'") e))))) + +(defn parse-account-id [i] + (try + (Long/parseLong (second + (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i)))) + (catch Exception e + (throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e))))) + +(defn parse-date [{:keys [raw-date]}] + (when-not + (re-find #"\d{1,2}/\d{1,2}/\d{4}" raw-date) + (throw (Exception. (str "Date " raw-date " must match MM/dd/yyyy")))) + (try + + (parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date) + (catch Exception e + (throw (Exception. (str "Could not parse date from '" raw-date "'") e))))) + +(defn parse-or-error [key f] + (fn [x] + (try + (assoc x key (f x)) + (catch Exception e + (update x :errors conj {:info (.getMessage e) + :details (str e)}))))) + +(defn assoc-or-error [m k f] + (try + (assoc m k (f)) + (catch Exception e + (update m :errors conj {:info (.getMessage e) + :details (str e)})))) diff --git a/src/clj/auto_ap/import/plaid.clj b/src/clj/auto_ap/import/plaid.clj new file mode 100644 index 00000000..774d4d4f --- /dev/null +++ b/src/clj/auto_ap/import/plaid.clj @@ -0,0 +1,60 @@ +(ns auto-ap.import.plaid + (:require + [auto-ap.datomic :refer [conn]] + [auto-ap.plaid.core :as p] + [auto-ap.utils :refer [allow-once]] + [auto-ap.import.transactions :as t] + [clj-time.core :as time] + [clojure.tools.logging :as log] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler] + [clj-time.coerce :as coerce])) + + +(defn get-plaid-accounts [db] + (-> (d/q '[:find ?ba ?c ?external-id ?t + :in $ + :where + [?c :client/bank-accounts ?ba] + [?ba :bank-account/plaid-account ?pa] + [?pa :plaid-account/external-id ?external-id] + [?pi :plaid-item/accounts ?pa] + [?pi :plaid-item/access-token ?t]] + db ))) + + +(defn plaid->transaction [t] + #:transaction {:description-original (:name t) + :raw-id (:transaction_id t) + :id (digest/sha-256 (:transaction_id t)) + :amount (double (:amount t)) + :date (coerce/to-date (auto-ap.time/parse (:date t) auto-ap.time/iso-date)) + :status "POSTED"}) + + +(defn import-plaid [] + (lc/with-context {:source "Import plaid transactions"} + (let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user") + end (auto-ap.time/local-now) + start (time/plus end (time/days -30))] + (try + (doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn)) + transaction (:transactions (p/get-transactions access-token external-id start end))] + (when (not (:pending transaction)) + (t/import-transaction! import-batch (assoc (plaid->transaction transaction) + :transaction/bank-account bank-account-id + :transaction/client client-id)))) + (t/finish! import-batch) + (catch Exception e + (t/fail! import-batch e)))))) + +(def import-plaid (allow-once import-plaid)) + +(mount/defstate import-worker + :start (scheduler/every (* 1000 60 60 3) import-plaid) + :stop (scheduler/stop import-worker)) + + + diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj new file mode 100644 index 00000000..f2b2034a --- /dev/null +++ b/src/clj/auto_ap/import/transactions.clj @@ -0,0 +1,336 @@ +(ns auto-ap.import.transactions + (:require [auto-ap.datomic :refer [audit-transact conn remove-nils uri]] + [auto-ap.datomic.accounts :as a] + [auto-ap.datomic.checks :as d-checks] + [auto-ap.datomic.transaction-rules :as tr] + [auto-ap.datomic.transactions :as d-transactions] + [auto-ap.rule-matching :as rm] + [auto-ap.utils :refer [dollars=]] + [auto-ap.yodlee.core :as client] + [clj-time.coerce :as coerce] + [clj-time.core :as t] + [clojure.core.cache :as cache] + [clojure.tools.logging :as log] + [datomic.api :as d] + [digest :refer [sha-256]])) + +(defn rough-match [client-id bank-account-id amount] + (if (and client-id bank-account-id amount) + (let [[matching-checks] (d-checks/get-graphql {:client-id client-id + :bank-account-id bank-account-id + :amount (- amount) + :status :payment-status/pending})] + (if (= 1 (count matching-checks)) + (first matching-checks) + nil)) + nil)) + + +(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id] + (log/info "Searching for a matching check for " + {:client-id client-id + :check-number check-number + :bank-account-id bank-account-id + :amount amount}) + (cond (not (and client-id bank-account-id)) + nil + + (:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))])) + nil + + check-number + (or (-> (d-checks/get-graphql {:client-id client-id + :bank-account-id bank-account-id + :check-number check-number + :amount (- amount) + :status :payment-status/pending}) + first + first) + (rough-match client-id bank-account-id amount)) + + :else + (rough-match client-id bank-account-id amount))) + +(defn match-transaction-to-unfulfilled-autopayments [amount client-id] + (log/info "trying to find uncleared autopay invoices") + (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd] + :in ['$ '?client-id] + :where ['[?e :invoice/client ?client-id] + '[?e :invoice/scheduled-payment ?sd] + '[?e :invoice/status :invoice-status/paid] + '(not [_ :invoice-payment/invoice ?e]) + '[?e :invoice/vendor ?vendor-id] + '[?e :invoice/total ?total]]} + :args [(d/db conn) client-id]}) + (sort-by last) ;; sort by scheduled payment date + (group-by first) ;; group by vendors + vals) + considerations (for [candidate-invoices candidate-invoices-vendor-groups + invoice-count (range 1 32) + consideration (partition invoice-count 1 candidate-invoices) + :when (dollars= (reduce (fn [acc [_ _ amount]] + (+ acc amount)) 0.0 consideration) + (- amount))] + consideration)] + (log/info "Found " (count considerations) "considerations for transaction of" amount) + considerations + )) + +(defn match-transaction-to-unpaid-invoices [amount client-id] + (log/info "trying to find unpaid invoices for " client-id amount) + (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d] + :in ['$ '?client-id] + :where ['[?e :invoice/client ?client-id] + '[?e :invoice/status :invoice-status/unpaid] + '(not [_ :invoice-payment/invoice ?e]) + '[?e :invoice/vendor ?vendor-id] + '[?e :invoice/outstanding-balance ?outstanding-balance] + '[?e :invoice/date ?d]]} + :args [(d/db conn) client-id]}) + (sort-by last) ;; sort by scheduled payment date + (group-by first) ;; group by vendors + vals) + considerations (for [candidate-invoices candidate-invoices-vendor-groups + invoice-count (range 1 32) + consideration (partition invoice-count 1 candidate-invoices) + :when (dollars= (reduce (fn [acc [_ _ amount]] + (+ acc amount)) 0.0 consideration) + (- amount))] + consideration)] + (log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount) + considerations)) + +(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id] + (let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)] + (if (= 1 (count considerations)) + (first considerations) + []))) + +(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id] + (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) + (let [payment-id (d/tempid :db.part/user)] + (-> tx + + (conj {:payment/bank-account bank-account-id + :payment/client client-id + :payment/amount (- (:transaction/amount transaction)) + :payment/vendor vendor + :payment/date (:transaction/date transaction) + :payment/type :payment-type/debit + :payment/status :payment-status/cleared + :db/id payment-id}) + (into (mapcat (fn [[vendor invoice-id invoice-amount]] + [{:invoice-payment/invoice invoice-id + :invoice-payment/payment payment-id + :invoice-payment/amount invoice-amount} + {:db/id invoice-id + :invoice/outstanding-balance 0.0 + :invoice/status :invoice-status/paid}]) + invoice-payments)) + (update 0 assoc + :transaction/payment payment-id + :transaction/approval-status :transaction-approval-status/approved + :transaction/vendor vendor + :transaction/location "A") + (conj [:reset (:db/id transaction) :transaction/accounts + [#:transaction-account + {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :location "A" + :amount (Math/abs (:transaction/amount transaction))}]])))) + +(defn extract-check-number [{:transaction/keys [description-original]}] + + (if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)] + (try + (Integer/parseInt check-number) + (catch NumberFormatException e + nil)) + nil)) + +(defn find-expected-deposit [client-id amount date] + (when date + (-> (d/q + '[:find ?ed + :in $ ?c ?a ?d-start + :where + [?ed :expected-deposit/client ?c] + (not [?ed :expected-deposit/status :expected-deposit-status/cleared]) + [?ed :expected-deposit/date ?d] + [(>= ?d ?d-start)] + [?ed :expected-deposit/total ?a2] + [(auto-ap.utils/dollars= ?a2 ?a)] + ] + (d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) + first + first))) + + +(defn categorize-transaction [transaction bank-account existing] + (let [bank-account-id (:db/id bank-account) + client (:client/_bank-accounts bank-account) + client-id (:db/id client) + valid-locations (or (:bank-account/locations bank-account) (:client/locations client))] + (cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction))) + :suppressed + + (existing (:transaction/id transaction)) + :extant + + (not (:transaction/client transaction)) + :error + + (not (:transaction/bank-account transaction)) + :error + + (not (:transaction/id transaction)) + :error + + (not= "POSTED" (:transaction/status transaction)) + :not-posted + + (and (:bank-account/start-date bank-account) + (not (t/after? (coerce/to-date-time (:transaction/date transaction)) + (-> bank-account :bank-account/start-date coerce/to-date-time)))) + :not-ready + + :else + :import))) + +(defn transaction->txs [transaction bank-account apply-rules] + (let [bank-account-id (:db/id bank-account) + client (:client/_bank-accounts bank-account) + client-id (:db/id client) + valid-locations (or (:bank-account/locations bank-account) (:client/locations client))] + (into [] + (let [{:transaction/keys [amount id date]} transaction + check-number (extract-check-number transaction) + existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id) + autopay-invoices-matches (when-not existing-check + (match-transaction-to-unfulfilled-autopayments amount client-id)) + unpaid-invoices-matches (when-not existing-check + (match-transaction-to-unpaid-invoices amount client-id )) + expected-deposit (when (and (> amount 0.0) + (not existing-check)) + (find-expected-deposit (:db/id client) amount (coerce/to-date-time date)))] + (cond-> + [(assoc transaction :transaction/approval-status :transaction-approval-status/unapproved)] + check-number (update 0 #(assoc % :transaction/check-number check-number)) + existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved + :transaction/payment {:db/id (:db/id existing-check) + :payment/status :payment-status/cleared} + :transaction/vendor (:db/id (:payment/vendor existing-check)) + :transaction/location "A" + :transaction/accounts [#:transaction-account + {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :location "A" + :amount (Math/abs (double amount))}])) + + ;; temporarily removed to automatically match autopaid invoices + #_(and (not existing-check) + (seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id) + expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit + :expected-deposit/status :expected-deposit-status/cleared})) + + + (and (not (seq autopay-invoices-matches)) + (not (seq unpaid-invoices-matches)) + (not expected-deposit)) (update 0 #(apply-rules % valid-locations)) + true (update 0 remove-nils)))))) + + + + + +(defn get-existing [bank-account] + (log/info "looking up bank account data for" bank-account) + (into {} + (d/query {:query {:find ['?tid '?as2] + :in ['$ '?ba] + :where ['[?e :transaction/bank-account ?ba] + '[?e :transaction/id ?tid] + '[?e :transaction/approval-status ?as] + '[?as :db/ident ?as2]]} + :args [(d/db (d/connect uri)) bank-account]}))) + +(defprotocol ImportBatch + (import-transaction! [this transaction]) + (get-stats [this ]) + (finish! [this]) + (fail! [this error])) + +(defn start-import-batch [source user] + (let [stats (atom {:import-batch/imported 0 + :import-batch/suppressed 0 + :import-batch/error 0 + :import-batch/not-ready 0 + :import-batch/extant 0}) + extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 )) + import-id (get (:tempids @(d/transact conn [{:db/id "import-batch" + :import-batch/date (coerce/to-date (t/now)) + :import-batch/source source + :import-batch/status :import-status/started + :import-batch/user-name user}])) "import-batch") + rule-applying-function (rm/rule-applying-fn (tr/get-all))] + (log/info "Importing transactions from " source) + + (reify ImportBatch + (import-transaction! [this transaction] + (let [bank-account (d/pull (d/db conn) + [:bank-account/code + :db/id + :bank-account/locations + :bank-account/start-date + {:client/_bank-accounts [:client/code :client/locations :db/id]} ] + (:transaction/bank-account transaction)) + extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing) + (:transaction/bank-account transaction)) + action (categorize-transaction transaction bank-account extant) + transaction (assoc transaction :import-batch/_entry import-id)] + (swap! stats + #(update % (condp = action + :import :import-batch/imported + :extant :import-batch/extant + :suppressed :import-batch/suppressed + :error :import-batch/error + :not-read :import-batch/not-ready) inc)) + (when (= :import action) + (audit-transact (transaction->txs transaction bank-account rule-applying-function) + {:user/name user + :user/role ":admin"})))) + + (get-stats [this] + @stats) + + (fail! [this error] + (log/errorf "Couldn't complete import %d with error." import-id) + (log/error error) + @(d/transact conn [(merge {:db/id import-id + :import-batch/status :import-status/completed + :import-batch/error-message (str error)} + @stats)])) + + (finish! [this] + (log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats)) + @(d/transact conn [(merge {:db/id import-id + + :import-batch/status :import-status/completed} + @stats)]))))) + + + +(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index] + (str (str (some-> date coerce/to-date-time auto-ap.time/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client)) + +(defn apply-synthetic-ids [transactions] + (->> transactions + (group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client])) + (vals) + (mapcat (fn [group] + (map (fn [index transaction] + (let [raw-id (synthetic-key transaction index)] + (assoc transaction + :transaction/id (digest/sha-256 raw-id) + :transaction/raw-id raw-id))) + (range) + group))))) + diff --git a/src/clj/auto_ap/import/yodlee.clj b/src/clj/auto_ap/import/yodlee.clj new file mode 100644 index 00000000..0b2cbe7d --- /dev/null +++ b/src/clj/auto_ap/import/yodlee.clj @@ -0,0 +1,70 @@ +(ns auto-ap.import.yodlee + (:require [auto-ap.datomic :refer [conn]] + [auto-ap.import.transactions :as t] + [auto-ap.time :as atime] + [auto-ap.utils :refer [allow-once]] + [auto-ap.yodlee.core :as client] + [clj-time.coerce :as coerce] + [clojure.string :as str] + [datomic.api :as d] + [digest :refer [sha-256]] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler] + [clojure.tools.logging :as log])) + +(defn yodlee->transaction [transaction] + (let [{post-date :postDate + account-id :accountId + date :date + id :id + {amount :amount} :amount + {description-original :original + description-simple :simple} :description + {merchant-id :id + merchant-name :name} :merchant + base-type :baseType + type :type + status :status} transaction + amount (if (= "DEBIT" base-type) + (- amount) + amount) + date (atime/parse date "YYYY-MM-dd")] + #:transaction + {:post-date (coerce/to-date (atime/parse post-date "YYYY-MM-dd")) + :id (sha-256 (str id)) + :raw-id (str id) + :account-id account-id + :date (coerce/to-date date) + :amount (double amount) + :description-original (some-> description-original (str/replace #"\s+" " ")) + :description-simple (some-> description-simple (str/replace #"\s+" " ")) + :type type + :status status})) + +(defn import-yodlee [] + (lc/with-context {:source "Import yodlee transactions"} + (let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")] + (try + (let [account-lookup (d/q '[:find ?ya ?ba ?c + :in $ + :where [?ba :bank-account/yodlee-account-id ?ya] + [?c :client/bank-accounts ?ba]] + (d/db conn))] + (doseq [[yodlee-account bank-account client-id] account-lookup + transaction (client/get-specific-transactions yodlee-account)] + (log/info "importing") + (t/import-transaction! import-batch (assoc (yodlee->transaction transaction) + :transaction/bank-account bank-account + :transaction/client client-id))) + + (t/finish! import-batch)) + (catch Exception e + (t/fail! import-batch e)))))) + +(def import-yodlee (allow-once import-yodlee)) + + +(mount/defstate import-worker + :start (scheduler/every (* 1000 60 60 4) import-yodlee) + :stop (scheduler/stop import-worker)) diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj new file mode 100644 index 00000000..5f1b9198 --- /dev/null +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -0,0 +1,45 @@ +(ns auto-ap.import.yodlee2 + (:require [auto-ap.datomic :refer [conn]] + [auto-ap.import.transactions :as t] + [auto-ap.import.yodlee :as y] + [auto-ap.utils :refer [allow-once]] + [auto-ap.yodlee.core2 :as client2] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) + +(defn import-yodlee2 [] + (lc/with-context {:source "Import yodlee2 transactions"} + (let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")] + (try + (let [account-lookup (d/q '[:find ?ya ?ba ?cd + :in $ + :where + [?ba :bank-account/yodlee-account ?y] + [?c :client/bank-accounts ?ba] + [?c :client/code ?cd] + [?y :yodlee-account/id ?ya]] + (d/db conn))] + (doseq [[yodlee-account bank-account client-code] account-lookup + transaction (client2/get-specific-transactions client-code yodlee-account)] + (t/import-transaction! import-batch (assoc (y/yodlee->transaction transaction) + :transaction/bank-account bank-account + :transaction/client [:client/code client-code]))) + + (t/finish! import-batch)) + (catch Exception e + (t/fail! import-batch e)))))) + + + +(def import-yodlee2 (allow-once import-yodlee2)) + + +(mount/defstate import-worker + :start (scheduler/every (* 1000 60 60 4) import-yodlee2) + :stop (scheduler/stop import-worker)) + +(mount/defstate account-worker + :start (scheduler/every (* 5 60 1000) client2/upsert-accounts) + :stop (scheduler/stop account-worker)) diff --git a/src/clj/auto_ap/intuit/import.clj b/src/clj/auto_ap/intuit/import.clj deleted file mode 100644 index cd191215..00000000 --- a/src/clj/auto_ap/intuit/import.clj +++ /dev/null @@ -1,95 +0,0 @@ -(ns auto-ap.intuit.import - (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [conn remove-nils]] - [auto-ap.intuit.core :as i] - [auto-ap.utils :refer [by allow-once]] - [auto-ap.yodlee.import :as y] - [clj-time.coerce :as coerce] - [clj-time.core :as time] - [clj-time.format :as f] - [clojure.string :as str] - [clojure.tools.logging :as log] - [datomic.api :as d] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) - -(def client-whitelist #{"PPFB"}) - -(defn get-intuit-bank-accounts [db] - (-> (d/q '[:find ?ba ?external-id - :in $ - :where - [?c :client/bank-accounts ?ba] - [?ba :bank-account/intuit-bank-account ?iab] - [?iab :intuit-bank-account/external-id ?external-id]] - db #_client-whitelist))) - - -(defn upsert-transactions [] - (lc/with-context {:source "Importing intuit transactions"} - (let [db (d/db conn)] - (->> - (for [[bank-account external-id] (get-intuit-bank-accounts db) - :let [bank-account (d/entity db bank-account) - end (auto-ap.time/local-now) - start (time/plus end (time/days -30)) - _ (log/infof "importing from %s to %s for %s" start end external-id)] - transaction (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) - (auto-ap.time/unparse end auto-ap.time/iso-date) - external-id)] - {:client-id (:db/id (:client/_bank-accounts bank-account)) - :bank-account-id (:db/id bank-account) - :description-original (:Memo/Description transaction) - :amount (Double/parseDouble (:Amount transaction)) - :date (auto-ap.time/parse (:Date transaction) auto-ap.time/iso-date) - :status "posted"}) - (y/grouped-import :import-source/intuit "Automated Intuit User")) - (log/info "Intuit transactions imported")))) - -(def upsert-transactions (allow-once upsert-transactions)) - -(defn dry-run-upsert-transactions [] - (let [db (d/db conn)] - (clojure.data.csv/write-csv - *out* - (->> - (for [[bank-account external-id] (get-intuit-bank-accounts db) - :let [bank-account (d/entity db bank-account) - end (auto-ap.time/local-now) - start (time/plus end (time/days -30))] - r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) - (auto-ap.time/unparse end auto-ap.time/iso-date) - external-id)] - {:client-id (:db/id (:client/_bank-accounts bank-account)) - :client-code (:client/code (:client/_bank-accounts bank-account)) - :bank-account-id (:db/id bank-account) - :bank-account-code (:bank-account/code bank-account) - :external-id external-id - :description-original (:Memo/Description r) - :amount (Double/parseDouble (:Amount r)) - :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) - :status "posted"}) - (y/grouped-new) - :import - (mapcat identity) - (map (juxt #(:bank-account/code (d/entity db(:transaction/bank-account %))) :transaction/id :transaction/raw-id :transaction/amount :transaction/description-original #(auto-ap.time/unparse-local (coerce/to-date-time (:transaction/date %)) auto-ap.time/normal-date) ))) - :separator \tab))) - - -(defn upsert-accounts [] - (let [token (i/get-fresh-access-token) - bank-accounts (i/get-bank-accounts token)] - @(d/transact conn (mapv - (fn [ba] - {:intuit-bank-account/external-id (:name ba) - :intuit-bank-account/name (:name ba)}) - bank-accounts)))) - -(mount/defstate upsert-transaction-worker - :start (scheduler/every (* 1000 60 60 24) upsert-transactions) - :stop (scheduler/stop upsert-transaction-worker)) - -(mount/defstate upsert-account-worker - :start (scheduler/every (* 1000 60 60 24) upsert-accounts) - :stop (scheduler/stop upsert-account-worker)) diff --git a/src/clj/auto_ap/plaid/core.clj b/src/clj/auto_ap/plaid/core.clj index c4bb0680..a0ab0888 100644 --- a/src/clj/auto_ap/plaid/core.clj +++ b/src/clj/auto_ap/plaid/core.clj @@ -58,6 +58,7 @@ :body)) (defn get-transactions [access-token account-id start end] + (log/infof "looking up transactions from %s to %s for %s" start end account-id) (-> (client/post (str base-url "/transactions/get") {:as :json :headers {"Content-Type" "application/json"} diff --git a/src/clj/auto_ap/plaid/import.clj b/src/clj/auto_ap/plaid/import.clj index 4ed77a27..f124d709 100644 --- a/src/clj/auto_ap/plaid/import.clj +++ b/src/clj/auto_ap/plaid/import.clj @@ -9,10 +9,11 @@ [datomic.api :as d] [mount.core :as mount] [unilog.context :as lc] - [yang.scheduler :as scheduler])) + [yang.scheduler :as scheduler] + [clj-time.coerce :as coerce])) (defn get-plaid-accounts [db] - (-> (d/q '[:find ?ba ?external-id ?t + (-> (d/q '[:find ?ba ?c ?external-id ?t :in $ :where [?c :client/bank-accounts ?ba] @@ -22,47 +23,36 @@ [?pi :plaid-item/access-token ?t]] db ))) -(defn import-plaid - [] - (lc/with-context {:source "Importing plaid transactions"} - (let [db (d/db conn) - import-id (y/start-import :import-source/plaid "Automated Plaid User")] + +(defn plaid->transaction [t] + #:transaction {:description-original (:name t) + :raw-id (:transaction_id t) + :id (digest/sha-256 (:transaction_id t)) + :amount (double (:amount t)) + :date (coerce/to-date (auto-ap.time/parse (:date t) auto-ap.time/iso-date)) + :status "POSTED"}) + + +(defn import-plaid [] + (lc/with-context {:source "Import plaid transactions"} + (let [import-batch (y/start-import-batch :import-source/plaid "Automated plaid user") + end (auto-ap.time/local-now) + start (time/plus end (time/days -30))] (try - (let [result (->> - (for [[bank-account external-id access-token] (get-plaid-accounts db) - :let [end (auto-ap.time/local-now) - start (time/plus end (time/days -30)) - _ (log/infof "importing from %s to %s for %s" start end external-id) - transactions (p/get-transactions access-token external-id start end) - transactions (->> transactions - :transactions - (filter (fn [t] - (not (:pending t)))) - (mapv (fn [t] - {:description {:original (:name t) - :simple (:name t)} - :id (:transaction_id t) - :amount {:amount (:amount t)} - :date (:date t) - :status "POSTED"})) - )]] - - - (y/import-for-bank-account transactions bank-account import-id)) - y/aggregate-results)] - (log/info "Plaid transactions imported" result) - (y/finish-import (assoc result :db/id import-id))) + (doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn)) + transaction (:transactions (p/get-transactions access-token external-id start end))] + (when (not (:pending transaction)) + (y/import-transaction! import-batch (assoc (plaid->transaction transaction) + :transaction/bank-account bank-account-id + :transaction/client client-id)))) + (y/finish! import-batch) (catch Exception e - (log/error e) - (y/finish-import {:db/id import-id})))))) + (y/fail! import-batch e)))))) -(defn do-import [] - (import-plaid)) - -(def do-import (allow-once do-import)) +(def import-plaid (allow-once import-plaid)) (mount/defstate import-worker - :start (scheduler/every (* 1000 60 60 3) do-import) + :start (scheduler/every (* 1000 60 60 3) import-plaid) :stop (scheduler/stop import-worker)) diff --git a/src/clj/auto_ap/routes/events.clj b/src/clj/auto_ap/routes/events.clj index a9707ba6..63fb04e1 100644 --- a/src/clj/auto_ap/routes/events.clj +++ b/src/clj/auto_ap/routes/events.clj @@ -1,6 +1,5 @@ (ns auto-ap.routes.events (:require [auto-ap.routes.utils :refer [wrap-secure]] - [auto-ap.yodlee.import :as yodlee-import] [config.core :refer [env]] [clj-http.client :as http] [clj-time.coerce :as c] diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 9ce3ba29..cc0289fd 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -5,13 +5,14 @@ [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] + [auto-ap.import.manual :as manual] + [auto-ap.import.manual.common :as c] [auto-ap.logging :refer [info-event]] [auto-ap.parse :as parse] [auto-ap.parse.util :as parse-u] [auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.time-utils :refer [next-dom]] [auto-ap.utils :refer [by]] - [auto-ap.yodlee.import :refer [grouped-import] :as y] [clj-time.coerce :as coerce :refer [to-date]] [clj-time.core :as time] [clojure.data.csv :as csv] @@ -73,50 +74,6 @@ (get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor)) client-id)) -(defn parse-amount [i] - (try - (Double/parseDouble (str/replace (or (second - (re-matches #"[^0-9\.,\\-]*([0-9\.,\\-]+)[^0-9\.,]*" (:amount i))) - "0") - #"," "")) - (catch Exception e - (throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e))))) - -(defn parse-account-numeric-code [i] - (try - (when-let [account-numeric-code (:account-numeric-code i)] - (:db/id (d-accounts/get-account-by-numeric-code-and-sets (Integer/parseInt account-numeric-code) - ["default"]))) - (catch Exception e - (throw (Exception. (str "Could not parse expense account from value '" (:account-numeric-code i) "'") e))))) - -(defn parse-account-id [i] - (try - (Long/parseLong (second - (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i)))) - (catch Exception e - (throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e))))) - - - -(defn parse-date [{:keys [raw-date]}] - (when-not - (re-find #"\d{1,2}/\d{1,2}/\d{4}" raw-date) - (throw (Exception. (str "Date " raw-date " must match MM/dd/yyyy")))) - (try - - (parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date) - (catch Exception e - (throw (Exception. (str "Could not parse date from '" raw-date "'") e))))) - -(defn parse-or-error [key f] - (fn [x] - (try - (assoc x key (f x)) - (catch Exception e - (update x :errors conj {:info (.getMessage e) - :details (str e)}))))) - (defn parse-invoice-rows [excel-rows] (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code] all-vendors (by :vendor/name (d-vendors/get-graphql {})) @@ -127,15 +84,15 @@ (map #(into {} (map (fn [c k] [k c] ) % columns))) (map reset-id) (map assoc-client-code) - (map (parse-or-error :client-id #(parse-client % all-clients))) - (map (parse-or-error :vendor #(parse-vendor % all-vendors))) - (map (parse-or-error :vendor-id #(parse-vendor-id %))) - (map (parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %))) - (map (parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %))) - (map (parse-or-error :account-id parse-account-numeric-code)) - (map (parse-or-error :invoice-number parse-invoice-number)) - (map (parse-or-error :total parse-amount)) - (map (parse-or-error :date parse-date)))] + (map (c/parse-or-error :client-id #(parse-client % all-clients))) + (map (c/parse-or-error :vendor #(parse-vendor % all-vendors))) + (map (c/parse-or-error :vendor-id #(parse-vendor-id %))) + (map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %))) + (map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %))) + (map (c/parse-or-error :account-id c/parse-account-numeric-code)) + (map (c/parse-or-error :invoice-number parse-invoice-number)) + (map (c/parse-or-error :total c/parse-amount)) + (map (c/parse-or-error :date c/parse-date)))] rows)) @@ -428,6 +385,9 @@ rows)] @(d/transact (d/connect uri) txes))) + + + (defroutes routes (wrap-routes (context "/" [] @@ -435,55 +395,18 @@ (POST "/batch-upload" {{:keys [data]} :edn-params user :identity} (assert-admin user) - (try - #_(throw (Exception. "Unexpected error")) - (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code] - all-clients (d-clients/get-all) - all-bank-accounts (mapcat :client/bank-accounts all-clients) - all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) - all-bank-accounts (merge (by :bank-account/code all-bank-accounts)) - rows (->> (str/split data #"\n" ) - (drop 1) - (map #(str/split % #"\t")) - (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) - (map (parse-or-error :amount parse-amount)) - (map (parse-or-error :date parse-date)) - (map (fn [{:keys [bank-account-code] :as row}] - (if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))] - (assoc row :bank-account-id bank-account-id) - (update row :errors conj {:info (str "Cannot find bank by code " bank-account-code) - :details (str "Cannot find bank by code " bank-account-code)})))) - (map (fn [{:keys [client-code] :as row}] - (if-let [client-id (:db/id (all-clients client-code))] - (assoc row :client-id client-id) - (update row :errors conj {:info (str "Cannot find client by code " client-code) - :details (str "Cannot find client by code " client-code)}))))) - error-rows (filter :errors rows) - _ (log/info "Importing " (count rows) "raw transactions") - raw-transactions (vec (->> rows - (filter #(not (seq (:errors %))) ) - (map (fn [row] - (select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id]))))) - import-id (y/start-import :import-source/manual (:user/name user)) - result (grouped-import raw-transactions import-id)] - (y/finish-import {:db/id import-id - :import-batch/imported (count (:import result)) - :import-batch/suppressed (count (:suppressed result)) - :import-batch/extant (count (:extant result))}) - - {:status 200 - :body (pr-str {:imported (count (:import result)) - :errors (map #(dissoc % :date) error-rows)}) - :headers {"Content-Type" "application/edn"}}) + (let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))] + {:status 200 + :body (pr-str stats) + :headers {"Content-Type" "application/edn"}}) (catch Exception e (log/error e) {:status 500 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) - :headers {"Content-Type" "application/edn"}})) - )) + :headers {"Content-Type" "application/edn"}})))) (context "/invoices" [] (POST "/upload" diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 0681e660..0fd2417d 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -2,7 +2,6 @@ (:require [auto-ap.handler :refer [app]] [auto-ap.ledger :as ledger] [auto-ap.yodlee.core] - [auto-ap.yodlee.core2 :as yodlee2] [auto-ap.graphql.clients :as gq-clients] [auto-ap.background.invoices] [auto-ap.background.requests :as requests] @@ -10,8 +9,10 @@ [auto-ap.background.vendor :as vendor] [auto-ap.square.core :as square] [auto-ap.datomic.migrate :as migrate] - [auto-ap.yodlee.import :as yodlee] - [auto-ap.intuit.import :as intuit] + [auto-ap.import.yodlee :as yodlee] + [auto-ap.import.yodlee2 :as yodlee2] + [auto-ap.import.intuit :as intuit] + [auto-ap.import.plaid :as plaid] [nrepl.server :refer [start-server stop-server]] [config.core :refer [env]] [ring.adapter.jetty :refer [run-jetty]] @@ -37,8 +38,6 @@ (not (env :run-web? )) (into [#'jetty]) (not (env :run-background?)) (into [#'square/square-loader #'vendor/refresh-vendor-usages-worker - #'intuit/upsert-transaction-worker - #'intuit/upsert-account-worker #'ledger/touch-broken-ledger-worker #'ledger/process-txes-worker #'ledger/ledger-reconciliation-worker @@ -46,8 +45,14 @@ #'sysco/sysco-invoice-importer #'auto-ap.background.invoices/close-auto-invoices-worker #'gq-clients/current-balance-worker - #'yodlee/import-transaction-worker - #'yodlee2/yodlee-sync-worker + + #'yodlee/import-worker + #'yodlee2/import-worker + #'yodlee2/account-worker + #'intuit/import-worker + #'intuit/account-worker + #'plaid/import-worker + #'migrate/migrate-start]))] (log/info "starting without " without) diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj index e08ffe13..15548635 100644 --- a/src/clj/auto_ap/yodlee/core.clj +++ b/src/clj/auto_ap/yodlee/core.clj @@ -209,6 +209,7 @@ (defn get-specific-transactions [account] + (log/infof "Getting yodlee transactions for account %s" account) (let [cob-session (login-cobrand) user-session (login-user cob-session) batch-size 100 diff --git a/src/clj/auto_ap/yodlee/core2.clj b/src/clj/auto_ap/yodlee/core2.clj index 20f116e6..c29fbb6b 100644 --- a/src/clj/auto_ap/yodlee/core2.clj +++ b/src/clj/auto_ap/yodlee/core2.clj @@ -322,9 +322,7 @@ (log/info "Current yodlee state is " result) @(d/transact conn result)))) -(mount/defstate yodlee-sync-worker - :start (scheduler/every (* 5 60 1000) upsert-accounts) - :stop (scheduler/stop upsert-accounts)) + (defn update-yodlee [id] (update-provider-account id) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj deleted file mode 100644 index 98ff5647..00000000 --- a/src/clj/auto_ap/yodlee/import.clj +++ /dev/null @@ -1,485 +0,0 @@ -(ns auto-ap.yodlee.import - (:require - [auto-ap.datomic :refer [audit-transact conn remove-nils uri]] - [auto-ap.datomic.accounts :as a] - [auto-ap.datomic.checks :as d-checks] - [auto-ap.datomic.clients :as d-clients] - [auto-ap.datomic.transaction-rules :as tr] - [auto-ap.datomic.transactions :as d-transactions] - [auto-ap.rule-matching :as rm] - [auto-ap.time :as time] - [auto-ap.utils :refer [allow-once by dollars=]] - [auto-ap.yodlee.core :as client] - [auto-ap.yodlee.core2 :as client2] - [clj-time.coerce :as coerce] - [clj-time.core :as t] - [clojure.string :as str] - [clojure.tools.logging :as log] - [datomic.api :as d] - [digest :refer [sha-256]] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) - -(defn rough-match [client-id bank-account-id amount] - (if (and client-id bank-account-id amount) - (let [[matching-checks] (d-checks/get-graphql {:client-id client-id - :bank-account-id bank-account-id - :amount (- amount) - :status :payment-status/pending})] - (if (= 1 (count matching-checks)) - (first matching-checks) - nil)) - nil)) - - -(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id] - (log/info "Searching for a matching check for " - {:client-id client-id - :check-number check-number - :bank-account-id bank-account-id - :amount amount}) - (cond (not (and client-id bank-account-id)) - nil - - (:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))])) - nil - - check-number - (or (-> (d-checks/get-graphql {:client-id client-id - :bank-account-id bank-account-id - :check-number check-number - :amount (- amount) - :status :payment-status/pending}) - first - first) - (rough-match client-id bank-account-id amount)) - - :else - (rough-match client-id bank-account-id amount))) - -(defn aggregate-results [results] - (reduce - (fn [acc result] - (merge-with + acc - {:import-batch/imported (count (:import result)) - :import-batch/extant (count (:extant result)) - :import-batch/suppressed (count (:suppressed result))})) - - {:import-batch/imported 0 - :import-batch/extant 0 - :import-batch/suppressed 0} - results)) - - - -(defn match-transaction-to-unfulfilled-autopayments [amount client-id] - (log/info "trying to find uncleared autopay invoices") - (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd] - :in ['$ '?client-id] - :where ['[?e :invoice/client ?client-id] - '[?e :invoice/scheduled-payment ?sd] - '[?e :invoice/status :invoice-status/paid] - '(not [_ :invoice-payment/invoice ?e]) - '[?e :invoice/vendor ?vendor-id] - '[?e :invoice/total ?total]]} - :args [(d/db conn) client-id]}) - (sort-by last) ;; sort by scheduled payment date - (group-by first) ;; group by vendors - vals) - considerations (for [candidate-invoices candidate-invoices-vendor-groups - invoice-count (range 1 32) - consideration (partition invoice-count 1 candidate-invoices) - :when (dollars= (reduce (fn [acc [_ _ amount]] - (+ acc amount)) 0.0 consideration) - (- amount))] - consideration)] - (log/info "Found " (count considerations) "considerations for transaction of" amount) - considerations - )) - -(defn match-transaction-to-unpaid-invoices [amount client-id] - (log/info "trying to find unpaid invoices for " client-id amount) - (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d] - :in ['$ '?client-id] - :where ['[?e :invoice/client ?client-id] - '[?e :invoice/status :invoice-status/unpaid] - '(not [_ :invoice-payment/invoice ?e]) - '[?e :invoice/vendor ?vendor-id] - '[?e :invoice/outstanding-balance ?outstanding-balance] - '[?e :invoice/date ?d]]} - :args [(d/db conn) client-id]}) - (sort-by last) ;; sort by scheduled payment date - (group-by first) ;; group by vendors - vals) - considerations (for [candidate-invoices candidate-invoices-vendor-groups - invoice-count (range 1 32) - consideration (partition invoice-count 1 candidate-invoices) - :when (dollars= (reduce (fn [acc [_ _ amount]] - (+ acc amount)) 0.0 consideration) - (- amount))] - consideration)] - (log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount) - considerations)) - -(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id] - (let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)] - (if (= 1 (count considerations)) - (first considerations) - []))) - -(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id] - (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) - (let [payment-id (d/tempid :db.part/user)] - (-> tx - - (conj {:payment/bank-account bank-account-id - :payment/client client-id - :payment/amount (- (:transaction/amount transaction)) - :payment/vendor vendor - :payment/date (:transaction/date transaction) - :payment/type :payment-type/debit - :payment/status :payment-status/cleared - :db/id payment-id}) - (into (mapcat (fn [[vendor invoice-id invoice-amount]] - [{:invoice-payment/invoice invoice-id - :invoice-payment/payment payment-id - :invoice-payment/amount invoice-amount} - {:db/id invoice-id - :invoice/outstanding-balance 0.0 - :invoice/status :invoice-status/paid}]) - invoice-payments)) - (update 0 assoc - :transaction/payment payment-id - :transaction/approval-status :transaction-approval-status/approved - :transaction/vendor vendor - :transaction/location "A") - (conj [:reset (:db/id transaction) :transaction/accounts - [#:transaction-account - {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :location "A" - :amount (Math/abs (:transaction/amount transaction))}]])))) - -(defn extract-check-number [{{description-original :original} :description}] - - (if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)] - (try - (Integer/parseInt check-number) - (catch NumberFormatException e - nil)) - nil)) - -(defn find-expected-deposit [client-id amount date] - (when date - (-> (d/q - '[:find ?ed - :in $ ?c ?a ?d-start - :where - [?ed :expected-deposit/client ?c] - (not [?ed :expected-deposit/status :expected-deposit-status/cleared]) - [?ed :expected-deposit/date ?d] - [(>= ?d ?d-start)] - [?ed :expected-deposit/total ?a2] - [(auto-ap.utils/dollars= ?a2 ?a)] - ] - (d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) - first - first))) - -(defn transactions->txs [transactions bank-account import-id apply-rules existing] - (let [client (:client/_bank-accounts bank-account) - client-id (:db/id client) - bank-account-id (:db/id bank-account) - valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) - grouped-transactions (group-by - (fn [{id :id - date :date - status :status :as transaction}] - (let [date (time/parse date "YYYY-MM-dd") - id (sha-256 (str id))] - (cond (= :transaction-approval-status/suppressed (existing id)) - :suppressed - - (existing id) - :extant - - (not client) - :error - - (not bank-account) - :error - - (not= "POSTED" status) - :not-posted - - (and (:bank-account/start-date bank-account) - (not (t/after? date (-> bank-account :bank-account/start-date coerce/to-date-time)))) - :not-ready - - :else - :import))) - transactions)] - - (update grouped-transactions :import - (fn [transactions] - (into [] - - (for [transaction transactions - :let [{post-date :postDate - account-id :accountId - date :date - id :id - {amount :amount} :amount - {description-original :original - description-simple :simple} :description - {merchant-id :id - merchant-name :name} :merchant - base-type :baseType - type :type - status :status} - transaction - amount (if (= "DEBIT" base-type) - (- amount) - amount) - check-number (extract-check-number transaction) - date (time/parse date "YYYY-MM-dd")]] - (let [ - existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id) - autopay-invoices-matches (when-not existing-check - (match-transaction-to-unfulfilled-autopayments amount client-id)) - unpaid-invoices-matches (when-not existing-check - (match-transaction-to-unpaid-invoices amount client-id )) - expected-deposit (when (and (> amount 0.0) - (not existing-check)) - (find-expected-deposit (:db/id client) amount date))] - (cond-> - [#:transaction - {:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd")) - :id (sha-256 (str id)) - :raw-id (str id) - :account-id account-id - :date (coerce/to-date date) - :amount (double amount) - :import-batch/_entry import-id - :description-original (some-> description-original (str/replace #"\s+" " ")) - :description-simple (some-> description-simple (str/replace #"\s+" " ")) - :approval-status :transaction-approval-status/unapproved - :type type - :status status - :client client-id - :check-number check-number - :bank-account bank-account-id}] - existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved - :transaction/payment {:db/id (:db/id existing-check) - :payment/status :payment-status/cleared} - :transaction/vendor (:db/id (:payment/vendor existing-check)) - :transaction/location "A" - :transaction/accounts [#:transaction-account - {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :location "A" - :amount (Math/abs (double amount))}])) - - ;; temporarily removed to automatically match autopaid invoices - #_(and (not existing-check) - (seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id) - expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit - :expected-deposit/status :expected-deposit-status/cleared})) - - - (and (not (seq autopay-invoices-matches)) - (not (seq unpaid-invoices-matches)) - (not expected-deposit)) (update 0 #(apply-rules % valid-locations)) - true (update 0 remove-nils))))))))) - - -(defn batch-transact [transactions] - (reduce (fn [acc batch] - (into acc (->> batch - (d/transact (d/connect uri) ) - (deref) - :tempids - vals))) - [] - (partition-all 100 transactions))) - - - -(defn get-existing [bank-account] - (into {} - (d/query {:query {:find ['?tid '?as2] - :in ['$ '?ba] - :where ['[?e :transaction/bank-account ?ba] - '[?e :transaction/id ?tid] - '[?e :transaction/approval-status ?as] - '[?as :db/ident ?as2]]} - :args [(d/db (d/connect uri)) bank-account]}))) - -(defn get-all-bank-accounts [] - (->> (d-clients/get-all) - (mapcat (fn [client] - (->> client - :client/bank-accounts - #_(filter :bank-account/yodlee-account-id) - (filter :db/id) - (map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}] - (assoc bank-account - :client/_bank-accounts client)))))))) -(defn start-import [source user] - (get (:tempids @(d/transact conn [{:db/id "import-batch" - :import-batch/date (coerce/to-date (t/now)) - :import-batch/source source - :import-batch/status :import-status/started - :import-batch/user-name user}])) "import-batch")) - -(defn finish-import [i] - @(d/transact conn [(assoc i :import-batch/status :import-status/completed)])) - -(defn import-for-bank-account [transactions bank-account-id import-id] - (let [all-rules (tr/get-all) - bank-account (d/pull (d/db conn) - [:bank-account/code - :db/id - :bank-account/locations - :bank-account/start-date - {:client/_bank-accounts [:client/code :client/locations :db/id]} ] - bank-account-id) - _ (log/infof "Importing %d transactions for bank account %s for client %s" (count transactions) (-> bank-account :bank-account/code) (-> bank-account :client/_bank-accounts :client/code)) - transactions (transactions->txs transactions bank-account import-id (rm/rule-applying-fn all-rules) (get-existing bank-account-id))] - (doseq [tx (:import transactions)] - (audit-transact tx {:user/name "Yodlee import" - :user/role ":admin"})) - (log/infof "Completed import for %s" bank-account) - transactions)) - -(defn grouped-import [source user manual-transactions] - (let [import-id (start-import source user)] - (lc/with-context {:import-id import-id} - (try - (let [transactions-by-bank-account (->> manual-transactions - (filter #(= "posted" (:status %))) - (group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id])) - (vals) - (mapcat (fn [transaction-group] - (map - (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] - {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) - :bank-account-id bank-account-id - :date (time/unparse date "YYYY-MM-dd") - :amount {:amount amount} - :description {:original description-original - :simple high-level-category} - :status "POSTED"}) - (range) - transaction-group))) - (group-by :bank-account-id)) - result (->> transactions-by-bank-account - (filter (fn [[bank-account]] - bank-account)) - (map (fn [[bank-account transactions]] - (import-for-bank-account transactions bank-account import-id))) - aggregate-results)] - (finish-import (assoc result :db/id import-id))) - (catch Exception e - (log/error e) - (finish-import {:db/id import-id}) - ))))) - -(defn grouped-new [manual-transactions] - (let [transformed-transactions (->> manual-transactions - (filter #(= "posted" (:status %))) - (group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id])) - (vals) - (mapcat (fn [transaction-group] - (map - (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] - {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) - :bank-account-id bank-account-id - :date (time/unparse date "YYYY-MM-dd") - :amount {:amount amount} - :description {:original description-original - :simple high-level-category} - :status "POSTED"}) - (range) - transaction-group)))) - all-rules (tr/get-all) - all-bank-accounts (by :db/id (get-all-bank-accounts)) - transaction->bank-account (comp all-bank-accounts :bank-account-id) - transactions (transactions->txs transformed-transactions nil transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] - transactions)) - - - -(defn import-yodlee [transactions] - (lc/with-context {:source "Import yodlee transactions"} - (let [import-id (start-import :import-source/yodlee "Automated Yodlee User")] - (lc/with-context {:import-id import-id} - (try - (log/info "importing from yodlee") - (let [account-lookup (->> (d/q '[:find ?ya ?ba - :in $ - :where [?ba :bank-account/yodlee-account-id ?ya]] - (d/db conn)) - (into {})) - transactions-by-bank-account (group-by - #(account-lookup (:accountId %)) - transactions) - result (->> transactions-by-bank-account - (filter (fn [[bank-account]] - bank-account)) - (map (fn [[bank-account transactions]] - (import-for-bank-account transactions bank-account import-id))) - aggregate-results)] - - - (finish-import (assoc result :db/id import-id))) - (catch Exception e - (log/error e) - (finish-import {:db/id import-id}))))))) - -(defn do-import [] - (import-yodlee (client/get-transactions))) - -(def do-import (allow-once do-import)) - - -(defn import-yodlee2 - [transactions] - (lc/with-context {:source "Import yodlee2 transactions"} - (let [import-id (start-import :import-source/yodlee2 "Automated Yodlee User")] - (lc/with-context {:import-id import-id} - (try - (let [account-lookup (->> (d/q '[:find ?ya ?ba - :in $ - :where - [?ba :bank-account/yodlee-account ?y] - [?y :yodlee-account/id ?ya]] - (d/db conn)) - (into {})) - transactions-by-bank-account (group-by - #(account-lookup (:accountId %)) - transactions) - result (->> transactions-by-bank-account - (filter (fn [[bank-account]] - bank-account)) - (map (fn [[bank-account transactions]] - (import-for-bank-account transactions bank-account import-id))) - aggregate-results)] - - - (finish-import (assoc result :db/id import-id))) - (catch Exception e - (log/error e) - (finish-import {:db/id import-id}))))))) - -(defn do-import2 [] - (log/info "starting import") - (import-yodlee2 (client2/get-transactions "NGGL"))) - -(def do-import2 (allow-once do-import2)) - - - - -(mount/defstate import-transaction-worker - :start (scheduler/every (* 1000 60 60 4) do-import) - :stop (scheduler/stop import-transaction-worker)) diff --git a/src/clj/user.clj b/src/clj/user.clj index d7b9ec86..4f94f0f9 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -727,24 +727,3 @@ (mapcat identity) vec)) -(defn manually-add-transaction [] - (auto-ap.yodlee.import/transactions->txs [{:postDate "2014-01-04" - :accountId 1234 - :date "2021-06-05" - :id 1 - :amount {:amount -1743.25} - :description {:original "original-description" - :simple "simple-description"} - :merchant {:id "123" - :name "456"} - :baseType "DEBIT" - :status "POSTED" - - :bank-account {:db/id [:bank-account/code "NGAK-1"] - :client/_bank-accounts {:db/id 17592186045456 - :client/locations ["MH"]}}}] - :bank-account - (fn noop-rule [transaction locations] - transaction) - #{})) - diff --git a/src/cljs/auto_ap/views/pages/transactions.cljs b/src/cljs/auto_ap/views/pages/transactions.cljs index 93b593df..d323efd1 100644 --- a/src/cljs/auto_ap/views/pages/transactions.cljs +++ b/src/cljs/auto_ap/views/pages/transactions.cljs @@ -17,7 +17,8 @@ [re-frame.core :as re-frame] [reagent.core :as reagent] [vimsical.re-frame.fx.track :as track] - [auto-ap.status :as status])) + [auto-ap.status :as status] + [clojure.string :as str])) @@ -134,12 +135,21 @@ {:id ::manual-import :events #{::manual/import-completed} :event-fn (fn [[_ {:keys [imported errors] :as result}]] - [::status/info ::manual-import (str "Successfully imported " imported " transactions" - - (when (seq errors) - (str - ", " (count errors) " errors. " - "Example: '" (str (:info (first (:errors (first errors))))) "'")))])}]})) + (println result) + [::status/info ::manual-import + (str "Successfully " + (str/join ", " + [(when-let [imported (:import-batch/imported result)] + (str "imported " imported)) + (when-let [extant (:import-batch/extant result)] + (str "extant " extant)) + (when-let [suppressed (:import-batch/suppressed result)] + (str "suppressed " suppressed)) + (when-let [error (:validation-error result)] + (str "errored " error))]) + " transactions." + (when (:sample-error result) + (str " Sample error: " (:info (:sample-error result)))))])}]})) (defn content [] diff --git a/test/clj/auto_ap/import/manual_test.clj b/test/clj/auto_ap/import/manual_test.clj new file mode 100644 index 00000000..381287e3 --- /dev/null +++ b/test/clj/auto_ap/import/manual_test.clj @@ -0,0 +1,60 @@ +(ns auto-ap.import.manual-test + (:require [auto-ap.datomic :refer [conn uri]] + [auto-ap.import.manual :as sut] + [auto-ap.time :as time] + [clj-time.coerce :as coerce] + [clojure.test :as t] + [datomic.api :as d] + [auto-ap.datomic.migrate :as m])) + +(def raw-tsv "Status Date Original Description High Level Category Category-Subcategory End Here Amount Amount Split Type Note Account Name SLO Acct Name SLO F1 Code Comp ID +posted 8/23/2021 MOUNTAIN MIKES PIZZA - -24.27 -24.27 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC +posted 9/1/2021 Prime Video*258CB55D0 - -3.99 -3.99 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC +posted 5/21/2021 Prime Video*2L1I62RT2 - -5.99 -5.99 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC +posted 9/6/2021 STARBUCKS STORE 06536 - -38.85 -38.85 - - MVSC - BofA Corp Card Sean - 7187 MVSC - BofA Corp Card Sean - 7187 MVSC-6 MVSC") + +(def base-transaction {:status "posted" + :raw-date "8/23/2021" + :description-original "MOUNTAIN MIKES PIZZA" + :high-level-category "" + :amount "-24.27" + :bank-account-code "MVSC-6" + :client-code "MVSC"}) + +(t/deftest tabulate-data + (t/testing "Should tabulate a single row" + (t/is (= base-transaction (first (sut/tabulate-data raw-tsv)))))) + +(t/deftest manual->transactions + (t/testing "Should transform a single transaction" + (t/is (= #:transaction{:bank-account 1 + :client 12 + :date #inst "2021-08-23T00:00:00.000-07:00" + :amount -24.27 + :description-original "MOUNTAIN MIKES PIZZA" + :status "POSTED"} + (sut/manual->transaction base-transaction {"MVSC-6" 1} {"MVSC-6" 12})))) + + (t/testing "Should regard a bad date as an error" + (let [row (sut/manual->transaction (assoc base-transaction :raw-date "124") {"MVSC-6" 1} {"MVSC-6" 12})] + (t/is (= + [{:info "Date 124 must match MM/dd/yyyy" + :details "java.lang.Exception: Date 124 must match MM/dd/yyyy"}] + (:errors row))))) + + (t/testing "Should consider an unparseable amount as an error" + (let [row (sut/manual->transaction (assoc base-transaction :amount "212.23,.41") {"MVSC-6" 1} {"MVSC-6" 12})] + (t/is (= + [{:info "Could not parse total from value '212.23,.41'" + :details "java.lang.Exception: Could not parse total from value '212.23,.41'"}] + (:errors row))))) + + (t/testing "Should consider a nonexistant bank account as an error" + (let [row (sut/manual->transaction (assoc base-transaction :bank-account-code "DUMMY") {"MVSC-6" 1} {"MVSC-6" 12})] + (t/is (= + [{:info "Cannot find bank account by code DUMMY" + :details "java.lang.Exception: Cannot find bank account by code DUMMY"} + {:info "Cannot find client for bank account code DUMMY" + :details "java.lang.Exception: Cannot find client for bank account code DUMMY"}] + (:errors row)))))) + diff --git a/test/clj/auto_ap/integration/yodlee/import.clj b/test/clj/auto_ap/import/transactions_test.clj similarity index 50% rename from test/clj/auto_ap/integration/yodlee/import.clj rename to test/clj/auto_ap/import/transactions_test.clj index ac067e9e..20e7cb7a 100644 --- a/test/clj/auto_ap/integration/yodlee/import.clj +++ b/test/clj/auto_ap/import/transactions_test.clj @@ -1,13 +1,10 @@ -(ns auto-ap.integration.yodlee.import - (:require [auto-ap.yodlee.import :as sut] - [auto-ap.yodlee.core :as c] - [datomic.api :as d] - [auto-ap.datomic :refer [uri conn]] - [auto-ap.rule-matching :as rm] +(ns auto-ap.import.transactions-test + (:require [auto-ap.datomic :refer [conn uri]] [auto-ap.datomic.migrate :as m] + [auto-ap.import.transactions :as sut] [clojure.test :as t] - [clojure.tools.logging :as log] - [clojure.set :as set])) + [datomic.api :as d] + [clj-time.coerce :as coerce])) (defn wrap-setup [f] @@ -24,311 +21,72 @@ (defn noop-rule [transaction locations] transaction) -(def base-transaction {:postDate "2014-01-04" - :accountId 1234 - :date "2014-01-02" - :id 1 - :amount {:amount 12.0} - :description {:original "original-description" - :simple "simple-description"} - :merchant {:id "123" - :name "456"} - :baseType "DEBIT" - :status "POSTED" +(def base-transaction #:transaction {:date #inst "2020-01-02T00:00:00-08:00" + :raw-id "1" + :id (digest/sha-256 "1") + :amount 12.0 + :description-original "original-description" + :status "POSTED" + :client 123 + :bank-account 456}) - :bank-account {:db/id 456 - :client/_bank-accounts {:db/id 123 - :client/locations ["Z" "E"]}}}) +(t/deftest categorize-transactions + (let [bank-account {:db/id 456 + :client/_bank-accounts {:db/id 123 + :client/locations ["MH"]}}] + (t/testing "Should exclude a transaction before start date" + (t/is (= :not-ready + (sut/categorize-transaction (assoc base-transaction :transaction/date #inst "2020-01-01") + (assoc bank-account :bank-account/start-date #inst "2030-01-01") + {}))) + (t/is (= :import + (sut/categorize-transaction (assoc base-transaction :transaction/date #inst "2020-01-02") + (assoc bank-account :bank-account/start-date #inst "2020-01-01") + {})))) + (t/testing "Should error a transaction without a client" + (t/is (= :error + (sut/categorize-transaction (dissoc base-transaction :transaction/client) + bank-account + {})))) -(t/deftest add-transactions + (t/testing "Should error a transaction without a bank-account" + (t/is (= :error + (sut/categorize-transaction (dissoc base-transaction :transaction/bank-account) + bank-account + {})))) + (t/testing "Should error a transaction without an id" + (t/is (= :error + (sut/categorize-transaction (dissoc base-transaction :transaction/id) + bank-account + {})))) + (t/testing "Should ignore a transaction that exists" + (t/is (= :extant + (sut/categorize-transaction (assoc base-transaction :transaction/id "123") + bank-account + {"123" :transaction-approval-status/unapproved})))) + (t/testing "Should import a transaction" + (t/is (= :import + (sut/categorize-transaction base-transaction + bank-account + {})))))) + + +(t/deftest transaction->txs (t/testing "Should import and code transactions" - (let [[result] (sut/transactions->txs [base-transaction] - :bank-account - noop-rule - #{})] - (t/is (= [#:transaction {:amount -12.0 - :date #inst "2014-01-02T08:00:00.000-00:00" - :bank-account 456 - :client 123 - :post-date #inst "2014-01-04T08:00:00.000-00:00" - :account-id 1234 - :description-original "original-description" - :status "POSTED" - :id "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" - :raw-id "1" - :approval-status :transaction-approval-status/unapproved - :description-simple "simple-description"}] - result))))) + (t/testing "Should import one transaction" + (let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id" + :bank-account/code "TEST-1"} + {:db/id "client-id" + :client/code "TEST" + :client/locations ["Z" "E"] + :client/bank-accounts ["bank-account-id"]}])) + result (sut/transaction->txs base-transaction + (d/entity (d/db conn) bank-account-id) + noop-rule)] + (t/is (= [(assoc base-transaction :transaction/approval-status :transaction-approval-status/unapproved)] + result)))) -(t/deftest do-import - (t/testing "Should import single transaction" - (println "HER") - (let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id" - :bank-account/code "TEST-1"} - {:db/id "client-id" - :client/code "TEST" - :client/locations ["Z" "E"] - :client/bank-accounts ["bank-account-id"]}])) - result (sut/transactions->txs [base-transaction] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - #{})] - (t/is (= {:import [[#:transaction {:amount -12.0 - :date #inst "2014-01-02T08:00:00.000-00:00" - :bank-account bank-account-id - :client client-id - :post-date #inst "2014-01-04T08:00:00.000-00:00" - :account-id 1234 - :description-original "original-description" - :status "POSTED" - :id "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" - :raw-id "1" - :approval-status :transaction-approval-status/unapproved - :description-simple "simple-description" - :import-batch/_entry 1}]]} - result)))) - - (t/testing "Should exclude a transaction before start date" - (let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id" - :bank-account/code "TEST-1" - :bank-account/start-date #inst "2030-01-01"} - {:db/id "client-id" - :client/code "TEST" - :client/locations ["Z" "E"] - :client/bank-accounts ["bank-account-id"]}])) - result (sut/transactions->txs [(assoc base-transaction :date "2020-05-01")] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - #{})] - (t/is (= 1 (count (:not-ready result)))))) - - (t/testing "Should not reimport an existing transaction" - (let [{:strs [bank-account-id client-id]} (:tempids @(d/transact conn [{:db/id "bank-account-id" - :bank-account/code "TEST-1" - :bank-account/start-date #inst "2030-01-01"} - {:db/id "client-id" - :client/code "TEST" - :client/locations ["Z" "E"] - :client/bank-accounts ["bank-account-id"]}])) - result (sut/transactions->txs [(assoc base-transaction :id "123")] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - {(digest/sha-256 "123") :transaction-approval-status/unapproved})] - (t/is (= 1 (count (:extant result)))))) - - - (t/testing "Should skip transaction if no client is found" - (let [result (sut/transactions->txs [base-transaction] - (d/entity (d/db conn) 1238970123) - 1 - noop-rule - {})] - (t/is (= 1 (count (:error result)))))) - - (t/testing "Should match an uncleared check" - (let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending - :date #inst "2019-01-01" - :bank-account "bank-account-id" - :client "client-id" - :check-number 10001 - :amount 30.0 - :db/id "payment-id"} - #:bank-account {:name "Bank account" - :db/id "bank-account-id"} - #:client {:name "Client" - :db/id "client-id" - :bank-accounts ["bank-account-id"]}] - (d/transact (d/connect uri)) - deref - :tempids)] - - - (println (sut/transactions->txs [(assoc base-transaction - :description {:original "CHECK 10001" - :simple ""} - :amount {:amount 30.0})] - (d/entity (d/db conn ) bank-account-id) - 1 - noop-rule - {})) - (let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction - :description {:original "CHECK 10001" - :simple ""} - :amount {:amount 30.0})] - (d/entity (d/db conn ) bank-account-id) - 1 - noop-rule - {}))] - - (t/is (= {:db/id payment-id - :payment/status :payment-status/cleared} - (:transaction/payment transaction-result)))) - - (t/testing "Should not match an already matched check" - @(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}]) - (let [[result] (:import (sut/transactions->txs [(assoc base-transaction - :description {:original "CHECK 10001" - :simple ""} - :amount {:amount 30.0} - :id 789)] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - {}))] - - (t/is (= nil - (:transaction/payment result))))))) - - (t/testing "Should match expected-deposits" - (let [{:strs [bank-account-id client-id expected-deposit-id]} (->> [#:expected-deposit {:client "client-id" - :date #inst "2021-07-01T00:00:00-08:00" - :total 100.0 - :location "MF" - :status :expected-deposit-status/pending - :db/id "expected-deposit-id"} - #:bank-account {:name "Bank account" - :db/id "bank-account-id"} - #:client {:name "Client" - :db/id "client-id" - :locations ["MF"] - :bank-accounts ["bank-account-id"]}] - (d/transact (d/connect uri)) - deref - :tempids)] - - - (t/testing "Should match within 10 days" - (let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction - :date "2021-07-03" - :amount {:amount -100.0})] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - #{}))] - (t/is (= expected-deposit-id - (sut/find-expected-deposit client-id 100.0 (clj-time.coerce/to-date-time #inst "2021-07-03T00:00:00-08:00")))) - - (t/is (= {:db/id expected-deposit-id - :expected-deposit/status :expected-deposit-status/cleared} - (:transaction/expected-deposit transaction-result))))) - - (t/testing "Should not match old expected deposits" - (let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction - :date "2021-07-13" - :amount {:amount -100.0})] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - {}))] - (t/is (not (:transaction/expected-deposit transaction-result))))) - - (t/testing "Should only match exact." - (let [[[transaction-result]] (:import (sut/transactions->txs [(assoc base-transaction - :date "2021-07-03" - :amount {:amount -100.01})] - (d/entity (d/db conn) bank-account-id) - 1 - noop-rule - {}))] - (t/is (not (:transaction/expected-deposit transaction-result))))))) - - #_(t/testing "Auto-pay Invoices" - (t/testing "Should match paid invoice that doesn't have a payment yet" - (let [{:strs [bank-account-id client-id invoice1-id invoice2-id vendor-id]} (->> [#:invoice {:status :invoice-status/paid - :vendor "vendor-id" - :scheduled-payment #inst "2019-01-04" - :date #inst "2019-01-01" - :client "client-id" - :total 20.0 - :db/id "invoice1-id"} - #:invoice {:status :invoice-status/paid - :vendor "vendor-id" - :scheduled-payment #inst "2019-01-04" - :date #inst "2019-01-01" - :client "client-id" - :total 10.0 - :db/id "invoice2-id"} - #:vendor {:name "Autopay vendor" - :db/id "vendor-id"} - #:bank-account {:name "Bank account" - :db/id "bank-account-id"} - #:client {:name "Client" - :db/id "client-id" - :bank-accounts ["bank-account-id"]}] - (d/transact (d/connect uri)) - deref - :tempids) - [[transaction-tx payment-tx invoice-payments1-tx invoice-payments2-tx]] (sut/transactions->txs [(assoc base-transaction - :amount {:amount 30.0} - :bank-account {:db/id bank-account-id - :client/_bank-accounts {:db/id client-id - :client/locations ["A"]}})] - :bank-account - noop-rule - #{})] - - (t/is (= :transaction-approval-status/approved - (:transaction/approval-status transaction-tx)) - (str "Should have approved transaction " transaction-tx)) - (t/is (= #:payment{:status :payment-status/cleared - :type :payment-type/debit - :date (:transaction/date transaction-tx) - :client client-id - :bank-account bank-account-id - :vendor vendor-id - :amount 30.0} - - (dissoc payment-tx :db/id)) - (str "Should have created payment " payment-tx)) - (t/is (= #:invoice-payment{:invoice invoice1-id - :amount 20.0 - :payment (:db/id payment-tx)} - - (dissoc invoice-payments1-tx :db/id)) - (str "Should have paid invoice 1" invoice-payments1-tx)) - (t/is (= #:invoice-payment{:invoice invoice2-id - :amount 10.0 - :payment (:db/id payment-tx)} - - (dissoc invoice-payments2-tx :db/id)) - (str "Should have paid invoice 2" invoice-payments2-tx)))) - - (t/testing "Should not match paid invoice that isn't a scheduled payment" - (let [{:strs [bank-account-id client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid - :vendor "vendor-id" - :date #inst "2019-01-01" - :client "client-id" - :total 30.0 - :db/id "invoice-id"} - #:vendor {:name "Autopay vendor" - :db/id "vendor-id"} - #:bank-account {:name "Bank account" - :db/id "bank-account-id"} - #:client {:name "Client" - :db/id "client-id" - :bank-accounts ["bank-account-id"]}] - (d/transact (d/connect uri)) - deref - :tempids) - [[transaction-tx payment-tx]] (sut/transactions->txs [(assoc base-transaction - :amount {:amount 30.0} - :bank-account {:db/id bank-account-id - :client/_bank-accounts {:db/id client-id - :client/locations ["A"]}})] - :bank-account - noop-rule - #{})] - - (t/is (= :transaction-approval-status/unapproved - (:transaction/approval-status transaction-tx))) - (t/is (nil? (:transaction/payment transaction-tx)))))) - - - (t/testing "Rules" - (t/testing "Should apply rules to imported transaction" + (t/testing "Should match an uncleared check" (let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending :date #inst "2019-01-01" :bank-account "bank-account-id" @@ -343,198 +101,114 @@ :bank-accounts ["bank-account-id"]}] (d/transact (d/connect uri)) deref - :tempids) - [[transaction-tx]] (:import (sut/transactions->txs [(assoc base-transaction - :description {:original "Hello XXX039" - :simple ""} - :amount {:amount 31.0} - :id 789)] - (d/entity (d/db conn) bank-account-id) - 1 - (rm/rule-applying-fn [{:transaction-rule/description "XXX039" - :transaction-rule/transaction-approval-status :transaction-approval-status/approved}]) - #{}))] - - (t/is (= :transaction-approval-status/approved - (:transaction/approval-status transaction-tx))))) + :tempids)] - (t/testing "Should apply vendor and approval status" - (let [apply-rules (rm/rule-applying-fn [{:db/id 1 - :transaction-rule/description "XXX039" - :transaction-rule/transaction-approval-status :transaction-approval-status/approved - :transaction-rule/vendor {:db/id 123} - :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} - :transaction-rule-account/location "Shared" - :transaction-rule-account/percentage 1.0}]} - {:db/id 2 - :transaction-rule/description "OtherMatch" - :transaction-rule/transaction-approval-status :transaction-approval-status/requires-feedback - :transaction-rule/vendor {:db/id 456} - :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} - :transaction-rule-account/location "Z" - :transaction-rule-account/percentage 1.0}]}])] - (t/is (= {:transaction/description-original "Hello XXX039", - :transaction/vendor 123 - :transaction/approval-status :transaction-approval-status/approved - :transaction/accounts [{:transaction-account/account 9 - :transaction-account/amount 30.0 - :transaction-account/location "Z"}] - :transaction/matched-rule 1 - :transaction/amount 30.0} - (-> {:transaction/description-original "Hello XXX039" - :transaction/amount 30.0} - (apply-rules ["Z"])))) + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/description-original "CHECK 10001" + :transaction/amount -30.0) + (d/entity (d/db conn ) bank-account-id) + noop-rule)] + + (t/is (= {:db/id payment-id + :payment/status :payment-status/cleared} + (:transaction/payment transaction-result)))) - (t/is (= {:transaction/description-original "OtherMatch", - :transaction/approval-status :transaction-approval-status/requires-feedback - :transaction/vendor 456 - :transaction/amount 30.0 - :transaction/matched-rule 2 - :transaction/accounts [{:transaction-account/account 9 - :transaction-account/amount 30.0 - :transaction-account/location "Z"}]} - (-> {:transaction/description-original "OtherMatch" - :transaction/amount 30.0} - (apply-rules ["Z"])))) + (t/testing "Should not match an already matched check" + @(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}]) + (let [[result] (sut/transaction->txs (assoc base-transaction + :transaction/description-original "CHECK 10001" + :transaction/amount -30.0) + (d/entity (d/db conn) bank-account-id) + noop-rule)] + + (t/is (= nil + (:transaction/payment result))))))) - (t/is (= {:transaction/description-original "Hello Not match"} - (-> {:transaction/description-original "Hello Not match"} - (apply-rules [])))))) - (t/testing "Should match if day of month matches" - (let [apply-rules (rm/rule-applying-fn [{:db/id 123 - :transaction-rule/dom-gte 3 - :transaction-rule/dom-lte 9 - :transaction-rule/transaction-approval-status :transaction-approval-status/approved - :transaction-rule/vendor {:db/id 123}}])] - (t/is (= 123 - (-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00" - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 123 - (-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00" - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 123 - (-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00" - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00" - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00" - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))))) + (t/testing "Should match expected-deposits" + (let [{:strs [bank-account-id client-id expected-deposit-id]} (->> [#:expected-deposit {:client "client-id" + :date #inst "2021-07-01T00:00:00-08:00" + :total 100.0 + :location "MF" + :status :expected-deposit-status/pending + :db/id "expected-deposit-id"} + #:bank-account {:name "Bank account" + :db/id "bank-account-id"} + #:client {:name "Client" + :db/id "client-id" + :locations ["MF"] + :bank-accounts ["bank-account-id"]}] + (d/transact (d/connect uri)) + deref + :tempids)] - (t/testing "Should match if amount matches" - (let [apply-rules (rm/rule-applying-fn [{:db/id 123 - :transaction-rule/amount-gte 3.0 - :transaction-rule/amount-lte 9.0 - :transaction-rule/transaction-approval-status :transaction-approval-status/approved - :transaction-rule/accounts [#:transaction-rule-account {:percentage 1.0 - :locatoin "HQ"}] - :transaction-rule/vendor {:db/id 123}}])] - (t/is (= 123 - (-> {:transaction/amount 4.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 123 - (-> {:transaction/amount 3.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 123 - (-> {:transaction/amount 9.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/amount 9.01} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/amount 2.99} - (apply-rules []) - :transaction/matched-rule))))) - (t/testing "Should match if client matches" - (let [apply-rules (rm/rule-applying-fn [{:db/id 123 - :transaction-rule/client {:db/id 456}}])] - (t/is (= 123 - (-> {:transaction/client 456 - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/client 89 - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))))) - (t/testing "Should match if bank-account matches" - (let [apply-rules (rm/rule-applying-fn [{:db/id 123 - :transaction-rule/bank-account {:db/id 456}}])] - (t/is (= 123 - (-> {:transaction/bank-account 456 - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (nil? - (-> {:transaction/bank-account 89 - :transaction/amount 1.0} - (apply-rules []) - :transaction/matched-rule))))) + (t/testing "Should match within 10 days" + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/date #inst "2021-07-03T00:00:00-08:00" + :transaction/amount 100.0) + (d/entity (d/db conn) bank-account-id) + noop-rule)] + (t/is (= expected-deposit-id + (sut/find-expected-deposit client-id 100.0 (clj-time.coerce/to-date-time #inst "2021-07-03T00:00:00-08:00")))) + + (t/is (= {:db/id expected-deposit-id + :expected-deposit/status :expected-deposit-status/cleared} + (:transaction/expected-deposit transaction-result))))) - (t/testing "Should prioritize rules" - (let [apply-rules (rm/rule-applying-fn (shuffle [{:db/id 2 - :transaction-rule/description "Hello" - :transaction-rule/amount-gte 5.0} - {:db/id 1 - :transaction-rule/description "Hello"} - {:db/id 0 - :transaction-rule/description "Hello"} - {:db/id 3 - :transaction-rule/description "Hello" - :transaction-rule/client {:db/id 789}} - {:db/id 4 - :transaction-rule/description "Hello" - :transaction-rule/client {:db/id 789} - :transaction-rule/bank-account {:db/id 456}}]))] + (t/testing "Should not match old expected deposits" + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/date #inst "2021-07-13" + :transaction/amount 100.0) + (d/entity (d/db conn) bank-account-id) + noop-rule)] + (t/is (not (:transaction/expected-deposit transaction-result))))) - (t/is (= 4 - (-> {:transaction/bank-account 456 - :transaction/client 789 - :transaction/description-original "Hello" - :transaction/amount 6.0} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 3 - (-> {:transaction/bank-account 457 - :transaction/client 789 - :transaction/amount 6.0 - :transaction/description-original "Hello"} - (apply-rules []) - :transaction/matched-rule))) - (t/is (= 3 - (-> {:transaction/bank-account 457 - :transaction/client 789 - :transaction/amount 6.0 - :transaction/description-original "Hello"} - (apply-rules []) - :transaction/matched-rule))) + (t/testing "Should only match exact." + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/date "2021-07-03" + :transaction/amount 100.01) + (d/entity (d/db conn) bank-account-id) + noop-rule)] + (t/is (not (:transaction/expected-deposit transaction-result))))))))) - (t/testing "Should only apply if there is a single rule at that specificity level" - (t/is (nil? - (-> {:transaction/bank-account 3 - :transaction/client 1 - :transaction/description-original "Hello" - :transaction/amount 0.0} - (apply-rules []) - :transaction/matched-rule)))))))) +(t/deftest apply-synthetic-ids + (t/testing "Should increment index for duplicate transactions" + (t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-1-123"] + (map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction base-transaction]))))) + + (t/testing "Should use unique ids if a parameter is different" + (t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + "2020-01-02T00:00:00.000-08:00-456-original-description-13.0-0-123"] + (map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction (assoc base-transaction :transaction/amount 13.0)]))))) + + (t/testing "Should increment index if an unimportant field is set" + (t/is (= ["2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-1-123"] + (map :transaction/raw-id (sut/apply-synthetic-ids [base-transaction (assoc base-transaction :transaction/other-random-value 10.0)]))))) + + (t/testing "Should be forgiving if dates come in other formats" + (t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + (->> (sut/apply-synthetic-ids [(assoc base-transaction + :transaction/date #inst "2020-01-02T00:00:00-08:00")]) + first + :transaction/raw-id))) + (t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + (->> (sut/apply-synthetic-ids [(assoc base-transaction + :transaction/date #inst "2020-01-02T08:00:00-00:00")]) + first + :transaction/raw-id))) + (t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + (->> (sut/apply-synthetic-ids [(assoc base-transaction + :transaction/date (coerce/to-date-time #inst "2020-01-02T08:00:00-00:00"))]) + first + :transaction/raw-id))) + (t/is (= "2020-01-02T00:00:00.000-08:00-456-original-description-12.0-0-123" + (->> (sut/apply-synthetic-ids [(assoc base-transaction + :transaction/date (coerce/to-date-time #inst "2020-01-02T00:00:00-08:00"))]) + first + :transaction/raw-id))))) (t/deftest match-transaction-to-single-unfulfilled-payments @@ -580,7 +254,7 @@ (t/testing "Should not match unpaid invoice" (let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/unpaid :scheduled-payment #inst "2019-01-04" - :vendor vendor1-id + :vendor vendor1-id :date #inst "2019-01-01" :client "client-id" :total 30.0 @@ -609,7 +283,7 @@ deref :tempids) invoices-matches (sut/match-transaction-to-single-unfulfilled-autopayments -30.0 - client-id)] + client-id)] (t/is (= [] invoices-matches)))) (t/testing "Should match multiple invoices for same vendor that total to transaction amount" (let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid @@ -709,3 +383,98 @@ (str "Expected to match with the chronologically adjacent invoice-1 and invoice-3.")) (t/is (= [] (sut/match-transaction-to-single-unfulfilled-autopayments -31.0 client-id)) (str "Expected to not match, because there is invoice-3 is between invoice-1 and invoice-2."))))))) + + + + + +#_(t/testing "Auto-pay Invoices" + (t/testing "Should match paid invoice that doesn't have a payment yet" + (let [{:strs [bank-account-id client-id invoice1-id invoice2-id vendor-id]} (->> [#:invoice {:status :invoice-status/paid + :vendor "vendor-id" + :scheduled-payment #inst "2019-01-04" + :date #inst "2019-01-01" + :client "client-id" + :total 20.0 + :db/id "invoice1-id"} + #:invoice {:status :invoice-status/paid + :vendor "vendor-id" + :scheduled-payment #inst "2019-01-04" + :date #inst "2019-01-01" + :client "client-id" + :total 10.0 + :db/id "invoice2-id"} + #:vendor {:name "Autopay vendor" + :db/id "vendor-id"} + #:bank-account {:name "Bank account" + :db/id "bank-account-id"} + #:client {:name "Client" + :db/id "client-id" + :bank-accounts ["bank-account-id"]}] + (d/transact (d/connect uri)) + deref + :tempids) + [[transaction-tx payment-tx invoice-payments1-tx invoice-payments2-tx]] (sut/yodlees->transactions [(assoc base-yodlee-transaction + :amount {:amount 30.0} + :bank-account {:db/id bank-account-id + :client/_bank-accounts {:db/id client-id + :client/locations ["A"]}})] + :bank-account + noop-rule + #{})] + + (t/is (= :transaction-approval-status/approved + (:transaction/approval-status transaction-tx)) + (str "Should have approved transaction " transaction-tx)) + (t/is (= #:payment{:status :payment-status/cleared + :type :payment-type/debit + :date (:transaction/date transaction-tx) + :client client-id + :bank-account bank-account-id + :vendor vendor-id + :amount 30.0} + + (dissoc payment-tx :db/id)) + (str "Should have created payment " payment-tx)) + (t/is (= #:invoice-payment{:invoice invoice1-id + :amount 20.0 + :payment (:db/id payment-tx)} + + (dissoc invoice-payments1-tx :db/id)) + (str "Should have paid invoice 1" invoice-payments1-tx)) + (t/is (= #:invoice-payment{:invoice invoice2-id + :amount 10.0 + :payment (:db/id payment-tx)} + + (dissoc invoice-payments2-tx :db/id)) + (str "Should have paid invoice 2" invoice-payments2-tx)))) + + (t/testing "Should not match paid invoice that isn't a scheduled payment" + (let [{:strs [bank-account-id client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid + :vendor "vendor-id" + :date #inst "2019-01-01" + :client "client-id" + :total 30.0 + :db/id "invoice-id"} + #:vendor {:name "Autopay vendor" + :db/id "vendor-id"} + #:bank-account {:name "Bank account" + :db/id "bank-account-id"} + #:client {:name "Client" + :db/id "client-id" + :bank-accounts ["bank-account-id"]}] + (d/transact (d/connect uri)) + deref + :tempids) + [[transaction-tx payment-tx]] (sut/yodlees->transactions [(assoc base-yodlee-transaction + :amount {:amount 30.0} + :bank-account {:db/id bank-account-id + :client/_bank-accounts {:db/id client-id + :client/locations ["A"]}})] + :bank-account + noop-rule + #{})] + + (t/is (= :transaction-approval-status/unapproved + (:transaction/approval-status transaction-tx))) + (t/is (nil? (:transaction/payment transaction-tx)))))) diff --git a/test/clj/auto_ap/import/yodlee_test.clj b/test/clj/auto_ap/import/yodlee_test.clj new file mode 100644 index 00000000..00625ec9 --- /dev/null +++ b/test/clj/auto_ap/import/yodlee_test.clj @@ -0,0 +1,29 @@ +(ns auto-ap.import.yodlee-test + (:require [auto-ap.import.yodlee :as sut] + [clojure.test :as t])) + + +(def base-transaction {:postDate "2014-01-04" + :accountId 1234 + :date "2014-01-02" + :id 1 + :amount {:amount 12.0} + :description {:original "original-description" + :simple "simple-description"} + :merchant {:id "123" + :name "456"} + :baseType "DEBIT" + :status "POSTED"}) + +(t/deftest yodlee->transaction + (t/testing "Should parse dates" + (t/is (= #inst "2021-01-01T00:00:00-08:00" (:transaction/date (sut/yodlee->transaction (assoc base-transaction :date "2021-01-01"))))) + (t/is (= #inst "2021-06-01T00:00:00-07:00" (:transaction/date (sut/yodlee->transaction (assoc base-transaction :date "2021-06-01")))))) + + (t/testing "Should invert amount for debits" + (t/is (= -12.0 (:transaction/amount (sut/yodlee->transaction (assoc base-transaction + :amount {:amount 12.0} + :baseType "DEBIT"))))) + (t/is (= 12.0 (:transaction/amount (sut/yodlee->transaction (assoc base-transaction + :amount {:amount 12.0} + :baseType "CREDIT")))))))