(ns auto-ap.integration.yodlee.import (:require [auto-ap.yodlee.import :as sut] [auto-ap.yodlee.core :as c] [datomic.api :as d] [auto-ap.datomic :refer [uri]] [auto-ap.rule-matching :as rm] [auto-ap.datomic.migrate :as m] [clojure.test :as t] [clojure.tools.logging :as log] [clojure.set :as set])) (defn wrap-setup [f] (with-redefs [auto-ap.datomic/uri "datomic:mem://datomic-transactor:4334/invoice"] (d/create-database uri) (with-redefs [auto-ap.datomic/conn (d/connect uri)] (m/migrate auto-ap.datomic/conn) (f) (d/release auto-ap.datomic/conn) (d/delete-database uri)))) (t/use-fixtures :each wrap-setup) (defn noop-rule [transaction locations] transaction) (def base-transaction {:postDate "2014-01-04" :accountId 1234 :date "2014-01-02" :id 1 :amount {:amount 12.0} :description {:original "original-description" :simple "simple-description"} :merchant {:id "123" :name "456"} :baseType "DEBIT" :status "POSTED" :bank-account {:db/id 456 :client/_bank-accounts {:db/id 123 :client/locations ["Z" "E"]}}}) (t/deftest do-import (t/testing "Should import single transaction" (let [[result] (sut/transactions->txs [base-transaction] :bank-account noop-rule #{})] (t/is (= [#:transaction {:amount -12.0 :date #inst "2014-01-02T08:00:00.000-00:00" :bank-account 456 :client 123 :post-date #inst "2014-01-04T08:00:00.000-00:00" :account-id 1234 :description-original "original-description" :status "POSTED" :id "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" :approval-status :transaction-approval-status/unapproved :description-simple "simple-description"}] result)))) (t/testing "Should exclude a transaction before start date" (let [result (sut/transactions->txs [(assoc-in base-transaction [:bank-account :start-date] (clj-time.coerce/to-date-time #inst "2020-01-01"))] :bank-account noop-rule #{})] (t/is (= [] result)))) (t/testing "Should not reimport an existing transaction" (let [result (sut/transactions->txs [base-transaction] :bank-account noop-rule #{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})] (t/is (= [] result)))) (t/testing "Should skip transaction if no client is found" (let [result (sut/transactions->txs [(assoc base-transaction :bank-account nil)] :bank-account noop-rule #{})] (t/is (= [] result)))) (t/testing "Should match an uncleared check" (let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending :date #inst "2019-01-01" :bank-account "bank-account-id" :client "client-id" :check-number 10001 :amount 30.0 :db/id "payment-id"} #:bank-account {:name "Bank account" :db/id "bank-account-id"} #:client {:name "Client" :db/id "client-id" :bank-accounts ["bank-account-id"]}] (d/transact (d/connect uri)) deref :tempids)] (let [[[transaction-result]] (sut/transactions->txs [(assoc base-transaction :description {:original "CHECK 10001" :simple ""} :amount {:amount 30.0} :bank-account {:db/id bank-account-id :client/_bank-accounts {:db/id client-id :client/locations ["A"]}})] :bank-account noop-rule #{})] (t/is (= {:db/id payment-id :payment/status :payment-status/cleared} (:transaction/payment transaction-result)))) (t/testing "Should not match an already matched check" @(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}]) (let [[result] (sut/transactions->txs [(assoc base-transaction :description {:original "CHECK 10001" :simple ""} :amount {:amount 30.0} :id 789 :bank-account {:db/id bank-account-id :client/_bank-accounts {:db/id client-id :client/locations ["A"]}})] :bank-account noop-rule #{})] (t/is (= nil (:transaction/payment result))))))) (t/testing "Auto-pay Invoices" (t/testing "Should match paid invoice that doesn't have a payment yet" (let [{:strs [bank-account-id client-id invoice1-id invoice2-id vendor-id]} (->> [#:invoice {:status :invoice-status/paid :vendor "vendor-id" :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 20.0 :db/id "invoice1-id"} #:invoice {:status :invoice-status/paid :vendor "vendor-id" :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 10.0 :db/id "invoice2-id"} #:vendor {:name "Autopay vendor" :db/id "vendor-id"} #:bank-account {:name "Bank account" :db/id "bank-account-id"} #:client {:name "Client" :db/id "client-id" :bank-accounts ["bank-account-id"]}] (d/transact (d/connect uri)) deref :tempids) [[transaction-tx payment-tx invoice-payments1-tx invoice-payments2-tx]] (sut/transactions->txs [(assoc base-transaction :amount {:amount 30.0} :bank-account {:db/id bank-account-id :client/_bank-accounts {:db/id client-id :client/locations ["A"]}})] :bank-account noop-rule #{})] (t/is (= :transaction-approval-status/approved (:transaction/approval-status transaction-tx)) (str "Should have approved transaction " transaction-tx)) (t/is (= #:payment{:status :payment-status/cleared :client client-id :bank-account bank-account-id :vendor vendor-id :amount 30.0} (dissoc payment-tx :db/id)) (str "Should have created payment " payment-tx)) (t/is (= #:invoice-payment{:invoice invoice1-id :amount 20.0 :payment (:db/id payment-tx)} (dissoc invoice-payments1-tx :db/id)) (str "Should have paid invoice 1" invoice-payments1-tx)) (t/is (= #:invoice-payment{:invoice invoice2-id :amount 10.0 :payment (:db/id payment-tx)} (dissoc invoice-payments2-tx :db/id)) (str "Should have paid invoice 2" invoice-payments2-tx)))) (t/testing "Should not match paid invoice that isn't a scheduled payment" (let [{:strs [bank-account-id client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid :vendor "vendor-id" :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice-id"} #:vendor {:name "Autopay vendor" :db/id "vendor-id"} #:bank-account {:name "Bank account" :db/id "bank-account-id"} #:client {:name "Client" :db/id "client-id" :bank-accounts ["bank-account-id"]}] (d/transact (d/connect uri)) deref :tempids) [[transaction-tx payment-tx]] (sut/transactions->txs [(assoc base-transaction :amount {:amount 30.0} :bank-account {:db/id bank-account-id :client/_bank-accounts {:db/id client-id :client/locations ["A"]}})] :bank-account noop-rule #{})] (t/is (= :transaction-approval-status/unapproved (:transaction/approval-status transaction-tx))) (t/is (nil? (:transaction/payment transaction-tx)))))) (t/testing "Rules" (t/testing "Should apply rules to imported transaction" (let [{:strs [bank-account-id client-id payment-id]} (->> [#:payment {:status :payment-status/pending :date #inst "2019-01-01" :bank-account "bank-account-id" :client "client-id" :check-number 10001 :amount 30.0 :db/id "payment-id"} #:bank-account {:name "Bank account" :db/id "bank-account-id"} #:client {:name "Client" :db/id "client-id" :bank-accounts ["bank-account-id"]}] (d/transact (d/connect uri)) deref :tempids) [[transaction-tx]] (sut/transactions->txs [(assoc base-transaction :description {:original "Hello XXX039" :simple ""} :amount {:amount 31.0} :id 789 :bank-account {:db/id bank-account-id :client/_bank-accounts {:db/id client-id :client/locations ["A"]}})] :bank-account (rm/rule-applying-fn [{:transaction-rule/description "XXX039" :transaction-rule/transaction-approval-status :transaction-approval-status/approved}]) #{})] (t/is (= :transaction-approval-status/approved (:transaction/approval-status transaction-tx))))) (t/testing "Should apply vendor and approval status" (let [apply-rules (rm/rule-applying-fn [{:db/id 1 :transaction-rule/description "XXX039" :transaction-rule/transaction-approval-status :transaction-approval-status/approved :transaction-rule/vendor {:db/id 123} :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} :transaction-rule-account/location "Shared" :transaction-rule-account/percentage 1.0}]} {:db/id 2 :transaction-rule/description "OtherMatch" :transaction-rule/transaction-approval-status :transaction-approval-status/requires-feedback :transaction-rule/vendor {:db/id 456} :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} :transaction-rule-account/location "Z" :transaction-rule-account/percentage 1.0}]}])] (t/is (= {:transaction/description-original "Hello XXX039", :transaction/vendor 123 :transaction/approval-status :transaction-approval-status/approved :transaction/accounts [{:transaction-account/account 9 :transaction-account/amount 30.0 :transaction-account/location "Z"}] :transaction/matched-rule 1 :transaction/amount 30.0} (-> {:transaction/description-original "Hello XXX039" :transaction/amount 30.0} (apply-rules ["Z"])))) (t/is (= {:transaction/description-original "OtherMatch", :transaction/approval-status :transaction-approval-status/requires-feedback :transaction/vendor 456 :transaction/amount 30.0 :transaction/matched-rule 2 :transaction/accounts [{:transaction-account/account 9 :transaction-account/amount 30.0 :transaction-account/location "Z"}]} (-> {:transaction/description-original "OtherMatch" :transaction/amount 30.0} (apply-rules ["Z"])))) (t/is (= {:transaction/description-original "Hello Not match"} (-> {:transaction/description-original "Hello Not match"} (apply-rules [])))))) (t/testing "Should match if day of month matches" (let [apply-rules (rm/rule-applying-fn [{:db/id 123 :transaction-rule/dom-gte 3 :transaction-rule/dom-lte 9 :transaction-rule/transaction-approval-status :transaction-approval-status/approved :transaction-rule/vendor {:db/id 123}}])] (t/is (= 123 (-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00" :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00" :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00" :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00" :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00" :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if amount matches" (let [apply-rules (rm/rule-applying-fn [{:db/id 123 :transaction-rule/amount-gte 3.0 :transaction-rule/amount-lte 9.0 :transaction-rule/transaction-approval-status :transaction-approval-status/approved :transaction-rule/accounts [#:transaction-rule-account {:percentage 1.0 :locatoin "HQ"}] :transaction-rule/vendor {:db/id 123}}])] (t/is (= 123 (-> {:transaction/amount 4.0} (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/amount 3.0} (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/amount 9.0} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/amount 9.01} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/amount 2.99} (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if client matches" (let [apply-rules (rm/rule-applying-fn [{:db/id 123 :transaction-rule/client {:db/id 456}}])] (t/is (= 123 (-> {:transaction/client 456 :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/client 89 :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if bank-account matches" (let [apply-rules (rm/rule-applying-fn [{:db/id 123 :transaction-rule/bank-account {:db/id 456}}])] (t/is (= 123 (-> {:transaction/bank-account 456 :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/bank-account 89 :transaction/amount 1.0} (apply-rules []) :transaction/matched-rule))))) (t/testing "Should prioritize rules" (let [apply-rules (rm/rule-applying-fn (shuffle [{:db/id 2 :transaction-rule/description "Hello" :transaction-rule/amount-gte 5.0} {:db/id 1 :transaction-rule/description "Hello"} {:db/id 0 :transaction-rule/description "Hello"} {:db/id 3 :transaction-rule/description "Hello" :transaction-rule/client {:db/id 789}} {:db/id 4 :transaction-rule/description "Hello" :transaction-rule/client {:db/id 789} :transaction-rule/bank-account {:db/id 456}}]))] (t/is (= 4 (-> {:transaction/bank-account 456 :transaction/client 789 :transaction/description-original "Hello" :transaction/amount 6.0} (apply-rules []) :transaction/matched-rule))) (t/is (= 3 (-> {:transaction/bank-account 457 :transaction/client 789 :transaction/amount 6.0 :transaction/description-original "Hello"} (apply-rules []) :transaction/matched-rule))) (t/is (= 3 (-> {:transaction/bank-account 457 :transaction/client 789 :transaction/amount 6.0 :transaction/description-original "Hello"} (apply-rules []) :transaction/matched-rule))) (t/testing "Should only apply if there is a single rule at that specificity level" (t/is (nil? (-> {:transaction/bank-account 3 :transaction/client 1 :transaction/description-original "Hello" :transaction/amount 0.0} (apply-rules []) :transaction/matched-rule)))))))) (t/deftest match-transaction-to-unfulfilled-payments (t/testing "Auto-pay Invoices" (let [{:strs [vendor1-id vendor2-id]} (->> [#:vendor {:name "Autopay vendor 1" :db/id "vendor1-id"} #:vendor {:name "Autopay vendor 2" :db/id "vendor2-id"}] (d/transact (d/connect uri)) deref :tempids)] (t/testing "Should find a single invoice that matches exactly" (let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= 1 (count invoices-matches))) )) (t/testing "Should not match paid invoice that isn't a scheduled payment" (let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= [] invoices-matches)))) (t/testing "Should not match unpaid invoice" (let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/unpaid :scheduled-payment #inst "2019-01-04" :vendor vendor1-id :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= [] invoices-matches)))) (t/testing "Should not match invoice that already has a payment" (let [{:strs [client-id invoice-id]} (->> [#:invoice {:status :invoice-status/paid :scheduled-payment #inst "2019-01-04" :vendor vendor1-id :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice-id"} {:invoice-payment/amount 30.0 :invoice-payment/invoice "invoice-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= [] invoices-matches)))) (t/testing "Should match multiple invoices for same vendor that total to transaction amount" (let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 15.0 :db/id "invoice1-id"} #:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 15.0 :db/id "invoice2-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= 2 (count invoices-matches)) (str "Expected " (vec invoices-matches) " to have a singular match of two invoices.")))) (t/testing "Should not match if there are multiple candidate matches" (let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice1-id"} #:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice2-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= 0 (count invoices-matches)) (str "Expected " (vec invoices-matches) " to not match due to multiple possibilities.")))) (t/testing "Should not match if invoices are for different vendors" (let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 10.0 :db/id "invoice1-id"} #:invoice {:status :invoice-status/paid :vendor vendor2-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 20.0 :db/id "invoice2-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids) invoices-matches (sut/match-transaction-to-unfulfilled-autopayments -30.0 client-id)] (t/is (= 0 (count invoices-matches)) (str "Expected " (vec invoices-matches) " to only consider invoices for the same vendor.")))) (t/testing "Should only consider invoices chronologically" (let [{:strs [client-id invoice1-id invoice2-id]} (->> [#:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-04" :date #inst "2019-01-01" :client "client-id" :total 10.0 :db/id "invoice1-id"} #:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-06" :date #inst "2019-01-01" :client "client-id" :total 21.0 :db/id "invoice2-id"} #:invoice {:status :invoice-status/paid :vendor vendor1-id :scheduled-payment #inst "2019-01-05" :date #inst "2019-01-01" :client "client-id" :total 30.0 :db/id "invoice3-id"} #:client {:name "Client" :db/id "client-id"}] (d/transact (d/connect uri)) deref :tempids)] (t/is (= 2 (count (sut/match-transaction-to-unfulfilled-autopayments -40.0 client-id))) (str "Expected to match with the chronologically adjacent invoice-1 and invoice-3.")) (t/is (= [] (sut/match-transaction-to-unfulfilled-autopayments -31.0 client-id)) (str "Expected to not match, because there is invoice-3 is between invoice-1 and invoice-2.")))))))