From 93ef07f4b14b8df967b384d3d10585b9817d99ba Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 16 May 2019 20:16:12 -0700 Subject: [PATCH] much cleaner matching solution... won't rematch --- src/clj/auto_ap/yodlee/import.clj | 106 +++++++++++++++-------------- test/clj/auto_ap/yodlee/import.clj | 103 ++++++++++++++-------------- 2 files changed, 105 insertions(+), 104 deletions(-) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 38863cce..d0ce0a0c 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -51,7 +51,7 @@ nil)) nil)) -(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules existing] +(defn transactions->txs [transactions transaction->bank-account apply-rules existing] (into [] (for [transaction transactions @@ -72,8 +72,11 @@ (- amount) amount) check-number (extract-check-number transaction) - client-id (transaction->client transaction) - bank-account-id (transaction->bank-account-id transaction) + bank-account (transaction->bank-account transaction) + bank-account-id (:db/id bank-account) + client (:client/_bank-accounts bank-account) + client-id (:db/id client) + valid-locations (:client/locations client) check (transaction->payment transaction check-number client-id bank-account-id amount id)] :when (and client-id (not (existing (sha-256 (str id)))))] @@ -94,11 +97,10 @@ :status status :client client-id :check-number check-number - :bank-account (transaction->bank-account-id transaction) + :bank-account bank-account-id :payment (when check {:db/id (:db/id check) - :payment/status :payment-status/cleared} - ) + :payment/status :payment-status/cleared}) :vendor (when check (:db/id (:payment/vendor check))) @@ -109,7 +111,7 @@ :location "A" :amount (Math/abs (double amount))}])} - apply-rules + (apply-rules valid-locations) remove-nils)))) @@ -162,19 +164,6 @@ (:db/id bank-account)) true)))) -(defn more-specific? [left right key] - (cond (and (get left key) - (get right key)) - nil - (get left key) - true - - (get right key) - false - - :else - nil)) - (defn rule-priority [rule] (or (->> [[:transaction-rule/bank-account 0] @@ -209,21 +198,34 @@ (defn rule-applying-fn [rules] (let [rules-by-priority (group-rules-by-priority rules)] - (fn [transaction] - (let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )] - (if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))] - (assoc transaction - :transaction/matched-rule (:db/id top-match) - :transaction/approval-status (:transaction-rule/transaction-approval-status top-match) - :transaction/accounts (map - (fn [tra] - {:transaction-account/account (:db/id (:transaction-rule-account/account tra)) - :transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra) - (:transaction/amount transaction))) - :transaction-account/location (:transaction-rule-account/location tra)}) - (:transaction-rule/accounts top-match)) - :transaction/vendor (:db/id (:transaction-rule/vendor top-match))) - transaction))))) + (fn [transaction valid-locations] + (if (:transaction/payment transaction) + transaction + (let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )] + (if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))] + (assoc transaction + :transaction/matched-rule (:db/id top-match) + :transaction/approval-status (:transaction-rule/transaction-approval-status top-match) + :transaction/accounts (mapcat + (fn [tra] + (if (= "Shared" (:transaction-rule-account/location tra)) + (map + (fn [location] + {:transaction-account/account (:db/id (:transaction-rule-account/account tra)) + :transaction-account/amount (Math/abs (* (/ 1.0 (count valid-locations)) + (:transaction-rule-account/percentage tra) + (:transaction/amount transaction))) + :transaction-account/location location}) + + + valid-locations) + [{:transaction-account/account (:db/id (:transaction-rule-account/account tra)) + :transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra) + (:transaction/amount transaction))) + :transaction-account/location (:transaction-rule-account/location tra)}])) + (:transaction-rule/accounts top-match)) + :transaction/vendor (:db/id (:transaction-rule/vendor top-match))) + transaction)))))) (defn get-existing [] (transduce (map first) conj #{} @@ -232,6 +234,17 @@ :where ['[_ :transaction/id ?tid]]} :args [(d/db (d/connect uri))]}))) +(defn get-all-bank-accounts [] + (->> (d-clients/get-all) + (mapcat (fn [client] + (->> client + :client/bank-accounts + (filter :bank-account/yodlee-account-id) + (filter :db/id) + (map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}] + (assoc bank-account + :client/_bank-accounts client)))))))) + (defn manual-import [manual-transactions] (let [transformed-transactions (->> manual-transactions (filter #(= "posted" (:status %))) @@ -241,7 +254,6 @@ (map (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) - :client-id client-id :bank-account-id bank-account-id :date (time/unparse date "YYYY-MM-dd") :amount {:amount amount} @@ -250,27 +262,19 @@ :status "POSTED"}) (range) transaction-group)))) - all-rules (tr/get-all)] + all-rules (tr/get-all) + all-bank-accounts (get-all-bank-accounts) + transaction->bank-account (comp (by :db/id all-bank-accounts) :bank-account-id)] (println "importing manual transactions" transformed-transactions) (batch-transact - (transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn all-rules) (get-existing))))) - - + (transactions->txs transformed-transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing))))) (defn do-import ([] (do-import (client/get-transactions))) ([transactions] - (let [all-bank-accounts (mapcat (fn [c] (map - (fn [{:keys [:db/id :bank-account/yodlee-account-id]}] - (when (and id yodlee-account-id) - {:bank-account-id id - :client-id (:db/id c) - :yodlee-account-id yodlee-account-id})) - (:client/bank-accounts c))) - (d-clients/get-all)) - transaction->client (comp (by :yodlee-account-id :client-id all-bank-accounts) :accountId) - transaction->bank-account-id (comp (by :yodlee-account-id :bank-account-id all-bank-accounts) :accountId) + (let [all-bank-accounts (get-all-bank-accounts) + transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId) all-rules (tr/get-all)] - (batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules) (get-existing)))))) + (batch-transact (transactions->txs transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing)))))) diff --git a/test/clj/auto_ap/yodlee/import.clj b/test/clj/auto_ap/yodlee/import.clj index 80a30201..5af7d83f 100644 --- a/test/clj/auto_ap/yodlee/import.clj +++ b/test/clj/auto_ap/yodlee/import.clj @@ -17,6 +17,9 @@ (t/use-fixtures :each wrap-setup) +(defn noop-rule [transaction locations] + transaction) + (t/deftest do-import (let [base-transaction {:postDate "2014-01-04" :accountId 1234 @@ -30,13 +33,13 @@ :baseType "DEBIT" :status "PENDING" - :client-id 123 - :bank-account-id 456}] + :bank-account {:db/id 456 + :client/_bank-accounts {:db/id 123 + :client/locations ["Z" "E"]}}}] (t/testing "Should import single transaction" (let [result (sut/transactions->txs [base-transaction] - :client-id - :bank-account-id - identity + :bank-account + noop-rule #{})] (t/is (= [#:transaction {:amount -12.0 :date #inst "2014-01-02T08:00:00.000-00:00" @@ -55,19 +58,17 @@ (t/testing "Should not reimport an existing transaction" (let [result (sut/transactions->txs [base-transaction] - :client-id - :bank-account-id - identity + :bank-account + noop-rule #{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})] (t/is (= [] result)))) (t/testing "Should skip transaction if no client is found" - (let [result (sut/transactions->txs [(assoc base-transaction :client-id nil)] - :client-id - :bank-account-id - identity + (let [result (sut/transactions->txs [(assoc base-transaction :bank-account nil)] + :bank-account + noop-rule #{})] (t/is (= [] result)))) @@ -93,11 +94,11 @@ :description {:original "CHECK 10001" :simple ""} :amount {:amount 30.0} - :client-id client-id - :bank-account-id bank-account-id)] - :client-id - :bank-account-id - identity + :bank-account {:db/id bank-account-id + :client/_bank-accounts {:db/id client-id + :client/locations ["A"]}})] + :bank-account + noop-rule #{})] (t/is (= {:db/id payment-id @@ -111,11 +112,11 @@ :simple ""} :amount {:amount 30.0} :id 789 - :client-id client-id - :bank-account-id bank-account-id)] - :client-id - :bank-account-id - identity + :bank-account {:db/id bank-account-id + :client/_bank-accounts {:db/id client-id + :client/locations ["A"]}})] + :bank-account + noop-rule #{})] (t/is (= nil @@ -144,10 +145,10 @@ :simple ""} :amount {:amount 30.0} :id 789 - :client-id client-id - :bank-account-id bank-account-id)] - :client-id - :bank-account-id + :bank-account {:db/id bank-account-id + :client/_bank-accounts {:db/id client-id + :client/locations ["A"]}})] + :bank-account (sut/rule-applying-fn [{:transaction-rule/description "XXX039" :transaction-rule/transaction-approval-status :transaction-approval-status/approved}]) #{})] @@ -161,7 +162,7 @@ :transaction-rule/transaction-approval-status :transaction-approval-status/approved :transaction-rule/vendor {:db/id 123} :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} - :transaction-rule-account/location "Z" + :transaction-rule-account/location "Shared" :transaction-rule-account/percentage 1.0}]} {:db/id 2 :transaction-rule/description "OtherMatch" @@ -180,7 +181,7 @@ :transaction/amount 30.0} (-> {:transaction/description-original "Hello XXX039" :transaction/amount 30.0} - apply-rules))) + (apply-rules ["Z"])))) (t/is (= {:transaction/description-original "OtherMatch", :transaction/approval-status :transaction-approval-status/requires-feedback @@ -192,11 +193,11 @@ :transaction-account/location "Z"}]} (-> {:transaction/description-original "OtherMatch" :transaction/amount 30.0} - apply-rules))) + (apply-rules ["Z"])))) (t/is (= {:transaction/description-original "Hello Not match"} (-> {:transaction/description-original "Hello Not match"} - apply-rules))))) + (apply-rules [])))))) (t/testing "Should match if day of month matches" (let [apply-rules (sut/rule-applying-fn [{:db/id 123 @@ -206,23 +207,23 @@ :transaction-rule/vendor {:db/id 123}}])] (t/is (= 123 (-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00"} - apply-rules + (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if amount matches" @@ -233,23 +234,23 @@ :transaction-rule/vendor {:db/id 123}}])] (t/is (= 123 (-> {:transaction/amount 4.0} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/amount 3.0} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 123 (-> {:transaction/amount 9.0} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/amount 9.01} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/amount 2.99} - apply-rules + (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if client matches" @@ -257,22 +258,22 @@ :transaction-rule/client {:db/id 456}}])] (t/is (= 123 (-> {:transaction/client 456} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/client 89} - apply-rules + (apply-rules []) :transaction/matched-rule))))) (t/testing "Should match if bank-account matches" (let [apply-rules (sut/rule-applying-fn [{:db/id 123 :transaction-rule/bank-account {:db/id 456}}])] (t/is (= 123 (-> {:transaction/bank-account 456} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (nil? (-> {:transaction/bank-account 89} - apply-rules + (apply-rules []) :transaction/matched-rule))))) (t/testing "Should prioritize rules" @@ -296,21 +297,21 @@ :transaction/client 789 :transaction/description-original "Hello" :transaction/amount 6.0} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 3 (-> {:transaction/bank-account 457 :transaction/client 789 :transaction/amount 6.0 :transaction/description-original "Hello"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/is (= 3 (-> {:transaction/bank-account 457 :transaction/client 789 :transaction/amount 6.0 :transaction/description-original "Hello"} - apply-rules + (apply-rules []) :transaction/matched-rule))) (t/testing "Should only apply if there is a single rule at that specificity level" @@ -319,9 +320,5 @@ :transaction/client 1 :transaction/description-original "Hello" :transaction/amount 0.0} - apply-rules - :transaction/matched-rule)))) - #_(t/is (nil? - (-> {:transaction/bank-account 89} - apply-rules - :transaction/matched-rule)))))))) + (apply-rules []) + :transaction/matched-rule)))))))))