From dde757e8fcab39f500ccf516f01ad19e6642b597 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 16 May 2019 19:01:33 -0700 Subject: [PATCH] supports grouping rules by priority. --- src/clj/auto_ap/yodlee/import.clj | 75 +++++++++++++++++++----------- test/clj/auto_ap/yodlee/import.clj | 36 ++++++++++++-- 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index cfa8726e..38863cce 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -51,7 +51,7 @@ nil)) nil)) -(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules] +(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules existing] (into [] (for [transaction transactions @@ -75,7 +75,8 @@ client-id (transaction->client transaction) bank-account-id (transaction->bank-account-id transaction) check (transaction->payment transaction check-number client-id bank-account-id amount id)] - :when client-id] + :when (and client-id + (not (existing (sha-256 (str id)))))] (-> #:transaction {:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd")) @@ -174,31 +175,43 @@ :else nil)) -(defn prioritize-rule [left right] - (first (filter #(not (nil? %)) - [(more-specific? left right :transaction-rule/bank-account) - (more-specific? left right :transaction-rule/client) - (more-specific? left right :transaction-rule/dom-lte) - (more-specific? left right :transaction-rule/dom-gte) - (more-specific? left right :transaction-rule/amount-lte) - (more-specific? left right :transaction-rule/amount-gte) - (more-specific? left right :transaction-rule/description) - (more-specific? left right :transaction-rule/yodlee-merchant) - - true]))) +(defn rule-priority [rule] + (or + (->> [[:transaction-rule/bank-account 0] + [:transaction-rule/client 1] + [:transaction-rule/dom-lte 2] + [:transaction-rule/dom-gte 2] + [:transaction-rule/amount-lte 3] + [:transaction-rule/amount-gte 3] + [:transaction-rule/description 4] + [:transaction-rule/yodlee-merchant 5]] + (filter (fn [[key]] + (get rule key))) + (map second) + first) + 6)) + +(defn get-matching-rules-by-priority [rules-by-priority transaction] + (loop [[rule-set & rules] rules-by-priority] + (if rule-set + (let [matching-rules (into [] (filter #(rule-applies? transaction %) rule-set))] + (if (seq matching-rules) + matching-rules + (recur rules))) + []))) + +(defn group-rules-by-priority [rules] + (->> rules + (map (fn [r] (update r :transaction-rule/description #(some-> % re-pattern)))) + (group-by rule-priority) + (sort-by first) + (map second))) (defn rule-applying-fn [rules] - (let [rules (transduce (map (fn [r] - (update r :transaction-rule/description #(some-> % re-pattern)))) - conj - [] - (sort prioritize-rule rules))] - - + (let [rules-by-priority (group-rules-by-priority rules)] (fn [transaction] - (let [matching-rules (->> rules - (filter #(rule-applies? transaction %) ))] - (if-let [top-match (first matching-rules)] + (let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )] + (if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))] (assoc transaction :transaction/matched-rule (:db/id top-match) :transaction/approval-status (:transaction-rule/transaction-approval-status top-match) @@ -212,6 +225,13 @@ :transaction/vendor (:db/id (:transaction-rule/vendor top-match))) transaction))))) +(defn get-existing [] + (transduce (map first) conj #{} + (d/query {:query {:find ['?tid] + :in ['$] + :where ['[_ :transaction/id ?tid]]} + :args [(d/db (d/connect uri))]}))) + (defn manual-import [manual-transactions] (let [transformed-transactions (->> manual-transactions (filter #(= "posted" (:status %))) @@ -229,10 +249,11 @@ :simple high-level-category} :status "POSTED"}) (range) - transaction-group))))] + transaction-group)))) + all-rules (tr/get-all)] (println "importing manual transactions" transformed-transactions) (batch-transact - (transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn []))))) + (transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn all-rules) (get-existing))))) @@ -251,5 +272,5 @@ transaction->client (comp (by :yodlee-account-id :client-id all-bank-accounts) :accountId) transaction->bank-account-id (comp (by :yodlee-account-id :bank-account-id all-bank-accounts) :accountId) all-rules (tr/get-all)] - (batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules)))))) + (batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules) (get-existing)))))) diff --git a/test/clj/auto_ap/yodlee/import.clj b/test/clj/auto_ap/yodlee/import.clj index 4ff6b0f5..80a30201 100644 --- a/test/clj/auto_ap/yodlee/import.clj +++ b/test/clj/auto_ap/yodlee/import.clj @@ -36,7 +36,8 @@ (let [result (sut/transactions->txs [base-transaction] :client-id :bank-account-id - identity)] + identity + #{})] (t/is (= [#:transaction {:amount -12.0 :date #inst "2014-01-02T08:00:00.000-00:00" :bank-account 456 @@ -52,11 +53,22 @@ :description-simple "simple-description"}] result)))) + (t/testing "Should not reimport an existing transaction" + (let [result (sut/transactions->txs [base-transaction] + :client-id + :bank-account-id + identity + #{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})] + (t/is (= [] + result)))) + + (t/testing "Should skip transaction if no client is found" (let [result (sut/transactions->txs [(assoc base-transaction :client-id nil)] :client-id :bank-account-id - identity)] + identity + #{})] (t/is (= [] result)))) (t/testing "Should match an uncleared check" @@ -85,7 +97,8 @@ :bank-account-id bank-account-id)] :client-id :bank-account-id - identity)] + identity + #{})] (t/is (= {:db/id payment-id :payment/status :payment-status/cleared} @@ -102,7 +115,8 @@ :bank-account-id bank-account-id)] :client-id :bank-account-id - identity)] + identity + #{})] (t/is (= nil (:transaction/payment result))))))) @@ -135,7 +149,8 @@ :client-id :bank-account-id (sut/rule-applying-fn [{:transaction-rule/description "XXX039" - :transaction-rule/transaction-approval-status :transaction-approval-status/approved}]))] + :transaction-rule/transaction-approval-status :transaction-approval-status/approved}]) + #{})] (t/is (= :transaction-approval-status/approved (:transaction/approval-status result))))) @@ -266,6 +281,8 @@ :transaction-rule/amount-gte 5.0} {:db/id 1 :transaction-rule/description "Hello"} + {:db/id 0 + :transaction-rule/description "Hello"} {:db/id 3 :transaction-rule/description "Hello" :transaction-rule/client {:db/id 789}} @@ -295,6 +312,15 @@ :transaction/description-original "Hello"} apply-rules :transaction/matched-rule))) + + (t/testing "Should only apply if there is a single rule at that specificity level" + (t/is (nil? + (-> {:transaction/bank-account 3 + :transaction/client 1 + :transaction/description-original "Hello" + :transaction/amount 0.0} + apply-rules + :transaction/matched-rule)))) #_(t/is (nil? (-> {:transaction/bank-account 89} apply-rules