notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

much cleaner matching solution... won't rematch

Browse Source
This commit is contained in:
Bryce Covert
2019-05-16 20:16:12 -07:00
parent dde757e8fc
commit 93ef07f4b1
2 changed files with 105 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/clj/auto_ap/yodlee/import.clj
View File

@@ -51,7 +51,7 @@
nil)) nil))
nil)) nil))
(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules existing] (defn transactions->txs [transactions transaction->bank-account apply-rules existing]
(into [] (into []
(for [transaction transactions (for [transaction transactions
@@ -72,8 +72,11 @@
(- amount) (- amount)
amount) amount)
check-number (extract-check-number transaction) check-number (extract-check-number transaction)
client-id (transaction->client transaction) bank-account (transaction->bank-account transaction)
bank-account-id (transaction->bank-account-id transaction) bank-account-id (:db/id bank-account)
client (:client/_bank-accounts bank-account)
client-id (:db/id client)
valid-locations (:client/locations client)
check (transaction->payment transaction check-number client-id bank-account-id amount id)] check (transaction->payment transaction check-number client-id bank-account-id amount id)]
:when (and client-id :when (and client-id
(not (existing (sha-256 (str id)))))] (not (existing (sha-256 (str id)))))]
@@ -94,11 +97,10 @@
:status status :status status
:client client-id :client client-id
:check-number check-number :check-number check-number
:bank-account (transaction->bank-account-id transaction) :bank-account bank-account-id
:payment (when check :payment (when check
{:db/id (:db/id check) {:db/id (:db/id check)
:payment/status :payment-status/cleared} :payment/status :payment-status/cleared})
)
:vendor (when check :vendor (when check
(:db/id (:payment/vendor check))) (:db/id (:payment/vendor check)))
@@ -109,7 +111,7 @@
:location "A" :location "A"
:amount (Math/abs (double amount))}])} :amount (Math/abs (double amount))}])}
apply-rules (apply-rules valid-locations)
remove-nils)))) remove-nils))))
@@ -162,19 +164,6 @@
(:db/id bank-account)) (:db/id bank-account))
true)))) true))))
(defn more-specific? [left right key]
(cond (and (get left key)
(get right key))
nil
(get left key)
true
(get right key)
false
:else
nil))
(defn rule-priority [rule] (defn rule-priority [rule]
(or (or
(->> [[:transaction-rule/bank-account 0] (->> [[:transaction-rule/bank-account 0]
@@ -209,21 +198,34 @@
(defn rule-applying-fn [rules] (defn rule-applying-fn [rules]
(let [rules-by-priority (group-rules-by-priority rules)] (let [rules-by-priority (group-rules-by-priority rules)]
(fn [transaction] (fn [transaction valid-locations]
(let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )] (if (:transaction/payment transaction)
(if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))] transaction
(assoc transaction (let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )]
:transaction/matched-rule (:db/id top-match) (if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))]
:transaction/approval-status (:transaction-rule/transaction-approval-status top-match) (assoc transaction
:transaction/accounts (map :transaction/matched-rule (:db/id top-match)
(fn [tra] :transaction/approval-status (:transaction-rule/transaction-approval-status top-match)
{:transaction-account/account (:db/id (:transaction-rule-account/account tra)) :transaction/accounts (mapcat
:transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra) (fn [tra]
(:transaction/amount transaction))) (if (= "Shared" (:transaction-rule-account/location tra))
:transaction-account/location (:transaction-rule-account/location tra)}) (map
(:transaction-rule/accounts top-match)) (fn [location]
:transaction/vendor (:db/id (:transaction-rule/vendor top-match))) {:transaction-account/account (:db/id (:transaction-rule-account/account tra))
transaction))))) :transaction-account/amount (Math/abs (* (/ 1.0 (count valid-locations))
(:transaction-rule-account/percentage tra)
(:transaction/amount transaction)))
:transaction-account/location location})
valid-locations)
[{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra)
(:transaction/amount transaction)))
:transaction-account/location (:transaction-rule-account/location tra)}]))
(:transaction-rule/accounts top-match))
:transaction/vendor (:db/id (:transaction-rule/vendor top-match)))
transaction))))))
(defn get-existing [] (defn get-existing []
(transduce (map first) conj #{} (transduce (map first) conj #{}
@@ -232,6 +234,17 @@
:where ['[_ :transaction/id ?tid]]} :where ['[_ :transaction/id ?tid]]}
:args [(d/db (d/connect uri))]}))) :args [(d/db (d/connect uri))]})))
(defn get-all-bank-accounts []
(->> (d-clients/get-all)
(mapcat (fn [client]
(->> client
:client/bank-accounts
(filter :bank-account/yodlee-account-id)
(filter :db/id)
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
(assoc bank-account
:client/_bank-accounts client))))))))
(defn manual-import [manual-transactions] (defn manual-import [manual-transactions]
(let [transformed-transactions (->> manual-transactions (let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %))) (filter #(= "posted" (:status %)))
@@ -241,7 +254,6 @@
(map (map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:client-id client-id
:bank-account-id bank-account-id :bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd") :date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount} :amount {:amount amount}
@@ -250,27 +262,19 @@
:status "POSTED"}) :status "POSTED"})
(range) (range)
transaction-group)))) transaction-group))))
all-rules (tr/get-all)] all-rules (tr/get-all)
all-bank-accounts (get-all-bank-accounts)
transaction->bank-account (comp (by :db/id all-bank-accounts) :bank-account-id)]
(println "importing manual transactions" transformed-transactions) (println "importing manual transactions" transformed-transactions)
(batch-transact (batch-transact
(transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn all-rules) (get-existing))))) (transactions->txs transformed-transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing)))))
(defn do-import (defn do-import
([] ([]
(do-import (client/get-transactions))) (do-import (client/get-transactions)))
([transactions] ([transactions]
(let [all-bank-accounts (mapcat (fn [c] (map (let [all-bank-accounts (get-all-bank-accounts)
(fn [{:keys [:db/id :bank-account/yodlee-account-id]}] transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
(when (and id yodlee-account-id)
{:bank-account-id id
:client-id (:db/id c)
:yodlee-account-id yodlee-account-id}))
(:client/bank-accounts c)))
(d-clients/get-all))
transaction->client (comp (by :yodlee-account-id :client-id all-bank-accounts) :accountId)
transaction->bank-account-id (comp (by :yodlee-account-id :bank-account-id all-bank-accounts) :accountId)
all-rules (tr/get-all)] all-rules (tr/get-all)]
(batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules) (get-existing)))))) (batch-transact (transactions->txs transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing))))))

103
test/clj/auto_ap/yodlee/import.clj
View File

@@ -17,6 +17,9 @@
(t/use-fixtures :each wrap-setup) (t/use-fixtures :each wrap-setup)
(defn noop-rule [transaction locations]
transaction)
(t/deftest do-import (t/deftest do-import
(let [base-transaction {:postDate "2014-01-04" (let [base-transaction {:postDate "2014-01-04"
:accountId 1234 :accountId 1234
@@ -30,13 +33,13 @@
:baseType "DEBIT" :baseType "DEBIT"
:status "PENDING" :status "PENDING"
:client-id 123 :bank-account {:db/id 456
:bank-account-id 456}] :client/_bank-accounts {:db/id 123
:client/locations ["Z" "E"]}}}]
(t/testing "Should import single transaction" (t/testing "Should import single transaction"
(let [result (sut/transactions->txs [base-transaction] (let [result (sut/transactions->txs [base-transaction]
:client-id :bank-account
:bank-account-id noop-rule
identity
#{})] #{})]
(t/is (= [#:transaction {:amount -12.0 (t/is (= [#:transaction {:amount -12.0
:date #inst "2014-01-02T08:00:00.000-00:00" :date #inst "2014-01-02T08:00:00.000-00:00"
@@ -55,19 +58,17 @@
(t/testing "Should not reimport an existing transaction" (t/testing "Should not reimport an existing transaction"
(let [result (sut/transactions->txs [base-transaction] (let [result (sut/transactions->txs [base-transaction]
:client-id :bank-account
:bank-account-id noop-rule
identity
#{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})] #{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})]
(t/is (= [] (t/is (= []
result)))) result))))
(t/testing "Should skip transaction if no client is found" (t/testing "Should skip transaction if no client is found"
(let [result (sut/transactions->txs [(assoc base-transaction :client-id nil)] (let [result (sut/transactions->txs [(assoc base-transaction :bank-account nil)]
:client-id :bank-account
:bank-account-id noop-rule
identity
#{})] #{})]
(t/is (= [] result)))) (t/is (= [] result))))
@@ -93,11 +94,11 @@
:description {:original "CHECK 10001" :description {:original "CHECK 10001"
:simple ""} :simple ""}
:amount {:amount 30.0} :amount {:amount 30.0}
:client-id client-id :bank-account {:db/id bank-account-id
:bank-account-id bank-account-id)] :client/_bank-accounts {:db/id client-id
:client-id :client/locations ["A"]}})]
:bank-account-id :bank-account
identity noop-rule
#{})] #{})]
(t/is (= {:db/id payment-id (t/is (= {:db/id payment-id
@@ -111,11 +112,11 @@
:simple ""} :simple ""}
:amount {:amount 30.0} :amount {:amount 30.0}
:id 789 :id 789
:client-id client-id :bank-account {:db/id bank-account-id
:bank-account-id bank-account-id)] :client/_bank-accounts {:db/id client-id
:client-id :client/locations ["A"]}})]
:bank-account-id :bank-account
identity noop-rule
#{})] #{})]
(t/is (= nil (t/is (= nil
@@ -144,10 +145,10 @@
:simple ""} :simple ""}
:amount {:amount 30.0} :amount {:amount 30.0}
:id 789 :id 789
:client-id client-id :bank-account {:db/id bank-account-id
:bank-account-id bank-account-id)] :client/_bank-accounts {:db/id client-id
:client-id :client/locations ["A"]}})]
:bank-account-id :bank-account
(sut/rule-applying-fn [{:transaction-rule/description "XXX039" (sut/rule-applying-fn [{:transaction-rule/description "XXX039"
:transaction-rule/transaction-approval-status :transaction-approval-status/approved}]) :transaction-rule/transaction-approval-status :transaction-approval-status/approved}])
#{})] #{})]
@@ -161,7 +162,7 @@
:transaction-rule/transaction-approval-status :transaction-approval-status/approved :transaction-rule/transaction-approval-status :transaction-approval-status/approved
:transaction-rule/vendor {:db/id 123} :transaction-rule/vendor {:db/id 123}
:transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9} :transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9}
:transaction-rule-account/location "Z" :transaction-rule-account/location "Shared"
:transaction-rule-account/percentage 1.0}]} :transaction-rule-account/percentage 1.0}]}
{:db/id 2 {:db/id 2
:transaction-rule/description "OtherMatch" :transaction-rule/description "OtherMatch"
@@ -180,7 +181,7 @@
:transaction/amount 30.0} :transaction/amount 30.0}
(-> {:transaction/description-original "Hello XXX039" (-> {:transaction/description-original "Hello XXX039"
:transaction/amount 30.0} :transaction/amount 30.0}
apply-rules))) (apply-rules ["Z"]))))
(t/is (= {:transaction/description-original "OtherMatch", (t/is (= {:transaction/description-original "OtherMatch",
:transaction/approval-status :transaction-approval-status/requires-feedback :transaction/approval-status :transaction-approval-status/requires-feedback
@@ -192,11 +193,11 @@
:transaction-account/location "Z"}]} :transaction-account/location "Z"}]}
(-> {:transaction/description-original "OtherMatch" (-> {:transaction/description-original "OtherMatch"
:transaction/amount 30.0} :transaction/amount 30.0}
apply-rules))) (apply-rules ["Z"]))))
(t/is (= {:transaction/description-original "Hello Not match"} (t/is (= {:transaction/description-original "Hello Not match"}
(-> {:transaction/description-original "Hello Not match"} (-> {:transaction/description-original "Hello Not match"}
apply-rules))))) (apply-rules []))))))
(t/testing "Should match if day of month matches" (t/testing "Should match if day of month matches"
(let [apply-rules (sut/rule-applying-fn [{:db/id 123 (let [apply-rules (sut/rule-applying-fn [{:db/id 123
@@ -206,23 +207,23 @@
:transaction-rule/vendor {:db/id 123}}])] :transaction-rule/vendor {:db/id 123}}])]
(t/is (= 123 (t/is (= 123
(-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00"} (-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 123 (t/is (= 123
(-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00"} (-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 123 (t/is (= 123
(-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00"} (-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00"} (-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00"} (-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00"}
apply-rules (apply-rules [])
:transaction/matched-rule))))) :transaction/matched-rule)))))
(t/testing "Should match if amount matches" (t/testing "Should match if amount matches"
@@ -233,23 +234,23 @@
:transaction-rule/vendor {:db/id 123}}])] :transaction-rule/vendor {:db/id 123}}])]
(t/is (= 123 (t/is (= 123
(-> {:transaction/amount 4.0} (-> {:transaction/amount 4.0}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 123 (t/is (= 123
(-> {:transaction/amount 3.0} (-> {:transaction/amount 3.0}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 123 (t/is (= 123
(-> {:transaction/amount 9.0} (-> {:transaction/amount 9.0}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/amount 9.01} (-> {:transaction/amount 9.01}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/amount 2.99} (-> {:transaction/amount 2.99}
apply-rules (apply-rules [])
:transaction/matched-rule))))) :transaction/matched-rule)))))
(t/testing "Should match if client matches" (t/testing "Should match if client matches"
@@ -257,22 +258,22 @@
:transaction-rule/client {:db/id 456}}])] :transaction-rule/client {:db/id 456}}])]
(t/is (= 123 (t/is (= 123
(-> {:transaction/client 456} (-> {:transaction/client 456}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/client 89} (-> {:transaction/client 89}
apply-rules (apply-rules [])
:transaction/matched-rule))))) :transaction/matched-rule)))))
(t/testing "Should match if bank-account matches" (t/testing "Should match if bank-account matches"
(let [apply-rules (sut/rule-applying-fn [{:db/id 123 (let [apply-rules (sut/rule-applying-fn [{:db/id 123
:transaction-rule/bank-account {:db/id 456}}])] :transaction-rule/bank-account {:db/id 456}}])]
(t/is (= 123 (t/is (= 123
(-> {:transaction/bank-account 456} (-> {:transaction/bank-account 456}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (nil? (t/is (nil?
(-> {:transaction/bank-account 89} (-> {:transaction/bank-account 89}
apply-rules (apply-rules [])
:transaction/matched-rule))))) :transaction/matched-rule)))))
(t/testing "Should prioritize rules" (t/testing "Should prioritize rules"
@@ -296,21 +297,21 @@
:transaction/client 789 :transaction/client 789
:transaction/description-original "Hello" :transaction/description-original "Hello"
:transaction/amount 6.0} :transaction/amount 6.0}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 3 (t/is (= 3
(-> {:transaction/bank-account 457 (-> {:transaction/bank-account 457
:transaction/client 789 :transaction/client 789
:transaction/amount 6.0 :transaction/amount 6.0
:transaction/description-original "Hello"} :transaction/description-original "Hello"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/is (= 3 (t/is (= 3
(-> {:transaction/bank-account 457 (-> {:transaction/bank-account 457
:transaction/client 789 :transaction/client 789
:transaction/amount 6.0 :transaction/amount 6.0
:transaction/description-original "Hello"} :transaction/description-original "Hello"}
apply-rules (apply-rules [])
:transaction/matched-rule))) :transaction/matched-rule)))
(t/testing "Should only apply if there is a single rule at that specificity level" (t/testing "Should only apply if there is a single rule at that specificity level"
@@ -319,9 +320,5 @@
:transaction/client 1 :transaction/client 1
:transaction/description-original "Hello" :transaction/description-original "Hello"
:transaction/amount 0.0} :transaction/amount 0.0}
apply-rules (apply-rules [])
:transaction/matched-rule)))) :transaction/matched-rule)))))))))
#_(t/is (nil?
(-> {:transaction/bank-account 89}
apply-rules
:transaction/matched-rule))))))))