notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

much cleaner matching solution... won't rematch

Browse Source
This commit is contained in:
Bryce Covert
2019-05-16 20:16:12 -07:00
parent dde757e8fc
commit 93ef07f4b1
2 changed files with 105 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/clj/auto_ap/yodlee/import.clj
View File

@@ -51,7 +51,7 @@
nil))
nil))
(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules existing]
(defn transactions->txs [transactions transaction->bank-account apply-rules existing]
(into []
(for [transaction transactions
@@ -72,8 +72,11 @@
(- amount)
amount)
check-number (extract-check-number transaction)
client-id (transaction->client transaction)
bank-account-id (transaction->bank-account-id transaction)
bank-account (transaction->bank-account transaction)
bank-account-id (:db/id bank-account)
client (:client/_bank-accounts bank-account)
client-id (:db/id client)
valid-locations (:client/locations client)
check (transaction->payment transaction check-number client-id bank-account-id amount id)]
:when (and client-id
(not (existing (sha-256 (str id)))))]
@@ -94,11 +97,10 @@
:status status
:client client-id
:check-number check-number
:bank-account (transaction->bank-account-id transaction)
:bank-account bank-account-id
:payment (when check
{:db/id (:db/id check)
:payment/status :payment-status/cleared}
)
:payment/status :payment-status/cleared})
:vendor (when check
(:db/id (:payment/vendor check)))
@@ -109,7 +111,7 @@
:location "A"
:amount (Math/abs (double amount))}])}
apply-rules
(apply-rules valid-locations)
remove-nils))))
@@ -162,19 +164,6 @@
(:db/id bank-account))
true))))
(defn more-specific? [left right key]
(cond (and (get left key)
(get right key))
nil
(get left key)
true
(get right key)
false
:else
nil))
(defn rule-priority [rule]
(or
(->> [[:transaction-rule/bank-account 0]
@@ -209,21 +198,34 @@
(defn rule-applying-fn [rules]
(let [rules-by-priority (group-rules-by-priority rules)]
(fn [transaction]
(let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )]
(if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))]
(assoc transaction
:transaction/matched-rule (:db/id top-match)
:transaction/approval-status (:transaction-rule/transaction-approval-status top-match)
:transaction/accounts (map
(fn [tra]
{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra)
(:transaction/amount transaction)))
:transaction-account/location (:transaction-rule-account/location tra)})
(:transaction-rule/accounts top-match))
:transaction/vendor (:db/id (:transaction-rule/vendor top-match)))
transaction)))))
(fn [transaction valid-locations]
(if (:transaction/payment transaction)
transaction
(let [matching-rules (get-matching-rules-by-priority rules-by-priority transaction )]
(if-let [top-match (and (= (count matching-rules) 1) (first matching-rules))]
(assoc transaction
:transaction/matched-rule (:db/id top-match)
:transaction/approval-status (:transaction-rule/transaction-approval-status top-match)
:transaction/accounts (mapcat
(fn [tra]
(if (= "Shared" (:transaction-rule-account/location tra))
(map
(fn [location]
{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (Math/abs (* (/ 1.0 (count valid-locations))
(:transaction-rule-account/percentage tra)
(:transaction/amount transaction)))
:transaction-account/location location})
valid-locations)
[{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
:transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra)
(:transaction/amount transaction)))
:transaction-account/location (:transaction-rule-account/location tra)}]))
(:transaction-rule/accounts top-match))
:transaction/vendor (:db/id (:transaction-rule/vendor top-match)))
transaction))))))
(defn get-existing []
(transduce (map first) conj #{}
@@ -232,6 +234,17 @@
:where ['[_ :transaction/id ?tid]]}
:args [(d/db (d/connect uri))]})))
(defn get-all-bank-accounts []
(->> (d-clients/get-all)
(mapcat (fn [client]
(->> client
:client/bank-accounts
(filter :bank-account/yodlee-account-id)
(filter :db/id)
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
(assoc bank-account
:client/_bank-accounts client))))))))
(defn manual-import [manual-transactions]
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
@@ -241,7 +254,6 @@
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:client-id client-id
:bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount}
@@ -250,27 +262,19 @@
:status "POSTED"})
(range)
transaction-group))))
all-rules (tr/get-all)]
all-rules (tr/get-all)
all-bank-accounts (get-all-bank-accounts)
transaction->bank-account (comp (by :db/id all-bank-accounts) :bank-account-id)]
(println "importing manual transactions" transformed-transactions)
(batch-transact
(transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn all-rules) (get-existing)))))
(transactions->txs transformed-transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing)))))
(defn do-import
([]
(do-import (client/get-transactions)))
([transactions]
(let [all-bank-accounts (mapcat (fn [c] (map
(fn [{:keys [:db/id :bank-account/yodlee-account-id]}]
(when (and id yodlee-account-id)
{:bank-account-id id
:client-id (:db/id c)
:yodlee-account-id yodlee-account-id}))
(:client/bank-accounts c)))
(d-clients/get-all))
transaction->client (comp (by :yodlee-account-id :client-id all-bank-accounts) :accountId)
transaction->bank-account-id (comp (by :yodlee-account-id :bank-account-id all-bank-accounts) :accountId)
(let [all-bank-accounts (get-all-bank-accounts)
transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
all-rules (tr/get-all)]
(batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules) (get-existing))))))
(batch-transact (transactions->txs transactions transaction->bank-account (rule-applying-fn all-rules) (get-existing))))))

103
test/clj/auto_ap/yodlee/import.clj
View File

@@ -17,6 +17,9 @@
(t/use-fixtures :each wrap-setup)
(defn noop-rule [transaction locations]
transaction)
(t/deftest do-import
(let [base-transaction {:postDate "2014-01-04"
:accountId 1234
@@ -30,13 +33,13 @@
:baseType "DEBIT"
:status "PENDING"
:client-id 123
:bank-account-id 456}]
:bank-account {:db/id 456
:client/_bank-accounts {:db/id 123
:client/locations ["Z" "E"]}}}]
(t/testing "Should import single transaction"
(let [result (sut/transactions->txs [base-transaction]
:client-id
:bank-account-id
identity
:bank-account
noop-rule
#{})]
(t/is (= [#:transaction {:amount -12.0
:date #inst "2014-01-02T08:00:00.000-00:00"
@@ -55,19 +58,17 @@
(t/testing "Should not reimport an existing transaction"
(let [result (sut/transactions->txs [base-transaction]
:client-id
:bank-account-id
identity
:bank-account
noop-rule
#{"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b"})]
(t/is (= []
result))))
(t/testing "Should skip transaction if no client is found"
(let [result (sut/transactions->txs [(assoc base-transaction :client-id nil)]
:client-id
:bank-account-id
identity
(let [result (sut/transactions->txs [(assoc base-transaction :bank-account nil)]
:bank-account
noop-rule
#{})]
(t/is (= [] result))))
@@ -93,11 +94,11 @@
:description {:original "CHECK 10001"
:simple ""}
:amount {:amount 30.0}
:client-id client-id
:bank-account-id bank-account-id)]
:client-id
:bank-account-id
identity
:bank-account {:db/id bank-account-id
:client/_bank-accounts {:db/id client-id
:client/locations ["A"]}})]
:bank-account
noop-rule
#{})]
(t/is (= {:db/id payment-id
@@ -111,11 +112,11 @@
:simple ""}
:amount {:amount 30.0}
:id 789
:client-id client-id
:bank-account-id bank-account-id)]
:client-id
:bank-account-id
identity
:bank-account {:db/id bank-account-id
:client/_bank-accounts {:db/id client-id
:client/locations ["A"]}})]
:bank-account
noop-rule
#{})]
(t/is (= nil
@@ -144,10 +145,10 @@
:simple ""}
:amount {:amount 30.0}
:id 789
:client-id client-id
:bank-account-id bank-account-id)]
:client-id
:bank-account-id
:bank-account {:db/id bank-account-id
:client/_bank-accounts {:db/id client-id
:client/locations ["A"]}})]
:bank-account
(sut/rule-applying-fn [{:transaction-rule/description "XXX039"
:transaction-rule/transaction-approval-status :transaction-approval-status/approved}])
#{})]
@@ -161,7 +162,7 @@
:transaction-rule/transaction-approval-status :transaction-approval-status/approved
:transaction-rule/vendor {:db/id 123}
:transaction-rule/accounts [{:transaction-rule-account/account {:db/id 9}
:transaction-rule-account/location "Z"
:transaction-rule-account/location "Shared"
:transaction-rule-account/percentage 1.0}]}
{:db/id 2
:transaction-rule/description "OtherMatch"
@@ -180,7 +181,7 @@
:transaction/amount 30.0}
(-> {:transaction/description-original "Hello XXX039"
:transaction/amount 30.0}
apply-rules)))
(apply-rules ["Z"]))))
(t/is (= {:transaction/description-original "OtherMatch",
:transaction/approval-status :transaction-approval-status/requires-feedback
@@ -192,11 +193,11 @@
:transaction-account/location "Z"}]}
(-> {:transaction/description-original "OtherMatch"
:transaction/amount 30.0}
apply-rules)))
(apply-rules ["Z"]))))
(t/is (= {:transaction/description-original "Hello Not match"}
(-> {:transaction/description-original "Hello Not match"}
apply-rules)))))
(apply-rules []))))))
(t/testing "Should match if day of month matches"
(let [apply-rules (sut/rule-applying-fn [{:db/id 123
@@ -206,23 +207,23 @@
:transaction-rule/vendor {:db/id 123}}])]
(t/is (= 123
(-> {:transaction/date #inst "2019-01-04T00:00:00.000-08:00"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 123
(-> {:transaction/date #inst "2019-01-03T00:00:00.000-08:00"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 123
(-> {:transaction/date #inst "2019-01-09T00:00:00.000-08:00"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/date #inst "2019-01-01T00:00:00.000-08:00"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/date #inst "2019-01-10T00:00:00.000-08:00"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))))
(t/testing "Should match if amount matches"
@@ -233,23 +234,23 @@
:transaction-rule/vendor {:db/id 123}}])]
(t/is (= 123
(-> {:transaction/amount 4.0}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 123
(-> {:transaction/amount 3.0}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 123
(-> {:transaction/amount 9.0}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/amount 9.01}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/amount 2.99}
apply-rules
(apply-rules [])
:transaction/matched-rule)))))
(t/testing "Should match if client matches"
@@ -257,22 +258,22 @@
:transaction-rule/client {:db/id 456}}])]
(t/is (= 123
(-> {:transaction/client 456}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/client 89}
apply-rules
(apply-rules [])
:transaction/matched-rule)))))
(t/testing "Should match if bank-account matches"
(let [apply-rules (sut/rule-applying-fn [{:db/id 123
:transaction-rule/bank-account {:db/id 456}}])]
(t/is (= 123
(-> {:transaction/bank-account 456}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (nil?
(-> {:transaction/bank-account 89}
apply-rules
(apply-rules [])
:transaction/matched-rule)))))
(t/testing "Should prioritize rules"
@@ -296,21 +297,21 @@
:transaction/client 789
:transaction/description-original "Hello"
:transaction/amount 6.0}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 3
(-> {:transaction/bank-account 457
:transaction/client 789
:transaction/amount 6.0
:transaction/description-original "Hello"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/is (= 3
(-> {:transaction/bank-account 457
:transaction/client 789
:transaction/amount 6.0
:transaction/description-original "Hello"}
apply-rules
(apply-rules [])
:transaction/matched-rule)))
(t/testing "Should only apply if there is a single rule at that specificity level"
@@ -319,9 +320,5 @@
:transaction/client 1
:transaction/description-original "Hello"
:transaction/amount 0.0}
apply-rules
:transaction/matched-rule))))
#_(t/is (nil?
(-> {:transaction/bank-account 89}
apply-rules
:transaction/matched-rule))))))))
(apply-rules [])
:transaction/matched-rule)))))))))