256 lines
11 KiB
Clojure
256 lines
11 KiB
Clojure
(ns auto-ap.yodlee.import
|
|
(:require [auto-ap.yodlee.core :as client]
|
|
[auto-ap.utils :refer [by]]
|
|
[datomic.api :as d]
|
|
[auto-ap.datomic :refer [uri remove-nils]]
|
|
[auto-ap.datomic.accounts :as a]
|
|
[clj-time.coerce :as coerce]
|
|
[digest :refer [sha-256]]
|
|
[auto-ap.datomic.checks :as d-checks]
|
|
[auto-ap.datomic.transactions :as d-transactions]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[auto-ap.time :as time]
|
|
[auto-ap.datomic.transaction-rules :as tr]))
|
|
|
|
|
|
|
|
(defn transaction->payment [_ check-number client-id bank-account-id amount id]
|
|
(cond (not (and client-id bank-account-id))
|
|
nil
|
|
|
|
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
|
nil
|
|
|
|
check-number
|
|
(-> (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:check-number check-number
|
|
:amount (- amount)
|
|
:status :payment-status/pending})
|
|
first
|
|
first)
|
|
|
|
(and client-id bank-account-id amount)
|
|
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:amount (- amount)
|
|
:status :payment-status/pending})]
|
|
(if (= 1 (count matching-checks))
|
|
(first matching-checks)
|
|
nil))
|
|
|
|
:else
|
|
nil))
|
|
|
|
(defn extract-check-number [{{description-original :original} :description}]
|
|
|
|
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
|
(try
|
|
(Integer/parseInt check-number)
|
|
(catch NumberFormatException e
|
|
nil))
|
|
nil))
|
|
|
|
(defn transactions->txs [transactions transaction->client transaction->bank-account-id apply-rules]
|
|
(into []
|
|
|
|
(for [transaction transactions
|
|
:let [{post-date :postDate
|
|
account-id :accountId
|
|
date :date
|
|
id :id
|
|
{amount :amount} :amount
|
|
{description-original :original
|
|
description-simple :simple} :description
|
|
{merchant-id :id
|
|
merchant-name :name} :merchant
|
|
base-type :baseType
|
|
type :type
|
|
status :status}
|
|
transaction
|
|
amount (if (= "DEBIT" base-type)
|
|
(- amount)
|
|
amount)
|
|
check-number (extract-check-number transaction)
|
|
client-id (transaction->client transaction)
|
|
bank-account-id (transaction->bank-account-id transaction)
|
|
check (transaction->payment transaction check-number client-id bank-account-id amount id)]
|
|
:when client-id]
|
|
(->
|
|
#:transaction
|
|
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
|
|
:id (sha-256 (str id))
|
|
:account-id account-id
|
|
:date (coerce/to-date (time/parse date "YYYY-MM-dd"))
|
|
:yodlee-merchant (when (and merchant-id merchant-name)
|
|
{:yodlee-merchant/yodlee-id merchant-id
|
|
:yodlee-merchant/name merchant-name})
|
|
:amount (double amount)
|
|
:description-original description-original
|
|
:description-simple description-simple
|
|
:exclude-from-ledger false
|
|
:type type
|
|
:status status
|
|
:client client-id
|
|
:check-number check-number
|
|
:bank-account (transaction->bank-account-id transaction)
|
|
:payment (when check
|
|
{:db/id (:db/id check)
|
|
:payment/status :payment-status/cleared}
|
|
)
|
|
|
|
:vendor (when check
|
|
(:db/id (:payment/vendor check)))
|
|
:location (when check
|
|
"A")
|
|
:accounts (when check
|
|
[#:transaction-account {:account (:db/id (a/get-account-by-numeric-code-and-sets 2110 ["default"]))
|
|
:location "A"
|
|
:amount (Math/abs (double amount))}])}
|
|
|
|
apply-rules
|
|
remove-nils))))
|
|
|
|
|
|
(defn batch-transact [transactions]
|
|
(reduce (fn [acc batch]
|
|
(into acc (->> batch
|
|
(d/transact (d/connect uri) )
|
|
(deref)
|
|
:tempids
|
|
vals)))
|
|
[]
|
|
(partition-all 100 transactions)))
|
|
|
|
(defn rule-applies? [transaction {:keys [:transaction-rule/description
|
|
:transaction-rule/dom-gte :transaction-rule/dom-lte
|
|
:transaction-rule/amount-gte :transaction-rule/amount-lte
|
|
:transaction-rule/client :transaction-rule/bank-account
|
|
:transaction-rule/yodlee-merchant]} ]
|
|
(let [transaction-dom (some-> transaction
|
|
:transaction/date
|
|
.toInstant
|
|
(.atZone (java.time.ZoneId/of "US/Pacific"))
|
|
(.get java.time.temporal.ChronoField/DAY_OF_MONTH))]
|
|
(and
|
|
(if description
|
|
(re-find description (:transaction/description-original transaction))
|
|
true)
|
|
(if dom-gte
|
|
(>= transaction-dom dom-gte)
|
|
true)
|
|
(if dom-lte
|
|
(<= transaction-dom dom-lte)
|
|
true)
|
|
(if amount-gte
|
|
(>= (:transaction/amount transaction) amount-gte)
|
|
true)
|
|
(if amount-lte
|
|
(<= (:transaction/amount transaction) amount-lte)
|
|
true)
|
|
(if client
|
|
(= (:transaction/client transaction)
|
|
(:db/id client))
|
|
true)
|
|
(if yodlee-merchant
|
|
(= (:yodlee-merchant/yodlee-id (:transaction/yodlee-merchant transaction))
|
|
(:yodlee-merchant/yodlee-id yodlee-merchant))
|
|
true)
|
|
(if bank-account
|
|
(= (:transaction/bank-account transaction)
|
|
(:db/id bank-account))
|
|
true))))
|
|
|
|
(defn more-specific? [left right key]
|
|
(cond (and (get left key)
|
|
(get right key))
|
|
nil
|
|
(get left key)
|
|
true
|
|
|
|
(get right key)
|
|
false
|
|
|
|
:else
|
|
nil))
|
|
|
|
(defn prioritize-rule [left right]
|
|
(first (filter #(not (nil? %))
|
|
[(more-specific? left right :transaction-rule/bank-account)
|
|
(more-specific? left right :transaction-rule/client)
|
|
(more-specific? left right :transaction-rule/dom-lte)
|
|
(more-specific? left right :transaction-rule/dom-gte)
|
|
(more-specific? left right :transaction-rule/amount-lte)
|
|
(more-specific? left right :transaction-rule/amount-gte)
|
|
(more-specific? left right :transaction-rule/description)
|
|
(more-specific? left right :transaction-rule/yodlee-merchant)
|
|
|
|
true])))
|
|
|
|
(defn rule-applying-fn [rules]
|
|
(let [rules (transduce (map (fn [r]
|
|
(update r :transaction-rule/description #(some-> % re-pattern))))
|
|
conj
|
|
[]
|
|
(sort prioritize-rule rules))]
|
|
|
|
|
|
(fn [transaction]
|
|
(let [matching-rules (->> rules
|
|
(filter #(rule-applies? transaction %) ))]
|
|
(if-let [top-match (first matching-rules)]
|
|
(assoc transaction
|
|
:transaction/matched-rule (:db/id top-match)
|
|
:transaction/approval-status (:transaction-rule/transaction-approval-status top-match)
|
|
:transaction/accounts (map
|
|
(fn [tra]
|
|
{:transaction-account/account (:db/id (:transaction-rule-account/account tra))
|
|
:transaction-account/amount (Math/abs (* (:transaction-rule-account/percentage tra)
|
|
(:transaction/amount transaction)))
|
|
:transaction-account/location (:transaction-rule-account/location tra)})
|
|
(:transaction-rule/accounts top-match))
|
|
:transaction/vendor (:db/id (:transaction-rule/vendor top-match)))
|
|
transaction)))))
|
|
|
|
(defn manual-import [manual-transactions]
|
|
(let [transformed-transactions (->> manual-transactions
|
|
(filter #(= "posted" (:status %)))
|
|
(group-by #(select-keys % [:date :description-original :amount]))
|
|
(vals)
|
|
(mapcat (fn [transaction-group]
|
|
(map
|
|
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
|
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
|
:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:date (time/unparse date "YYYY-MM-dd")
|
|
:amount {:amount amount}
|
|
:description {:original description-original
|
|
:simple high-level-category}
|
|
:status "POSTED"})
|
|
(range)
|
|
transaction-group))))]
|
|
(println "importing manual transactions" transformed-transactions)
|
|
(batch-transact
|
|
(transactions->txs transformed-transactions :client-id :bank-account-id (rule-applying-fn [])))))
|
|
|
|
|
|
|
|
(defn do-import
|
|
([]
|
|
(do-import (client/get-transactions)))
|
|
([transactions]
|
|
(let [all-bank-accounts (mapcat (fn [c] (map
|
|
(fn [{:keys [:db/id :bank-account/yodlee-account-id]}]
|
|
(when (and id yodlee-account-id)
|
|
{:bank-account-id id
|
|
:client-id (:db/id c)
|
|
:yodlee-account-id yodlee-account-id}))
|
|
(:client/bank-accounts c)))
|
|
(d-clients/get-all))
|
|
transaction->client (comp (by :yodlee-account-id :client-id all-bank-accounts) :accountId)
|
|
transaction->bank-account-id (comp (by :yodlee-account-id :bank-account-id all-bank-accounts) :accountId)
|
|
all-rules (tr/get-all)]
|
|
(batch-transact (transactions->txs transactions transaction->client transaction->bank-account-id (rule-applying-fn all-rules))))))
|
|
|