183 lines
8.1 KiB
Clojure
183 lines
8.1 KiB
Clojure
(ns auto-ap.yodlee.import
|
|
(:require [auto-ap.yodlee.core :as client]
|
|
[auto-ap.utils :refer [by]]
|
|
[datomic.api :as d]
|
|
[auto-ap.datomic :refer [uri remove-nils]]
|
|
[auto-ap.datomic.accounts :as a]
|
|
[clj-time.coerce :as coerce]
|
|
[digest :refer [sha-256]]
|
|
[auto-ap.datomic.checks :as d-checks]
|
|
[auto-ap.datomic.transactions :as d-transactions]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[auto-ap.time :as time]
|
|
[auto-ap.datomic.transaction-rules :as tr]
|
|
[auto-ap.rule-matching :as rm]
|
|
[clojure.string :as str]))
|
|
|
|
|
|
|
|
(defn transaction->payment [_ check-number client-id bank-account-id amount id]
|
|
(cond (not (and client-id bank-account-id))
|
|
nil
|
|
|
|
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
|
nil
|
|
|
|
check-number
|
|
(-> (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:check-number check-number
|
|
:amount (- amount)
|
|
:status :payment-status/pending})
|
|
first
|
|
first)
|
|
|
|
(and client-id bank-account-id amount)
|
|
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
|
:bank-account-id bank-account-id
|
|
:amount (- amount)
|
|
:status :payment-status/pending})]
|
|
(if (= 1 (count matching-checks))
|
|
(first matching-checks)
|
|
nil))
|
|
|
|
:else
|
|
nil))
|
|
|
|
(defn extract-check-number [{{description-original :original} :description}]
|
|
|
|
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
|
(try
|
|
(Integer/parseInt check-number)
|
|
(catch NumberFormatException e
|
|
nil))
|
|
nil))
|
|
|
|
(defn transactions->txs [transactions transaction->bank-account apply-rules existing]
|
|
(into []
|
|
|
|
(for [transaction transactions
|
|
:let [{post-date :postDate
|
|
account-id :accountId
|
|
date :date
|
|
id :id
|
|
{amount :amount} :amount
|
|
{description-original :original
|
|
description-simple :simple} :description
|
|
{merchant-id :id
|
|
merchant-name :name} :merchant
|
|
base-type :baseType
|
|
type :type
|
|
status :status}
|
|
transaction
|
|
amount (if (= "DEBIT" base-type)
|
|
(- amount)
|
|
amount)
|
|
check-number (extract-check-number transaction)
|
|
bank-account (transaction->bank-account transaction)
|
|
bank-account-id (:db/id bank-account)
|
|
client (:client/_bank-accounts bank-account)
|
|
client-id (:db/id client)
|
|
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
|
check (transaction->payment transaction check-number client-id bank-account-id amount id)]
|
|
:when (and client-id
|
|
(not (existing (sha-256 (str id)))))]
|
|
(->
|
|
#:transaction
|
|
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
|
|
:id (sha-256 (str id))
|
|
:account-id account-id
|
|
:date (coerce/to-date (time/parse date "YYYY-MM-dd"))
|
|
:yodlee-merchant (when (and merchant-id merchant-name (not (str/blank? merchant-id)))
|
|
{:yodlee-merchant/yodlee-id merchant-id
|
|
:yodlee-merchant/name merchant-name})
|
|
:amount (double amount)
|
|
:description-original description-original
|
|
:description-simple description-simple
|
|
:approval-status :transaction-approval-status/unapproved
|
|
:type type
|
|
:status status
|
|
:client client-id
|
|
:check-number check-number
|
|
:bank-account bank-account-id
|
|
:payment (when check
|
|
{:db/id (:db/id check)
|
|
:payment/status :payment-status/cleared})
|
|
|
|
:vendor (when check
|
|
(:db/id (:payment/vendor check)))
|
|
:location (when check
|
|
"A")
|
|
:accounts (when check
|
|
[#:transaction-account {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
|
:location "A"
|
|
:amount (Math/abs (double amount))}])}
|
|
|
|
(apply-rules valid-locations)
|
|
remove-nils))))
|
|
|
|
|
|
(defn batch-transact [transactions]
|
|
(reduce (fn [acc batch]
|
|
(into acc (->> batch
|
|
(d/transact (d/connect uri) )
|
|
(deref)
|
|
:tempids
|
|
vals)))
|
|
[]
|
|
(partition-all 100 transactions)))
|
|
|
|
|
|
|
|
(defn get-existing []
|
|
(transduce (map first) conj #{}
|
|
(d/query {:query {:find ['?tid]
|
|
:in ['$]
|
|
:where ['[_ :transaction/id ?tid]]}
|
|
:args [(d/db (d/connect uri))]})))
|
|
|
|
(defn get-all-bank-accounts []
|
|
(->> (d-clients/get-all)
|
|
(mapcat (fn [client]
|
|
(->> client
|
|
:client/bank-accounts
|
|
(filter :bank-account/yodlee-account-id)
|
|
(filter :db/id)
|
|
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
|
|
(assoc bank-account
|
|
:client/_bank-accounts client))))))))
|
|
|
|
(defn manual-import [manual-transactions]
|
|
(let [transformed-transactions (->> manual-transactions
|
|
(filter #(= "posted" (:status %)))
|
|
(group-by #(select-keys % [:date :description-original :amount]))
|
|
(vals)
|
|
(mapcat (fn [transaction-group]
|
|
(map
|
|
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
|
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
|
:bank-account-id bank-account-id
|
|
:date (time/unparse date "YYYY-MM-dd")
|
|
:amount {:amount amount}
|
|
:description {:original description-original
|
|
:simple high-level-category}
|
|
:status "POSTED"})
|
|
(range)
|
|
transaction-group))))
|
|
all-rules (tr/get-all)
|
|
all-bank-accounts (get-all-bank-accounts)
|
|
transaction->bank-account (comp (by :db/id all-bank-accounts) :bank-account-id)]
|
|
(println "importing manual transactions" transformed-transactions)
|
|
(batch-transact
|
|
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))))
|
|
|
|
(defn do-import
|
|
([]
|
|
(do-import (client/get-transactions)))
|
|
([transactions]
|
|
(let [all-bank-accounts (get-all-bank-accounts)
|
|
transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
|
|
all-rules (tr/get-all)]
|
|
(batch-transact (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))))))
|
|
|