added suggestions
This commit is contained in:
@@ -1,426 +0,0 @@
|
||||
#_{:clj-kondo/ignore [:unused-namespace]}
|
||||
(ns auto-ap.yodlee.core
|
||||
(:require
|
||||
[auto-ap.utils :refer [by]]
|
||||
[clj-http.client :as client]
|
||||
[clojure.core.async :as async]
|
||||
[clojure.data.json :as json]
|
||||
[clojure.tools.logging :as log]
|
||||
[config.core :refer [env]]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(def known-bad-yodlee-ids ;; these yodlee ids are ones we wish we could manually delete
|
||||
(set [
|
||||
3432269925
|
||||
3432269925
|
||||
3440074569
|
||||
3440074568
|
||||
3451368964
|
||||
3453413513
|
||||
3453413398
|
||||
3325396245
|
||||
3451368962
|
||||
2987488002
|
||||
3021618678
|
||||
3021618674
|
||||
3025625892
|
||||
3052944039
|
||||
3052944035
|
||||
3079067192
|
||||
3079067193
|
||||
3079067193
|
||||
3122287847
|
||||
3122287842
|
||||
3189405320
|
||||
3189405320
|
||||
3189405321
|
||||
3189405321
|
||||
3189405319
|
||||
3189405319
|
||||
3248171256
|
||||
3248171256
|
||||
3248171257
|
||||
3248171257
|
||||
3248171254
|
||||
3291944265
|
||||
3291944267
|
||||
3291944267
|
||||
3310943251
|
||||
3387770449
|
||||
3387770447
|
||||
3432269929]))
|
||||
|
||||
(defn auth-header
|
||||
([cob-session] (str "{cobSession=" cob-session "}"))
|
||||
([cob-session user-session] (str "{cobSession=" cob-session ",userSession=" user-session "}")))
|
||||
|
||||
(def other-config
|
||||
(if (:yodlee-proxy-host env)
|
||||
{:proxy-host (:yodlee-proxy-host env)
|
||||
:proxy-port (:yodlee-proxy-port env)
|
||||
:retry-handler (fn [ex _ _]
|
||||
(log/error "yodlee Error." ex)
|
||||
false)
|
||||
:socket-timeout 60000
|
||||
:connection-timeout 60000}
|
||||
{:retry-handler (fn [ex _ _]
|
||||
(log/error "yodlee Error." ex)
|
||||
false)
|
||||
:socket-timeout 60000
|
||||
:connection-timeout 60000}))
|
||||
|
||||
(defn retry-thrice
|
||||
([x] (retry-thrice x 0))
|
||||
([x i]
|
||||
(try
|
||||
(x)
|
||||
(catch Exception e
|
||||
(if (>= i 3)
|
||||
(throw e)
|
||||
(do
|
||||
(Thread/sleep 5000)
|
||||
(retry-thrice x (inc i))))))))
|
||||
|
||||
|
||||
(def base-headers {"Api-Version" "1.1"
|
||||
"Cobrand-Name" (:yodlee-cobrand-name env)
|
||||
"Content-Type" "application/json"})
|
||||
|
||||
|
||||
(defn login-cobrand []
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/cobrand/login")
|
||||
(client/post (merge {:headers base-headers
|
||||
:body
|
||||
(json/write-str {:cobrand {:cobrandLogin (:yodlee-cobrand-login env)
|
||||
:cobrandPassword (:yodlee-cobrand-password env)
|
||||
:locale "en_US"}})
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:session
|
||||
:cobSession))))
|
||||
|
||||
|
||||
(defn login-user
|
||||
([cob-session] (login-user cob-session (:yodlee-user-login env) (:yodlee-user-password env)))
|
||||
([cob-session user password]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/user/login")
|
||||
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
||||
:body
|
||||
(json/write-str {:user {:loginName user
|
||||
:password password
|
||||
:locale "en_US"}})
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:user
|
||||
:session
|
||||
:userSession)))))
|
||||
|
||||
|
||||
(defn get-auth-header []
|
||||
(let [cob-session (login-cobrand)
|
||||
user-session (login-user cob-session)]
|
||||
{"Authorization" (auth-header cob-session user-session)}))
|
||||
|
||||
|
||||
(defn get-accounts [auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/accounts")
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:account))))
|
||||
|
||||
|
||||
(defn get-accounts-for-provider-account [provider-account-id auth-header]
|
||||
(try
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/accounts?providerAccountId=" provider-account-id)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:account)))
|
||||
(catch Exception e
|
||||
(log/error (str "Couldn't get accounts for provider account '" provider-account-id "'")
|
||||
e)
|
||||
[])))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn get-account [i auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) (str "/accounts/" i))
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:account))))
|
||||
|
||||
|
||||
(defn get-provider-accounts [auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts")
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:providerAccount))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn get-provider-account [id auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:providerAccount))))
|
||||
|
||||
|
||||
(defn get-provider-account-detail [id auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:query-params {"include" "credentials,preferences"}
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:providerAccount
|
||||
first))))
|
||||
|
||||
|
||||
(defn update-provider-account [pa auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
||||
(client/put (merge {:headers (merge base-headers auth-header)
|
||||
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
|
||||
:as :json}
|
||||
other-config))))))
|
||||
|
||||
|
||||
(defn delete-provider-account [id auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
|
||||
(client/delete (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:providerAccount
|
||||
first))))
|
||||
|
||||
|
||||
(defn get-transaction-page
|
||||
([account batch-size skip auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:transaction))))
|
||||
([account start end batch-size skip auth-header]
|
||||
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&fromDate=" start "&toDate=" end "&skip=" skip "&accountId=" account)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:transaction)))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn count-specific-transactions [account auth-header]
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
|
||||
(client/get (merge {:headers (merge base-headers auth-header)
|
||||
:as :json}
|
||||
other-config))
|
||||
:body
|
||||
:transaction))))
|
||||
|
||||
|
||||
(defn get-access-token []
|
||||
(try
|
||||
(let [cob-session (login-cobrand)
|
||||
user-session (login-user cob-session)
|
||||
token (->
|
||||
(str (:yodlee-base-url env) "/user/accessTokens?appIds=" 10003600)
|
||||
|
||||
(client/get
|
||||
(merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json}
|
||||
other-config))
|
||||
(doto log/info)
|
||||
:body
|
||||
:user
|
||||
:accessTokens
|
||||
first
|
||||
:value)]
|
||||
[user-session token])
|
||||
(catch Exception e
|
||||
(log/error e)
|
||||
(throw e))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn create-user []
|
||||
(let [cob-session (login-cobrand)]
|
||||
(-> (str (:yodlee-base-url env) "/user/register")
|
||||
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
||||
:body (json/write-str {:user {:loginName "brycesPersoonal2"
|
||||
:password "kV@mdv3TU11"
|
||||
:email "yodleepersonal2@brycecovertoperations.com"}})
|
||||
:as :json}
|
||||
other-config))
|
||||
:body)))
|
||||
|
||||
|
||||
(defn reauthenticate [pa data auth-header]
|
||||
(try
|
||||
(retry-thrice
|
||||
(fn []
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
||||
(client/put (merge {:headers (merge base-headers auth-header)
|
||||
:body (json/write-str data)
|
||||
:as :json}
|
||||
other-config)))))
|
||||
(catch Exception e
|
||||
(log/error e))))
|
||||
|
||||
|
||||
;; helpers
|
||||
|
||||
|
||||
(defn get-specific-transactions [account auth-header]
|
||||
(log/infof "Getting yodlee transactions for account %s" account)
|
||||
(let [batch-size 100]
|
||||
(loop [transactions []
|
||||
skip 0]
|
||||
(let [transaction-batch (get-transaction-page account batch-size skip auth-header)]
|
||||
(if (seq transaction-batch)
|
||||
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
||||
transactions)))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn get-specific-transactions-with-date [account start end auth-header]
|
||||
(let [batch-size 100]
|
||||
(loop [transactions []
|
||||
skip 0]
|
||||
(let [transaction-batch (get-transaction-page account start end batch-size skip auth-header)]
|
||||
(if (seq transaction-batch)
|
||||
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
||||
transactions)))))
|
||||
|
||||
|
||||
(defn get-provider-accounts-with-details [auth-header]
|
||||
(let [provider-accounts (get-provider-accounts auth-header)
|
||||
concurrent 10
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(map (fn [provider-account]
|
||||
(lc/with-context {:provider-account-id (:id provider-account)}
|
||||
(get-provider-account-detail (:id provider-account) auth-header))))
|
||||
(async/to-chan! provider-accounts)
|
||||
true
|
||||
(fn [e]
|
||||
(lc/with-context {:source "Yodlee"}
|
||||
(log/error "Yodlee pipeline error" e))))
|
||||
(async/<!! (async/into [] output-chan))))
|
||||
|
||||
|
||||
(defn concurrent-get-accounts-for-providers [provider-account-ids auth-header]
|
||||
(let [concurrent 20
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(map (fn [provider-account-id]
|
||||
(lc/with-context {:provider-account-id provider-account-id}
|
||||
[provider-account-id
|
||||
(get-accounts-for-provider-account provider-account-id auth-header)])))
|
||||
(async/to-chan! provider-account-ids)
|
||||
true
|
||||
(fn [e]
|
||||
(lc/with-context {:source "Yodlee"}
|
||||
(log/error "Yodlee pipeline error" e))))
|
||||
(async/<!! (async/into {} output-chan))))
|
||||
|
||||
|
||||
(defn get-provider-accounts-with-accounts [auth-header]
|
||||
(let [provider-accounts (by :id (get-provider-accounts-with-details auth-header))
|
||||
accounts (concurrent-get-accounts-for-providers (keys provider-accounts) auth-header)]
|
||||
(->> accounts
|
||||
(reduce
|
||||
(fn [provider-accounts [which accounts]]
|
||||
(assoc-in provider-accounts [which :accounts] accounts))
|
||||
provider-accounts)
|
||||
vals)))
|
||||
|
||||
|
||||
(mount/defstate in-memory-cache
|
||||
:start (atom []))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn refresh-in-memory-cache []
|
||||
(lc/with-context {:source "refreshing-in-memory-cache"}
|
||||
(try
|
||||
(log/info "Refreshing Yodlee in memory cache")
|
||||
(reset! in-memory-cache (get-provider-accounts-with-accounts (get-auth-header)))
|
||||
(catch Exception e
|
||||
(log/error e)))))
|
||||
|
||||
|
||||
#_(mount/defstate in-memory-cache-worker
|
||||
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)
|
||||
:stop (scheduler/stop in-memory-cache-worker))
|
||||
|
||||
|
||||
(defn refresh-provider-account [id auth-header]
|
||||
(swap! in-memory-cache
|
||||
(fn [i]
|
||||
(-> (by :id i)
|
||||
(assoc id (assoc (get-provider-account-detail id auth-header)
|
||||
:accounts (get-accounts-for-provider-account id auth-header)))
|
||||
vals))))
|
||||
|
||||
|
||||
(defn delete-and-uncache-provider-account [id auth-header]
|
||||
(delete-provider-account id auth-header)
|
||||
(swap! in-memory-cache
|
||||
(fn [i]
|
||||
(-> (by :id i)
|
||||
(dissoc id)
|
||||
vals))))
|
||||
|
||||
|
||||
(defn update-yodlee [id auth-header]
|
||||
(update-provider-account id auth-header)
|
||||
(refresh-provider-account id auth-header))
|
||||
|
||||
|
||||
(defn reauthenticate-and-recache [pa data auth-header]
|
||||
(reauthenticate pa data auth-header)
|
||||
(refresh-provider-account pa auth-header))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user