338 lines
14 KiB
Clojure
338 lines
14 KiB
Clojure
(ns auto-ap.yodlee.core2
|
|
(:require [clj-http.client :as client]
|
|
[auto-ap.utils :refer [by]]
|
|
[cemerick.url :as u]
|
|
[unilog.context :as lc]
|
|
[clojure.tools.logging :as log]
|
|
[clojure.data.json :as json]
|
|
[clojure.core.async :as async]
|
|
[config.core :refer [env]]
|
|
[mount.core :as mount]
|
|
[yang.scheduler :as scheduler]
|
|
[clj-time.coerce :as coerce]
|
|
[auto-ap.time :as time2]
|
|
[datomic.api :as d]
|
|
[auto-ap.datomic :refer [conn]]
|
|
[auto-ap.datomic.clients :as d-clients]))
|
|
;; switch all of this to use tokens instead of passing around client codes, particularly because the codes
|
|
;; need to be tweaked for repeats
|
|
(defn client-code->login [client-code]
|
|
(if (< (count client-code) 3)
|
|
(str client-code "_" client-code)
|
|
client-code))
|
|
|
|
(defn auth-header
|
|
([cob-session] (str "Bearer " cob-session)))
|
|
|
|
|
|
|
|
(def other-config
|
|
(if (:yodlee2-proxy-host env)
|
|
{:proxy-host (:yodlee2-proxy-host env)
|
|
:proxy-port (:yodlee2-proxy-port env)
|
|
:retry-handler (fn [ex try-count http-context]
|
|
(log/error "yodlee Error." ex)
|
|
false)}
|
|
{:retry-handler (fn [ex try-count http-context]
|
|
(log/error "yodlee Error." ex)
|
|
false)}))
|
|
|
|
(def base-headers {"Api-Version" "1.1"
|
|
"Content-Type" "application/json"})
|
|
|
|
(defn login-cobrand []
|
|
(-> (str (:yodlee2-base-url env) "/auth/token")
|
|
(client/post (merge {:headers (assoc base-headers
|
|
"loginName" (:yodlee2-admin-user env)
|
|
"Content-Type" "application/x-www-form-urlencoded")
|
|
:body (str "clientId=" (:yodlee2-client-id env) " &secret=" (:yodlee2-client-secret env))
|
|
:as :json}
|
|
other-config)
|
|
)
|
|
:body
|
|
:token
|
|
:accessToken))
|
|
|
|
(defn login-user [client-code]
|
|
(-> (str (:yodlee2-base-url env) "/auth/token")
|
|
(client/post (merge {:headers (assoc base-headers
|
|
"loginName" client-code
|
|
"Content-Type" "application/x-www-form-urlencoded")
|
|
:body (str "clientId=" (:yodlee2-client-id env) " &secret=" (:yodlee2-client-secret env))
|
|
:as :json}
|
|
other-config)
|
|
)
|
|
:body
|
|
:token
|
|
:accessToken))
|
|
|
|
(defn get-accounts [client-code ]
|
|
(let [cob-session (login-user client-code)]
|
|
(-> (str (:yodlee2-base-url env) "/accounts")
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (str "Bearer " cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:account)))
|
|
|
|
(defn get-accounts-for-provider-account [client-code provider-account-id]
|
|
(try
|
|
(let [cob-session (login-user client-code)]
|
|
(-> (str (:yodlee2-base-url env) "/accounts?providerAccountId=" provider-account-id)
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:account))
|
|
(catch Exception e
|
|
(log/error (str "Couldn't get accounts for provider account '" provider-account-id "'")
|
|
e)
|
|
[])))
|
|
|
|
(defn get-provider-accounts [client-code ]
|
|
(let [cob-session (login-user client-code)]
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts")
|
|
(-> (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session )})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount))))
|
|
|
|
|
|
|
|
(defn get-transactions [client-code]
|
|
(let [cob-session (login-user client-code)
|
|
batch-size 100
|
|
get-transaction-batch (fn [skip]
|
|
(-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip)
|
|
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:transaction
|
|
))]
|
|
|
|
(loop [transactions []
|
|
skip 0]
|
|
(let [transaction-batch (get-transaction-batch skip)]
|
|
(if (seq transaction-batch)
|
|
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
|
transactions)))))
|
|
|
|
|
|
|
|
(defn get-provider-account [client-code id]
|
|
(let [cob-session (login-user client-code)
|
|
batch-size 100]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id)
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first)))
|
|
|
|
(defn get-provider-account-detail [client-code id]
|
|
(let [cob-session (login-user client-code)]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id )
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:query-params {"include" "credentials,preferences"}
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first)))
|
|
|
|
(defn update-provider-account [client-code pa]
|
|
(let [cob-session (login-user client-code)]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
|
|
|
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
|
|
:as :json}
|
|
other-config)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
(defn get-specific-transactions [client-code account]
|
|
(let [cob-session (login-user client-code)
|
|
batch-size 100
|
|
get-transaction-batch (fn [skip]
|
|
(-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:transaction
|
|
))]
|
|
|
|
(loop [transactions []
|
|
skip 0]
|
|
(let [transaction-batch (get-transaction-batch skip)]
|
|
(if (seq transaction-batch)
|
|
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
|
transactions)))))
|
|
|
|
|
|
(defn get-access-token [client-code]
|
|
(try
|
|
(let [cob-session (login-user client-code)]
|
|
cob-session)
|
|
(catch Exception e
|
|
(log/error e)
|
|
(throw e))))
|
|
|
|
(defn create-user [client-code]
|
|
(let [cob-session (login-cobrand)]
|
|
(-> (str (:yodlee2-base-url env) "/user/register")
|
|
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:body (json/write-str {"user" {
|
|
"loginName" client-code
|
|
"email" "bryce@integreatconsult.com"
|
|
"name" {"first" client-code
|
|
"last" client-code}
|
|
"address" {"address1" "200 Lincoln Ave"
|
|
"state" "CA"
|
|
"city" "Salinas"
|
|
"zip" "93901"
|
|
"country" "USA"}
|
|
"preferences" {"currency" "USD"
|
|
"timeZone" "GMT"
|
|
"dateFormat" "YYYY-MMM-DD"
|
|
"locale" "en_US"}}})
|
|
:as :json}
|
|
other-config))
|
|
:body)))
|
|
|
|
|
|
(defn assert-user [client-code]
|
|
(let [cob-session (login-cobrand client-code)]
|
|
cob-session))
|
|
|
|
|
|
|
|
(defn get-provider-accounts-with-details [client-code ]
|
|
(let [provider-accounts (get-provider-accounts client-code)]
|
|
(let [concurrent 20
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(map (fn [provider-account]
|
|
(lc/with-context {:provider-account-id (:id provider-account)}
|
|
(get-provider-account-detail client-code (:id provider-account)))))
|
|
(async/to-chan provider-accounts))
|
|
(async/<!! (async/into [] output-chan)))))
|
|
|
|
(defn get-accounts-for-providers [client-code provider-account-ids]
|
|
(log/info "looking up " (count provider-account-ids) " provider accounts's accounts for client " client-code ".")
|
|
(into {}
|
|
(mapv (fn [provider-account-id]
|
|
(lc/with-context {:provider-account-id provider-account-id}
|
|
[provider-account-id
|
|
(get-accounts-for-provider-account client-code provider-account-id)]))
|
|
provider-account-ids)))
|
|
|
|
|
|
|
|
(defn get-provider-accounts-with-accounts [client-code]
|
|
(let [provider-accounts (by :id (get-provider-accounts-with-details client-code))
|
|
accounts (get-accounts-for-providers client-code (keys provider-accounts))]
|
|
(->> accounts
|
|
(reduce
|
|
(fn [provider-accounts [which accounts]]
|
|
(assoc-in provider-accounts [which :accounts] accounts))
|
|
provider-accounts)
|
|
vals)))
|
|
|
|
|
|
|
|
|
|
|
|
(defn delete-provider-account [client-code id]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id )
|
|
|
|
(client/delete (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first))
|
|
@(d/transact conn [[:db/retractEntity (:db/id (d/entity (d/db conn) [:yodlee-provider-account/id id]))]]))
|
|
|
|
(defn upsert-accounts-tx
|
|
([client-code]
|
|
(upsert-accounts-tx (get-provider-accounts (client-code->login client-code))))
|
|
([client-code provider-accounts]
|
|
(let [accounts (get-accounts-for-providers (client-code->login client-code) (map :id provider-accounts))]
|
|
(map (fn [pa]
|
|
{:yodlee-provider-account/id (:id pa)
|
|
:yodlee-provider-account/status (:status pa)
|
|
:yodlee-provider-account/detailed-status (-> pa :dataset first :additionalStatus)
|
|
:yodlee-provider-account/client [:client/code client-code]
|
|
:yodlee-provider-account/last-updated (-> pa :dataset first :lastUpdated coerce/to-date)
|
|
:yodlee-provider-account/accounts (mapv
|
|
(fn [a]
|
|
{:yodlee-account/id (:id a)
|
|
:yodlee-account/name (str (:providerName a) " (" (:accountName a) ")")
|
|
:yodlee-account/number (:accountNumber a)
|
|
:yodlee-account/status (-> a :dataset first :additionalStatus)
|
|
:yodlee-account/available-balance (or (-> a :currentBalance :amount)
|
|
0.0)})
|
|
(get accounts (:id pa)))})
|
|
provider-accounts))))
|
|
|
|
(defn refresh-provider-account [client-code id]
|
|
(log/info "refreshing yodlee provider account id" id)
|
|
@(d/transact conn (upsert-accounts-tx client-code
|
|
[(get-provider-account (client-code->login client-code) id)])))
|
|
|
|
(defn upsert-accounts []
|
|
(let [concurrent 20
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(mapcat (fn [client]
|
|
(log/info "Upserting Yodlee Accounts for " (:client/code client))
|
|
(lc/with-context {:client-code (:client/code client)}
|
|
(upsert-accounts-tx (:client/code client)))))
|
|
(async/to-chan (d-clients/get-all)))
|
|
(let [result (async/<!! (async/into [] output-chan))]
|
|
(log/info "Current yodlee state is " result)
|
|
@(d/transact conn result))))
|
|
|
|
(mount/defstate yodlee-sync-worker
|
|
:start (scheduler/every (* 5 60 1000) upsert-accounts)
|
|
:stop (scheduler/stop upsert-accounts))
|
|
|
|
(defn update-yodlee [id]
|
|
(update-provider-account id)
|
|
(refresh-provider-account id))
|
|
|
|
(defn reauthenticate [pa data]
|
|
(let [cob-session (login-cobrand)]
|
|
(try
|
|
|
|
(doto (-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
|
|
|
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:body (json/write-str data)
|
|
:as :json}
|
|
other-config)))
|
|
log/info)
|
|
(refresh-provider-account pa)
|
|
(catch Exception e
|
|
(log/error e)))))
|