362 lines
16 KiB
Clojure
362 lines
16 KiB
Clojure
(ns auto-ap.yodlee.core2
|
|
(:require [clj-http.client :as client]
|
|
[auto-ap.utils :refer [by]]
|
|
[unilog.context :as lc]
|
|
[clojure.tools.logging :as log]
|
|
[clojure.data.json :as json]
|
|
[clojure.core.async :as async]
|
|
[config.core :refer [env]]
|
|
#_{:clj-kondo/ignore [:unused-namespace]}
|
|
[mount.core :as mount]
|
|
#_{:clj-kondo/ignore [:unused-namespace]}
|
|
[yang.scheduler :as scheduler]
|
|
[clj-time.coerce :as coerce]
|
|
[datomic.api :as dc]
|
|
[auto-ap.datomic :refer [conn]]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[clojure.string :as str]))
|
|
;; switch all of this to use tokens instead of passing around client codes, particularly because the codes
|
|
;; need to be tweaked for repeats
|
|
(defn client-code->login [client-code]
|
|
(if (< (count client-code) 3)
|
|
(str client-code "_" client-code)
|
|
client-code))
|
|
|
|
(defn auth-header
|
|
([cob-session] (str "Bearer " cob-session)))
|
|
|
|
(defn retry-thrice
|
|
([x] (retry-thrice x 0))
|
|
([x i]
|
|
(try
|
|
(x)
|
|
(catch Exception e
|
|
(if (>= i 3)
|
|
(throw e)
|
|
(do
|
|
(Thread/sleep 5000)
|
|
(retry-thrice x (inc i))))))))
|
|
|
|
(def other-config
|
|
(if (:yodlee2-proxy-host env)
|
|
{:proxy-host (:yodlee2-proxy-host env)
|
|
:proxy-port (:yodlee2-proxy-port env)
|
|
:socket-timeout 60000
|
|
:connection-timeout 60000
|
|
:retry-handler (fn [ex _ _]
|
|
(log/error "yodlee Error." ex)
|
|
false)}
|
|
{:retry-handler (fn [ex _ _]
|
|
(log/error "yodlee Error." ex)
|
|
false)
|
|
:socket-timeout 60000
|
|
:connection-timeout 60000}))
|
|
|
|
(def base-headers {"Api-Version" "1.1"
|
|
"Content-Type" "application/json"})
|
|
|
|
(defn login-cobrand []
|
|
(retry-thrice
|
|
(fn []
|
|
(-> (str (:yodlee2-base-url env) "/auth/token")
|
|
(client/post (merge {:headers (assoc base-headers
|
|
"loginName" (:yodlee2-admin-user env)
|
|
"Content-Type" "application/x-www-form-urlencoded")
|
|
:body (str "clientId=" (:yodlee2-client-id env) "&secret=" (:yodlee2-client-secret env))
|
|
:as :json}
|
|
other-config)
|
|
)
|
|
:body
|
|
:token
|
|
:accessToken))))
|
|
|
|
(defn login-user [client-code]
|
|
(retry-thrice
|
|
(fn []
|
|
(log/info "logging in as " client-code)
|
|
(-> (str (:yodlee2-base-url env) "/auth/token")
|
|
(client/post (merge {:headers (assoc base-headers
|
|
"loginName" (if (:yodlee2-test-user env)
|
|
(:yodlee2-test-user env)
|
|
(if (<= (count client-code) 3)
|
|
(str client-code client-code)
|
|
client-code))
|
|
"Content-Type" "application/x-www-form-urlencoded")
|
|
:body (str "clientId=" (:yodlee2-client-id env) "&secret=" (:yodlee2-client-secret env))
|
|
:as :json}
|
|
other-config)
|
|
)
|
|
:body
|
|
:token
|
|
:accessToken))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn get-accounts [client-code ]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
(-> (str (:yodlee2-base-url env) "/accounts")
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (str "Bearer " cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:account)))
|
|
|
|
(defn get-accounts-for-provider-account [client-code provider-account-id]
|
|
(try
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
(-> (str (:yodlee2-base-url env) "/accounts?providerAccountId=" provider-account-id)
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:account))
|
|
(catch Exception e
|
|
(log/error (str "Couldn't get accounts for provider account '" provider-account-id "'")
|
|
e)
|
|
[])))
|
|
|
|
(defn get-provider-accounts [client-code ]
|
|
(retry-thrice
|
|
(fn []
|
|
(log/info "logging in user " client-code)
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts")
|
|
(-> (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session )})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount))))))
|
|
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn get-transactions [client-code]
|
|
(let [cob-session (login-user (client-code->login client-code))
|
|
batch-size 100
|
|
get-transaction-batch (fn [skip]
|
|
(-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip)
|
|
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:transaction
|
|
))]
|
|
|
|
(loop [transactions []
|
|
skip 0]
|
|
(let [transaction-batch (get-transaction-batch skip)]
|
|
(if (seq transaction-batch)
|
|
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
|
transactions)))))
|
|
|
|
|
|
|
|
(defn get-provider-account [client-code id]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id)
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first)))
|
|
|
|
(defn get-provider-account-detail [client-code id]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id )
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:query-params {"include" "credentials,preferences"}
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first)))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn update-provider-account [client-code pa]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
|
|
|
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
|
|
:as :json}
|
|
other-config)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
(defn get-specific-transactions [client-code account]
|
|
(let [cob-session (login-user (client-code->login client-code))
|
|
batch-size 100
|
|
get-transaction-batch (fn [skip]
|
|
(-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
|
|
|
|
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:transaction
|
|
))]
|
|
|
|
(loop [transactions []
|
|
skip 0]
|
|
(let [transaction-batch (get-transaction-batch skip)]
|
|
(if (seq transaction-batch)
|
|
(recur (concat transactions transaction-batch) (+ batch-size skip))
|
|
transactions)))))
|
|
|
|
|
|
(defn get-access-token [client-code]
|
|
(try
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
cob-session)
|
|
(catch Exception e
|
|
(log/error e)
|
|
(throw e))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn create-user [client-code]
|
|
(let [cob-session (login-cobrand)]
|
|
(-> (str (:yodlee2-base-url env) "/user/register")
|
|
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:body (json/write-str {"user" {
|
|
"loginName" client-code
|
|
"email" "bryce@integreatconsult.com"
|
|
"name" {"first" client-code
|
|
"last" client-code}
|
|
"address" {"address1" "200 Lincoln Ave"
|
|
"state" "CA"
|
|
"city" "Salinas"
|
|
"zip" "93901"
|
|
"country" "USA"}
|
|
"preferences" {"currency" "USD"
|
|
"timeZone" "GMT"
|
|
"dateFormat" "YYYY-MMM-DD"
|
|
"locale" "en_US"}}})
|
|
:as :json}
|
|
other-config))
|
|
:body)))
|
|
|
|
|
|
|
|
|
|
(defn get-provider-accounts-with-details [client-code ]
|
|
(let [provider-accounts (get-provider-accounts client-code)
|
|
concurrent 20
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(map (fn [provider-account]
|
|
(lc/with-context {:provider-account-id (:id provider-account)}
|
|
(get-provider-account-detail client-code (:id provider-account)))))
|
|
(async/to-chan! provider-accounts))
|
|
(async/<!! (async/into [] output-chan))))
|
|
|
|
(defn get-accounts-for-providers [client-code provider-account-ids]
|
|
(log/info "looking up " (count provider-account-ids) " provider accounts's accounts for client " client-code ".")
|
|
(into {}
|
|
(mapv (fn [provider-account-id]
|
|
(lc/with-context {:provider-account-id provider-account-id}
|
|
[provider-account-id
|
|
(get-accounts-for-provider-account client-code provider-account-id)]))
|
|
provider-account-ids)))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn get-provider-accounts-with-accounts [client-code]
|
|
(let [provider-accounts (by :id (get-provider-accounts-with-details client-code))
|
|
accounts (get-accounts-for-providers client-code (keys provider-accounts))]
|
|
(->> accounts
|
|
(reduce
|
|
(fn [provider-accounts [which accounts]]
|
|
(assoc-in provider-accounts [which :accounts] accounts))
|
|
provider-accounts)
|
|
vals)))
|
|
|
|
(defn delete-provider-account [client-code id]
|
|
(let [cob-session (login-user (client-code->login client-code))]
|
|
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts/" id )
|
|
|
|
(client/delete (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
|
|
:as :json}
|
|
other-config))
|
|
:body
|
|
:providerAccount
|
|
first))
|
|
@(dc/transact conn [[:db/retractEntity [:yodlee-provider-account/id id]]]))
|
|
|
|
(defn upsert-accounts-tx
|
|
([client-code]
|
|
(upsert-accounts-tx client-code (get-provider-accounts client-code)))
|
|
([client-code provider-accounts]
|
|
(let [accounts (get-accounts-for-providers client-code (map :id provider-accounts))]
|
|
(map (fn [pa]
|
|
(cond->
|
|
{:yodlee-provider-account/id (:id pa)
|
|
:yodlee-provider-account/status (:status pa)
|
|
:yodlee-provider-account/detailed-status (or (-> pa :dataset first :additionalStatus) "unknown")
|
|
:yodlee-provider-account/client [:client/code client-code]
|
|
|
|
:yodlee-provider-account/accounts (mapv
|
|
(fn [a]
|
|
{:yodlee-account/id (:id a)
|
|
:yodlee-account/name (str (:providerName a) " (" (:accountName a) ")")
|
|
:yodlee-account/number (or (:accountNumber a) "Unknown")
|
|
:yodlee-account/status (or (-> a :dataset first :additionalStatus) "unknown")
|
|
:yodlee-account/available-balance (or (-> a :currentBalance :amount)
|
|
0.0)})
|
|
(get accounts (:id pa)))}
|
|
|
|
(-> pa :dataset first :lastUpdated) (assoc :yodlee-provider-account/last-updated (-> pa :dataset first :lastUpdated coerce/to-date)) ))
|
|
provider-accounts))))
|
|
|
|
(defn refresh-provider-account [client-code id]
|
|
(log/info "refreshing yodlee provider account id" id)
|
|
@(dc/transact conn (upsert-accounts-tx client-code
|
|
[(get-provider-account client-code id)])))
|
|
|
|
(defn upsert-accounts []
|
|
(let [concurrent 20
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(mapcat (fn [client]
|
|
(log/info "Upserting Yodlee Accounts for " (:client/code client))
|
|
(lc/with-context {:client-code (:client/code client)}
|
|
(try
|
|
(upsert-accounts-tx (:client/code client))
|
|
(catch Exception e
|
|
(log/error "Could not update client " (:client/code client)
|
|
e))))))
|
|
(async/to-chan! (d-clients/get-all)))
|
|
(let [result (async/<!! (async/into [] output-chan))]
|
|
(log/info "Current yodlee state is " result)
|
|
@(dc/transact conn result))))
|
|
|
|
|
|
|
|
|
|
(defn reauthenticate [client-code pa data]
|
|
(try
|
|
(-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
|
|
|
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header (login-user (client-code->login client-code)))})
|
|
:body (json/write-str data)
|
|
:as :json}
|
|
other-config)))
|
|
(catch Exception e
|
|
(log/error e))) )
|
|
|
|
(defn reauthenticate-and-recache [client-code pa data]
|
|
(reauthenticate client-code pa data)
|
|
(refresh-provider-account client-code pa))
|