notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Hopefully makes yodlee less busy.

Browse Source
This commit is contained in:
Bryce Covert
2022-01-09 08:34:20 -08:00
parent 51e9e6a83b
commit 6c83b40d95
5 changed files with 302 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/clj/auto_ap/import/transactions.clj
View File

@@ -218,7 +218,7 @@
(defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}] (defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}]
(when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))] (when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))]
(add-new-payment autopay-invoices-matches bank-account client))) (add-new-payment transaction autopay-invoices-matches bank-account client)))
(defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}] (defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}]
(when (>= amount 0.0) (when (>= amount 0.0)

2
src/clj/auto_ap/import/yodlee.clj
View File

@@ -59,7 +59,7 @@
(d/db conn))] (d/db conn))]
(doseq [[yodlee-account bank-account client-id] account-lookup (doseq [[yodlee-account bank-account client-id] account-lookup
transaction (try transaction (try
(client/get-specific-transactions yodlee-account) (client/get-specific-transactions yodlee-account (client/get-auth-header))
(catch Exception e (catch Exception e
(log/warn e) (log/warn e)
[]))] []))]

5
src/clj/auto_ap/routes/invoices.clj
View File

@@ -174,14 +174,13 @@
(lc/with-context {:area "upload-invoice"} (lc/with-context {:area "upload-invoice"}
(log/info "Number of invoices to import is" (count imports) "sample: " (first imports)) (log/info "Number of invoices to import is" (count imports) "sample: " (first imports))
(let [clients (d-clients/get-all) (let [clients (d-clients/get-all)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}] transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text]}]
(let [ (let [
matching-client (or (and account-number matching-client (or (and account-number
(parse/best-match clients account-number 0.0)) (parse/best-match clients account-number 0.0))
(and customer-identifier (and customer-identifier
(parse/best-match clients customer-identifier)) (parse/best-match clients customer-identifier))
(if client (when client
(first (filter (fn [c] (first (filter (fn [c]
(= (:db/id c) (Long/parseLong client))) (= (:db/id c) (Long/parseLong client)))
clients)))) clients))))

164
src/clj/auto_ap/routes/yodlee.clj
View File

@@ -1,99 +1,83 @@
(ns auto-ap.routes.yodlee (ns auto-ap.routes.yodlee
(:require (:require
[auto-ap.graphql :as graphql] [auto-ap.graphql.utils :refer [assert-admin]]
[clj-http.client :as http]
[auto-ap.yodlee.core :as yodlee]
[auto-ap.graphql.utils :refer [->graphql assert-admin]]
[auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.routes.utils :refer [wrap-secure]]
[clj-time.coerce :refer [to-date]] [auto-ap.yodlee.core :as yodlee]
[ring.middleware.json :refer [wrap-json-response]] [clojure.tools.logging :as log]
[compojure.core :refer [GET POST context defroutes wrap-routes]] [compojure.core :refer [context defroutes GET POST wrap-routes]]
[clojure.string :as str] [config.core :refer [env]]))
[config.core :refer [env]]
[clojure.tools.logging :as log]))
(defroutes routes (defroutes routes
(wrap-routes (wrap-routes
(context "/yodlee" [] (context "/yodlee" []
(GET "/fastlink" {:keys [query-params identity] :as request} (GET "/fastlink" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(let [[session token] (yodlee/get-access-token)] (let [[session token] (yodlee/get-access-token)]
{:status 200
:headers {"Content-Type" "application/edn"}
{:status 200 :body (pr-str {:session session
:headers {"Content-Type" "application/edn"} :token token
:body (pr-str {:session session :app (:yodlee-app env)
:token token
:app (:yodlee-app env)
:url (:yodlee-fastlink env)
}) })) :url (:yodlee-fastlink env)})}))
(GET "/accounts" {:keys [query-params identity] :as request}
(assert-admin identity)
(let [[session token] (yodlee/get-access-token)]
{:status 200
:headers {"Content-Type" "application/edn"}
:body (pr-str (yodlee/get-accounts)) }))
(GET "/provider-accounts" {:keys [query-params identity] :as request} (GET "/accounts" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(log/info "working on provider accounts...") {:status 200
{:status 200 :headers {"Content-Type" "application/edn"}
:headers {"Content-Type" "application/edn"} :body (pr-str (yodlee/get-accounts (yodlee/get-auth-header)))})
:body (pr-str @yodlee/in-memory-cache) })
(POST "/reauthenticate/:id" {:keys [query-params identity] {:keys [id]} :route-params (GET "/provider-accounts" {:keys [identity]}
data :edn-params (assert-admin identity)
:as request} (log/info "working on provider accounts...")
(assert-admin identity) {:status 200
(try :headers {"Content-Type" "application/edn"}
(let [[session token] (yodlee/get-access-token)] :body (pr-str @yodlee/in-memory-cache)})
{:status 200 (POST "/reauthenticate/:id" {:keys [identity] {:keys [id]} :route-params
:headers {"Content-Type" "application/edn"} data :edn-params}
:body (pr-str (yodlee/reauthenticate (Long/parseLong id) data)) }) (assert-admin identity)
(catch Exception e (try
(log/error e) {:status 200
{:status 500 :headers {"Content-Type" "application/edn"}
:headers {"Content-Type" "application/edn"} :body (pr-str (yodlee/reauthenticate-and-recache (Long/parseLong id) data (yodlee/get-auth-header)))}
:body (pr-str {:message (.getMessage e) (catch Exception e
:error (.toString e)})}))) (log/error e)
(POST "/provider-accounts/refresh/:id" {:keys [query-params identity] {:keys [id]} :route-params :as request} {:status 500
(assert-admin identity) :headers {"Content-Type" "application/edn"}
(try :body (pr-str {:message (.getMessage e)
(let [[session token] (yodlee/get-access-token)] :error (.toString e)})})))
(yodlee/refresh-provider-account (Long/parseLong id)) (POST "/provider-accounts/refresh/:id" {:keys [identity] {:keys [id]} :route-params}
{:status 200 (assert-admin identity)
:headers {"Content-Type" "application/edn"} (try
:body (pr-str @yodlee/in-memory-cache) }) (yodlee/refresh-provider-account (Long/parseLong id) (yodlee/get-auth-header))
(catch Exception e {:status 200
{:status 400 :headers {"Content-Type" "application/edn"}
:headers {"Content-Type" "application/edn"} :body (pr-str @yodlee/in-memory-cache)}
:body (pr-str {:message (.getMessage e) (catch Exception e
:error (.toString e)})}))) {:status 400
(POST "/provider-accounts/delete/:id" {:keys [query-params identity] {:keys [id]} :route-params :as request} :headers {"Content-Type" "application/edn"}
(assert-admin identity) :body (pr-str {:message (.getMessage e)
(try :error (.toString e)})})))
(let [[session token] (yodlee/get-access-token)] (POST "/provider-accounts/delete/:id" {:keys [identity] {:keys [id]} :route-params}
(yodlee/delete-provider-account (Long/parseLong id)) (assert-admin identity)
{:status 200 (try
:headers {"Content-Type" "application/edn"} (yodlee/delete-and-uncache-provider-account (Long/parseLong id) (yodlee/get-auth-header))
:body (pr-str @yodlee/in-memory-cache) }) {:status 200
(catch Exception e :headers {"Content-Type" "application/edn"}
{:status 400 :body (pr-str @yodlee/in-memory-cache)}
:headers {"Content-Type" "application/edn"} (catch Exception e
:body (pr-str {:message (.getMessage e) {:status 400
:error (.toString e)})}))) :headers {"Content-Type" "application/edn"}
(POST "/provider-accounts/:id" {:keys [query-params identity] {:keys [id]} :route-params :as request} :body (pr-str {:message (.getMessage e)
(assert-admin identity) :error (.toString e)})})))
(try (POST "/provider-accounts/:id" {:keys [identity] {:keys [id]} :route-params}
(let [[session token] (yodlee/get-access-token)] (assert-admin identity)
{:status 200 (try
:headers {"Content-Type" "application/edn"} {:status 200
:body (pr-str (yodlee/update-yodlee (Long/parseLong id))) }) :headers {"Content-Type" "application/edn"}
(catch Exception e :body (pr-str (yodlee/update-yodlee (Long/parseLong id) (yodlee/get-auth-header)))}
{:status 400 (catch Exception e
:headers {"Content-Type" "application/edn"} {:status 400
:body (pr-str e)})))) :headers {"Content-Type" "application/edn"}
:body (pr-str e)}))))
wrap-secure)) wrap-secure))

485
src/clj/auto_ap/yodlee/core.clj
View File

@@ -1,14 +1,15 @@
#_{:clj-kondo/ignore [:unused-namespace]}
(ns auto-ap.yodlee.core (ns auto-ap.yodlee.core
(:require [clj-http.client :as client] (:require
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
[cemerick.url :as u] [clj-http.client :as client]
[unilog.context :as lc] [clojure.core.async :as async]
[clojure.tools.logging :as log] [clojure.data.json :as json]
[clojure.data.json :as json] [clojure.tools.logging :as log]
[clojure.core.async :as async] [config.core :refer [env]]
[config.core :refer [env]] [mount.core :as mount]
[mount.core :as mount] [unilog.context :as lc]
[yang.scheduler :as scheduler])) [yang.scheduler :as scheduler]))
(defn auth-header (defn auth-header
([cob-session] (str "{cobSession=" cob-session "}")) ([cob-session] (str "{cobSession=" cob-session "}"))
@@ -18,12 +19,12 @@
(if (:yodlee-proxy-host env) (if (:yodlee-proxy-host env)
{:proxy-host (:yodlee-proxy-host env) {:proxy-host (:yodlee-proxy-host env)
:proxy-port (:yodlee-proxy-port env) :proxy-port (:yodlee-proxy-port env)
:retry-handler (fn [ex try-count http-context] :retry-handler (fn [ex _ _]
(log/error "yodlee Error." ex) (log/error "yodlee Error." ex)
false) false)
:socket-timeout 60000 :socket-timeout 60000
:connection-timeout 60000} :connection-timeout 60000}
{:retry-handler (fn [ex try-count http-context] {:retry-handler (fn [ex _ _]
(log/error "yodlee Error." ex) (log/error "yodlee Error." ex)
false) false)
:socket-timeout 60000 :socket-timeout 60000
@@ -37,32 +38,33 @@
(catch Exception e (catch Exception e
(if (>= i 3) (if (>= i 3)
(throw e) (throw e)
(do (do
(Thread/sleep 5000) (Thread/sleep 5000)
(retry-thrice x (inc i)))))))) (retry-thrice x (inc i))))))))
(def base-headers {"Api-Version" "1.1" (def base-headers {"Api-Version" "1.1"
"Cobrand-Name" (:yodlee-cobrand-name env) "Cobrand-Name" (:yodlee-cobrand-name env)
"Content-Type" "application/json"}) "Content-Type" "application/json"})
(defn login-cobrand [] (defn login-cobrand []
(retry-thrice (retry-thrice
(fn [] (fn []
(-> (str (:yodlee-base-url env) "/cobrand/login") (-> (str (:yodlee-base-url env) "/cobrand/login")
(client/post (merge {:headers base-headers (client/post (merge {:headers base-headers
:body :body
(json/write-str {:cobrand {:cobrandLogin (:yodlee-cobrand-login env) (json/write-str {:cobrand {:cobrandLogin (:yodlee-cobrand-login env)
:cobrandPassword (:yodlee-cobrand-password env) :cobrandPassword (:yodlee-cobrand-password env)
:locale "en_US"}}) :locale "en_US"}})
:as :json} :as :json}
other-config) other-config))
)
:body :body
:session :session
:cobSession)))) :cobSession))))
(defn login-user (defn login-user
([cob-session] (login-user cob-session (:yodlee-user-login env) (:yodlee-user-password env))) ([cob-session] (login-user cob-session (:yodlee-user-login env) (:yodlee-user-password env)))
([cob-session user password] ([cob-session user password]
(retry-thrice (retry-thrice
@@ -71,214 +73,159 @@
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) (client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
:body :body
(json/write-str {:user {:loginName user (json/write-str {:user {:loginName user
:password password :password password
:locale "en_US"}}) :locale "en_US"}})
:as :json} :as :json}
other-config)) other-config))
:body :body
:user :user
:session :session
:userSession)))) :userSession)))))
)
(defn get-accounts []
(defn get-auth-header []
(let [cob-session (login-cobrand) (let [cob-session (login-cobrand)
user-session (login-user cob-session)] user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/accounts") {"Authorization" (auth-header cob-session user-session)}))
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:account)))
(defn get-accounts-for-provider-account [provider-account-id]
(try (defn get-accounts [auth-header]
(let [cob-session (login-cobrand) (retry-thrice
user-session (login-user cob-session)] (fn []
(retry-thrice (-> (str (:yodlee-base-url env) "/accounts")
(fn [] (client/get (merge {:headers (merge base-headers auth-header)
(-> (str (:yodlee-base-url env) "/accounts?providerAccountId=" provider-account-id) :as :json}
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) other-config))
:as :json} :body
other-config)) :account))))
:body
:account))))
(defn get-accounts-for-provider-account [provider-account-id auth-header]
(try
(retry-thrice
(fn []
(-> (str (:yodlee-base-url env) "/accounts?providerAccountId=" provider-account-id)
(client/get (merge {:headers (merge base-headers auth-header)
:as :json}
other-config))
:body
:account)))
(catch Exception e (catch Exception e
(log/error (str "Couldn't get accounts for provider account '" provider-account-id "'") (log/error (str "Couldn't get accounts for provider account '" provider-account-id "'")
e) e)
[]))) [])))
(defn get-account [i]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) (str "/accounts/" i))
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:account)))
(defn get-provider-accounts [] #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(let [cob-session (login-cobrand) (defn get-account [i auth-header]
user-session (login-user cob-session)] (retry-thrice
(-> (str (:yodlee-base-url env) "/providerAccounts") (fn []
(-> (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) (-> (str (:yodlee-base-url env) (str "/accounts/" i))
:query-params {"include" "credentials,questions,preferences"} (client/get (merge {:headers (merge base-headers auth-header)
:as :json} :as :json}
other-config)) other-config))
:body :body
:providerAccount)))) :account))))
(defn get-provider-accounts [auth-header]
(defn get-transactions [] (retry-thrice
(let [cob-session (login-cobrand) (fn []
user-session (login-user cob-session) (-> (str (:yodlee-base-url env) "/providerAccounts")
batch-size 100 (client/get (merge {:headers (merge base-headers auth-header)
get-transaction-batch (fn [skip] :as :json}
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip) other-config))
:body
:providerAccount))))
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:transaction
))]
(loop [transactions []
skip 0]
(let [transaction-batch (get-transaction-batch skip)]
(if (seq transaction-batch)
(recur (concat transactions transaction-batch) (+ batch-size skip))
transactions)))))
(defn get-provider-accounts []
(let [cob-session (login-cobrand)
user-session (login-user cob-session)
batch-size 100]
(-> (str (:yodlee-base-url env) "/providerAccounts")
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:providerAccount
)))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-provider-account [id] (defn get-provider-account [id auth-header]
(let [cob-session (login-cobrand) (retry-thrice
user-session (login-user cob-session) (fn []
batch-size 100] (-> (str (:yodlee-base-url env) "/providerAccounts/" id)
(client/get (merge {:headers (merge base-headers auth-header)
(-> (str (:yodlee-base-url env) "/providerAccounts/" id) :as :json}
other-config))
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) :body
:as :json} :providerAccount))))
other-config))
:body
:providerAccount)))
(defn get-provider-account-detail [id]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)
batch-size 100]
(-> (str (:yodlee-base-url env) "/providerAccounts/" id )
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:query-params {"include" "credentials,preferences"}
:as :json}
other-config))
:body
:providerAccount
first)))
(defn update-provider-account [pa]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)
batch-size 100]
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json}
other-config)))))
(defn get-provider-account-detail [id auth-header]
(retry-thrice
(fn []
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
(client/get (merge {:headers (merge base-headers auth-header)
:query-params {"include" "credentials,preferences"}
:as :json}
other-config))
:body
:providerAccount
first))))
(defn update-provider-account [pa auth-header]
(retry-thrice
(fn []
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put (merge {:headers (merge base-headers auth-header)
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json}
other-config))))))
(defn get-specific-transactions [account] (defn delete-provider-account [id auth-header]
(log/infof "Getting yodlee transactions for account %s" account) (retry-thrice
(let [cob-session (login-cobrand) (fn []
user-session (login-user cob-session) (-> (str (:yodlee-base-url env) "/providerAccounts/" id)
batch-size 100 (client/delete (merge {:headers (merge base-headers auth-header)
get-transaction-batch (fn [skip] :as :json}
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account) other-config))
:body
:providerAccount
first))))
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:transaction
))]
(loop [transactions [] (defn get-transaction-page
skip 0] ([account batch-size skip auth-header]
(let [transaction-batch (get-transaction-batch skip)] (retry-thrice
(if (seq transaction-batch) (fn []
(recur (concat transactions transaction-batch) (+ batch-size skip)) (-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
transactions))))) (client/get (merge {:headers (merge base-headers auth-header)
:as :json}
other-config))
:body
:transaction))))
([account start end batch-size skip auth-header]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&fromDate=" start "&toDate=" end "&skip=" skip "&accountId=" account)
(client/get (merge {:headers (merge base-headers auth-header)
:as :json}
other-config))
:body
:transaction)))
(defn get-specific-transactions-with-date [account start end]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)
batch-size 100
get-transaction-batch (fn [skip]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&fromDate=" start "&toDate=" end "&skip=" skip "&accountId=" account)
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
:as :json} (defn count-specific-transactions [account auth-header]
other-config)) (retry-thrice
:body (fn []
:transaction (-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
))] (client/get (merge {:headers (merge base-headers auth-header)
:as :json}
other-config))
:body
:transaction))))
(loop [transactions []
skip 0]
(let [transaction-batch (get-transaction-batch skip)]
(if (seq transaction-batch)
(recur (concat transactions transaction-batch) (+ batch-size skip))
transactions)))))
(defn count-specific-transactions [account]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
(client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json}
other-config))
:body
:transaction
)))
(defn get-access-token [] (defn get-access-token []
(try (try
(let [cob-session (login-cobrand) (let [cob-session (login-cobrand)
user-session (login-user cob-session) user-session (login-user cob-session)
token (-> token (->
(str (:yodlee-base-url env) "/user/accessTokens?appIds=" 10003600) (str (:yodlee-base-url env) "/user/accessTokens?appIds=" 10003600)
(client/get (client/get
(merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json} :as :json}
other-config)) other-config))
(doto log/info) (doto log/info)
@@ -286,17 +233,18 @@
:user :user
:accessTokens :accessTokens
first first
:value :value)]
)]
[user-session token]) [user-session token])
(catch Exception e (catch Exception e
(log/error e) (log/error e)
(throw e)))) (throw e))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn create-user [] (defn create-user []
(let [cob-session (login-cobrand)] (let [cob-session (login-cobrand)]
(-> (str (:yodlee-base-url env) "/user/register") (-> (str (:yodlee-base-url env) "/user/register")
(client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) (client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
:body (json/write-str {:user {:loginName "brycesPersoonal2" :body (json/write-str {:user {:loginName "brycesPersoonal2"
:password "kV@mdv3TU11" :password "kV@mdv3TU11"
:email "yodleepersonal2@brycecovertoperations.com"}}) :email "yodleepersonal2@brycecovertoperations.com"}})
@@ -305,24 +253,62 @@
:body))) :body)))
(defn reauthenticate [pa data auth-header]
(try
(retry-thrice
(fn []
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put (merge {:headers (merge base-headers auth-header)
:body (json/write-str data)
:as :json}
other-config)))))
(catch Exception e
(log/error e))))
(defn get-provider-accounts-with-details []
(let [provider-accounts (get-provider-accounts)]
(let [concurrent 10
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account]
(lc/with-context {:provider-account-id (:id provider-account)}
(get-provider-account-detail (:id provider-account)))))
(async/to-chan provider-accounts)
true
(fn [e]
(lc/with-context {:source "Yodlee"}
(log/error "Yodlee pipeline error" e))))
(async/<!! (async/into [] output-chan)))))
(defn concurrent-get-accounts-for-providers [provider-account-ids] ;; helpers
(defn get-specific-transactions [account auth-header]
(log/infof "Getting yodlee transactions for account %s" account)
(let [batch-size 100]
(loop [transactions []
skip 0]
(let [transaction-batch (get-transaction-page account batch-size skip auth-header)]
(if (seq transaction-batch)
(recur (concat transactions transaction-batch) (+ batch-size skip))
transactions)))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-specific-transactions-with-date [account start end auth-header]
(let [batch-size 100]
(loop [transactions []
skip 0]
(let [transaction-batch (get-transaction-page account start end batch-size skip auth-header)]
(if (seq transaction-batch)
(recur (concat transactions transaction-batch) (+ batch-size skip))
transactions)))))
(defn get-provider-accounts-with-details [auth-header]
(let [provider-accounts (get-provider-accounts auth-header)
concurrent 10
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account]
(lc/with-context {:provider-account-id (:id provider-account)}
(get-provider-account-detail (:id provider-account) auth-header))))
(async/to-chan! provider-accounts)
true
(fn [e]
(lc/with-context {:source "Yodlee"}
(log/error "Yodlee pipeline error" e))))
(async/<!! (async/into [] output-chan))))
(defn concurrent-get-accounts-for-providers [provider-account-ids auth-header]
(let [concurrent 20 (let [concurrent 20
output-chan (async/chan)] output-chan (async/chan)]
(async/pipeline-blocking concurrent (async/pipeline-blocking concurrent
@@ -330,93 +316,70 @@
(map (fn [provider-account-id] (map (fn [provider-account-id]
(lc/with-context {:provider-account-id provider-account-id} (lc/with-context {:provider-account-id provider-account-id}
[provider-account-id [provider-account-id
(get-accounts-for-provider-account provider-account-id)]))) (get-accounts-for-provider-account provider-account-id auth-header)])))
(async/to-chan provider-account-ids) (async/to-chan! provider-account-ids)
true true
(fn [e] (fn [e]
(lc/with-context {:source "Yodlee"} (lc/with-context {:source "Yodlee"}
(log/error "Yodlee pipeline error" e)))) (log/error "Yodlee pipeline error" e))))
(async/<!! (async/into {} output-chan)))) (async/<!! (async/into {} output-chan))))
(defn get-provider-accounts-with-accounts []
(let [provider-accounts (by :id (get-provider-accounts-with-details)) (defn get-provider-accounts-with-accounts [auth-header]
accounts (concurrent-get-accounts-for-providers (keys provider-accounts))] (let [provider-accounts (by :id (get-provider-accounts-with-details auth-header))
(->> accounts accounts (concurrent-get-accounts-for-providers (keys provider-accounts) auth-header)]
(->> accounts
(reduce (reduce
(fn [provider-accounts [which accounts]] (fn [provider-accounts [which accounts]]
(assoc-in provider-accounts [which :accounts] accounts)) (assoc-in provider-accounts [which :accounts] accounts))
provider-accounts) provider-accounts)
vals))) vals)))
(mount/defstate in-memory-cache (mount/defstate in-memory-cache
:start (atom [])) :start (atom []))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn refresh-in-memory-cache [] (defn refresh-in-memory-cache []
(lc/with-context {:source "refreshing-in-memory-cache"} (lc/with-context {:source "refreshing-in-memory-cache"}
(try (try
(log/info "Refreshing Yodlee in memory cache") (log/info "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts)) (reset! in-memory-cache (get-provider-accounts-with-accounts (get-auth-header)))
(catch Exception e (catch Exception e
(log/error e))))) (log/error e)))))
(mount/defstate in-memory-cache-worker (mount/defstate in-memory-cache-worker
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache) :start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)
:stop (scheduler/stop in-memory-cache-worker)) :stop (scheduler/stop in-memory-cache-worker))
(defn refresh-provider-account [id] (defn refresh-provider-account [id auth-header]
(swap! in-memory-cache (swap! in-memory-cache
(fn [i] (fn [i]
(-> (by :id i) (-> (by :id i)
(assoc id (assoc (get-provider-account-detail id) (assoc id (assoc (get-provider-account-detail id auth-header)
:accounts (get-accounts-for-provider-account id))) :accounts (get-accounts-for-provider-account id auth-header)))
vals)))) vals))))
(defn delete-provider-account [id]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)
batch-size 100]
(-> (str (:yodlee-base-url env) "/providerAccounts/" id ) (defn delete-and-uncache-provider-account [id auth-header]
(delete-provider-account id auth-header)
(client/delete (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) (swap! in-memory-cache
:as :json}
other-config))
:body
:providerAccount
first))
(swap! in-memory-cache
(fn [i] (fn [i]
(-> (by :id i) (-> (by :id i)
(dissoc id) (dissoc id)
vals)))) vals))))
(defn update-yodlee [id]
(update-provider-account id)
(refresh-provider-account id)
)
(defn reauthenticate [pa data] (defn update-yodlee [id auth-header]
(let [cob-session (login-cobrand) (update-provider-account id auth-header)
user-session (login-user cob-session) (refresh-provider-account id auth-header))
batch-size 100]
(try
(doto (-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) (defn reauthenticate-and-recache [pa data auth-header]
:body (json/write-str data) (reauthenticate pa data auth-header)
:as :json} (refresh-provider-account pa auth-header))
other-config)))
log/info)
(refresh-provider-account pa)
(catch Exception e
(log/error e)))))
#_(defn get-users []
(let [cob-session (login-cobrand)]
(-> "https://developer.api.yodlee.com/ysl/user"
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session)})
:as :json})
:body)))