notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

revamped logging!

Browse Source
This commit is contained in:
Bryce Covert
2020-08-01 17:07:17 -07:00
parent 49f98522c0
commit 51b097766f
33 changed files with 598 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
src/clj/auto_ap/yodlee/core.clj
View File

@@ -2,7 +2,10 @@
(:require [clj-http.client :as client]
[auto-ap.utils :refer [by]]
[cemerick.url :as u]
[unilog.context :as lc]
[clojure.tools.logging :as log]
[clojure.data.json :as json]
[clojure.core.async :as async]
[config.core :refer [env]]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
@@ -92,9 +95,7 @@
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -114,9 +115,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts")
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount
@@ -131,9 +130,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount)))
@@ -145,9 +142,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id )
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:query-params {"include" "credentials,questions,preferences"}
:as :json})
:body
@@ -161,9 +156,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/put {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json}))))
@@ -179,9 +172,7 @@
get-transaction-batch (fn [skip]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -199,11 +190,8 @@
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
(doto println)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -236,34 +224,55 @@
:as :json})
:body)))
(defn get-provider-accounts-with-details []
(let [provider-accounts (get-provider-accounts)]
(reduce
(fn [pas pa]
(conj pas (try (get-provider-account-detail (:id pa))
(catch Exception e
pa))))
[]
provider-accounts)))
(let [concurrent 20
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account]
(lc/with-context {:provider-account-id (:id provider-account)}
(log/info "fetching details for provider" (:id provider-account))
(get-provider-account-detail (:id provider-account)))))
(async/to-chan provider-accounts))
(async/<!! (async/into [] output-chan)))))
(defn concurrent-get-accounts-for-providers [provider-account-ids]
(let [concurrent 20
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account-id]
(lc/with-context {:provider-account-id provider-account-id}
(log/info "fetching accounts for provider" provider-account-id)
[provider-account-id
(get-accounts-for-provider-account provider-account-id)])))
(async/to-chan provider-account-ids))
(async/<!! (async/into {} output-chan))))
(defn get-provider-accounts-with-accounts []
(let [provider-accounts (by :id (get-provider-accounts-with-details))
accounts (get-accounts)]
accounts (concurrent-get-accounts-for-providers (keys provider-accounts))]
(->> accounts
(reduce
(fn [provider-accounts a]
(update-in provider-accounts [(:providerAccountId a) :accounts] conj a)) provider-accounts)
(fn [provider-accounts [which accounts]]
(assoc-in provider-accounts [which :accounts] accounts))
provider-accounts)
vals)))
(mount/defstate in-memory-cache
:start (doto (atom (get-provider-accounts-with-accounts)) println))
:start (atom []))
(defn refresh-in-memory-cache []
(try
(println "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(println e))))
(lc/with-context {:source "refreshing-in-memory-cache"}
(try
(log/info "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(log/error e)))))
(mount/defstate in-memory-cache-worker
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)