revamped logging!
This commit is contained in:
Bryce Covert
@@ -2,7 +2,10 @@
|
||||
(:require [clj-http.client :as client]
|
||||
[auto-ap.utils :refer [by]]
|
||||
[cemerick.url :as u]
|
||||
[unilog.context :as lc]
|
||||
[clojure.tools.logging :as log]
|
||||
[clojure.data.json :as json]
|
||||
[clojure.core.async :as async]
|
||||
[config.core :refer [env]]
|
||||
[mount.core :as mount]
|
||||
[yang.scheduler :as scheduler]))
|
||||
@@ -92,9 +95,7 @@
|
||||
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip)
|
||||
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json})
|
||||
:body
|
||||
:transaction
|
||||
@@ -114,9 +115,7 @@
|
||||
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts")
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json})
|
||||
:body
|
||||
:providerAccount
|
||||
@@ -131,9 +130,7 @@
|
||||
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json})
|
||||
:body
|
||||
:providerAccount)))
|
||||
@@ -145,9 +142,7 @@
|
||||
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts/" id )
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:query-params {"include" "credentials,questions,preferences"}
|
||||
:as :json})
|
||||
:body
|
||||
@@ -161,9 +156,7 @@
|
||||
|
||||
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
|
||||
|
||||
(client/put {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/put {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
|
||||
:as :json}))))
|
||||
|
||||
@@ -179,9 +172,7 @@
|
||||
get-transaction-batch (fn [skip]
|
||||
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json})
|
||||
:body
|
||||
:transaction
|
||||
@@ -199,11 +190,8 @@
|
||||
user-session (login-user cob-session)]
|
||||
|
||||
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
|
||||
(doto println)
|
||||
|
||||
(client/get {:headers (doto
|
||||
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
println)
|
||||
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
|
||||
:as :json})
|
||||
:body
|
||||
:transaction
|
||||
@@ -236,34 +224,55 @@
|
||||
:as :json})
|
||||
:body)))
|
||||
|
||||
|
||||
|
||||
(defn get-provider-accounts-with-details []
|
||||
(let [provider-accounts (get-provider-accounts)]
|
||||
(reduce
|
||||
(fn [pas pa]
|
||||
(conj pas (try (get-provider-account-detail (:id pa))
|
||||
(catch Exception e
|
||||
pa))))
|
||||
[]
|
||||
provider-accounts)))
|
||||
(let [concurrent 20
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(map (fn [provider-account]
|
||||
(lc/with-context {:provider-account-id (:id provider-account)}
|
||||
(log/info "fetching details for provider" (:id provider-account))
|
||||
(get-provider-account-detail (:id provider-account)))))
|
||||
(async/to-chan provider-accounts))
|
||||
(async/<!! (async/into [] output-chan)))))
|
||||
|
||||
(defn concurrent-get-accounts-for-providers [provider-account-ids]
|
||||
(let [concurrent 20
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(map (fn [provider-account-id]
|
||||
(lc/with-context {:provider-account-id provider-account-id}
|
||||
(log/info "fetching accounts for provider" provider-account-id)
|
||||
[provider-account-id
|
||||
(get-accounts-for-provider-account provider-account-id)])))
|
||||
(async/to-chan provider-account-ids))
|
||||
(async/<!! (async/into {} output-chan))))
|
||||
|
||||
(defn get-provider-accounts-with-accounts []
|
||||
(let [provider-accounts (by :id (get-provider-accounts-with-details))
|
||||
accounts (get-accounts)]
|
||||
accounts (concurrent-get-accounts-for-providers (keys provider-accounts))]
|
||||
(->> accounts
|
||||
(reduce
|
||||
(fn [provider-accounts a]
|
||||
(update-in provider-accounts [(:providerAccountId a) :accounts] conj a)) provider-accounts)
|
||||
(fn [provider-accounts [which accounts]]
|
||||
(assoc-in provider-accounts [which :accounts] accounts))
|
||||
provider-accounts)
|
||||
vals)))
|
||||
|
||||
(mount/defstate in-memory-cache
|
||||
:start (doto (atom (get-provider-accounts-with-accounts)) println))
|
||||
:start (atom []))
|
||||
|
||||
(defn refresh-in-memory-cache []
|
||||
(try
|
||||
(println "Refreshing Yodlee in memory cache")
|
||||
(reset! in-memory-cache (get-provider-accounts-with-accounts))
|
||||
(catch Exception e
|
||||
(println e))))
|
||||
(lc/with-context {:source "refreshing-in-memory-cache"}
|
||||
(try
|
||||
(log/info "Refreshing Yodlee in memory cache")
|
||||
(reset! in-memory-cache (get-provider-accounts-with-accounts))
|
||||
|
||||
(catch Exception e
|
||||
(log/error e)))))
|
||||
|
||||
(mount/defstate in-memory-cache-worker
|
||||
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
[auto-ap.time :as time]
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
[clojure.string :as str]))
|
||||
[clojure.string :as str]
|
||||
[unilog.context :as lc]
|
||||
[clojure.tools.logging :as log]))
|
||||
|
||||
|
||||
|
||||
@@ -149,29 +151,31 @@
|
||||
:client/_bank-accounts client))))))))
|
||||
|
||||
(defn manual-import [manual-transactions]
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
||||
:bank-account-id bank-account-id
|
||||
:date (time/unparse date "YYYY-MM-dd")
|
||||
:amount {:amount amount}
|
||||
:description {:original description-original
|
||||
:simple high-level-category}
|
||||
:status "POSTED"})
|
||||
(range)
|
||||
transaction-group))))
|
||||
all-rules (tr/get-all)
|
||||
all-bank-accounts (by :db/id (get-all-bank-accounts))
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
|
||||
(println "importing manual transactions" transformed-transactions)
|
||||
(let [result (batch-transact
|
||||
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
|
||||
(println "imported " (count result)))))
|
||||
(lc/with-context {:source "manual import"}
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
||||
:bank-account-id bank-account-id
|
||||
:date (time/unparse date "YYYY-MM-dd")
|
||||
:amount {:amount amount}
|
||||
:description {:original description-original
|
||||
:simple high-level-category}
|
||||
:status "POSTED"})
|
||||
(range)
|
||||
transaction-group))))
|
||||
all-rules (tr/get-all)
|
||||
all-bank-accounts (by :db/id (get-all-bank-accounts))
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
|
||||
(log/info "Importing " (count transformed-transactions) " manual transactions")
|
||||
|
||||
(let [result (batch-transact
|
||||
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
|
||||
(log/info "Imported " (count result) " manual transactions")))))
|
||||
|
||||
(defn do-import
|
||||
([]
|
||||
|
||||
Reference in New Issue
Block a user