(ns auto-ap.yodlee.core2 (:require [clj-http.client :as client] [auto-ap.utils :refer [by]] [unilog.context :as lc] [clojure.tools.logging :as log] [clojure.data.json :as json] [clojure.core.async :as async] [config.core :refer [env]] #_{:clj-kondo/ignore [:unused-namespace]} [mount.core :as mount] #_{:clj-kondo/ignore [:unused-namespace]} [yang.scheduler :as scheduler] [clj-time.coerce :as coerce] [datomic.api :as dc] [auto-ap.datomic :refer [conn]] [auto-ap.datomic.clients :as d-clients] [clojure.string :as str])) ;; switch all of this to use tokens instead of passing around client codes, particularly because the codes ;; need to be tweaked for repeats (defn client-code->login [client-code] (if (< (count client-code) 3) (str client-code "_" client-code) client-code)) (defn auth-header ([cob-session] (str "Bearer " cob-session))) (defn retry-thrice ([x] (retry-thrice x 0)) ([x i] (try (x) (catch Exception e (if (>= i 3) (throw e) (do (Thread/sleep 5000) (retry-thrice x (inc i)))))))) (def other-config (if (:yodlee2-proxy-host env) {:proxy-host (:yodlee2-proxy-host env) :proxy-port (:yodlee2-proxy-port env) :socket-timeout 60000 :connection-timeout 60000 :retry-handler (fn [ex _ _] (log/error "yodlee Error." ex) false)} {:retry-handler (fn [ex _ _] (log/error "yodlee Error." ex) false) :socket-timeout 60000 :connection-timeout 60000})) (def base-headers {"Api-Version" "1.1" "Content-Type" "application/json"}) (defn login-cobrand [] (retry-thrice (fn [] (-> (str (:yodlee2-base-url env) "/auth/token") (client/post (merge {:headers (assoc base-headers "loginName" (:yodlee2-admin-user env) "Content-Type" "application/x-www-form-urlencoded") :body (str "clientId=" (:yodlee2-client-id env) "&secret=" (:yodlee2-client-secret env)) :as :json} other-config) ) :body :token :accessToken)))) (defn login-user [client-code] (retry-thrice (fn [] (log/info "logging in as " client-code) (-> (str (:yodlee2-base-url env) "/auth/token") (client/post (merge {:headers (assoc base-headers "loginName" (if (:yodlee2-test-user env) (:yodlee2-test-user env) (if (<= (count client-code) 3) (str client-code client-code) client-code)) "Content-Type" "application/x-www-form-urlencoded") :body (str "clientId=" (:yodlee2-client-id env) "&secret=" (:yodlee2-client-secret env)) :as :json} other-config) ) :body :token :accessToken)))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-accounts [client-code ] (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/accounts") (client/get (merge {:headers (merge base-headers {"Authorization" (str "Bearer " cob-session)}) :as :json} other-config)) :body :account))) (defn get-accounts-for-provider-account [client-code provider-account-id] (try (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/accounts?providerAccountId=" provider-account-id) (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :as :json} other-config)) :body :account)) (catch Exception e (log/error (str "Couldn't get accounts for provider account '" provider-account-id "'") e) []))) (defn get-provider-accounts [client-code ] (retry-thrice (fn [] (log/info "logging in user " client-code) (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/providerAccounts") (-> (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session )}) :as :json} other-config)) :body :providerAccount)))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-transactions [client-code] (let [cob-session (login-user (client-code->login client-code)) batch-size 100 get-transaction-batch (fn [skip] (-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip) (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :as :json} other-config)) :body :transaction ))] (loop [transactions [] skip 0] (let [transaction-batch (get-transaction-batch skip)] (if (seq transaction-batch) (recur (concat transactions transaction-batch) (+ batch-size skip)) transactions))))) (defn get-provider-account [client-code id] (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/providerAccounts/" id) (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :as :json} other-config)) :body :providerAccount first))) (defn get-provider-account-detail [client-code id] (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/providerAccounts/" id ) (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :query-params {"include" "credentials,preferences"} :as :json} other-config)) :body :providerAccount first))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn update-provider-account [client-code pa] (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa) (client/put (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}" :as :json} other-config))))) (defn get-specific-transactions [client-code account] (let [cob-session (login-user (client-code->login client-code)) batch-size 100 get-transaction-batch (fn [skip] (-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account) (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :as :json} other-config)) :body :transaction ))] (loop [transactions [] skip 0] (let [transaction-batch (get-transaction-batch skip)] (if (seq transaction-batch) (recur (concat transactions transaction-batch) (+ batch-size skip)) transactions))))) (defn get-access-token [client-code] (try (let [cob-session (login-user (client-code->login client-code))] cob-session) (catch Exception e (log/error e) (throw e)))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn create-user [client-code] (let [cob-session (login-cobrand)] (-> (str (:yodlee2-base-url env) "/user/register") (client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :body (json/write-str {"user" { "loginName" client-code "email" "bryce@integreatconsult.com" "name" {"first" client-code "last" client-code} "address" {"address1" "200 Lincoln Ave" "state" "CA" "city" "Salinas" "zip" "93901" "country" "USA"} "preferences" {"currency" "USD" "timeZone" "GMT" "dateFormat" "YYYY-MMM-DD" "locale" "en_US"}}}) :as :json} other-config)) :body))) (defn get-provider-accounts-with-details [client-code ] (let [provider-accounts (get-provider-accounts client-code) concurrent 20 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan (map (fn [provider-account] (lc/with-context {:provider-account-id (:id provider-account)} (get-provider-account-detail client-code (:id provider-account))))) (async/to-chan! provider-accounts)) (async/> accounts (reduce (fn [provider-accounts [which accounts]] (assoc-in provider-accounts [which :accounts] accounts)) provider-accounts) vals))) (defn delete-provider-account [client-code id] (let [cob-session (login-user (client-code->login client-code))] (-> (str (:yodlee2-base-url env) "/providerAccounts/" id ) (client/delete (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) :as :json} other-config)) :body :providerAccount first)) @(dc/transact conn [[:db/retractEntity [:yodlee-provider-account/id id]]])) (defn upsert-accounts-tx ([client-code] (upsert-accounts-tx client-code (get-provider-accounts client-code))) ([client-code provider-accounts] (let [accounts (get-accounts-for-providers client-code (map :id provider-accounts))] (map (fn [pa] (cond-> {:yodlee-provider-account/id (:id pa) :yodlee-provider-account/status (:status pa) :yodlee-provider-account/detailed-status (or (-> pa :dataset first :additionalStatus) "unknown") :yodlee-provider-account/client [:client/code client-code] :yodlee-provider-account/accounts (mapv (fn [a] {:yodlee-account/id (:id a) :yodlee-account/name (str (:providerName a) " (" (:accountName a) ")") :yodlee-account/number (or (:accountNumber a) "Unknown") :yodlee-account/status (or (-> a :dataset first :additionalStatus) "unknown") :yodlee-account/available-balance (or (-> a :currentBalance :amount) 0.0)}) (get accounts (:id pa)))} (-> pa :dataset first :lastUpdated) (assoc :yodlee-provider-account/last-updated (-> pa :dataset first :lastUpdated coerce/to-date)) )) provider-accounts)))) (defn refresh-provider-account [client-code id] (log/info "refreshing yodlee provider account id" id) @(dc/transact conn (upsert-accounts-tx client-code [(get-provider-account client-code id)]))) (defn upsert-accounts [] (let [concurrent 20 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan (mapcat (fn [client] (log/info "Upserting Yodlee Accounts for " (:client/code client)) (lc/with-context {:client-code (:client/code client)} (try (upsert-accounts-tx (:client/code client)) (catch Exception e (log/error "Could not update client " (:client/code client) e)))))) (async/to-chan! (d-clients/get-all))) (let [result (async/ (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa) (client/put (merge {:headers (merge base-headers {"Authorization" (auth-header (login-user (client-code->login client-code)))}) :body (json/write-str data) :as :json} other-config))) (catch Exception e (log/error e))) ) (defn reauthenticate-and-recache [client-code pa data] (reauthenticate client-code pa data) (refresh-provider-account client-code pa))