Yodlee exists as datomic entities!

src/clj/auto_ap/datomic/clients.clj

@@ -6,6 +6,7 @@
 
 (defn cleanse [e]
   (-> e
+      (assoc :client/yodlee-provider-accounts (get e :yodlee-provider-account/_client))
       (update :client/location-matches
               (fn [lms]
                 (map #(assoc % :location-match/match (first (:location-match/matches %))) lms)))
@@ -20,7 +21,9 @@
 (defn get-all []
   (->> (d/q '[:find (pull ?e [*
                               {:client/address [*]}
-                              {:client/bank-accounts [* {:bank-account/type [*]}]}])
+                              {:client/bank-accounts [* {:bank-account/type [*]
+                                                         :bank-account/yodlee-account [:yodlee-account/name :yodlee-account/id :yodlee-account/number]}]}
+                              {:yodlee-provider-account/_client [*]}])
               :where [?e :client/name]]
             (d/db (d/connect uri)))
        (map first)
@@ -29,7 +32,10 @@
        ))
 (defn get-by-id [id]
   (->> 
-   (d/query (-> {:query {:find ['(pull ?e [* {:client/bank-accounts [* {:bank-account/type [*]}]}])]
+   (d/query (-> {:query {:find ['(pull ?e [*
+                                           {:client/bank-accounts [* {:bank-account/type [*]
+                                                                      :bank-account/yodlee-account [:yodlee-account/name :yodlee-account/id :yodlee-account/number]}]}
+                                           {:yodlee-provider-account/_client [*]}])]
                          :in ['$ '?e]
                          :where [['?e]]}
                  :args [(d/db (d/connect uri)) id]}

src/clj/auto_ap/datomic/migrate/yodlee2.clj

@@ -0,0 +1,56 @@
+(ns auto-ap.datomic.migrate.yodlee2)
+
+(def norms-map {::add-yodlee-view
+                {:txes [[{:db/ident :yodlee-provider-account/id
+                          :db/doc "Yodlee Provider ACcount Id"
+                          :db/valueType :db.type/long
+                          :db/cardinality :db.cardinality/one
+                          :db/unique :db.unique/identity}
+                         {:db/ident :yodlee-provider-account/client
+                          :db/doc "Which client the provider account is for"
+                          :db/valueType :db.type/ref
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-provider-account/accounts
+                          :db/doc "Individual bank accounts"
+                          :db/isComponent true
+                          :db/valueType :db.type/ref
+                          :db/cardinality :db.cardinality/many}
+                         {:db/ident :yodlee-provider-account/status
+                          :db/doc "Current status"
+                          :db/valueType :db.type/string
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-provider-account/detailed-status
+                          :db/doc "Current status (detail)"
+                          :db/valueType :db.type/string
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-provider-account/last-updated
+                          :db/doc "Last updated date"
+                          :db/valueType :db.type/instant
+                          :db/cardinality :db.cardinality/one}
+
+                         {:db/ident :yodlee-account/id
+                          :db/doc "Yodlee account id"
+                          :db/valueType :db.type/long
+                          :db/cardinality :db.cardinality/one
+                          :db/unique :db.unique/identity}
+                         {:db/ident :yodlee-account/name
+                          :db/doc "account name"
+                          :db/valueType :db.type/string
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-account/number
+                          :db/doc "account number"
+                          :db/valueType :db.type/string
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-account/status
+                          :db/doc "Current status"
+                          :db/valueType :db.type/string
+                          :db/cardinality :db.cardinality/one}
+                         {:db/ident :yodlee-account/available-balance
+                          :db/doc "Available Balance"
+                          :db/valueType :db.type/double
+                          :db/cardinality :db.cardinality/one}
+
+                         {:db/ident :bank-account/yodlee-account
+                          :db/doc "Yodlee account for the bank account"
+                          :db/valueType :db.type/ref
+                          :db/cardinality :db.cardinality/one}]]}})

src/clj/auto_ap/datomic/yodlee2.clj

@@ -0,0 +1,69 @@
+(ns auto-ap.datomic.yodlee2
+  (:require [datomic.api :as d]
+            [auto-ap.datomic :refer [uri remove-nils merge-query apply-pagination apply-sort-3 add-sorter-fields conn]]
+            [auto-ap.graphql.utils :refer [limited-clients]]
+            [auto-ap.parse :as parse]
+            [clj-time.coerce :as c]
+            [clojure.set :refer [rename-keys]]
+            [clojure.string :as str]
+            [clojure.tools.logging :as log]))
+
+(def default-read '[*])
+
+(defn <-datomic [x]
+  (-> x (update  :yodlee-provider-account/last-updated c/from-date)))
+
+(defn raw-graphql-ids [db args]
+  (->> (cond-> {:query {:find []
+                        :in ['$]
+                        :where ['[?e :yodlee-provider-account/id]]}
+                :args [(d/db (d/connect uri))]}
+         
+         (limited-clients (:id args))
+         (merge-query {:query {:in ['[?xx ...]]
+                               :where ['[?e :yodlee-provider-account/client ?xx]]}
+                       :args [ (set (map :db/id (limited-clients (:id args))))]})
+
+         (:client-id args)
+         (merge-query {:query {:in ['?client-id]
+                               :where ['[?e :yodlee-provider-account/client ?client-id]]}
+                       :args [ (:client-id args)]})
+
+         (:client-code args)
+         (merge-query {:query {:in ['?client-code]
+                               :where ['[?e :yodlee-provider-account/client ?client-id]
+                                       '[?client-id :client/code ?client-code]]}
+                       :args [ (:client-code args)]})
+
+         (:sort args) (add-sorter-fields {"status" ['[?e :yodlee-provider-account/status ?sort-status]]}
+                                         args)
+         true
+         (merge-query {:query {:find ['?e ]
+                               :where ['[?e :yodlee-provider-account/id]]}}) )
+       
+
+       (d/query)
+       (apply-sort-3 args)
+       (apply-pagination args)))
+
+
+(defn graphql-results [ids db args]
+  (let [results (->> (d/pull-many db default-read ids)
+                     (group-by :db/id))]
+    (->> ids
+         (map results)
+         (map first)
+         (mapv <-datomic))))
+
+
+(defn get-graphql [args]
+  (let [db (d/db (d/connect uri))
+        {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
+    [(->> (graphql-results ids-to-retrieve db args))
+     matching-count]))
+
+(defn get-by-id [id]
+  (-> (d/db (d/connect uri))
+      (d/pull  default-read id)
+      (<-datomic)))
+

src/clj/auto_ap/graphql.clj

@@ -16,6 +16,7 @@
             [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
             [auto-ap.graphql.vendors :as gq-vendors]
             [auto-ap.graphql.yodlee-merchants :as ym]
+            [auto-ap.graphql.yodlee2 :as gq-yodlee2]
             [auto-ap.logging :refer [error-event info-event warn-event]]
             [auto-ap.time :as time]
             [clj-time.coerce :as coerce]
@@ -78,7 +79,24 @@
               :locations {:type '(list String)}
               :matches {:type '(list String)}
               :bank_accounts {:type '(list :bank_account)}
-              :forecasted_transactions {:type '(list :forecasted_transaction)}}}
+              :forecasted_transactions {:type '(list :forecasted_transaction)}
+              :yodlee_provider_accounts {:type '(list :yodlee_provider_account)}}}
+
+    :yodlee_provider_account
+    {:fields {:id {:type 'Int}
+              :client {:type :client}
+              :status {:type 'String}
+              :detailed_status {:type 'String}
+              :last_updated {:type :iso_date}
+              :accounts {:type '(list :yodlee_account)}}}
+    
+    :yodlee_account
+    {:fields {:id {:type 'Int}
+              :status {:type 'String}
+              :available_balance {:type :money}
+              :name {:type 'String}
+              :number {:type 'String}
+              :last_updated {:type :iso_date}}}
     :contact
     {:fields {:id {:type :id}
               :name {:type 'String}
@@ -100,6 +118,7 @@
               :bank_code {:type 'String}
               :bank_name {:type 'String}
               :yodlee_account_id {:type 'Int}
+              :yodlee_account {:type :yodlee_account}
               :locations {:type '(list String)}}}
     :forecasted_transaction {:fields {:identifier {:type 'String}
                                       :id {:type :id}
@@ -364,6 +383,11 @@
 
 
     
+    :yodlee_provider_account_page {:fields {:yodlee_provider_accounts {:type '(list :yodlee_provider_account)}
+                                            :count {:type 'Int}
+                                            :total {:type 'Int}
+                                            :start {:type 'Int}
+                                            :end {:type 'Int}}}
 
     :invoice_page {:fields {:invoices {:type '(list :invoice)}
                             :outstanding {:type :money}
@@ -479,6 +503,9 @@
                       :resolve :get-profit-and-loss}
 
     
+    :yodlee_provider_account_page {:type :yodlee_provider_account_page
+                                   :args {:client_id {:type :id}}
+                                   :resolve :get-yodlee-provider-account-page}
 
     :invoice_page {:type '(list :invoice_page)
                    :args {:import_status {:type 'String}
@@ -680,7 +707,8 @@
               :routing {:type 'String}
               :bank_name {:type 'String}
               :locations {:type '(list String)}
-              :yodlee_account_id {:type 'Int}}}
+              :yodlee_account_id {:type 'Int}
+              :yodlee_account {:type 'Int}}}
     :edit_user
     {:fields {:id {:type :id}
               :name {:type 'String}
@@ -1165,6 +1193,7 @@
   (-> integreat-schema
       (attach-resolvers {:get-invoice-page gq-invoices/get-invoice-page
                          :get-all-invoices gq-invoices/get-all-invoices
+                         :get-yodlee-provider-account-page gq-yodlee2/get-yodlee-provider-account-page
                          :get-all-payments get-all-payments
                          :get-all-expected-deposits gq-expected-deposit/get-all-expected-deposits
                          :get-all-sales-orders get-all-sales-orders

src/clj/auto_ap/graphql/clients.clj

@@ -1,7 +1,7 @@
 (ns auto-ap.graphql.clients
   (:require [auto-ap.datomic :refer [audit-transact conn remove-nils]]
             [auto-ap.datomic.clients :as d-clients]
-            [auto-ap.graphql.utils :refer [->graphql assert-admin can-see-client?]]
+            [auto-ap.graphql.utils :refer [->graphql assert-admin can-see-client? is-admin?]]
             [clj-time.coerce :as coerce]
             [config.core :refer [env]]
             [clojure.string :as str]
@@ -47,10 +47,16 @@
                              (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client))
                              (mapv (fn [m] [:db/retract (:db/id client) :client/matches m]) (:client/matches client)))
                             (:id context)))
-        reverts (->> (:bank_accounts edit_client)
+        reverts (into (->> (:bank_accounts edit_client)
                            (filter #(nil? (:yodlee_account_id %)))
                            (filter #(:bank-account/yodlee-account-id (d/entity (d/db conn) (:id %))))
                            (map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account-id (:bank-account/yodlee-account-id (d/entity (d/db conn) (:id ba)))])))
+
+                      (->> (:bank_accounts edit_client)
+                           (filter #(nil? (:yodlee_account %)))
+                           (filter #(:bank-account/yodlee-account (d/entity (d/db conn) (:id %))))
+                           (map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account (:db/id (:bank-account/yodlee-account (d/entity (d/db conn) (:id ba))))]))))
+
         transactions (into [(remove-nils {:db/id id
                                           :client/code (if (str/blank? (:client/code client))
                                                          (:code edit_client)
@@ -77,7 +83,7 @@
 
                                           
                                           :client/bank-accounts (map #(remove-nils
-                                                                       {:db/id (:id %)
+                                                                       (cond-> {:db/id (:id %)
                                                                                 :bank-account/code (:code %)
                                                                                 :bank-account/bank-name (:bank_name %)
                                                                                 :bank-account/bank-code (:bank_code %)
@@ -93,8 +99,10 @@
                                                                                 :bank-account/locations (:locations %)
 
                                                                                 :bank-account/yodlee-account-id (:yodlee_account_id %)
+                                                                                
                                                                                 :bank-account/type (keyword "bank-account-type" (name (:type %)))
                                                                                 }
+                                                                         (:yodlee_account %) (assoc :bank-account/yodlee-account [:yodlee-account/id (:yodlee_account %)]))
                                                                        ) (:bank_accounts edit_client))
 
                                           })
@@ -122,5 +130,9 @@
 
 (defn get-client [context args value]
   (->graphql
-   (filter #(can-see-client? (:id context) %)
+   (->> (d-clients/get-all)
-           (d-clients/get-all))))
+        (filter #(can-see-client? (:id context) %))
+        (map (fn [c]
+               (if (is-admin? (:id context))
+                 c
+                 (dissoc c :client/yodlee-provider-accounts)))))))

src/clj/auto_ap/graphql/yodlee2.clj

@@ -0,0 +1,29 @@
+(ns auto-ap.graphql.yodlee2
+  (:require [auto-ap.graphql.utils :refer [->graphql <-graphql assert-can-see-client assert-admin enum->keyword]]
+            [auto-ap.datomic.yodlee2 :as d-yodlee2]
+            [auto-ap.time :refer [parse iso-date]]
+            [auto-ap.utils :refer [dollars=]]
+            [datomic.api :as d]
+            [auto-ap.datomic :refer [uri remove-nils audit-transact conn]]
+            [clj-time.coerce :as coerce]
+            [clj-time.core :as time]
+            [clojure.set :as set]
+            [clojure.tools.logging :as log]))
+
+(defn get-yodlee-provider-account-page [context args value]
+  (assert-admin (:id context))  
+  (let [args (assoc args :id (:id context))
+        [yodlee-provider-accounts cnt] (d-yodlee2/get-graphql (<-graphql (assoc args :id (:id context))))]
+    {:yodlee_provider_accounts (map ->graphql yodlee-provider-accounts)
+     :total cnt
+     :count (count yodlee-provider-accounts)
+     :start (:start args 0)
+     :end (+ (:start args 0) (count yodlee-provider-accounts))}))
+
+(defn get-all-yodlee-provider-accounts [context args value]
+  (assert-admin (:id context))
+  (map
+   ->graphql
+   (first (d-yodlee2/get-graphql (assoc (<-graphql args)
+                                         :count Integer/MAX_VALUE)))))
+

src/clj/auto_ap/routes/yodlee2.clj

@@ -19,7 +19,7 @@
    (context "/yodlee2" []
             (GET "/fastlink" {:keys [query-params identity] :as request}
                  (assert-admin identity)
-                 (let [token (yodlee/get-access-token)]
+                 (let [token (yodlee/get-access-token (get query-params "client"))]
                    {:status 200 
                     :headers {"Content-Type" "application/edn"}
                     :body (pr-str {:token token

src/clj/auto_ap/time.clj

@@ -2,6 +2,9 @@
   (:require [clj-time.core :as time]
             [clj-time.format :as f]))
 
+(defn localize [d]
+  (time/to-time-zone d (time/time-zone-for-id "America/Los_Angeles")))
+
 (defn local-now []
   (time/to-time-zone (time/now) (time/time-zone-for-id "America/Los_Angeles")))
 

src/clj/auto_ap/yodlee/core2.clj

@@ -8,11 +8,22 @@
             [clojure.core.async :as async]
             [config.core :refer [env]]
             [mount.core :as mount]
-            [yang.scheduler :as scheduler]))
+            [yang.scheduler :as scheduler]
+            [clj-time.coerce :as coerce]
+            [auto-ap.time :as time2]
+            [datomic.api :as d]
+            [auto-ap.datomic :refer [conn]]
+            [auto-ap.datomic.clients :as d-clients]))
+(defn client-code->login [client-code]
+  (if (< (count client-code) 3)
+    (str client-code "_" client-code)
+    client-code))
 
 (defn auth-header
   ([cob-session] (str "Bearer " cob-session)))
 
+
+
 (def other-config
   (if (:yodlee2-proxy-host env)
     {:proxy-host (:yodlee2-proxy-host env)
@@ -40,10 +51,10 @@
       :token
       :accessToken))
 
-(defn login-user []
+(defn login-user [client-code]
   (-> (str (:yodlee2-base-url env) "/auth/token") 
       (client/post (merge {:headers (assoc base-headers
-                                           "loginName"  (:yodlee2-integreat-user env)
+                                           "loginName"  client-code
                                            "Content-Type" "application/x-www-form-urlencoded")
                            :body (str "clientId=" (:yodlee2-client-id env) " &secret=" (:yodlee2-client-secret env))
                            :as :json}
@@ -53,9 +64,8 @@
       :token
       :accessToken))
 
-
+(defn get-accounts [client-code ]
-(defn get-accounts []
+  (let [cob-session (login-user client-code)]
-  (let [cob-session (login-user)]
     (-> (str (:yodlee2-base-url env) "/accounts") 
         (client/get (merge {:headers (merge base-headers {"Authorization" (str "Bearer " cob-session)}) 
                             :as :json}
@@ -63,9 +73,9 @@
         :body
         :account)))
 
-(defn get-accounts-for-provider-account [provider-account-id]
+(defn get-accounts-for-provider-account [client-code provider-account-id]
   (try 
-    (let [cob-session (login-user)]
+    (let [cob-session (login-user client-code)]
       (-> (str (:yodlee2-base-url env) "/accounts?providerAccountId=" provider-account-id) 
           (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) 
                               :as :json}
@@ -77,8 +87,8 @@
                  e)
       [])))
 
-(defn get-provider-accounts []
+(defn get-provider-accounts [client-code ]
-  (let [cob-session (login-user)]
+  (let [cob-session (login-user client-code)]
     (-> (str (:yodlee2-base-url env) "/providerAccounts") 
         (-> (client/get (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session )})
                                 :as :json}
@@ -88,8 +98,8 @@
 
 
 
-(defn get-transactions []
+(defn get-transactions [client-code]
-  (let [cob-session (login-user)
+  (let [cob-session (login-user client-code)
         batch-size 100
         get-transaction-batch (fn [skip]
                                 (-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip) 
@@ -111,8 +121,8 @@
 
 
 
-(defn get-provider-account [id]
+(defn get-provider-account [client-code id]
-  (let [cob-session (login-user)
+  (let [cob-session (login-user client-code)
         batch-size 100]
 
     (-> (str (:yodlee2-base-url env) "/providerAccounts/" id) 
@@ -123,8 +133,8 @@
         :body
         :providerAccount)))
 
-(defn get-provider-account-detail [id]
+(defn get-provider-account-detail [client-code id]
-  (let [cob-session (login-user)]
+  (let [cob-session (login-user client-code)]
 
     (-> (str (:yodlee2-base-url env) "/providerAccounts/" id ) 
 
@@ -136,8 +146,8 @@
         :providerAccount
         first)))
 
-(defn update-provider-account [pa]
+(defn update-provider-account [client-code pa]
-  (let [cob-session (login-user)]
+  (let [cob-session (login-user client-code)]
 
     (-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa) 
 
@@ -151,8 +161,8 @@
 
 
 
-(defn get-specific-transactions [account]
+(defn get-specific-transactions [client-code account]
-  (let [cob-session (login-user)
+  (let [cob-session (login-user client-code)
         batch-size 100
         get-transaction-batch (fn [skip]
                                 (-> (str (:yodlee2-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account) 
@@ -172,23 +182,23 @@
           transactions)))))
 
 
-(defn get-access-token []
+(defn get-access-token [client-code]
   (try 
-    (let [cob-session (login-user)]
+    (let [cob-session (login-user client-code)]
       cob-session)
     (catch Exception e
       (log/error e)
       (throw e))))
 
-(defn create-user []
+(defn create-user [client-code]
   (let [cob-session (login-cobrand)]
     (-> (str (:yodlee2-base-url env) "/user/register") 
         (client/post (merge {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) 
                              :body (json/write-str {"user" {
-                                                            "loginName" "integreat-main"
+                                                            "loginName" client-code
                                                             "email" "bryce@integreatconsult.com"
-                                                            "name" {"first" "Bryce"
+                                                            "name" {"first" client-code
-                                                                    "last" "Covert"}
+                                                                    "last" client-code}
                                                             "address" {"address1" "200 Lincoln Ave"
                                                                        "state" "CA"
                                                                        "city" "Salinas"
@@ -203,34 +213,38 @@
         :body)))
 
 
+(defn assert-user [client-code]
+  (let [cob-session (login-cobrand client-code)]
+    cob-session))
 
-(defn get-provider-accounts-with-details []
+
-  (let [provider-accounts (get-provider-accounts)]
+
+(defn get-provider-accounts-with-details [client-code ]
+  (let [provider-accounts (get-provider-accounts client-code)]
     (let [concurrent 20
           output-chan (async/chan)]
       (async/pipeline-blocking concurrent
                                output-chan
                                (map (fn [provider-account]
                                       (lc/with-context {:provider-account-id (:id provider-account)}
-                                        (get-provider-account-detail (:id provider-account)))))
+                                        (get-provider-account-detail client-code (:id provider-account)))))
                                (async/to-chan provider-accounts))
       (async/<!! (async/into [] output-chan)))))
 
-(defn concurrent-get-accounts-for-providers [provider-account-ids]
+(defn get-accounts-for-providers [client-code provider-account-ids]
-  (let [concurrent 20
+  (log/info "looking up " (count provider-account-ids) " provider accounts for client " client-code ".")
-        output-chan (async/chan)]
+  (into {}
-    (async/pipeline-blocking concurrent
+        (mapv (fn [provider-account-id]
-                             output-chan
-                             (map (fn [provider-account-id]
                 (lc/with-context {:provider-account-id provider-account-id}
                   [provider-account-id
-                                       (get-accounts-for-provider-account provider-account-id)])))
+                   (get-accounts-for-provider-account client-code provider-account-id)]))
-                             (async/to-chan provider-account-ids))
+              provider-account-ids)))
-    (async/<!! (async/into {} output-chan))))
 
-(defn get-provider-accounts-with-accounts []
+
-  (let [provider-accounts (by :id (get-provider-accounts-with-details))
+
-        accounts (concurrent-get-accounts-for-providers (keys provider-accounts))]
+(defn get-provider-accounts-with-accounts [client-code]
+  (let [provider-accounts (by :id (get-provider-accounts-with-details client-code))
+        accounts (get-accounts-for-providers client-code (keys provider-accounts))]
     (->> accounts 
          (reduce
           (fn [provider-accounts [which accounts]]
@@ -238,33 +252,12 @@
           provider-accounts)
          vals)))
 
-(mount/defstate in-memory-cache
-  :start (atom []))
-
-(defn refresh-in-memory-cache []
-  (lc/with-context {:source "refreshing-in-memory-cache"}
-    (try 
-      (log/info "Refreshing Yodlee in memory cache")
-      (reset! in-memory-cache (get-provider-accounts-with-accounts))
-      
-      (catch Exception e
-        (log/error e)))))
-
-(mount/defstate in-memory-cache-worker
-  :start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)
-  :stop (scheduler/stop in-memory-cache-worker))
 
 
-(defn refresh-provider-account [id]
-  (swap! in-memory-cache 
-         (fn [i]
-           (-> (by :id i)
-               (assoc id (assoc  (get-provider-account-detail id)
-                                 :accounts (get-accounts-for-provider-account id)))
-               vals))))
 
-(defn delete-provider-account [id]
+
-  (let [cob-session (login-user)]
+(defn delete-provider-account [client-code id]
+  (let [cob-session (login-user client-code)]
 
     (-> (str (:yodlee2-base-url env) "/providerAccounts/" id ) 
 
@@ -274,11 +267,49 @@
         :body
         :providerAccount
         first))
-  (swap! in-memory-cache 
+  @(d/transact conn [:db/retractEntity [:yodlee-provider-account/id id]]))
-         (fn [i]
+
-           (-> (by :id i)
+(defn upsert-accounts-tx [client-code]
-               (dissoc id)
+  (let [provider-accounts (get-provider-accounts client-code)
-               vals))))
+        accounts (get-accounts-for-providers client-code (map :id provider-accounts))]
+    (map (fn [pa]
+           {:yodlee-provider-account/id  (:id pa)
+            :yodlee-provider-account/status (:status pa)
+            :yodlee-provider-account/detailed-status (-> pa :dataset first :additionalStatus)
+            :yodlee-provider-account/client [:client/code client-code]
+            :yodlee-provider-account/last-updated (-> pa :dataset first :lastUpdated coerce/to-date)
+            :yodlee-provider-account/accounts (mapv
+                                               (fn [a]
+                                                 {:yodlee-account/id (:id a)
+                                                  :yodlee-account/name (str (:providerName a) " (" (:accountName a) ")")
+                                                  :yodlee-account/number (:accountNumber a)
+                                                  :yodlee-account/status (-> a :dataset first :additionalStatus)
+                                                  :yodlee-account/available-balance (or (-> a :currentBalance :amount)
+                                                                                        0.0)})
+                                               (get accounts (:id pa)))})
+         provider-accounts)))
+
+(defn refresh-provider-account [client-code id]
+  @(d/transact conn (upsert-accounts-tx (client-code->login (client-code->login client-code)
+                                                            id))))
+
+(defn upsert-accounts []
+  (let [concurrent 20
+        output-chan (async/chan)]
+    (async/pipeline-blocking concurrent
+                             output-chan
+                             (mapcat (fn [client]
+                                    (log/info "Upserting Yodlee Accounts for " (:client/code client))
+                                    (lc/with-context {:client-code (:client/code client)}
+                                      (upsert-accounts-tx (client-code->login (:client/code client))))))
+                             (async/to-chan (d-clients/get-all)))
+    (let [result (async/<!! (async/into [] output-chan))]
+      (log/info "Current yodlee state is " result)
+      @(d/transact conn result))))
+
+(mount/defstate yodlee-sync-worker
+  :start (scheduler/every (* 5 60 1000) upsert-accounts)
+  :stop (scheduler/stop upsert-accounts))
 
 (defn update-yodlee [id]
   (update-provider-account id)
@@ -286,7 +317,6 @@
 
 (defn reauthenticate [pa data]
   (let [cob-session (login-cobrand)]
-
     (try
       
       (doto (-> (str (:yodlee2-base-url env) "/providerAccounts?providerAccountIds=" pa) 

src/cljs/auto_ap/core.cljs

@@ -13,12 +13,12 @@
             [auto-ap.history :as p]
             [bidi.bidi :as bidi]))
 
-(set! *warn-on-infer* true)
+#_(set! *warn-on-infer* true)
 
 (defn dev-setup []
   (when true
     (enable-console-print!)
-    (println "dev mode")))
+    (println "dev mode enabled")))
 
 (defn mount-root []
   (re-frame/clear-subscription-cache!)

src/cljs/auto_ap/events.cljs

@@ -27,6 +27,16 @@
    :print-as :invoice-reminder-schedule :code
    [:address [:street1 :street2 :city :state :zip]]])
 
+(defn client-query [token]
+  (cond-> [:id :name :signature-file :code :email :matches :week-a-debits :week-a-credits :week-b-debits :week-b-credits :locations
+           [:location-matches [:id :location :match]]
+           [:bank-accounts [:id :start-date :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id
+                            [:yodlee-account [:name :id :number]]
+                            :locations :include-in-reports] ]
+           [:address [:street1 :street2 :city :state :zip]]
+           [:forecasted-transactions [:id :amount :identifier :day-of-month]]]
+    (= "admin" (or (get (jwt->data token) "role") (get (jwt->data token) "user/role")) ) (conj [:yodlee-provider-accounts [:id [:accounts [:id :name :number :available-balance]]]])))
+
 (re-frame/reg-event-fx
  ::initialize-db
  (fn  [{:keys [db]} [_ token]]
@@ -56,9 +66,7 @@
         :graphql {:token token
                   :query-obj {:venia/queries [[:client
 
-                                               [:id :name :signature-file :code :email :matches :week-a-debits :week-a-credits :week-b-debits :week-b-credits :locations [:location-matches [:id :location :match]] [:bank-accounts [:id :start-date :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id :locations :include-in-reports] ]
+                                               (client-query token)]
-                                                [:address [:street1 :street2 :city :state :zip]]
-                                                [:forecasted-transactions [:id :amount :identifier :day-of-month]]]]
                                               [:vendor
                                                vendor-query]
                                               [:accounts [:numeric-code :location :name :type :account_set :applicability :id [:client-overrides [:name :id [:client [:name :id]]]]]]]}
@@ -76,10 +84,7 @@
  (fn  [{:keys [db]} [_ token user]]
    {:graphql {:token token
               :query-obj {:venia/queries [[:client
-                                           [:id :name :code :matches :locations :week-a-debits :week-a-credits :week-b-debits :week-b-credits [:location-matches [:id :location :match]]
+                                           (client-query token)]
-                                            [:address [:street1 :street2 :city :state :zip]]
-                                            [:bank-accounts [:id :start-date :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id :locations :include-in-reports] ]
-                                            [:forecasted-transactions [:id :amount :identifier :day-of-month]]]]
                                           [:vendor
                                            vendor-query]
                                           [:accounts [:numeric-code :name :location :type :account_set :applicability :id [:client-overrides [:name [:client [:name :id]]]]]]]}

src/cljs/auto_ap/reload.cljs

@@ -1,4 +1,5 @@
 (ns ^:figwheel-hooks auto-ap.reload)
 
 (defn ^:after-load reload []
+  (println "HERE")
   (@(resolve 'auto-ap.core/mount-root)))

src/cljs/auto_ap/views/main.cljs

@@ -105,7 +105,7 @@
   (admin-yodlee-page))
 
 (defmethod page :admin-yodlee2 [_]
-  (yodlee2/admin-yodlee-page))
+  (yodlee2/admin-yodle-provider-accounts-page))
 
 (defmethod page :admin-accounts [_]
   (admin-accounts-page))

src/cljs/auto_ap/views/pages/admin/clients/form.cljs

@@ -2,17 +2,24 @@
   (:require [auto-ap.entities.clients :as entity]
             [auto-ap.forms :as forms]
             [auto-ap.subs :as subs]
+            [auto-ap.events :as events]
             [auto-ap.views.components.address :refer [address-field]]
+            [auto-ap.views.components.typeahead :refer [typeahead-entity]]
             [auto-ap.views.components.layouts :refer [side-bar]]
-            [auto-ap.views.utils :refer [dispatch-event horizontal-field nf multi-field date-picker standard date->str]]
+            [auto-ap.views.utils
+             :refer
+             [date->str
+              date-picker
+              dispatch-event
+              horizontal-field
+              multi-field
+              standard]]
             [cljs-time.coerce :as coerce]
             [cljs-time.core :as t]
-            [clojure.spec.alpha :as s]
+            [clojure.string :as str]
             [re-frame.core :as re-frame]
             [reagent.core :as r]
-            [clojure.string :as str]
+            [react-signature-canvas :as canvas]))
-            [react-signature-canvas :as canvas]
-            [auto-ap.views.components.buttons :as buttons]))
 
 (def signature-canvas (r/adapt-react-class (.-default canvas)))
 
@@ -98,7 +105,7 @@
                                          :identifier identifier
                                          :amount amount})
                                       (:forecasted-transactions new-client-data))
-        :bank-accounts (map (fn [{:keys [number name check-number include-in-reports type id code start-date bank-name routing bank-code new? sort-order visible yodlee-account-id locations]}]
+        :bank-accounts (map (fn [{:keys [number name check-number include-in-reports type id code start-date bank-name routing bank-code new? sort-order visible yodlee-account-id locations yodlee-account]}]
                               {:number number
                                :name name
                                :check-number check-number
@@ -119,6 +126,7 @@
                                :locations (mapv :location locations)
                                :yodlee-account-id (when-not (str/blank? yodlee-account-id)
                                                     (js/parseInt yodlee-account-id))
+                               :yodlee-account (:id yodlee-account)
                                :code (if new?
                                        (str (:code new-client-data) "-" code)
                                        code)
@@ -160,11 +168,7 @@
                                      :operation/name "EditClient"}
                    :venia/queries [{:query/data [:edit-client
                                                  {:edit-client new-client-req}
-                                                 [:id :name :signature-file :code :email :locations :matches :week-a-debits :week-a-credits :week-b-debits :week-b-credits
+                                                 (events/client-query user)]}]}
-                                                  [:location-matches [:location :match :id]]
-                                                  [:address [:street1 :street2 :city :state :zip]]
-                                                  [:forecasted-transactions [:id :amount :identifier :day-of-month]]
-                                                  [:bank-accounts [:id :number :start-date :check-number :name :code :bank-code :bank-name :routing :type :visible :yodlee-account-id :sort-order :locations]]]]}]}
        :on-success [::save-complete]
        :on-error [::forms/save-error ::form]}})))
 (re-frame/reg-event-db
@@ -238,6 +242,16 @@
   (= 0 (mod (t/in-weeks (t/interval first-week-a d)) 2)))
 
 
+(re-frame/reg-sub
+ ::yodlee-accounts
+ :<- [::subs/clients-by-id]
+ (fn [clients [_ id]]
+   
+   (if id
+     (mapcat :accounts (:yodlee-provider-accounts (get clients id) ))
+     [])))
+
+
 (defn bank-account-card [new-client {:keys [active? new? type visible code name number check-number id sort-order] :as bank-account} first? last?]
   (let [{:keys [form field raw-field error-notification submit-button ]} client-form]
     [:div.card {:style {:margin-bottom "1em"}}
@@ -340,8 +354,11 @@
             [:input.input {:placeholder "Yodlee Account #"
                            :type "text"
                            :field [:bank-accounts sort-order :yodlee-account-id]}]]
-
+           [field "Yodlee Account (new)"
-           ])
+            [typeahead-entity {:matches @(re-frame/subscribe [::yodlee-accounts (:id new-client)])
+                               :match->text (fn [m] (str (:name m) " - " (:number m)))
+                               :type "typeahead"
+                               :field [:bank-accounts sort-order :yodlee-account]}]]])
         (when (#{:credit ":credit"} type )
           [:div
 
@@ -364,7 +381,12 @@
            [field "Yodlee Account"
             [:input.input {:placeholder "Yodlee Account #"
                            :type "text"
-                           :field [:bank-accounts sort-order :yodlee-account-id]}]]])
+                           :field [:bank-accounts sort-order :yodlee-account-id]}]]
+           [field "Yodlee Account (new)"
+            [typeahead-entity {:matches @(re-frame/subscribe [::yodlee-accounts (:id new-client)])
+                               :match->text (fn [m] (str (:name m) " - " (:number m)))
+                               :type "typeahead"
+                               :field [:bank-accounts sort-order :yodlee-account]}]]])
         [:div.field
             [:label.label "Locations"]
             [:div.control

src/cljs/auto_ap/views/pages/admin/yodlee2.cljs

@@ -16,14 +16,17 @@
             [auto-ap.status :as status]
             [cljs.reader :as edn]
             [auto-ap.routes :as routes]
-            [bidi.bidi :as bidi]))
+            [bidi.bidi :as bidi]
+            [auto-ap.views.pages.admin.yodlee2.table :as table]
+            [auto-ap.views.pages.admin.yodlee2.form :as form]
+            [auto-ap.views.components.grid :as grid]
+            [auto-ap.effects.forward :as forward]))
+
+(comment 
+
 
 
   
-(re-frame/reg-sub
- ::authentication
- (fn [db]
-   (-> db ::yodlee :authentication)))
 
   (re-frame/reg-sub
    ::can-submit
@@ -55,16 +58,7 @@
    (fn [db]
      (-> db ::provider-accounts)))
 
-(re-frame/reg-event-fx
+  
- ::authenticate-with-yodlee
- (fn [{:keys [db]} _]
-   {:db (assoc-in db [::yodlee :loading?] true)
-    :http {:token (:user db)
-           :method :get 
-           :headers {"Content-Type" "application/edn"}
-           :uri (str "/api/yodlee2/fastlink")
-           :on-success [::authenticated]
-           :on-error [::save-error]}}))
 
   (re-frame/reg-event-fx
    ::mounted
@@ -115,12 +109,7 @@
               (assoc-in [::provider-accounts] accounts)
               (assoc-in [::provider-accounts-loading?] false))}))
 
-(re-frame/reg-event-fx
+  
- ::authenticated
- (fn [{:keys [db]} [_ authentication]]
-   {:db (-> db
-            (assoc-in [::yodlee :authentication] authentication)
-            (assoc-in [::yodlee :loading?] false))}))
 
   (re-frame/reg-event-fx
    ::authenticated-mfa
@@ -135,28 +124,7 @@
    (fn [{:keys [db]} [_ authentication]]
      {:db (assoc :db ::load-error "error")}))
 
-(defn yodlee-link-button []
-  [:div
-   (let [authentication @(re-frame/subscribe [::authentication])
-         loading? @(re-frame/subscribe [::loading?])]
   
-     (if authentication
-       [:div
-        "Authentication successful!"
-        [:div#fa-spot]
-        [:button.button.is-primary {:on-click (fn []
-                                                #_(println #js {"fastLinkUrl" (:url authentication)
-                                                          "accessToken" (:token authentication)
-                                                          "params" #js { "configName" "Aggregation"}})
-                                                (.open (.-fastlink js/window)
-                                                       (doto #js {"fastLinkURL" (:url authentication)
-                                                                  "accessToken" (:token authentication)
-                                                                  "params" #js { "configName" "Aggregation"}}
-                                                         println)
-                                                       "fa-spot")
-                                                )}[:span [:span.icon [:i.fa.fa-external-link]] " Go to yodlee"]]]
-       
-       [:button.button.is-primary {:class (if loading? "is-loading" "") :on-click (dispatch-event [::authenticate-with-yodlee])} "Authenticate with Yodlee"]))])
 
   (defn yodlee-date->date [d]
     (try
@@ -409,4 +377,123 @@
 
   (defn admin-yodlee-page []
     [side-bar-layout {:side-bar [admin-side-bar {}]
-                    :main [admin-yodlee-content]}])
+                      :main [admin-yodlee-content]}]))
+
+
+(re-frame/reg-sub
+ ::authentication
+ (fn [db]
+   (-> db ::authentication)))
+
+(re-frame/reg-sub
+ ::params
+ :<- [::table/params]
+ (fn [table-params]
+   table-params))
+
+(re-frame/reg-sub
+ ::yodlee-provider-accounts
+ (fn [db]
+   (::yodlee-provider-accounts db)))
+
+(re-frame/reg-event-fx
+ ::params-change
+ (fn [_ [_ params]]
+   {:set-uri-params params}))
+
+(re-frame/reg-sub
+ ::page
+ :<- [::params]
+ :<- [::yodlee-provider-accounts]
+ (fn [[params all-yodlee-provider-accounts]]
+   (assoc (grid/virtual-paginate-controls (:start params ) (:per-page params) (:yodlee-provider-accounts all-yodlee-provider-accounts) )
+          :data (grid/virtual-paginate (:start params) (:per-page params) (:yodlee-provider-accounts all-yodlee-provider-accounts)))))
+
+(re-frame/reg-event-fx
+ ::mounted
+ (fn [{:keys [db]} _]
+   {:graphql {:token (:user db)
+              :owns-state {:single ::page}
+              :query-obj {:venia/queries [[:yodlee-provider-account-page {:client-id (:id @(re-frame/subscribe [::subs/client]))}
+                                           [[:yodlee-provider-accounts [:id :last-updated :status :detailed-status
+                                                                                                      [:accounts [:id :name :number :available-balance]]]]
+                                                                          :count]]]} 
+              :on-success [::received]}
+    #_#_::forward/register {:id ::edited-yodlee-provider-account
+                        #_#_:events #{::form/saved}
+                        #_#_:event-fn (fn [[_ query-result]]
+                                        [::saved query-result])}
+    :db (dissoc db ::authentication)}))
+
+(re-frame/reg-event-fx
+ ::unmounted
+ (fn [{:keys [db]} _]
+   #_{::forward/dispose {:id ::edited-yodlee-provider-account}}))
+
+
+(re-frame/reg-event-fx
+ ::authenticate-with-yodlee
+ (fn [{:keys [db]} [_ client]]
+   {:http {:token (:user db)
+           :method :get 
+           :headers {"Content-Type" "application/edn"}
+           :uri (str "/api/yodlee2/fastlink?client=" client)
+           :owns-state {:single ::authenticating}
+           :on-success [::authenticated]
+           :on-error [::save-error]}}))
+
+(re-frame/reg-event-fx
+ ::authenticated
+ (fn [{:keys [db]} [_ authentication]]
+   {:db (-> db
+            (assoc-in [::authentication] authentication))}))
+
+(re-frame/reg-event-db
+ ::received
+ (fn [db [_ d]]
+   (assoc-in db [::yodlee-provider-accounts] (:yodlee-provider-account-page d))))
+
+
+(defn yodlee-provider-accounts-table []
+  [table/table {:page @(re-frame/subscribe [::page])
+                :status @(re-frame/subscribe [::status/single ::page])}])
+
+
+(defn yodlee-link-button []
+  [:div
+   (let [authentication @(re-frame/subscribe [::authentication])
+         status @(re-frame/subscribe [::status/single ::authenticating])
+         client-code (:code @(re-frame/subscribe [::subs/client]))]
+     (if (and authentication client-code)
+       [:div
+        "Authentication successful!"
+        [:div#fa-spot]
+        [:button.button.is-primary {:on-click (fn []
+                                                (.open (.-fastlink js/window)
+                                                       #js {"fastLinkURL" (:url authentication)
+                                                            "accessToken" (:token authentication)
+                                                            "params" #js { "configName" "Aggregation"}}
+                                                       
+                                                       "fa-spot")
+                                                )}
+         [:span [:span.icon [:i.fa.fa-external-link]] " Go to yodlee"]]]
+       [:button.button.is-primary {:disabled (status/disabled-for status)
+                                   :class (status/class-for status)
+                                   :on-click (dispatch-event [::authenticate-with-yodlee client-code])}
+        "Authenticate with Yodlee (" client-code ")"]))])
+
+(defn admin-yodlee-provider-accounts-content []
+  [:div
+   [:h1.title "Yodlee Provider Accounts"]
+   [yodlee-provider-accounts-table]
+   [yodlee-link-button]])
+
+
+(defn admin-yodle-provider-accounts-page []
+  (reagent/create-class
+   {:component-will-unmount #(re-frame/dispatch [::unmounted])
+    :component-did-mount #(re-frame/dispatch [::mounted])
+    :reagent-render (fn []
+                      [side-bar-layout {:side-bar [admin-side-bar {}]
+                                        :main [admin-yodlee-provider-accounts-content]}])}))
+

src/cljs/auto_ap/views/pages/admin/yodlee2/form.cljs

@@ -0,0 +1,4 @@
+(ns auto-ap.views.pages.admin.yodlee2.form)
+
+(defn form []
+  [:div])

src/cljs/auto_ap/views/pages/admin/yodlee2/table.cljs

@@ -0,0 +1,52 @@
+(ns auto-ap.views.pages.admin.yodlee2.table
+  (:require 
+   [clojure.string :as str]
+   [re-frame.core :as re-frame]
+   [auto-ap.views.utils :refer [action-cell-width date->str]]
+   [auto-ap.views.pages.admin.users.form :as form]
+   [auto-ap.views.components.buttons :as buttons]
+   [auto-ap.views.components.grid :as grid]))
+
+(re-frame/reg-event-fx
+ ::params-changed
+ (fn [{:keys [db]} [_ p]]
+   {:db (assoc db ::params p)}))
+
+(re-frame/reg-sub
+ ::params
+ (fn [db]
+   (-> db ::params)))
+
+(defn table [{:keys [status page]}]
+  (let [params @(re-frame/subscribe [::params])]
+    [grid/grid {:status status
+                :on-params-change (fn [p]
+                                    (re-frame/dispatch [::params-changed p]))
+                :params  params
+                :column-count 4}
+     [grid/controls page]
+     [grid/table {:fullwidth true}
+      [grid/header
+       [grid/row {}
+        [grid/header-cell {} "Provider Account"]
+        [grid/header-cell {} "Status"]
+        [grid/header-cell {} "Detailed Status"]
+        [grid/header-cell {} "Last Updated"]
+        [grid/header-cell {} "Accounts"]
+        [grid/header-cell {:style {:width (action-cell-width 1)}} ]]]
+      [grid/body
+       (for [{:keys [id name accounts status detailed-status last-updated clients] :as c} (:data page)]
+         ^{:key (str name "-" id )}
+         [grid/row {:class (:class c) :id id}
+          [grid/cell {} id]
+          [grid/cell {} status]
+          [grid/cell {} detailed-status]
+          [grid/cell {} (date->str last-updated)]
+          [grid/cell {}
+           [:ul
+            (for [a accounts]
+              ^{:key (:id a)}
+              [:li (:name a) " - " (:number a)])]]
+          [grid/cell {}
+           [buttons/fa-icon {:event [::form/editing c]
+                             :icon "fa-pencil"}]]])]]]))