From 789914b3f7a7e8e3aed4abddc140a7b9ecfa3947 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 8 Jul 2022 11:59:59 -0700 Subject: [PATCH] adds integration statuses to bank account pages. --- .clj-kondo/config.edn | 1 + .../clj_kondo/slingshot/try_plus.clj | 44 ++++++++++ .clj-kondo/slingshot/config.edn | 2 + src/clj/auto_ap/datomic/clients.clj | 16 +++- src/clj/auto_ap/datomic/migrate.clj | 10 +-- .../auto_ap/datomic/migrate/integrations.clj | 40 +++++++++ src/clj/auto_ap/graphql.clj | 9 ++ src/clj/auto_ap/graphql/checks.clj | 4 +- src/clj/auto_ap/graphql/clients.clj | 2 + src/clj/auto_ap/import/yodlee2.clj | 29 +++++- src/clj/auto_ap/square/core.clj | 88 ++++++++++--------- src/cljs/auto_ap/events.cljs | 3 +- .../views/pages/admin/clients/table.cljs | 44 ++++++++-- 13 files changed, 234 insertions(+), 58 deletions(-) create mode 100644 .clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj create mode 100644 .clj-kondo/slingshot/config.edn create mode 100644 src/clj/auto_ap/datomic/migrate/integrations.clj diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 56776b0b..5495176e 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -4,4 +4,5 @@ :private-call {:level :off} :mount/defstate {:level :warning}} + :config-paths ["slingshot"] :hooks {:analyze-call {mount.core/defstate hooks.defstate/defstate}} } diff --git a/.clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj b/.clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj new file mode 100644 index 00000000..48a4c174 --- /dev/null +++ b/.clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj @@ -0,0 +1,44 @@ +(ns clj-kondo.slingshot.try-plus + (:require [clj-kondo.hooks-api :as api])) + +(defn expand-catch [catch-node] + (let [[catch catchee & exprs] (:children catch-node) + catchee-sexpr (api/sexpr catchee)] + (cond (vector? catchee-sexpr) + (let [[selector & exprs] exprs] + (api/list-node + [catch (api/token-node 'Exception) (api/token-node '_e#) + (api/list-node + (list* (api/token-node 'let) + (api/vector-node [selector (api/token-node nil)]) + exprs))])) + :else catch-node))) + +(defn try+ [{:keys [node]}] + (let [children (rest (:children node)) + [body catches] + (loop [body children + body-exprs [] + catches []] + (if (seq body) + (let [f (first body) + f-sexpr (api/sexpr f)] + (if (and (seq? f-sexpr) (= 'catch (first f-sexpr))) + (recur (rest body) + body-exprs + (conj catches (expand-catch f))) + (recur (rest body) + (conj body-exprs f) + catches))) + [body-exprs catches])) + new-node (api/list-node + [(api/token-node 'let) + (api/vector-node + [(api/token-node '&throw-context) (api/token-node nil)]) + (api/token-node '&throw-context) ;; use throw-context to avoid warning + (with-meta (api/list-node (list* (api/token-node 'try) + (concat body catches))) + (meta node))])] + ;; (prn (api/sexpr new-node)) + {:node new-node})) + diff --git a/.clj-kondo/slingshot/config.edn b/.clj-kondo/slingshot/config.edn new file mode 100644 index 00000000..446d4f0c --- /dev/null +++ b/.clj-kondo/slingshot/config.edn @@ -0,0 +1,2 @@ +{:hooks + {:analyze-call {slingshot.slingshot/try+ clj-kondo.slingshot.try-plus/try+}}} diff --git a/src/clj/auto_ap/datomic/clients.clj b/src/clj/auto_ap/datomic/clients.clj index 4be5c952..413158ed 100644 --- a/src/clj/auto_ap/datomic/clients.clj +++ b/src/clj/auto_ap/datomic/clients.clj @@ -9,6 +9,9 @@ (assoc :client/yodlee-provider-accounts (get e :yodlee-provider-account/_client)) (assoc :client/plaid-items (get e :plaid-item/_client)) (update :client/locked-until #(some-> % coerce/to-date-time)) + (update-in [:client/square-integration-status :integration-status/state] :db/ident) + (update-in [:client/square-integration-status :integration-status/last-attempt] #(some-> % coerce/to-date-time)) + (update-in [:client/square-integration-status :integration-status/last-updated] #(some-> % coerce/to-date-time)) (update :client/location-matches (fn [lms] (map #(assoc % :location-match/match (first (:location-match/matches %))) lms))) @@ -17,11 +20,18 @@ (map (fn [i ba] (-> ba (update :bank-account/type :db/ident ) + (update-in [:bank-account/integration-status :integration-status/state] :db/ident) + (update-in [:bank-account/integration-status :integration-status/last-attempt] #(some-> % coerce/to-date-time)) + (update-in [:bank-account/integration-status :integration-status/last-updated] #(some-> % coerce/to-date-time)) (update :bank-account/start-date #(some-> % (coerce/to-date-time))) (update :bank-account/sort-order (fn [so] (or so i))))) (range) bas))))) (defn get-all [] (->> (d/q '[:find (pull ?e [* + {:client/square-integration-status [:integration-status/message + :integration-status/last-attempt + :integration-status/last-updated + {:integration-status/state [:db/ident]}]} {:client/address [*]} {:client/square-locations [:square-location/square-id :square-location/name @@ -30,7 +40,11 @@ {:client/bank-accounts [* {:bank-account/type [*] :bank-account/yodlee-account [:yodlee-account/name :yodlee-account/id :yodlee-account/number] :bank-account/plaid-account [:plaid-account/name :db/id :plaid-account/number :plaid-account/balance] - :bank-account/intuit-bank-account [:intuit-bank-account/name :intuit-bank-account/external-id :db/id]} + :bank-account/intuit-bank-account [:intuit-bank-account/name :intuit-bank-account/external-id :db/id] + :bank-account/integration-status [:integration-status/message + :integration-status/last-attempt + :integration-status/last-updated + {:integration-status/state [:db/ident]}]} ]} {:yodlee-provider-account/_client [*]} {:plaid-item/_client [*]} diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 451317d0..da9831ac 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -2,15 +2,14 @@ (:gen-class) (:require [auto-ap.datomic :refer [conn]] - [auto-ap.datomic.migrate.add-bank-account-codes - :refer [add-bank-account-codes]] - [auto-ap.datomic.migrate.add-client-codes :refer [add-client-codes]] + [auto-ap.datomic.migrate.add-bank-account-codes] + [auto-ap.datomic.migrate.add-client-codes] [auto-ap.datomic.migrate.add-general-ledger :as add-general-ledger] + [auto-ap.datomic.migrate.integrations :as integrations] [auto-ap.datomic.migrate.audit :as audit] [auto-ap.datomic.migrate.clients :as clients] [auto-ap.datomic.migrate.reports :as reports] - [auto-ap.datomic.migrate.invoice-converter - :refer [add-import-status-existing-invoices]] + [auto-ap.datomic.migrate.invoice-converter] [auto-ap.datomic.migrate.ledger :as ledger] [auto-ap.datomic.migrate.queries :as queries] [auto-ap.datomic.migrate.plaid :as plaid] @@ -531,6 +530,7 @@ clients/norms-map ledger/norms-map yodlee2/norms-map + integrations/norms-map reports/norms-map plaid/norms-map audit/norms-map diff --git a/src/clj/auto_ap/datomic/migrate/integrations.clj b/src/clj/auto_ap/datomic/migrate/integrations.clj new file mode 100644 index 00000000..b6d94414 --- /dev/null +++ b/src/clj/auto_ap/datomic/migrate/integrations.clj @@ -0,0 +1,40 @@ +(ns auto-ap.datomic.migrate.integrations) + +(def norms-map {::add-integration-status3 + {:txes [[{:db/ident :bank-account/integration-status + :db/doc "A status for integration for the bank account" + :db/valueType :db.type/ref + :db/isComponent true + :db/cardinality :db.cardinality/one} + + {:db/ident :client/square-integration-status + :db/doc "Square's integration status" + :db/valueType :db.type/ref + :db/isComponent true + :db/cardinality :db.cardinality/one} + + {:db/ident :integration-status/last-updated + :db/doc "When was this integration updated" + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/noHistory true} + + {:db/ident :integration-status/last-attempt + :db/doc "When was this integration attempted." + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/noHistory true} + + {:db/ident :integration-status/state + :db/doc "A status for the integration" + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + + {:db/ident :integration-status/message + :db/doc "A message from the last attempt" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + + {:db/ident :integration-state/failed} + {:db/ident :integration-state/success} + {:db/ident :integration-state/unauthorized}]]}}) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index c1b68d20..deae12d2 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -114,6 +114,12 @@ :last_updated {:type :iso_date} :accounts {:type '(list :yodlee_account)}}} + :integration_status + {:fields {:last_attempt {:type :iso_date} + :last_updated {:type :iso_date} + :message {:type 'String} + :state {:type :integration_state}}} + :yodlee_account {:fields {:id {:type 'Int} :status {:type 'String} @@ -501,6 +507,9 @@ {:enum-value :doordash} {:enum-value :uber_eats} {:enum-value :grubhub}]} + :integration_state {:values [{:enum-value :failed} + {:enum-value :success} + {:enum-value :unauthorized}]} :tin_type {:values [{:enum-value :ein} {:enum-value :ssn}]} :type_1099 {:values [{:enum-value :none} diff --git a/src/clj/auto_ap/graphql/checks.clj b/src/clj/auto_ap/graphql/checks.clj index 4e9c3551..a39398e0 100644 --- a/src/clj/auto_ap/graphql/checks.clj +++ b/src/clj/auto_ap/graphql/checks.clj @@ -259,8 +259,8 @@ :check (str (+ index (:bank-account/check-number bank-account))) :memo memo :date (date->str (local-now)) - :client (dissoc client :client/bank-accounts :client/locked-until :client/emails :client/square-auth-token :client/square-locations) - :bank-account (dissoc bank-account :bank-account/start-date) + :client (dissoc client :client/bank-accounts :client/square-integration-status :client/locked-until :client/emails :client/square-auth-token :client/square-locations) + :bank-account (dissoc bank-account :bank-account/start-date :bank-account/integration-status) #_#_:client {:name (:name client) :address (:address client) :signature-file (:signature-file client) diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index 5b990bfc..60b14676 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -441,6 +441,7 @@ :code {:type 'String} :square_auth_token {:type 'String} :signature_file {:type 'String} + :square_integration_status {:type :integration_status} :week_a_debits {:type :money} :week_a_credits {:type :money} :week_b_debits {:type :money} @@ -459,6 +460,7 @@ :bank_account {:fields {:id {:type :id} + :integration_status {:type :integration_status} :type {:type :ident} :start_date {:type :iso_date} :number {:type 'String} diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj index f581d59f..bc10191d 100644 --- a/src/clj/auto_ap/import/yodlee2.clj +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -9,7 +9,31 @@ [datomic.api :as d] [mount.core :as mount] [unilog.context :as lc] - [yang.scheduler :as scheduler])) + [yang.scheduler :as scheduler] + [clojure.tools.logging :as log])) + +(defn bank-account->integration-id [bank-account] + (or (->> bank-account + (d/pull (d/db conn) [:bank-account/integration-status]) + :bank-account/integration-status + :db/id) + #db/id[:db.part/user])) + +(defn get-client-transactions-or-mark-integration-fail [client-code yodlee-account bank-account] + (try + (let [result (client2/get-specific-transactions client-code yodlee-account)] + @(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account) + :integration-status/state :integration-state/success + :integration-status/last-attempt (java.util.Date.) + :integration-status/last-updated (java.util.Date.)}}]) + result) + (catch Exception e + @(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account) + :integration-status/state :integration-state/failed + :integration-status/last-attempt (java.util.Date.) + :integration-status/message (.getMessage e)}}]) + (log/warn e) + []))) (defn import-yodlee2 [] (lc/with-context {:source "Import yodlee2 transactions"} @@ -30,7 +54,8 @@ ] (d/db conn))] (doseq [[yodlee-account bank-account client-code use-date-instead-of-post-date?] account-lookup - transaction (client2/get-specific-transactions client-code yodlee-account)] + transaction (get-client-transactions-or-mark-integration-fail client-code yodlee-account bank-account)] + (println transaction) (t/import-transaction! import-batch (assoc (y/yodlee->transaction transaction use-date-instead-of-post-date?) :transaction/bank-account bank-account :transaction/client [:client/code client-code]))) diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index ee4054a9..73dd31fc 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -14,7 +14,8 @@ [mount.core :as mount] [unilog.context :as lc] [yang.scheduler :as scheduler] - [clojure.core.async :as async])) + [clojure.core.async :as async] + [slingshot.slingshot :refer [try+]])) (defn client-base-headers [client] {"Square-Version" "2021-08-18" @@ -385,22 +386,19 @@ (upsert client square-location (time/plus (time/now) (time/days -3)) (time/now)))) ([client location start end] (lc/with-context {:source "Square loading"} - (try - (let [existing (->> (d/query {:query {:find ['?external-id] - :in ['$ '?client] - :where ['[?o :sales-order/client ?client] - '[?o :sales-order/external-id ?external-id]]} - :args [(d/db conn) (:db/id client)]}) - (map first) - set) - _ (log/info (count existing) "Sales orders already exist") - to-create (filter #(not (existing (:sales-order/external-id %))) - (daily-results client location start end))] - (doseq [x (partition-all 20 to-create)] - (log/info "Loading " (count x)) - @(d/transact conn x))) - (catch Exception e - (log/error e)))))) + (let [existing (->> (d/query {:query {:find ['?external-id] + :in ['$ '?client] + :where ['[?o :sales-order/client ?client] + '[?o :sales-order/external-id ?external-id]]} + :args [(d/db conn) (:db/id client)]}) + (map first) + set) + _ (log/info (count existing) "Sales orders already exist") + to-create (filter #(not (existing (:sales-order/external-id %))) + (daily-results client location start end))] + (doseq [x (partition-all 20 to-create)] + (log/info "Loading " (count x)) + @(d/transact conn x)))))) (defn upsert-settlements ([client] @@ -409,12 +407,9 @@ (upsert-settlements client square-location))) ([client location] (lc/with-context {:source "Square settlements loading"} - (try - (doseq [x (partition-all 20 (daily-settlements client location))] - (log/info "Loading expected deposit" (count x)) - @(d/transact conn x)) - (catch Exception e - (log/error e))) + (doseq [x (partition-all 20 (daily-settlements client location))] + (log/info "Loading expected deposit" (count x)) + @(d/transact conn x)) (log/info "Done loading settlements")))) (defn upsert-refunds @@ -426,12 +421,9 @@ (lc/with-context {:source "Loading Square Settlements" :client (:client/code client) :location (:square-location/client-location client)} - (try - (doseq [x (partition-all 20 (refunds client location))] - (log/info "Loading refund" (count x)) - @(d/transact conn x)) - (catch Exception e - (log/error e))) + (doseq [x (partition-all 20 (refunds client location))] + (log/info "Loading refund" (count x)) + @(d/transact conn x)) (log/info "Done loading refunds")))) (def square-read [:db/id @@ -442,6 +434,7 @@ (defn get-square-clients ([] (d/q '[:find [(pull ?c [:db/id + :client/square-integration-status :client/code :client/square-auth-token {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] @@ -487,17 +480,33 @@ (map first) (map (fn [x] [:db/retractEntity x])))) -(defn upsert-all [] - (doseq [client (get-square-clients) +(defn mark-integration-status [client integration-status] + @(d/transact conn + [{:db/id (:db/id client) + :client/square-integration-status (assoc integration-status + :db/id (or (-> client :client/square-integration-status :db/id) + #db/id [:db.part/user]))}])) + +(defn upsert-all [ & clients] + (doseq [client (apply get-square-clients clients) :when (seq (filter :square-location/client-location (:client/square-locations client)))] (lc/with-context {:client (:client/code client)} - (upsert-locations client) - (log/info "Loading Orders") - (upsert client) - (log/info "Loading Settlements") - (upsert-settlements client) - (log/info "Loading refunds") - (upsert-refunds client)))) + (mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))}) + (try+ + (upsert-locations client) + (upsert client) + (upsert-settlements client) + (upsert-refunds client) + (mark-integration-status client {:integration-status/state :integration-state/success + :integration-status/last-updated (coerce/to-date (time/now))}) + + (catch [:status 401] data + (mark-integration-status client {:integration-status/state :integration-state/unauthorized + :integration-status/message (-> data :body )})) + (catch Exception e + (log/warn e) + (mark-integration-status client {:integration-status/state :integration-state/failed + :integration-status/message (.getMessage e)})))))) (mount/defstate square-loader :start (scheduler/every (* 4 59 60 1000) (heartbeat upsert-all "square-loading")) @@ -505,6 +514,3 @@ - - - diff --git a/src/cljs/auto_ap/events.cljs b/src/cljs/auto_ap/events.cljs index 028a2654..b5a2456c 100644 --- a/src/cljs/auto_ap/events.cljs +++ b/src/cljs/auto_ap/events.cljs @@ -16,10 +16,12 @@ (defn client-query [token] (cond-> [:id :name :signature-file :code :email :matches :week-a-debits :week-a-credits :week-b-debits :week-b-credits :locations :locked-until :square-auth-token + [:square-integration-status [:last-updated :last-attempt :message :state]] [:square-locations [:square-id :id :name :client-location]] [:emails [:id :email :description]] [:location-matches [:id :location :match]] [:bank-accounts [:id :start-date :numeric-code :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id + [:integration-status [:last-updated :last-attempt :message :state]] [:yodlee-account [:name :id :number]] [:plaid-account [:name :id :number]] [:intuit-bank-account [:name :id :external-id]] @@ -88,7 +90,6 @@ (re-frame/reg-event-fx ::received-initial (fn [{:keys [db]} [_ {clients :client}]] - {:db (-> db (assoc :clients (by :id clients) ) (assoc :is-initial-loading? false) diff --git a/src/cljs/auto_ap/views/pages/admin/clients/table.cljs b/src/cljs/auto_ap/views/pages/admin/clients/table.cljs index f60bb84d..a369d68e 100644 --- a/src/cljs/auto_ap/views/pages/admin/clients/table.cljs +++ b/src/cljs/auto_ap/views/pages/admin/clients/table.cljs @@ -49,6 +49,32 @@ (merge (select-keys query-params #{:start :sort}) specific-params ))) +(defn integration-status-badge [name status] + (condp = (:state status) + :success + [:div.tag.has-tooltip-right.has-tooltip-arrow {:data-tooltip (str "Last updated:" (date->str (:last-updated status)) + "\n" + "Last Attempted:" (date->str (:last-attempt status)))} [:span.icon [:i.has-text-success.fa.fa-check]] [:span name]] + + :failed + [:div.tag.is-danger.is-light.has-tooltip-right.has-tooltip-arrow {:data-tooltip (str "Last updated:" (date->str (:last-updated status)) + "\n" + "Last Attempted:" (date->str (:last-attempt status)) + "\n" + (:message status)) + } [:span.icon [:i.has-text-danger.fa.fa-warning]] [:span name]] + + :unauthorized + [:div.tag.is-danger.is-light.has-tooltip-right.has-tooltip-arrow {:data-tooltip (str "Last updated:" (date->str (:last-updated status)) + "\n" + "Last Attempted:" (date->str (:last-attempt status)) + "\n" + "Your user is unauthorized. Detail:\n" + (:message status)) + } [:span.icon [:i.has-text-danger.fa.fa-warning]] [:span name]] + nil + )) + (defn clients-table [{:keys [page status]}] (let [states @(re-frame/subscribe [::status/multi ::setup-sales-queries])] [grid/grid {:on-params-change (fn [p] @@ -63,21 +89,27 @@ [grid/header-cell {} "Name"] [grid/header-cell {:style {:width "20em"}} "Code"] [grid/header-cell {} "Locations"] - [grid/header-cell {} "Locked Until"] + [grid/header-cell {} "Status"] [grid/header-cell {} "Email"] [grid/header-cell {:style {:width (action-cell-width 2)}}]] ] [grid/body - (for [{:keys [id name email locked-until code locations]} (:data page)] + (for [{:keys [id name email square-integration-status locked-until code locations bank-accounts]} (:data page)] ^{:key (str name "-" id )} [grid/row {:id id} [grid/cell {} name] [grid/cell {} code] [grid/cell {} (str/join ", " locations)] - [grid/cell {} [:div.tag (or (some-> locked-until date->str) - "Not locked" - - )]] + [grid/cell {} [:div.tags + + [:div.tag (or (some-> locked-until date->str (#(str "Locked " %))) "Not locked")] + [integration-status-badge "Square" square-integration-status] + [:<> + (for [bank-account bank-accounts + :let [code (:code bank-account) + integration-status (:integration-status bank-account)] + :when integration-status] + [integration-status-badge code integration-status])]]] [grid/cell {} email] [grid/cell {} [:div.buttons [buttons/fa-icon {:event [::setup-sales-queries id] :class (status/class-for (get states id))