notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Makes logging unified

Browse Source
This commit is contained in:
Bryce
2023-10-30 12:35:18 -07:00
parent f0a7c378f7
commit 930b900849
44 changed files with 485 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
src/clj/auto_ap/square/core.clj
View File

@@ -11,10 +11,9 @@
[clojure.data.json :as json]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.api :as dc]
[slingshot.slingshot :refer [try+]]
[unilog.context :as lc]))
))
(defn client-base-headers [client]
{"Square-Version" "2021-08-18"
@@ -22,7 +21,6 @@
"Content-Type" "application/json"})
(defn retry-4 [ex try-count _]
(log/warn "Retrying after failure " ex)
(if (> try-count 4) false true))
(defn lookup-dates []
@@ -42,13 +40,12 @@
:body
:locations)
(catch Exception e
(log/warn e)
[])))
(defn fetch-catalog [client i]
(if i
(try
(log/trace "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i))
(->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i)
{:headers (client-base-headers client)
:query-params {"include_related_items" "true"}
@@ -56,9 +53,7 @@
:body
:object)
(catch Exception e
(log/error e)
nil))
(log/warn "Trying to look up non existant ")))
nil))))
(def fetch-catalog-fast (memoize fetch-catalog))
@@ -75,7 +70,7 @@
:else
(do
(log/warn "couldn't look up" i)
"Uncategorized"))))
(defn pc [start end]
@@ -98,7 +93,7 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-order
([client location order-id]
(log/info "Searching for" (:square-location/client-location location))
(let [result (->> (client/get (str "https://connect.squareup.com/v2/orders/" order-id)
{:headers (client-base-headers client)
:as :json})
@@ -107,7 +102,7 @@
result)))
(defn continue-search [client location start end cursor]
(log/info "Continuing search for" cursor)
(let [result (->> (client/post "https://connect.squareup.com/v2/orders/search"
{:headers (client-base-headers client)
:body (json/write-str (cond-> {"location_ids" [(:square-location/square-id location)]
@@ -116,21 +111,21 @@
start (merge (pc start end))))
:as :json})
:body)]
(log/info "found " (count (:orders result)))
(if (not-empty (:cursor result))
(concat (:orders result) (continue-search client location start end (:cursor result)))
(:orders result))))
(defn search
([client location start end]
(log/info "Searching for" (:square-location/client-location location))
(let [result (->> (client/post "https://connect.squareup.com/v2/orders/search"
{:headers (client-base-headers client)
:body (json/write-str (cond-> {"location_ids" [(:square-location/square-id location)] "limit" 10000}
start (merge (pc start end))))
:as :json})
:body)]
(log/info "found " (count (:orders result)))
(if (not-empty (:cursor result))
(concat (:orders result) (continue-search client location start end (:cursor result)))
(:orders result)))))
@@ -247,9 +242,8 @@
(try
(f)
(catch Exception e
(log/warn "error pulling http " e)
(retry f (inc i))))
(log/warn "Too many failures"))))
(retry f (inc i)))))))
(defn get-payment [client p]
(:payment (:body (retry #(client/get (str "https://connect.squareup.com/v2/payments/" p)
@@ -267,13 +261,11 @@
(async/pipeline-blocking concurrent
output-chan
(map (fn [p]
(lc/with-context {:source "Square settlements loading "}
(log/trace "looking up payment " p " for settlement " (:id settlement))
(or
(-> (get-payment client p)
:created_at
coerce/to-date)
(coerce/to-date (time/now))))))
(or
(-> (get-payment client p)
:created_at
coerce/to-date)
(coerce/to-date (time/now)))))
(async/to-chan! (->> settlement
:entries
(filter #(= "CHARGE" (:type %)))
@@ -284,9 +276,7 @@
))
true
(fn [e]
(lc/with-context {:source "Square settlements loading "}
(log/warn "Error loading sales date details" e)
e)))
e))
(->> (async/<!! (async/into [] output-chan))
(map halt-if-error)
sort
@@ -294,38 +284,32 @@
first)))
(defn get-settlement-details [client location settlements] ;; pairs of [location settlement]
(log/info "getting settlement details for " settlements)
(let [concurrent 10
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [s]
(lc/with-context {:source "Square settlements loading "}
(log/info "Looking up settlement " s " for location " (:square-location/client-location location))
(let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" s)
{:headers (client-base-headers client)
:as :json
:retry-handler retry-4})))
sales-date (get-settlement-sales-date client settlement)]
(log/info "sales date for " s " is " sales-date)
(assoc settlement :sales-date sales-date)))))
(let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" s)
{:headers (client-base-headers client)
:as :json
:retry-handler retry-4})))
sales-date (get-settlement-sales-date client settlement)]
(assoc settlement :sales-date sales-date))))
(async/to-chan! settlements)
true
(fn [e]
(lc/with-context {:source "Square settlements loading "}
(log/warn "Error loading settlements details" e)
e)))
e))
(->> (async/<!! (async/into [] output-chan))
(map halt-if-error))))
(defn settlements
([client location] (settlements client location (lookup-dates)))
([client location lookup-dates]
(log/info "Searching for" (:square-location/client-location location))
(->> lookup-dates
(mapcat (fn [[start-date end-date]]
(log/info "looking up settlements for " (:square-location/client-location location) " on dates " start-date " to " end-date)
(let [settlements (->> (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements")
{:headers (client-base-headers client)
:query-params {"begin_time" start-date
@@ -394,20 +378,17 @@
:when (:square-location/client-location square-location)]
(upsert client square-location (time/plus (time/now) (time/days -3)) (time/now))))
([client location start end]
(lc/with-context {:source "Square loading"}
(let [existing (->> (dc/q {:find ['?external-id]
:in ['$ '?client]
:where ['[?o :sales-order/client ?client]
'[?o :sales-order/external-id ?external-id]]}
(dc/db conn) (:db/id client))
(map first)
set)
_ (log/info (count existing) "Sales orders already exist")
to-create (filter #(not (existing (:sales-order/external-id %)))
(daily-results client location start end))]
(doseq [x (partition-all 20 to-create)]
(log/info "Loading " (count x))
@(dc/transact-async conn x))))))
(let [existing (->> (dc/q {:find ['?external-id]
:in ['$ '?client]
:where ['[?o :sales-order/client ?client]
'[?o :sales-order/external-id ?external-id]]}
(dc/db conn) (:db/id client))
(map first)
set)
to-create (filter #(not (existing (:sales-order/external-id %)))
(daily-results client location start end))]
(doseq [x (partition-all 20 to-create)]
@(dc/transact-async conn x)))))
(defn upsert-settlements
([client]
@@ -415,12 +396,9 @@
:when (:square-location/client-location square-location)]
(upsert-settlements client square-location)))
([client location]
(lc/with-context {:source "Square settlements loading"
:client (:client/code client)}
(doseq [x (partition-all 20 (daily-settlements client location))]
(log/info "Loading expected deposit" (count x))
@(dc/transact-async conn x))
(log/info "Done loading settlements"))))
(doseq [x (partition-all 20 (daily-settlements client location))]
@(dc/transact-async conn x))))
(defn upsert-refunds
([client]
@@ -428,13 +406,9 @@
:when (:square-location/client-location square-location)]
(upsert-refunds client square-location)))
([client location]
(lc/with-context {:source "Loading Square Settlements"
:client (:client/code client)
:location (:square-location/client-location client)}
(doseq [x (partition-all 20 (refunds client location))]
(log/info "Loading refund" (count x))
@(dc/transact-async conn x))
(log/info "Done loading refunds"))))
(doseq [x (partition-all 20 (refunds client location))]
@(dc/transact-async conn x))))
(def square-read [:db/id
:client/code
@@ -501,29 +475,27 @@
(defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients)
:when (seq (filter :square-location/client-location (:client/square-locations client)))]
(lc/with-context {:client (:client/code client)}
(mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))})
(mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))})
(try+
(upsert-locations client)
(upsert client)
#_(upsert-settlements client) ;; currently settlements v1 is broken
(upsert-refunds client)
(mark-integration-status client {:integration-status/state :integration-state/success
:integration-status/last-updated (coerce/to-date (time/now))})
(try+
(upsert-locations client)
(upsert client)
#_(upsert-settlements client) ;; currently settlements v1 is broken
(upsert-refunds client)
(mark-integration-status client {:integration-status/state :integration-state/success
:integration-status/last-updated (coerce/to-date (time/now))})
(catch [:status 401] data
(mark-integration-status client {:integration-status/state :integration-state/unauthorized
:integration-status/message (-> data :body str)}))
(catch [:status 503] data
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (-> data :body str)}))
(catch Object _
(log/warn &throw-context)
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (or (some-> (:wrapper &throw-context) (.getMessage ))
(some-> (:object &throw-context) str)
"Unknown error")}))))))
(catch [:status 401] data
(mark-integration-status client {:integration-status/state :integration-state/unauthorized
:integration-status/message (-> data :body str)}))
(catch [:status 503] data
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (-> data :body str)}))
(catch Object _
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (or (some-> (:wrapper &throw-context) (.getMessage ))
(some-> (:object &throw-context) str)
"Unknown error")})))))

119
src/clj/auto_ap/square/core2.clj
View File

@@ -11,11 +11,10 @@
[clojure.data.json :as json]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[cemerick.url :as url]
[datomic.api :as dc]
[slingshot.slingshot :refer [try+]]
[unilog.context :as lc]))
))
(defn client-base-headers [client]
{"Square-Version" "2021-08-18"
@@ -23,7 +22,6 @@
"Content-Type" "application/json"})
(defn retry-4 [ex try-count _]
(log/warn "Retrying after failure " ex)
(if (> try-count 4) false true))
(defn lookup-dates []
@@ -43,13 +41,11 @@
:body
:locations)
(catch Exception e
(log/warn e)
[])))
(defn fetch-catalog [client i]
(if i
(try
(log/trace "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i))
(->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i)
{:headers (client-base-headers client)
:query-params {"include_related_items" "true"}
@@ -57,9 +53,7 @@
:body
:object)
(catch Exception e
(log/error e)
nil))
(log/warn "Trying to look up non existant ")))
nil))))
(def fetch-catalog-fast (memoize fetch-catalog))
@@ -76,7 +70,7 @@
:else
(do
(log/warn "couldn't look up" i)
"Uncategorized"))))
(defn pc [start end]
@@ -99,7 +93,7 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-order
([client location order-id]
(log/info "Searching for" (:square-location/client-location location))
(let [result (->> (client/get (str "https://connect.squareup.com/v2/orders/" order-id)
{:headers (client-base-headers client)
:as :json})
@@ -108,7 +102,7 @@
result)))
(defn continue-search [client location start end cursor]
(log/info "Continuing search for" cursor)
(let [result (->> (client/post "https://connect.squareup.com/v2/orders/search"
{:headers (client-base-headers client)
:body (json/write-str (cond-> {"location_ids" [(:square-location/square-id location)]
@@ -117,21 +111,21 @@
start (merge (pc start end))))
:as :json})
:body)]
(log/info "found " (count (:orders result)))
(if (not-empty (:cursor result))
(concat (:orders result) (continue-search client location start end (:cursor result)))
(:orders result))))
(defn search
([client location start end]
(log/info "Searching for" (:square-location/client-location location))
(let [result (->> (client/post "https://connect.squareup.com/v2/orders/search"
{:headers (client-base-headers client)
:body (json/write-str (cond-> {"location_ids" [(:square-location/square-id location)] "limit" 10000}
start (merge (pc start end))))
:as :json})
:body)]
(log/info "found " (count (:orders result)))
(if (not-empty (:cursor result))
(concat (:orders result) (continue-search client location start end (:cursor result)))
(:orders result)))))
@@ -268,9 +262,9 @@
(try
(f)
(catch Exception e
(log/warn "error pulling http " e)
(retry f (inc i))))
(log/warn "Too many failures"))))
)))
(defn get-payment [client p]
(:payment (:body (retry #(client/get (str "https://connect.squareup.com/v2/payments/" p)
@@ -283,34 +277,30 @@
x))
(defn get-settlement-details [client location settlements] ;; pairs of [location settlement]
(log/info "getting settlement details for " settlements)
(let [concurrent 4
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [s]
(lc/with-context {:source "Square settlements loading "}
(log/info "Looking up settlement " s " for location " (:square-location/client-location location))
(:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" (:id s))
{:headers (client-base-headers client)
:as :json
:retry-handler retry-4}))))))
(:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" (:id s))
{:headers (client-base-headers client)
:as :json
:retry-handler retry-4})))))
(async/to-chan! settlements)
true
(fn [e]
(lc/with-context {:source "Square settlements loading "}
(log/warn "Error loading settlements details" e)
e)))
e))
(->> (async/<!! (async/into [] output-chan))
(map halt-if-error))))
(defn settlements
([client location] (settlements client location (lookup-dates)))
([client location lookup-dates]
(log/info "Searching for" (:square-location/client-location location))
(->> lookup-dates
(mapcat (fn [[start-date end-date]]
(log/info "looking up settlements for " (:square-location/client-location location) " on dates " start-date " to " end-date)
(let [settlements (->> (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements")
{:headers (client-base-headers client)
:query-params {"begin_time" start-date
@@ -396,10 +386,9 @@
:when (:square-location/client-location square-location)]
(upsert client square-location (time/plus (time/now) (time/days -45)) (time/now))))
([client location start end]
(lc/with-context {:source "Square loading"}
(doseq [x (partition-all 20 (daily-results client location start end))]
(log/info "Loading " (count x))
@(dc/transact conn x)))))
(doseq [x (partition-all 20 (daily-results client location start end))]
@(dc/transact conn x))))
(defn upsert-settlements
([client]
@@ -407,12 +396,9 @@
:when (:square-location/client-location square-location)]
(upsert-settlements client square-location)))
([client location]
(lc/with-context {:source "Square settlements loading"
:client (:client/code client)}
(doseq [x (partition-all 20 (daily-settlements client location))]
(log/info "Loading expected deposit" (count x))
@(dc/transact conn x))
(log/info "Done loading settlements"))))
(doseq [x (partition-all 20 (daily-settlements client location))]
@(dc/transact conn x))))
(defn upsert-refunds
([client]
@@ -420,13 +406,9 @@
:when (:square-location/client-location square-location)]
(upsert-refunds client square-location)))
([client location]
(lc/with-context {:source "Loading Square Settlements"
:client (:client/code client)
:location (:square-location/client-location client)}
(doseq [x (partition-all 20 (refunds client location))]
(log/info "Loading refund" (count x))
@(dc/transact conn x))
(log/info "Done loading refunds"))))
(doseq [x (partition-all 20 (refunds client location))]
@(dc/transact conn x))))
(def square-read [:db/id
:client/code
@@ -493,32 +475,30 @@
(defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients)
:when (seq (filter :square-location/client-location (:client/square-locations client)))]
(lc/with-context {:client (:client/code client)}
(log/info "Importing square2 " (:client/code client))
(mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))})
(mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))})
(try+
(upsert-locations client)
(upsert client)
(upsert-settlements client)
(upsert-refunds client)
(mark-integration-status client {:integration-status/state :integration-state/success
:integration-status/last-updated (coerce/to-date (time/now))})
(try+
(upsert-locations client)
(upsert client)
(upsert-settlements client)
(upsert-refunds client)
(mark-integration-status client {:integration-status/state :integration-state/success
:integration-status/last-updated (coerce/to-date (time/now))})
(catch [:status 401] data
(mark-integration-status client {:integration-status/state :integration-state/unauthorized
:integration-status/message (-> data :body str)}))
(catch [:status 401] data
(mark-integration-status client {:integration-status/state :integration-state/unauthorized
:integration-status/message (-> data :body str)}))
(catch [:status 503] data
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (-> data :body str)}))
(catch Object _
(catch [:status 503] data
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (-> data :body str)}))
(catch Object _
(log/warn &throw-context)
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (or (some-> (:wrapper &throw-context) (.getMessage ))
(some-> (:object &throw-context) str)
"Unknown error")}))))))
(mark-integration-status client {:integration-status/state :integration-state/failed
:integration-status/message (or (some-> (:wrapper &throw-context) (.getMessage ))
(some-> (:object &throw-context) str)
"Unknown error")})))))
(defn preview-changes
([client]
@@ -528,5 +508,4 @@
:when (:square-location/client-location square-location)]
(preview-changes client square-location (time/plus (time/now) (time/days -30)) (time/now)))))
([client location start end]
(lc/with-context {:source "Square loading"}
(daily-results client location start end))))
(daily-results client location start end)))