From 0c08e6a92979cd93cf8386ff5b00dcbf4c7f4941 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 6 Jan 2023 17:37:01 -0800 Subject: [PATCH] fixes sales queries, minor logging tweaks --- src/clj/auto_ap/background/metrics.clj | 6 +- src/clj/auto_ap/logging.clj | 22 +++++- src/clj/auto_ap/square/core3.clj | 100 +++++++++++-------------- src/clj/user.clj | 4 +- 4 files changed, 74 insertions(+), 58 deletions(-) diff --git a/src/clj/auto_ap/background/metrics.clj b/src/clj/auto_ap/background/metrics.clj index 950a030d..8f74498e 100644 --- a/src/clj/auto_ap/background/metrics.clj +++ b/src/clj/auto_ap/background/metrics.clj @@ -36,9 +36,13 @@ (when (seq container-data) (lc/push-context "container" (:DockerId container-data)) (lc/push-context "ip" (-> container-data :Networks first :IPv4Addresses first)) + + (mu/start-publisher! {:type :console-json}) (mu/set-global-context! {:container (:DockerId container-data) - :ip (-> container-data :Networks first :IPv4Addresses first)}))) + :ip (-> container-data :Networks first :IPv4Addresses first) + :env (:dd-env env) + :service (:dd-service env)}))) (defn stop-logging-context [] (when (seq container-data) diff --git a/src/clj/auto_ap/logging.clj b/src/clj/auto_ap/logging.clj index c429ae7e..5f1a5ed1 100644 --- a/src/clj/auto_ap/logging.clj +++ b/src/clj/auto_ap/logging.clj @@ -1,6 +1,8 @@ (ns auto-ap.logging (:require [clojure.tools.logging :as log] - [unilog.context :as lc])) + [unilog.context :as lc] + [com.brunobonacci.mulog :as mu])) + (defn info-event [message context] (lc/with-context context (log/info message))) @@ -13,3 +15,21 @@ (defn error-event [message context] (lc/with-context context (log/warn message))) + +(defmacro with-context-as [ctx s & body] + `(mu/with-context ~ctx + (let [~s (mu/local-context)] + ~@body))) + +(defmacro capture-context->lc [& body] + `(let [~'lc (mu/local-context)] + ~@body)) + +(defmacro info [x & kvs] + `(mu/log ~x :status "INFO" ~@kvs )) + +(defmacro warn [x & kvs] + `(mu/log ~x :status "WARN" ~@kvs )) + +(defmacro error [x & kvs] + `(mu/log ~x :status "ERROR" ~@kvs )) diff --git a/src/clj/auto_ap/square/core3.clj b/src/clj/auto_ap/square/core3.clj index 77273b59..30fa4f86 100644 --- a/src/clj/auto_ap/square/core3.clj +++ b/src/clj/auto_ap/square/core3.clj @@ -11,7 +11,6 @@ [clojure.data.json :as json] [clojure.set :as set] [clojure.string :as str] - [clojure.tools.logging :as log] [cemerick.url :as url] [datomic.api :as d] [slingshot.slingshot :refer [try+]] @@ -19,7 +18,8 @@ [manifold.deferred :as de] [manifold.time :as mt] [manifold.stream :as s] - [com.brunobonacci.mulog :as mu])) + [com.brunobonacci.mulog :as mu] + [auto-ap.logging :refer [with-context-as capture-context->lc] :as log])) (defn client-base-headers [client] {"Square-Version" "2021-08-18" @@ -28,20 +28,11 @@ (defn retry-4 [ex try-count _] - (mu/log ::aborting-request - :attempt try-count - :exception ex) + (log/error ::aborting-request + :attempt try-count + :exception ex) (if (> try-count 4) false true)) -(defmacro with-context-as [ctx s & body] - `(mu/with-context ~ctx - (let [~s (mu/local-context)] - ~@body))) - -(defmacro capture-context->lc [& body] - `(let [~'lc (mu/local-context)] - ~@body)) - (def manifold-api-stream (let [stream (s/stream 100)] (->> stream @@ -51,7 +42,7 @@ (de/chain (de/loop [attempt 0] (-> (de/chain (de/future-with (ex/execute-pool) - (mu/log ::request-started + (log/info ::request-started :url (:url request) :attempt attempt :source "Square 3" @@ -75,9 +66,9 @@ (s/buffer 50) (s/realize-each) (s/consume (fn [result] - (mu/log ::request-completed - :source "Square 3" - :background-job "Square 3")))) + (log/info ::request-completed + :source "Square 3" + :background-job "Square 3")))) stream)) (defn manifold-api-call @@ -110,8 +101,8 @@ :locations) (fn [error] (mu/with-context lc - (mu/log ::no-locations-found - :exception error)) + (log/error ::no-locations-found + :exception error)) [])))) @@ -158,9 +149,8 @@ :else (do - (mu/log ::no-look-up-item - :item item - ) + (log/warn ::no-look-up-item + :item item) "Uncategorized")))) @@ -190,9 +180,9 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-order ([client location order-id] - (mu/log ::searching-for-order - :location location - :order-id order-id) + (log/info ::searching-for-order + :location location + :order-id order-id) (let [result (->> (client/get (str "https://connect.squareup.com/v2/orders/" order-id) {:headers (client-base-headers client) :as :json}) @@ -201,8 +191,8 @@ result))) (defn continue-search [client location start end cursor] - (mu/log ::continue-order-search - :cursor cursor) + (log/info ::continue-order-search + :cursor cursor) (capture-context->lc (de/chain (manifold-api-call @@ -218,8 +208,8 @@ (fn [result] (mu/with-context lc - (mu/log ::orders-found - :count (count (:orders result))) + (log/info ::orders-found + :count (count (:orders result))) (if (not-empty (:cursor result)) (de/chain (continue-search client location start end (:cursor result)) (fn [continued-results] @@ -232,7 +222,7 @@ (defn search ([client location start end] (capture-context->lc - (mu/log ::searching + (log/info ::searching :location (:square-location/client-location location)) (de/chain (manifold-api-call {:url "https://connect.squareup.com/v2/orders/search" :method :post @@ -243,8 +233,8 @@ :body (fn [result] (mu/with-context lc - (mu/log ::orders-found - :count (count (:orders result))) + (log/info ::orders-found + :count (count (:orders result))) (if (not-empty (:cursor result)) (de/chain (continue-search client location start end (:cursor result)) (fn [continued-results] @@ -411,9 +401,9 @@ (s/->source) (s/map (fn [[start-date end-date]] (mu/with-context lc - (mu/log ::searching-settlements - :start-date start-date - :end-date end-date) + (log/info ::searching-settlements + :start-date start-date + :end-date end-date) (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements") :method :get @@ -433,8 +423,8 @@ (s/->source) (s/map (fn [settlement-id] (mu/with-context lc - (mu/log ::looking-up-settlement - :settlement-id settlement-id) + (log/info ::looking-up-settlement + :settlement-id settlement-id) (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" settlement-id) @@ -536,8 +526,8 @@ (fn [results ] (mu/with-context lc (doseq [x (partition-all 100 results)] - (mu/log ::loading-orders - :count (count x)) + (log/info ::loading-orders + :count (count x)) @(d/transact conn x)))))))) @@ -554,11 +544,11 @@ (de/chain (daily-settlements client location) (fn [settlements] (mu/with-context lc - (doseq [x (partition-all 20 settlements)] - (mu/log ::loading-deposits + (doseq [x (partition-all 20 settlements)] + (log/info ::loading-deposits :count (count x)) - @(d/transact conn x)) - (mu/log ::done-loading-deposits))))))) + @(d/transact conn x)) + (log/info ::done-loading-deposits))))))) (defn upsert-refunds ([client] @@ -573,10 +563,10 @@ :client (:client/code client) :location (:square-location/client-location client)} (doseq [x (partition-all 100 refunds)] - (mu/log ::loading-refunds - :count (count x)) + (log/info ::loading-refunds + :count (count x)) @(d/transact conn x)) - (mu/log ::done-loading-refunds)))))) + (log/info ::done-loading-refunds)))))) (def square-read [:db/id :client/code @@ -653,34 +643,34 @@ (seq (filter :square-location/client-location (:client/square-locations client))))) (s/map (fn [client] (with-context-as (merge lc {:client (:client/code client)}) lc - (mu/log ::import-started) + (log/info ::import-started) (mark-integration-status client {:integration-status/last-attempt (coerce/to-date (time/now))}) (-> (de/chain (upsert-locations client) (fn [_] (mu/with-context lc - (mu/log ::upsert-orders-started) + (log/info ::upsert-orders-started) (upsert client))) (fn [_] (mu/with-context lc - (mu/log ::upsert-settlements-started) + (log/info ::upsert-settlements-started) (upsert-settlements client))) (fn [_] (mu/with-context lc - (mu/log ::upsert-refunds-started) + (log/info ::upsert-refunds-started) (upsert-refunds client))) (fn [_] (mu/with-context lc - (mu/log ::upsert-done)) + (log/info ::upsert-done)) (mark-integration-status client {:integration-status/state :integration-state/success :integration-status/last-updated (coerce/to-date (time/now))}))) (de/catch (fn [e] (mu/with-context lc (let [data (ex-data e)] - (mu/log ::upsert-all-failed - :severity :warn - :exception e) + (log/info ::upsert-all-failed + :severity :warn + :exception e) (cond (= (:status data) 401) (mark-integration-status client {:integration-status/state :integration-state/unauthorized :integration-status/message (-> data :body str)}) diff --git a/src/clj/user.clj b/src/clj/user.clj index aee27369..0e31c6b5 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -20,7 +20,8 @@ [datomic.api :as d] [mount.core :as mount] [nrepl.middleware.print] - [unilog.context :as lc]) + [unilog.context :as lc] + [com.brunobonacci.mulog :as mu]) (:import (org.apache.commons.io.input BOMInputStream))) @@ -396,6 +397,7 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn start-db [] + (mu/start-publisher! {:type :console-json}) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn})))