Makes logging unified

2023-10-30 12:35:18 -07:00
parent f0a7c378f7
commit 930b900849
44 changed files with 485 additions and 555 deletions
--- a/src/clj/auto_ap/jobs/bulk_journal_import.clj
+++ b/src/clj/auto_ap/jobs/bulk_journal_import.clj
@@ -7,7 +7,7 @@
   [clojure.data.csv :as csv]
   [clojure.java.io :as io]
   [clojure.string :as str]
-   [clojure.tools.logging :as log]
+   [auto-ap.logging :as alog]
   [config.core :refer [env]]))

 (defn line->id [{:keys [source id client-code]}]
@@ -67,7 +67,7 @@

 (defn bulk-journal-import [args]
  (let [{:keys [ledger-url]} args
-        _ (log/info "importing ledger from" ledger-url)
+        _ (alog/info ::importing :url ledger-url)
        csv-stream (s3->csv ledger-url)
        import-rows (csv->graphql-rows csv-stream)]
    (import-ledger {:id {:user/name "Bulk-import" :user/role "admin"}}
--- a/src/clj/auto_ap/jobs/close_auto_invoices.clj
+++ b/src/clj/auto_ap/jobs/close_auto_invoices.clj
@@ -5,7 +5,7 @@
   [auto-ap.jobs.core :refer [execute]]
   [auto-ap.time :as time]
   [clj-time.coerce :as coerce]
-   [clojure.tools.logging :as log]
+   [auto-ap.logging :as alog]
   [datomic.api :as dc]))

 (defn close-auto-invoices []
@@ -15,15 +15,16 @@
                                         '[?e :invoice/status :invoice-status/unpaid]
                                         '[(<= ?d ?today)]]}
                                (dc/db conn) (coerce/to-date (time/local-now)))]
-    (log/info "Closing " (count invoices-to-close) "scheduled invoices")
+    (alog/info ::closing :count (count invoices-to-close))
    @(dc/transact conn (some->> invoices-to-close
-                               seq
+                                seq

-                               (mapv (fn [[i]] {:db/id                       i
-                                                :invoice/outstanding-balance 0.0
-                                                :invoice/status              :invoice-status/paid}))
-                               ))
-    (log/info "Closed " (count invoices-to-close) "scheduled invoices")))
+                                (mapv (fn [[i]] {:db/id                       i
+                                                 :invoice/outstanding-balance 0.0
+                                                 :invoice/status              :invoice-status/paid}))
+                                ))
+
+    (alog/info ::closed :count (count invoices-to-close))))


 (defn -main [& _]
--- a/src/clj/auto_ap/jobs/core.clj
+++ b/src/clj/auto_ap/jobs/core.clj
@@ -2,26 +2,26 @@
  (:require [auto-ap.utils :refer [heartbeat]]
            [mount.core :as mount]
            [auto-ap.datomic :refer [conn ]]
-            [clojure.tools.logging :as log]
+            [auto-ap.logging :as alog]
            [nrepl.server :refer [start-server]]
            [auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
-            [unilog.context :as lc]
            [com.brunobonacci.mulog :as mu]))

 (defn execute [name f]
-  (try 
-    (lc/with-context {:background-job name}
-      (mu/with-context {:background-job name
-                        :service name}
-        (mount/start (mount/only #{#'conn  #'metrics-setup #'container-tags #'logging-context #'container-data }))
-        (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
-        ((heartbeat f name))
-        (log/info "Stopping " name)
-        (Thread/sleep 15000)
-        (mount/stop)))
-    (catch Exception e
-      (log/error "ERROR" e)
-      (println e)
-      (throw e))
-    (finally
-      (System/exit 0))))
+  (mu/with-context {:background-job name
+                    :source name
+                    :service        name}
+    (mu/trace ::execute-background-job
+              []
+              (try 
+                (mount/start (mount/only #{#'conn  #'metrics-setup #'container-tags #'logging-context #'container-data }))
+                (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
+                ((heartbeat f name))
+                (alog/info ::stopping :job name)
+                (Thread/sleep 15000)
+                (mount/stop)
+                (catch Exception e
+                  (alog/error ::job-error :error e)
+                  (throw e))
+                (finally
+                  (System/exit 0))))))
--- a/src/clj/auto_ap/jobs/load_historical_sales.clj
+++ b/src/clj/auto_ap/jobs/load_historical_sales.clj
@@ -5,18 +5,16 @@
   [auto-ap.jobs.core :refer [execute]]
   [auto-ap.square.core :as square]
   [auto-ap.square.core3 :as square3]
-   [auto-ap.time :as atime]
   [clj-time.coerce :as coerce]
   [clj-time.core :as time]
   [clj-time.periodic :as per]
-   [clojure.tools.logging :as log]
+   [auto-ap.logging :as alog]
   [config.core :refer [env]]
-   [datomic.api :as dc]
-   [unilog.context :as lc]))
+   [datomic.api :as dc]))

 #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
 (defn historical-load-sales [client days]
-  (log/info "loading old approach")
+  (alog/info ::old-sales-loading)
  (let [client (dc/pull (dc/db auto-ap.datomic/conn)
                       square/square-read
                       client)]
@@ -24,12 +22,11 @@
            :when           (:square-location/client-location square-location)]
      
      (println "orders")
-      (lc/with-context {:source "Historical loading data"}
-        (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
-                                    (time/today)
-                                    (time/days 1))]
-          (println d)
-          (square/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1))))))
+      (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
+                                  (time/today)
+                                  (time/days 1))]
+        (println d)
+        (square/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1)))))

      (println "refunds")
      (square/upsert-refunds client square-location)
@@ -48,7 +45,7 @@


 (defn historical-load-sales2 [client days]
-  (log/info "loading new approach")
+  (alog/info ::new-sales-loading)
  (let [client (dc/pull (dc/db auto-ap.datomic/conn)
                       square/square-read
                       client)
@@ -57,12 +54,11 @@
            :when           (:square-location/client-location square-location)]
      
      (println "orders")
-      (lc/with-context {:source "Historical loading data"}
-        (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
-                                    (time/today)
-                                    (time/days 1))]
-          (println d)
-          @(square3/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1))))))
+      (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
+                                  (time/today)
+                                  (time/days 1))]
+        (println d)
+        @(square3/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1)))))

      (println "refunds")
     @(square3/upsert-refunds client square-location)
--- a/src/clj/auto_ap/jobs/register_invoice_import.clj
+++ b/src/clj/auto_ap/jobs/register_invoice_import.clj
@@ -7,10 +7,10 @@
   [auto-ap.time :as atime]
   [auto-ap.utils :refer [dollars=]]
   [clj-time.coerce :as coerce]
+   [auto-ap.logging :as alog]
   [clojure.data.csv :as csv]
   [clojure.java.io :as io]
   [clojure.string :as str]
-   [clojure.tools.logging :as log]
   [config.core :refer [env]]
   [datomic.api :as dc])
  (:import
@@ -26,8 +26,10 @@
             io/reader
             csv/read-csv))
    (catch Exception e
-      (log/error (str "Could not read the file " url ". Are you sure you uploaded it?"))
-      (log/error e)
+      (alog/error
+        :file-not-found
+        :error e
+        :url url)
      (throw e))))

 (defn register-invoice-import* [data]
@@ -138,9 +140,12 @@
 (defn register-invoice-import [args]
  (let [{:keys [invoice-url]} args
        data                 (s3->csv invoice-url)]
-    (log/info "contains " (count data) " rows")
+    (alog/info ::rows
+      :count (count data))
    (doseq [n (partition-all 50 (register-invoice-import* data))]
-      (log/info "transacting" n)
+      (alog/info ::transacting
+        :count (count n)
+        :sample (take 2 n))
      (audit-transact n {:user/name "register-invoice-import"
                         :user/role "admin"}))))

--- a/src/clj/auto_ap/jobs/sysco.clj
+++ b/src/clj/auto_ap/jobs/sysco.clj
@@ -12,8 +12,8 @@
   [clojure.data.csv :as csv]
   [clojure.java.io :as io]
   [com.brunobonacci.mulog :as mu]
+   [auto-ap.logging :as alog]
   [clojure.string :as str]
-   [clojure.tools.logging :as log]
   [com.unbounce.dogstatsd.core :as statsd]
   [config.core :refer [env]]
   [datomic.api :as dc]
@@ -77,7 +77,9 @@
        date (t/parse
              (header-row "InvoiceDate")
              "yyMMdd")]
-    (log/infof "Importing %s for %s" (header-row "InvoiceNumber") (header-row "CustomerName"))
+    (alog/info ::importing
+               :invoice-number (header-row "InvoiceNumber")
+               :customer-name (header-row "CustomerName"))

    (cond-> #:invoice {:invoice-number      (header-row "InvoiceNumber")
                       :db/id (random-tempid)
@@ -124,7 +126,9 @@
                  (map :key))]
    

-    (log/infof "Found %d sysco invoice to import: %s" (count keys) (pr-str keys))
+    (alog/info ::importing-sysco
+               :count (count keys)
+               :keys (pr-str keys))
    
    (let [transaction (->> keys
                           (mapcat (fn [k]
@@ -141,15 +145,16 @@
                                               (extract-invoice-details sysco-vendor)
                                               (assoc :invoice/source-url invoice-url))]])
                                       (catch Exception e
-                                         (log/error (str "Cannot load file " k) e)
-                                         (log/info 
-                                           (s3/copy-object {:source-bucket-name      (:data-bucket env) 
-                                                            :destination-bucket-name (:data-bucket env)
-                                                            :source-key              k
-                                                            :destination-key         (str "sysco/error/"
-                                                                                          (.getName (io/file k)))}))
+                                         (alog/error ::cant-load-file
+                                                     :file k
+                                                     :error e e)
+                                         (s3/copy-object {:source-bucket-name      (:data-bucket env) 
+                                                          :destination-bucket-name (:data-bucket env)
+                                                          :source-key              k
+                                                          :destination-key         (str "sysco/error/"
+                                                                                        (.getName (io/file k)))})
                                         [])))))
-result      (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
+          result      (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
    (doseq [k keys]
      (mark-key k))))