From 099fa4f97d23965f36e583fb090ca169613bc730 Mon Sep 17 00:00:00 2001
From: Bryce <bryce@integreatconsult.com>
Date: Tue, 1 Aug 2023 17:11:48 -0700
Subject: [PATCH] makes clearer logging to identify problems

---
 .../auto_ap/jobs/import_uploaded_invoices.clj | 69 ++++++++++---------
 1 file changed, 36 insertions(+), 33 deletions(-)

diff --git a/src/clj/auto_ap/jobs/import_uploaded_invoices.clj b/src/clj/auto_ap/jobs/import_uploaded_invoices.clj
index 69ce319e..9460108c 100644
--- a/src/clj/auto_ap/jobs/import_uploaded_invoices.clj
+++ b/src/clj/auto_ap/jobs/import_uploaded_invoices.clj
@@ -10,8 +10,9 @@
    [clojure.data.json :as json]
    [clojure.java.io :as io]
    [clojure.string :as str]
-   [clojure.tools.logging :as log]
-   [config.core :refer [env]])
+   [config.core :refer [env]]
+   [auto-ap.logging :as alog]
+   [com.brunobonacci.mulog :as mu])
   (:import
    (java.util Properties UUID)
    (javax.mail Session)
@@ -22,7 +23,7 @@
         target-url (str "http://" (:data-bucket env)
                         ".s3-website-us-east-1.amazonaws.com/"
                         target-key)]
-    (log/info "sending email to " (:import-failure-destination-emails env))
+    (alog/info ::sending-failure-email :who (:import-failure-destination-emails env))
     (s3/copy-object mail-bucket mail-key (:data-bucket env) target-key)
     (ses/send-email {:destination {:to-addresses (:import-failure-destination-emails env)}
                      :source      (:invoice-email env)
@@ -32,7 +33,7 @@
 
 
 (defn process-sqs []
-  (log/info "Fetching messages from sqs...")
+  (alog/info ::fetching-sqs)
   (doseq [message (:messages (sqs/receive-message {:queue-url              (:invoice-import-queue-url env)
                                                    :wait-time-seconds      5
                                                    :max-number-of-messages 10
@@ -40,40 +41,42 @@
     (let [message-body (json/read-str (:body message)
                                       :key-fn keyword)]
       (doseq [r (:Records message-body)] 
-        (log/info "Processing record " r)
+        (alog/info ::processing-record :record r )
         (let [props          (Session/getDefaultInstance (Properties.))
               message-stream (-> (s3/get-object {:key         (-> r :s3 :object :key)
                                                  :bucket-name (-> r :s3 :bucket :name)})
                                  :input-stream)
               mail           (message/read-message (MimeMessage. props message-stream))]
-          (log/info "reading mail" (->> mail :body (filter :content-type)  (map :body)  ))
-          (doseq [pdf-stream (->> (-> mail :body)
-                                  (filter :content-type)
-                                  #_(filter #(re-find #"application/pdf" (:content-type %)) ))
-                  :let       [filename    (str "/tmp/" (UUID/randomUUID) ".pdf")]]
-            (try 
-              (let [
-                    _           (io/copy (:body pdf-stream) (io/file filename))
-                    extension   (last (str/split (.getName (io/file filename)) #"\."))
-                    s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)
-                    _           (s3/put-object :bucket-name (:data-bucket env)
-                                               :key s3-location
-                                               :input-stream (io/input-stream filename)
-                                               :metadata {:content-type "application/pdf"
-                                                          :content-length (.length (io/file filename))})
-                    imports     (->> (parse/parse-file filename filename)
-                                     (map #(assoc %
-                                                  :source-url (str "http://" (:data-bucket env)
-                                                                   ".s3-website-us-east-1.amazonaws.com/"
-                                                                   s3-location)
-                                                  :import-status :import-status/approved)))]
-                (log/info "Found imports" imports)
-                (invoices/import-uploaded-invoice {:user/role "admin"} imports ))
-              (catch Exception e
-                (log/warn e)
-                (send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key)))
-              (finally 
-                (io/delete-file filename)))))))
+          (alog/info ::reading-mail :body (->> mail :body (filter :content-type)  (map :body)  ))
+          (mu/with-context {:email-record {:key         (-> r :s3 :object :key)
+                                           :bucket-name (-> r :s3 :bucket :name)}}
+            (doseq [pdf-stream (->> (-> mail :body)
+                                    (filter :content-type)
+                                    #_(filter #(re-find #"application/pdf" (:content-type %)) ))
+                    :let       [filename    (str "/tmp/" (UUID/randomUUID) ".pdf")]]
+              (try 
+                (let [_           (io/copy (:body pdf-stream) (io/file filename))
+                      extension   (last (str/split (.getName (io/file filename)) #"\."))
+                      s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)]
+                  (mu/with-context {:s3-location s3-location}
+                    (s3/put-object :bucket-name (:data-bucket env)
+                                   :key s3-location
+                                   :input-stream (io/input-stream filename)
+                                   :metadata {:content-type   "application/pdf"
+                                              :content-length (.length (io/file filename))})
+                    (let [imports (->> (parse/parse-file filename filename)
+                                       (map #(assoc %
+                                                    :source-url (str "http://" (:data-bucket env)
+                                                                     ".s3-website-us-east-1.amazonaws.com/"
+                                                                     s3-location)
+                                                    :import-status :import-status/approved)))]
+                      (alog/info ::found-imports :imports imports)
+                      (invoices/import-uploaded-invoice {:user/role "admin"} imports ))))
+                (catch Exception e
+                  (alog/warn ::cant-import e)
+                  (send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key)))
+                (finally 
+                  (io/delete-file filename))))))))
     (sqs/delete-message (assoc message :queue-url (:invoice-import-queue-url env) ))))
 
 (defn -main [& _]