notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

makes clearer logging to identify problems

Browse Source
This commit is contained in:
Bryce
2023-08-01 17:11:48 -07:00
parent 02f0d25571
commit 099fa4f97d
1 changed files with 36 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/clj/auto_ap/jobs/import_uploaded_invoices.clj
View File

@@ -10,8 +10,9 @@
[clojure.data.json :as json] [clojure.data.json :as json]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [config.core :refer [env]]
[config.core :refer [env]]) [auto-ap.logging :as alog]
[com.brunobonacci.mulog :as mu])
(:import (:import
(java.util Properties UUID) (java.util Properties UUID)
(javax.mail Session) (javax.mail Session)
@@ -22,7 +23,7 @@
target-url (str "http://" (:data-bucket env) target-url (str "http://" (:data-bucket env)
".s3-website-us-east-1.amazonaws.com/" ".s3-website-us-east-1.amazonaws.com/"
target-key)] target-key)]
(log/info "sending email to " (:import-failure-destination-emails env)) (alog/info ::sending-failure-email :who (:import-failure-destination-emails env))
(s3/copy-object mail-bucket mail-key (:data-bucket env) target-key) (s3/copy-object mail-bucket mail-key (:data-bucket env) target-key)
(ses/send-email {:destination {:to-addresses (:import-failure-destination-emails env)} (ses/send-email {:destination {:to-addresses (:import-failure-destination-emails env)}
:source (:invoice-email env) :source (:invoice-email env)
@@ -32,7 +33,7 @@
(defn process-sqs [] (defn process-sqs []
(log/info "Fetching messages from sqs...") (alog/info ::fetching-sqs)
(doseq [message (:messages (sqs/receive-message {:queue-url (:invoice-import-queue-url env) (doseq [message (:messages (sqs/receive-message {:queue-url (:invoice-import-queue-url env)
:wait-time-seconds 5 :wait-time-seconds 5
:max-number-of-messages 10 :max-number-of-messages 10
@@ -40,40 +41,42 @@
(let [message-body (json/read-str (:body message) (let [message-body (json/read-str (:body message)
:key-fn keyword)] :key-fn keyword)]
(doseq [r (:Records message-body)] (doseq [r (:Records message-body)]
(log/info "Processing record " r) (alog/info ::processing-record :record r )
(let [props (Session/getDefaultInstance (Properties.)) (let [props (Session/getDefaultInstance (Properties.))
message-stream (-> (s3/get-object {:key (-> r :s3 :object :key) message-stream (-> (s3/get-object {:key (-> r :s3 :object :key)
:bucket-name (-> r :s3 :bucket :name)}) :bucket-name (-> r :s3 :bucket :name)})
:input-stream) :input-stream)
mail (message/read-message (MimeMessage. props message-stream))] mail (message/read-message (MimeMessage. props message-stream))]
(log/info "reading mail" (->> mail :body (filter :content-type) (map :body) )) (alog/info ::reading-mail :body (->> mail :body (filter :content-type) (map :body) ))
(doseq [pdf-stream (->> (-> mail :body) (mu/with-context {:email-record {:key (-> r :s3 :object :key)
(filter :content-type) :bucket-name (-> r :s3 :bucket :name)}}
#_(filter #(re-find #"application/pdf" (:content-type %)) )) (doseq [pdf-stream (->> (-> mail :body)
:let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]] (filter :content-type)
(try #_(filter #(re-find #"application/pdf" (:content-type %)) ))
(let [ :let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]]
_ (io/copy (:body pdf-stream) (io/file filename)) (try
extension (last (str/split (.getName (io/file filename)) #"\.")) (let [_ (io/copy (:body pdf-stream) (io/file filename))
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension) extension (last (str/split (.getName (io/file filename)) #"\."))
_ (s3/put-object :bucket-name (:data-bucket env) s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)]
:key s3-location (mu/with-context {:s3-location s3-location}
:input-stream (io/input-stream filename) (s3/put-object :bucket-name (:data-bucket env)
:metadata {:content-type "application/pdf" :key s3-location
:content-length (.length (io/file filename))}) :input-stream (io/input-stream filename)
imports (->> (parse/parse-file filename filename) :metadata {:content-type "application/pdf"
(map #(assoc % :content-length (.length (io/file filename))})
:source-url (str "http://" (:data-bucket env) (let [imports (->> (parse/parse-file filename filename)
".s3-website-us-east-1.amazonaws.com/" (map #(assoc %
s3-location) :source-url (str "http://" (:data-bucket env)
:import-status :import-status/approved)))] ".s3-website-us-east-1.amazonaws.com/"
(log/info "Found imports" imports) s3-location)
(invoices/import-uploaded-invoice {:user/role "admin"} imports )) :import-status :import-status/approved)))]
(catch Exception e (alog/info ::found-imports :imports imports)
(log/warn e) (invoices/import-uploaded-invoice {:user/role "admin"} imports ))))
(send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key))) (catch Exception e
(finally (alog/warn ::cant-import e)
(io/delete-file filename))))))) (send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key)))
(finally
(io/delete-file filename))))))))
(sqs/delete-message (assoc message :queue-url (:invoice-import-queue-url env) )))) (sqs/delete-message (assoc message :queue-url (:invoice-import-queue-url env) ))))
(defn -main [& _] (defn -main [& _]