notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

makes clearer logging to identify problems

Browse Source
This commit is contained in:
Bryce
2023-08-01 17:11:48 -07:00
parent 02f0d25571
commit 099fa4f97d
1 changed files with 36 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/clj/auto_ap/jobs/import_uploaded_invoices.clj
View File

@@ -10,8 +10,9 @@
[clojure.data.json :as json]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.logging :as log]
[config.core :refer [env]])
[config.core :refer [env]]
[auto-ap.logging :as alog]
[com.brunobonacci.mulog :as mu])
(:import
(java.util Properties UUID)
(javax.mail Session)
@@ -22,7 +23,7 @@
target-url (str "http://" (:data-bucket env)
".s3-website-us-east-1.amazonaws.com/"
target-key)]
(log/info "sending email to " (:import-failure-destination-emails env))
(alog/info ::sending-failure-email :who (:import-failure-destination-emails env))
(s3/copy-object mail-bucket mail-key (:data-bucket env) target-key)
(ses/send-email {:destination {:to-addresses (:import-failure-destination-emails env)}
:source (:invoice-email env)
@@ -32,7 +33,7 @@
(defn process-sqs []
(log/info "Fetching messages from sqs...")
(alog/info ::fetching-sqs)
(doseq [message (:messages (sqs/receive-message {:queue-url (:invoice-import-queue-url env)
:wait-time-seconds 5
:max-number-of-messages 10
@@ -40,40 +41,42 @@
(let [message-body (json/read-str (:body message)
:key-fn keyword)]
(doseq [r (:Records message-body)]
(log/info "Processing record " r)
(alog/info ::processing-record :record r )
(let [props (Session/getDefaultInstance (Properties.))
message-stream (-> (s3/get-object {:key (-> r :s3 :object :key)
:bucket-name (-> r :s3 :bucket :name)})
:input-stream)
mail (message/read-message (MimeMessage. props message-stream))]
(log/info "reading mail" (->> mail :body (filter :content-type) (map :body) ))
(doseq [pdf-stream (->> (-> mail :body)
(filter :content-type)
#_(filter #(re-find #"application/pdf" (:content-type %)) ))
:let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]]
(try
(let [
_ (io/copy (:body pdf-stream) (io/file filename))
extension (last (str/split (.getName (io/file filename)) #"\."))
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)
_ (s3/put-object :bucket-name (:data-bucket env)
:key s3-location
:input-stream (io/input-stream filename)
:metadata {:content-type "application/pdf"
:content-length (.length (io/file filename))})
imports (->> (parse/parse-file filename filename)
(map #(assoc %
:source-url (str "http://" (:data-bucket env)
".s3-website-us-east-1.amazonaws.com/"
s3-location)
:import-status :import-status/approved)))]
(log/info "Found imports" imports)
(invoices/import-uploaded-invoice {:user/role "admin"} imports ))
(catch Exception e
(log/warn e)
(send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key)))
(finally
(io/delete-file filename)))))))
(alog/info ::reading-mail :body (->> mail :body (filter :content-type) (map :body) ))
(mu/with-context {:email-record {:key (-> r :s3 :object :key)
:bucket-name (-> r :s3 :bucket :name)}}
(doseq [pdf-stream (->> (-> mail :body)
(filter :content-type)
#_(filter #(re-find #"application/pdf" (:content-type %)) ))
:let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]]
(try
(let [_ (io/copy (:body pdf-stream) (io/file filename))
extension (last (str/split (.getName (io/file filename)) #"\."))
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)]
(mu/with-context {:s3-location s3-location}
(s3/put-object :bucket-name (:data-bucket env)
:key s3-location
:input-stream (io/input-stream filename)
:metadata {:content-type "application/pdf"
:content-length (.length (io/file filename))})
(let [imports (->> (parse/parse-file filename filename)
(map #(assoc %
:source-url (str "http://" (:data-bucket env)
".s3-website-us-east-1.amazonaws.com/"
s3-location)
:import-status :import-status/approved)))]
(alog/info ::found-imports :imports imports)
(invoices/import-uploaded-invoice {:user/role "admin"} imports ))))
(catch Exception e
(alog/warn ::cant-import e)
(send-email-about-failed-message (-> r :s3 :bucket :name) (-> r :s3 :object :key)))
(finally
(io/delete-file filename))))))))
(sqs/delete-message (assoc message :queue-url (:invoice-import-queue-url env) ))))
(defn -main [& _]