From 26168cd5236b631bee1ca849ddeda6089d2e3016 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 5 Apr 2018 22:47:16 -0700 Subject: [PATCH] replying to emails --- project.clj | 1 + src/clj/auto_ap/background/mail.clj | 69 +++++++++++++++++++++++------ src/clj/auto_ap/handler.clj | 16 ++----- src/clj/auto_ap/parse.clj | 13 +++++- src/clj/auto_ap/server.clj | 4 +- 5 files changed, 75 insertions(+), 28 deletions(-) diff --git a/project.clj b/project.clj index 35c9fb2c..4ac6b9a0 100644 --- a/project.clj +++ b/project.clj @@ -25,6 +25,7 @@ [buddy/buddy-auth "2.1.0"] [buddy/buddy-sign "2.1.0"] [clj-time "0.14.2"] + [io.forward/clojure-mail "1.0.7"] [amazonica "0.3.121" :exclusions [com.amazonaws/aws-java-sdk com.amazonaws/amazon-kinesis-client]] diff --git a/src/clj/auto_ap/background/mail.clj b/src/clj/auto_ap/background/mail.clj index 4542d4e7..1908d022 100644 --- a/src/clj/auto_ap/background/mail.clj +++ b/src/clj/auto_ap/background/mail.clj @@ -1,18 +1,61 @@ (ns auto-ap.background.mail (:require [amazonica.aws.sqs :as sqs] [amazonica.aws.s3 :as s3] - [clojure.data.json :as json]) - ) + [clojure.data.json :as json] + [clojure-mail.message :as message] + [clojure.string :as str] + [clojure.java.io :as io] + [auto-ap.parse :as parse] + [auto-ap.db.invoices :as invoices] + [auto-ap.db.companies :as companies] + ) + (:import [java.util Properties] + [java.util UUID] + [javax.mail.search FlagTerm] + [java.io FileInputStream File] + [javax.mail.internet MimeMessage] + [javax.mail Session + Folder + Flags + Flags$Flag AuthenticationFailedException] + (com.sun.mail.imap IMAPStore))) + +(def queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod") (defn process-sqs [] - (doseq [message (:messages (sqs/receive-message {:queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" - :wait-time-seconds 5 - :max-number-of-messages 10 - #_#_:attribute-names ["All"]}))] - (let [message-body (json/read-str (:body message) - :key-fn keyword)] - (doseq [r (:Records message-body)] - (println (-> (s3/get-object {:key (-> r :s3 :object :key) - :bucket-name (-> r :s3 :bucket :name)}) - :input-stream - slurp)))))) + (println "Fetching messages from sqs...") + (let [companies (companies/get-all)] + (doseq [message (:messages (sqs/receive-message {:queue-url queue-url + :wait-time-seconds 5 + :max-number-of-messages 10 + #_#_:attribute-names ["All"]}))] + (let [message-body (json/read-str (:body message) + :key-fn keyword)] + (doseq [r (:Records message-body)] + (println "Processing record " r) + (let [props (Session/getDefaultInstance (Properties.)) + message-stream (-> (s3/get-object {:key (-> r :s3 :object :key) + :bucket-name (-> r :s3 :bucket :name)}) + :input-stream) + mail (message/read-message (MimeMessage. props message-stream))] + (doseq [pdf-stream (->> (-> mail :body) + (filter :content-type) + (filter #(re-find #"application/pdf" (:content-type %)) ))] + (let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")] + (io/copy (:body pdf-stream) (io/file filename)) + (invoices/insert-multi! + (for [{:keys [total date invoice-number customer-identifier vendor] :as row} + (parse/parse-file filename filename)] + (do + (println row) + (assoc row + :company (:name (parse/best-match companies customer-identifier)) + :imported false + :potential-duplicate false)))) + + (io/delete-file filename)))))) + (sqs/delete-message (assoc message :queue-url queue-url ))))) + +(defn always-process-sqs [] + (while (not (Thread/interrupted)) + (process-sqs))) diff --git a/src/clj/auto_ap/handler.clj b/src/clj/auto_ap/handler.clj index a98811f4..91d145da 100644 --- a/src/clj/auto_ap/handler.clj +++ b/src/clj/auto_ap/handler.clj @@ -13,7 +13,6 @@ [ring.middleware.reload :refer [wrap-reload]] [ring.middleware.edn :refer [wrap-edn-params]] [clojure.java.jdbc :as j] - [clj-fuzzy.metrics :as m] [clj-http.client :as http] [clj-time.core :as time] [config.core :refer [env]] @@ -27,14 +26,7 @@ [auto-ap.db.vendors :as vendors] [amazonica.core :refer [defcredential]] [amazonica.aws.simpleemail :as ses])) -(defn best-match [companies company-identifier] - (->> companies - (map (fn [company] - [company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))])) - (filter #(< (second %) 0.25)) - (sort-by second) - - ffirst)) + (def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com") (def google-client-secret "OC-WemHurPXYpuIw5cT-B90g") @@ -168,12 +160,12 @@ (for [{:keys [total date invoice-number customer-identifier vendor] :as row} (parse/parse-file (.getPath tempfile) filename)] (assoc row - :company (:name (best-match companies customer-identifier)) + :company (:name (parse/best-match companies customer-identifier)) :imported false :potential-duplicate (boolean (seq (filter #(and (= vendor (:vendor %)) - (= invoice-number (:invoice-number %))) - existing-invoices))) + (= invoice-number (:invoice-number %))) + existing-invoices))) ))) {:status 200 :body (pr-str (invoices/get-pending ((:query-params params ) "company"))) diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index bd6310c2..d9f0d273 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -3,7 +3,9 @@ [clojure.string :as str] [clojure.java.shell :as sh] [auto-ap.parse.excel :as excel] - [auto-ap.parse.templates :as t])) + [auto-ap.parse.templates :as t] + [clj-fuzzy.metrics :as m] + )) @@ -50,3 +52,12 @@ "xlsx" [file filename] (excel/parse-file file filename)) + +(defn best-match [companies company-identifier] + (->> companies + (map (fn [company] + [company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))])) + (filter #(< (second %) 0.25)) + (sort-by second) + + ffirst)) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 4791fbb3..7d4e90e5 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -1,11 +1,11 @@ (ns auto-ap.server (:require [auto-ap.handler :refer [app]] - [auto-ap.background.mail :refer [process-sqs]] + [auto-ap.background.mail :refer [always-process-sqs]] [config.core :refer [env]] [ring.adapter.jetty :refer [run-jetty]]) (:gen-class)) (defn -main [& args] (let [port (Integer/parseInt (or (env :port) "3000"))] - (future (process-sqs)) + (future (always-process-sqs)) (run-jetty app {:port port :join? false})))