replying to emails

project.clj

@@ -25,6 +25,7 @@
                  [buddy/buddy-auth "2.1.0"]
                  [buddy/buddy-sign "2.1.0"]
                  [clj-time "0.14.2"]
+                 [io.forward/clojure-mail "1.0.7"]
                  [amazonica "0.3.121"
                   :exclusions [com.amazonaws/aws-java-sdk
                                com.amazonaws/amazon-kinesis-client]]

src/clj/auto_ap/background/mail.clj

@@ -1,18 +1,61 @@
 (ns auto-ap.background.mail
   (:require [amazonica.aws.sqs :as sqs]
             [amazonica.aws.s3 :as s3]
-            [clojure.data.json :as json])
+            [clojure.data.json :as json]
+            [clojure-mail.message :as message]
+            [clojure.string :as str]
+            [clojure.java.io :as io]
+            [auto-ap.parse :as parse]
+            [auto-ap.db.invoices :as invoices]
+            [auto-ap.db.companies :as companies]
             )
+  (:import [java.util Properties]
+           [java.util UUID]
+           [javax.mail.search FlagTerm]
+           [java.io FileInputStream File]
+           [javax.mail.internet MimeMessage]
+           [javax.mail Session
+            Folder
+            Flags
+            Flags$Flag AuthenticationFailedException]
+           (com.sun.mail.imap IMAPStore)))
+
+(def  queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod")
 
 (defn process-sqs []
-  (doseq [message (:messages (sqs/receive-message {:queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
+  (println "Fetching messages from sqs...")
+  (let [companies (companies/get-all)]
+    (doseq [message (:messages (sqs/receive-message {:queue-url queue-url 
                                                      :wait-time-seconds 5
                                                      :max-number-of-messages 10
                                                      #_#_:attribute-names ["All"]}))]
       (let [message-body (json/read-str (:body message)
                                         :key-fn keyword)]
         (doseq [r (:Records message-body)] 
-        (println (-> (s3/get-object {:key (-> r :s3 :object :key)
+          (println "Processing record " r)
+          (let [props (Session/getDefaultInstance (Properties.))
+                message-stream (-> (s3/get-object {:key (-> r :s3 :object :key)
                                                    :bucket-name (-> r :s3 :bucket :name)})
-                     :input-stream
+                                   :input-stream)
-                     slurp))))))
+                mail (message/read-message (MimeMessage. props message-stream))]
+            (doseq [pdf-stream (->> (-> mail :body)
+                                    (filter :content-type)
+                                    (filter #(re-find #"application/pdf" (:content-type %)) ))]
+              (let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]
+                (io/copy (:body pdf-stream) (io/file filename))
+                (invoices/insert-multi!
+                 (for [{:keys [total date invoice-number customer-identifier vendor] :as row}
+                       (parse/parse-file filename filename)]
+                   (do
+                     (println row)
+                     (assoc row
+                            :company (:name (parse/best-match companies customer-identifier))
+                            :imported false
+                            :potential-duplicate false))))
+                
+                (io/delete-file filename))))))
+      (sqs/delete-message (assoc message :queue-url queue-url )))))
+
+(defn always-process-sqs []
+  (while (not (Thread/interrupted))
+    (process-sqs)))

src/clj/auto_ap/handler.clj

@@ -13,7 +13,6 @@
             [ring.middleware.reload :refer [wrap-reload]]
             [ring.middleware.edn :refer [wrap-edn-params]]
             [clojure.java.jdbc :as j]
-            [clj-fuzzy.metrics :as m]
             [clj-http.client :as http]
             [clj-time.core :as time]
             [config.core :refer [env]]
@@ -27,14 +26,7 @@
             [auto-ap.db.vendors :as vendors]
             [amazonica.core :refer [defcredential]]
             [amazonica.aws.simpleemail :as ses]))
-(defn best-match [companies company-identifier]
-  (->> companies
-       (map (fn [company]
-              [company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))]))
-       (filter #(< (second %) 0.25))
-       (sort-by second)
 
-       ffirst))
 
 (def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com")
 (def google-client-secret "OC-WemHurPXYpuIw5cT-B90g")
@@ -168,7 +160,7 @@
            (for [{:keys [total date invoice-number customer-identifier vendor] :as row}
                  (parse/parse-file (.getPath tempfile) filename)]
              (assoc row
-                    :company (:name (best-match companies customer-identifier))
+                    :company (:name (parse/best-match companies customer-identifier))
 
                     :imported false
                     :potential-duplicate (boolean (seq (filter #(and (= vendor (:vendor %))

src/clj/auto_ap/parse.clj

@@ -3,7 +3,9 @@
             [clojure.string :as str]
             [clojure.java.shell :as sh]
             [auto-ap.parse.excel :as excel]
-            [auto-ap.parse.templates :as t]))
+            [auto-ap.parse.templates :as t]
+            [clj-fuzzy.metrics :as m]
+            ))
 
 
 
@@ -50,3 +52,12 @@
   "xlsx"
   [file filename]
   (excel/parse-file file filename))
+
+(defn best-match [companies company-identifier]
+  (->> companies
+       (map (fn [company]
+              [company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))]))
+       (filter #(< (second %) 0.25))
+       (sort-by second)
+       
+       ffirst))

src/clj/auto_ap/server.clj

@@ -1,11 +1,11 @@
 (ns auto-ap.server
   (:require [auto-ap.handler :refer [app]]
-            [auto-ap.background.mail :refer [process-sqs]]
+            [auto-ap.background.mail :refer [always-process-sqs]]
             [config.core :refer [env]]
             [ring.adapter.jetty :refer [run-jetty]])
   (:gen-class))
 
  (defn -main [& args]
    (let [port (Integer/parseInt (or (env :port) "3000"))]
-     (future (process-sqs))
+     (future (always-process-sqs))
      (run-jetty app {:port port :join? false})))