notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replying to emails

Browse Source
This commit is contained in:
Bryce Covert
2018-04-05 22:47:16 -07:00
parent 4eace8144e
commit 26168cd523
5 changed files with 75 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
project.clj
View File

@@ -25,6 +25,7 @@
[buddy/buddy-auth "2.1.0"]
[buddy/buddy-sign "2.1.0"]
[clj-time "0.14.2"]
[io.forward/clojure-mail "1.0.7"]
[amazonica "0.3.121"
:exclusions [com.amazonaws/aws-java-sdk
com.amazonaws/amazon-kinesis-client]]

69
src/clj/auto_ap/background/mail.clj
View File

@@ -1,18 +1,61 @@
(ns auto-ap.background.mail
(:require [amazonica.aws.sqs :as sqs]
[amazonica.aws.s3 :as s3]
[clojure.data.json :as json])
)
[clojure.data.json :as json]
[clojure-mail.message :as message]
[clojure.string :as str]
[clojure.java.io :as io]
[auto-ap.parse :as parse]
[auto-ap.db.invoices :as invoices]
[auto-ap.db.companies :as companies]
)
(:import [java.util Properties]
[java.util UUID]
[javax.mail.search FlagTerm]
[java.io FileInputStream File]
[javax.mail.internet MimeMessage]
[javax.mail Session
Folder
Flags
Flags$Flag AuthenticationFailedException]
(com.sun.mail.imap IMAPStore)))
(def queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod")
(defn process-sqs []
(doseq [message (:messages (sqs/receive-message {:queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
:wait-time-seconds 5
:max-number-of-messages 10
#_#_:attribute-names ["All"]}))]
(let [message-body (json/read-str (:body message)
:key-fn keyword)]
(doseq [r (:Records message-body)]
(println (-> (s3/get-object {:key (-> r :s3 :object :key)
:bucket-name (-> r :s3 :bucket :name)})
:input-stream
slurp))))))
(println "Fetching messages from sqs...")
(let [companies (companies/get-all)]
(doseq [message (:messages (sqs/receive-message {:queue-url queue-url
:wait-time-seconds 5
:max-number-of-messages 10
#_#_:attribute-names ["All"]}))]
(let [message-body (json/read-str (:body message)
:key-fn keyword)]
(doseq [r (:Records message-body)]
(println "Processing record " r)
(let [props (Session/getDefaultInstance (Properties.))
message-stream (-> (s3/get-object {:key (-> r :s3 :object :key)
:bucket-name (-> r :s3 :bucket :name)})
:input-stream)
mail (message/read-message (MimeMessage. props message-stream))]
(doseq [pdf-stream (->> (-> mail :body)
(filter :content-type)
(filter #(re-find #"application/pdf" (:content-type %)) ))]
(let [filename (str "/tmp/" (UUID/randomUUID) ".pdf")]
(io/copy (:body pdf-stream) (io/file filename))
(invoices/insert-multi!
(for [{:keys [total date invoice-number customer-identifier vendor] :as row}
(parse/parse-file filename filename)]
(do
(println row)
(assoc row
:company (:name (parse/best-match companies customer-identifier))
:imported false
:potential-duplicate false))))
(io/delete-file filename))))))
(sqs/delete-message (assoc message :queue-url queue-url )))))
(defn always-process-sqs []
(while (not (Thread/interrupted))
(process-sqs)))

16
src/clj/auto_ap/handler.clj
View File

@@ -13,7 +13,6 @@
[ring.middleware.reload :refer [wrap-reload]]
[ring.middleware.edn :refer [wrap-edn-params]]
[clojure.java.jdbc :as j]
[clj-fuzzy.metrics :as m]
[clj-http.client :as http]
[clj-time.core :as time]
[config.core :refer [env]]
@@ -27,14 +26,7 @@
[auto-ap.db.vendors :as vendors]
[amazonica.core :refer [defcredential]]
[amazonica.aws.simpleemail :as ses]))
(defn best-match [companies company-identifier]
(->> companies
(map (fn [company]
[company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))]))
(filter #(< (second %) 0.25))
(sort-by second)
ffirst))
(def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com")
(def google-client-secret "OC-WemHurPXYpuIw5cT-B90g")
@@ -168,12 +160,12 @@
(for [{:keys [total date invoice-number customer-identifier vendor] :as row}
(parse/parse-file (.getPath tempfile) filename)]
(assoc row
:company (:name (best-match companies customer-identifier))
:company (:name (parse/best-match companies customer-identifier))
:imported false
:potential-duplicate (boolean (seq (filter #(and (= vendor (:vendor %))
(= invoice-number (:invoice-number %)))
existing-invoices)))
(= invoice-number (:invoice-number %)))
existing-invoices)))
)))
{:status 200
:body (pr-str (invoices/get-pending ((:query-params params ) "company")))

13
src/clj/auto_ap/parse.clj
View File

@@ -3,7 +3,9 @@
[clojure.string :as str]
[clojure.java.shell :as sh]
[auto-ap.parse.excel :as excel]
[auto-ap.parse.templates :as t]))
[auto-ap.parse.templates :as t]
[clj-fuzzy.metrics :as m]
))
@@ -50,3 +52,12 @@
"xlsx"
[file filename]
(excel/parse-file file filename))
(defn best-match [companies company-identifier]
(->> companies
(map (fn [company]
[company (apply min (map #(m/jaccard (.toLowerCase company-identifier) %) (:matches company)))]))
(filter #(< (second %) 0.25))
(sort-by second)
ffirst))

4
src/clj/auto_ap/server.clj
View File

@@ -1,11 +1,11 @@
(ns auto-ap.server
(:require [auto-ap.handler :refer [app]]
[auto-ap.background.mail :refer [process-sqs]]
[auto-ap.background.mail :refer [always-process-sqs]]
[config.core :refer [env]]
[ring.adapter.jetty :refer [run-jetty]])
(:gen-class))
(defn -main [& args]
(let [port (Integer/parseInt (or (env :port) "3000"))]
(future (process-sqs))
(future (always-process-sqs))
(run-jetty app {:port port :join? false})))