notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

merged.

Browse Source
This commit is contained in:
Bryce Covert
2020-01-25 11:26:59 -08:00
parent 903f9ceb18 84c22be822
commit 3926d5f6d7
4 changed files with 134 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/clj/auto_ap/parse/csv.clj
View File

@@ -18,6 +18,17 @@
:default (fn default [rows] :default (fn default [rows]
nil)) nil))
(defn parse-date-fallover [d fmts]
(if-let [valid-fmt (->> fmts
(filter (fn [f]
(try
(u/parse-value :clj-time f d)
(catch Exception e
nil))
))
(first))]
(u/parse-value :clj-time valid-fmt d)))
(defmethod parse-csv :mama-lus (defmethod parse-csv :mama-lus
[rows] [rows]
(println "MAMA LU4") (println "MAMA LU4")
@@ -27,9 +38,7 @@
{:vendor-code "Mama Lu's Foods" {:vendor-code "Mama Lu's Foods"
:customer-identifier customer :customer-identifier customer
:invoice-number (str po-number "-" invoice-number ) :invoice-number (str po-number "-" invoice-number )
:date (try (u/parse-value :clj-time "M/d/yyyy HH:ss" invoice-date) :date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
(catch Exception _
(u/parse-value :clj-time "M/d/yyyy" invoice-date)))
:total (str/replace value #"," "") :total (str/replace value #"," "")
:text (str/join " " row) :text (str/join " " row)
:full-text (str/join " " row)}))) :full-text (str/join " " row)})))

63
src/clj/auto_ap/parse/templates.clj
View File

@@ -121,12 +121,12 @@
;; Young's Market Co - INVOICE ;; Young's Market Co - INVOICE
{:vendor "Youngs Market" {:vendor "Youngs Market"
:keywords [#"P.O.Box 743564"] :keywords [#"P.O.Box 743564"]
:extract {:date #"INVOICE DATE\n(?:.*?)(\S+)\n" :extract {:date #"(?:INVOICE|CREDIT) DATE\n(?:.*?)(\S+)\n"
:customer-identifier #"INVOICE DATE\n [0-9]+\s+(.*?)\s{2,}" :customer-identifier #"(?:INVOICE|CREDIT) DATE\n [0-9]+\s+(.*?)\s{2,}"
:invoice-number #"INVOICE DATE\n(?:.*?)\s{2,}(\d+?)\s+\S+\n" :invoice-number #"(?:INVOICE|CREDIT) DATE\n(?:.*?)\s{2,}(\d+?)\s+\S+\n"
:total #"Net Amount(?:.*\n){4}(?:.*?)([0-9\.]+)\n"} :total #"Net Amount(?:.*\n){4}(?:.*?)([\-]?[0-9\.]+)\n"}
:parser {:date [:clj-time "dd-MMM-yy"] :parser {:date [:clj-time "dd-MMM-yy"]
:total [:trim-commas nil]}} :total [:trim-commas-and-negate nil]}}
;; WINE WAREHOUSE ;; WINE WAREHOUSE
{:vendor "Wine Warehouse" {:vendor "Wine Warehouse"
@@ -168,6 +168,16 @@
:parser {:date [:clj-time "MM/dd/yyyy"] :parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}} :total [:trim-commas nil]}}
;; P&R
{:vendor "P & R PAPER SUPPLY CO"
:keywords [#"PAPER SUPPLY COMPANY"]
:extract {:date #"Invoiced.*\n\s+\S+\s+(\S+)"
:customer-identifier #"Bill To.*\n\s*(.*?)\s{2,}"
:invoice-number #"Invoice#.*\n.*\n.*?(\S+)\s+\d+\n"
:total #"INVOICE TOTAL\s+([\-]?[\d,]+\.\d+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
;; SUNCREST STATEMENT ;; SUNCREST STATEMENT
{:vendor "Suncrest USA Inc" {:vendor "Suncrest USA Inc"
:keywords [#"Suncrest.*\n.*Statement"] :keywords [#"Suncrest.*\n.*Statement"]
@@ -196,8 +206,9 @@
:extract {:date #"INVOICE NUMBER[^\n]+\n([^\n]+)\n" :extract {:date #"INVOICE NUMBER[^\n]+\n([^\n]+)\n"
:customer-identifier #"INVOICE NUMBER[^\n]+\n[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,} :customer-identifier #"INVOICE NUMBER[^\n]+\n[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})" :invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})"
:total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+)"} :total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+[\-]?)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}} :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
;; LE BOULANGER ;; LE BOULANGER
{:vendor "Le Boulanger" {:vendor "Le Boulanger"
@@ -263,6 +274,18 @@
:multi #"\n" :multi #"\n"
:multi-match? #"^\s+[\d]{6,8}\s+\d+"} :multi-match? #"^\s+[\d]{6,8}\s+\d+"}
;; ACME BREAD
{:vendor "Acme Bread"
:keywords [#"acmebread\.com"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Print Date.*\n.*\n(.*)"
:invoice-number #"^\s*(\d+)"
:total #"\s{2,}(\d+\.\d{2})\s{2,}"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}
:multi #"\n"
:multi-match? #"^\s*\d+\s+([0-9]+/[0-9]+/[0-9]+)"}
;; PFG - ROMA ;; PFG - ROMA
{:vendor "Performance Food Group - ROMA" {:vendor "Performance Food Group - ROMA"
:keywords [#"Performance Food Group, Inc\n\f"] :keywords [#"Performance Food Group, Inc\n\f"]
@@ -328,6 +351,30 @@
:keywords [#"Alt_invoice_number"] :keywords [#"Alt_invoice_number"]
:extract (fn [wb vendor] :extract (fn [wb vendor]
(let [[sheet] (d/sheet-seq wb)] (let [[sheet] (d/sheet-seq wb)]
(println "COUNT" (count (transduce (comp
(drop-while (fn [c]
(not (re-find #"Customer_id" (str (d/read-cell c))))))
(drop 9)
(filter (fn [c]
(= 0 (.getColumnIndex c))))
(filter (fn [c]
(not (str/blank? (str/trim (or (d/read-cell (d/select-cell (offset c 1 0) sheet)) ""))))))
(map (fn [c]
{:customer-identifier (str/trim (d/read-cell (d/select-cell (offset c 1 0) sheet)))
:text (d/read-cell (d/select-cell (offset c 1 0) sheet))
:full-text (d/read-cell (d/select-cell (offset c 1 0) sheet))
:date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet))))
:invoice-number (->>
(re-find #"^(?:0+([A-Z0-9]+))|([A-Z]+[A-Z0-9]+)" (str/trim (d/read-cell (d/select-cell (offset c 2 0) sheet))))
(drop 1 )
(filter identity)
first)
:total (str (d/read-cell (d/select-cell (offset c 7 0) sheet)))
:vendor-code vendor}))
(filter :customer-identifier))
conj
[]
(d/cell-seq sheet))))
(transduce (comp (transduce (comp
(drop-while (fn [c] (drop-while (fn [c]
(not (re-find #"Customer_id" (str (d/read-cell c)))))) (not (re-find #"Customer_id" (str (d/read-cell c))))))
@@ -342,7 +389,7 @@
:full-text (d/read-cell (d/select-cell (offset c 1 0) sheet)) :full-text (d/read-cell (d/select-cell (offset c 1 0) sheet))
:date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet)))) :date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet))))
:invoice-number (->> :invoice-number (->>
(re-find #"^(?:0+([A-Z0-9]+))|([A-Z]+[A-Z0-9]+)" (str/trim (d/read-cell (d/select-cell (offset c 3 0) sheet)))) (re-find #"^(?:0+([A-Z0-9]+))|([A-Z]+[A-Z0-9]+)" (str/trim (d/read-cell (d/select-cell (offset c 2 0) sheet))))
(drop 1 ) (drop 1 )
(filter identity) (filter identity)
first) first)

58
src/clj/auto_ap/routes/invoices.clj
View File

@@ -176,23 +176,36 @@
'[?vendor :vendor/default-account ?default-account]]} '[?vendor :vendor/default-account ?default-account]]}
:args [(d/db (d/connect uri)) vendor-code]}) :args [(d/db (d/connect uri)) vendor-code]})
first) first)
_ (when-not matching-vendor
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
{:invoice-number invoice-number
:customer-identifier customer-identifier
:vendor-code vendor-code})))
_ (println "matching" customer-identifier "-" matching-vendor) _ (println "matching" customer-identifier "-" matching-vendor)
matching-client (parse/best-match clients customer-identifier) matching-client (parse/best-match clients customer-identifier)
_ (println "New invoice matches client" matching-client) _ (println "invoice \"" invoice-number "\"matches client" (:client/name matching-client))
matching-location (parse/best-location-match matching-client text full-text) matching-location (parse/best-location-match matching-client text full-text)
[existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location) [existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
(try
(->> (d/query (->> (d/query
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client] :in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number] :where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor] [?e :invoice/vendor ?vendor]
[?e :invoice/client ?client] [?e :invoice/client ?client]
[?e :invoice/outstanding-balance ?outstanding-balance] [?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/status ?status] [?e :invoice/status ?status]
[?e :invoice/import-status ?import-status] [?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]} [?import-status :db/ident ?import-status2]]}
:args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]})) :args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]}))
first))] first)
(catch Exception e
(throw (ex-info (str "Failed to find potential matching invoice with"
" invoice " invoice-number
" vendor " matching-vendor
" client " (:client/name matching-client))
{:args [ invoice-number matching-vendor (:db/id matching-client)]})))
))]
(cond (cond
(not (and matching-location matching-client)) (not (and matching-location matching-client))
@@ -219,7 +232,12 @@
)) ))
[] []
imports)] imports)]
(when-not (seq transactions)
(throw (ex-info "No invoices found."
{:imports imports})))
@(d/transact (d/connect uri) (vec (set transactions))))) @(d/transact (d/connect uri) (vec (set transactions)))))
(defroutes routes (defroutes routes
(wrap-routes (wrap-routes
@@ -260,6 +278,7 @@
(context "/invoices" [] (context "/invoices" []
(POST "/upload" (POST "/upload"
<<<<<<< HEAD
{{ file :file } :params :as params} {{ file :file } :params :as params}
#_(clojure.pprint/pprint params) #_(clojure.pprint/pprint params)
(let [{:keys [filename tempfile]} file] (let [{:keys [filename tempfile]} file]
@@ -267,6 +286,21 @@
{:status 200 {:status 200
:body (pr-str {}) :body (pr-str {})
:headers {"Content-Type" "application/edn"}})) :headers {"Content-Type" "application/edn"}}))
=======
{{ files "file"} :params :as params}
(let [{:keys [filename tempfile]} files]
(try
(import-uploaded-invoice (parse/parse-file (.getPath tempfile) filename))
{:status 200
:body (pr-str {})
:headers {"Content-Type" "application/edn"}}
(catch Exception e
{:status 500
:body (pr-str {:message (.getMessage e)
:data (ex-data e)})
:headers {"Content-Type" "application/edn"}}))
))
>>>>>>> master
(POST "/upload-integreat" (POST "/upload-integreat"
{{:keys [excel-rows]} :edn-params user :identity} {{:keys [excel-rows]} :edn-params user :identity}
(assert-admin user) (assert-admin user)

24
src/cljs/auto_ap/views/pages/import_invoices.cljs
View File

@@ -28,7 +28,9 @@
(js/Dropzone. (reagent/dom-node this) (js/Dropzone. (reagent/dom-node this)
(clj->js {:init (fn [] (clj->js {:init (fn []
(.on (js-this) "success" (fn [_ files] (.on (js-this) "success" (fn [_ files]
(re-frame/dispatch [::invalidated])))) (re-frame/dispatch [::invalidated])))
(.on (js-this) "error" (fn [_ error]
(re-frame/dispatch [::errored error]))))
:paramName "file" :paramName "file"
:headers {"Authorization" (str "Token " @token)} :headers {"Authorization" (str "Token " @token)}
:url (str "/api/invoices/upload" :url (str "/api/invoices/upload"
@@ -42,6 +44,11 @@
(fn [db] (fn [db]
(-> db ::invoice-page))) (-> db ::invoice-page)))
(re-frame/reg-sub
::error
(fn [db]
(-> db ::error)))
(re-frame/reg-sub (re-frame/reg-sub
::params ::params
(fn [db] (fn [db]
@@ -57,6 +64,7 @@
(fn [cofx [_ params]] (fn [cofx [_ params]]
{:db (-> (:db cofx) {:db (-> (:db cofx)
(dissoc ::error)
(assoc-in [:status :loading] true) (assoc-in [:status :loading] true)
(assoc-in [::params] params)) (assoc-in [::params] params))
:graphql {:token (-> cofx :db :user) :graphql {:token (-> cofx :db :user)
@@ -72,6 +80,11 @@
(assoc ip :checked (by :id (:invoices ip))))) (assoc ip :checked (by :id (:invoices ip)))))
(assoc-in [:status :loading] false)))) (assoc-in [:status :loading] false))))
(re-frame/reg-event-db
::errored
(fn [db [_ error]]
(assoc db ::error (:message (edn/read-string error)))))
(re-frame/reg-event-fx (re-frame/reg-event-fx
::reject-invoices-clicked ::reject-invoices-clicked
(fn [{:keys [db]} [_ invoices on-success]] (fn [{:keys [db]} [_ invoices on-success]]
@@ -154,12 +167,17 @@
(with-meta (with-meta
(fn [] (fn []
(let [invoice-page (re-frame/subscribe [::invoice-page]) (let [invoice-page (re-frame/subscribe [::invoice-page])
status (re-frame/subscribe [::subs/status])] status (re-frame/subscribe [::subs/status])
error (re-frame/subscribe [::error])]
[:div [:div
[:h1.title "Upload invoices"] [:h1.title "Upload invoices"]
[dropzone] [dropzone]
[:div {:class "section"}] [:div {:class "section"}
(when @error
[:div.notification.is-warning @error])]
[:div {:class "card found-invoices",} [:div {:class "card found-invoices",}
[:div {:class "card-header"} [:div {:class "card-header"}
[:span {:class "card-header-title"} "Found Invoices"]] [:span {:class "card-header-title"} "Found Invoices"]]