From 2e9659557d74f42babdca9a44cee596205c8efed Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 8 Jan 2020 07:10:56 -0800 Subject: [PATCH 1/2] Adding new types. --- src/clj/auto_ap/parse/templates.clj | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index a0bba1a6..6fb6cf1c 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -1,5 +1,6 @@ (ns auto-ap.parse.templates (:require [dk.ative.docjure.spreadsheet :as d] + [auto-ap.parse.util :as u] [clojure.string :as str]) (:import (org.apache.poi.ss.util CellAddress))) @@ -50,7 +51,7 @@ :keywords [#"DVW Commercial"] :extract {:date #"\s*([0-9]+/[0-9]+/[0-9]+)" :customer-identifier #"Bill To:[^\n]+\n[^\n]*\n\s*([\w ]+) \(" - :invoice-number #"Invoice\s*\n\s*([\w\./]+)*" + :invoice-number #"Invoice\s*\n\s*[S]?([\w\./]+)*" :total #"Total:\s+\$ ([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yy"]}} @@ -298,4 +299,29 @@ :vendor-code vendor})))) conj [] + (d/cell-seq sheet))))} + {:vendor "Chef's Choice Produce Co" + :keywords [#"Alt_invoice_number"] + :extract (fn [wb vendor] + (let [[sheet] (d/sheet-seq wb)] + (transduce (comp + (drop-while (fn [c] + (not (re-find #"Customer_id" (str (d/read-cell c)))))) + (drop 9) + (filter (fn [c] + (= 0 (.getColumnIndex c)))) + (filter (fn [c] + (not (str/blank? (str/trim (or (d/read-cell (d/select-cell (offset c 1 0) sheet)) "")))))) + (map (fn [c] + (doto {:customer-identifier (str/trim (d/read-cell (d/select-cell (offset c 1 0) sheet))) + :text (d/read-cell (d/select-cell (offset c 1 0) sheet)) + :full-text (d/read-cell (d/select-cell (offset c 1 0) sheet)) + :date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet)))) + :invoice-number (str/trim (d/read-cell (d/select-cell (offset c 3 0) sheet))) + :total (str (d/read-cell (d/select-cell (offset c 7 0) sheet))) + :vendor-code vendor} + println))) + (filter :customer-identifier)) + conj + [] (d/cell-seq sheet))))}]) From fe8256a488fe7fdda8ba107cdcbc14deba5c416c Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 24 Jan 2020 18:18:31 -0800 Subject: [PATCH 2/2] changes. --- src/clj/auto_ap/parse.clj | 4 +- src/clj/auto_ap/parse/templates.clj | 69 +++++++++++++------ src/cljs/auto_ap/views/components/modal.cljs | 6 +- .../views/components/vendor_dialog.cljs | 28 ++++++-- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index 52fb658f..c68b3c8b 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -38,7 +38,9 @@ (first (map second (re-seq v full-text)))) str/trim ) [value-parser parser-params] (-> template :parser k)] - (assoc result k (try (u/parse-value value-parser parser-params value) + (assoc result k (try + (println "applying parser" value-parser "to value" value) + (u/parse-value value-parser parser-params value) (catch Exception e (println e)))))) {:vendor-code (:vendor template) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index 6fb6cf1c..e5ee60af 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -98,13 +98,36 @@ ;; GOLDEN BRANDS {:vendor "Golden Brands San Jose" :keywords [#"GOLDEN BRANDS"] - :extract {:date #"0430\n(.*)" + :extract {:date #"(?:.*\n){4}(.*)" ;; HOW TO GO TO SPCIFIC LINE :customer-identifier #"Account:(?:.*\n)(.*(?=\s{2,}))" :invoice-number #"Invoice#: (\d+)" :total #"Invoice Total\s+([0-9,]+\.[0-9]{2})"} :parser {:date [:clj-time "EEE MMM dd, yyyy HH:mm aa"] :total [:trim-commas nil]}} + + ;; Young's Market Co + {:vendor "Youngs Market" + :keywords [#"Young's Market Co"] + :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"Customer Name\s+(.*)" + :invoice-number #"^(?:.*?)\s{2,}([0-9]+)" + :total #"([0-9\.,]+)\s+[0-9\.,]+$"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas nil]} + :multi #"\n" + :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+[0-9]+\s+INV "} + + ;; Young's Market Co - INVOICE + {:vendor "Youngs Market" + :keywords [#"P.O.Box 743564"] + :extract {:date #"INVOICE DATE\n(?:.*?)(\S+)\n" + :customer-identifier #"INVOICE DATE\n [0-9]+\s+(.*?)\s{2,}" + :invoice-number #"INVOICE DATE\n(?:.*?)\s{2,}(\d+?)\s+\S+\n" + :total #"Net Amount(?:.*\n){4}(?:.*?)([0-9\.]+)\n"} + :parser {:date [:clj-time "dd-MMM-yy"] + :total [:trim-commas nil]}} + ;; WINE WAREHOUSE {:vendor "Wine Warehouse" :keywords [#"WINE WAREHOUSE"] @@ -228,17 +251,6 @@ :total #"Balance Due\s+\$([0-9\.]+)"} :parser {:date [:clj-time "MM/dd/yyyy"]}} - - ;; PFG - ROMA - {:vendor "Performance Food Group - ROMA" - :keywords [#"Performance Food Group, Inc\n\f"] - :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" - :invoice-number #"INVOICE NO.\s+ ([\d]+)" - :total #"([\d\.,]+)\s+INVOICE TOTAL"} - :parser {:date [:clj-time "MM/dd/yy"] - :total [:trim-commas nil]}} - ;; PFG - ROMA LOOK 1 {:vendor "Performance Food Group - ROMA" :keywords [#"inquiries call 1-800-233-6211"] @@ -249,7 +261,19 @@ :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas-and-negate nil]} :multi #"\n" - :multi-match? #"^\s+[\d]{6,8}\s+\d+"} + :multi-match? #"^\s+[\d]{6,8}\s+\d+"} + + ;; PFG - ROMA + {:vendor "Performance Food Group - ROMA" + :keywords [#"Performance Food Group, Inc\n\f"] + :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :invoice-number #"INVOICE NO.\s+ ([\d]+)" + :total #"([\d\.,]+)\s+INVOICE TOTAL"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas nil]}} + + ;; JFC {:vendor "JFC International" @@ -313,14 +337,17 @@ (filter (fn [c] (not (str/blank? (str/trim (or (d/read-cell (d/select-cell (offset c 1 0) sheet)) "")))))) (map (fn [c] - (doto {:customer-identifier (str/trim (d/read-cell (d/select-cell (offset c 1 0) sheet))) - :text (d/read-cell (d/select-cell (offset c 1 0) sheet)) - :full-text (d/read-cell (d/select-cell (offset c 1 0) sheet)) - :date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet)))) - :invoice-number (str/trim (d/read-cell (d/select-cell (offset c 3 0) sheet))) - :total (str (d/read-cell (d/select-cell (offset c 7 0) sheet))) - :vendor-code vendor} - println))) + {:customer-identifier (str/trim (d/read-cell (d/select-cell (offset c 1 0) sheet))) + :text (d/read-cell (d/select-cell (offset c 1 0) sheet)) + :full-text (d/read-cell (d/select-cell (offset c 1 0) sheet)) + :date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet)))) + :invoice-number (->> + (re-find #"^(?:0+([A-Z0-9]+))|([A-Z]+[A-Z0-9]+)" (str/trim (d/read-cell (d/select-cell (offset c 3 0) sheet)))) + (drop 1 ) + (filter identity) + first) + :total (str (d/read-cell (d/select-cell (offset c 7 0) sheet))) + :vendor-code vendor})) (filter :customer-identifier)) conj [] diff --git a/src/cljs/auto_ap/views/components/modal.cljs b/src/cljs/auto_ap/views/components/modal.cljs index a5155e33..b37d733d 100644 --- a/src/cljs/auto_ap/views/components/modal.cljs +++ b/src/cljs/auto_ap/views/components/modal.cljs @@ -21,7 +21,7 @@ [:footer.modal-card-foot foot])]]) -(defn action-modal [{:keys [title action-text id save-event can-submit? status-from] :or {can-submit? true}} & rest] +(defn action-modal [{:keys [title warning action-text id save-event can-submit? status-from] :or {can-submit? true}} & rest] (let [{:keys [visible? saving? error-message]} @(re-frame/subscribe [::subs/modal-state id status-from])] (when visible? [:form {:id id @@ -41,6 +41,8 @@ :id id :hide-event [::events/modal-status id {:visible? false}]} (when error-message - [:div.notification.is-warning error-message])] + [:div.notification.is-warning error-message]) + (when warning + [:div.notification.is-info warning])] (into (r/children (r/current-component))) (into [(when saving? [:div.is-overlay {:style {"backgroundColor" "rgba(150,150,150, 0.5)"}}])]))]))) diff --git a/src/cljs/auto_ap/views/components/vendor_dialog.cljs b/src/cljs/auto_ap/views/components/vendor_dialog.cljs index 61e3f8c9..923651d6 100644 --- a/src/cljs/auto_ap/views/components/vendor_dialog.cljs +++ b/src/cljs/auto_ap/views/components/vendor_dialog.cljs @@ -6,14 +6,34 @@ [auto-ap.views.components.typeahead :refer [typeahead]] [auto-ap.expense-accounts :refer [chooseable-expense-accounts]] + [clj-fuzzy.metrics :refer [jaccard]] [clojure.spec.alpha :as s] + [clojure.string :as str] [auto-ap.entities.vendors :as entity] [auto-ap.entities.contact :as contact] [auto-ap.subs :as subs])) +(defn ngrams [text len] + (mapv #(.toLowerCase (str/join "" %)) + (partition len 1 text))) + +(defn partial-matches-vendor? [text vendors] + (when (> (count text) 4) + (let [text (.toLowerCase text)] + (->> vendors + (map :name) + (mapcat + (fn [v] (mapv + (fn [n] [v (jaccard text n )]) + (ngrams v (count text))))) + #_(sort-by second) + (filter #(< (second %) 0.3)) + (map first) + first)))) (defn vendor-dialog [{:keys [vendor save-event change-event id] {:keys [name]} :vendor}] - (println (s/explain ::entity/vendor vendor) ) - (let [clients-by-id @(re-frame/subscribe [::subs/clients-by-id])] + + (let [clients-by-id @(re-frame/subscribe [::subs/clients-by-id]) + all-vendors @(re-frame/subscribe [::subs/vendors])] [action-modal {:id id :title [:span (if (:id vendor) (str "Edit " (or name "")) @@ -21,12 +41,12 @@ (when (:error vendor) [:span.icon.has-text-danger [:i.fa.fa-exclamation-triangle]])] + :warning (when-let [vendor (partial-matches-vendor? (:name vendor) all-vendors)] + (str "Are you sure you don't mean " vendor "?")) :action-text "Save" :save-event save-event :can-submit? (s/valid? ::entity/vendor vendor)} - - [horizontal-field [:label.label "Name"] [:div.control