From 806e882d5cdd61b8a4d5f064330dba1b448ad569 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 29 Oct 2019 22:11:57 -0700 Subject: [PATCH 01/14] lots of improvements. --- src/clj/auto_ap/graphql/clients.clj | 14 ++++++++++- src/clj/auto_ap/parse/csv.clj | 11 +++++--- src/clj/auto_ap/routes/auth.clj | 25 +++++++++++-------- .../auto_ap/views/pages/admin/clients.cljs | 19 ++++++++++++-- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index fd4a3fe2..38429723 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -25,6 +25,9 @@ id (or (:db/id client) "new-client") _ (println id) _ (println edit_client) + _ (when client + @(d/transact (d/connect uri) + (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client)))) transactions [(remove-nils {:db/id id :client/code (if (str/blank? (:client/code client)) (:code edit_client) @@ -64,7 +67,16 @@ })] result @(d/transact (d/connect uri) transactions)] (println result "ID" id) - (-> result :tempids (get id) (or id) d-clients/get-by-id ->graphql))) + (-> result :tempids (get id) (or id) d-clients/get-by-id + (update :client/location-matches + (fn [lms] + (mapcat (fn [lm] + (map (fn [m] + {:location-match/match m + :location-match/location (:location-match/location lm)}) + (:location-match/matches lm))) + lms))) + ->graphql))) (defn get-client [context args value] diff --git a/src/clj/auto_ap/parse/csv.clj b/src/clj/auto_ap/parse/csv.clj index f70b7727..318ee98d 100644 --- a/src/clj/auto_ap/parse/csv.clj +++ b/src/clj/auto_ap/parse/csv.clj @@ -20,16 +20,19 @@ (defmethod parse-csv :mama-lus [rows] - (println "MAMA LU") + (println "MAMA LU4") (transduce (comp (drop 1) (map (fn [[_ po-number despatch-number invoice-number invoice-date customer value :as row]] {:vendor-code "Mama Lu's Foods" :customer-identifier customer :invoice-number (str po-number "-" invoice-number ) - :date (u/parse-value :clj-time "MM/dd/yy HH:ss" invoice-date) - :total value - :text (str/join " " row)}))) + :date (try (u/parse-value :clj-time "M/d/yyyy HH:ss" invoice-date) + (catch Exception _ + (u/parse-value :clj-time "M/d/yyyy" invoice-date))) + :total (str/replace value #"," "") + :text (str/join " " row) + :full-text (str/join " " row)}))) conj [] rows)) diff --git a/src/clj/auto_ap/routes/auth.clj b/src/clj/auto_ap/routes/auth.clj index 1a8e3141..627cc7d6 100644 --- a/src/clj/auto_ap/routes/auth.clj +++ b/src/clj/auto_ap/routes/auth.clj @@ -19,6 +19,7 @@ (defroutes routes (GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers} + (println "Authenticating with" r "..." code) (try (let [auth (-> "https://accounts.google.com/o/oauth2/token" (http/post @@ -44,17 +45,19 @@ ;; TODO - these namespaces are not being transmitted/deserialized properly (if (and token user) - {:status 301 - :headers {"Location" (str "/?jwt=" (jwt/sign (doto {:user (:name profile) - :exp (time/plus (time/now) (time/days 30)) - :user/clients (map (fn [c] - (dissoc c :client/bank-accounts )) - (:user/clients user)) - :user/role (name (:user/role user)) - :user/name (:name profile)} - println) - (:jwt-secret env) - {:alg :hs512}))}} + (let [jwt (jwt/sign (doto {:user (:name profile) + :exp (time/plus (time/now) (time/days 30)) + :user/clients (map (fn [c] + (dissoc c :client/bank-accounts :client/location-matches)) + (:user/clients user)) + :user/role (name (:user/role user)) + :user/name (:name profile)} + println) + (:jwt-secret env) + {:alg :hs512})] + (println "authenticated. using jwt" jwt) + {:status 301 + :headers {"Location" (str "/?jwt=" jwt)}}) {:status 401 :body "Couldn't authenticate"})) (catch Exception e diff --git a/src/cljs/auto_ap/views/pages/admin/clients.cljs b/src/cljs/auto_ap/views/pages/admin/clients.cljs index cc2b0ee9..41e4098c 100644 --- a/src/cljs/auto_ap/views/pages/admin/clients.cljs +++ b/src/cljs/auto_ap/views/pages/admin/clients.cljs @@ -31,6 +31,7 @@ ::edit-client-clicked (fn [{:keys [db]} [_ client-id]] {:db (-> db + (forms/stop-form ::new-client) (forms/start-form ::new-client (get (:clients db) client-id)))})) (re-frame/reg-sub @@ -120,6 +121,18 @@ (update :location-matches conj (:location-match client)) (dissoc :location-match)))) +(re-frame/reg-event-db + ::remove-location-match + [(forms/in-form ::new-client) (re-frame/path [:data])] + (fn [client [_ i]] + (-> client + (update :location-matches (fn [lm] + (->> lm + (map vector (range)) + (filter (fn [[index item]] + (not= index i))) + (map second))))))) + (re-frame/reg-event-db ::add-new-bank-account [(forms/in-form ::new-client) (re-frame/path [:data])] @@ -402,9 +415,11 @@ :event change-event :subscription new-client}]]] [:p.control [:button.button.is-primary {:on-click (dispatch-event [::add-new-location-match])} "Add"]]] + [:ul - (for [{:keys [location match]} (:location-matches new-client)] - ^{:key location} [:li match "->" location ])]]] + (for [[index {:keys [location match]}] (map vector (range) (:location-matches new-client))] + ^{:key index} [:li match "->" location [:a {:on-click (dispatch-event [::remove-location-match index])} [:span.icon + [:span.fa.fa-times]]]])]]] [:div {:style {:padding-bottom "0.75em" :padding-top "0.75em"}} [:h2.subtitle "Address"] From fdf1dab0ac895c885befe86be6946512e6cf4989 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 1 Nov 2019 17:29:16 -0700 Subject: [PATCH 02/14] le boulanger, others. --- src/clj/auto_ap/graphql.clj | 2 ++ src/clj/auto_ap/graphql/clients.clj | 5 ++- src/clj/auto_ap/parse.clj | 2 -- src/clj/auto_ap/parse/templates.clj | 19 ++++++++-- src/clj/auto_ap/yodlee/import.clj | 5 +++ src/cljs/auto_ap/events.cljs | 4 +-- .../auto_ap/views/pages/admin/clients.cljs | 36 ++++++++++++++++++- 7 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 4abfe4a0..88e62287 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -50,6 +50,7 @@ :address {:type :address} :location_matches {:type '(list :location_match)} :locations {:type '(list String)} + :matches {:type '(list String)} :bank_accounts {:type '(list :bank_account)}}} :contact {:fields {:id {:type :id} @@ -298,6 +299,7 @@ :email {:type 'String} :address {:type :add_address} :locations {:type '(list String)} + :matches {:type '(list String)} :location_matches {:type '(list :edit_location_match)} :bank_accounts {:type '(list :edit_bank_account)}}} :edit_bank_account diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index 38429723..b1de91a0 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -27,12 +27,15 @@ _ (println edit_client) _ (when client @(d/transact (d/connect uri) - (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client)))) + (into + (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client)) + (mapv (fn [m] [:db/retract (:db/id client) :client/matches m]) (:client/matches client))))) transactions [(remove-nils {:db/id id :client/code (if (str/blank? (:client/code client)) (:code edit_client) (:client/code client)) :client/name (:name edit_client) + :client/matches (:matches edit_client) :client/email (:email edit_client) :client/locations (filter identity (:locations edit_client)) :client/location-matches (->> (:location_matches edit_client) diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index e666340d..cd6b0908 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -79,7 +79,6 @@ (let [fuzzy-match (->> clients (mapcat (fn [{:keys [:db/id :client/matches :client/name] :as client :or {matches []}}] (map (fn [m] - (println m invoice-client-name) [client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))]) (conj matches name)))) (filter #(< (second %) 0.25)) @@ -111,7 +110,6 @@ (map (fn [match] [location match]) matches))) (filter (fn [[location match]] - (println "loc " location match text) (re-find (re-pattern (str "(?i)" match)) text)) ) first first) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index f9e8fd7c..b4c2edba 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -61,7 +61,7 @@ :parser {:date [:clj-time "MM/dd/yyyy"]} :multi #"\n" :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"} - {:vendor "Performance Food Group" + {:vendor "Performance Food Group - LEDYARD" :keywords [#"performancefoodservice"] :extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)" :customer-identifier #"DELIVER TO[^\n]+\n\s*[\S ]+?(?=\s{2,}([\S ]+?)\s{2,})" ;; ([\S ]+)\s{2,} @@ -81,7 +81,22 @@ :customer-identifier #"INVOICE NUMBER[^\n]+\n[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,} :invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})" :total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+)"} - :parser {:date [:clj-time "MM/dd/yyyy"]}}]) + :parser {:date [:clj-time "MM/dd/yyyy"]}} + {:vendor "Le Boulanger" + :keywords [#"Le Boulanger"] + :extract {:date #"Invoice Date: ([^\n]+)\n" + :customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})" + :invoice-number #"Invoice No: ([^\n]+)\n" + :total #" Total:\s+([\d\.]+)"} + :parser {:date [:clj-time "MMM dd, yyyy"]}} + {:vendor "Performance Food Group - ROMA" + :keywords [#"Performance Food Group, Inc\n\f"] + :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :invoice-number #"INVOICE NO.\s+ ([\d]+)" + :total #"([\d\.,]+)\s+INVOICE TOTAL"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas nil]}}]) (defn offset [c x y] (.toString (CellAddress. (+ y (.getRow (.getAddress c))) (+ x (.getColumn (.getAddress c))) ))) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 942e0485..03e3edf1 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -115,6 +115,11 @@ (defn do-import [] (let [transactions (client/get-transactions) + _ (println "Count of transactions" (->> transactions + (group-by transactions :accountId) + (reduce-kv (fn [acc k v] + (assoc acc k (count v))) + {}))) #_#__ (println "All accounts:" (client/get-accounts)) #_#__ (println "ALL Transactions:" transactions) all-bank-accounts (mapcat (fn [c] (map diff --git a/src/cljs/auto_ap/events.cljs b/src/cljs/auto_ap/events.cljs index 7ec41a5a..fd04f401 100644 --- a/src/cljs/auto_ap/events.cljs +++ b/src/cljs/auto_ap/events.cljs @@ -42,7 +42,7 @@ :graphql {:token token :query-obj {:venia/queries [[:client - [:id :name :code :email :locations [:location-matches [:location :match]] [:bank-accounts [:id :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id] ] + [:id :name :code :email :matches :locations [:location-matches [:location :match]] [:bank-accounts [:id :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id] ] [:address [:street1 :street2 :city :state :zip]]]] [:vendor [:id :name :default-expense-account [:primary-contact [:name :phone :email :id]] [:secondary-contact [:id :name :phone :email]] :print-as :invoice-reminder-schedule :code]]]} @@ -65,7 +65,7 @@ (fn [{:keys [db]} [_ token user]] {:graphql {:token token :query-obj {:venia/queries [[:client - [:id :name :code [:location-matches [:location :match]] [:address [:street1 :street2 :city :state :zip]] [:bank-accounts [:id :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id] ]]] + [:id :name :code :matches :locations [:location-matches [:location :match]] [:address [:street1 :street2 :city :state :zip]] [:bank-accounts [:id :code :number :bank-name :bank-code :check-number :name :routing :type :sort-order :visible :yodlee-account-id] ]]] [:vendor [:id :name :default-expense-account [:primary-contact [:name :phone :email :id]] [:secondary-contact [:id :name :phone :email]] :print-as :invoice-reminder-schedule :code]]]} diff --git a/src/cljs/auto_ap/views/pages/admin/clients.cljs b/src/cljs/auto_ap/views/pages/admin/clients.cljs index 41e4098c..36b46c8e 100644 --- a/src/cljs/auto_ap/views/pages/admin/clients.cljs +++ b/src/cljs/auto_ap/views/pages/admin/clients.cljs @@ -43,6 +43,7 @@ :code (:code new-client-data) ;; TODO add validation can't change :email (:email new-client-data) :locations (:locations new-client-data) + :matches (vec (:matches new-client-data)) :location-matches (:location-matches new-client-data) :address {:street1 (:street1 (:address new-client-data)) :street2 (:street2 (:address new-client-data)), @@ -87,7 +88,7 @@ :operation/name "EditClient"} :venia/queries [{:query/data [:edit-client {:edit-client new-client-req} - [:id :name :code :email :locations [:location-matches [:location :match]] [:address [:street1 :street2 :city :state :zip]] [:bank-accounts [:id :number :check-number :name :code :bank-code :bank-name :routing :type :visible :yodlee-account-id :sort-order]]]]}]} + [:id :name :code :email :locations :matches [:location-matches [:location :match]] [:address [:street1 :street2 :city :state :zip]] [:bank-accounts [:id :number :check-number :name :code :bank-code :bank-name :routing :type :visible :yodlee-account-id :sort-order]]]]}]} :on-success [::save-complete] :on-error [::forms/save-error ::new-client]}} {:db new-client-form})))) @@ -113,6 +114,23 @@ (update :locations conj (:location client)) (dissoc :location)))) +(re-frame/reg-event-db + ::add-new-match + [(forms/in-form ::new-client) (re-frame/path [:data])] + (fn [client _] + (-> client + (update :matches conj (:match client)) + (update :matches set) + (dissoc :match)))) + +(re-frame/reg-event-db + ::remove-match + [(forms/in-form ::new-client) (re-frame/path [:data])] + (fn [client [_ which]] + (-> client + (update :matches set) + (update :matches disj which)))) + (re-frame/reg-event-db ::add-new-location-match [(forms/in-form ::new-client) (re-frame/path [:data])] @@ -378,6 +396,22 @@ :spec ::entity/email :event change-event :subscription new-client}]]]] + [:div.field + [:p.help "Matches"] + [:div.control + [:div.field.has-addons + [:p.control + [bind-field + [:input.input {:type "text" + :field :match + :event change-event + :subscription new-client}]]] + [:p.control [:button.button.is-primary {:on-click (dispatch-event [::add-new-match])} "Add"]]]] + [:ul + (for [match (:matches new-client)] + ^{:key match} [:li match [:a {:on-click (dispatch-event [::remove-match match])} [:span.icon [:span.fa.fa-times]]]])]] + + [:div.field [:p.help "Locations"] From d2e48125d1a30660917d61a3a6fdf85989c46136 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 5 Nov 2019 07:22:48 -0800 Subject: [PATCH 03/14] fix. --- src/clj/auto_ap/yodlee/import.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 03e3edf1..b3d494d4 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -115,8 +115,9 @@ (defn do-import [] (let [transactions (client/get-transactions) + _ (println "Total transactions" (count transactions)) _ (println "Count of transactions" (->> transactions - (group-by transactions :accountId) + (group-by :accountId) (reduce-kv (fn [acc k v] (assoc acc k (count v))) {}))) From 141e5b06f1eb39f4b7011ec7085f5cf1a5ee43d2 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 17 Nov 2019 12:07:06 -0800 Subject: [PATCH 04/14] shows the matching information. --- src/clj/auto_ap/datomic/migrate.clj | 7 +++++++ src/clj/auto_ap/graphql.clj | 1 + src/clj/auto_ap/routes/invoices.clj | 1 + src/cljs/auto_ap/views/components/invoice_table.cljs | 8 +++++--- src/cljs/auto_ap/views/pages/import_invoices.cljs | 5 +++++ 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 769c1505..3c3b4101 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -46,6 +46,12 @@ :invoice-status/paid :invoice-status/unpaid)]])})}]] ) +(def add-client-identifier + [[{:db/ident :invoice/client-identifier + :db/doc "An identifier found in an uploaded invoice" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one}]]) + (defn -main [& args] (println "Creating database...") (d/create-database uri) @@ -78,6 +84,7 @@ :auto-ap/add-new-vendors {:txes-fn 'auto-ap.datomic.migrate.add-new-vendors/add-new-vendors :requires [:auto-ap/fix-check-numbers]} :auto-ap/add-account-visibility-fields {:txes-fn 'auto-ap.datomic.migrate.account-sorting/add-account-visibility-fields :requires [:auto-ap/add-new-vendors]} :auto-ap/make-every-account-visible {:txes-fn 'auto-ap.datomic.migrate.account-sorting/make-every-account-visible :requires [:auto-ap/add-account-visibility-fields]} + :auto-ap/add-client-identifier2 {:txes add-client-identifier :requires [:auto-ap/make-every-account-visible]} }] (println "Conforming database...") diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 88e62287..2bc56c98 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -174,6 +174,7 @@ :invoice {:fields {:id {:type :id} :original_id {:type 'Int} + :client_identifier {:type 'String} :total {:type 'String} :outstanding_balance {:type 'String} :invoice_number {:type 'String} diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 7ec2e141..d5aff19b 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -199,6 +199,7 @@ :else (conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client) + :invoice/client-identifier customer-identifier :invoice/vendor matching-vendor :invoice/invoice-number invoice-number :invoice/total (Double/parseDouble total) diff --git a/src/cljs/auto_ap/views/components/invoice_table.cljs b/src/cljs/auto_ap/views/components/invoice_table.cljs index e7e9515a..99e997f9 100644 --- a/src/cljs/auto_ap/views/components/invoice_table.cljs +++ b/src/cljs/auto_ap/views/components/invoice_table.cljs @@ -47,7 +47,7 @@ {:venia/queries [[:invoice_page (assoc params :client-id (:id @(re-frame/subscribe [::subs/client]))) - [[:invoices [:id :total :outstanding-balance :invoice-number :date :status + [[:invoices [:id :total :outstanding-balance :invoice-number :date :status :client-identifier [:vendor [:name :id]] [:expense_accounts [:amount :id :expense_account_id :location [:expense_account [:id :name :location [:parent [:id :name]]]]]] @@ -58,7 +58,7 @@ :start :end]]]}) -(defn invoice-table [{:keys [id invoice-page status on-params-change vendors params check-boxes checked on-check-changed on-edit-invoice on-void-invoice on-unvoid-invoice expense-event]}] +(defn invoice-table [{:keys [id invoice-page status on-params-change vendors params check-boxes checked on-check-changed on-edit-invoice on-void-invoice on-unvoid-invoice expense-event overrides]}] (let [visible-checks @(re-frame/subscribe [::visible-checks]) visible-expense-accounts @(re-frame/subscribe [::visible-expense-accounts]) selected-client @(re-frame/subscribe [::subs/client]) @@ -158,7 +158,9 @@ (on-check-changed id i)))} ]]) (when-not selected-client - [:td (:name client)]) + [:td (if-let [client-override (:client overrides)] + (client-override i) + (:name client))]) [:td (:name vendor)] [:td invoice-number] [:td (date->str date) ] diff --git a/src/cljs/auto_ap/views/pages/import_invoices.cljs b/src/cljs/auto_ap/views/pages/import_invoices.cljs index 0c889f3c..31db5e41 100644 --- a/src/cljs/auto_ap/views/pages/import_invoices.cljs +++ b/src/cljs/auto_ap/views/pages/import_invoices.cljs @@ -171,6 +171,11 @@ (if (seq (:invoices @invoice-page)) [invoice-table {:id :approved :invoice-page invoice-page + :overrides {:client (fn [row] + [:p (:name (:client row)) + [:p [:i.is-size-7 (:client-identifier row)]]] + + )} :check-boxes true :checked (:checked @invoice-page) :on-check-changed (fn [which invoice] From ed592aaa983882644974c300737133d530a1bfb2 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 17 Nov 2019 19:44:24 -0800 Subject: [PATCH 05/14] Added new vendors --- src/clj/auto_ap/parse/templates.clj | 17 +++++++++++++---- src/clj/auto_ap/yodlee/core.clj | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index b4c2edba..7bacb01f 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -55,12 +55,12 @@ {:vendor "Southbay Fresh Produce" :keywords [#"SOUTH BAY FRESH PRODUCE"] :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"FAX:[^\n]+\n\s+([A-Za-z ]+)\s{2}" - :invoice-number #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)" - :total #"\$([0-9.]+)"} + :customer-identifier #"To:[^\n]*\n\s+([A-Za-z' ]+)\s{2}" + :invoice-number #"INV #\/(\d+)" + :total #"\$([0-9.]+)\."} :parser {:date [:clj-time "MM/dd/yyyy"]} :multi #"\n" - :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"} + :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+INV "} {:vendor "Performance Food Group - LEDYARD" :keywords [#"performancefoodservice"] :extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)" @@ -89,6 +89,15 @@ :invoice-number #"Invoice No: ([^\n]+)\n" :total #" Total:\s+([\d\.]+)"} :parser {:date [:clj-time "MMM dd, yyyy"]}} + {:vendor "A&B Produce" + :keywords [#"ABProduce"] + :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"BILL TO:[^\n]+\n[^\n]+\n[^\n]+\n(.*)\s{2,}" + :invoice-number #"(\d+)\s+INV" + :total #" INV\s+([\d\.]+)"} + :parser {:date [:clj-time "MM/dd/yyyy"]} + :multi #"\n" + :multi-match? #"^\s+[0-9]+/[0-9]+/[0-9]+\s+\d+\s+INV\s+"} {:vendor "Performance Food Group - ROMA" :keywords [#"Performance Food Group, Inc\n\f"] :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj index e7c04a43..1554a240 100644 --- a/src/clj/auto_ap/yodlee/core.clj +++ b/src/clj/auto_ap/yodlee/core.clj @@ -160,6 +160,21 @@ (recur (concat transactions transaction-batch) (+ batch-size skip)) transactions))))) +(defn count-specific-transactions [account] + (let [cob-session (login-cobrand) + user-session (login-user cob-session)] + + (-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account) + (doto println) + + (client/get {:headers (doto + (merge base-headers {"Authorization" (auth-header cob-session user-session)}) + println) + :as :json}) + :body + :transaction + ))) + (defn get-access-token [] (let [cob-session (login-cobrand) user-session (login-user cob-session) From 23b374c52977de2268a98620fc0f3f0dc05e6791 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 17 Nov 2019 19:58:10 -0800 Subject: [PATCH 06/14] another vendor. --- src/clj/auto_ap/parse/templates.clj | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index 7bacb01f..f0929264 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -5,7 +5,9 @@ (def pdf-templates - [{:vendor "CHFW" + [ + ;; CHEF's WAREHOUSE + {:vendor "CHFW" :keywords [#"CHEF'S WAREHOUSE"] :extract {:total #"2 WKS C\.C\.\s+([\d.,]+)" :customer-identifier #"\n([A-Z][A-Z ]+)\s{2,}" @@ -13,6 +15,7 @@ :invoice-number #"\s+[0-9]+/[0-9]+/[0-9]+\s+([0-9]+)"} :parser {:date [:clj-time "MM/dd/yyyy"]}} + ;; GGM {:vendor "Golden Gate Meat Company, Inc" :keywords [#"Golden Gate Meat"] :extract {:total #"Invoice Total\:\s+\$([\d.,]+)" @@ -22,6 +25,7 @@ :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas nil]}} + ;; CINTAS {:vendor "CINTAS" :keywords [#"CINTAS CORPORATION"] :extract {:invoice-number #"INVOICE\s#\s+([\d.,]+)" @@ -31,6 +35,7 @@ :parser {:date [:clj-time "MM/dd/yy"]} :multi #"\f\f"} + ;; CARBONIC {:vendor "Carbonic Service Inc" :keywords [#"CARBONIC SERVICE INC"] :extract {:invoice-number #"Invoice #\s*\n\s*[\w\.]+\s+[\w\./]+(.*)\s*\n" @@ -38,6 +43,8 @@ :date #"Invoice #\s*\n\s*[\w\.]+\s+([\w\./]+)" :total #"Total\s+\$([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yy"]}} + + ;; DVW {:vendor "DVW Commercial" :keywords [#"DVW Commercial"] :extract {:date #"\s*([0-9]+/[0-9]+/[0-9]+)" @@ -45,6 +52,8 @@ :invoice-number #"Invoice\s*\n\s*([\w\./]+)*" :total #"Total:\s+\$ ([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yy"]}} + + ;; DAYLIGHT FOOD {:vendor "Daylight Foods" :keywords [#"DAYLIGHT FOODS"] :extract {:date #"\n\s*Date[^\n]+\n\s*([0-9]+/[0-9]+/[0-9]+)" @@ -52,6 +61,8 @@ :invoice-number #"Invoice\s([\w\./]+)*" :total #"Total Invoice\s+([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yy"]}} + + ;; SOUTHBAY FRESH {:vendor "Southbay Fresh Produce" :keywords [#"SOUTH BAY FRESH PRODUCE"] :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" @@ -61,6 +72,8 @@ :parser {:date [:clj-time "MM/dd/yyyy"]} :multi #"\n" :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+INV "} + + ;; PFG - LEDYARD {:vendor "Performance Food Group - LEDYARD" :keywords [#"performancefoodservice"] :extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)" @@ -68,6 +81,8 @@ :invoice-number #"DELIVER TO[^\n]+\n.+?(?=\d+)(\d+)\s*\n" :total #"([0-9.]+)\s+Status Code"} :parser {:date [:clj-time "MM/dd/yy"]}} + + ;; US FOODS {:vendor "US Foods" :keywords [#"US Foods"] :extract {:date #"INVOICE NUMBER[^\n]+\n\n\d+\s+\d+\s+([0-9]+/[0-9]+/[0-9]+)" @@ -75,6 +90,8 @@ :invoice-number #"INVOICE NUMBER[^\n]+\n\n\d+\s+(\d+)" :total #"DELIVERED AMOUNT\s+\$([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yyyy"]}} + + ;; SYSCO {:vendor "Sysco" :keywords [#"SYSCO"] :extract {:date #"INVOICE NUMBER[^\n]+\n([^\n]+)\n" @@ -82,6 +99,8 @@ :invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})" :total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yyyy"]}} + + ;; LE BOULANGER {:vendor "Le Boulanger" :keywords [#"Le Boulanger"] :extract {:date #"Invoice Date: ([^\n]+)\n" @@ -89,6 +108,9 @@ :invoice-number #"Invoice No: ([^\n]+)\n" :total #" Total:\s+([\d\.]+)"} :parser {:date [:clj-time "MMM dd, yyyy"]}} + + + ;; A&B {:vendor "A&B Produce" :keywords [#"ABProduce"] :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" @@ -98,6 +120,20 @@ :parser {:date [:clj-time "MM/dd/yyyy"]} :multi #"\n" :multi-match? #"^\s+[0-9]+/[0-9]+/[0-9]+\s+\d+\s+INV\s+"} + + ;; CHEF's CHOICE + {:vendor "Chef's Choice Produce Co" + :keywords [#"2170 MARTIN AVENUE"] + :extract {:date #"([0-9]{10,10})\s+INVOICE" + :customer-identifier #"\n B\s+([A-Za-z ']+)\s+I " + :invoice-number #"([0-9]+)\s+[0-9]{10,10}\s+INVOICE" + :total #"INVOICE\s+([\d\.]+)"} + :parser {:date [:clj-time "MM1dd1yyyy"]} + :multi #"\n" + :multi-match? #"\s+INVOICE\s+"} + + + ;; PFG - ROMA {:vendor "Performance Food Group - ROMA" :keywords [#"Performance Food Group, Inc\n\f"] :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" From 34bad0dba25c425f878d96bd6c9316a19b08d7c7 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 17 Nov 2019 20:08:09 -0800 Subject: [PATCH 07/14] new one. --- src/clj/auto_ap/parse/templates.clj | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index f0929264..c36e87dc 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -132,6 +132,15 @@ :multi #"\n" :multi-match? #"\s+INVOICE\s+"} + ;; FRESH AND BELT + {:vendor "Fresh and Best Produce" + :keywords [#"freshbestproduce"] + :extract {:date #"\n\s+([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"Bill To[^\n]+\n([A-Za-z ']+)" + :invoice-number #"\n\s+[0-9/]+\s+(\d+)" + :total #"Balance Due\s+\$([0-9\.]+)"} + :parser {#_#_:date [:clj-time "MM/dd/yyyy"]}} + ;; PFG - ROMA {:vendor "Performance Food Group - ROMA" From 12d38710f0f2e3e8fd7b16a20eebe786698fb8d8 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 20 Nov 2019 18:59:00 -0800 Subject: [PATCH 08/14] fix. --- src/clj/auto_ap/parse/templates.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index c36e87dc..09f65c45 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -124,11 +124,11 @@ ;; CHEF's CHOICE {:vendor "Chef's Choice Produce Co" :keywords [#"2170 MARTIN AVENUE"] - :extract {:date #"([0-9]{10,10})\s+INVOICE" - :customer-identifier #"\n B\s+([A-Za-z ']+)\s+I " - :invoice-number #"([0-9]+)\s+[0-9]{10,10}\s+INVOICE" + :extract {:date #"([0-9/]{10,10})" + :customer-identifier #"\n B\s+([\S ]+?)(?=\s{2,}I) " + :invoice-number #"\n([0-9]+)" :total #"INVOICE\s+([\d\.]+)"} - :parser {:date [:clj-time "MM1dd1yyyy"]} + :parser {:date [:clj-time "MM/dd/yyyy"]} ;; may want to try two approaches [:clj-time ["MM/dd/yyyy" "MM1dd1yyyy"]] :multi #"\n" :multi-match? #"\s+INVOICE\s+"} From dec5a800c434f1dacec87445c76eae8086039156 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 23 Nov 2019 07:05:41 -0800 Subject: [PATCH 09/14] minor fix. --- src/clj/auto_ap/routes/invoices.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index d5aff19b..485460ef 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -216,7 +216,7 @@ [] imports)] - @(d/transact (d/connect uri) transactions) + @(d/transact (d/connect uri) (vec (set transactions))) )) (defroutes routes From 94cc8f0f9a537cbe5d64ddf6d4aed5c135ad73d1 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 30 Nov 2019 16:24:07 -0800 Subject: [PATCH 10/14] forgiveness. --- src/clj/auto_ap/parse.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index cd6b0908..6b2eceb9 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -19,6 +19,7 @@ (defn extract-template ([text template] + (println "template" template) (if (:multi template) (mapcat #(extract-template % text (dissoc template :multi)) @@ -37,12 +38,15 @@ (first (map second (re-seq v full-text)))) str/trim ) [value-parser parser-params] (-> template :parser k)] - (assoc result k (u/parse-value value-parser parser-params value)))) + (assoc result k (try (u/parse-value value-parser parser-params value) + (catch Exception e + (println e)))))) {:vendor-code (:vendor template) :text text :full-text full-text}))]))) (defn parse [text] + (println "Parsing PDF " text) (reset! last-text text) (->> t/pdf-templates (filter (partial template-applies? text)) From 8cdc91d05782185f4e2266501b2f7bf97deee2e8 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 30 Nov 2019 18:07:27 -0800 Subject: [PATCH 11/14] support for new templates. --- src/clj/auto_ap/parse/templates.clj | 26 ++++++++++++++++++++++++-- src/clj/auto_ap/parse/util.clj | 11 +++++++++++ src/clj/auto_ap/routes/invoices.clj | 1 - 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index 09f65c45..81f6604b 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -139,7 +139,7 @@ :customer-identifier #"Bill To[^\n]+\n([A-Za-z ']+)" :invoice-number #"\n\s+[0-9/]+\s+(\d+)" :total #"Balance Due\s+\$([0-9\.]+)"} - :parser {#_#_:date [:clj-time "MM/dd/yyyy"]}} + :parser {:date [:clj-time "MM/dd/yyyy"]}} ;; PFG - ROMA @@ -150,7 +150,29 @@ :invoice-number #"INVOICE NO.\s+ ([\d]+)" :total #"([\d\.,]+)\s+INVOICE TOTAL"} :parser {:date [:clj-time "MM/dd/yy"] - :total [:trim-commas nil]}}]) + :total [:trim-commas nil]}} + + ;; PFG - ROMA LOOK 1 + {:vendor "Performance Food Group - ROMA" + :keywords [#"inquiries call 1-800-233-6211"] + :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :invoice-number #"^\s+([\dA-Z]+)" + :total #"([\d\.,\-]+\.[\d\-]+)"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas-and-negate nil]} + :multi #"\n" + :multi-match? #"^\s+[\d]{6,8}\s+\d+"} + + ;; JFC + {:vendor "JFC International" + :keywords [#"48490 MILMONT DRIVE"] + :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))" + :invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)" + :total #"(?:INVOICE|TOTAL)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas-and-negate nil]}}]) (defn offset [c x y] (.toString (CellAddress. (+ y (.getRow (.getAddress c))) (+ x (.getColumn (.getAddress c))) ))) diff --git a/src/clj/auto_ap/parse/util.clj b/src/clj/auto_ap/parse/util.clj index 380d9b7c..b07f3e2a 100644 --- a/src/clj/auto_ap/parse/util.clj +++ b/src/clj/auto_ap/parse/util.clj @@ -14,6 +14,17 @@ (str/replace value #"," "") ) +(defmethod parse-value :trim-commas-and-negate + [_ _ value] + (let [[_ raw-value] (re-find #"([\d\.]+)" + (-> value + (str/replace #"," "") + (str/replace #"-" "")))] + (if (or (str/includes? value "-") + (str/includes? value "CR")) + (str (- (Double/parseDouble raw-value))) + (str raw-value)))) + (defmethod parse-value :clj-time [_ format value] (time/from-time-zone (f/parse (f/formatter format) value) diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 485460ef..8f4e2c7c 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -161,7 +161,6 @@ (defn import-uploaded-invoice [imports] (let [clients (d-clients/get-all) - _ (clojure.pprint/pprint imports) transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text full-text] :as info}] (println "searching for" vendor-code) From dbbaf37bdd2f7885020b56657229ac8535635eb0 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 11 Dec 2019 20:02:58 -0800 Subject: [PATCH 12/14] minor fixes of invoices. --- src/clj/auto_ap/parse/templates.clj | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index 81f6604b..d993c127 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -88,8 +88,9 @@ :extract {:date #"INVOICE NUMBER[^\n]+\n\n\d+\s+\d+\s+([0-9]+/[0-9]+/[0-9]+)" :customer-identifier #"BILL TO[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,} :invoice-number #"INVOICE NUMBER[^\n]+\n\n\d+\s+(\d+)" - :total #"DELIVERED AMOUNT\s+\$([0-9.]+)"} - :parser {:date [:clj-time "MM/dd/yyyy"]}} + :total #"(?:DELIVERED AMOUNT|PLEASE REMIT).*(?=\$)\$([0-9.,]+)\s*\n"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas nil]}} ;; SYSCO {:vendor "Sysco" @@ -115,18 +116,19 @@ :keywords [#"ABProduce"] :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" :customer-identifier #"BILL TO:[^\n]+\n[^\n]+\n[^\n]+\n(.*)\s{2,}" - :invoice-number #"(\d+)\s+INV" - :total #" INV\s+([\d\.]+)"} - :parser {:date [:clj-time "MM/dd/yyyy"]} + :invoice-number #"(\d+)\s+(?:INV|C/M)" + :total #" (?:INV|C/M)\s+([\d\.\-]+)"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas-and-negate nil]} :multi #"\n" - :multi-match? #"^\s+[0-9]+/[0-9]+/[0-9]+\s+\d+\s+INV\s+"} + :multi-match? #"^\s+[0-9]+/[0-9]+/[0-9]+\s+\d+\s+(INV|C/M)\s+"} ;; CHEF's CHOICE {:vendor "Chef's Choice Produce Co" - :keywords [#"2170 MARTIN AVENUE"] + :keywords [#"(2170 MARTIN AVENUE|213-3886)"] :extract {:date #"([0-9/]{10,10})" :customer-identifier #"\n B\s+([\S ]+?)(?=\s{2,}I) " - :invoice-number #"\n([0-9]+)" + :invoice-number #"^0*([0-9]+)" :total #"INVOICE\s+([\d\.]+)"} :parser {:date [:clj-time "MM/dd/yyyy"]} ;; may want to try two approaches [:clj-time ["MM/dd/yyyy" "MM1dd1yyyy"]] :multi #"\n" From ca2f1d84ded51eef10d7487ad0dead54126ccc20 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 19 Dec 2019 10:16:01 -0800 Subject: [PATCH 13/14] added implementations --- src/clj/auto_ap/parse.clj | 7 +++++ src/clj/auto_ap/parse/templates.clj | 40 ++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index 6b2eceb9..52fb658f 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -126,3 +126,10 @@ first) (:client/default-location client) (first (:client/locations client)))) + +(defn dbg-parse [v] + (doto + (map + (fn [x] (dissoc x :full-text :text)) + (parse v)) + clojure.pprint/pprint )) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index d993c127..f9373b0f 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -41,8 +41,9 @@ :extract {:invoice-number #"Invoice #\s*\n\s*[\w\.]+\s+[\w\./]+(.*)\s*\n" :customer-identifier #"Bill To[^\n]+\n[^\n]*\n([\w ]+)\s{2,}" :date #"Invoice #\s*\n\s*[\w\.]+\s+([\w\./]+)" - :total #"Total\s+\$([0-9.]+)"} - :parser {:date [:clj-time "MM/dd/yy"]}} + :total #"Total\s+\$([0-9.,]+)"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas nil]}} ;; DVW {:vendor "DVW Commercial" @@ -79,8 +80,39 @@ :extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)" :customer-identifier #"DELIVER TO[^\n]+\n\s*[\S ]+?(?=\s{2,}([\S ]+?)\s{2,})" ;; ([\S ]+)\s{2,} :invoice-number #"DELIVER TO[^\n]+\n.+?(?=\d+)(\d+)\s*\n" - :total #"([0-9.]+)\s+Status Code"} - :parser {:date [:clj-time "MM/dd/yy"]}} + :total #"([0-9.\-]+)\s+Status Code"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas-and-negate nil]}} + + ;; SOUTHERN GLAZER'S + {:vendor "Southern Glazers" + :keywords [#"Southern Glazer's"] + :extract {:date #"INVOICE DATE(?s:.*)(?= (?:[0-9]+/[0-9]+/[0-9]+)\s+([0-9]+/[0-9]+/[0-9]+)) " + :customer-identifier #"SOLD TO:(?:.*)(?=\n)\n(.*)(?=\s{2,})" ;; ([\S ]+)\s{2,} + :invoice-number #"INVOICE\n(?:.*?)(?=\d{4,})(\d+)" + :total #"PAY THIS AMOUNT(?s:.*)(?= ([0-9,]+\.[0-9]{2}))"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas nil]}} + + ;; GOLDEN BRANDS + {:vendor "Golden Brands San Jose" + :keywords [#"GOLDEN BRANDS"] + :extract {:date #"0430\n(.*)" + :customer-identifier #"Account:(?:.*\n)(.*(?=\s{2,}))" + :invoice-number #"Invoice#: (\d+)" + :total #"Invoice Total\s+([0-9]+\.[0-9]{2})"} + :parser {:date [:clj-time "EEE MMM dd, yyyy HH:mm aa"] + :total [:trim-commas nil]}} + + ;; WINE WAREHOUSE + {:vendor "Wine Warehouse" + :keywords [#"WINE WAREHOUSE"] + :extract {:date #"INVOICE DATE\s+([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"SHIP-TO-PARTY.*\n(.*?)(?=\s{2,})" + :invoice-number #"INV #\s+(\d+)" + :total #"PLEASE PAY THIS AMOUNT\s+([0-9]+\.[0-9]{2})"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas nil]}} ;; US FOODS {:vendor "US Foods" From 7bc9d434572e6f5871cfe5e4e7484323142560c2 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 20 Dec 2019 11:50:48 -0800 Subject: [PATCH 14/14] even more. --- src/clj/auto_ap/parse/templates.clj | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index f9373b0f..07087551 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -114,6 +114,48 @@ :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas nil]}} + ;; REGAL + {:vendor "Regal Wine Co" + :keywords [#"REGAL WINE"] + :extract {:date #"INVOICE DATE.*\n\n(?:.*?)([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"INVOICE\n(.*?)\s{2,}" + :invoice-number #"INVOICE NUMBER.*\n\n(?:.*?)(\d+)" + :total #"Total Amount Due(?:.*?)([0-9,]+\.[0-9]{2})"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas nil]}} + + ;; ALSCO + {:vendor "Alsco" + :keywords [#"Alsco"] + :extract {:date #"Invoice Date:\s+(.*)" + :customer-identifier #"Invoice F o r(?:.*?)\n\s+(.*?)\s{2,}" + :invoice-number #" (\S+)\n\s+Invoice Date" + :total #"Invoice Total\s+\$([0-9,]+\.[0-9]{2})"} + :parser {:date [:clj-time "MMM dd yyyy"] + :total [:trim-commas nil]}} + + ;; SUNCREST + {:vendor "Suncrest USA Inc" + :keywords [#"Suncrest.*Invoice"] + :extract {:date #"Date.*\n\s*\n(?:.*?)([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"Bill To(?:.*?)\n\n(.*?)\s{2,}" + :invoice-number #"Invoice #.*\n\s*\n(?:.*?)\s{2,}(\d{5,})" + :total #"Balance Due\s+\$([0-9,]+\.[0-9]{2})"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas nil]}} + + ;; SUNCREST STATEMENT + {:vendor "Suncrest USA Inc" + :keywords [#"Suncrest.*\n.*Statement"] + :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"To:(?:.*?)\n\s*(.*?)\s{2,}" + :invoice-number #"INV #(\d+)" + :total #"Orig\. Amount \$([0-9,]+\.[0-9]{2})"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas nil]} + :multi #"\n" + :multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+INV "} + ;; US FOODS {:vendor "US Foods" :keywords [#"US Foods"]