diff --git a/migrator/migrations/1525382176-UP-more-customer-fields.sql b/migrator/migrations/1525382176-UP-more-customer-fields.sql index 428ca84a..4dcdb371 100644 --- a/migrator/migrations/1525382176-UP-more-customer-fields.sql +++ b/migrator/migrations/1525382176-UP-more-customer-fields.sql @@ -1,5 +1,8 @@ -- 1525382176 UP more-customer-fields ALTER TABLE companies ADD code varchar(255); +delete from invoices; +delete from companies; +delete from vendors; INSERT INTO companies (code, name, data) VALUES @@ -18,3 +21,99 @@ VALUES ('RCI', 'Roberto''s Cantina Inc', '{}'), ('SIB', 'Sorelle Italian Bistro', '{}'), ('TSL', 'The Socialight', '{}'); + + +INSERT INTO vendors (name, address1, primary_contact, primary_email, primary_phone, secondary_email) +VALUES +('Acme Bread', '', '', '', '', ''), +('Alfredo''s Produce', '55 Sunol St', 'Oscar', 'candelas_oscar@yahoo.com', '408-806-4150', 'oscar.candelas1@gmail.com'), +('Aloha Paid Out', '', '', '', '', ''), +('Alsco', '2275 Junction Ave', 'Suki', 'spham@alsco.com', '408-279-2345', ''), +('American Wine & Spirits', '', 'Nicholas Stagnaro', 'American Wine & Spirits', '408-838-1005', ''), +('Aramark', '', '', '', '', ''), +('Auto-Chlor', '3400 Thomas Rd', '', '', '', ''), +('Bassian Farms Inc', '', 'Flora Hernandez', 'fhernandez@bassianfarms.com', 'Mike Maltese, Sales: 408-691-8487', ''), +('Better Brands', '', 'Fred', 'fredmontalvo@hotmail.com', '', ''), +('Bigoli Fresh Pasta', '', '', 'info@bigolifreshpasta.com', '', ''), +('BiRite', '123 South Hill Drive', 'Justin', '', '', ''), +('BSG Craftbrewing', '', 'Amanda Steele', 'asteele@bsgcraft.com', '952-465-0596', ''), +('Carbonic Service Inc', '', 'Cynthia', 'cynthia@carbonicservice.com', '408-727-8835', ''), +('Chavez Distributors', '', 'Chavez', 'chavezdist@gmail.com', '510-352-8431', 'Alejandra'), +('Chef''s Choice Produce Co', '2170 Martin Avenue', 'Anna', 'Accounting ', '', ''), +('Chef''s Warehouse', '1250 Whipple Rd.', 'Deborah', 'dvalle@chefswarehouse.com', '510.627.0093', ''), +('Chesbro', '', 'Robin Voss', 'risingstarwinegroup@hotmail.com', '831-392-7984', ''), +('Chrissa Imports Ltd', '', 'Segev', 'orders@chrissaimports.com', '', ''), +('Cintas', '', '', 'Loc00W01@cintas.com', '916.576.4104 ', 'Ruth Moya- prev the contact'), +('Cintas - Towels', '', '', 'Loc00W01@cintas.com', '916.576.4104 ', 'Ruth Moya- prev the contact'), +('Classic Mix - Galloway Company', 'PO Box 617', 'Linda', 'lfournier@gallowaycompany.com', '', ''), +('Classic Wines', '', 'AR', 'bronco.ar@broncowine.com', '800-692-5780', 'Laura, AR person I talked to'), +('Daylight Foods', '', '', '', '', ''), +('DBI Beverage', '', 'Grace Miller', 'grace.miller@dbibeverage.com', '408-380-5235', ''), +('Del Monte Meat Co', 'PO Box #101831', 'Vicki', 'vickic@delmontemeat.com', '', ''), +('Digital Dining Paid Out', '', '', '', '', ''), +('DVW Commercial', '38507 Cherry St., Suite I', 'Lily', 'dvwcommercial@hotmail.com', '', ''), +('Eddie''s Produce', '', 'Eddie', 'eddiesfreshproduce.sj@gmail.com', '', ''), +('Epic Wines & Spirits', 'P.O. Box 8366', '', '', '888-565-4352', ''), +('Frank Family Vineyards', 'PO Box 1012', '', 'ar@vintagewinemarketing.com', '', ''), +('Fresh and Best Produce', '', 'Abraham', 'freshbestarap@gmail.com', '', 'Abraham - Sales Rep'), +('GE Mobile Water, Inc', 'PO Box 742132', '', '', '', ''), +('Golden Gate Meat Company, Inc', '803 Wright Ave', 'Greg', 'GregM@ggmeatco.com', '', ''), +('Half Moon Bay Brewing Co, Inc', 'PO Box 879', 'Emily', 'emily@hmbbrewingco.com', '650-728-2739', ''), +('Il Pastaio Foods', '', '', '', '', ''), +('International Food Distributing', '822 American Street', 'Dan', 'dan@inter-national-foods.com', '650-593-7183', ''), +('ISP Productions', '2468 Karen Dr, Unit 1', 'Greg', 'super851@gmail.com', '(408) 891-8122', ''), +('ItalFoods Inc', 'PO BOX 7511', 'Paris', 'Paris Glass ', '650 877 0724, ext 326', ''), +('ItalFoods Inc', 'PO BOX 2563', 'Jennifer', 'jenniferc@lusamerica.com', '408 778 7200', ''), +('Julius Meinl', '4115 N. Ravenswood Ave', 'Tiffany', 'tiffany.marchan@meinl.com', '773-954-7571', ''), +('JVS Wine Imports', '360 Swift Ave Side B Ste 9', '', '', '', ''), +('Laird Family Wine', '', 'Robin Voss', 'risingstarwinegroup@hotmail.com', '831-392-7984', ''), +('Le Boulanger', '', 'Anna Lisa Daniel', 'adaniel@leboulanger.com', '408-774-9000 ext 278', ''), +('Lettieri & Co', '120 Park Lane', 'Selo', 'selo@lettieri.com', '', ''), +('Lusamerica Foods, Inc', '', 'Jennifer', 'jenniferc@lusamerica.com', '', ''), +('Mama Lu''s Foods', '', 'Sam Orozco', 'Sam Orozco ; robert.guzman@mamalusfoods.com', '', ''), +('Mama Lu''s Invoices', '', 'Sam Orozco', 'Sam Orozco ; robert.guzman@mamalusfoods.com', '', ''), +('Marianne''s Ice Cream', '2100 B Delaware Ave', 'Alice', 'kd@mariannesicecream.com', '', ''), +('Mission Linen Supply', '601 Swift St', '', '', '831-423-1630', ''), +('Mt Eden Winery', '22020 Mount Eden Rd', '', '', '', ''), +('New York Style Sausage Co.', '1228 Reamwood Ave', 'Kay', 'nysdon@aol.com', '', ''), +('Newport Fish Co', '', 'Christina Brooks', 'newportfish@earthlink.net', '650-877-7200', ''), +('Original Roasters', '', 'Joaquin Gomez ', 'jgomez@originalroasters.com', '', ''), +('Pacific Edge Wine & Spirits', '5155 Clareton Drive, #100', '', '', '', ''), +('Palo Alto Foods', '6691 Clark Ave', 'Evelyn', 'evelyn@paloaltofoods.com', '', ''), +('Pelerin Wines', '', 'Robin Voss', 'risingstarwinegroup@hotmail.com', '831-392-7984', ''), +('Performance Food Group', '', 'Cristin Garden', 'Cristin.Garden@pfgc.com', '831-465-3212', ''), +('Performance Food Group - Joan', '', 'Joant Yount', 'Joan Yount ', '831-465-3212', ''), +('Performance Food Group - KOG', '', 'Greg Sosebee', 'greg.sosebee@pfgc.com', '408-499-9863', ''), +('Performance Food Group - LFT', 'PO Box 951080', 'Michael Forman', 'michael.forman@pfgc.com', '408-799-3772', ''), +('Performance Food Group - RCI', '', 'Carlos Gonzalez', 'carlos.gonzalez@pfgc.com', '831-465-3212', ''), +('Performance Food Group - ROMA', '', '', '', '', ''), +('Performance Food Group - SIB', '', 'Joant Yount', 'Joan Yount ', '831-465-3212', ''), +('Pride Mountain Vineyards', '', 'Robin Voss', 'risingstarwinegroup@hotmail.com', '831-392-7984', ''), +('Prudential Overall Supply', 'PO Box 11210', '', '', '', ''), +('Ralph Martinez', '1071 Topaz Ave Apt. C', 'Ralph', '', '', ''), +('Regal Wine Co', 'PO Box 2160', '', '', '', ''), +('Rhodes-To-Wine com', '', 'Rebecca Rhodes ', 'rebecca@rhodes-to-wine.com', '', ''), +('Royal Hawaiian', '213 E. Harris Ave ', '', '', '', ''), +('S. Ramirez Distribution', '', 'Lupe', 'Guadalupe.Ramirez@unionbank.com', '650-365-2110 (W) 650-568-5883 (Direct)', ''), +('Saportio Foods Inc', '', '', '', '', ''), +('Scanned Invoices', '', 'Ron', 'rjb.jr62@gmail.com', '', ''), +('Southern Glazers', 'P.O. Box 742313', 'Mark', '', '', 'For Online Modules'), +('Suprema Meat Company', '', 'Berta Gonzalez', 'Berta Gonzalez ', '', ''), +('Sysco', '', '', '', '', ''), +('System Services of America', '', '', '', '', ''), +('Testarossa Winery', 'PO Box 969', 'Jessica Morocco', 'jmorocco@testarossa.com', '408-354-6150 xt 49', ''), +('The Alarm Company', 'PO Box 178', '', '', '', ''), +('Top Hat Provisions, LLC', '1 South Park St., Unit 205', 'Auto-email', 'Orders@TopHatProvisions.com', '415-902-7821', ''), +('Treat Ice Cream', '11 South 19th Street', 'Bob', 'bob@treaticecream.com', '', ''), +('United Hop Farm', '', 'Ro Nayyar ', 'ro@unitedhops.com', '(916) 899-9578', ''), +('US Foods', '', 'Erol', '', '', ''), +('Vance Wine Selections', '2060-D East Avenida de Los Arboles 306', 'Sami', 'notic+invoices@vinosmith.com', '805 368 0707', ''), +('Vigneron Imports', '6 Barner Place', '', '', '', ''), +('Vince''s Shellfish Co Inc', '', '', '', '', ''), +('Vino Favoloso LLC', '', '', '', '', ''), +('Vintage Wine Marketing', 'PO Box 5149', '', '', '', ''), +('Wine Bridge Imports', 'PO Box 2272', '', '', '', ''), +('Wine Warehouse', 'PO Box 45616', 'Robert Favela', 'r.favela@winewarehouse.com', '', ''), +('Wine Wise', '', 'Devo Price', '', '', 'Brian Greenwood'), +('Youngs Market', '', 'Hello,', 'fileclerk@youngsmarket.com', '', ''), +('PG&E', '', '', '', '', '') diff --git a/src/clj/auto_ap/db/invoices.clj b/src/clj/auto_ap/db/invoices.clj index f7811390..ad3989f3 100644 --- a/src/clj/auto_ap/db/invoices.clj +++ b/src/clj/auto_ap/db/invoices.clj @@ -1,5 +1,5 @@ (ns auto-ap.db.invoices - (:require [auto-ap.db.utils :refer [clj->db db->clj get-conn query] :as utils] + (:require [auto-ap.db.utils :refer [clj->db kebab->snake db->clj get-conn query] :as utils] [auto-ap.parse :as parse] [auto-ap.db.companies :as companies] [auto-ap.db.vendors :as vendors] @@ -17,6 +17,26 @@ :invoices (map clj->db rows))) + +(defn upsert-multi! [rows] + (let [k (vec (map #(keyword (kebab->snake (name %))) [:company-id :vendor-id :invoice-number :total :date :imported])) + column-names (str/join "," (map name k))] + (doseq [rows (partition-all 2000 rows)] + (j/db-do-prepared (get-conn) + (sql/format {:with [[[:v (str "(" column-names ")")] + (helpers/values (->> rows + (map clj->db ) + (map (apply juxt k))))]] + :insert-into [[:invoices k] + {:select [:v.company-id :v.vendor-id :v.invoice-number :v.total (sql/raw "cast(v.date as timestamp)") :v.imported ] + :from [:v] + :left-join [[:invoices :exist] + [:and + [:= :exist.invoice-number :v.invoice-number] + [:= :exist.company-id :v.company-id] + [:= :exist.vendor-id :v.vendor-id]]] + :where [:= :exist.id nil] }] }))))) + (def base-query (sql/build :select :invoices.* :from :invoices)) @@ -75,8 +95,7 @@ (defn base-graphql [{:keys [imported company-id]}] (cond-> base-query (not (nil? imported)) (helpers/merge-where [:= :imported imported]) - (not (nil? company-id)) (helpers/merge-where [:= :company-id company-id]) - )) + (not (nil? company-id)) (helpers/merge-where [:= :company-id company-id]))) (defn get-graphql [{:keys [start sort-by asc] :as args}] (query diff --git a/src/clj/auto_ap/db/reminders.clj b/src/clj/auto_ap/db/reminders.clj index 6cf9b8e8..845f45da 100644 --- a/src/clj/auto_ap/db/reminders.clj +++ b/src/clj/auto_ap/db/reminders.clj @@ -98,7 +98,7 @@ (doseq [{:keys [vendor-name id email subject body]} reminders] (when email (println "Sending email to" email) - (ses/send-email :destination {:to-addresses [email]} + #_(ses/send-email :destination {:to-addresses [email]} :source (:invoice-email env) :message {:subject subject :body {:html (str "

Hello " vendor-name ",

" body "

- Integreat.

") diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index a0209bf8..47c26567 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -18,21 +18,42 @@ (defn assoc-company-code [i] (assoc i :company-code (first (str/split (:location i) #"-" )))) -(defn assoc-company [i companies] - (assoc i :company-id (or (-> i - :company-code - companies - :id) - (-> i - :company - companies - :id)))) +(defn parse-company [{:keys [company-code company]} companies] + (if-let [id (:id (or (companies company-code) + (companies company)))] + id + (throw (Exception. (str "Company code '" company-code "' and company named '" company "' not found."))))) -(defn assoc-vendor [i vendors] - (assoc i :vendor-id (-> i - :vendor-name - vendors - :id))) +(defn parse-invoice-number [{:keys [invoice-number]}] + (or invoice-number "")) + +(defn parse-vendor [{:keys [vendor-name]} vendors] + (if-let [id (:id (vendors vendor-name))] + id + (throw (Exception. (str "Vendor '" vendor-name "' not found."))))) + +(defn parse-amount [i] + (try + (Double/parseDouble (str/replace (or (second + (re-matches #"[^0-9\.,]*([0-9\.,]+)[^0-9\.,]*" (:amount i))) + "0") + #"," "")) + (catch Exception e + (throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e))))) + +(defn parse-date [{:keys [raw-date]}] + (try + (parse/parse-value :clj-time "MM/dd/yyyy" raw-date) + (catch Exception e + (throw (Exception. (str "Could not parse date from '" raw-date "'") e))))) + +(defn parse-or-error [key f] + (fn [x] + (try + (assoc x key (f x)) + (catch Exception e + (update x :errors conj {:info (.getMessage e) + :details (str e)}))))) (defroutes routes (wrap-routes @@ -78,7 +99,7 @@ (POST "/upload-integreat" {{:keys [excel-rows]} :edn-params} - (let [columns [:date :vendor-name :check :location :invoice-number :amount :company :bill-entered :bill-rejected :added-on :exported-on] + (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :company :bill-entered :bill-rejected :added-on :exported-on] all-vendors (reduce (fn [x y] @@ -100,23 +121,27 @@ (map #(into {} (map (fn [c k] [k c] ) % columns))) (map reset-id) (map assoc-company-code) - (map #(assoc-company % all-companies)) - (map #(assoc-vendor % all-vendors)) - (map (fn [{:keys [vendor-id company-id amount date invoice-number]}] - {:vendor-id vendor-id - :company-id company-id - :total (Double/parseDouble (second - (re-matches #"[^0-9\.]*([0-9\.]+)[^0-9\.]*" amount))) - :imported true - :status "unpaid" - :invoice-number invoice-number - :date (parse/parse-value :clj-time "MM/dd/yyyy" date ) - })) - )] - (invoices/insert-multi! rows) + (map (parse-or-error :company-id #(parse-company % all-companies))) + (map (parse-or-error :vendor-id #(parse-vendor % all-vendors))) + (map (parse-or-error :invoice-number parse-invoice-number)) + (map (parse-or-error :total parse-amount)) + (map (parse-or-error :date parse-date))) + error-rows (filter :errors rows) + insert-rows (vec (->> (filter #(not (seq (:errors %))) rows) + (map (fn [{:keys [vendor-id total company-id amount date invoice-number]}] + {:vendor-id vendor-id + :company-id company-id + :total total + :imported true + :status "unpaid" + :invoice-number invoice-number + :date date}))))] + + (invoices/upsert-multi! insert-rows) {:status 200 - :body (pr-str rows) + :body (pr-str {:imported (count insert-rows) + :errors (map #(dissoc % :date) error-rows)}) :headers {"Content-Type" "application/edn"}})) ;; Removing the export view for now... diff --git a/src/cljs/auto_ap/views/pages/admin/excel-import.cljs b/src/cljs/auto_ap/views/pages/admin/excel-import.cljs index 07ddfc22..f00168d1 100644 --- a/src/cljs/auto_ap/views/pages/admin/excel-import.cljs +++ b/src/cljs/auto_ap/views/pages/admin/excel-import.cljs @@ -8,7 +8,8 @@ [auto-ap.views.utils :refer [login-url dispatch-value-change bind-field horizontal-field dispatch-event]] [cljs.reader :as edn] [auto-ap.routes :as routes] - [bidi.bidi :as bidi])) + [bidi.bidi :as bidi] + [clojure.string :as str])) (re-frame/reg-sub ::excel-import @@ -36,7 +37,8 @@ :headers {"Content-Type" "application/edn"} :uri (str "/api/invoices/upload-integreat") :on-success [::save-complete] - :on-error [::save-error]}}))) + :on-error [::save-error]} + :db (assoc-in db [::excel-import :rows] nil)}))) (re-frame/reg-event-fx ::save-complete @@ -62,18 +64,28 @@ :subscription excel-import-data}]] [:button.button.is-large.is-pulled-right.is-primary {:on-click (dispatch-event [::save])} "Import"] - [:div - [:table.table.is-fullwidth - [:thead - [:td "Date"] - [:td "Invoice #"] - [:td "Vendor"] - [:td "Company"]] - [:tbody - (for [{:keys [invoice-number date vendor-name company]} (:rows excel-import-data)] - [:tr - - [:td date] - [:td invoice-number] - [:td vendor-name] - [:td company]])]]]])]) + + [:div.is-clearfix + (when-let [imported (:imported (:rows excel-import-data))] + (str imported " rows imported.")) + ] + (when-let [errors (:errors (:rows excel-import-data))] + + [:div + [:h3 (str "Import errors (" (min 100 (count errors)) ")")] + [:table.table.is-fullwidth + [:thead + [:th "Date"] + [:th "Invoice #"] + [:th "Company"] + [:th "Vendor"] + [:th "Amount"] + [:th "Errors"]] + (for [{:keys [raw-date invoice-number company vendor-name amount] row-errors :errors} (take 100 errors)] + [:tr + [:td raw-date] + [:td invoice-number] + [:td company] + [:td vendor-name] + [:td amount] + [:td (map #(vector :p (:info %)) row-errors)]])]])])])