diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj index ae361bc0..d3bd8aa9 100644 --- a/src/clj/auto_ap/graphql/vendors.clj +++ b/src/clj/auto_ap/graphql/vendors.clj @@ -199,15 +199,14 @@ (defn rebuild-search-index [] (de/future-with - single-thread - (auto-ap.solr/index-documents-raw - auto-ap.solr/impl - "vendors" - (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) - :in $ - :where [?v :vendor/search-terms ]] - :args [(dc/db conn)]})] - {"id" (:db/id result) - "name" (or (first (:vendor/search-terms result)) - (:vendor/name result)) - "hidden" (boolean (:vendor/hidden result))})))) + single-thread + (auto-ap.solr/index-documents-raw + auto-ap.solr/impl + "vendors" + (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) + :in $ + :where [?v :vendor/name]] + :args [(dc/db conn)]})] + {"id" (:db/id result) + "name" (:vendor/name result) + "hidden" (boolean (:vendor/hidden result))})))) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index d3b270cc..7e5c93e1 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -4,8 +4,7 @@ (def pdf-templates - [ - ;; CHEF's WAREHOUSE + [;; CHEF's WAREHOUSE {:vendor "CHFW" :keywords [#"CHEF'S WAREHOUSE"] :extract {:total #"2 WKS C\.C\.\s+([\d.,]+)" @@ -75,9 +74,7 @@ :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas-and-negate nil]}} - - - ;; SOUTHBAY FRESH +;; SOUTHBAY FRESH {:vendor "Southbay Fresh Produce" :keywords [#"SOUTH BAY FRESH PRODUCE"] :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" @@ -173,10 +170,9 @@ :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas nil]} :multi (. java.util.regex.Pattern (compile (-> \formfeed str) java.util.regex.Pattern/CASE_INSENSITIVE)) - :multi-match? #"(Total\s+[0-9\.]+|Total Order)" - } + :multi-match? #"(Total\s+[0-9\.]+|Total Order)"} - ;; AUTO-CHLOR +;; AUTO-CHLOR {:vendor "Auto-Chlor" :keywords [#"AUTO-CHLOR"] :extract {:date #"DATE : ([0-9]+/[0-9]+/[0-9]+)" @@ -208,8 +204,7 @@ :multi #"\n" :multi-match? #"^\s+.*?\d{6,}.*?\$"} - - ;; C & L +;; C & L {:vendor "C&L Produce" :keywords [#"440 Franklin Street"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" @@ -239,8 +234,7 @@ :parser {:date [:clj-time "dd-MMM-yy"] :total [:trim-commas-and-negate nil]}} - - ;;; credits don't have the same format +;;; credits don't have the same format {:vendor "General Produce Company" :keywords [#"1330 NORTH B"] :extract {:date #"DATE.*\n.*\n.*?([0-9]+/[0-9]+/[0-9]+)" @@ -362,9 +356,7 @@ :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas nil]}} - - - ;; PACIFIC SEAFOOD +;; PACIFIC SEAFOOD {:vendor "Pacific Seafood" :keywords [#"(pacseafood|PACIFIC FRESH)"] :extract {:date #"DATE(?:.*\n.*(?=([0-9]+/[0-9]+/[0-9]+)))([0-9]+/[0-9]+/[0-9]+)" @@ -421,13 +413,12 @@ {:vendor "Le Boulanger" :keywords [#"Le Boulanger"] :extract {:date #"Invoice Date: ([^\n]+)\n" - :customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})" :invoice-number #"Invoice No: ([^\n]+)\n" :total #" Total:\s+([\d\.]+)"} :parser {:date [:clj-time "MMM dd, yyyy"]}} - - ;; A&B +;; A&B {:vendor "A&B Produce" :keywords [#"ABProduce"] :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" @@ -530,19 +521,19 @@ {:vendor "Performance Food Group - ROMA" :keywords [#"inquiries call 1-800-233-6211"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" :invoice-number #"^\s+([\dA-Z]+)" :total #"([\d\.,\-]+\.[\d\-]+)"} :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas-and-negate nil]} :multi #"\n" - :multi-match? #"^\s+[\d]{6,8}\s+\d+"} + :multi-match? #"^\s+[\d]{6,8}\s+\d+"} ;; ACME BREAD {:vendor "Acme Bread" :keywords [#"acmebread\.com"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Print Date.*\n.*\n(.*)" + :customer-identifier #"Print Date.*\n.*\n(.*)" :invoice-number #"^\s*(\d+)" :total #"\s{2,}(\d+\.\d{2})\s{2,}"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -554,19 +545,17 @@ {:vendor "Performance Food Group - ROMA" :keywords [#"Performance Food Group, Inc\n\f"] :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" :invoice-number #"INVOICE NO.\s+ ([\d]+)" :total #"([\d\.,]+)\s+INVOICE TOTAL"} :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas nil]}} - - - ;; JFC +;; JFC {:vendor "JFC International" :keywords [#"48490 MILMONT DRIVE"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))" + :customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))" :invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)" :total #"(?:INVOICE|TOTAL|CREDIT)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -576,7 +565,7 @@ {:vendor "Roma Bakery Inc." :keywords [#"Roma Bakery Inc"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" + :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" :invoice-number #"Invoice (\d+)" :total #"Total\s+([\d\-\.]+)"} :parser {:date [:clj-time "MM/dd/yy"] @@ -586,7 +575,7 @@ {:vendor "Kael Foods" :keywords [#"kaelfoods.com"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" + :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" :invoice-number #"INVOICE 0*(\d+)" :total #"TOTAL:\s+\$([\d\-\.,]+)"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -596,7 +585,7 @@ {:vendor "Starter Bakery" :keywords [#"starterbakery.com"] :extract {:date #"INVOICE DATE:\s+(.*?)\s{2,}" - :customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}" + :customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}" :invoice-number #"Invoice.*?(\d+)" :total #"Total:.*?([\d\-,]+\.\d{2,2}+)"} :parser {:date [:clj-time "MMMM dd, yyyy"] @@ -606,9 +595,18 @@ {:vendor "TriMark R.W. Smith" :keywords [#"TriMark"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To\s+(.*?)\s{2,}" + :customer-identifier #"Bill To\s+(.*?)\s{2,}" :invoice-number #"Invoice #\n.*?([\d\-]+)\n" :total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas-and-negate nil]}} + + {:vendor "Reel Produce" + :keywords [#"reelproduce.com"] + :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}" + :invoice-number #"Invoice #\n.*?([\d\-]+)\n" + :total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"} :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas-and-negate nil]}}]) diff --git a/src/clj/auto_ap/solr.clj b/src/clj/auto_ap/solr.clj index 1ecb8d2f..b1d3dcc8 100644 --- a/src/clj/auto_ap/solr.clj +++ b/src/clj/auto_ap/solr.clj @@ -136,7 +136,8 @@ (index-documents-raw [this index xs] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 5000})) + :query {"commitWithin" 5000 + "commit" true})) {:headers {"Content-Type" "application/json"} :socket-timeout 30000 @@ -147,7 +148,8 @@ (index-documents [this index xs] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 5000})) + :query {"commitWithin" 5000 + "commit" true})) {:headers {"Content-Type" "application/json"} :socket-timeout 30000 :connection-timeout 30000 @@ -168,7 +170,8 @@ (delete [this index] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 15000})) + :query {"commitWithin" 15000 + "commit" true})) {:headers {"Content-Type" "application/json"} :method "POST" :body (json/write-str {"delete" {"query" "*:*"}})})))