diff --git a/scratch-sessions/fixing-duplicate-vendors.clj b/scratch-sessions/fixing-duplicate-vendors.clj new file mode 100644 index 00000000..38e51b16 --- /dev/null +++ b/scratch-sessions/fixing-duplicate-vendors.clj @@ -0,0 +1,165 @@ +;; This buffer is for Clojure experiments and evaluation. + +;; Press C-j to evaluate the last expression. + +;; You can also press C-u C-j to evaluate the expression and pretty-print its result. + +(init-repl) + +(def best 17592232929447) +best + + +(def others (->> (dc/q '[:find ?v + :in $ [?name ...] ?b + :where [?v :vendor/name ?name] + (not [(= ?v ?b)])] + (dc/db conn) + ["CCP Grubhub" "CCP Grub Hub" "CCP GrubHub"] + best) + (map first) + (into #{}))) + +(count others) + +#_@(dc/transact auto-ap.datomic/conn + (map (fn [o] + [:db/retractEntity o]) + others)) + + +(count + (for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from] + :in '[$ [?a ...] [?vendor-from ...] ] + :where ['[?x ?a ?vendor-from] + '[?a :db/ident ?a2]]} + (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others) + part [[:db/retract incoming incoming-attr from-vendor] + [:db/add incoming incoming-attr best]]] + part + )) + +(auto-ap.datomic/audit-transact-batch + (for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from] + :in '[$ [?a ...] [?vendor-from ...] ] + :where ['[?x ?a ?vendor-from] + '[?a :db/ident ?a2]]} + (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others) + part [[:db/retract incoming incoming-attr from-vendor] + [:db/add incoming incoming-attr best]]] + part + ) + {:user/name "Fixing duplicate vendors"}) + + +(set (map second (dc/q '[:find ?x ?a2 ?vendor-from + :in $ [?a ...] [?vendor-from ...] + :where [?x ?a ?vendor-from] + [?a :db/ident ?a2]] + (dc/db conn) #{ :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others))) + + + +(->> (dc/q '[:find ?v ?c (count ?e) + :in $ ?v + :where + [?v :vendor/name] + (or-join [?v ?c ?e] + (and + [?e :invoice/vendor ?v] + [?e :invoice/client ?c]) + (and + [?e :transaction/vendor ?v] + [?e :transaction/client ?c]) + (and + [?e :journal-entry/vendor ?v] + [?e :journal-entry/client ?c]))] + (dc/db conn) + best) + (map (fn [[v c cnt]] + #:vendor-usage {:vendor v + :client c + :key (str v "-" c) + :count cnt}))) + + +(auto-ap.datomic/audit-transact-batch + (for [[e a] (dc/q '[:find ?e ?a + :in $$ [?tx ...] [?other ...] ?false-best + :where + [$$ ?e ?a ?false-best ?tx true] + [$$ ?e ?a ?other ?tx false] + [$$ ?other :vendor/name ?vn] + [(clojure.string/includes? ?vn "rub")] + ] + + (dc/history (dc/db conn)) + (map (comp dc/t->tx :t) (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00")) + #_(dc/tx-range (dc/log conn) #inst "2023-06-06T22:15:00-08:00" #inst "2023-06-06T22:31:00-08:00") + others + 17592232586913) + part [[:db/retract e a 17592232586913] + [:db/add e a 17592270032394]]] + part + ) + {:user/name "re-fixing grub-hub"}) + + +(map :t (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00")) + +(dc/pull (dc/db conn) '[*] 17592232929447) + + + +(let [best 17592232586918 + others (->> (dc/q '[:find ?v + :in $ [?name ...] ?b + :where [?v :vendor/name ?name] + (not [(= ?v ?b)])] + (dc/db conn) + ["CCP Uber Eats" "CCP UBER" "CCP Ubereats" "CCP Uber Eats MP" "CCP Uber Eats SC" "CCP UberEats" "CCP Ubereats" "CCP Uber"] + best) + (map first) + (into #{}))] + + (auto-ap.datomic/audit-transact-batch + (for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from] + :in '[$ [?a ...] [?vendor-from ...] ] + :where ['[?x ?a ?vendor-from] + '[?a :db/ident ?a2]]} + (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others) + part [[:db/retract incoming incoming-attr from-vendor] + [:db/add incoming incoming-attr best]]] + part + ) + {:user/name "Fixing duplicate vendors"}) + + @(dc/transact auto-ap.datomic/conn + (map (fn [o] + [:db/retractEntity o]) + others))) + + +(require '[datomic.api :as dc]) + +(require '[auto-ap.datomic :refer [conn]]) +(user/init-repl) + +(auto-ap.datomic/audit-transact-batch + (for [[bad] (seq (dc/q '[:find ?x ?vn + :in $ + :where + [?x :vendor/name ?vn] + (not [_ :expected-deposit/vendor ?x]) + (not [_ :invoice/vendor ?x]) + (not [_ :journal-entry/vendor ?x]) + (not [_ :payment/vendor ?x]) + (not [_ :sales-order/vendor ?x]) + (not [_ :payment/vendor ?x]) + (not [_ :sales-refund/vendor ?x]) + (not [_ :transaction/vendor ?x]) + (not [_ :transaction-rule/vendor ?x]) + (not [_ :cash-drawer-shift/vendor ?x])] + (dc/db conn)))] + [:db/retractEntity bad]) + {:user/name "fix-unused-vendors"}) diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index ca27bc18..70c68e3f 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -71,89 +71,88 @@ (assert-admin (:id context)) (when-not (:id edit_client) (assert-client-code-is-unique (:code edit_client))) - + (let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client))) id (or (:db/id client) "new-client") signature-file (upload-signature-data (:signature_data edit_client)) client-code (if (str/blank? (:client/code client)) (:code edit_client) (:client/code client)) - updated-entity {:db/id id - :client/code client-code - :client/name (:name edit_client) - :client/matches (:matches edit_client) - :client/signature-file signature-file - :client/email (:email edit_client) - :client/locked-until (some-> (:locked_until edit_client) (coerce/to-date)) - :client/locations (filter identity (:locations edit_client)) - :client/week-a-debits (:week_a_debits edit_client) - :client/week-a-credits (:week_a_credits edit_client) - :client/week-b-debits (:week_b_debits edit_client) - :client/square-auth-token (:square_auth_token edit_client) - :client/square-locations (map - (fn [sl] - {:db/id (or (:id sl) (random-tempid)) - :square-location/client-location (:client_location sl)}) - (:square_locations edit_client)) + updated-entity (cond-> {:db/id id + :client/code client-code + :client/name (:name edit_client) + :client/matches (:matches edit_client) + :client/email (:email edit_client) + :client/locked-until (some-> (:locked_until edit_client) (coerce/to-date)) + :client/locations (filter identity (:locations edit_client)) + :client/week-a-debits (:week_a_debits edit_client) + :client/week-a-credits (:week_a_credits edit_client) + :client/week-b-debits (:week_b_debits edit_client) + :client/square-auth-token (:square_auth_token edit_client) + :client/square-locations (map + (fn [sl] + {:db/id (or (:id sl) (random-tempid)) + :square-location/client-location (:client_location sl)}) + (:square_locations edit_client)) - :client/emails (map (fn [e] - {:db/id (or (:id e) - (random-tempid)) - :email-contact/email (:email e) - :email-contact/description (:description e)}) - (:emails edit_client)) + :client/emails (map (fn [e] + {:db/id (or (:id e) + (random-tempid)) + :email-contact/email (:email e) + :email-contact/description (:description e)}) + (:emails edit_client)) - :client/feature-flags (:feature_flags edit_client) - :client/ezcater-locations (map - (fn [el] - {:db/id (or (:id el) (random-tempid)) - :ezcater-location/location (:location el) - :ezcater-location/caterer (:caterer el)}) - (:ezcater_locations edit_client)) - :client/week-b-credits (:week_b_credits edit_client) - :client/location-matches (->> (:location_matches edit_client) - (filter (fn [lm] (and (:location lm) (:match lm)))) - (map (fn [lm] {:db/id (or (:id lm ) (random-tempid)) - :location-match/location (:location lm) - :location-match/matches [(:match lm)]}))) - :client/address (when (seq (filter identity (vals (:address edit_client)))) - {:db/id (or (:id (:address edit_client)) (random-tempid)) - :address/street1 (:street1 (:address edit_client)) - :address/street2 (:street2 (:address edit_client)) - :address/city (:city (:address edit_client)) - :address/state (:state (:address edit_client)) - :address/zip (:zip (:address edit_client))}) - :client/bank-accounts (map (fn [ba] - {:db/id (or (:id ba) (random-tempid)) - :bank-account/code (:code ba) - :bank-account/bank-name (:bank_name ba) - :bank-account/bank-code (:bank_code ba) - :bank-account/start-date (-> (:start_date ba) (coerce/to-date)) - :bank-account/routing (:routing ba) - :bank-account/include-in-reports (:include_in_reports ba) + :client/feature-flags (:feature_flags edit_client) + :client/ezcater-locations (map + (fn [el] + {:db/id (or (:id el) (random-tempid)) + :ezcater-location/location (:location el) + :ezcater-location/caterer (:caterer el)}) + (:ezcater_locations edit_client)) + :client/week-b-credits (:week_b_credits edit_client) + :client/location-matches (->> (:location_matches edit_client) + (filter (fn [lm] (and (:location lm) (:match lm)))) + (map (fn [lm] {:db/id (or (:id lm) (random-tempid)) + :location-match/location (:location lm) + :location-match/matches [(:match lm)]}))) + :client/address (when (seq (filter identity (vals (:address edit_client)))) + {:db/id (or (:id (:address edit_client)) (random-tempid)) + :address/street1 (:street1 (:address edit_client)) + :address/street2 (:street2 (:address edit_client)) + :address/city (:city (:address edit_client)) + :address/state (:state (:address edit_client)) + :address/zip (:zip (:address edit_client))}) + :client/bank-accounts (map (fn [ba] + {:db/id (or (:id ba) (random-tempid)) + :bank-account/code (:code ba) + :bank-account/bank-name (:bank_name ba) + :bank-account/bank-code (:bank_code ba) + :bank-account/start-date (-> (:start_date ba) (coerce/to-date)) + :bank-account/routing (:routing ba) + :bank-account/include-in-reports (:include_in_reports ba) - :bank-account/name (:name ba) - :bank-account/visible (:visible ba) - :bank-account/number (:number ba) - :bank-account/check-number (:check_number ba) - :bank-account/numeric-code (:numeric_code ba) - :bank-account/sort-order (:sort_order ba) - :bank-account/locations (:locations ba) - :bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba)) + :bank-account/name (:name ba) + :bank-account/visible (:visible ba) + :bank-account/number (:number ba) + :bank-account/check-number (:check_number ba) + :bank-account/numeric-code (:numeric_code ba) + :bank-account/sort-order (:sort_order ba) + :bank-account/locations (:locations ba) + :bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba)) - :bank-account/yodlee-account-id (:yodlee_account_id ba) - :bank-account/type (keyword "bank-account-type" (name (:type ba))) - :bank-account/yodlee-account (when (:yodlee_account ba) - [:yodlee-account/id (:yodlee_account ba)]) - :bank-account/plaid-account (:plaid_account ba) - :bank-account/intuit-bank-account (:intuit_bank_account ba)}) - (:bank_accounts edit_client)) + :bank-account/yodlee-account-id (:yodlee_account_id ba) + :bank-account/type (keyword "bank-account-type" (name (:type ba))) + :bank-account/yodlee-account (when (:yodlee_account ba) + [:yodlee-account/id (:yodlee_account ba)]) + :bank-account/plaid-account (:plaid_account ba) + :bank-account/intuit-bank-account (:intuit_bank_account ba)}) + (:bank_accounts edit_client))} + signature-file (assoc :client/signature-file signature-file)) - } _ (mu/log ::upserting :up updated-entity) _ (assert-no-shared-transaction-sources client-code [[:upsert-entity updated-entity]]) _ (log/info "upserting client" updated-entity) - + result (audit-transact [[:upsert-entity updated-entity]] (:id context))] (when (:square_auth_token edit_client) (square/upsert-locations (-> result :tempids (get id) (or id) d-clients/get-by-id))) @@ -167,18 +166,18 @@ (:client/name updated-client)))}]) (-> updated-client - (update :client/bank-accounts - (fn [bas] - (map #(set/rename-keys % {:bank-account/use-date-instead-of-post-date? :use-date-instead-of-post-date}) bas))) - (update :client/location-matches - (fn [lms] - (mapcat (fn [lm] - (map (fn [m] - {:location-match/match m - :location-match/location (:location-match/location lm)}) - (:location-match/matches lm))) - lms))) - ->graphql)))) + (update :client/bank-accounts + (fn [bas] + (map #(set/rename-keys % {:bank-account/use-date-instead-of-post-date? :use-date-instead-of-post-date}) bas))) + (update :client/location-matches + (fn [lms] + (mapcat (fn [lm] + (map (fn [m] + {:location-match/match m + :location-match/location (:location-match/location lm)}) + (:location-match/matches lm))) + lms))) + ->graphql)))) (defn refresh-all-current-balance [] diff --git a/src/clj/auto_ap/graphql/ledger.clj b/src/clj/auto_ap/graphql/ledger.clj index 77c3f03d..da2573ce 100644 --- a/src/clj/auto_ap/graphql/ledger.clj +++ b/src/clj/auto_ap/graphql/ledger.clj @@ -530,10 +530,17 @@ (when (seq ignore-retraction) (audit-transact-batch ignore-retraction (:id context)))) #_(log/info (map :tx success)) - (mu/trace ::success-tx - [:count (count success)] - (doseq [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))] - (solr/touch n))) + (let [invalidated + (mu/trace ::success-tx + [:count (count success)] + (for [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))] + n))] + (future + (mu/log ::indexing-solr :count (count invalidated)) + (mu/trace ::indexed-external-solr + [:count (count invalidated)] + (doseq [n invalidated] + (solr/touch n))))) {:successful (map (fn [x] {:external_id (:external_id x)}) success) :ignored (map (fn [x] diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj index ae361bc0..d3bd8aa9 100644 --- a/src/clj/auto_ap/graphql/vendors.clj +++ b/src/clj/auto_ap/graphql/vendors.clj @@ -199,15 +199,14 @@ (defn rebuild-search-index [] (de/future-with - single-thread - (auto-ap.solr/index-documents-raw - auto-ap.solr/impl - "vendors" - (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) - :in $ - :where [?v :vendor/search-terms ]] - :args [(dc/db conn)]})] - {"id" (:db/id result) - "name" (or (first (:vendor/search-terms result)) - (:vendor/name result)) - "hidden" (boolean (:vendor/hidden result))})))) + single-thread + (auto-ap.solr/index-documents-raw + auto-ap.solr/impl + "vendors" + (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) + :in $ + :where [?v :vendor/name]] + :args [(dc/db conn)]})] + {"id" (:db/id result) + "name" (:vendor/name result) + "hidden" (boolean (:vendor/hidden result))})))) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index d3b270cc..7e5c93e1 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -4,8 +4,7 @@ (def pdf-templates - [ - ;; CHEF's WAREHOUSE + [;; CHEF's WAREHOUSE {:vendor "CHFW" :keywords [#"CHEF'S WAREHOUSE"] :extract {:total #"2 WKS C\.C\.\s+([\d.,]+)" @@ -75,9 +74,7 @@ :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas-and-negate nil]}} - - - ;; SOUTHBAY FRESH +;; SOUTHBAY FRESH {:vendor "Southbay Fresh Produce" :keywords [#"SOUTH BAY FRESH PRODUCE"] :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" @@ -173,10 +170,9 @@ :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas nil]} :multi (. java.util.regex.Pattern (compile (-> \formfeed str) java.util.regex.Pattern/CASE_INSENSITIVE)) - :multi-match? #"(Total\s+[0-9\.]+|Total Order)" - } + :multi-match? #"(Total\s+[0-9\.]+|Total Order)"} - ;; AUTO-CHLOR +;; AUTO-CHLOR {:vendor "Auto-Chlor" :keywords [#"AUTO-CHLOR"] :extract {:date #"DATE : ([0-9]+/[0-9]+/[0-9]+)" @@ -208,8 +204,7 @@ :multi #"\n" :multi-match? #"^\s+.*?\d{6,}.*?\$"} - - ;; C & L +;; C & L {:vendor "C&L Produce" :keywords [#"440 Franklin Street"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" @@ -239,8 +234,7 @@ :parser {:date [:clj-time "dd-MMM-yy"] :total [:trim-commas-and-negate nil]}} - - ;;; credits don't have the same format +;;; credits don't have the same format {:vendor "General Produce Company" :keywords [#"1330 NORTH B"] :extract {:date #"DATE.*\n.*\n.*?([0-9]+/[0-9]+/[0-9]+)" @@ -362,9 +356,7 @@ :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas nil]}} - - - ;; PACIFIC SEAFOOD +;; PACIFIC SEAFOOD {:vendor "Pacific Seafood" :keywords [#"(pacseafood|PACIFIC FRESH)"] :extract {:date #"DATE(?:.*\n.*(?=([0-9]+/[0-9]+/[0-9]+)))([0-9]+/[0-9]+/[0-9]+)" @@ -421,13 +413,12 @@ {:vendor "Le Boulanger" :keywords [#"Le Boulanger"] :extract {:date #"Invoice Date: ([^\n]+)\n" - :customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})" :invoice-number #"Invoice No: ([^\n]+)\n" :total #" Total:\s+([\d\.]+)"} :parser {:date [:clj-time "MMM dd, yyyy"]}} - - ;; A&B +;; A&B {:vendor "A&B Produce" :keywords [#"ABProduce"] :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" @@ -530,19 +521,19 @@ {:vendor "Performance Food Group - ROMA" :keywords [#"inquiries call 1-800-233-6211"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" :invoice-number #"^\s+([\dA-Z]+)" :total #"([\d\.,\-]+\.[\d\-]+)"} :parser {:date [:clj-time "MM/dd/yyyy"] :total [:trim-commas-and-negate nil]} :multi #"\n" - :multi-match? #"^\s+[\d]{6,8}\s+\d+"} + :multi-match? #"^\s+[\d]{6,8}\s+\d+"} ;; ACME BREAD {:vendor "Acme Bread" :keywords [#"acmebread\.com"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Print Date.*\n.*\n(.*)" + :customer-identifier #"Print Date.*\n.*\n(.*)" :invoice-number #"^\s*(\d+)" :total #"\s{2,}(\d+\.\d{2})\s{2,}"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -554,19 +545,17 @@ {:vendor "Performance Food Group - ROMA" :keywords [#"Performance Food Group, Inc\n\f"] :extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" + :customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})" :invoice-number #"INVOICE NO.\s+ ([\d]+)" :total #"([\d\.,]+)\s+INVOICE TOTAL"} :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas nil]}} - - - ;; JFC +;; JFC {:vendor "JFC International" :keywords [#"48490 MILMONT DRIVE"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))" + :customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))" :invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)" :total #"(?:INVOICE|TOTAL|CREDIT)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -576,7 +565,7 @@ {:vendor "Roma Bakery Inc." :keywords [#"Roma Bakery Inc"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" + :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" :invoice-number #"Invoice (\d+)" :total #"Total\s+([\d\-\.]+)"} :parser {:date [:clj-time "MM/dd/yy"] @@ -586,7 +575,7 @@ {:vendor "Kael Foods" :keywords [#"kaelfoods.com"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" + :customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}" :invoice-number #"INVOICE 0*(\d+)" :total #"TOTAL:\s+\$([\d\-\.,]+)"} :parser {:date [:clj-time "MM/dd/yyyy"] @@ -596,7 +585,7 @@ {:vendor "Starter Bakery" :keywords [#"starterbakery.com"] :extract {:date #"INVOICE DATE:\s+(.*?)\s{2,}" - :customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}" + :customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}" :invoice-number #"Invoice.*?(\d+)" :total #"Total:.*?([\d\-,]+\.\d{2,2}+)"} :parser {:date [:clj-time "MMMM dd, yyyy"] @@ -606,9 +595,18 @@ {:vendor "TriMark R.W. Smith" :keywords [#"TriMark"] :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" - :customer-identifier #"Bill To\s+(.*?)\s{2,}" + :customer-identifier #"Bill To\s+(.*?)\s{2,}" :invoice-number #"Invoice #\n.*?([\d\-]+)\n" :total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"} + :parser {:date [:clj-time "MM/dd/yy"] + :total [:trim-commas-and-negate nil]}} + + {:vendor "Reel Produce" + :keywords [#"reelproduce.com"] + :extract {:date #"([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}" + :invoice-number #"Invoice #\n.*?([\d\-]+)\n" + :total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"} :parser {:date [:clj-time "MM/dd/yy"] :total [:trim-commas-and-negate nil]}}]) diff --git a/src/clj/auto_ap/solr.clj b/src/clj/auto_ap/solr.clj index 57fc6900..18b535f2 100644 --- a/src/clj/auto_ap/solr.clj +++ b/src/clj/auto_ap/solr.clj @@ -136,7 +136,8 @@ (index-documents-raw [this index xs] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 5000})) + :query {"commitWithin" 5000 + "commit" true})) {:headers {"Content-Type" "application/json"} :socket-timeout 30000 @@ -147,7 +148,8 @@ (index-documents [this index xs] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 5000})) + :query {"commitWithin" 5000 + "commit" true})) {:headers {"Content-Type" "application/json"} :socket-timeout 30000 :connection-timeout 30000 @@ -168,7 +170,8 @@ (delete [this index] (client/post (str (assoc (url/url solr-uri "solr" index "update") - :query {"commitWithin" 15000})) + :query {"commitWithin" 15000 + "commit" true})) {:headers {"Content-Type" "application/json"} :method "POST" :body (json/write-str {"delete" {"query" "*:*"}})})))