This commit is contained in:
Bryce
2023-06-13 08:48:26 -07:00
6 changed files with 301 additions and 130 deletions

View File

@@ -0,0 +1,165 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(init-repl)
(def best 17592232929447)
best
(def others (->> (dc/q '[:find ?v
:in $ [?name ...] ?b
:where [?v :vendor/name ?name]
(not [(= ?v ?b)])]
(dc/db conn)
["CCP Grubhub" "CCP Grub Hub" "CCP GrubHub"]
best)
(map first)
(into #{})))
(count others)
#_@(dc/transact auto-ap.datomic/conn
(map (fn [o]
[:db/retractEntity o])
others))
(count
(for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from]
:in '[$ [?a ...] [?vendor-from ...] ]
:where ['[?x ?a ?vendor-from]
'[?a :db/ident ?a2]]}
(dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
part [[:db/retract incoming incoming-attr from-vendor]
[:db/add incoming incoming-attr best]]]
part
))
(auto-ap.datomic/audit-transact-batch
(for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from]
:in '[$ [?a ...] [?vendor-from ...] ]
:where ['[?x ?a ?vendor-from]
'[?a :db/ident ?a2]]}
(dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
part [[:db/retract incoming incoming-attr from-vendor]
[:db/add incoming incoming-attr best]]]
part
)
{:user/name "Fixing duplicate vendors"})
(set (map second (dc/q '[:find ?x ?a2 ?vendor-from
:in $ [?a ...] [?vendor-from ...]
:where [?x ?a ?vendor-from]
[?a :db/ident ?a2]]
(dc/db conn) #{ :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)))
(->> (dc/q '[:find ?v ?c (count ?e)
:in $ ?v
:where
[?v :vendor/name]
(or-join [?v ?c ?e]
(and
[?e :invoice/vendor ?v]
[?e :invoice/client ?c])
(and
[?e :transaction/vendor ?v]
[?e :transaction/client ?c])
(and
[?e :journal-entry/vendor ?v]
[?e :journal-entry/client ?c]))]
(dc/db conn)
best)
(map (fn [[v c cnt]]
#:vendor-usage {:vendor v
:client c
:key (str v "-" c)
:count cnt})))
(auto-ap.datomic/audit-transact-batch
(for [[e a] (dc/q '[:find ?e ?a
:in $$ [?tx ...] [?other ...] ?false-best
:where
[$$ ?e ?a ?false-best ?tx true]
[$$ ?e ?a ?other ?tx false]
[$$ ?other :vendor/name ?vn]
[(clojure.string/includes? ?vn "rub")]
]
(dc/history (dc/db conn))
(map (comp dc/t->tx :t) (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00"))
#_(dc/tx-range (dc/log conn) #inst "2023-06-06T22:15:00-08:00" #inst "2023-06-06T22:31:00-08:00")
others
17592232586913)
part [[:db/retract e a 17592232586913]
[:db/add e a 17592270032394]]]
part
)
{:user/name "re-fixing grub-hub"})
(map :t (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00"))
(dc/pull (dc/db conn) '[*] 17592232929447)
(let [best 17592232586918
others (->> (dc/q '[:find ?v
:in $ [?name ...] ?b
:where [?v :vendor/name ?name]
(not [(= ?v ?b)])]
(dc/db conn)
["CCP Uber Eats" "CCP UBER" "CCP Ubereats" "CCP Uber Eats MP" "CCP Uber Eats SC" "CCP UberEats" "CCP Ubereats" "CCP Uber"]
best)
(map first)
(into #{}))]
(auto-ap.datomic/audit-transact-batch
(for [[incoming incoming-attr from-vendor] (dc/q {:find '[?x ?a2 ?vendor-from]
:in '[$ [?a ...] [?vendor-from ...] ]
:where ['[?x ?a ?vendor-from]
'[?a :db/ident ?a2]]}
(dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
part [[:db/retract incoming incoming-attr from-vendor]
[:db/add incoming incoming-attr best]]]
part
)
{:user/name "Fixing duplicate vendors"})
@(dc/transact auto-ap.datomic/conn
(map (fn [o]
[:db/retractEntity o])
others)))
(require '[datomic.api :as dc])
(require '[auto-ap.datomic :refer [conn]])
(user/init-repl)
(auto-ap.datomic/audit-transact-batch
(for [[bad] (seq (dc/q '[:find ?x ?vn
:in $
:where
[?x :vendor/name ?vn]
(not [_ :expected-deposit/vendor ?x])
(not [_ :invoice/vendor ?x])
(not [_ :journal-entry/vendor ?x])
(not [_ :payment/vendor ?x])
(not [_ :sales-order/vendor ?x])
(not [_ :payment/vendor ?x])
(not [_ :sales-refund/vendor ?x])
(not [_ :transaction/vendor ?x])
(not [_ :transaction-rule/vendor ?x])
(not [_ :cash-drawer-shift/vendor ?x])]
(dc/db conn)))]
[:db/retractEntity bad])
{:user/name "fix-unused-vendors"})

View File

@@ -78,78 +78,77 @@
client-code (if (str/blank? (:client/code client)) client-code (if (str/blank? (:client/code client))
(:code edit_client) (:code edit_client)
(:client/code client)) (:client/code client))
updated-entity {:db/id id updated-entity (cond-> {:db/id id
:client/code client-code :client/code client-code
:client/name (:name edit_client) :client/name (:name edit_client)
:client/matches (:matches edit_client) :client/matches (:matches edit_client)
:client/signature-file signature-file :client/email (:email edit_client)
:client/email (:email edit_client) :client/locked-until (some-> (:locked_until edit_client) (coerce/to-date))
:client/locked-until (some-> (:locked_until edit_client) (coerce/to-date)) :client/locations (filter identity (:locations edit_client))
:client/locations (filter identity (:locations edit_client)) :client/week-a-debits (:week_a_debits edit_client)
:client/week-a-debits (:week_a_debits edit_client) :client/week-a-credits (:week_a_credits edit_client)
:client/week-a-credits (:week_a_credits edit_client) :client/week-b-debits (:week_b_debits edit_client)
:client/week-b-debits (:week_b_debits edit_client) :client/square-auth-token (:square_auth_token edit_client)
:client/square-auth-token (:square_auth_token edit_client) :client/square-locations (map
:client/square-locations (map (fn [sl]
(fn [sl] {:db/id (or (:id sl) (random-tempid))
{:db/id (or (:id sl) (random-tempid)) :square-location/client-location (:client_location sl)})
:square-location/client-location (:client_location sl)}) (:square_locations edit_client))
(:square_locations edit_client))
:client/emails (map (fn [e] :client/emails (map (fn [e]
{:db/id (or (:id e) {:db/id (or (:id e)
(random-tempid)) (random-tempid))
:email-contact/email (:email e) :email-contact/email (:email e)
:email-contact/description (:description e)}) :email-contact/description (:description e)})
(:emails edit_client)) (:emails edit_client))
:client/feature-flags (:feature_flags edit_client) :client/feature-flags (:feature_flags edit_client)
:client/ezcater-locations (map :client/ezcater-locations (map
(fn [el] (fn [el]
{:db/id (or (:id el) (random-tempid)) {:db/id (or (:id el) (random-tempid))
:ezcater-location/location (:location el) :ezcater-location/location (:location el)
:ezcater-location/caterer (:caterer el)}) :ezcater-location/caterer (:caterer el)})
(:ezcater_locations edit_client)) (:ezcater_locations edit_client))
:client/week-b-credits (:week_b_credits edit_client) :client/week-b-credits (:week_b_credits edit_client)
:client/location-matches (->> (:location_matches edit_client) :client/location-matches (->> (:location_matches edit_client)
(filter (fn [lm] (and (:location lm) (:match lm)))) (filter (fn [lm] (and (:location lm) (:match lm))))
(map (fn [lm] {:db/id (or (:id lm ) (random-tempid)) (map (fn [lm] {:db/id (or (:id lm) (random-tempid))
:location-match/location (:location lm) :location-match/location (:location lm)
:location-match/matches [(:match lm)]}))) :location-match/matches [(:match lm)]})))
:client/address (when (seq (filter identity (vals (:address edit_client)))) :client/address (when (seq (filter identity (vals (:address edit_client))))
{:db/id (or (:id (:address edit_client)) (random-tempid)) {:db/id (or (:id (:address edit_client)) (random-tempid))
:address/street1 (:street1 (:address edit_client)) :address/street1 (:street1 (:address edit_client))
:address/street2 (:street2 (:address edit_client)) :address/street2 (:street2 (:address edit_client))
:address/city (:city (:address edit_client)) :address/city (:city (:address edit_client))
:address/state (:state (:address edit_client)) :address/state (:state (:address edit_client))
:address/zip (:zip (:address edit_client))}) :address/zip (:zip (:address edit_client))})
:client/bank-accounts (map (fn [ba] :client/bank-accounts (map (fn [ba]
{:db/id (or (:id ba) (random-tempid)) {:db/id (or (:id ba) (random-tempid))
:bank-account/code (:code ba) :bank-account/code (:code ba)
:bank-account/bank-name (:bank_name ba) :bank-account/bank-name (:bank_name ba)
:bank-account/bank-code (:bank_code ba) :bank-account/bank-code (:bank_code ba)
:bank-account/start-date (-> (:start_date ba) (coerce/to-date)) :bank-account/start-date (-> (:start_date ba) (coerce/to-date))
:bank-account/routing (:routing ba) :bank-account/routing (:routing ba)
:bank-account/include-in-reports (:include_in_reports ba) :bank-account/include-in-reports (:include_in_reports ba)
:bank-account/name (:name ba) :bank-account/name (:name ba)
:bank-account/visible (:visible ba) :bank-account/visible (:visible ba)
:bank-account/number (:number ba) :bank-account/number (:number ba)
:bank-account/check-number (:check_number ba) :bank-account/check-number (:check_number ba)
:bank-account/numeric-code (:numeric_code ba) :bank-account/numeric-code (:numeric_code ba)
:bank-account/sort-order (:sort_order ba) :bank-account/sort-order (:sort_order ba)
:bank-account/locations (:locations ba) :bank-account/locations (:locations ba)
:bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba)) :bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba))
:bank-account/yodlee-account-id (:yodlee_account_id ba) :bank-account/yodlee-account-id (:yodlee_account_id ba)
:bank-account/type (keyword "bank-account-type" (name (:type ba))) :bank-account/type (keyword "bank-account-type" (name (:type ba)))
:bank-account/yodlee-account (when (:yodlee_account ba) :bank-account/yodlee-account (when (:yodlee_account ba)
[:yodlee-account/id (:yodlee_account ba)]) [:yodlee-account/id (:yodlee_account ba)])
:bank-account/plaid-account (:plaid_account ba) :bank-account/plaid-account (:plaid_account ba)
:bank-account/intuit-bank-account (:intuit_bank_account ba)}) :bank-account/intuit-bank-account (:intuit_bank_account ba)})
(:bank_accounts edit_client)) (:bank_accounts edit_client))}
signature-file (assoc :client/signature-file signature-file))
}
_ (mu/log ::upserting :up updated-entity) _ (mu/log ::upserting :up updated-entity)
_ (assert-no-shared-transaction-sources client-code [[:upsert-entity updated-entity]]) _ (assert-no-shared-transaction-sources client-code [[:upsert-entity updated-entity]])
_ (log/info "upserting client" updated-entity) _ (log/info "upserting client" updated-entity)
@@ -167,18 +166,18 @@
(:client/name updated-client)))}]) (:client/name updated-client)))}])
(-> updated-client (-> updated-client
(update :client/bank-accounts (update :client/bank-accounts
(fn [bas] (fn [bas]
(map #(set/rename-keys % {:bank-account/use-date-instead-of-post-date? :use-date-instead-of-post-date}) bas))) (map #(set/rename-keys % {:bank-account/use-date-instead-of-post-date? :use-date-instead-of-post-date}) bas)))
(update :client/location-matches (update :client/location-matches
(fn [lms] (fn [lms]
(mapcat (fn [lm] (mapcat (fn [lm]
(map (fn [m] (map (fn [m]
{:location-match/match m {:location-match/match m
:location-match/location (:location-match/location lm)}) :location-match/location (:location-match/location lm)})
(:location-match/matches lm))) (:location-match/matches lm)))
lms))) lms)))
->graphql)))) ->graphql))))
(defn refresh-all-current-balance [] (defn refresh-all-current-balance []

View File

@@ -530,10 +530,17 @@
(when (seq ignore-retraction) (when (seq ignore-retraction)
(audit-transact-batch ignore-retraction (:id context)))) (audit-transact-batch ignore-retraction (:id context))))
#_(log/info (map :tx success)) #_(log/info (map :tx success))
(mu/trace ::success-tx (let [invalidated
[:count (count success)] (mu/trace ::success-tx
(doseq [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))] [:count (count success)]
(solr/touch n))) (for [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))]
n))]
(future
(mu/log ::indexing-solr :count (count invalidated))
(mu/trace ::indexed-external-solr
[:count (count invalidated)]
(doseq [n invalidated]
(solr/touch n)))))
{:successful (map (fn [x] {:external_id (:external_id x)}) success) {:successful (map (fn [x] {:external_id (:external_id x)}) success)
:ignored (map (fn [x] :ignored (map (fn [x]

View File

@@ -199,15 +199,14 @@
(defn rebuild-search-index [] (defn rebuild-search-index []
(de/future-with (de/future-with
single-thread single-thread
(auto-ap.solr/index-documents-raw (auto-ap.solr/index-documents-raw
auto-ap.solr/impl auto-ap.solr/impl
"vendors" "vendors"
(for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
:in $ :in $
:where [?v :vendor/search-terms ]] :where [?v :vendor/name]]
:args [(dc/db conn)]})] :args [(dc/db conn)]})]
{"id" (:db/id result) {"id" (:db/id result)
"name" (or (first (:vendor/search-terms result)) "name" (:vendor/name result)
(:vendor/name result)) "hidden" (boolean (:vendor/hidden result))}))))
"hidden" (boolean (:vendor/hidden result))}))))

View File

@@ -4,8 +4,7 @@
(def pdf-templates (def pdf-templates
[ [;; CHEF's WAREHOUSE
;; CHEF's WAREHOUSE
{:vendor "CHFW" {:vendor "CHFW"
:keywords [#"CHEF'S WAREHOUSE"] :keywords [#"CHEF'S WAREHOUSE"]
:extract {:total #"2 WKS C\.C\.\s+([\d.,]+)" :extract {:total #"2 WKS C\.C\.\s+([\d.,]+)"
@@ -75,9 +74,7 @@
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}} :total [:trim-commas-and-negate nil]}}
;; SOUTHBAY FRESH
;; SOUTHBAY FRESH
{:vendor "Southbay Fresh Produce" {:vendor "Southbay Fresh Produce"
:keywords [#"SOUTH BAY FRESH PRODUCE"] :keywords [#"SOUTH BAY FRESH PRODUCE"]
:extract {:date #"^([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"^([0-9]+/[0-9]+/[0-9]+)"
@@ -173,10 +170,9 @@
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]} :total [:trim-commas nil]}
:multi (. java.util.regex.Pattern (compile (-> \formfeed str) java.util.regex.Pattern/CASE_INSENSITIVE)) :multi (. java.util.regex.Pattern (compile (-> \formfeed str) java.util.regex.Pattern/CASE_INSENSITIVE))
:multi-match? #"(Total\s+[0-9\.]+|Total Order)" :multi-match? #"(Total\s+[0-9\.]+|Total Order)"}
}
;; AUTO-CHLOR ;; AUTO-CHLOR
{:vendor "Auto-Chlor" {:vendor "Auto-Chlor"
:keywords [#"AUTO-CHLOR"] :keywords [#"AUTO-CHLOR"]
:extract {:date #"DATE : ([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"DATE : ([0-9]+/[0-9]+/[0-9]+)"
@@ -208,8 +204,7 @@
:multi #"\n" :multi #"\n"
:multi-match? #"^\s+.*?\d{6,}.*?\$"} :multi-match? #"^\s+.*?\d{6,}.*?\$"}
;; C & L
;; C & L
{:vendor "C&L Produce" {:vendor "C&L Produce"
:keywords [#"440 Franklin Street"] :keywords [#"440 Franklin Street"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
@@ -239,8 +234,7 @@
:parser {:date [:clj-time "dd-MMM-yy"] :parser {:date [:clj-time "dd-MMM-yy"]
:total [:trim-commas-and-negate nil]}} :total [:trim-commas-and-negate nil]}}
;;; credits don't have the same format
;;; credits don't have the same format
{:vendor "General Produce Company" {:vendor "General Produce Company"
:keywords [#"1330 NORTH B"] :keywords [#"1330 NORTH B"]
:extract {:date #"DATE.*\n.*\n.*?([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"DATE.*\n.*\n.*?([0-9]+/[0-9]+/[0-9]+)"
@@ -362,9 +356,7 @@
:parser {:date [:clj-time "MM/dd/yyyy"] :parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}} :total [:trim-commas nil]}}
;; PACIFIC SEAFOOD
;; PACIFIC SEAFOOD
{:vendor "Pacific Seafood" {:vendor "Pacific Seafood"
:keywords [#"(pacseafood|PACIFIC FRESH)"] :keywords [#"(pacseafood|PACIFIC FRESH)"]
:extract {:date #"DATE(?:.*\n.*(?=([0-9]+/[0-9]+/[0-9]+)))([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"DATE(?:.*\n.*(?=([0-9]+/[0-9]+/[0-9]+)))([0-9]+/[0-9]+/[0-9]+)"
@@ -426,8 +418,7 @@
:total #" Total:\s+([\d\.]+)"} :total #" Total:\s+([\d\.]+)"}
:parser {:date [:clj-time "MMM dd, yyyy"]}} :parser {:date [:clj-time "MMM dd, yyyy"]}}
;; A&B
;; A&B
{:vendor "A&B Produce" {:vendor "A&B Produce"
:keywords [#"ABProduce"] :keywords [#"ABProduce"]
:extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)"
@@ -560,9 +551,7 @@
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]}} :total [:trim-commas nil]}}
;; JFC
;; JFC
{:vendor "JFC International" {:vendor "JFC International"
:keywords [#"48490 MILMONT DRIVE"] :keywords [#"48490 MILMONT DRIVE"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
@@ -609,6 +598,15 @@
:customer-identifier #"Bill To\s+(.*?)\s{2,}" :customer-identifier #"Bill To\s+(.*?)\s{2,}"
:invoice-number #"Invoice #\n.*?([\d\-]+)\n" :invoice-number #"Invoice #\n.*?([\d\-]+)\n"
:total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"} :total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
{:vendor "Reel Produce"
:keywords [#"reelproduce.com"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice #\n.*?([\d\-]+)\n"
:total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}]) :total [:trim-commas-and-negate nil]}}])

View File

@@ -136,7 +136,8 @@
(index-documents-raw [this index xs] (index-documents-raw [this index xs]
(client/post (client/post
(str (assoc (url/url solr-uri "solr" index "update") (str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 5000})) :query {"commitWithin" 5000
"commit" true}))
{:headers {"Content-Type" "application/json"} {:headers {"Content-Type" "application/json"}
:socket-timeout 30000 :socket-timeout 30000
@@ -147,7 +148,8 @@
(index-documents [this index xs] (index-documents [this index xs]
(client/post (client/post
(str (assoc (url/url solr-uri "solr" index "update") (str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 5000})) :query {"commitWithin" 5000
"commit" true}))
{:headers {"Content-Type" "application/json"} {:headers {"Content-Type" "application/json"}
:socket-timeout 30000 :socket-timeout 30000
:connection-timeout 30000 :connection-timeout 30000
@@ -168,7 +170,8 @@
(delete [this index] (delete [this index]
(client/post (client/post
(str (assoc (url/url solr-uri "solr" index "update") (str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 15000})) :query {"commitWithin" 15000
"commit" true}))
{:headers {"Content-Type" "application/json"} {:headers {"Content-Type" "application/json"}
:method "POST" :method "POST"
:body (json/write-str {"delete" {"query" "*:*"}})}))) :body (json/write-str {"delete" {"query" "*:*"}})})))