tweak

scratch-sessions/fixing-duplicate-vendors.clj

@@ -0,0 +1,165 @@
+;; This buffer is for Clojure experiments and evaluation.
+
+;; Press C-j to evaluate the last expression.
+
+;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
+
+(init-repl)
+
+(def best 17592232929447)
+best
+
+
+(def others (->> (dc/q '[:find ?v
+                         :in $ [?name ...] ?b
+                         :where [?v :vendor/name ?name]
+                         (not [(= ?v ?b)])]
+                       (dc/db conn)
+                       ["CCP Grubhub" "CCP Grub Hub" "CCP GrubHub"]
+                       best)
+                 (map first)
+                 (into #{})))
+
+(count others)
+
+#_@(dc/transact auto-ap.datomic/conn
+             (map (fn [o]
+                    [:db/retractEntity o])
+                  others))
+
+
+(count 
+  (for [[incoming incoming-attr from-vendor] (dc/q {:find  '[?x ?a2 ?vendor-from]
+                                                    :in    '[$ [?a ...] [?vendor-from ...] ]
+                                                    :where ['[?x ?a ?vendor-from]
+                                                            '[?a :db/ident ?a2]]}
+                                                   (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
+        part                                 [[:db/retract incoming incoming-attr from-vendor]
+                                              [:db/add incoming incoming-attr best]]]
+    part
+    ))
+
+(auto-ap.datomic/audit-transact-batch
+  (for [[incoming incoming-attr from-vendor] (dc/q {:find  '[?x ?a2 ?vendor-from]
+                                                    :in    '[$ [?a ...] [?vendor-from ...] ]
+                                                    :where ['[?x ?a ?vendor-from]
+                                                            '[?a :db/ident ?a2]]}
+                                                   (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
+        part                     [[:db/retract incoming incoming-attr from-vendor]
+                                  [:db/add incoming incoming-attr best]]]
+    part
+    )
+  {:user/name "Fixing duplicate vendors"})
+
+
+(set (map second (dc/q '[:find  ?x ?a2 ?vendor-from
+                         :in    $ [?a ...] [?vendor-from ...]
+                         :where [?x ?a ?vendor-from]
+                         [?a :db/ident ?a2]]
+                      (dc/db conn) #{ :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)))
+
+
+
+(->> (dc/q '[:find  ?v ?c (count ?e)
+             :in    $ ?v
+             :where
+             [?v :vendor/name]
+             (or-join [?v ?c ?e]
+                      (and 
+                        [?e :invoice/vendor ?v]
+                        [?e :invoice/client ?c])
+                      (and
+                        [?e :transaction/vendor ?v]
+                        [?e :transaction/client ?c])
+                      (and
+                        [?e :journal-entry/vendor ?v]
+                        [?e :journal-entry/client ?c]))]
+           (dc/db conn)
+           best)
+     (map (fn [[v c cnt]]
+            #:vendor-usage {:vendor v
+                            :client c
+                            :key    (str v "-" c)
+                            :count  cnt})))
+
+
+(auto-ap.datomic/audit-transact-batch 
+  (for [[e a] (dc/q '[:find ?e ?a 
+                      :in $$ [?tx ...] [?other ...] ?false-best
+                      :where
+                      [$$ ?e ?a ?false-best ?tx true]
+                      [$$ ?e ?a ?other ?tx false]
+                      [$$ ?other :vendor/name ?vn]
+                      [(clojure.string/includes? ?vn "rub")]
+                      ]
+
+                    (dc/history (dc/db conn))
+                    (map (comp dc/t->tx :t) (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00"))
+                    #_(dc/tx-range (dc/log conn) #inst "2023-06-06T22:15:00-08:00" #inst "2023-06-06T22:31:00-08:00")
+                    others
+                    17592232586913)
+        part [[:db/retract e a 17592232586913]
+              [:db/add e a 17592270032394]]]
+    part
+    )
+  {:user/name "re-fixing grub-hub"})
+
+
+(map :t (dc/tx-range (dc/log conn) #inst "2023-06-06T21:00:00-08:00" #inst "2023-06-06T22:31:00-08:00"))
+
+(dc/pull (dc/db conn) '[*] 17592232929447)
+
+
+
+(let [best 17592232586918
+      others (->> (dc/q '[:find ?v
+                          :in $ [?name ...] ?b
+                          :where [?v :vendor/name ?name]
+                          (not [(= ?v ?b)])]
+                        (dc/db conn)
+                        ["CCP Uber Eats" "CCP UBER" "CCP Ubereats" "CCP Uber Eats MP" "CCP Uber Eats SC" "CCP UberEats" "CCP Ubereats" "CCP Uber"]
+                        best)
+                  (map first)
+                  (into #{}))]
+  
+  (auto-ap.datomic/audit-transact-batch
+    (for [[incoming incoming-attr from-vendor] (dc/q {:find  '[?x ?a2 ?vendor-from]
+                                                      :in    '[$ [?a ...] [?vendor-from ...] ]
+                                                      :where ['[?x ?a ?vendor-from]
+                                                              '[?a :db/ident ?a2]]}
+                                                     (dc/db conn) #{:expected-deposit/vendor, :invoice/vendor, :journal-entry/vendor, :payment/vendor, :sales-order/vendor, :sales-refund/vendor, :transaction/vendor, :transaction-rule/vendor, :cash-drawer-shift/vendor} others)
+          part                                 [[:db/retract incoming incoming-attr from-vendor]
+                                                [:db/add incoming incoming-attr best]]]
+      part
+      )
+    {:user/name "Fixing duplicate vendors"})
+
+  @(dc/transact auto-ap.datomic/conn
+                (map (fn [o]
+                       [:db/retractEntity o])
+                     others)))
+
+
+(require '[datomic.api :as dc])
+
+(require '[auto-ap.datomic :refer [conn]])
+(user/init-repl)
+
+(auto-ap.datomic/audit-transact-batch
+ (for [[bad] (seq (dc/q '[:find  ?x ?vn
+                          :in    $
+                          :where
+                          [?x :vendor/name ?vn]
+                          (not [_ :expected-deposit/vendor ?x])
+                          (not [_ :invoice/vendor ?x])
+                          (not [_ :journal-entry/vendor ?x])
+                          (not [_ :payment/vendor ?x])
+                          (not [_ :sales-order/vendor ?x])
+                          (not [_ :payment/vendor ?x])
+                          (not [_ :sales-refund/vendor ?x])
+                          (not [_ :transaction/vendor ?x])
+                          (not [_ :transaction-rule/vendor ?x])
+                          (not [_ :cash-drawer-shift/vendor ?x])]
+                        (dc/db conn)))]
+   [:db/retractEntity bad])
+ {:user/name "fix-unused-vendors"})

src/clj/auto_ap/graphql/clients.clj

@@ -78,11 +78,10 @@
         client-code (if (str/blank? (:client/code client))
                       (:code edit_client)
                       (:client/code client))
-        updated-entity {:db/id                    id
+        updated-entity (cond-> {:db/id                    id
                                 :client/code              client-code
                                 :client/name              (:name edit_client)
                                 :client/matches           (:matches edit_client)
-                        :client/signature-file    signature-file
                                 :client/email             (:email edit_client)
                                 :client/locked-until      (some-> (:locked_until edit_client) (coerce/to-date))
                                 :client/locations         (filter identity (:locations edit_client))
@@ -147,9 +146,9 @@
                                                                                                       [:yodlee-account/id (:yodlee_account ba)])
                                                                   :bank-account/plaid-account       (:plaid_account ba)
                                                                   :bank-account/intuit-bank-account (:intuit_bank_account ba)})
-                                                       (:bank_accounts edit_client))
+                                                               (:bank_accounts edit_client))}
+                         signature-file (assoc :client/signature-file    signature-file))
 
-                        }
         _ (mu/log ::upserting :up updated-entity)
         _ (assert-no-shared-transaction-sources client-code [[:upsert-entity updated-entity]])
         _ (log/info "upserting client" updated-entity)

src/clj/auto_ap/graphql/ledger.clj

@@ -530,10 +530,17 @@
       (when (seq ignore-retraction)
         (audit-transact-batch ignore-retraction (:id context))))
     #_(log/info (map :tx success))
+    (let [invalidated 
           (mu/trace ::success-tx
                     [:count (count success)]
-      (doseq [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))]
+                    (for [[_ n] (:tempids (audit-transact-batch (map :tx success) (:id context)))]
-        (solr/touch n)))
+                      n))]
+      (future 
+        (mu/log ::indexing-solr :count (count invalidated))
+        (mu/trace ::indexed-external-solr
+                  [:count (count invalidated)]
+                  (doseq [n invalidated]
+                    (solr/touch n)))))
     
     {:successful (map (fn [x] {:external_id (:external_id x)}) success)
      :ignored    (map (fn [x]

src/clj/auto_ap/graphql/vendors.clj

@@ -205,9 +205,8 @@
     "vendors"
     (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
                                       :in $
-                                        :where [?v :vendor/search-terms ]]
+                                      :where [?v :vendor/name]]
                              :args  [(dc/db conn)]})]
       {"id"     (:db/id result)
-         "name"   (or (first (:vendor/search-terms result))
+       "name"   (:vendor/name result)
-                      (:vendor/name result))
        "hidden" (boolean (:vendor/hidden result))}))))

src/clj/auto_ap/parse/templates.clj

@@ -4,8 +4,7 @@
 
 
 (def pdf-templates
-  [
+  [;; CHEF's WAREHOUSE
-   ;; CHEF's WAREHOUSE
    {:vendor "CHFW"
     :keywords [#"CHEF'S WAREHOUSE"]
     :extract {:total #"2 WKS C\.C\.\s+([\d.,]+)"
@@ -75,8 +74,6 @@
     :parser {:date [:clj-time "MM/dd/yy"]
              :total [:trim-commas-and-negate nil]}}
 
-
-
 ;; SOUTHBAY FRESH
    {:vendor "Southbay Fresh Produce"
     :keywords [#"SOUTH BAY FRESH PRODUCE"]
@@ -173,8 +170,7 @@
     :parser {:date [:clj-time "MM/dd/yy"]
              :total [:trim-commas nil]}
     :multi (. java.util.regex.Pattern (compile (-> \formfeed str) java.util.regex.Pattern/CASE_INSENSITIVE))
-    :multi-match? #"(Total\s+[0-9\.]+|Total Order)"
+    :multi-match? #"(Total\s+[0-9\.]+|Total Order)"}
-    }
 
 ;; AUTO-CHLOR
    {:vendor "Auto-Chlor"
@@ -208,7 +204,6 @@
     :multi #"\n"
     :multi-match? #"^\s+.*?\d{6,}.*?\$"}
 
-
 ;; C & L
    {:vendor "C&L Produce"
     :keywords [#"440 Franklin Street"]
@@ -239,7 +234,6 @@
     :parser   {:date  [:clj-time "dd-MMM-yy"]
                :total [:trim-commas-and-negate nil]}}
 
-
 ;;; credits don't have the same format
    {:vendor "General Produce Company"
     :keywords [#"1330 NORTH B"]
@@ -362,8 +356,6 @@
     :parser {:date [:clj-time "MM/dd/yyyy"]
              :total [:trim-commas nil]}}
 
-
-
 ;; PACIFIC SEAFOOD
    {:vendor "Pacific Seafood"
     :keywords [#"(pacseafood|PACIFIC FRESH)"]
@@ -426,7 +418,6 @@
               :total #" Total:\s+([\d\.]+)"}
     :parser {:date [:clj-time "MMM dd, yyyy"]}}
 
-
 ;; A&B
    {:vendor "A&B Produce"
     :keywords [#"ABProduce"]
@@ -560,8 +551,6 @@
     :parser {:date [:clj-time "MM/dd/yy"]
              :total [:trim-commas nil]}}
 
-   
-
 ;; JFC
    {:vendor "JFC International"
     :keywords [#"48490 MILMONT DRIVE"]
@@ -609,6 +598,15 @@
               :customer-identifier #"Bill To\s+(.*?)\s{2,}"
               :invoice-number #"Invoice #\n.*?([\d\-]+)\n"
               :total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"}
+    :parser {:date [:clj-time "MM/dd/yy"]
+             :total [:trim-commas-and-negate nil]}}
+
+   {:vendor "Reel Produce"
+    :keywords [#"reelproduce.com"]
+    :extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
+              :customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}"
+              :invoice-number #"Invoice #\n.*?([\d\-]+)\n"
+              :total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"}
     :parser {:date [:clj-time "MM/dd/yy"]
              :total [:trim-commas-and-negate nil]}}])
 

src/clj/auto_ap/solr.clj

@@ -136,7 +136,8 @@
   (index-documents-raw [this index xs]
     (client/post
       (str (assoc (url/url solr-uri "solr" index "update")
-                  :query {"commitWithin" 5000}))
+                  :query {"commitWithin" 5000
+                          "commit" true}))
       
       {:headers            {"Content-Type" "application/json"}
        :socket-timeout     30000
@@ -147,7 +148,8 @@
   (index-documents [this index xs]
     (client/post
       (str (assoc (url/url solr-uri "solr" index "update")
-                   :query {"commitWithin" 5000}))
+                   :query {"commitWithin" 5000
+                           "commit" true}))
       {:headers            {"Content-Type" "application/json"}
        :socket-timeout     30000
        :connection-timeout 30000
@@ -168,7 +170,8 @@
   (delete [this index]
     (client/post
       (str (assoc (url/url solr-uri "solr" index "update")
-                  :query {"commitWithin" 15000}))
+                  :query {"commitWithin" 15000
+                          "commit" true}))
       {:headers {"Content-Type" "application/json"}
        :method  "POST"
        :body    (json/write-str {"delete" {"query" "*:*"}})})))