(cloud) adjustments for ezcater tenders, search fixes.

2023-05-18 16:56:58 -07:00
parent 3ce9cceee0
commit 0e6a5c6749
6 changed files with 141 additions and 125 deletions
--- a/src/clj/auto_ap/graphql/accounts.clj
+++ b/src/clj/auto_ap/graphql/accounts.clj
@@ -7,16 +7,17 @@
            <-graphql
            assert-admin
            assert-can-see-client
+            can-see-client?
            cleanse-query
            enum->keyword
            is-admin?
+            limited-clients
            result->page]]
   [auto-ap.search :as search]
-   [auto-ap.utils :refer [heartbeat]]
+   [auto-ap.solr :as solr]
   [datomic.api :as dc]
   [iol-ion.tx :refer [random-tempid]]
-   [mount.core :as mount]
-   [yang.scheduler :as scheduler]))
+   [com.brunobonacci.mulog :as mu]))

 (defn get-graphql [context args _]
  (assert-admin (:id context))
@@ -68,9 +69,29 @@
                                                                             :account-client-override/search-terms (:name client-override)})
                                                                          client-overrides)}
                                       id (dissoc :account/numeric-code :account/code))]]
-                                 (:id context))]
+                                 (:id context))
+          updated-account (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))]
+      (mu/log ::account-updated :account updated-account)
+      (solr/index-documents-raw solr/impl
+                                "accounts"
+                                (into [{"id"            (:db/id updated-account)
+                                        "account_id"    (:db/id updated-account)
+                                        "name"          (:account/name updated-account)
+                                        "numeric_code"  (:account/numeric-code updated-account)
+                                        "location"      (:account/location updated-account)
+                                        "applicability" (clojure.core/name (:account/applicability updated-account))}]
+                                      (for [o (:account/client-overrides updated-account)]
+                                        {"id"            (:db/id o)
+                                         "account_id"    (:db/id updated-account)
+                                         "name"          (:account-client-override/name o)
+                                         "numeric_code"  (:account/numeric-code updated-account)
+                                         "location"      (:account/location updated-account)
+                                         "applicability" (clojure.core/name (:account/applicability updated-account))
+                                         "client_id" (:db/id (:account-client-override/client o))
+                                         "account_client_override_id" (:db/id o)}))
+                                )
      (->graphql 
-        (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))))))
+        updated-account))))

 (def search-pattern [:db/id
                             :account/numeric-code
@@ -78,12 +99,25 @@
                             {:account/vendor-allowance  [:db/ident]
                              :account/default-allowance [:db/ident]
                              :account/invoice-allowance [:db/ident]}])
+(defn search- [id query client]
+  (let [client-part (if (some->> client (can-see-client? id))
+                      (format "((applicability:(global OR optional) AND -client_id:*) OR (account_client_override_id:* AND client_id:%s))" client)
+                      "(applicability:(global OR optional) AND -client_id:*)"
+
+                      )
+        query (format "_text_:(%s) AND %s" (cleanse-query query) client-part)]
+    (mu/log ::searching :search-query query)
+    (for [{:keys [account_id name] :as g} (solr/query solr/impl "accounts"
+                                                 {"query" query
+                                                  "fields" "id, name, client_id, numeric_code, applicability, account_id"})]
+     
+      {:account_id (first account_id)
+      :name       (first name)})))
+
 (defn search [context {query :query client :client_id allowance :allowance vendor-id :vendor_id} _]
  (when client
    (assert-can-see-client (:id context) client))
-  (let [query (cleanse-query query)
-        _ (println query)
-        num (some-> (re-find #"([0-9]+)" query)
+  (let [num (some-> (re-find #"([0-9]+)" query)
                    second
                    (not-empty )
                    Integer/parseInt)
@@ -130,41 +164,34 @@
                  allowance
                  search-pattern)
             (sequence xform))
-        (->> (dc/q '[:find ?n (pull ?i pattern) ?s
-                     :in $ [[?i ?n ?s]] pattern
-                     :where (or [?i :account/applicability :account-applicability/global]
-                                [?i :account/applicability :account-applicability/optional])]
-                  (dc/db conn)
-                  (search/search {:q query} "account")
-                  search-pattern)
-             (concat (when client
-                       (dc/q '[:find ?n (pull ?i pattern) ?s
-                              :in $ [[?i ?n ?s]] pattern]
-                            (dc/db conn)
-                            (search/search {:q query :client (str client)} "account-client-override")
-                            search-pattern)))
-             (sort-by (comp - last))
-             (sequence xform)))
+        (->> (search- (:id context) query client)
+             (sequence
+               (comp (map (fn [i] [(:name i) (dc/pull (dc/db conn) search-pattern (:account_id i))]))
+                     xform))))
      [])))

 (defn rebuild-search-index []
-  (search/full-index-query  
-    (for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}])
+  (solr/index-documents-raw
+    solr/impl
+    "accounts"
+    (for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id {:account/applicability [:db/ident]}]}])
                                               :in $
                                               :where [?aco :account-client-override/client ]
                                               [?aco :account-client-override/search-terms ]
                                               [_ :account/client-overrides ?aco]]
                                      :args  [(dc/db conn)]}))
-          :when (:account/numeric-code (:account/_client-overrides result))]
-      {:id     (:db/id (:account/_client-overrides result))
-       :account-client-override-id (:db/id result)
-       :text   (:account-client-override/search-terms result)
-       :client (str (:db/id (:account-client-override/client result)))
-       :numeric-code (:account/numeric-code (:account/_client-overrides result))
-       :location (:account/location (:account/_client-overrides result))})
-    "account-client-override")
-
-  (search/full-index-query  
+          :when  (:account/numeric-code (:account/_client-overrides result))]
+      {"id"                         (:db/id result)
+       "account_id" (:db/id (:account/_client-overrides result))
+       "account_client_override_id" (str (:db/id result))
+       "name"                       (:account-client-override/search-terms result)
+       "client_id"                  (str (:db/id (:account-client-override/client result)))
+       "numeric_code"               (:account/numeric-code (:account/_client-overrides result))
+       "location"                   (:account/location (:account/_client-overrides result))
+       "applicability" (name (:db/ident (:account/applicability (:account/_client-overrides result))))}))
+  (solr/index-documents-raw
+    solr/impl
+    "accounts"
    (for [result (map first (dc/qseq {:query '[:find (pull ?a [:account/numeric-code
                                                               :account/search-terms
                                                               {:account/applicability [:db/ident]}
@@ -173,14 +200,14 @@
                                               :in $
                                               :where [?a :account/search-terms ]]
                                      :args  [(dc/db conn)]}))
-          :when (:account/search-terms result)
+          :when  (:account/search-terms result)
          ]
-      {:id     (:db/id result)
-       :text   (:account/search-terms result)
-       :numeric-code (:account/numeric-code result)
-       :location (:account/location result)
-       :applicability (name (:db/ident (:account/applicability result)))})
-    "account"))
+      {"id"            (:db/id result)
+       "account_id"            (:db/id result)
+       "name"          (:account/search-terms result)
+       "numeric_code"  (:account/numeric-code result)
+       "location"      (:account/location result)
+       "applicability" (name (:db/ident (:account/applicability result)))})))

 #_(dc/transact conn
               [{:db/ident       :account-client-override/k2
@@ -195,8 +222,3 @@


 #_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k])
-
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(mount/defstate indexer
-  :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index"))
-  :stop (scheduler/stop indexer))
--- a/src/clj/auto_ap/graphql/vendors.clj
+++ b/src/clj/auto_ap/graphql/vendors.clj
@@ -3,6 +3,7 @@
   [auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]]
   [iol-ion.tx :refer [random-tempid]]
   [auto-ap.datomic.vendors :as d-vendors]
+   [auto-ap.solr :as solr]
   [auto-ap.graphql.utils
    :refer [->graphql
            <-graphql
@@ -138,10 +139,18 @@
                                                                  :vendor/automatically-paid-when-due (:automatically_paid_when_due in)))]

        
-        transaction-result (audit-transact [transaction] (:id context))]
+        transaction-result (audit-transact [transaction] (:id context))
+        new-vendor (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
+                                            id))]
    
-    (-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
-                                 id))
+    (auto-ap.solr/index-documents-raw
+      auto-ap.solr/impl
+      "vendors"
+      [{"id"     (:db/id new-vendor)
+        "name"   (:vendor/name new-vendor)
+        "hidden" (boolean (:vendor/hidden new-vendor))}])
+    
+    (-> new-vendor
        (->graphql))))

 (defn merge-vendors [context {:keys [from to]} _]
@@ -183,64 +192,28 @@
    matches))

 (defn search [context args _]
-  (let [search-query (cleanse-query (:query args))]
-    (for [[id name] (search/search (cond-> {:q search-query}
-                                     (not (is-admin? (:id context))) (assoc :hidden false))
-                                   "vendor")]
-      {:name name
-       :id   id})))
+  (let [search-query (str "name:(" (cleanse-query (:query args)) ")")]
+    
+
+    (for [{:keys [id name]} (solr/query solr/impl "vendors" {"query"  (cond-> search-query
+                                                                       (not (is-admin? (:id context))) (str " hidden:false"))
+                                                             "fields" "id, name"})]
+      {:id (Long/parseLong id)
+       :name (first name)})))

 (def single-thread (ex/fixed-thread-executor 1))

 (defn rebuild-search-index []
  (de/future-with
    single-thread
-    (search/full-index-query  
+    (auto-ap.solr/index-documents-raw
+      auto-ap.solr/impl
+      "vendors"
      (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
                                        :in $
                                        :where [?v :vendor/search-terms ]]
                               :args  [(dc/db conn)]})]
-        {:id     (:db/id result)
-         :text   (or (first (:vendor/search-terms result))
-                     (:vendor/name result))
-         :hidden (boolean (:vendor/hidden result))})
-      "vendor")))
-
-(def last-run-basis (atom nil))
-
-(defn add-incremental-changes []
-  (de/future-with
-    single-thread
-    (if-let [last-run-basis-value  @last-run-basis]
-      (let [db         (dc/db conn)
-            recent     (dc/since db last-run-basis-value)
-            _          (mu/log ::indexing
-                      :last-run last-run-basis-value
-                      :starting-from (:basisT db))
-            results    (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
-                                                         :in $ $$
-                                                         :where [$ ?v :vendor/name ]
-                                                         [$$ ?v]]
-                                                :args  [db recent]})]
-                         {:id     (:db/id result)
-                          :text   (or (first (:vendor/search-terms result))
-                                      (:vendor/name result))
-                          :hidden (boolean (:vendor/hidden result))})]
-        (when (seq results)
-          (mu/log ::adding-to-index
-                  :sample (first results)
-                  :count (count results))
-          (search/full-index-query results "vendor" false))
-        (reset! last-run-basis (:basisT db))
-        (count results))
-      (reset! last-run-basis (:basisT (dc/db conn))))))
-
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(mount/defstate indexer
-  :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index"))
-  :stop (scheduler/stop indexer))
-
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(mount/defstate incremental-indexer
-  :start (scheduler/every (* 5 1000) (heartbeat add-incremental-changes "incremental-indexing"))
-  :stop (scheduler/stop incremental-indexer))
+        {"id"     (:db/id result)
+         "name"   (or (first (:vendor/search-terms result))
+                      (:vendor/name result))
+         "hidden" (boolean (:vendor/hidden result))}))))
--- a/src/clj/auto_ap/routes/ezcater_xls.clj
+++ b/src/clj/auto_ap/routes/ezcater_xls.clj
@@ -89,7 +89,8 @@
                              :tip         (fmt-amount tip)}]
            :total          (fmt-amount (+ food-total
                                           tax
-                                           tip))
+                                           tip
+                                           (or adjustments 0.0)))
            :discount       (fmt-amount (or adjustments 0.0))
            :service-charge (fmt-amount (+ fee commission))
            :tax            (fmt-amount tax)
--- a/src/clj/auto_ap/solr.clj
+++ b/src/clj/auto_ap/solr.clj
@@ -1,6 +1,7 @@
 (ns auto-ap.solr
  (:require
   [auto-ap.datomic :refer [conn pull-id]]
+   [cemerick.url :as url]
   [auto-ap.time :as atime]
   [clj-http.client :as client]
   [clj-time.coerce :as c]
@@ -118,26 +119,38 @@


 (defprotocol SolrClient
-  (index-documents [this xs])
-  (query [this q])
-  (delete [this]))
+  (index-documents-raw [this index xs])
+  (index-documents [this index xs])
+  (query [this index q])
+  (delete [this index]))

 (defrecord RealSolrClient [solr-uri]

  SolrClient
-  (index-documents [this xs]
+  (index-documents-raw [this index xs]
    (client/post
-      (str solr-uri "/solr/invoices/update?commitWithin=15000")
+      (str (assoc (url/url solr-uri "solr" index "update")
+                  :query {"commitWithin" 60000}))
+      
+      {:headers            {"Content-Type" "application/json"}
+       :socket-timeout     30000
+       :connection-timeout 30000
+       :method             "POST"
+       :body               (json/write-str xs)}))
+
+  (index-documents [this index xs]
+    (client/post
+      (str (assoc (url/url solr-uri "solr" index "update")
+                   :query {"commitWithin" 60000}))
      {:headers            {"Content-Type" "application/json"}
       :socket-timeout     30000
       :connection-timeout 30000
       :method             "POST"
       :body               (json/write-str (filter identity (map datomic->solr xs)))}))

-  (query [this q]
-    (-> (client/post (str solr-uri "/solr/invoices/query")
-                     {:body               (json/write-str {"query"  q 
-                                                           "fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"})
+  (query [this index q]
+    (-> (client/post (str (url/url solr-uri "solr" index "query"))
+                     {:body               (json/write-str q )
                      :socket-timeout     30000
                      :connection-timeout 30000
                      :headers            {"Content-Type" "application/json"}
@@ -146,21 +159,25 @@
        :body
        :response
        :docs))
-  (delete [this]
+  (delete [this index]
    (client/post
-      (str solr-uri "/solr/invoices/update?commitWithin=1000")
+      (str (assoc (url/url solr-uri "solr" index "update")
+                  :query {"commitWithin" 15000}))
      {:headers {"Content-Type" "application/json"}
       :method  "POST"
       :body    (json/write-str {"delete" {"query" "*:*"}})})))

 (defrecord MockSolrClient []
  SolrClient
-  (index-documents [this xs]
+  (index-documents [this index xs]
    nil)

-  (query [this q]
+  (index-documents-raw [this index xs]
    nil)
-  (delete [this]
+
+  (query [this index q]
+    nil)
+  (delete [this index]
    nil))

 (def impl (if (= :solr (:solr-impl env))
@@ -172,9 +189,11 @@


 (defn touch-with-ledger [i]
-  (index-documents impl [i [:journal-entry/original-entity i]]))
+  (index-documents impl  "invoices" [i [:journal-entry/original-entity i]]))

-(defn touch [i]
-  (index-documents impl [i]))
+(defn touch
+  ([i] (touch i "invoices"))
+  ([i index]
+   (index-documents impl index [i])))


--- a/src/clj/auto_ap/ssr/search.clj
+++ b/src/clj/auto_ap/ssr/search.clj
@@ -53,7 +53,8 @@
  (into []
        (filter (fn [d]
                  (can-see-client? id (first (:client_id d)))))
-        (solr/query solr/impl (q->solr-q q))))
+        (solr/query solr/impl "invoices" {"query" (q->solr-q q)
+                                          "fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"})))


 (defn search-results* [q id]
--- a/src/clj/user.clj
+++ b/src/clj/user.clj
@@ -346,7 +346,7 @@
  (mu/start-publisher! {:type :dev})
  (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn })))

-(defn start-search []
+#_(defn start-search []
  (mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer})))

 (defn restart-db []
@@ -573,7 +573,7 @@
                     (partition-all 1000))]
    (print ".")
    (flush)
-    (solr/index-documents solr/impl batch))
+    (solr/index-documents solr/impl "invoices" batch))

  (doseq [batch (->> (dc/qseq {:query '[:find ?i
                                        :in $
@@ -584,7 +584,7 @@
                     (partition-all 1000))]
    (print ".")
    (flush)
-    (solr/index-documents solr/impl batch))
+    (solr/index-documents solr/impl "invoices" batch))

  (doseq [batch (->> (dc/qseq {:query '[:find ?i
                                        :in $
@@ -595,7 +595,7 @@
                     (partition-all 1000))]
    (print ".")
    (flush)
-    (solr/index-documents solr/impl batch))
+    (solr/index-documents solr/impl "invoices" batch))
  (doseq [batch (->> (dc/qseq {:query '[:find ?i
                                        :in $
                                        :where [?i :journal-entry/date]]
@@ -604,7 +604,7 @@
                     (partition-all 1000))]
    (print ".")
    (flush)
-    (solr/index-documents solr/impl batch)))
+    (solr/index-documents solr/impl "invoices" batch)))

 (defn setup-sales-orders []
  (doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]})