cloud - finishes search implementation

2023-05-18 21:05:48 -07:00
parent 0e6a5c6749
commit cf34b4af7e
13 changed files with 1901 additions and 158 deletions
--- a/src/clj/auto_ap/datomic/clients.clj
+++ b/src/clj/auto_ap/datomic/clients.clj
@@ -9,11 +9,12 @@
            pull-many
            query2]]
   [auto-ap.graphql.utils :refer [limited-clients]]
-   [auto-ap.search :as search]
   [clj-time.coerce :as coerce]
   [clojure.string :as str]
   [com.brunobonacci.mulog :as mu]
-   [datomic.api :as dc]))
+   [datomic.api :as dc]
+   [clj-http.client :as client]
+   [auto-ap.solr :as solr]))

 (def full-read '[*
                 {:client/square-integration-status [:integration-status/message
@@ -107,23 +108,38 @@
   (first)))

 (defn best-match [identifier]
-  (first (search/search-ids {:q (str/replace identifier #"[\(\)\-/\*\]\[\#:\&]" " ")} "client")))
+  (when (and identifier (not-empty identifier))  
+    (some-> (solr/query solr/impl "clients"
+                        {"query"  (format "_text_:\"%s\"" (str/upper-case (solr/escape identifier)))
+                         "fields" "id"})
+            first
+            :id
+            Long/parseLong)))
+
+

 (defn exact-match [identifier]
-  (first (search/search-ids {:exact-match (str/upper-case identifier)} "client")))
+  (when (and identifier (not-empty identifier))  
+    (some-> (solr/query solr/impl "clients"
+                        {"query"  (format "exact:\"%s\"" (str/upper-case (solr/escape identifier)))
+                         "fields" "id"})
+            first
+            :id
+            Long/parseLong)))

 (defn rebuild-search-index []
-  (search/full-index-query  
-    (for [result (map first (dc/q '[:find (pull ?v [:client/name :client/matches :db/id])
-                                       :in $
-                                       :where [?v :client/code]]
-                                     (dc/db conn)))
-          match  (conj (or (:client/matches result) [])
-                      (:client/name result))]
-      {:id   (:db/id result)
-       :text match
-       :exact-match (str/upper-case match)})
-    "client"))
+  (solr/index-documents-raw solr/impl
+                            "clients"
+                            (for [result (map first (dc/q '[:find (pull ?v [:client/name :client/matches :db/id :client/code])
+                                                            :in $
+                                                            :where [?v :client/code]]
+                                                          (dc/db conn)))
+                                  :let   [matches (conj (or (:client/matches result) [])
+                                                        (:client/name result))]]
+                              {"id"    (:db/id result)
+                               "name"  matches
+                               "code"  (:client/code result)
+                               "exact" (map str/upper-case matches)})))


 (defn raw-graphql-ids [db args]
--- a/src/clj/auto_ap/graphql/accounts.clj
+++ b/src/clj/auto_ap/graphql/accounts.clj
@@ -13,7 +13,6 @@
            is-admin?
            limited-clients
            result->page]]
-   [auto-ap.search :as search]
   [auto-ap.solr :as solr]
   [datomic.api :as dc]
   [iol-ion.tx :refer [random-tempid]]
--- a/src/clj/auto_ap/graphql/clients.clj
+++ b/src/clj/auto_ap/graphql/clients.clj
@@ -24,7 +24,8 @@
   [iol-ion.tx :refer [random-tempid]]
   [mount.core :as mount]
   [unilog.context :as lc]
-   [yang.scheduler :as scheduler])
+   [yang.scheduler :as scheduler]
+   [auto-ap.solr :as solr])
  (:import
   (java.util UUID)
   (org.apache.commons.codec.binary Base64)))
@@ -156,7 +157,15 @@
        result (audit-transact [[:upsert-entity updated-entity]] (:id context))]
    (when (:square_auth_token edit_client)
      (square/upsert-locations (-> result :tempids (get id) (or id) d-clients/get-by-id)))
-    (-> (-> result :tempids (get id) (or id) d-clients/get-by-id)
+    (let [updated-client (-> result :tempids (get id) (or id) d-clients/get-by-id)]
+      (solr/index-documents-raw solr/impl "clients"
+                                [{"id"    (:db/id updated-client)
+                                  "name"  (conj (or (:client/matches updated-client) [])
+                                                (:client/name updated-client))
+                                  "code"  (:client/code updated-client)
+                                  "exact" (map str/upper-case (conj (or (:client/matches updated-client) [])
+                                                                    (:client/name updated-client)))}])
+      (-> updated-client

        (update :client/bank-accounts 
                (fn [bas]
@@ -165,11 +174,11 @@
                (fn [lms]
                  (mapcat (fn [lm]
                            (map (fn [m]
-                                      {:location-match/match m
-                                       :location-match/location (:location-match/location lm)})
-                                    (:location-match/matches lm)))
+                                   {:location-match/match    m
+                                    :location-match/location (:location-match/location lm)})
+                                 (:location-match/matches lm)))
                          lms)))
-        ->graphql)))
+        ->graphql))))


 (defn refresh-all-current-balance []
@@ -629,8 +638,3 @@
                :input-objects input-objects
                :enums enums})
   (attach-tracing-resolvers resolvers)))
-
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(mount/defstate indexer
-  :start (scheduler/every (* 5 60 1000) (heartbeat d-clients/rebuild-search-index "rebuild-search-index"))
-  :stop (scheduler/stop indexer))
--- a/src/clj/auto_ap/graphql/ezcater.clj
+++ b/src/clj/auto_ap/graphql/ezcater.clj
@@ -1,35 +1,22 @@
 (ns auto-ap.graphql.ezcater
  (:require
   [auto-ap.datomic :refer [conn]]
-   [auto-ap.graphql.utils
-    :refer [assert-admin attach-tracing-resolvers cleanse-query]]
-   [auto-ap.search :as search]
-   [auto-ap.utils :refer [heartbeat]]
-   [datomic.api :as dc]
-   [mount.core :as mount]
-   [yang.scheduler :as scheduler]))
+   [auto-ap.graphql.utils :refer [assert-admin attach-tracing-resolvers]]
+   [clojure.string :as str]
+   [datomic.api :as dc]))

 (defn search [context args _]
  (assert-admin (:id context))
-  (let [search-query (cleanse-query (:query args))]
-    (for [[id name] (search/search {:q search-query}
-                                   "ezcater-caterer")]
-      {:name name
-       :id   (Long/parseLong id)})))
+  (for [[id name] (dc/q '[:find ?i ?n
+                          :in $ ?s
+                          :where [?i :ezcater-caterer/name ?n]
+                          [(clojure.string/upper-case ?n) ?n2]
+                          [(clojure.string/includes? ?n2 ?s)]]
+                        (dc/db conn)
+                        (str/upper-case (:query args)))]
+    {:name name
+     :id   id}))

-(defn rebuild-search-index []
-  (search/full-index-query  
-    (for [result (map first (dc/qseq {:query '[:find (pull ?a [:ezcater-caterer/search-terms :db/id :ezcater-caterer/name])
-                                               :in $
-                                               :where [?a :ezcater-caterer/search-terms ]]
-                                      :args  [(dc/db conn)]}))]
-      {:id            (:db/id result)
-       :text  (:ezcater-caterer/search-terms result)})
-    "ezcater-caterer"))
-
-(mount/defstate indexer
-  :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index"))
-  :stop (scheduler/stop indexer))

 (def objects
  {:ezcater_caterer  {:fields {:name   {:type 'String}
--- a/src/clj/auto_ap/graphql/vendors.clj
+++ b/src/clj/auto_ap/graphql/vendors.clj
@@ -1,9 +1,7 @@
 (ns auto-ap.graphql.vendors
  (:require
-   [auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]]
-   [iol-ion.tx :refer [random-tempid]]
+   [auto-ap.datomic :refer [audit-transact conn remove-nils]]
   [auto-ap.datomic.vendors :as d-vendors]
-   [auto-ap.solr :as solr]
   [auto-ap.graphql.utils
    :refer [->graphql
            <-graphql
@@ -13,19 +11,13 @@
            enum->keyword
            is-admin?
            result->page]]
-   [auto-ap.search :as search]
-   [auto-ap.utils :refer [heartbeat]]
+   [auto-ap.solr :as solr]
   [clojure.set :as set]
   [clojure.string :as str]
-   [manifold.executor :as ex]
-   [manifold.deferred :as de]
-   [clojure.tools.logging :as log]
   [datomic.api :as dc]
-   [yang.scheduler :as scheduler]
-   [mount.core :as mount]
-   [clj-time.core :as time]
-   [clj-time.coerce :as coerce]
-   [com.brunobonacci.mulog :as mu]))
+   [iol-ion.tx :refer [random-tempid]]
+   [manifold.deferred :as de]
+   [manifold.executor :as ex]))

 (defn can-user-edit-vendor? [vendor-id id]
  (if (is-admin? id)
--- a/src/clj/auto_ap/search.clj
+++ b/src/clj/auto_ap/search.clj
@@ -1,75 +0,0 @@
-(ns auto-ap.search
-  (:require [config.core :refer [env]])
-  (:import
-   (java.nio.file Paths)
-   (org.apache.lucene.analysis.standard StandardAnalyzer)
-   (org.apache.lucene.document Document Field$Store StoredField StringField TextField)
-   (org.apache.lucene.index DirectoryReader IndexWriter IndexWriterConfig Term)
-   (org.apache.lucene.queryparser.classic QueryParser)
-   (org.apache.lucene.search BooleanClause$Occur BooleanQuery$Builder IndexSearcher PhraseQuery$Builder Query TermQuery)
-   (org.apache.lucene.store FSDirectory)))
-
-(defn full-index-query
-  ([results index-name]
-   (full-index-query results index-name true))
-  ([results index-name delete?]
-   (let [directory           (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/search/" (:dd-env env) "/"  index-name))))
-         analyzer            (StandardAnalyzer.)
-         index-writer-config (IndexWriterConfig. analyzer)
-         index-writer        (IndexWriter. directory index-writer-config)]
-     (when delete?
-       (.deleteAll index-writer))
-     (try 
-       (doseq [{:keys [text id] :as x} results
-               :let                    [doc (doto 
-                                  (Document.)
-                                (.add (TextField. "name" text Field$Store/YES))
-                                (.add (StoredField. "id" (long id))))]]
-         (doseq [k (filter (complement #{:text :id}) (keys x))]
-           (.add doc (StringField. (name k) (str (get x k)) Field$Store/YES)))
-         (.addDocument index-writer doc))
-       (finally 
-         (.close index-writer))))))
-
-(defn make-query [n]
-  (let [
-        text-query   (when (:q n)
-                       (.parse (QueryParser. "name" (StandardAnalyzer.)) (:q n)))
-        text-query-exact (when (:q-exact n)
-                       (.build (doto (PhraseQuery$Builder. )
-                                 (.add (Term. "name" (:q-exact n)) 0))))
-        full-query (BooleanQuery$Builder.)
-        ]
-    (when text-query 
-      (.add full-query text-query BooleanClause$Occur/MUST))
-    (when text-query-exact
-      (.add full-query text-query-exact BooleanClause$Occur/MUST))
-    (doseq [[k v] (dissoc n :q :q-exact)]
-      (if (instance? Query v)
-        (.add full-query v BooleanClause$Occur/MUST)
-        (.add full-query (TermQuery. (Term. (name k) (str v))) BooleanClause$Occur/MUST)))
-    (.build full-query)))
-
-(defn search
-  ([n index-name]
-   (search n index-name []))
-  ([n index-name other-keys]
-   (let [directory      (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/search/" (:dd-env env) "/"  index-name))))
-         index-reader   (DirectoryReader/open directory)
-         index-searcher (IndexSearcher. index-reader)]
-     (for [x (seq (.scoreDocs (.search index-searcher (make-query n) 10)))]
-       (into 
-         [(Long/parseLong (.get (.doc index-searcher (.-doc x)) "id"))
-          (.get (.doc index-searcher (.-doc x)) "name")
-          (.-score x)]
-         (map (fn [o]
-                (.get (.doc index-searcher (.-doc x)) o))
-              other-keys)))))
-  )
-
-(defn search-ids [n index-name]
-  (let [directory      (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/search/" (:dd-env env) "/"  index-name))))
-        index-reader   (DirectoryReader/open directory)
-        index-searcher (IndexSearcher. index-reader)]
-    (for [x (seq (.scoreDocs (.search index-searcher (make-query n) 100)))]
-      (Long/parseLong (.get (.doc index-searcher (.-doc x)) "id")))))
--- a/src/clj/auto_ap/solr.clj
+++ b/src/clj/auto_ap/solr.clj
@@ -10,6 +10,12 @@
   [config.core :refer [env]]
   [datomic.api :as dc]))

+(defn escape [s]
+  (str/escape s
+              (into {}
+                    (for [c "\\+-&&||!(){}[]^\"~*?:/"]
+                      [c (str "\\" c)]))))
+
 (def solr-uri (:solr-uri env))

 (defn fmt-amount [a]