From 4c90e72a1f7a75eba2231f484f3044c92b969809 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 21 Mar 2023 11:35:35 -0700 Subject: [PATCH] (cloud) Should make searching function again for --- src/clj/auto_ap/graphql/accounts.clj | 13 ++++--- src/clj/auto_ap/graphql/clients.clj | 20 +++++++--- src/clj/auto_ap/graphql/vendors.clj | 56 ++++++++++++++-------------- src/clj/auto_ap/search.clj | 5 +-- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/clj/auto_ap/graphql/accounts.clj b/src/clj/auto_ap/graphql/accounts.clj index e4c2249f..89b42766 100644 --- a/src/clj/auto_ap/graphql/accounts.clj +++ b/src/clj/auto_ap/graphql/accounts.clj @@ -1,7 +1,6 @@ (ns auto-ap.graphql.accounts (:require [auto-ap.datomic :refer [audit-transact conn]] - [iol-ion.tx :refer [upsert-entity]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.graphql.utils :refer [->graphql @@ -13,11 +12,10 @@ is-admin? result->page]] [auto-ap.search :as search] + [auto-ap.utils :refer [heartbeat]] [datomic.client.api :as dc] - [clojure.string :as str]) - (:import - (org.apache.lucene.search TermInSetQuery) - (org.apache.lucene.util BytesRef))) + [mount.core :as mount] + [yang.scheduler :as scheduler])) (defn get-graphql [context args _] (assert-admin (:id context)) @@ -195,3 +193,8 @@ #_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k]) + +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(mount/defstate indexer + :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index")) + :stop (scheduler/stop indexer)) diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index 533ab718..fc483bc2 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -1,13 +1,17 @@ (ns auto-ap.graphql.clients (:require [amazonica.aws.s3 :as s3] - [iol-ion.tx :refer [upsert-entity random-tempid]] [auto-ap.datomic :refer [audit-transact conn]] [auto-ap.datomic.clients :as d-clients] - [auto-ap.square.core :as square] [auto-ap.graphql.utils - :refer [->graphql assert-admin can-see-client? is-admin? attach-tracing-resolvers]] + :refer [->graphql + assert-admin + attach-tracing-resolvers + can-see-client? + is-admin?]] [auto-ap.routes.queries :as q] + [auto-ap.square.core :as square] + [auto-ap.utils :refer [heartbeat]] [clj-time.coerce :as coerce] [clojure.java.io :as io] [clojure.set :as set] @@ -15,8 +19,9 @@ [clojure.tools.logging :as log] [datomic.client.api :as dc] [unilog.context :as lc] - [com.brunobonacci.mulog :as mu] - [datomic.client.api :as dc]) + [mount.core :as mount] + [yang.scheduler :as scheduler] + [iol-ion.tx :refer [random-tempid]]) (:import (java.util UUID) (org.apache.commons.codec.binary Base64))) @@ -598,3 +603,8 @@ :input-objects input-objects :enums enums}) (attach-tracing-resolvers resolvers))) + +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(mount/defstate indexer + :start (scheduler/every (* 5 60 1000) (heartbeat d-clients/rebuild-search-index "rebuild-search-index")) + :stop (scheduler/stop indexer)) diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj index 3d93b0bd..43aaff1c 100644 --- a/src/clj/auto_ap/graphql/vendors.clj +++ b/src/clj/auto_ap/graphql/vendors.clj @@ -11,11 +11,14 @@ enum->keyword is-admin? result->page]] + [auto-ap.search :as search] + [auto-ap.utils :refer [heartbeat]] [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] [datomic.client.api :as dc] - [auto-ap.search :as search])) + [yang.scheduler :as scheduler] + [mount.core :as mount])) (defn can-user-edit-vendor? [vendor-id id] (if (is-admin? id) @@ -183,30 +186,27 @@ matches)) (defn search [context args _] - (comment (let [search-query (cleanse-query (:query args))] - (for [[id name] (search/search (cond-> {:q search-query} - (not (is-admin? (:id context))) (assoc :hidden false)) - "vendors")] - {:name name - :id (Long/parseLong id)}) - )) - (if-let [search-query (cleanse-query (:query args))] - (let [data (if (is-admin? (:id context)) - (dc/q '[:find ?n ?i ?s - :in $ ?q - :where [(fulltext $ :vendor/search-terms ?q) [[?i ?n _ ?s]]]] - (dc/db conn) - search-query) - (dc/q '[:find ?n ?i ?s - :in $ ?q - :where [(fulltext $ :vendor/search-terms ?q) [[?i ?n _ ?s]]] - (not [?i :vendor/hidden true])] - (dc/db conn) - search-query))] - (->> data - (sort-by (comp - last)) - (partial-match-first (:query args)) - (map (fn [[n i]] - {:name n - :id i})))) - [])) + (let [search-query (cleanse-query (:query args))] + (for [[id name] (search/search (cond-> {:q search-query} + (not (is-admin? (:id context))) (assoc :hidden false)) + "vendor")] + {:name name + :id id}) + )) + +(defn rebuild-search-index [] + (search/full-index-query + (for [result (map first (dc/qseq '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) + :in $ + :where [?v :vendor/search-terms ]] + (dc/db conn)))] + {:id (:db/id result) + :text (or (first (:vendor/search-terms result)) + (:vendor/name result)) + :hidden (boolean (:vendor/hidden result))}) + "vendor")) + +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(mount/defstate indexer + :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index")) + :stop (scheduler/stop indexer)) diff --git a/src/clj/auto_ap/search.clj b/src/clj/auto_ap/search.clj index 50f69a70..01c02392 100644 --- a/src/clj/auto_ap/search.clj +++ b/src/clj/auto_ap/search.clj @@ -10,7 +10,7 @@ (org.apache.lucene.store FSDirectory))) (defn full-index-query [results index-name] - (let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/search" (:dd-env env) "/" index-name)))) + (let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/search/" (:dd-env env) "/" index-name)))) analyzer (StandardAnalyzer.) index-writer-config (IndexWriterConfig. analyzer) index-writer (IndexWriter. directory index-writer-config)] @@ -22,10 +22,7 @@ (.add (TextField. "name" text Field$Store/YES)) (.add (StoredField. "id" (long id))))]] (doseq [k (filter (complement #{:text :id}) (keys x))] - (println "K" (name k) (get x k)) (.add doc (StringField. (name k) (str (get x k)) Field$Store/YES))) - (println "adding" text) - (flush) (.addDocument index-writer doc)) (finally (.close index-writer)))))