diff --git a/src/clj/auto_ap/graphql/accounts.clj b/src/clj/auto_ap/graphql/accounts.clj index 8f119b5e..4d16ad56 100644 --- a/src/clj/auto_ap/graphql/accounts.clj +++ b/src/clj/auto_ap/graphql/accounts.clj @@ -7,16 +7,17 @@ <-graphql assert-admin assert-can-see-client + can-see-client? cleanse-query enum->keyword is-admin? + limited-clients result->page]] [auto-ap.search :as search] - [auto-ap.utils :refer [heartbeat]] + [auto-ap.solr :as solr] [datomic.api :as dc] [iol-ion.tx :refer [random-tempid]] - [mount.core :as mount] - [yang.scheduler :as scheduler])) + [com.brunobonacci.mulog :as mu])) (defn get-graphql [context args _] (assert-admin (:id context)) @@ -68,9 +69,29 @@ :account-client-override/search-terms (:name client-override)}) client-overrides)} id (dissoc :account/numeric-code :account/code))]] - (:id context))] + (:id context)) + updated-account (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))] + (mu/log ::account-updated :account updated-account) + (solr/index-documents-raw solr/impl + "accounts" + (into [{"id" (:db/id updated-account) + "account_id" (:db/id updated-account) + "name" (:account/name updated-account) + "numeric_code" (:account/numeric-code updated-account) + "location" (:account/location updated-account) + "applicability" (clojure.core/name (:account/applicability updated-account))}] + (for [o (:account/client-overrides updated-account)] + {"id" (:db/id o) + "account_id" (:db/id updated-account) + "name" (:account-client-override/name o) + "numeric_code" (:account/numeric-code updated-account) + "location" (:account/location updated-account) + "applicability" (clojure.core/name (:account/applicability updated-account)) + "client_id" (:db/id (:account-client-override/client o)) + "account_client_override_id" (:db/id o)})) + ) (->graphql - (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"]))))))) + updated-account)))) (def search-pattern [:db/id :account/numeric-code @@ -78,12 +99,25 @@ {:account/vendor-allowance [:db/ident] :account/default-allowance [:db/ident] :account/invoice-allowance [:db/ident]}]) +(defn search- [id query client] + (let [client-part (if (some->> client (can-see-client? id)) + (format "((applicability:(global OR optional) AND -client_id:*) OR (account_client_override_id:* AND client_id:%s))" client) + "(applicability:(global OR optional) AND -client_id:*)" + + ) + query (format "_text_:(%s) AND %s" (cleanse-query query) client-part)] + (mu/log ::searching :search-query query) + (for [{:keys [account_id name] :as g} (solr/query solr/impl "accounts" + {"query" query + "fields" "id, name, client_id, numeric_code, applicability, account_id"})] + + {:account_id (first account_id) + :name (first name)}))) + (defn search [context {query :query client :client_id allowance :allowance vendor-id :vendor_id} _] (when client (assert-can-see-client (:id context) client)) - (let [query (cleanse-query query) - _ (println query) - num (some-> (re-find #"([0-9]+)" query) + (let [num (some-> (re-find #"([0-9]+)" query) second (not-empty ) Integer/parseInt) @@ -130,41 +164,34 @@ allowance search-pattern) (sequence xform)) - (->> (dc/q '[:find ?n (pull ?i pattern) ?s - :in $ [[?i ?n ?s]] pattern - :where (or [?i :account/applicability :account-applicability/global] - [?i :account/applicability :account-applicability/optional])] - (dc/db conn) - (search/search {:q query} "account") - search-pattern) - (concat (when client - (dc/q '[:find ?n (pull ?i pattern) ?s - :in $ [[?i ?n ?s]] pattern] - (dc/db conn) - (search/search {:q query :client (str client)} "account-client-override") - search-pattern))) - (sort-by (comp - last)) - (sequence xform))) + (->> (search- (:id context) query client) + (sequence + (comp (map (fn [i] [(:name i) (dc/pull (dc/db conn) search-pattern (:account_id i))])) + xform)))) []))) (defn rebuild-search-index [] - (search/full-index-query - (for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}]) + (solr/index-documents-raw + solr/impl + "accounts" + (for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id {:account/applicability [:db/ident]}]}]) :in $ :where [?aco :account-client-override/client ] [?aco :account-client-override/search-terms ] [_ :account/client-overrides ?aco]] :args [(dc/db conn)]})) - :when (:account/numeric-code (:account/_client-overrides result))] - {:id (:db/id (:account/_client-overrides result)) - :account-client-override-id (:db/id result) - :text (:account-client-override/search-terms result) - :client (str (:db/id (:account-client-override/client result))) - :numeric-code (:account/numeric-code (:account/_client-overrides result)) - :location (:account/location (:account/_client-overrides result))}) - "account-client-override") - - (search/full-index-query + :when (:account/numeric-code (:account/_client-overrides result))] + {"id" (:db/id result) + "account_id" (:db/id (:account/_client-overrides result)) + "account_client_override_id" (str (:db/id result)) + "name" (:account-client-override/search-terms result) + "client_id" (str (:db/id (:account-client-override/client result))) + "numeric_code" (:account/numeric-code (:account/_client-overrides result)) + "location" (:account/location (:account/_client-overrides result)) + "applicability" (name (:db/ident (:account/applicability (:account/_client-overrides result))))})) + (solr/index-documents-raw + solr/impl + "accounts" (for [result (map first (dc/qseq {:query '[:find (pull ?a [:account/numeric-code :account/search-terms {:account/applicability [:db/ident]} @@ -173,14 +200,14 @@ :in $ :where [?a :account/search-terms ]] :args [(dc/db conn)]})) - :when (:account/search-terms result) + :when (:account/search-terms result) ] - {:id (:db/id result) - :text (:account/search-terms result) - :numeric-code (:account/numeric-code result) - :location (:account/location result) - :applicability (name (:db/ident (:account/applicability result)))}) - "account")) + {"id" (:db/id result) + "account_id" (:db/id result) + "name" (:account/search-terms result) + "numeric_code" (:account/numeric-code result) + "location" (:account/location result) + "applicability" (name (:db/ident (:account/applicability result)))}))) #_(dc/transact conn [{:db/ident :account-client-override/k2 @@ -195,8 +222,3 @@ #_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k]) - -#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(mount/defstate indexer - :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index")) - :stop (scheduler/stop indexer)) diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj index a5910677..b47e28de 100644 --- a/src/clj/auto_ap/graphql/vendors.clj +++ b/src/clj/auto_ap/graphql/vendors.clj @@ -3,6 +3,7 @@ [auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]] [iol-ion.tx :refer [random-tempid]] [auto-ap.datomic.vendors :as d-vendors] + [auto-ap.solr :as solr] [auto-ap.graphql.utils :refer [->graphql <-graphql @@ -138,10 +139,18 @@ :vendor/automatically-paid-when-due (:automatically_paid_when_due in)))] - transaction-result (audit-transact [transaction] (:id context))] + transaction-result (audit-transact [transaction] (:id context)) + new-vendor (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor")) + id))] - (-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor")) - id)) + (auto-ap.solr/index-documents-raw + auto-ap.solr/impl + "vendors" + [{"id" (:db/id new-vendor) + "name" (:vendor/name new-vendor) + "hidden" (boolean (:vendor/hidden new-vendor))}]) + + (-> new-vendor (->graphql)))) (defn merge-vendors [context {:keys [from to]} _] @@ -183,64 +192,28 @@ matches)) (defn search [context args _] - (let [search-query (cleanse-query (:query args))] - (for [[id name] (search/search (cond-> {:q search-query} - (not (is-admin? (:id context))) (assoc :hidden false)) - "vendor")] - {:name name - :id id}))) + (let [search-query (str "name:(" (cleanse-query (:query args)) ")")] + + + (for [{:keys [id name]} (solr/query solr/impl "vendors" {"query" (cond-> search-query + (not (is-admin? (:id context))) (str " hidden:false")) + "fields" "id, name"})] + {:id (Long/parseLong id) + :name (first name)}))) (def single-thread (ex/fixed-thread-executor 1)) (defn rebuild-search-index [] (de/future-with single-thread - (search/full-index-query + (auto-ap.solr/index-documents-raw + auto-ap.solr/impl + "vendors" (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) :in $ :where [?v :vendor/search-terms ]] :args [(dc/db conn)]})] - {:id (:db/id result) - :text (or (first (:vendor/search-terms result)) - (:vendor/name result)) - :hidden (boolean (:vendor/hidden result))}) - "vendor"))) - -(def last-run-basis (atom nil)) - -(defn add-incremental-changes [] - (de/future-with - single-thread - (if-let [last-run-basis-value @last-run-basis] - (let [db (dc/db conn) - recent (dc/since db last-run-basis-value) - _ (mu/log ::indexing - :last-run last-run-basis-value - :starting-from (:basisT db)) - results (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) - :in $ $$ - :where [$ ?v :vendor/name ] - [$$ ?v]] - :args [db recent]})] - {:id (:db/id result) - :text (or (first (:vendor/search-terms result)) - (:vendor/name result)) - :hidden (boolean (:vendor/hidden result))})] - (when (seq results) - (mu/log ::adding-to-index - :sample (first results) - :count (count results)) - (search/full-index-query results "vendor" false)) - (reset! last-run-basis (:basisT db)) - (count results)) - (reset! last-run-basis (:basisT (dc/db conn)))))) - -#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(mount/defstate indexer - :start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index")) - :stop (scheduler/stop indexer)) - -#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(mount/defstate incremental-indexer - :start (scheduler/every (* 5 1000) (heartbeat add-incremental-changes "incremental-indexing")) - :stop (scheduler/stop incremental-indexer)) + {"id" (:db/id result) + "name" (or (first (:vendor/search-terms result)) + (:vendor/name result)) + "hidden" (boolean (:vendor/hidden result))})))) diff --git a/src/clj/auto_ap/routes/ezcater_xls.clj b/src/clj/auto_ap/routes/ezcater_xls.clj index dc45f3cf..ebf995d1 100644 --- a/src/clj/auto_ap/routes/ezcater_xls.clj +++ b/src/clj/auto_ap/routes/ezcater_xls.clj @@ -89,7 +89,8 @@ :tip (fmt-amount tip)}] :total (fmt-amount (+ food-total tax - tip)) + tip + (or adjustments 0.0))) :discount (fmt-amount (or adjustments 0.0)) :service-charge (fmt-amount (+ fee commission)) :tax (fmt-amount tax) diff --git a/src/clj/auto_ap/solr.clj b/src/clj/auto_ap/solr.clj index 697dfc13..1d866036 100644 --- a/src/clj/auto_ap/solr.clj +++ b/src/clj/auto_ap/solr.clj @@ -1,6 +1,7 @@ (ns auto-ap.solr (:require [auto-ap.datomic :refer [conn pull-id]] + [cemerick.url :as url] [auto-ap.time :as atime] [clj-http.client :as client] [clj-time.coerce :as c] @@ -118,26 +119,38 @@ (defprotocol SolrClient - (index-documents [this xs]) - (query [this q]) - (delete [this])) + (index-documents-raw [this index xs]) + (index-documents [this index xs]) + (query [this index q]) + (delete [this index])) (defrecord RealSolrClient [solr-uri] SolrClient - (index-documents [this xs] + (index-documents-raw [this index xs] (client/post - (str solr-uri "/solr/invoices/update?commitWithin=15000") + (str (assoc (url/url solr-uri "solr" index "update") + :query {"commitWithin" 60000})) + + {:headers {"Content-Type" "application/json"} + :socket-timeout 30000 + :connection-timeout 30000 + :method "POST" + :body (json/write-str xs)})) + + (index-documents [this index xs] + (client/post + (str (assoc (url/url solr-uri "solr" index "update") + :query {"commitWithin" 60000})) {:headers {"Content-Type" "application/json"} :socket-timeout 30000 :connection-timeout 30000 :method "POST" :body (json/write-str (filter identity (map datomic->solr xs)))})) - (query [this q] - (-> (client/post (str solr-uri "/solr/invoices/query") - {:body (json/write-str {"query" q - "fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"}) + (query [this index q] + (-> (client/post (str (url/url solr-uri "solr" index "query")) + {:body (json/write-str q ) :socket-timeout 30000 :connection-timeout 30000 :headers {"Content-Type" "application/json"} @@ -146,21 +159,25 @@ :body :response :docs)) - (delete [this] + (delete [this index] (client/post - (str solr-uri "/solr/invoices/update?commitWithin=1000") + (str (assoc (url/url solr-uri "solr" index "update") + :query {"commitWithin" 15000})) {:headers {"Content-Type" "application/json"} :method "POST" :body (json/write-str {"delete" {"query" "*:*"}})}))) (defrecord MockSolrClient [] SolrClient - (index-documents [this xs] + (index-documents [this index xs] nil) - (query [this q] + (index-documents-raw [this index xs] nil) - (delete [this] + + (query [this index q] + nil) + (delete [this index] nil)) (def impl (if (= :solr (:solr-impl env)) @@ -172,9 +189,11 @@ (defn touch-with-ledger [i] - (index-documents impl [i [:journal-entry/original-entity i]])) + (index-documents impl "invoices" [i [:journal-entry/original-entity i]])) -(defn touch [i] - (index-documents impl [i])) +(defn touch + ([i] (touch i "invoices")) + ([i index] + (index-documents impl index [i]))) diff --git a/src/clj/auto_ap/ssr/search.clj b/src/clj/auto_ap/ssr/search.clj index 40cc40d3..d7340507 100644 --- a/src/clj/auto_ap/ssr/search.clj +++ b/src/clj/auto_ap/ssr/search.clj @@ -53,7 +53,8 @@ (into [] (filter (fn [d] (can-see-client? id (first (:client_id d))))) - (solr/query solr/impl (q->solr-q q)))) + (solr/query solr/impl "invoices" {"query" (q->solr-q q) + "fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"}))) (defn search-results* [q id] diff --git a/src/clj/user.clj b/src/clj/user.clj index bb06ea25..12b2c5e4 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -346,7 +346,7 @@ (mu/start-publisher! {:type :dev}) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn }))) -(defn start-search [] +#_(defn start-search [] (mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer}))) (defn restart-db [] @@ -573,7 +573,7 @@ (partition-all 1000))] (print ".") (flush) - (solr/index-documents solr/impl batch)) + (solr/index-documents solr/impl "invoices" batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ @@ -584,7 +584,7 @@ (partition-all 1000))] (print ".") (flush) - (solr/index-documents solr/impl batch)) + (solr/index-documents solr/impl "invoices" batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ @@ -595,7 +595,7 @@ (partition-all 1000))] (print ".") (flush) - (solr/index-documents solr/impl batch)) + (solr/index-documents solr/impl "invoices" batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ :where [?i :journal-entry/date]] @@ -604,7 +604,7 @@ (partition-all 1000))] (print ".") (flush) - (solr/index-documents solr/impl batch))) + (solr/index-documents solr/impl "invoices" batch))) (defn setup-sales-orders [] (doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]})