notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

(cloud) adjustments for ezcater tenders, search fixes.

Browse Source
This commit is contained in:
Bryce
2023-05-18 16:56:58 -07:00
parent 3ce9cceee0
commit 0e6a5c6749
6 changed files with 141 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
src/clj/auto_ap/graphql/accounts.clj
View File

@@ -7,16 +7,17 @@
<-graphql <-graphql
assert-admin assert-admin
assert-can-see-client assert-can-see-client
can-see-client?
cleanse-query cleanse-query
enum->keyword enum->keyword
is-admin? is-admin?
limited-clients
result->page]] result->page]]
[auto-ap.search :as search] [auto-ap.search :as search]
[auto-ap.utils :refer [heartbeat]] [auto-ap.solr :as solr]
[datomic.api :as dc] [datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]] [iol-ion.tx :refer [random-tempid]]
[mount.core :as mount] [com.brunobonacci.mulog :as mu]))
[yang.scheduler :as scheduler]))
(defn get-graphql [context args _] (defn get-graphql [context args _]
(assert-admin (:id context)) (assert-admin (:id context))
@@ -68,9 +69,29 @@
:account-client-override/search-terms (:name client-override)}) :account-client-override/search-terms (:name client-override)})
client-overrides)} client-overrides)}
id (dissoc :account/numeric-code :account/code))]] id (dissoc :account/numeric-code :account/code))]]
(:id context))] (:id context))
updated-account (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))]
(mu/log ::account-updated :account updated-account)
(solr/index-documents-raw solr/impl
"accounts"
(into [{"id" (:db/id updated-account)
"account_id" (:db/id updated-account)
"name" (:account/name updated-account)
"numeric_code" (:account/numeric-code updated-account)
"location" (:account/location updated-account)
"applicability" (clojure.core/name (:account/applicability updated-account))}]
(for [o (:account/client-overrides updated-account)]
{"id" (:db/id o)
"account_id" (:db/id updated-account)
"name" (:account-client-override/name o)
"numeric_code" (:account/numeric-code updated-account)
"location" (:account/location updated-account)
"applicability" (clojure.core/name (:account/applicability updated-account))
"client_id" (:db/id (:account-client-override/client o))
"account_client_override_id" (:db/id o)}))
)
(->graphql (->graphql
(d-accounts/get-by-id (or id (get-in result [:tempids "new-account"]))))))) updated-account))))
(def search-pattern [:db/id (def search-pattern [:db/id
:account/numeric-code :account/numeric-code
@@ -78,12 +99,25 @@
{:account/vendor-allowance [:db/ident] {:account/vendor-allowance [:db/ident]
:account/default-allowance [:db/ident] :account/default-allowance [:db/ident]
:account/invoice-allowance [:db/ident]}]) :account/invoice-allowance [:db/ident]}])
(defn search- [id query client]
(let [client-part (if (some->> client (can-see-client? id))
(format "((applicability:(global OR optional) AND -client_id:*) OR (account_client_override_id:* AND client_id:%s))" client)
"(applicability:(global OR optional) AND -client_id:*)"
)
query (format "_text_:(%s) AND %s" (cleanse-query query) client-part)]
(mu/log ::searching :search-query query)
(for [{:keys [account_id name] :as g} (solr/query solr/impl "accounts"
{"query" query
"fields" "id, name, client_id, numeric_code, applicability, account_id"})]
{:account_id (first account_id)
:name (first name)})))
(defn search [context {query :query client :client_id allowance :allowance vendor-id :vendor_id} _] (defn search [context {query :query client :client_id allowance :allowance vendor-id :vendor_id} _]
(when client (when client
(assert-can-see-client (:id context) client)) (assert-can-see-client (:id context) client))
(let [query (cleanse-query query) (let [num (some-> (re-find #"([0-9]+)" query)
_ (println query)
num (some-> (re-find #"([0-9]+)" query)
second second
(not-empty ) (not-empty )
Integer/parseInt) Integer/parseInt)
@@ -130,41 +164,34 @@
allowance allowance
search-pattern) search-pattern)
(sequence xform)) (sequence xform))
(->> (dc/q '[:find ?n (pull ?i pattern) ?s (->> (search- (:id context) query client)
:in $ [[?i ?n ?s]] pattern (sequence
:where (or [?i :account/applicability :account-applicability/global] (comp (map (fn [i] [(:name i) (dc/pull (dc/db conn) search-pattern (:account_id i))]))
[?i :account/applicability :account-applicability/optional])] xform))))
(dc/db conn)
(search/search {:q query} "account")
search-pattern)
(concat (when client
(dc/q '[:find ?n (pull ?i pattern) ?s
:in $ [[?i ?n ?s]] pattern]
(dc/db conn)
(search/search {:q query :client (str client)} "account-client-override")
search-pattern)))
(sort-by (comp - last))
(sequence xform)))
[]))) [])))
(defn rebuild-search-index [] (defn rebuild-search-index []
(search/full-index-query (solr/index-documents-raw
(for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}]) solr/impl
"accounts"
(for [result (map first (dc/qseq {:query '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id {:account/applicability [:db/ident]}]}])
:in $ :in $
:where [?aco :account-client-override/client ] :where [?aco :account-client-override/client ]
[?aco :account-client-override/search-terms ] [?aco :account-client-override/search-terms ]
[_ :account/client-overrides ?aco]] [_ :account/client-overrides ?aco]]
:args [(dc/db conn)]})) :args [(dc/db conn)]}))
:when (:account/numeric-code (:account/_client-overrides result))] :when (:account/numeric-code (:account/_client-overrides result))]
{:id (:db/id (:account/_client-overrides result)) {"id" (:db/id result)
:account-client-override-id (:db/id result) "account_id" (:db/id (:account/_client-overrides result))
:text (:account-client-override/search-terms result) "account_client_override_id" (str (:db/id result))
:client (str (:db/id (:account-client-override/client result))) "name" (:account-client-override/search-terms result)
:numeric-code (:account/numeric-code (:account/_client-overrides result)) "client_id" (str (:db/id (:account-client-override/client result)))
:location (:account/location (:account/_client-overrides result))}) "numeric_code" (:account/numeric-code (:account/_client-overrides result))
"account-client-override") "location" (:account/location (:account/_client-overrides result))
"applicability" (name (:db/ident (:account/applicability (:account/_client-overrides result))))}))
(search/full-index-query (solr/index-documents-raw
solr/impl
"accounts"
(for [result (map first (dc/qseq {:query '[:find (pull ?a [:account/numeric-code (for [result (map first (dc/qseq {:query '[:find (pull ?a [:account/numeric-code
:account/search-terms :account/search-terms
{:account/applicability [:db/ident]} {:account/applicability [:db/ident]}
@@ -173,14 +200,14 @@
:in $ :in $
:where [?a :account/search-terms ]] :where [?a :account/search-terms ]]
:args [(dc/db conn)]})) :args [(dc/db conn)]}))
:when (:account/search-terms result) :when (:account/search-terms result)
] ]
{:id (:db/id result) {"id" (:db/id result)
:text (:account/search-terms result) "account_id" (:db/id result)
:numeric-code (:account/numeric-code result) "name" (:account/search-terms result)
:location (:account/location result) "numeric_code" (:account/numeric-code result)
:applicability (name (:db/ident (:account/applicability result)))}) "location" (:account/location result)
"account")) "applicability" (name (:db/ident (:account/applicability result)))})))
#_(dc/transact conn #_(dc/transact conn
[{:db/ident :account-client-override/k2 [{:db/ident :account-client-override/k2
@@ -195,8 +222,3 @@
#_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k]) #_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k])
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(mount/defstate indexer
:start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index"))
:stop (scheduler/stop indexer))

81
src/clj/auto_ap/graphql/vendors.clj
View File

@@ -3,6 +3,7 @@
[auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]] [auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]]
[iol-ion.tx :refer [random-tempid]] [iol-ion.tx :refer [random-tempid]]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
[auto-ap.solr :as solr]
[auto-ap.graphql.utils [auto-ap.graphql.utils
:refer [->graphql :refer [->graphql
<-graphql <-graphql
@@ -138,10 +139,18 @@
:vendor/automatically-paid-when-due (:automatically_paid_when_due in)))] :vendor/automatically-paid-when-due (:automatically_paid_when_due in)))]
transaction-result (audit-transact [transaction] (:id context))] transaction-result (audit-transact [transaction] (:id context))
new-vendor (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
id))]
(-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor")) (auto-ap.solr/index-documents-raw
id)) auto-ap.solr/impl
"vendors"
[{"id" (:db/id new-vendor)
"name" (:vendor/name new-vendor)
"hidden" (boolean (:vendor/hidden new-vendor))}])
(-> new-vendor
(->graphql)))) (->graphql))))
(defn merge-vendors [context {:keys [from to]} _] (defn merge-vendors [context {:keys [from to]} _]
@@ -183,64 +192,28 @@
matches)) matches))
(defn search [context args _] (defn search [context args _]
(let [search-query (cleanse-query (:query args))] (let [search-query (str "name:(" (cleanse-query (:query args)) ")")]
(for [[id name] (search/search (cond-> {:q search-query}
(not (is-admin? (:id context))) (assoc :hidden false))
"vendor")] (for [{:keys [id name]} (solr/query solr/impl "vendors" {"query" (cond-> search-query
{:name name (not (is-admin? (:id context))) (str " hidden:false"))
:id id}))) "fields" "id, name"})]
{:id (Long/parseLong id)
:name (first name)})))
(def single-thread (ex/fixed-thread-executor 1)) (def single-thread (ex/fixed-thread-executor 1))
(defn rebuild-search-index [] (defn rebuild-search-index []
(de/future-with (de/future-with
single-thread single-thread
(search/full-index-query (auto-ap.solr/index-documents-raw
auto-ap.solr/impl
"vendors"
(for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden]) (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
:in $ :in $
:where [?v :vendor/search-terms ]] :where [?v :vendor/search-terms ]]
:args [(dc/db conn)]})] :args [(dc/db conn)]})]
{:id (:db/id result) {"id" (:db/id result)
:text (or (first (:vendor/search-terms result)) "name" (or (first (:vendor/search-terms result))
(:vendor/name result)) (:vendor/name result))
:hidden (boolean (:vendor/hidden result))}) "hidden" (boolean (:vendor/hidden result))}))))
"vendor")))
(def last-run-basis (atom nil))
(defn add-incremental-changes []
(de/future-with
single-thread
(if-let [last-run-basis-value @last-run-basis]
(let [db (dc/db conn)
recent (dc/since db last-run-basis-value)
_ (mu/log ::indexing
:last-run last-run-basis-value
:starting-from (:basisT db))
results (for [[result] (dc/qseq {:query '[:find (pull ?v [:vendor/search-terms :db/id :vendor/name :vendor/hidden])
:in $ $$
:where [$ ?v :vendor/name ]
[$$ ?v]]
:args [db recent]})]
{:id (:db/id result)
:text (or (first (:vendor/search-terms result))
(:vendor/name result))
:hidden (boolean (:vendor/hidden result))})]
(when (seq results)
(mu/log ::adding-to-index
:sample (first results)
:count (count results))
(search/full-index-query results "vendor" false))
(reset! last-run-basis (:basisT db))
(count results))
(reset! last-run-basis (:basisT (dc/db conn))))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(mount/defstate indexer
:start (scheduler/every (* 5 60 1000) (heartbeat rebuild-search-index "rebuild-search-index"))
:stop (scheduler/stop indexer))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(mount/defstate incremental-indexer
:start (scheduler/every (* 5 1000) (heartbeat add-incremental-changes "incremental-indexing"))
:stop (scheduler/stop incremental-indexer))

3
src/clj/auto_ap/routes/ezcater_xls.clj
View File

@@ -89,7 +89,8 @@
:tip (fmt-amount tip)}] :tip (fmt-amount tip)}]
:total (fmt-amount (+ food-total :total (fmt-amount (+ food-total
tax tax
tip)) tip
(or adjustments 0.0)))
:discount (fmt-amount (or adjustments 0.0)) :discount (fmt-amount (or adjustments 0.0))
:service-charge (fmt-amount (+ fee commission)) :service-charge (fmt-amount (+ fee commission))
:tax (fmt-amount tax) :tax (fmt-amount tax)

53
src/clj/auto_ap/solr.clj
View File

@@ -1,6 +1,7 @@
(ns auto-ap.solr (ns auto-ap.solr
(:require (:require
[auto-ap.datomic :refer [conn pull-id]] [auto-ap.datomic :refer [conn pull-id]]
[cemerick.url :as url]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[clj-http.client :as client] [clj-http.client :as client]
[clj-time.coerce :as c] [clj-time.coerce :as c]
@@ -118,26 +119,38 @@
(defprotocol SolrClient (defprotocol SolrClient
(index-documents [this xs]) (index-documents-raw [this index xs])
(query [this q]) (index-documents [this index xs])
(delete [this])) (query [this index q])
(delete [this index]))
(defrecord RealSolrClient [solr-uri] (defrecord RealSolrClient [solr-uri]
SolrClient SolrClient
(index-documents [this xs] (index-documents-raw [this index xs]
(client/post (client/post
(str solr-uri "/solr/invoices/update?commitWithin=15000") (str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 60000}))
{:headers {"Content-Type" "application/json"}
:socket-timeout 30000
:connection-timeout 30000
:method "POST"
:body (json/write-str xs)}))
(index-documents [this index xs]
(client/post
(str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 60000}))
{:headers {"Content-Type" "application/json"} {:headers {"Content-Type" "application/json"}
:socket-timeout 30000 :socket-timeout 30000
:connection-timeout 30000 :connection-timeout 30000
:method "POST" :method "POST"
:body (json/write-str (filter identity (map datomic->solr xs)))})) :body (json/write-str (filter identity (map datomic->solr xs)))}))
(query [this q] (query [this index q]
(-> (client/post (str solr-uri "/solr/invoices/query") (-> (client/post (str (url/url solr-uri "solr" index "query"))
{:body (json/write-str {"query" q {:body (json/write-str q )
"fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"})
:socket-timeout 30000 :socket-timeout 30000
:connection-timeout 30000 :connection-timeout 30000
:headers {"Content-Type" "application/json"} :headers {"Content-Type" "application/json"}
@@ -146,21 +159,25 @@
:body :body
:response :response
:docs)) :docs))
(delete [this] (delete [this index]
(client/post (client/post
(str solr-uri "/solr/invoices/update?commitWithin=1000") (str (assoc (url/url solr-uri "solr" index "update")
:query {"commitWithin" 15000}))
{:headers {"Content-Type" "application/json"} {:headers {"Content-Type" "application/json"}
:method "POST" :method "POST"
:body (json/write-str {"delete" {"query" "*:*"}})}))) :body (json/write-str {"delete" {"query" "*:*"}})})))
(defrecord MockSolrClient [] (defrecord MockSolrClient []
SolrClient SolrClient
(index-documents [this xs] (index-documents [this index xs]
nil) nil)
(query [this q] (index-documents-raw [this index xs]
nil) nil)
(delete [this]
(query [this index q]
nil)
(delete [this index]
nil)) nil))
(def impl (if (= :solr (:solr-impl env)) (def impl (if (= :solr (:solr-impl env))
@@ -172,9 +189,11 @@
(defn touch-with-ledger [i] (defn touch-with-ledger [i]
(index-documents impl [i [:journal-entry/original-entity i]])) (index-documents impl "invoices" [i [:journal-entry/original-entity i]]))
(defn touch [i] (defn touch
(index-documents impl [i])) ([i] (touch i "invoices"))
([i index]
(index-documents impl index [i])))

3
src/clj/auto_ap/ssr/search.clj
View File

@@ -53,7 +53,8 @@
(into [] (into []
(filter (fn [d] (filter (fn [d]
(can-see-client? id (first (:client_id d))))) (can-see-client? id (first (:client_id d)))))
(solr/query solr/impl (q->solr-q q)))) (solr/query solr/impl "invoices" {"query" (q->solr-q q)
"fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"})))
(defn search-results* [q id] (defn search-results* [q id]

10
src/clj/user.clj
View File

@@ -346,7 +346,7 @@
(mu/start-publisher! {:type :dev}) (mu/start-publisher! {:type :dev})
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn }))) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn })))
(defn start-search [] #_(defn start-search []
(mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer}))) (mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer})))
(defn restart-db [] (defn restart-db []
@@ -573,7 +573,7 @@
(partition-all 1000))] (partition-all 1000))]
(print ".") (print ".")
(flush) (flush)
(solr/index-documents solr/impl batch)) (solr/index-documents solr/impl "invoices" batch))
(doseq [batch (->> (dc/qseq {:query '[:find ?i (doseq [batch (->> (dc/qseq {:query '[:find ?i
:in $ :in $
@@ -584,7 +584,7 @@
(partition-all 1000))] (partition-all 1000))]
(print ".") (print ".")
(flush) (flush)
(solr/index-documents solr/impl batch)) (solr/index-documents solr/impl "invoices" batch))
(doseq [batch (->> (dc/qseq {:query '[:find ?i (doseq [batch (->> (dc/qseq {:query '[:find ?i
:in $ :in $
@@ -595,7 +595,7 @@
(partition-all 1000))] (partition-all 1000))]
(print ".") (print ".")
(flush) (flush)
(solr/index-documents solr/impl batch)) (solr/index-documents solr/impl "invoices" batch))
(doseq [batch (->> (dc/qseq {:query '[:find ?i (doseq [batch (->> (dc/qseq {:query '[:find ?i
:in $ :in $
:where [?i :journal-entry/date]] :where [?i :journal-entry/date]]
@@ -604,7 +604,7 @@
(partition-all 1000))] (partition-all 1000))]
(print ".") (print ".")
(flush) (flush)
(solr/index-documents solr/impl batch))) (solr/index-documents solr/impl "invoices" batch)))
(defn setup-sales-orders [] (defn setup-sales-orders []
(doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]}) (doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]})