134 lines
6.1 KiB
Clojure
134 lines
6.1 KiB
Clojure
(ns auto-ap.routes.queries
|
|
(:require
|
|
[amazonica.aws.s3 :as s3]
|
|
[auto-ap.datomic :refer [conn pull-attr pull-id]]
|
|
[auto-ap.graphql.utils :refer [assert-admin]]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.edn :as edn]
|
|
[clojure.java.io :as io]
|
|
[clojure.string :as str]
|
|
[auto-ap.logging :as alog]
|
|
[com.unbounce.dogstatsd.core :as statsd]
|
|
[com.brunobonacci.mulog :as mu]
|
|
[config.core :refer [env]]
|
|
[datomic.api :as dc]
|
|
[iol-ion.tx :refer [random-tempid]]
|
|
[ring.middleware.json :refer [wrap-json-response]]
|
|
[ring.util.request :refer [body-string]])
|
|
(:import
|
|
(java.util UUID)))
|
|
|
|
(defn wrap-csv-response [handler]
|
|
(fn [request]
|
|
(let [response (handler request)]
|
|
(update response :body #(with-open [w (java.io.StringWriter.)]
|
|
(csv/write-csv w %)
|
|
(.toString w))))))
|
|
|
|
|
|
|
|
(defn execute-query [query-params params]
|
|
(let [{:keys [query-id]} params]
|
|
(mu/with-context {:query-id query-id}
|
|
|
|
(alog/info ::executing-query :query-id query-id)
|
|
(let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env)
|
|
:key (str "queries/" (:query-id params))))))]
|
|
(into (list) (apply dc/q (edn/read-string query-string)
|
|
(into [(dc/db conn)] (edn/read-string (get query-params "args" "[]")))))))))
|
|
|
|
|
|
(defn put-query [guid body note & [lookup-key client]]
|
|
(let [id (pull-id (dc/db conn) [:saved-query/lookup-key lookup-key])
|
|
guid (if lookup-key
|
|
(or (pull-attr (dc/db conn) :saved-query/guid [:saved-query/lookup-key lookup-key])
|
|
guid)
|
|
guid)]
|
|
@(dc/transact conn [[:upsert-entity {:db/id (or id (random-tempid))
|
|
:saved-query/guid guid
|
|
:saved-query/description note
|
|
:saved-query/key (str "queries/" guid)
|
|
:saved-query/client client
|
|
:saved-query/lookup-key lookup-key}]])
|
|
(s3/put-object :bucket-name (:data-bucket env)
|
|
:key (str "queries/" guid)
|
|
:input-stream (io/make-input-stream (.getBytes body) {})
|
|
:metadata {:content-type "application/text"
|
|
:content-length (count (.getBytes body))
|
|
:user-metadata {:note note}})
|
|
{:body {:query body
|
|
:id guid
|
|
:results-url (str "/api/queries/" guid "/results")
|
|
:csv-results-url (str "/api/queries/" guid "/results/csv")
|
|
:json-results-url (str "/api/queries/" guid "/results/json")}}))
|
|
|
|
|
|
|
|
(defn get-queries [{:keys [identity]}]
|
|
(assert-admin identity)
|
|
(let [obj (s3/list-objects :bucket-name (:data-bucket env)
|
|
:prefix (str "queries/"))]
|
|
{:body (->> (:object-summaries obj)
|
|
(map (fn [o]
|
|
{:last-modified (.toString (:last-modified o))
|
|
:key (str/replace (:key o) #"^queries\/" "")})))}))
|
|
|
|
(defn create-query [{:keys [query-params identity] :as request}]
|
|
(assert-admin identity)
|
|
(put-query (str (UUID/randomUUID)) (body-string request) (query-params "note")))
|
|
|
|
(defn get-query [{:keys [identity params]} ]
|
|
(assert-admin identity)
|
|
(let [{:keys [query-id]} params
|
|
obj (s3/get-object :bucket-name (:data-bucket env)
|
|
:key (str "queries/" query-id))
|
|
query-string (str (slurp (:object-content obj)))]
|
|
{:body {:query query-string
|
|
:note (:note (:user-metadata (:object-metadata obj)))
|
|
:id query-id
|
|
:csv-results-url (str "/api/queries/" query-id "/results/csv")
|
|
:json-results-url (str "/api/queries/" query-id "/results/json")}}))
|
|
|
|
(defn update-query [{:keys [query-params identity params] :as request} ]
|
|
(assert-admin identity)
|
|
(put-query (:query-id params) (body-string request) (query-params "note")))
|
|
|
|
(defn results-json-query [{:keys [query-params params]}]
|
|
(statsd/time! [(str "export.query.time") {:tags #{(str "query:" (:query-id params))}}]
|
|
{:body (execute-query query-params params)}))
|
|
|
|
(defn raw-query [{:keys [identity params]}]
|
|
(assert-admin identity)
|
|
(let [{:keys [query-id]} params
|
|
obj (s3/get-object :bucket-name (:data-bucket env)
|
|
:key (str "queries/" query-id))
|
|
query-string (str (slurp (:object-content obj)))]
|
|
{:body query-string}))
|
|
|
|
(defn results-csv-query [{:keys [query-params params]}]
|
|
(statsd/time! [(str "export.query.time") {:tags #{(str "query:" (:query-id params))}}]
|
|
{:body (execute-query query-params params)}))
|
|
|
|
(def routes2 {"api/" {#"queries/?" {[:query-id "/raw"] {:get :raw-query}
|
|
[:query-id "/results/csv"] {:get :results-csv-query}
|
|
[:query-id "/results/json"] {:get :results-json-query}
|
|
[:query-id] {:get :get-query
|
|
:put :update-query}
|
|
:get :get-queries
|
|
:post :create-query}}})
|
|
|
|
(def match->handler {:get-queries get-queries
|
|
:create-query create-query
|
|
:raw-query raw-query
|
|
:get-query (-> get-query
|
|
wrap-json-response)
|
|
:update-query (-> update-query
|
|
wrap-json-response)
|
|
|
|
:results-json-query (-> results-json-query
|
|
wrap-json-response)
|
|
:results-csv-query (-> results-csv-query
|
|
wrap-csv-response)})
|
|
|
|
|