(ns auto-ap.routes.queries (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [conn pull-attr pull-id]] [auto-ap.graphql.utils :refer [assert-admin]] [clojure.data.csv :as csv] [clojure.edn :as edn] [clojure.java.io :as io] [clojure.string :as str] [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] [config.core :refer [env]] [datomic.api :as dc] [iol-ion.tx :refer [random-tempid]] [ring.middleware.json :refer [wrap-json-response]] [ring.util.request :refer [body-string]] [unilog.context :as lc]) (:import (java.util UUID))) (defn wrap-csv-response [handler] (fn [request] (let [response (handler request)] (update response :body #(with-open [w (java.io.StringWriter.)] (csv/write-csv w %) (.toString w)))))) (defn execute-query [query-params params] (let [{:keys [query-id]} params] (lc/with-context {:query-id query-id} (log/info "Executing raw query " query-id) (let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env) :key (str "queries/" (:query-id params))))))] (log/info "Executing query " query-string) (into (list) (apply dc/q (edn/read-string query-string) (into [(dc/db conn)] (edn/read-string (get query-params "args" "[]"))))))))) (defn put-query [guid body note & [lookup-key client]] (let [id (pull-id (dc/db conn) [:saved-query/lookup-key lookup-key]) guid (if lookup-key (or (pull-attr (dc/db conn) :saved-query/guid [:saved-query/lookup-key lookup-key]) guid) guid)] @(dc/transact conn [[:upsert-entity {:db/id (or id (random-tempid)) :saved-query/guid guid :saved-query/description note :saved-query/key (str "queries/" guid) :saved-query/client client :saved-query/lookup-key lookup-key}]]) (s3/put-object :bucket-name (:data-bucket env) :key (str "queries/" guid) :input-stream (io/make-input-stream (.getBytes body) {}) :metadata {:content-type "application/text" :content-length (count (.getBytes body)) :user-metadata {:note note}}) {:body {:query body :id guid :results-url (str "/api/queries/" guid "/results") :csv-results-url (str "/api/queries/" guid "/results/csv") :json-results-url (str "/api/queries/" guid "/results/json")}})) (defn get-queries [{:keys [identity]}] (assert-admin identity) (let [obj (s3/list-objects :bucket-name (:data-bucket env) :prefix (str "queries/"))] (log/info obj) {:body (->> (:object-summaries obj) (map (fn [o] {:last-modified (.toString (:last-modified o)) :key (str/replace (:key o) #"^queries\/" "")})))})) (defn create-query [{:keys [query-params identity] :as request}] (assert-admin identity) (log/info "Note" (query-params "note")) (put-query (str (UUID/randomUUID)) (body-string request) (query-params "note"))) (defn get-query [{:keys [identity params]} ] (assert-admin identity) (let [{:keys [query-id]} params obj (s3/get-object :bucket-name (:data-bucket env) :key (str "queries/" query-id)) query-string (str (slurp (:object-content obj)))] (log/info obj) {:body {:query query-string :note (:note (:user-metadata (:object-metadata obj))) :id query-id :csv-results-url (str "/api/queries/" query-id "/results/csv") :json-results-url (str "/api/queries/" query-id "/results/json")}})) (defn update-query [{:keys [query-params identity params] :as request} ] (assert-admin identity) (log/info "Note" (query-params "note")) (put-query (:query-id params) (body-string request) (query-params "note"))) (defn results-json-query [{:keys [query-params params]}] (statsd/time! [(str "export.query.time") {:tags #{(str "query:" (:query-id params))}}] {:body (execute-query query-params params)})) (defn raw-query [{:keys [identity params]}] (assert-admin identity) (let [{:keys [query-id]} params obj (s3/get-object :bucket-name (:data-bucket env) :key (str "queries/" query-id)) query-string (str (slurp (:object-content obj)))] (log/info obj) {:body query-string})) (defn results-csv-query [{:keys [query-params params]}] (statsd/time! [(str "export.query.time") {:tags #{(str "query:" (:query-id params))}}] {:body (execute-query query-params params)})) (def routes2 {"api/" {#"queries/?" {[:query-id "/raw"] {:get :raw-query} [:query-id "/results/csv"] {:get :results-csv-query} [:query-id "/results/json"] {:get :results-json-query} [:query-id] {:get :get-query :put :update-query} :get :get-queries :post :create-query}}}) (def match->handler {:get-queries get-queries :create-query create-query :raw-query raw-query :get-query (-> get-query wrap-json-response) :update-query (-> update-query wrap-json-response) :results-json-query (-> results-json-query wrap-json-response) :results-csv-query (-> results-csv-query wrap-csv-response)})