From 311b07543c406b8ddaffb28258a5e1e68c4a9657 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 3 Jun 2021 07:18:03 -0700 Subject: [PATCH] Adds ability to create/save raw queries --- src/clj/auto_ap/handler.clj | 2 + src/clj/auto_ap/routes/queries.clj | 118 +++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 src/clj/auto_ap/routes/queries.clj diff --git a/src/clj/auto_ap/handler.clj b/src/clj/auto_ap/handler.clj index 23986f79..3deb281b 100644 --- a/src/clj/auto_ap/handler.clj +++ b/src/clj/auto_ap/handler.clj @@ -3,6 +3,7 @@ [auto-ap.routes.auth :as auth] [auto-ap.routes.events :as events] [auto-ap.routes.exports :as exports] + [auto-ap.routes.queries :as queries] [auto-ap.routes.graphql :as graphql] [auto-ap.routes.invoices :as invoices] [auto-ap.routes.yodlee :as yodlee] @@ -56,6 +57,7 @@ exports/export-routes yodlee/routes yodlee2/routes + queries/query-routes invoices/routes graphql/routes auth/routes diff --git a/src/clj/auto_ap/routes/queries.clj b/src/clj/auto_ap/routes/queries.clj new file mode 100644 index 00000000..bd8a748f --- /dev/null +++ b/src/clj/auto_ap/routes/queries.clj @@ -0,0 +1,118 @@ +(ns auto-ap.routes.queries + (:require [auto-ap.datomic :refer [conn]] + [auto-ap.datomic.clients :as d-clients] + [auto-ap.datomic.ledger :as d-ledger] + [auto-ap.datomic.transactions :as d-transactions] + [auto-ap.datomic.vendors :as d-vendors] + + [unilog.context :as lc] + [buddy.sign.jwt :as jwt] + [auto-ap.graphql :as graphql] + [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]] + [auto-ap.routes.utils :refer [wrap-secure]] + [clojure.tools.logging :as log] + [auto-ap.logging :refer [error-event info-event warn-event]] + [clj-time.coerce :as coerce :refer [to-date]] + [amazonica.aws.s3 :as s3] + [clj-time.core :as time] + [clojure.java.io :as io] + [clojure.data.csv :as csv] + [config.core :refer [env]] + [compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]] + [datomic.api :as d] + [ring.middleware.json :refer [wrap-json-response]] + [venia.core :as venia] + [yang.time :refer [time-it]] + [ring.util.request :refer [body-string]] + ) + (:import java.util.UUID)) + + +(defn wrap-csv-response [handler] + (fn [request] + (let [response (handler request)] + (update response :body #(with-open [w (java.io.StringWriter.)] + (csv/write-csv w %) + (.toString w)))))) + +#_(GET "/raw" {:keys [query-params identity]} + (assert-admin identity) + (log/info "Executing raw query " (get query-params "query" )) + (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))) + +(defn execute-query [query-params params] + (let [{:keys [query-id]} params] + (lc/with-context {:query-id query-id} + + (log/info "Executing raw query " query-id) + (let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env) + :key (str "queries/" (:query-id params))))))] + (log/info "Executing query " query-string) + (into (list) (apply d/q (clojure.edn/read-string query-string) + (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))) + +(def query-routes + (routes + (wrap-json-response (POST "/queries" {:keys [query-params identity] :as request} + (assert-admin identity) + (let [uuid (str (UUID/randomUUID)) + body (body-string request)] + (s3/put-object :bucket-name (:data-bucket env) + :key (str "queries/" uuid) + :input-stream (io/make-input-stream (.getBytes body) {}) + :metadata {:content-type "application/text" + :user-metadata {:note (query-params "note")}}) + {:body {:query body + :id uuid + :results-url (str "/api/queries/" uuid "/results") + :csv-results-url (str "/api/queries/" uuid "/results/csv") + :json-results-url (str "/api/queries/" uuid "/results/json")}}) + + )) + (wrap-json-response (PUT "/queries/:query-id" {:keys [query-params identity params] :as request} + (assert-admin identity) + (log/info "Note" (query-params "note")) + (let [body (body-string request)] + (s3/put-object :bucket-name (:data-bucket env) + :key (str "queries/" (:query-id params)) + :input-stream (io/make-input-stream (.getBytes body) {}) + :metadata {:content-type "application/text" + :user-metadata {:note (query-params "note")}}) + {:body {:query body + :id (:query-id params) + :csv-results-url (str "/api/queries/" (:query-id params) "/results/csv") + :json-results-url (str "/api/queries/" (:query-id params) "/results/json")}}))) + (wrap-json-response (GET "/queries/:query-id" {:keys [query-params identity params]} + (assert-admin identity) + (let [{:keys [query-id]} params + obj (s3/get-object :bucket-name (:data-bucket env) + :key (str "queries/" query-id)) + query-string (str (slurp (:object-content obj)))] + (log/info obj) + {:body {:query query-string + :note (:note (:user-metadata (:object-metadata obj))) + :id query-id + :csv-results-url (str "/api/queries/" query-id "/results/csv") + :json-results-url (str "/api/queries/" query-id "/results/json")}}))) + (GET "/queries/:query-id/raw" {:keys [query-params identity params]} + (assert-admin identity) + (let [{:keys [query-id]} params + obj (s3/get-object :bucket-name (:data-bucket env) + :key (str "queries/" query-id)) + query-string (str (slurp (:object-content obj)))] + (log/info obj) + {:body query-string})) + + (wrap-json-response (GET "/queries/" {:keys [query-params identity params]} + (assert-admin identity) + (let [{:keys [query-id]} params + obj (s3/list-objects :bucket-name (:data-bucket env) + :prefix (str "queries/"))] + (log/info obj) + {:body {}}))) + + (wrap-json-response (GET "/queries/:query-id/results/json" {:keys [query-params identity params]} + {:body (execute-query query-params params)})) + (wrap-csv-response (GET "/queries/:query-id/results/csv" {:keys [query-params identity params]} + {:body (execute-query query-params params)})) + ))