notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Improves performance of ledger export

Browse Source
This commit is contained in:
Bryce Covert
2022-01-13 12:57:57 -08:00
parent 0d09a3f702
commit 8a38030b36
1 changed files with 66 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/clj/auto_ap/routes/exports.clj
View File

@@ -1,15 +1,14 @@
(ns auto-ap.routes.exports (ns auto-ap.routes.exports
(:require [auto-ap.datomic :refer [conn]] (:require [auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.ledger :as d-ledger]
[auto-ap.datomic.transactions :as d-transactions] [auto-ap.datomic.transactions :as d-transactions]
[clojure.edn :refer [read-string]]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
[buddy.sign.jwt :as jwt] [buddy.sign.jwt :as jwt]
[auto-ap.graphql :as graphql] [auto-ap.graphql :as graphql]
[auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]] [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
[auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.routes.utils :refer [wrap-secure]]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[auto-ap.logging :refer [error-event info-event warn-event]]
[clj-time.coerce :as coerce :refer [to-date]] [clj-time.coerce :as coerce :refer [to-date]]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
@@ -18,8 +17,8 @@
[datomic.api :as d] [datomic.api :as d]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia] [venia.core :as venia]
[yang.time :refer [time-it]] [com.unbounce.dogstatsd.core :as statsd]
[com.unbounce.dogstatsd.core :as statsd])) [auto-ap.time :as atime]))
(defn wrap-csv-response [handler] (defn wrap-csv-response [handler]
(fn [request] (fn [request]
(let [response (handler request)] (let [response (handler request)]
@@ -30,7 +29,7 @@
(def api-key-authed-routes (def api-key-authed-routes
(context "/" [] (context "/" []
(GET "/sales/aggregated/export" {:keys [query-params identity :as params]} (GET "/sales/aggregated/export" {:keys [query-params]}
(let [client-id (Long/parseLong (get query-params "client-id")) (let [client-id (Long/parseLong (get query-params "client-id"))
identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})] identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})]
(assert-can-see-client identity client-id) (assert-can-see-client identity client-id)
@@ -55,7 +54,7 @@
(def admin-only-routes (def admin-only-routes
(context "/" [] (context "/" []
(GET "/invoices/export" {:keys [query-params identity] :as request} (GET "/invoices/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:invoice"}}] "export:invoice"}}]
@@ -89,9 +88,8 @@
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
(list (:all-payments (:data payments)))))) (list (:all-payments (:data payments))))))
(GET "/sales/export" {:keys [query-params identity :as params]} (GET "/sales/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(println params)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:sales"}}] "export:sales"}}]
(let [query [[:all_sales_orders (let [query [[:all_sales_orders
@@ -165,28 +163,19 @@
(update :fee #(some-> % Double/parseDouble)) (update :fee #(some-> % Double/parseDouble))
(update :total #(some-> % Double/parseDouble)))) (update :total #(some-> % Double/parseDouble))))
(:all-expected-deposits (:data payments))))))) (:all-expected-deposits (:data payments)))))))
(GET "/clients/export" {:keys [query-params identity]} (GET "/clients/export" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(map <-graphql (d-clients/get-all))) (map <-graphql (d-clients/get-all)))
(GET "/vendors/export" {:keys [query-params identity]} (GET "/vendors/export" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{"export:vendors"}}] (statsd/time! [(str "export.time") {:tags #{"export:vendors"}}]
(map <-graphql (d-vendors/get-graphql {})))) (map <-graphql (d-vendors/get-graphql {}))))
(GET "/ledger/export" {:keys [query-params identity]} (GET "/ledger/export" {:keys [identity query-params]}
(log/info "exporting for " (query-params "client-code")) (let [start-date (or (some-> (query-params "start-date")
(assert-admin identity) (atime/parse atime/iso-date))
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (time/plus (time/now) (time/days -120)))]
"export:ledger"}}] (log/info "exporting for " (query-params "client-code") "starting" start-date)
(transduce (comp
(map #(update % :journal-entry/date to-date))
(map <-graphql))
conj
(list)
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE
:client-code (query-params "client-code")})))))
(GET "/ledger2/export" {:keys [query-params identity]}
(log/info "exporting for " (query-params "client-code"))
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:ledger2"}}] "export:ledger2"}}]
@@ -216,11 +205,13 @@
:account/location :account/location
{:account/type [:db/ident :db/id]} {:account/type [:db/ident :db/id]}
{:bank-account/type [:db/ident :db/id]}]}]}]) {:bank-account/type [:db/ident :db/id]}]}]}])
:in $ ?c :in $ ?c ?start-date
:where [?e :journal-entry/client ?c]] :where [?e :journal-entry/client ?c]
[?e :journal-entry/date ?date]
[(>= ?date ?start-date)]]
(d/db conn) (d/db conn)
[:client/code (query-params "client-code")]) [:client/code (query-params "client-code")]
) (coerce/to-date start-date)))
tf-result (transduce (comp tf-result (transduce (comp
(map first) (map first)
(filter (fn [je] (filter (fn [je]
@@ -234,7 +225,7 @@
conj conj
(list) (list)
results)] results)]
tf-result))) tf-result))))
@@ -317,7 +308,7 @@
(assert-admin identity) (assert-admin identity)
(log/info "Executing raw query " (get query-params "query" )) (log/info "Executing raw query " (get query-params "query" ))
(statsd/time! [(str "export.time") {:tags #{"export:raw"}}] (statsd/time! [(str "export.time") {:tags #{"export:raw"}}]
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))) (into (list) (apply d/q (read-string (get query-params "query" )) (into [(d/db conn)] (read-string (get query-params "args" "[]")))))))))
(defroutes export-routes (defroutes export-routes
(routes (routes
(wrap-routes api-key-authed-routes (wrap-routes api-key-authed-routes