Improves performance of ledger export
This commit is contained in:
@@ -1,15 +1,14 @@
|
||||
(ns auto-ap.routes.exports
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.datomic.clients :as d-clients]
|
||||
[auto-ap.datomic.ledger :as d-ledger]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[clojure.edn :refer [read-string]]
|
||||
[auto-ap.datomic.vendors :as d-vendors]
|
||||
[buddy.sign.jwt :as jwt]
|
||||
[auto-ap.graphql :as graphql]
|
||||
[auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
|
||||
[auto-ap.routes.utils :refer [wrap-secure]]
|
||||
[clojure.tools.logging :as log]
|
||||
[auto-ap.logging :refer [error-event info-event warn-event]]
|
||||
[clj-time.coerce :as coerce :refer [to-date]]
|
||||
[clj-time.core :as time]
|
||||
[clojure.data.csv :as csv]
|
||||
@@ -18,8 +17,8 @@
|
||||
[datomic.api :as d]
|
||||
[ring.middleware.json :refer [wrap-json-response]]
|
||||
[venia.core :as venia]
|
||||
[yang.time :refer [time-it]]
|
||||
[com.unbounce.dogstatsd.core :as statsd]))
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[auto-ap.time :as atime]))
|
||||
(defn wrap-csv-response [handler]
|
||||
(fn [request]
|
||||
(let [response (handler request)]
|
||||
@@ -30,7 +29,7 @@
|
||||
|
||||
(def api-key-authed-routes
|
||||
(context "/" []
|
||||
(GET "/sales/aggregated/export" {:keys [query-params identity :as params]}
|
||||
(GET "/sales/aggregated/export" {:keys [query-params]}
|
||||
(let [client-id (Long/parseLong (get query-params "client-id"))
|
||||
identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})]
|
||||
(assert-can-see-client identity client-id)
|
||||
@@ -55,7 +54,7 @@
|
||||
|
||||
(def admin-only-routes
|
||||
(context "/" []
|
||||
(GET "/invoices/export" {:keys [query-params identity] :as request}
|
||||
(GET "/invoices/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:invoice"}}]
|
||||
@@ -89,9 +88,8 @@
|
||||
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
(list (:all-payments (:data payments))))))
|
||||
|
||||
(GET "/sales/export" {:keys [query-params identity :as params]}
|
||||
(GET "/sales/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(println params)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:sales"}}]
|
||||
(let [query [[:all_sales_orders
|
||||
@@ -165,76 +163,69 @@
|
||||
(update :fee #(some-> % Double/parseDouble))
|
||||
(update :total #(some-> % Double/parseDouble))))
|
||||
(:all-expected-deposits (:data payments)))))))
|
||||
(GET "/clients/export" {:keys [query-params identity]}
|
||||
(GET "/clients/export" {:keys [identity]}
|
||||
(assert-admin identity)
|
||||
(map <-graphql (d-clients/get-all)))
|
||||
|
||||
(GET "/vendors/export" {:keys [query-params identity]}
|
||||
(GET "/vendors/export" {:keys [identity]}
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{"export:vendors"}}]
|
||||
(map <-graphql (d-vendors/get-graphql {}))))
|
||||
(GET "/ledger/export" {:keys [query-params identity]}
|
||||
(log/info "exporting for " (query-params "client-code"))
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:ledger"}}]
|
||||
(transduce (comp
|
||||
(map #(update % :journal-entry/date to-date))
|
||||
(map <-graphql))
|
||||
conj
|
||||
(list)
|
||||
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE
|
||||
:client-code (query-params "client-code")})))))
|
||||
(GET "/ledger2/export" {:keys [query-params identity]}
|
||||
(log/info "exporting for " (query-params "client-code"))
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:ledger2"}}]
|
||||
(let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id
|
||||
:journal-entry/cleared
|
||||
:journal-entry/alternate-description
|
||||
:journal-entry/date
|
||||
:journal-entry/note
|
||||
:journal-entry/amount
|
||||
:journal-entry/source
|
||||
:journal-entry/cleared-against
|
||||
:journal-entry/original-entity
|
||||
{:journal-entry/client [:client/name :client/code :db/id]
|
||||
:journal-entry/vendor [:vendor/name :db/id]
|
||||
:journal-entry/line-items [:journal-entry-line/location
|
||||
:journal-entry-line/debit
|
||||
:journal-entry-line/credit
|
||||
{:journal-entry-line/account [:bank-account/include-in-reports
|
||||
:bank-account/bank-name
|
||||
:bank-account/code
|
||||
:bank-account/visible
|
||||
:bank-account/name
|
||||
:bank-account/number
|
||||
:account/code
|
||||
:account/name
|
||||
:account/numeric-code
|
||||
:account/location
|
||||
{:account/type [:db/ident :db/id]}
|
||||
{:bank-account/type [:db/ident :db/id]}]}]}])
|
||||
:in $ ?c
|
||||
:where [?e :journal-entry/client ?c]]
|
||||
(d/db conn)
|
||||
[:client/code (query-params "client-code")])
|
||||
)
|
||||
tf-result (transduce (comp
|
||||
(map first)
|
||||
(filter (fn [je]
|
||||
(every?
|
||||
(fn [jel]
|
||||
(let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)]
|
||||
(or (nil? include-in-reports)
|
||||
(true? include-in-reports))))
|
||||
(:journal-entry/line-items je))))
|
||||
(map <-graphql))
|
||||
conj
|
||||
(list)
|
||||
results)]
|
||||
tf-result)))
|
||||
(GET "/ledger/export" {:keys [identity query-params]}
|
||||
(let [start-date (or (some-> (query-params "start-date")
|
||||
(atime/parse atime/iso-date))
|
||||
(time/plus (time/now) (time/days -120)))]
|
||||
(log/info "exporting for " (query-params "client-code") "starting" start-date)
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:ledger2"}}]
|
||||
(let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id
|
||||
:journal-entry/cleared
|
||||
:journal-entry/alternate-description
|
||||
:journal-entry/date
|
||||
:journal-entry/note
|
||||
:journal-entry/amount
|
||||
:journal-entry/source
|
||||
:journal-entry/cleared-against
|
||||
:journal-entry/original-entity
|
||||
{:journal-entry/client [:client/name :client/code :db/id]
|
||||
:journal-entry/vendor [:vendor/name :db/id]
|
||||
:journal-entry/line-items [:journal-entry-line/location
|
||||
:journal-entry-line/debit
|
||||
:journal-entry-line/credit
|
||||
{:journal-entry-line/account [:bank-account/include-in-reports
|
||||
:bank-account/bank-name
|
||||
:bank-account/code
|
||||
:bank-account/visible
|
||||
:bank-account/name
|
||||
:bank-account/number
|
||||
:account/code
|
||||
:account/name
|
||||
:account/numeric-code
|
||||
:account/location
|
||||
{:account/type [:db/ident :db/id]}
|
||||
{:bank-account/type [:db/ident :db/id]}]}]}])
|
||||
:in $ ?c ?start-date
|
||||
:where [?e :journal-entry/client ?c]
|
||||
[?e :journal-entry/date ?date]
|
||||
[(>= ?date ?start-date)]]
|
||||
(d/db conn)
|
||||
[:client/code (query-params "client-code")]
|
||||
(coerce/to-date start-date)))
|
||||
tf-result (transduce (comp
|
||||
(map first)
|
||||
(filter (fn [je]
|
||||
(every?
|
||||
(fn [jel]
|
||||
(let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)]
|
||||
(or (nil? include-in-reports)
|
||||
(true? include-in-reports))))
|
||||
(:journal-entry/line-items je))))
|
||||
(map <-graphql))
|
||||
conj
|
||||
(list)
|
||||
results)]
|
||||
tf-result))))
|
||||
|
||||
|
||||
|
||||
@@ -242,8 +233,8 @@
|
||||
(assert-admin identity)
|
||||
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
|
||||
"export:accounts"}}]
|
||||
(let [client-id (d-clients/code->id (query-params "client-code"))
|
||||
query [[:accounts
|
||||
(let [client-id (d-clients/code->id (query-params "client-code"))
|
||||
query [[:accounts
|
||||
[:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
|
||||
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
|
||||
@@ -317,7 +308,7 @@
|
||||
(assert-admin identity)
|
||||
(log/info "Executing raw query " (get query-params "query" ))
|
||||
(statsd/time! [(str "export.time") {:tags #{"export:raw"}}]
|
||||
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))))
|
||||
(into (list) (apply d/q (read-string (get query-params "query" )) (into [(d/db conn)] (read-string (get query-params "args" "[]")))))))))
|
||||
(defroutes export-routes
|
||||
(routes
|
||||
(wrap-routes api-key-authed-routes
|
||||
|
||||
Reference in New Issue
Block a user