From 8a38030b361cf5bfe3e215587a11bc3af970ecec Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 13 Jan 2022 12:57:57 -0800 Subject: [PATCH] Improves performance of ledger export --- src/clj/auto_ap/routes/exports.clj | 141 ++++++++++++++--------------- 1 file changed, 66 insertions(+), 75 deletions(-) diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index 203db861..19c4a1f0 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -1,15 +1,14 @@ (ns auto-ap.routes.exports (:require [auto-ap.datomic :refer [conn]] [auto-ap.datomic.clients :as d-clients] - [auto-ap.datomic.ledger :as d-ledger] [auto-ap.datomic.transactions :as d-transactions] + [clojure.edn :refer [read-string]] [auto-ap.datomic.vendors :as d-vendors] [buddy.sign.jwt :as jwt] [auto-ap.graphql :as graphql] [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]] [auto-ap.routes.utils :refer [wrap-secure]] [clojure.tools.logging :as log] - [auto-ap.logging :refer [error-event info-event warn-event]] [clj-time.coerce :as coerce :refer [to-date]] [clj-time.core :as time] [clojure.data.csv :as csv] @@ -18,8 +17,8 @@ [datomic.api :as d] [ring.middleware.json :refer [wrap-json-response]] [venia.core :as venia] - [yang.time :refer [time-it]] - [com.unbounce.dogstatsd.core :as statsd])) + [com.unbounce.dogstatsd.core :as statsd] + [auto-ap.time :as atime])) (defn wrap-csv-response [handler] (fn [request] (let [response (handler request)] @@ -30,7 +29,7 @@ (def api-key-authed-routes (context "/" [] - (GET "/sales/aggregated/export" {:keys [query-params identity :as params]} + (GET "/sales/aggregated/export" {:keys [query-params]} (let [client-id (Long/parseLong (get query-params "client-id")) identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})] (assert-can-see-client identity client-id) @@ -55,7 +54,7 @@ (def admin-only-routes (context "/" [] - (GET "/invoices/export" {:keys [query-params identity] :as request} + (GET "/invoices/export" {:keys [query-params identity]} (assert-admin identity) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) "export:invoice"}}] @@ -89,9 +88,8 @@ payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] (list (:all-payments (:data payments)))))) - (GET "/sales/export" {:keys [query-params identity :as params]} + (GET "/sales/export" {:keys [query-params identity]} (assert-admin identity) - (println params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) "export:sales"}}] (let [query [[:all_sales_orders @@ -165,76 +163,69 @@ (update :fee #(some-> % Double/parseDouble)) (update :total #(some-> % Double/parseDouble)))) (:all-expected-deposits (:data payments))))))) - (GET "/clients/export" {:keys [query-params identity]} + (GET "/clients/export" {:keys [identity]} (assert-admin identity) (map <-graphql (d-clients/get-all))) - (GET "/vendors/export" {:keys [query-params identity]} + (GET "/vendors/export" {:keys [identity]} (assert-admin identity) (statsd/time! [(str "export.time") {:tags #{"export:vendors"}}] (map <-graphql (d-vendors/get-graphql {})))) - (GET "/ledger/export" {:keys [query-params identity]} - (log/info "exporting for " (query-params "client-code")) - (assert-admin identity) - (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) - "export:ledger"}}] - (transduce (comp - (map #(update % :journal-entry/date to-date)) - (map <-graphql)) - conj - (list) - (first (d-ledger/get-graphql {:count Integer/MAX_VALUE - :client-code (query-params "client-code")}))))) - (GET "/ledger2/export" {:keys [query-params identity]} - (log/info "exporting for " (query-params "client-code")) - (assert-admin identity) - (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) - "export:ledger2"}}] - (let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id - :journal-entry/cleared - :journal-entry/alternate-description - :journal-entry/date - :journal-entry/note - :journal-entry/amount - :journal-entry/source - :journal-entry/cleared-against - :journal-entry/original-entity - {:journal-entry/client [:client/name :client/code :db/id] - :journal-entry/vendor [:vendor/name :db/id] - :journal-entry/line-items [:journal-entry-line/location - :journal-entry-line/debit - :journal-entry-line/credit - {:journal-entry-line/account [:bank-account/include-in-reports - :bank-account/bank-name - :bank-account/code - :bank-account/visible - :bank-account/name - :bank-account/number - :account/code - :account/name - :account/numeric-code - :account/location - {:account/type [:db/ident :db/id]} - {:bank-account/type [:db/ident :db/id]}]}]}]) - :in $ ?c - :where [?e :journal-entry/client ?c]] - (d/db conn) - [:client/code (query-params "client-code")]) - ) - tf-result (transduce (comp - (map first) - (filter (fn [je] - (every? - (fn [jel] - (let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)] - (or (nil? include-in-reports) - (true? include-in-reports)))) - (:journal-entry/line-items je)))) - (map <-graphql)) - conj - (list) - results)] - tf-result))) + (GET "/ledger/export" {:keys [identity query-params]} + (let [start-date (or (some-> (query-params "start-date") + (atime/parse atime/iso-date)) + (time/plus (time/now) (time/days -120)))] + (log/info "exporting for " (query-params "client-code") "starting" start-date) + (assert-admin identity) + (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) + "export:ledger2"}}] + (let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id + :journal-entry/cleared + :journal-entry/alternate-description + :journal-entry/date + :journal-entry/note + :journal-entry/amount + :journal-entry/source + :journal-entry/cleared-against + :journal-entry/original-entity + {:journal-entry/client [:client/name :client/code :db/id] + :journal-entry/vendor [:vendor/name :db/id] + :journal-entry/line-items [:journal-entry-line/location + :journal-entry-line/debit + :journal-entry-line/credit + {:journal-entry-line/account [:bank-account/include-in-reports + :bank-account/bank-name + :bank-account/code + :bank-account/visible + :bank-account/name + :bank-account/number + :account/code + :account/name + :account/numeric-code + :account/location + {:account/type [:db/ident :db/id]} + {:bank-account/type [:db/ident :db/id]}]}]}]) + :in $ ?c ?start-date + :where [?e :journal-entry/client ?c] + [?e :journal-entry/date ?date] + [(>= ?date ?start-date)]] + (d/db conn) + [:client/code (query-params "client-code")] + (coerce/to-date start-date))) + tf-result (transduce (comp + (map first) + (filter (fn [je] + (every? + (fn [jel] + (let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)] + (or (nil? include-in-reports) + (true? include-in-reports)))) + (:journal-entry/line-items je)))) + (map <-graphql)) + conj + (list) + results)] + tf-result)))) @@ -242,8 +233,8 @@ (assert-admin identity) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) "export:accounts"}}] - (let [client-id (d-clients/code->id (query-params "client-code")) - query [[:accounts + (let [client-id (d-clients/code->id (query-params "client-code")) + query [[:accounts [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]] all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] @@ -317,7 +308,7 @@ (assert-admin identity) (log/info "Executing raw query " (get query-params "query" )) (statsd/time! [(str "export.time") {:tags #{"export:raw"}}] - (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))) + (into (list) (apply d/q (read-string (get query-params "query" )) (into [(d/db conn)] (read-string (get query-params "args" "[]"))))))))) (defroutes export-routes (routes (wrap-routes api-key-authed-routes