notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Improves performance of ledger export

Browse Source
This commit is contained in:
Bryce Covert
2022-01-13 12:57:57 -08:00
parent 0d09a3f702
commit 8a38030b36
1 changed files with 66 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
src/clj/auto_ap/routes/exports.clj
View File

@@ -1,15 +1,14 @@
(ns auto-ap.routes.exports (ns auto-ap.routes.exports
(:require [auto-ap.datomic :refer [conn]] (:require [auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.ledger :as d-ledger]
[auto-ap.datomic.transactions :as d-transactions] [auto-ap.datomic.transactions :as d-transactions]
[clojure.edn :refer [read-string]]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
[buddy.sign.jwt :as jwt] [buddy.sign.jwt :as jwt]
[auto-ap.graphql :as graphql] [auto-ap.graphql :as graphql]
[auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]] [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
[auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.routes.utils :refer [wrap-secure]]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[auto-ap.logging :refer [error-event info-event warn-event]]
[clj-time.coerce :as coerce :refer [to-date]] [clj-time.coerce :as coerce :refer [to-date]]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
@@ -18,8 +17,8 @@
[datomic.api :as d] [datomic.api :as d]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia] [venia.core :as venia]
[yang.time :refer [time-it]] [com.unbounce.dogstatsd.core :as statsd]
[com.unbounce.dogstatsd.core :as statsd])) [auto-ap.time :as atime]))
(defn wrap-csv-response [handler] (defn wrap-csv-response [handler]
(fn [request] (fn [request]
(let [response (handler request)] (let [response (handler request)]
@@ -30,7 +29,7 @@
(def api-key-authed-routes (def api-key-authed-routes
(context "/" [] (context "/" []
(GET "/sales/aggregated/export" {:keys [query-params identity :as params]} (GET "/sales/aggregated/export" {:keys [query-params]}
(let [client-id (Long/parseLong (get query-params "client-id")) (let [client-id (Long/parseLong (get query-params "client-id"))
identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})] identity (jwt/unsign (get query-params "key") (:jwt-secret env) {:alg :hs512})]
(assert-can-see-client identity client-id) (assert-can-see-client identity client-id)
@@ -55,7 +54,7 @@
(def admin-only-routes (def admin-only-routes
(context "/" [] (context "/" []
(GET "/invoices/export" {:keys [query-params identity] :as request} (GET "/invoices/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:invoice"}}] "export:invoice"}}]
@@ -89,9 +88,8 @@
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
(list (:all-payments (:data payments)))))) (list (:all-payments (:data payments))))))
(GET "/sales/export" {:keys [query-params identity :as params]} (GET "/sales/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(println params)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:sales"}}] "export:sales"}}]
(let [query [[:all_sales_orders (let [query [[:all_sales_orders
@@ -165,76 +163,69 @@
(update :fee #(some-> % Double/parseDouble)) (update :fee #(some-> % Double/parseDouble))
(update :total #(some-> % Double/parseDouble)))) (update :total #(some-> % Double/parseDouble))))
(:all-expected-deposits (:data payments))))))) (:all-expected-deposits (:data payments)))))))
(GET "/clients/export" {:keys [query-params identity]} (GET "/clients/export" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(map <-graphql (d-clients/get-all))) (map <-graphql (d-clients/get-all)))
(GET "/vendors/export" {:keys [query-params identity]} (GET "/vendors/export" {:keys [identity]}
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{"export:vendors"}}] (statsd/time! [(str "export.time") {:tags #{"export:vendors"}}]
(map <-graphql (d-vendors/get-graphql {})))) (map <-graphql (d-vendors/get-graphql {}))))
(GET "/ledger/export" {:keys [query-params identity]} (GET "/ledger/export" {:keys [identity query-params]}
(log/info "exporting for " (query-params "client-code")) (let [start-date (or (some-> (query-params "start-date")
(assert-admin identity) (atime/parse atime/iso-date))
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (time/plus (time/now) (time/days -120)))]
"export:ledger"}}] (log/info "exporting for " (query-params "client-code") "starting" start-date)
(transduce (comp (assert-admin identity)
(map #(update % :journal-entry/date to-date)) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
(map <-graphql)) "export:ledger2"}}]
conj (let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id
(list) :journal-entry/cleared
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE :journal-entry/alternate-description
:client-code (query-params "client-code")}))))) :journal-entry/date
(GET "/ledger2/export" {:keys [query-params identity]} :journal-entry/note
(log/info "exporting for " (query-params "client-code")) :journal-entry/amount
(assert-admin identity) :journal-entry/source
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) :journal-entry/cleared-against
"export:ledger2"}}] :journal-entry/original-entity
(let [results (->> (d/q '[:find (pull ?e [:journal-entry/external-id {:journal-entry/client [:client/name :client/code :db/id]
:journal-entry/cleared :journal-entry/vendor [:vendor/name :db/id]
:journal-entry/alternate-description :journal-entry/line-items [:journal-entry-line/location
:journal-entry/date :journal-entry-line/debit
:journal-entry/note :journal-entry-line/credit
:journal-entry/amount {:journal-entry-line/account [:bank-account/include-in-reports
:journal-entry/source :bank-account/bank-name
:journal-entry/cleared-against :bank-account/code
:journal-entry/original-entity :bank-account/visible
{:journal-entry/client [:client/name :client/code :db/id] :bank-account/name
:journal-entry/vendor [:vendor/name :db/id] :bank-account/number
:journal-entry/line-items [:journal-entry-line/location :account/code
:journal-entry-line/debit :account/name
:journal-entry-line/credit :account/numeric-code
{:journal-entry-line/account [:bank-account/include-in-reports :account/location
:bank-account/bank-name {:account/type [:db/ident :db/id]}
:bank-account/code {:bank-account/type [:db/ident :db/id]}]}]}])
:bank-account/visible :in $ ?c ?start-date
:bank-account/name :where [?e :journal-entry/client ?c]
:bank-account/number [?e :journal-entry/date ?date]
:account/code [(>= ?date ?start-date)]]
:account/name (d/db conn)
:account/numeric-code [:client/code (query-params "client-code")]
:account/location (coerce/to-date start-date)))
{:account/type [:db/ident :db/id]} tf-result (transduce (comp
{:bank-account/type [:db/ident :db/id]}]}]}]) (map first)
:in $ ?c (filter (fn [je]
:where [?e :journal-entry/client ?c]] (every?
(d/db conn) (fn [jel]
[:client/code (query-params "client-code")]) (let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)]
) (or (nil? include-in-reports)
tf-result (transduce (comp (true? include-in-reports))))
(map first) (:journal-entry/line-items je))))
(filter (fn [je] (map <-graphql))
(every? conj
(fn [jel] (list)
(let [include-in-reports (-> jel :journal-entry-line/account :bank-account/include-in-reports)] results)]
(or (nil? include-in-reports) tf-result))))
(true? include-in-reports))))
(:journal-entry/line-items je))))
(map <-graphql))
conj
(list)
results)]
tf-result)))
@@ -242,8 +233,8 @@
(assert-admin identity) (assert-admin identity)
(statsd/time! [(str "export.time") {:tags #{(client-tag query-params) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)
"export:accounts"}}] "export:accounts"}}]
(let [client-id (d-clients/code->id (query-params "client-code")) (let [client-id (d-clients/code->id (query-params "client-code"))
query [[:accounts query [[:accounts
[:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]] [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
@@ -317,7 +308,7 @@
(assert-admin identity) (assert-admin identity)
(log/info "Executing raw query " (get query-params "query" )) (log/info "Executing raw query " (get query-params "query" ))
(statsd/time! [(str "export.time") {:tags #{"export:raw"}}] (statsd/time! [(str "export.time") {:tags #{"export:raw"}}]
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))) (into (list) (apply d/q (read-string (get query-params "query" )) (into [(d/db conn)] (read-string (get query-params "args" "[]")))))))))
(defroutes export-routes (defroutes export-routes
(routes (routes
(wrap-routes api-key-authed-routes (wrap-routes api-key-authed-routes