From cbc3c00b8ecf72bc79501fad9f36195401d7ebcf Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 9 Apr 2022 10:58:10 -0700 Subject: [PATCH] more perf tweaks. --- config/prod-background-worker.edn | 2 +- src/clj/auto_ap/background/mail.clj | 3 +- src/clj/auto_ap/routes/exports.clj | 66 +++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/config/prod-background-worker.edn b/config/prod-background-worker.edn index 550e6545..22e7a8d1 100644 --- a/config/prod-background-worker.edn +++ b/config/prod-background-worker.edn @@ -7,7 +7,7 @@ :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" :invoice-email "invoices@mail.app.integreatconsult.com" - :import-failure-destination-email "ben@integreatconsult.com" + :import-failure-destination-emails ["ben@integreatconsult.com" "bryce+invoices@integreatconsult.com"] :data-bucket "data.prod.app.integreatconsult.com" :yodlee-cobrand-name "qstartus12" :yodlee-cobrand-login "qstartus12" diff --git a/src/clj/auto_ap/background/mail.clj b/src/clj/auto_ap/background/mail.clj index 3ee9d75d..8d803a2c 100644 --- a/src/clj/auto_ap/background/mail.clj +++ b/src/clj/auto_ap/background/mail.clj @@ -23,8 +23,9 @@ target-url (str "http://" (:data-bucket env) ".s3-website-us-east-1.amazonaws.com/" target-key)] + (log/info "sending email to " (:import-failure-destination-emails env)) (s3/copy-object mail-bucket mail-key (:data-bucket env) target-key) - (ses/send-email {:destination {:to-addresses [(:import-failure-destination-email env)]} + (ses/send-email {:destination {:to-addresses (:import-failure-destination-emails env)} :source (:invoice-email env) :message {:subject "An email invoice import failed" :body {:html (str "
You can download the original email here.
") diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index 888d720c..5bd37416 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -19,6 +19,7 @@ [venia.core :as venia] [com.unbounce.dogstatsd.core :as statsd] [auto-ap.time :as atime])) + (defn wrap-csv-response [handler] (fn [request] (let [response (handler request)] @@ -52,26 +53,63 @@ (:client-code params))] (str "client:" code))) +(defn datomic-map->graphql-map [m] + (into {} (map (fn [[k v]] + [(keyword (name k)) + (cond (:db/ident v) + (str (keyword (name (:db/ident v)))) + + (map? v) + (datomic-map->graphql-map v) + + (and (seqable? v) (map? (first v))) + (into [] (map datomic-map->graphql-map) v) + + (inst? v) + (atime/unparse (clj-time.coerce/to-date-time v) atime/iso-date) + + (= :db/id k) + (str v) + + (double? v) + (str v) + + + :else + v) + ])) + m)) + (def admin-only-routes (context "/" [] (GET "/invoices/export" {:keys [query-params identity]} (assert-admin identity) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params) "export:invoice"}}] - (let [query [[:all_invoices - {:client-code (query-params "client-code") - :original-id (query-params "original")} - - [:id :total :outstanding-balance :invoice-number :date :status :original-id - [:payments [:amount [:payment [:check-number :memo [:bank_account [:id :name :number :bank-name :bank-code :code]]]]]] - [:vendor [:name :id [:primary_contact [:name]] [:address [:street1 :city :state :zip]]]] - [:expense_accounts [:amount :id :location - [:account [:id :numeric-code :name]]]] - [:client [:name :id :code :locations]]]]] - invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] - - (list (:all-invoices (:data invoices))))) - ) + [(into [] + (map datomic-map->graphql-map) + (d/q '[:find [(pull ?i [:db/id :invoice/total :invoice/outstanding-balance :invoice/invoice-number :invoice/date :invoice/original-id + { :invoice/status [:db/ident] + :invoice/payments + [:invoice-payment/amount + {:invoice-payment/payment [:payment/check-number + :payment/memo + {:payment/bank_account [:bank-account/id :bank-account/name :bank-account/number :bank-account/bank-name :bank-account/bank-code :bank-account/code]}]}] + :invoice/vendor [:vendor/name + :db/id + {:vendor/primary-contact [:contact/name] + :vendor/address [:address/street1 :address/city :address/state :address/zip]}] + :invoice/expense-accounts [:db/id + :invoice-expense-account/amount + :invoice-expense-account/id + :invoice-expense-account/location + {:invoice-expense-account/account + [:db/id :account/numeric-code :account/name]}] + :invoice/client [:client/name :db/id :client/code :client/locations]}]) ...] + :in $ ?c + :where [?i :invoice/client ?c]] + (d/db conn) + [:client/code (query-params "client-code")]))])) (GET "/payments/export" {:keys [query-params identity]} (assert-admin identity) (statsd/time! [(str "export.time") {:tags #{(client-tag query-params)