diff --git a/config/dev.edn b/config/dev.edn index 6b3941fb..e3834d5b 100644 --- a/config/dev.edn +++ b/config/dev.edn @@ -23,8 +23,7 @@ :yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver" :run-web? true :run-background? true - - + :dd-env "dev" :yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN" :yodlee2-integreat-user "integreat-main" :yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd" diff --git a/config/prod-background-worker.edn b/config/prod-background-worker.edn index b8e81d5d..fe92b7d9 100644 --- a/config/prod-background-worker.edn +++ b/config/prod-background-worker.edn @@ -1,6 +1,7 @@ {:db {:server "database"} :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod" :scheme "https" + :dd-env "prod" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" diff --git a/config/prod.edn b/config/prod.edn index bbd84d22..ee540e57 100644 --- a/config/prod.edn +++ b/config/prod.edn @@ -1,6 +1,7 @@ {:db {:server "database"} :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod" :scheme "https" + :dd-env "prod" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" diff --git a/config/staging-background-worker.edn b/config/staging-background-worker.edn index 8d9ad394..06e9753a 100644 --- a/config/staging-background-worker.edn +++ b/config/staging-background-worker.edn @@ -1,6 +1,7 @@ {:db {:server "database"} :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging" :scheme "https" + :dd-env "staging" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" diff --git a/config/staging.edn b/config/staging.edn index 1842a62c..940be73a 100644 --- a/config/staging.edn +++ b/config/staging.edn @@ -1,6 +1,7 @@ {:db {:server "database"} :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging" :scheme "https" + :dd-env "staging" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" diff --git a/src/clj/auto_ap/background/metrics.clj b/src/clj/auto_ap/background/metrics.clj index 9f81c1e4..67460a86 100644 --- a/src/clj/auto_ap/background/metrics.clj +++ b/src/clj/auto_ap/background/metrics.clj @@ -1,7 +1,8 @@ (ns auto-ap.background.metrics (:require [com.unbounce.dogstatsd.core :as statsd] - [mount.core :as mount])) + [mount.core :as mount] + [config.core :refer [env]])) (mount/defstate metrics-setup - :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "my.app") + :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(:dd-env env)}) :stop nil) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 5c9ace2b..83979fe0 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -1329,7 +1329,7 @@ (defn query-name [q] (try - (str/join "," (map name (:operations (p/operations (p/parse-query schema q))))) + (str/join "__" (map name (:operations (p/operations (p/parse-query schema q))))) (catch Exception e "unknown query"))) @@ -1337,9 +1337,9 @@ ([id q] (query id q nil)) ([id q v] - (statsd/increment "queries") - (statsd/increment (query-name q)) - (statsd/time! [(query-name q) {}] + (statsd/increment "query.count") + (statsd/increment (str "query." (query-name q) ".count")) + (statsd/time! [(str "query." (query-name q) ".time") {}] (lc/with-context {:query q} (log/info "executing query name" (query-name q)) (try diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index 73317da4..1e615e99 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -18,7 +18,8 @@ [datomic.api :as d] [ring.middleware.json :refer [wrap-json-response]] [venia.core :as venia] - [yang.time :refer [time-it]])) + [yang.time :refer [time-it]] + [com.unbounce.dogstatsd.core :as statsd])) (defn wrap-csv-response [handler] (fn [request] (let [response (handler request)] @@ -51,7 +52,9 @@ (context "/" [] (GET "/invoices/export" {:keys [query-params identity] :as request} (assert-admin identity) - (let [query [[:all_invoices + (statsd/increment (str "export.invoice.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.invoice.time") {:tags #{(query-params "client-code")}}] + (let [query [[:all_invoices {:client-code (query-params "client-code") :original-id (query-params "original")} @@ -64,74 +67,82 @@ invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] (list (:all-invoices (:data invoices))))) + ) (GET "/payments/export" {:keys [query-params identity]} (assert-admin identity) - (let [query [[:all_payments - {:client-code (query-params "client-code") - :original-id (query-params "original")} - [:id :check-number :amount :memo :date :status :type :original-id - [:invoices [[:invoice [:id :original-id]] :amount]] - [:bank-account [:number :code :bank-name :bank-code :id]] - [:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]] - [:client [:id :name :code]] - ]]] - payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] - (list (:all-payments (:data payments))))) + (statsd/increment (str "export.payment.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.payment.time") {:tags #{(query-params "client-code")}}] + (let [query [[:all_payments + {:client-code (query-params "client-code") + :original-id (query-params "original")} + [:id :check-number :amount :memo :date :status :type :original-id + [:invoices [[:invoice [:id :original-id]] :amount]] + [:bank-account [:number :code :bank-name :bank-code :id]] + [:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]] + [:client [:id :name :code]] + ]]] + payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] + (list (:all-payments (:data payments)))))) (GET "/sales/export" {:keys [query-params identity :as params]} (assert-admin identity) (println params) - (let [query [[:all_sales_orders - (cond-> {:client-code (query-params "client-code")} - (query-params "after") (assoc :date-range {:start (query-params "after") - :end nil})) - [:id - :location - :external_id - :total - :tip - :tax - :discount - :returns - :service_charge - :date - [:charges [:type_name :total :tip]] - [:line_items [:item_name :total :tax :discount :category]] - [:client [:id :name :code]]]]] - payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)})) - parsedouble #(some-> % Double/parseDouble) ] - (seq (map - (fn [s] - (-> s - (assoc :utc_date (:date s)) - (update :date (fn [d] - (coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles")))))) - (update :total parsedouble) - (update :tax parsedouble) - (update :discount parsedouble) - (update :tip parsedouble) - (update :line-items (fn [lis] - (map - (fn [li] - (-> li - (update :tax parsedouble) - (update :discount parsedouble) - (update :total parsedouble))) - lis))) - (update :charges (fn [charges] - (map - (fn [charge] - (-> charge - (update :tip parsedouble) - (update :total parsedouble))) - charges))))) - (:all-sales-orders (:data payments)))))) + (statsd/increment (str "export.sales.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.sales.time") {:tags #{(query-params "client-code")}}] + (let [query [[:all_sales_orders + (cond-> {:client-code (query-params "client-code")} + (query-params "after") (assoc :date-range {:start (query-params "after") + :end nil})) + [:id + :location + :external_id + :total + :tip + :tax + :discount + :returns + :service_charge + :date + [:charges [:type_name :total :tip]] + [:line_items [:item_name :total :tax :discount :category]] + [:client [:id :name :code]]]]] + payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)})) + parsedouble #(some-> % Double/parseDouble) ] + (seq (map + (fn [s] + (-> s + (assoc :utc_date (:date s)) + (update :date (fn [d] + (coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles")))))) + (update :total parsedouble) + (update :tax parsedouble) + (update :discount parsedouble) + (update :tip parsedouble) + (update :line-items (fn [lis] + (map + (fn [li] + (-> li + (update :tax parsedouble) + (update :discount parsedouble) + (update :total parsedouble))) + lis))) + (update :charges (fn [charges] + (map + (fn [charge] + (-> charge + (update :tip parsedouble) + (update :total parsedouble))) + charges))))) + (:all-sales-orders (:data payments)))))) + ) (GET "/expected-deposit/export" {:keys [query-params identity]} (assert-admin identity) - (let [query [[:all_expected_deposits + (statsd/increment (str "export.expected_deposit.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.expected_deposit.time") {:tags #{(query-params "client-code")}}] + (let [query [[:all_expected_deposits (cond-> {:client-code (query-params "client-code")} (query-params "after") (assoc :date-range {:start (query-params "after") :end nil})) @@ -148,76 +159,81 @@ (-> d (update :fee #(some-> % Double/parseDouble)) (update :total #(some-> % Double/parseDouble)))) - (:all-expected-deposits (:data payments)))))) - + (:all-expected-deposits (:data payments))))))) (GET "/clients/export" {:keys [query-params identity]} (assert-admin identity) (map <-graphql (d-clients/get-all))) (GET "/vendors/export" {:keys [query-params identity]} (assert-admin identity) - (map <-graphql (d-vendors/get-graphql {}))) + (statsd/increment (str "export.vendor.count" )) + (statsd/time! [(str "export.vendor.time") {}] + (map <-graphql (d-vendors/get-graphql {})))) (GET "/ledger/export" {:keys [query-params identity]} (log/info "exporting for " (query-params "client-code")) (assert-admin identity) - (let [[result time] (time-it (transduce (comp - (map #(update % :journal-entry/date to-date)) - (map <-graphql)) - conj - (list) - (first (d-ledger/get-graphql {:count Integer/MAX_VALUE - :client-code (query-params "client-code")}))))] - (info-event "Export completed" - {:time (:time time) - :errors (seq (:errors result))}) - result)) + (statsd/increment (str "export.ledger.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.ledger.time") {:tags #{(query-params "client-code")}}] + (transduce (comp + (map #(update % :journal-entry/date to-date)) + (map <-graphql)) + conj + (list) + (first (d-ledger/get-graphql {:count Integer/MAX_VALUE + :client-code (query-params "client-code")}))))) (GET "/accounts/export" {:keys [query-params identity]} (assert-admin identity) - - (let [client-id (d-clients/code->id (query-params "client-code")) - query [[:accounts - [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]] - all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] + (statsd/increment (str "export.accounts.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.accounts.time") {:tags #{(query-params "client-code")}}] + (let [client-id (d-clients/code->id (query-params "client-code")) + query [[:accounts + [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]] + all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] - (list (transduce - (comp - (filter (fn [a] - (let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))] - (or (= :global (:applicability a)) - (overriden-clients (str client-id)))))) - (map (fn [a] - (let [client->name (reduce - (fn [override co] - (assoc override (str (:id (:client co))) (:name co))) - {} - (:client-overrides a))] - (-> a - (assoc :global-name (:name a)) - (assoc :client-name (client->name (str client-id) (:name a))) - (dissoc :client-overrides)))))) - conj - [] - (:accounts (:data all-accounts)))))) + (list (transduce + (comp + (filter (fn [a] + (let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))] + (or (= :global (:applicability a)) + (overriden-clients (str client-id)))))) + (map (fn [a] + (let [client->name (reduce + (fn [override co] + (assoc override (str (:id (:client co))) (:name co))) + {} + (:client-overrides a))] + (-> a + (assoc :global-name (:name a)) + (assoc :client-name (client->name (str client-id) (:name a))) + (dissoc :client-overrides)))))) + conj + [] + (:accounts (:data all-accounts))))))) (GET "/transactions/export" {:keys [query-params identity]} (assert-admin identity) - (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code") - #_#_:original-id (Integer/parseInt (query-params "original")) - :count Integer/MAX_VALUE})] - - - (map (comp ->graphql (fn [i] - (cond-> i - true (update :transaction/date to-date) - true (update :transaction/post-date to-date) - (:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date) - (:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date)))) - transactions))) + (statsd/increment (str "export.transactions.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.transactions.time") {:tags #{(query-params "client-code")}}] + (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code") + #_#_:original-id (Integer/parseInt (query-params "original")) + :count Integer/MAX_VALUE})] + + + (map (comp ->graphql (fn [i] + (cond-> i + true (update :transaction/date to-date) + true (update :transaction/post-date to-date) + (:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date) + (:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date)))) + transactions))) + ) (GET "/transactions/export2" {:keys [query-params identity]} (assert-admin identity) - (let [db (d/db conn)] + (statsd/increment (str "export.transactions2.count") {:tags #{(query-params "client-code")}}) + (statsd/time! [(str "export.transactions2.time") {:tags #{(query-params "client-code")}}] + (let [db (d/db conn)] (->> (d/query {:query {:find ['?e] :in ['$ '?client-code] @@ -239,12 +255,15 @@ :approval_status (:transaction/approval-status e) :bank_account {:name (:bank-account/name bank-account) :code (:bank-account/code bank-account) - :id (:db/id bank-account)}})))))) + :id (:db/id bank-account)}}))))))) (GET "/raw" {:keys [query-params identity]} (assert-admin identity) (log/info "Executing raw query " (get query-params "query" )) - (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))) + + (statsd/increment (str "export.raw." (get query-params "query" ) ".count")) + (statsd/time! [(str "export.raw." (get query-params "query" ) ".time") {}] + (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))) (defroutes export-routes (routes (wrap-routes api-key-authed-routes diff --git a/src/clj/auto_ap/routes/queries.clj b/src/clj/auto_ap/routes/queries.clj index 0cf9008d..31b69d32 100644 --- a/src/clj/auto_ap/routes/queries.clj +++ b/src/clj/auto_ap/routes/queries.clj @@ -1,34 +1,22 @@ (ns auto-ap.routes.queries - (:require [auto-ap.datomic :refer [conn]] - [auto-ap.datomic.clients :as d-clients] - [auto-ap.datomic.ledger :as d-ledger] - [auto-ap.datomic.transactions :as d-transactions] - [auto-ap.datomic.vendors :as d-vendors] - [clojure.string :as str] - - [unilog.context :as lc] - [buddy.sign.jwt :as jwt] - [auto-ap.graphql :as graphql] - [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]] - [auto-ap.routes.utils :refer [wrap-secure]] - [clojure.tools.logging :as log] - [auto-ap.logging :refer [error-event info-event warn-event]] - [clj-time.coerce :as coerce :refer [to-date]] - [amazonica.aws.s3 :as s3] - [clj-time.core :as time] - [clojure.java.io :as io] + (:require [amazonica.aws.s3 :as s3] + [auto-ap.datomic :refer [conn]] + [auto-ap.graphql.utils :refer [assert-admin]] [clojure.data.csv :as csv] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.logging :as log] + [com.unbounce.dogstatsd.core :as statsd] + [compojure.core + :refer + [context defroutes GET POST PUT routes wrap-routes]] [config.core :refer [env]] - [compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]] [datomic.api :as d] [ring.middleware.json :refer [wrap-json-response]] - [venia.core :as venia] - [yang.time :refer [time-it]] [ring.util.request :refer [body-string]] - ) + [unilog.context :as lc]) (:import java.util.UUID)) - (defn wrap-csv-response [handler] (fn [request] (let [response (handler request)] @@ -101,8 +89,9 @@ :key (str/replace (:key o) #"^queries\/" "")})))})) (GET "/:query-id/results/json" {:keys [query-params identity params]} - {:body (execute-query query-params params)}) - )) + (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}}) + (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}] + {:body (execute-query query-params params)})))) (def raw-routes @@ -119,7 +108,9 @@ (def csv-routes (context "/queries" [] (GET "/:query-id/results/csv" {:keys [query-params identity params]} - {:body (execute-query query-params params)}))) + (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}}) + (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}] + {:body (execute-query query-params params)})))) (defroutes query2-routes (routes raw-routes