Datadog stats

2022-01-04 11:53:47 -08:00
parent e6229ef5d0
commit 6ca4f0fb60
9 changed files with 160 additions and 146 deletions
--- a/config/dev.edn
+++ b/config/dev.edn
@@ -23,8 +23,7 @@
 :yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver"
 :run-web? true
 :run-background? true
-
-
+ :dd-env "dev"
 :yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN"
 :yodlee2-integreat-user "integreat-main"
 :yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd"
--- a/config/prod-background-worker.edn
+++ b/config/prod-background-worker.edn
@@ -1,6 +1,7 @@
 {:db {:server "database"}
 :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
 :scheme "https"
+ :dd-env "prod"
 :jwt-secret "auto ap invoices are awesome"
 :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" 
 :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" 
--- a/config/prod.edn
+++ b/config/prod.edn
@@ -1,6 +1,7 @@
 {:db {:server "database"}
 :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
 :scheme "https"
+ :dd-env "prod"
 :jwt-secret "auto ap invoices are awesome"
 :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" 
 :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" 
--- a/config/staging-background-worker.edn
+++ b/config/staging-background-worker.edn
@@ -1,6 +1,7 @@
 {:db {:server "database"}
 :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
 :scheme "https"
+ :dd-env "staging"
 :jwt-secret "auto ap invoices are awesome"
 :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" 
 :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" 
--- a/config/staging.edn
+++ b/config/staging.edn
@@ -1,6 +1,7 @@
 {:db {:server "database"}
 :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
 :scheme "https"
+ :dd-env "staging"
 :jwt-secret "auto ap invoices are awesome"
 :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" 
 :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" 
--- a/src/clj/auto_ap/background/metrics.clj
+++ b/src/clj/auto_ap/background/metrics.clj
@@ -1,7 +1,8 @@
 (ns auto-ap.background.metrics
  (:require [com.unbounce.dogstatsd.core :as statsd]
-            [mount.core :as mount]))
+            [mount.core :as mount]
+            [config.core :refer [env]]))

 (mount/defstate metrics-setup
-  :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "my.app")
+  :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(:dd-env env)})
  :stop nil)
--- a/src/clj/auto_ap/graphql.clj
+++ b/src/clj/auto_ap/graphql.clj
@@ -1329,7 +1329,7 @@

 (defn query-name [q]
  (try 
-    (str/join "," (map name (:operations (p/operations (p/parse-query schema q)))))
+    (str/join "__" (map name (:operations (p/operations (p/parse-query schema q)))))
    (catch Exception e
      "unknown query")))

@@ -1337,9 +1337,9 @@
  ([id q]
   (query id q nil))
  ([id q v]
-   (statsd/increment "queries")
-   (statsd/increment (query-name q))
-   (statsd/time! [(query-name q) {}]
+   (statsd/increment "query.count")
+   (statsd/increment (str "query." (query-name q) ".count"))
+   (statsd/time! [(str "query." (query-name q) ".time") {}]
     (lc/with-context {:query q}
       (log/info "executing query name" (query-name q))
       (try 
--- a/src/clj/auto_ap/routes/exports.clj
+++ b/src/clj/auto_ap/routes/exports.clj
@@ -18,7 +18,8 @@
            [datomic.api :as d]
            [ring.middleware.json :refer [wrap-json-response]]
            [venia.core :as venia]
-            [yang.time :refer [time-it]]))
+            [yang.time :refer [time-it]]
+            [com.unbounce.dogstatsd.core :as statsd]))
 (defn wrap-csv-response [handler]
  (fn [request]
    (let [response (handler request)]
@@ -51,6 +52,8 @@
  (context "/" []
           (GET "/invoices/export" {:keys [query-params identity] :as request}
                (assert-admin identity)
+                (statsd/increment (str "export.invoice.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.invoice.time") {:tags #{(query-params "client-code")}}]
                  (let [query [[:all_invoices
                              {:client-code (query-params "client-code")
                               :original-id (query-params "original")} 
@@ -64,8 +67,11 @@
                      invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
                  
                  (list (:all-invoices (:data invoices)))))
+                )
           (GET "/payments/export" {:keys [query-params identity]}
                (assert-admin identity)
+                (statsd/increment (str "export.payment.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.payment.time") {:tags #{(query-params "client-code")}}]
                  (let [query [[:all_payments
                                {:client-code (query-params "client-code")
                                 :original-id (query-params "original")} 
@@ -76,11 +82,13 @@
                                 [:client [:id :name :code]]
                                 ]]]
                        payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
-                  (list (:all-payments (:data payments)))))
+                    (list (:all-payments (:data payments))))))

           (GET "/sales/export" {:keys [query-params identity :as params]}
                (assert-admin identity)
                (println params)
+                (statsd/increment (str "export.sales.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.sales.time") {:tags #{(query-params "client-code")}}]
                  (let [query [[:all_sales_orders
                                (cond-> {:client-code (query-params "client-code")}
                                  (query-params "after") (assoc :date-range {:start (query-params "after")
@@ -126,11 +134,14 @@
                                                          (update :total parsedouble)))
                                                    charges)))))
                          (:all-sales-orders (:data payments))))))
+                )

           

           (GET "/expected-deposit/export" {:keys [query-params identity]}
                (assert-admin identity)
+                (statsd/increment (str "export.expected_deposit.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.expected_deposit.time") {:tags #{(query-params "client-code")}}]
                  (let [query [[:all_expected_deposits
                              (cond-> {:client-code (query-params "client-code")}
                                (query-params "after") (assoc :date-range {:start (query-params "after")
@@ -148,33 +159,33 @@
                          (-> d
                              (update  :fee #(some-> % Double/parseDouble))
                              (update  :total #(some-> % Double/parseDouble))))
-                        (:all-expected-deposits (:data payments))))))
-
+                        (:all-expected-deposits (:data payments)))))))
           (GET "/clients/export" {:keys [query-params identity]}
                (assert-admin identity)
                (map <-graphql (d-clients/get-all)))

           (GET "/vendors/export" {:keys [query-params identity]}
                (assert-admin identity)
-                (map <-graphql (d-vendors/get-graphql {})))
+                (statsd/increment (str "export.vendor.count" ))
+                (statsd/time! [(str "export.vendor.time") {}]
+                  (map <-graphql (d-vendors/get-graphql {}))))
           (GET "/ledger/export" {:keys [query-params identity]}
                (log/info "exporting for " (query-params "client-code"))
                (assert-admin identity)
-                (let [[result time] (time-it (transduce (comp
+                (statsd/increment (str "export.ledger.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.ledger.time") {:tags #{(query-params "client-code")}}]
+                  (transduce (comp
                              (map #(update % :journal-entry/date to-date))
                              (map <-graphql))
                             conj
                             (list)
                             (first (d-ledger/get-graphql {:count Integer/MAX_VALUE
-                                                                                      :client-code (query-params "client-code")}))))]
-                  (info-event "Export completed"
-                              {:time (:time time)
-                               :errors (seq (:errors result))})
-                  result))
+                                                           :client-code (query-params "client-code")})))))

           (GET "/accounts/export" {:keys [query-params identity]}
                (assert-admin identity)
-                
+                (statsd/increment (str "export.accounts.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.accounts.time") {:tags #{(query-params "client-code")}}]
                  (let [client-id (d-clients/code->id (query-params "client-code"))
                        query [[:accounts
                                [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
@@ -198,10 +209,12 @@
                                         (dissoc :client-overrides))))))
                           conj
                           []
-                         (:accounts (:data all-accounts))))))
+                           (:accounts (:data all-accounts)))))))

           (GET "/transactions/export" {:keys [query-params identity]}
                (assert-admin identity)
+                (statsd/increment (str "export.transactions.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.transactions.time") {:tags #{(query-params "client-code")}}]
                  (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code")
                                                                    #_#_:original-id (Integer/parseInt (query-params "original")) 
                                                                    :count Integer/MAX_VALUE})]
@@ -214,9 +227,12 @@
                                             (:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date)
                                             (:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date))))
                         transactions)))
+                )

           (GET "/transactions/export2" {:keys [query-params identity]}
                (assert-admin identity)
+                (statsd/increment (str "export.transactions2.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.transactions2.time") {:tags #{(query-params "client-code")}}]
                  (let [db (d/db conn)]
                  (->> 
                   (d/query {:query {:find ['?e]
@@ -239,12 +255,15 @@
                             :approval_status (:transaction/approval-status e)
                             :bank_account {:name (:bank-account/name bank-account)
                                            :code (:bank-account/code bank-account)
-                                            :id (:db/id bank-account)}}))))))
+                                            :id (:db/id bank-account)}})))))))

           (GET "/raw" {:keys [query-params identity]}
                (assert-admin identity)
                (log/info "Executing raw query " (get query-params "query" ))
-                (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))
+
+                (statsd/increment (str "export.raw." (get query-params "query" ) ".count"))
+                (statsd/time! [(str "export.raw." (get query-params "query" ) ".time") {}]
+                              (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))))
 (defroutes export-routes
  (routes 
   (wrap-routes api-key-authed-routes
--- a/src/clj/auto_ap/routes/queries.clj
+++ b/src/clj/auto_ap/routes/queries.clj
@@ -1,34 +1,22 @@
 (ns auto-ap.routes.queries
-  (:require [auto-ap.datomic :refer [conn]]
-            [auto-ap.datomic.clients :as d-clients]
-            [auto-ap.datomic.ledger :as d-ledger]
-            [auto-ap.datomic.transactions :as d-transactions]
-            [auto-ap.datomic.vendors :as d-vendors]
-            [clojure.string :as str]
-
-            [unilog.context :as lc]
-            [buddy.sign.jwt :as jwt]
-            [auto-ap.graphql :as graphql]
-            [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
-            [auto-ap.routes.utils :refer [wrap-secure]]
-            [clojure.tools.logging :as log]
-            [auto-ap.logging :refer [error-event info-event warn-event]]
-            [clj-time.coerce :as coerce :refer [to-date]]
-            [amazonica.aws.s3 :as s3]
-            [clj-time.core :as time]
-            [clojure.java.io :as io]
+  (:require [amazonica.aws.s3 :as s3]
+            [auto-ap.datomic :refer [conn]]
+            [auto-ap.graphql.utils :refer [assert-admin]]
            [clojure.data.csv :as csv]
+            [clojure.java.io :as io]
+            [clojure.string :as str]
+            [clojure.tools.logging :as log]
+            [com.unbounce.dogstatsd.core :as statsd]
+            [compojure.core
+             :refer
+             [context defroutes GET POST PUT routes wrap-routes]]
            [config.core :refer [env]]
-            [compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]]
            [datomic.api :as d]
            [ring.middleware.json :refer [wrap-json-response]]
-            [venia.core :as venia]
-            [yang.time :refer [time-it]]
            [ring.util.request :refer [body-string]]
-            )
+            [unilog.context :as lc])
  (:import java.util.UUID))

-
 (defn wrap-csv-response [handler]
  (fn [request]
    (let [response (handler request)]
@@ -101,8 +89,9 @@
                                      :key (str/replace (:key o) #"^queries\/" "")})))}))

           (GET "/:query-id/results/json" {:keys [query-params identity params]}
-                {:body (execute-query query-params params)})
-           ))
+                (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
+                (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
+                  {:body (execute-query query-params params)}))))


 (def raw-routes 
@@ -119,7 +108,9 @@
 (def csv-routes 
  (context "/queries" []
           (GET "/:query-id/results/csv" {:keys [query-params identity params]}
-                {:body (execute-query query-params params)})))
+                (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
+                (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
+                              {:body (execute-query query-params params)}))))
 (defroutes query2-routes
  (routes 
   raw-routes