Datadog stats

config/dev.edn

@@ -23,8 +23,7 @@
  :yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver"
  :run-web? true
  :run-background? true
-
+ :dd-env "dev"
-
  :yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN"
  :yodlee2-integreat-user "integreat-main"
  :yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd"

config/prod-background-worker.edn

@@ -1,6 +1,7 @@
 {:db {:server "database"}
  :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
  :scheme "https"
+ :dd-env "prod"
  :jwt-secret "auto ap invoices are awesome"
  :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" 
  :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" 

config/prod.edn

@@ -1,6 +1,7 @@
 {:db {:server "database"}
  :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
  :scheme "https"
+ :dd-env "prod"
  :jwt-secret "auto ap invoices are awesome"
  :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" 
  :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" 

config/staging-background-worker.edn

@@ -1,6 +1,7 @@
 {:db {:server "database"}
  :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
  :scheme "https"
+ :dd-env "staging"
  :jwt-secret "auto ap invoices are awesome"
  :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" 
  :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" 

config/staging.edn

@@ -1,6 +1,7 @@
 {:db {:server "database"}
  :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
  :scheme "https"
+ :dd-env "staging"
  :jwt-secret "auto ap invoices are awesome"
  :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" 
  :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" 

src/clj/auto_ap/background/metrics.clj

@@ -1,7 +1,8 @@
 (ns auto-ap.background.metrics
   (:require [com.unbounce.dogstatsd.core :as statsd]
-            [mount.core :as mount]))
+            [mount.core :as mount]
+            [config.core :refer [env]]))
 
 (mount/defstate metrics-setup
-  :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "my.app")
+  :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(:dd-env env)})
   :stop nil)

src/clj/auto_ap/graphql.clj

@@ -1329,7 +1329,7 @@
 
 (defn query-name [q]
   (try 
-    (str/join "," (map name (:operations (p/operations (p/parse-query schema q)))))
+    (str/join "__" (map name (:operations (p/operations (p/parse-query schema q)))))
     (catch Exception e
       "unknown query")))
 
@@ -1337,9 +1337,9 @@
   ([id q]
    (query id q nil))
   ([id q v]
-   (statsd/increment "queries")
+   (statsd/increment "query.count")
-   (statsd/increment (query-name q))
+   (statsd/increment (str "query." (query-name q) ".count"))
-   (statsd/time! [(query-name q) {}]
+   (statsd/time! [(str "query." (query-name q) ".time") {}]
      (lc/with-context {:query q}
        (log/info "executing query name" (query-name q))
        (try 

src/clj/auto_ap/routes/exports.clj

@@ -18,7 +18,8 @@
             [datomic.api :as d]
             [ring.middleware.json :refer [wrap-json-response]]
             [venia.core :as venia]
-            [yang.time :refer [time-it]]))
+            [yang.time :refer [time-it]]
+            [com.unbounce.dogstatsd.core :as statsd]))
 (defn wrap-csv-response [handler]
   (fn [request]
     (let [response (handler request)]
@@ -51,6 +52,8 @@
   (context "/" []
            (GET "/invoices/export" {:keys [query-params identity] :as request}
                 (assert-admin identity)
+                (statsd/increment (str "export.invoice.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.invoice.time") {:tags #{(query-params "client-code")}}]
                   (let [query [[:all_invoices
                               {:client-code (query-params "client-code")
                                :original-id (query-params "original")} 
@@ -64,8 +67,11 @@
                       invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
                   
                   (list (:all-invoices (:data invoices)))))
+                )
            (GET "/payments/export" {:keys [query-params identity]}
                 (assert-admin identity)
+                (statsd/increment (str "export.payment.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.payment.time") {:tags #{(query-params "client-code")}}]
                   (let [query [[:all_payments
                                 {:client-code (query-params "client-code")
                                  :original-id (query-params "original")} 
@@ -76,11 +82,13 @@
                                  [:client [:id :name :code]]
                                  ]]]
                         payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
-                  (list (:all-payments (:data payments)))))
+                    (list (:all-payments (:data payments))))))
 
            (GET "/sales/export" {:keys [query-params identity :as params]}
                 (assert-admin identity)
                 (println params)
+                (statsd/increment (str "export.sales.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.sales.time") {:tags #{(query-params "client-code")}}]
                   (let [query [[:all_sales_orders
                                 (cond-> {:client-code (query-params "client-code")}
                                   (query-params "after") (assoc :date-range {:start (query-params "after")
@@ -126,11 +134,14 @@
                                                           (update :total parsedouble)))
                                                     charges)))))
                           (:all-sales-orders (:data payments))))))
+                )
 
            
 
            (GET "/expected-deposit/export" {:keys [query-params identity]}
                 (assert-admin identity)
+                (statsd/increment (str "export.expected_deposit.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.expected_deposit.time") {:tags #{(query-params "client-code")}}]
                   (let [query [[:all_expected_deposits
                               (cond-> {:client-code (query-params "client-code")}
                                 (query-params "after") (assoc :date-range {:start (query-params "after")
@@ -148,33 +159,33 @@
                           (-> d
                               (update  :fee #(some-> % Double/parseDouble))
                               (update  :total #(some-> % Double/parseDouble))))
-                        (:all-expected-deposits (:data payments))))))
+                        (:all-expected-deposits (:data payments)))))))
-
            (GET "/clients/export" {:keys [query-params identity]}
                 (assert-admin identity)
                 (map <-graphql (d-clients/get-all)))
 
            (GET "/vendors/export" {:keys [query-params identity]}
                 (assert-admin identity)
-                (map <-graphql (d-vendors/get-graphql {})))
+                (statsd/increment (str "export.vendor.count" ))
+                (statsd/time! [(str "export.vendor.time") {}]
+                  (map <-graphql (d-vendors/get-graphql {}))))
            (GET "/ledger/export" {:keys [query-params identity]}
                 (log/info "exporting for " (query-params "client-code"))
                 (assert-admin identity)
-                (let [[result time] (time-it (transduce (comp
+                (statsd/increment (str "export.ledger.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.ledger.time") {:tags #{(query-params "client-code")}}]
+                  (transduce (comp
                               (map #(update % :journal-entry/date to-date))
                               (map <-graphql))
                              conj
                              (list)
                              (first (d-ledger/get-graphql {:count Integer/MAX_VALUE
-                                                                                      :client-code (query-params "client-code")}))))]
+                                                           :client-code (query-params "client-code")})))))
-                  (info-event "Export completed"
-                              {:time (:time time)
-                               :errors (seq (:errors result))})
-                  result))
 
            (GET "/accounts/export" {:keys [query-params identity]}
                 (assert-admin identity)
-                
+                (statsd/increment (str "export.accounts.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.accounts.time") {:tags #{(query-params "client-code")}}]
                   (let [client-id (d-clients/code->id (query-params "client-code"))
                         query [[:accounts
                                 [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
@@ -198,10 +209,12 @@
                                          (dissoc :client-overrides))))))
                            conj
                            []
-                         (:accounts (:data all-accounts))))))
+                           (:accounts (:data all-accounts)))))))
 
            (GET "/transactions/export" {:keys [query-params identity]}
                 (assert-admin identity)
+                (statsd/increment (str "export.transactions.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.transactions.time") {:tags #{(query-params "client-code")}}]
                   (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code")
                                                                     #_#_:original-id (Integer/parseInt (query-params "original")) 
                                                                     :count Integer/MAX_VALUE})]
@@ -214,9 +227,12 @@
                                              (:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date)
                                              (:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date))))
                          transactions)))
+                )
 
            (GET "/transactions/export2" {:keys [query-params identity]}
                 (assert-admin identity)
+                (statsd/increment (str "export.transactions2.count") {:tags #{(query-params "client-code")}})
+                (statsd/time! [(str "export.transactions2.time") {:tags #{(query-params "client-code")}}]
                   (let [db (d/db conn)]
                   (->> 
                    (d/query {:query {:find ['?e]
@@ -239,12 +255,15 @@
                              :approval_status (:transaction/approval-status e)
                              :bank_account {:name (:bank-account/name bank-account)
                                             :code (:bank-account/code bank-account)
-                                            :id (:db/id bank-account)}}))))))
+                                            :id (:db/id bank-account)}})))))))
 
            (GET "/raw" {:keys [query-params identity]}
                 (assert-admin identity)
                 (log/info "Executing raw query " (get query-params "query" ))
-                (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))
+
+                (statsd/increment (str "export.raw." (get query-params "query" ) ".count"))
+                (statsd/time! [(str "export.raw." (get query-params "query" ) ".time") {}]
+                              (into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))))
 (defroutes export-routes
   (routes 
    (wrap-routes api-key-authed-routes

src/clj/auto_ap/routes/queries.clj

@@ -1,34 +1,22 @@
 (ns auto-ap.routes.queries
-  (:require [auto-ap.datomic :refer [conn]]
+  (:require [amazonica.aws.s3 :as s3]
-            [auto-ap.datomic.clients :as d-clients]
+            [auto-ap.datomic :refer [conn]]
-            [auto-ap.datomic.ledger :as d-ledger]
+            [auto-ap.graphql.utils :refer [assert-admin]]
-            [auto-ap.datomic.transactions :as d-transactions]
-            [auto-ap.datomic.vendors :as d-vendors]
-            [clojure.string :as str]
-
-            [unilog.context :as lc]
-            [buddy.sign.jwt :as jwt]
-            [auto-ap.graphql :as graphql]
-            [auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
-            [auto-ap.routes.utils :refer [wrap-secure]]
-            [clojure.tools.logging :as log]
-            [auto-ap.logging :refer [error-event info-event warn-event]]
-            [clj-time.coerce :as coerce :refer [to-date]]
-            [amazonica.aws.s3 :as s3]
-            [clj-time.core :as time]
-            [clojure.java.io :as io]
             [clojure.data.csv :as csv]
+            [clojure.java.io :as io]
+            [clojure.string :as str]
+            [clojure.tools.logging :as log]
+            [com.unbounce.dogstatsd.core :as statsd]
+            [compojure.core
+             :refer
+             [context defroutes GET POST PUT routes wrap-routes]]
             [config.core :refer [env]]
-            [compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]]
             [datomic.api :as d]
             [ring.middleware.json :refer [wrap-json-response]]
-            [venia.core :as venia]
-            [yang.time :refer [time-it]]
             [ring.util.request :refer [body-string]]
-            )
+            [unilog.context :as lc])
   (:import java.util.UUID))
 
-
 (defn wrap-csv-response [handler]
   (fn [request]
     (let [response (handler request)]
@@ -101,8 +89,9 @@
                                       :key (str/replace (:key o) #"^queries\/" "")})))}))
 
            (GET "/:query-id/results/json" {:keys [query-params identity params]}
-                {:body (execute-query query-params params)})
+                (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
-           ))
+                (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
+                  {:body (execute-query query-params params)}))))
 
 
 (def raw-routes 
@@ -119,7 +108,9 @@
 (def csv-routes 
   (context "/queries" []
            (GET "/:query-id/results/csv" {:keys [query-params identity params]}
-                {:body (execute-query query-params params)})))
+                (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
+                (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
+                              {:body (execute-query query-params params)}))))
 (defroutes query2-routes
   (routes 
    raw-routes