Datadog stats
This commit is contained in:
Bryce Covert
@@ -23,8 +23,7 @@
|
||||
:yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver"
|
||||
:run-web? true
|
||||
:run-background? true
|
||||
|
||||
|
||||
:dd-env "dev"
|
||||
:yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN"
|
||||
:yodlee2-integreat-user "integreat-main"
|
||||
:yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{:db {:server "database"}
|
||||
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
|
||||
:scheme "https"
|
||||
:dd-env "prod"
|
||||
:jwt-secret "auto ap invoices are awesome"
|
||||
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
|
||||
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{:db {:server "database"}
|
||||
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
|
||||
:scheme "https"
|
||||
:dd-env "prod"
|
||||
:jwt-secret "auto ap invoices are awesome"
|
||||
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
|
||||
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{:db {:server "database"}
|
||||
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
|
||||
:scheme "https"
|
||||
:dd-env "staging"
|
||||
:jwt-secret "auto ap invoices are awesome"
|
||||
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging"
|
||||
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{:db {:server "database"}
|
||||
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
|
||||
:scheme "https"
|
||||
:dd-env "staging"
|
||||
:jwt-secret "auto ap invoices are awesome"
|
||||
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging"
|
||||
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging"
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
(ns auto-ap.background.metrics
|
||||
(:require [com.unbounce.dogstatsd.core :as statsd]
|
||||
[mount.core :as mount]))
|
||||
[mount.core :as mount]
|
||||
[config.core :refer [env]]))
|
||||
|
||||
(mount/defstate metrics-setup
|
||||
:start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "my.app")
|
||||
:start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(:dd-env env)})
|
||||
:stop nil)
|
||||
|
||||
@@ -1329,7 +1329,7 @@
|
||||
|
||||
(defn query-name [q]
|
||||
(try
|
||||
(str/join "," (map name (:operations (p/operations (p/parse-query schema q)))))
|
||||
(str/join "__" (map name (:operations (p/operations (p/parse-query schema q)))))
|
||||
(catch Exception e
|
||||
"unknown query")))
|
||||
|
||||
@@ -1337,9 +1337,9 @@
|
||||
([id q]
|
||||
(query id q nil))
|
||||
([id q v]
|
||||
(statsd/increment "queries")
|
||||
(statsd/increment (query-name q))
|
||||
(statsd/time! [(query-name q) {}]
|
||||
(statsd/increment "query.count")
|
||||
(statsd/increment (str "query." (query-name q) ".count"))
|
||||
(statsd/time! [(str "query." (query-name q) ".time") {}]
|
||||
(lc/with-context {:query q}
|
||||
(log/info "executing query name" (query-name q))
|
||||
(try
|
||||
|
||||
@@ -18,7 +18,8 @@
|
||||
[datomic.api :as d]
|
||||
[ring.middleware.json :refer [wrap-json-response]]
|
||||
[venia.core :as venia]
|
||||
[yang.time :refer [time-it]]))
|
||||
[yang.time :refer [time-it]]
|
||||
[com.unbounce.dogstatsd.core :as statsd]))
|
||||
(defn wrap-csv-response [handler]
|
||||
(fn [request]
|
||||
(let [response (handler request)]
|
||||
@@ -51,7 +52,9 @@
|
||||
(context "/" []
|
||||
(GET "/invoices/export" {:keys [query-params identity] :as request}
|
||||
(assert-admin identity)
|
||||
(let [query [[:all_invoices
|
||||
(statsd/increment (str "export.invoice.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.invoice.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [query [[:all_invoices
|
||||
{:client-code (query-params "client-code")
|
||||
:original-id (query-params "original")}
|
||||
|
||||
@@ -64,74 +67,82 @@
|
||||
invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
|
||||
(list (:all-invoices (:data invoices)))))
|
||||
)
|
||||
(GET "/payments/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(let [query [[:all_payments
|
||||
{:client-code (query-params "client-code")
|
||||
:original-id (query-params "original")}
|
||||
[:id :check-number :amount :memo :date :status :type :original-id
|
||||
[:invoices [[:invoice [:id :original-id]] :amount]]
|
||||
[:bank-account [:number :code :bank-name :bank-code :id]]
|
||||
[:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]]
|
||||
[:client [:id :name :code]]
|
||||
]]]
|
||||
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
(list (:all-payments (:data payments)))))
|
||||
(statsd/increment (str "export.payment.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.payment.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [query [[:all_payments
|
||||
{:client-code (query-params "client-code")
|
||||
:original-id (query-params "original")}
|
||||
[:id :check-number :amount :memo :date :status :type :original-id
|
||||
[:invoices [[:invoice [:id :original-id]] :amount]]
|
||||
[:bank-account [:number :code :bank-name :bank-code :id]]
|
||||
[:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]]
|
||||
[:client [:id :name :code]]
|
||||
]]]
|
||||
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
(list (:all-payments (:data payments))))))
|
||||
|
||||
(GET "/sales/export" {:keys [query-params identity :as params]}
|
||||
(assert-admin identity)
|
||||
(println params)
|
||||
(let [query [[:all_sales_orders
|
||||
(cond-> {:client-code (query-params "client-code")}
|
||||
(query-params "after") (assoc :date-range {:start (query-params "after")
|
||||
:end nil}))
|
||||
[:id
|
||||
:location
|
||||
:external_id
|
||||
:total
|
||||
:tip
|
||||
:tax
|
||||
:discount
|
||||
:returns
|
||||
:service_charge
|
||||
:date
|
||||
[:charges [:type_name :total :tip]]
|
||||
[:line_items [:item_name :total :tax :discount :category]]
|
||||
[:client [:id :name :code]]]]]
|
||||
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))
|
||||
parsedouble #(some-> % Double/parseDouble) ]
|
||||
(seq (map
|
||||
(fn [s]
|
||||
(-> s
|
||||
(assoc :utc_date (:date s))
|
||||
(update :date (fn [d]
|
||||
(coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))))))
|
||||
(update :total parsedouble)
|
||||
(update :tax parsedouble)
|
||||
(update :discount parsedouble)
|
||||
(update :tip parsedouble)
|
||||
(update :line-items (fn [lis]
|
||||
(map
|
||||
(fn [li]
|
||||
(-> li
|
||||
(update :tax parsedouble)
|
||||
(update :discount parsedouble)
|
||||
(update :total parsedouble)))
|
||||
lis)))
|
||||
(update :charges (fn [charges]
|
||||
(map
|
||||
(fn [charge]
|
||||
(-> charge
|
||||
(update :tip parsedouble)
|
||||
(update :total parsedouble)))
|
||||
charges)))))
|
||||
(:all-sales-orders (:data payments))))))
|
||||
(statsd/increment (str "export.sales.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.sales.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [query [[:all_sales_orders
|
||||
(cond-> {:client-code (query-params "client-code")}
|
||||
(query-params "after") (assoc :date-range {:start (query-params "after")
|
||||
:end nil}))
|
||||
[:id
|
||||
:location
|
||||
:external_id
|
||||
:total
|
||||
:tip
|
||||
:tax
|
||||
:discount
|
||||
:returns
|
||||
:service_charge
|
||||
:date
|
||||
[:charges [:type_name :total :tip]]
|
||||
[:line_items [:item_name :total :tax :discount :category]]
|
||||
[:client [:id :name :code]]]]]
|
||||
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))
|
||||
parsedouble #(some-> % Double/parseDouble) ]
|
||||
(seq (map
|
||||
(fn [s]
|
||||
(-> s
|
||||
(assoc :utc_date (:date s))
|
||||
(update :date (fn [d]
|
||||
(coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))))))
|
||||
(update :total parsedouble)
|
||||
(update :tax parsedouble)
|
||||
(update :discount parsedouble)
|
||||
(update :tip parsedouble)
|
||||
(update :line-items (fn [lis]
|
||||
(map
|
||||
(fn [li]
|
||||
(-> li
|
||||
(update :tax parsedouble)
|
||||
(update :discount parsedouble)
|
||||
(update :total parsedouble)))
|
||||
lis)))
|
||||
(update :charges (fn [charges]
|
||||
(map
|
||||
(fn [charge]
|
||||
(-> charge
|
||||
(update :tip parsedouble)
|
||||
(update :total parsedouble)))
|
||||
charges)))))
|
||||
(:all-sales-orders (:data payments))))))
|
||||
)
|
||||
|
||||
|
||||
|
||||
(GET "/expected-deposit/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(let [query [[:all_expected_deposits
|
||||
(statsd/increment (str "export.expected_deposit.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.expected_deposit.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [query [[:all_expected_deposits
|
||||
(cond-> {:client-code (query-params "client-code")}
|
||||
(query-params "after") (assoc :date-range {:start (query-params "after")
|
||||
:end nil}))
|
||||
@@ -148,76 +159,81 @@
|
||||
(-> d
|
||||
(update :fee #(some-> % Double/parseDouble))
|
||||
(update :total #(some-> % Double/parseDouble))))
|
||||
(:all-expected-deposits (:data payments))))))
|
||||
|
||||
(:all-expected-deposits (:data payments)))))))
|
||||
(GET "/clients/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(map <-graphql (d-clients/get-all)))
|
||||
|
||||
(GET "/vendors/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(map <-graphql (d-vendors/get-graphql {})))
|
||||
(statsd/increment (str "export.vendor.count" ))
|
||||
(statsd/time! [(str "export.vendor.time") {}]
|
||||
(map <-graphql (d-vendors/get-graphql {}))))
|
||||
(GET "/ledger/export" {:keys [query-params identity]}
|
||||
(log/info "exporting for " (query-params "client-code"))
|
||||
(assert-admin identity)
|
||||
(let [[result time] (time-it (transduce (comp
|
||||
(map #(update % :journal-entry/date to-date))
|
||||
(map <-graphql))
|
||||
conj
|
||||
(list)
|
||||
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE
|
||||
:client-code (query-params "client-code")}))))]
|
||||
(info-event "Export completed"
|
||||
{:time (:time time)
|
||||
:errors (seq (:errors result))})
|
||||
result))
|
||||
(statsd/increment (str "export.ledger.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.ledger.time") {:tags #{(query-params "client-code")}}]
|
||||
(transduce (comp
|
||||
(map #(update % :journal-entry/date to-date))
|
||||
(map <-graphql))
|
||||
conj
|
||||
(list)
|
||||
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE
|
||||
:client-code (query-params "client-code")})))))
|
||||
|
||||
(GET "/accounts/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
|
||||
(let [client-id (d-clients/code->id (query-params "client-code"))
|
||||
query [[:accounts
|
||||
[:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
|
||||
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
(statsd/increment (str "export.accounts.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.accounts.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [client-id (d-clients/code->id (query-params "client-code"))
|
||||
query [[:accounts
|
||||
[:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
|
||||
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
|
||||
|
||||
(list (transduce
|
||||
(comp
|
||||
(filter (fn [a]
|
||||
(let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))]
|
||||
(or (= :global (:applicability a))
|
||||
(overriden-clients (str client-id))))))
|
||||
(map (fn [a]
|
||||
(let [client->name (reduce
|
||||
(fn [override co]
|
||||
(assoc override (str (:id (:client co))) (:name co)))
|
||||
{}
|
||||
(:client-overrides a))]
|
||||
(-> a
|
||||
(assoc :global-name (:name a))
|
||||
(assoc :client-name (client->name (str client-id) (:name a)))
|
||||
(dissoc :client-overrides))))))
|
||||
conj
|
||||
[]
|
||||
(:accounts (:data all-accounts))))))
|
||||
(list (transduce
|
||||
(comp
|
||||
(filter (fn [a]
|
||||
(let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))]
|
||||
(or (= :global (:applicability a))
|
||||
(overriden-clients (str client-id))))))
|
||||
(map (fn [a]
|
||||
(let [client->name (reduce
|
||||
(fn [override co]
|
||||
(assoc override (str (:id (:client co))) (:name co)))
|
||||
{}
|
||||
(:client-overrides a))]
|
||||
(-> a
|
||||
(assoc :global-name (:name a))
|
||||
(assoc :client-name (client->name (str client-id) (:name a)))
|
||||
(dissoc :client-overrides))))))
|
||||
conj
|
||||
[]
|
||||
(:accounts (:data all-accounts)))))))
|
||||
|
||||
(GET "/transactions/export" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code")
|
||||
#_#_:original-id (Integer/parseInt (query-params "original"))
|
||||
:count Integer/MAX_VALUE})]
|
||||
|
||||
|
||||
(map (comp ->graphql (fn [i]
|
||||
(cond-> i
|
||||
true (update :transaction/date to-date)
|
||||
true (update :transaction/post-date to-date)
|
||||
(:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date)
|
||||
(:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date))))
|
||||
transactions)))
|
||||
(statsd/increment (str "export.transactions.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.transactions.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code")
|
||||
#_#_:original-id (Integer/parseInt (query-params "original"))
|
||||
:count Integer/MAX_VALUE})]
|
||||
|
||||
|
||||
(map (comp ->graphql (fn [i]
|
||||
(cond-> i
|
||||
true (update :transaction/date to-date)
|
||||
true (update :transaction/post-date to-date)
|
||||
(:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date)
|
||||
(:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date))))
|
||||
transactions)))
|
||||
)
|
||||
|
||||
(GET "/transactions/export2" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(let [db (d/db conn)]
|
||||
(statsd/increment (str "export.transactions2.count") {:tags #{(query-params "client-code")}})
|
||||
(statsd/time! [(str "export.transactions2.time") {:tags #{(query-params "client-code")}}]
|
||||
(let [db (d/db conn)]
|
||||
(->>
|
||||
(d/query {:query {:find ['?e]
|
||||
:in ['$ '?client-code]
|
||||
@@ -239,12 +255,15 @@
|
||||
:approval_status (:transaction/approval-status e)
|
||||
:bank_account {:name (:bank-account/name bank-account)
|
||||
:code (:bank-account/code bank-account)
|
||||
:id (:db/id bank-account)}}))))))
|
||||
:id (:db/id bank-account)}})))))))
|
||||
|
||||
(GET "/raw" {:keys [query-params identity]}
|
||||
(assert-admin identity)
|
||||
(log/info "Executing raw query " (get query-params "query" ))
|
||||
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))
|
||||
|
||||
(statsd/increment (str "export.raw." (get query-params "query" ) ".count"))
|
||||
(statsd/time! [(str "export.raw." (get query-params "query" ) ".time") {}]
|
||||
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))))
|
||||
(defroutes export-routes
|
||||
(routes
|
||||
(wrap-routes api-key-authed-routes
|
||||
|
||||
@@ -1,34 +1,22 @@
|
||||
(ns auto-ap.routes.queries
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.datomic.clients :as d-clients]
|
||||
[auto-ap.datomic.ledger :as d-ledger]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.datomic.vendors :as d-vendors]
|
||||
[clojure.string :as str]
|
||||
|
||||
[unilog.context :as lc]
|
||||
[buddy.sign.jwt :as jwt]
|
||||
[auto-ap.graphql :as graphql]
|
||||
[auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
|
||||
[auto-ap.routes.utils :refer [wrap-secure]]
|
||||
[clojure.tools.logging :as log]
|
||||
[auto-ap.logging :refer [error-event info-event warn-event]]
|
||||
[clj-time.coerce :as coerce :refer [to-date]]
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[clj-time.core :as time]
|
||||
[clojure.java.io :as io]
|
||||
(:require [amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.graphql.utils :refer [assert-admin]]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[compojure.core
|
||||
:refer
|
||||
[context defroutes GET POST PUT routes wrap-routes]]
|
||||
[config.core :refer [env]]
|
||||
[compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]]
|
||||
[datomic.api :as d]
|
||||
[ring.middleware.json :refer [wrap-json-response]]
|
||||
[venia.core :as venia]
|
||||
[yang.time :refer [time-it]]
|
||||
[ring.util.request :refer [body-string]]
|
||||
)
|
||||
[unilog.context :as lc])
|
||||
(:import java.util.UUID))
|
||||
|
||||
|
||||
(defn wrap-csv-response [handler]
|
||||
(fn [request]
|
||||
(let [response (handler request)]
|
||||
@@ -101,8 +89,9 @@
|
||||
:key (str/replace (:key o) #"^queries\/" "")})))}))
|
||||
|
||||
(GET "/:query-id/results/json" {:keys [query-params identity params]}
|
||||
{:body (execute-query query-params params)})
|
||||
))
|
||||
(statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
|
||||
(statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
|
||||
{:body (execute-query query-params params)}))))
|
||||
|
||||
|
||||
(def raw-routes
|
||||
@@ -119,7 +108,9 @@
|
||||
(def csv-routes
|
||||
(context "/queries" []
|
||||
(GET "/:query-id/results/csv" {:keys [query-params identity params]}
|
||||
{:body (execute-query query-params params)})))
|
||||
(statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
|
||||
(statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
|
||||
{:body (execute-query query-params params)}))))
|
||||
(defroutes query2-routes
|
||||
(routes
|
||||
raw-routes
|
||||
|
||||
Reference in New Issue
Block a user