notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Datadog stats

Browse Source
This commit is contained in:
Bryce Covert
2022-01-04 11:53:47 -08:00
parent e6229ef5d0
commit 6ca4f0fb60
9 changed files with 160 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config/dev.edn
View File

@@ -23,8 +23,7 @@
:yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver" :yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver"
:run-web? true :run-web? true
:run-background? true :run-background? true
:dd-env "dev"
:yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN" :yodlee2-admin-user "e02b38f9-9865-4264-8e4f-6a5ac2c500b0_ADMIN"
:yodlee2-integreat-user "integreat-main" :yodlee2-integreat-user "integreat-main"
:yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd" :yodlee2-client-id "l6sUyK2NEq3mwopISHlFGWUcJ1U8OUQd"

1
config/prod-background-worker.edn
View File

@@ -1,6 +1,7 @@
{:db {:server "database"} {:db {:server "database"}
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod" :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
:scheme "https" :scheme "https"
:dd-env "prod"
:jwt-secret "auto ap invoices are awesome" :jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod"

1
config/prod.edn
View File

@@ -1,6 +1,7 @@
{:db {:server "database"} {:db {:server "database"}
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod" :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod"
:scheme "https" :scheme "https"
:dd-env "prod"
:jwt-secret "auto ap invoices are awesome" :jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod"

1
config/staging-background-worker.edn
View File

@@ -1,6 +1,7 @@
{:db {:server "database"} {:db {:server "database"}
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging" :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
:scheme "https" :scheme "https"
:dd-env "staging"
:jwt-secret "auto ap invoices are awesome" :jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging"

1
config/staging.edn
View File

@@ -1,6 +1,7 @@
{:db {:server "database"} {:db {:server "database"}
:datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging" :datomic-url "datomic:ddb://us-east-1/integreat/integreat-staging"
:scheme "https" :scheme "https"
:dd-env "staging"
:jwt-secret "auto ap invoices are awesome" :jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging" :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging"

5
src/clj/auto_ap/background/metrics.clj
View File

@@ -1,7 +1,8 @@
(ns auto-ap.background.metrics (ns auto-ap.background.metrics
(:require [com.unbounce.dogstatsd.core :as statsd] (:require [com.unbounce.dogstatsd.core :as statsd]
[mount.core :as mount])) [mount.core :as mount]
[config.core :refer [env]]))
(mount/defstate metrics-setup (mount/defstate metrics-setup
:start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "my.app") :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(:dd-env env)})
:stop nil) :stop nil)

8
src/clj/auto_ap/graphql.clj
View File

@@ -1329,7 +1329,7 @@
(defn query-name [q] (defn query-name [q]
(try (try
(str/join "," (map name (:operations (p/operations (p/parse-query schema q))))) (str/join "__" (map name (:operations (p/operations (p/parse-query schema q)))))
(catch Exception e (catch Exception e
"unknown query"))) "unknown query")))
@@ -1337,9 +1337,9 @@
([id q] ([id q]
(query id q nil)) (query id q nil))
([id q v] ([id q v]
(statsd/increment "queries") (statsd/increment "query.count")
(statsd/increment (query-name q)) (statsd/increment (str "query." (query-name q) ".count"))
(statsd/time! [(query-name q) {}] (statsd/time! [(str "query." (query-name q) ".time") {}]
(lc/with-context {:query q} (lc/with-context {:query q}
(log/info "executing query name" (query-name q)) (log/info "executing query name" (query-name q))
(try (try

243
src/clj/auto_ap/routes/exports.clj
View File

@@ -18,7 +18,8 @@
[datomic.api :as d] [datomic.api :as d]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia] [venia.core :as venia]
[yang.time :refer [time-it]])) [yang.time :refer [time-it]]
[com.unbounce.dogstatsd.core :as statsd]))
(defn wrap-csv-response [handler] (defn wrap-csv-response [handler]
(fn [request] (fn [request]
(let [response (handler request)] (let [response (handler request)]
@@ -51,7 +52,9 @@
(context "/" [] (context "/" []
(GET "/invoices/export" {:keys [query-params identity] :as request} (GET "/invoices/export" {:keys [query-params identity] :as request}
(assert-admin identity) (assert-admin identity)
(let [query [[:all_invoices (statsd/increment (str "export.invoice.count") {:tags #{(query-params "client-code")}})
(statsd/time! [(str "export.invoice.time") {:tags #{(query-params "client-code")}}]
(let [query [[:all_invoices
{:client-code (query-params "client-code") {:client-code (query-params "client-code")
:original-id (query-params "original")} :original-id (query-params "original")}
@@ -64,74 +67,82 @@
invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] invoices (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
(list (:all-invoices (:data invoices))))) (list (:all-invoices (:data invoices)))))
)
(GET "/payments/export" {:keys [query-params identity]} (GET "/payments/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(let [query [[:all_payments (statsd/increment (str "export.payment.count") {:tags #{(query-params "client-code")}})
{:client-code (query-params "client-code") (statsd/time! [(str "export.payment.time") {:tags #{(query-params "client-code")}}]
:original-id (query-params "original")} (let [query [[:all_payments
[:id :check-number :amount :memo :date :status :type :original-id {:client-code (query-params "client-code")
[:invoices [[:invoice [:id :original-id]] :amount]] :original-id (query-params "original")}
[:bank-account [:number :code :bank-name :bank-code :id]] [:id :check-number :amount :memo :date :status :type :original-id
[:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]] [:invoices [[:invoice [:id :original-id]] :amount]]
[:client [:id :name :code]] [:bank-account [:number :code :bank-name :bank-code :id]]
]]] [:vendor [:name :id [:primary-contact [:name :email :phone]] [:default-account [:name :numeric-code :id]] [:address [:street1 :city :state :zip]]]]
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] [:client [:id :name :code]]
(list (:all-payments (:data payments))))) ]]]
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
(list (:all-payments (:data payments))))))
(GET "/sales/export" {:keys [query-params identity :as params]} (GET "/sales/export" {:keys [query-params identity :as params]}
(assert-admin identity) (assert-admin identity)
(println params) (println params)
(let [query [[:all_sales_orders (statsd/increment (str "export.sales.count") {:tags #{(query-params "client-code")}})
(cond-> {:client-code (query-params "client-code")} (statsd/time! [(str "export.sales.time") {:tags #{(query-params "client-code")}}]
(query-params "after") (assoc :date-range {:start (query-params "after") (let [query [[:all_sales_orders
:end nil})) (cond-> {:client-code (query-params "client-code")}
[:id (query-params "after") (assoc :date-range {:start (query-params "after")
:location :end nil}))
:external_id [:id
:total :location
:tip :external_id
:tax :total
:discount :tip
:returns :tax
:service_charge :discount
:date :returns
[:charges [:type_name :total :tip]] :service_charge
[:line_items [:item_name :total :tax :discount :category]] :date
[:client [:id :name :code]]]]] [:charges [:type_name :total :tip]]
payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)})) [:line_items [:item_name :total :tax :discount :category]]
parsedouble #(some-> % Double/parseDouble) ] [:client [:id :name :code]]]]]
(seq (map payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))
(fn [s] parsedouble #(some-> % Double/parseDouble) ]
(-> s (seq (map
(assoc :utc_date (:date s)) (fn [s]
(update :date (fn [d] (-> s
(coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles")))))) (assoc :utc_date (:date s))
(update :total parsedouble) (update :date (fn [d]
(update :tax parsedouble) (coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))))))
(update :discount parsedouble) (update :total parsedouble)
(update :tip parsedouble) (update :tax parsedouble)
(update :line-items (fn [lis] (update :discount parsedouble)
(map (update :tip parsedouble)
(fn [li] (update :line-items (fn [lis]
(-> li (map
(update :tax parsedouble) (fn [li]
(update :discount parsedouble) (-> li
(update :total parsedouble))) (update :tax parsedouble)
lis))) (update :discount parsedouble)
(update :charges (fn [charges] (update :total parsedouble)))
(map lis)))
(fn [charge] (update :charges (fn [charges]
(-> charge (map
(update :tip parsedouble) (fn [charge]
(update :total parsedouble))) (-> charge
charges))))) (update :tip parsedouble)
(:all-sales-orders (:data payments)))))) (update :total parsedouble)))
charges)))))
(:all-sales-orders (:data payments))))))
)
(GET "/expected-deposit/export" {:keys [query-params identity]} (GET "/expected-deposit/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(let [query [[:all_expected_deposits (statsd/increment (str "export.expected_deposit.count") {:tags #{(query-params "client-code")}})
(statsd/time! [(str "export.expected_deposit.time") {:tags #{(query-params "client-code")}}]
(let [query [[:all_expected_deposits
(cond-> {:client-code (query-params "client-code")} (cond-> {:client-code (query-params "client-code")}
(query-params "after") (assoc :date-range {:start (query-params "after") (query-params "after") (assoc :date-range {:start (query-params "after")
:end nil})) :end nil}))
@@ -148,76 +159,81 @@
(-> d (-> d
(update :fee #(some-> % Double/parseDouble)) (update :fee #(some-> % Double/parseDouble))
(update :total #(some-> % Double/parseDouble)))) (update :total #(some-> % Double/parseDouble))))
(:all-expected-deposits (:data payments)))))) (:all-expected-deposits (:data payments)))))))
(GET "/clients/export" {:keys [query-params identity]} (GET "/clients/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(map <-graphql (d-clients/get-all))) (map <-graphql (d-clients/get-all)))
(GET "/vendors/export" {:keys [query-params identity]} (GET "/vendors/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(map <-graphql (d-vendors/get-graphql {}))) (statsd/increment (str "export.vendor.count" ))
(statsd/time! [(str "export.vendor.time") {}]
(map <-graphql (d-vendors/get-graphql {}))))
(GET "/ledger/export" {:keys [query-params identity]} (GET "/ledger/export" {:keys [query-params identity]}
(log/info "exporting for " (query-params "client-code")) (log/info "exporting for " (query-params "client-code"))
(assert-admin identity) (assert-admin identity)
(let [[result time] (time-it (transduce (comp (statsd/increment (str "export.ledger.count") {:tags #{(query-params "client-code")}})
(map #(update % :journal-entry/date to-date)) (statsd/time! [(str "export.ledger.time") {:tags #{(query-params "client-code")}}]
(map <-graphql)) (transduce (comp
conj (map #(update % :journal-entry/date to-date))
(list) (map <-graphql))
(first (d-ledger/get-graphql {:count Integer/MAX_VALUE conj
:client-code (query-params "client-code")}))))] (list)
(info-event "Export completed" (first (d-ledger/get-graphql {:count Integer/MAX_VALUE
{:time (:time time) :client-code (query-params "client-code")})))))
:errors (seq (:errors result))})
result))
(GET "/accounts/export" {:keys [query-params identity]} (GET "/accounts/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(statsd/increment (str "export.accounts.count") {:tags #{(query-params "client-code")}})
(let [client-id (d-clients/code->id (query-params "client-code")) (statsd/time! [(str "export.accounts.time") {:tags #{(query-params "client-code")}}]
query [[:accounts (let [client-id (d-clients/code->id (query-params "client-code"))
[:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]] query [[:accounts
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] [:id :numeric_code :type :applicability :location :name [:client_overrides [:name [:client [:id :code :name]]]]]]]
all-accounts (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))]
(list (transduce (list (transduce
(comp (comp
(filter (fn [a] (filter (fn [a]
(let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))] (let [overriden-clients (set (map (comp :id :client) (:client-overrides a)))]
(or (= :global (:applicability a)) (or (= :global (:applicability a))
(overriden-clients (str client-id)))))) (overriden-clients (str client-id))))))
(map (fn [a] (map (fn [a]
(let [client->name (reduce (let [client->name (reduce
(fn [override co] (fn [override co]
(assoc override (str (:id (:client co))) (:name co))) (assoc override (str (:id (:client co))) (:name co)))
{} {}
(:client-overrides a))] (:client-overrides a))]
(-> a (-> a
(assoc :global-name (:name a)) (assoc :global-name (:name a))
(assoc :client-name (client->name (str client-id) (:name a))) (assoc :client-name (client->name (str client-id) (:name a)))
(dissoc :client-overrides)))))) (dissoc :client-overrides))))))
conj conj
[] []
(:accounts (:data all-accounts)))))) (:accounts (:data all-accounts)))))))
(GET "/transactions/export" {:keys [query-params identity]} (GET "/transactions/export" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code") (statsd/increment (str "export.transactions.count") {:tags #{(query-params "client-code")}})
#_#_:original-id (Integer/parseInt (query-params "original")) (statsd/time! [(str "export.transactions.time") {:tags #{(query-params "client-code")}}]
:count Integer/MAX_VALUE})] (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code")
#_#_:original-id (Integer/parseInt (query-params "original"))
:count Integer/MAX_VALUE})]
(map (comp ->graphql (fn [i]
(cond-> i
true (update :transaction/date to-date) (map (comp ->graphql (fn [i]
true (update :transaction/post-date to-date) (cond-> i
(:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date) true (update :transaction/date to-date)
(:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date)))) true (update :transaction/post-date to-date)
transactions))) (:transaction/payment i) (update-in [:transaction/payment :payment/date] to-date)
(:transaction/expected-deposit i) (update-in [:transaction/expected-deposit :expected-deposit/date] to-date))))
transactions)))
)
(GET "/transactions/export2" {:keys [query-params identity]} (GET "/transactions/export2" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(let [db (d/db conn)] (statsd/increment (str "export.transactions2.count") {:tags #{(query-params "client-code")}})
(statsd/time! [(str "export.transactions2.time") {:tags #{(query-params "client-code")}}]
(let [db (d/db conn)]
(->> (->>
(d/query {:query {:find ['?e] (d/query {:query {:find ['?e]
:in ['$ '?client-code] :in ['$ '?client-code]
@@ -239,12 +255,15 @@
:approval_status (:transaction/approval-status e) :approval_status (:transaction/approval-status e)
:bank_account {:name (:bank-account/name bank-account) :bank_account {:name (:bank-account/name bank-account)
:code (:bank-account/code bank-account) :code (:bank-account/code bank-account)
:id (:db/id bank-account)}})))))) :id (:db/id bank-account)}})))))))
(GET "/raw" {:keys [query-params identity]} (GET "/raw" {:keys [query-params identity]}
(assert-admin identity) (assert-admin identity)
(log/info "Executing raw query " (get query-params "query" )) (log/info "Executing raw query " (get query-params "query" ))
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]"))))))))
(statsd/increment (str "export.raw." (get query-params "query" ) ".count"))
(statsd/time! [(str "export.raw." (get query-params "query" ) ".time") {}]
(into (list) (apply d/q (clojure.edn/read-string (get query-params "query" )) (into [(d/db conn)] (clojure.edn/read-string (get query-params "args" "[]")))))))))
(defroutes export-routes (defroutes export-routes
(routes (routes
(wrap-routes api-key-authed-routes (wrap-routes api-key-authed-routes

43
src/clj/auto_ap/routes/queries.clj
View File

@@ -1,34 +1,22 @@
(ns auto-ap.routes.queries (ns auto-ap.routes.queries
(:require [auto-ap.datomic :refer [conn]] (:require [amazonica.aws.s3 :as s3]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic :refer [conn]]
[auto-ap.datomic.ledger :as d-ledger] [auto-ap.graphql.utils :refer [assert-admin]]
[auto-ap.datomic.transactions :as d-transactions]
[auto-ap.datomic.vendors :as d-vendors]
[clojure.string :as str]
[unilog.context :as lc]
[buddy.sign.jwt :as jwt]
[auto-ap.graphql :as graphql]
[auto-ap.graphql.utils :refer [->graphql <-graphql assert-admin assert-can-see-client]]
[auto-ap.routes.utils :refer [wrap-secure]]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [error-event info-event warn-event]]
[clj-time.coerce :as coerce :refer [to-date]]
[amazonica.aws.s3 :as s3]
[clj-time.core :as time]
[clojure.java.io :as io]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[compojure.core
:refer
[context defroutes GET POST PUT routes wrap-routes]]
[config.core :refer [env]] [config.core :refer [env]]
[compojure.core :refer [context defroutes GET POST PUT wrap-routes routes]]
[datomic.api :as d] [datomic.api :as d]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia]
[yang.time :refer [time-it]]
[ring.util.request :refer [body-string]] [ring.util.request :refer [body-string]]
) [unilog.context :as lc])
(:import java.util.UUID)) (:import java.util.UUID))
(defn wrap-csv-response [handler] (defn wrap-csv-response [handler]
(fn [request] (fn [request]
(let [response (handler request)] (let [response (handler request)]
@@ -101,8 +89,9 @@
:key (str/replace (:key o) #"^queries\/" "")})))})) :key (str/replace (:key o) #"^queries\/" "")})))}))
(GET "/:query-id/results/json" {:keys [query-params identity params]} (GET "/:query-id/results/json" {:keys [query-params identity params]}
{:body (execute-query query-params params)}) (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
)) (statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
{:body (execute-query query-params params)}))))
(def raw-routes (def raw-routes
@@ -119,7 +108,9 @@
(def csv-routes (def csv-routes
(context "/queries" [] (context "/queries" []
(GET "/:query-id/results/csv" {:keys [query-params identity params]} (GET "/:query-id/results/csv" {:keys [query-params identity params]}
{:body (execute-query query-params params)}))) (statsd/increment (str "export.query.count") {:tags #{(:query-id params)}})
(statsd/time! [(str "export.query.time") {:tags #{(:query-id params)}}]
{:body (execute-query query-params params)}))))
(defroutes query2-routes (defroutes query2-routes
(routes (routes
raw-routes raw-routes