From 78db8ea4dd96b1dd56513e99d5c0188d83705bc1 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 4 Jan 2022 18:11:52 -0800 Subject: [PATCH] Adding more metrics --- src/clj/auto_ap/background/metrics.clj | 2 + src/clj/auto_ap/server.clj | 65 ++++++++++++++----- terraform/prod-background-worker-taskdef.json | 4 -- terraform/prod-taskdef.json | 4 -- terraform/staging-taskdef.json | 4 -- .../staging/terraform.tfstate | 18 ++--- .../staging/terraform.tfstate.backup | 21 +++--- 7 files changed, 70 insertions(+), 48 deletions(-) diff --git a/src/clj/auto_ap/background/metrics.clj b/src/clj/auto_ap/background/metrics.clj index ec4d5388..56af4ed8 100644 --- a/src/clj/auto_ap/background/metrics.clj +++ b/src/clj/auto_ap/background/metrics.clj @@ -7,3 +7,5 @@ :start (statsd/setup! :host "127.0.0.1" :port 8125 :prefix "integreat.app" :tags #{(str "env:" (:dd-env env)) (str "service:" (:dd-service env))}) :stop (statsd/shutdown!)) + + diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 5b7feda1..6cc28749 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -1,42 +1,73 @@ (ns auto-ap.server - (:require [auto-ap.handler :refer [app]] - [auto-ap.ledger :as ledger] - [auto-ap.yodlee.core] - [auto-ap.graphql.clients :as gq-clients] - [auto-ap.background.invoices] + (:gen-class) + (:require auto-ap.background.invoices [auto-ap.background.requests :as requests] [auto-ap.background.sysco :as sysco] [auto-ap.background.vendor :as vendor] - [auto-ap.background.metrics :as metrics] - [auto-ap.square.core :as square] [auto-ap.datomic.migrate :as migrate] - [auto-ap.import.yodlee :as yodlee] - [auto-ap.import.yodlee2 :as yodlee2] + [auto-ap.graphql.clients :as gq-clients] + [auto-ap.handler :refer [app]] [auto-ap.import.intuit :as intuit] [auto-ap.import.plaid :as plaid] - [nrepl.server :refer [start-server stop-server]] - [config.core :refer [env]] - [ring.adapter.jetty :refer [run-jetty]] + [auto-ap.import.yodlee :as yodlee] + [auto-ap.import.yodlee2 :as yodlee2] + [auto-ap.ledger :as ledger] + [auto-ap.square.core :as square] [clojure.tools.logging :as log] - [unilog.config] - [mount.core :as mount]) - (:gen-class)) + [config.core :refer [env]] + [mount.core :as mount] + [nrepl.server :refer [start-server]] + [ring.adapter.jetty :refer [run-jetty]] + [yang.scheduler :as scheduler] + [com.unbounce.dogstatsd.core :as statsd]) + (:import org.eclipse.jetty.server.handler.StatisticsHandler)) (defn add-shutdown-hook! [^Runnable f] (.addShutdownHook (Runtime/getRuntime) (Thread. f))) + + +(defn configure-jetty [server] + (let [stats-handler (StatisticsHandler.) + default-handler (.getHandler server)] + (.setHandler stats-handler default-handler) + (.setHandler server stats-handler)) + (.setStopAtShutdown server true)) + (mount/defstate port :start (Integer/parseInt (or (env :port) "3000"))) (mount/defstate jetty - :start (run-jetty app {:port port :join? false}) + :start (run-jetty app {:port port + :join? false + :configurator configure-jetty}) :stop (.stop jetty)) +(defn collect-jetty-stats [] + (try + (statsd/gauge "requests.active" (.getRequestsActive (.getHandler jetty))) + (statsd/gauge "requests.1xx" (.getResponses1xx (.getHandler jetty))) + (statsd/gauge "requests.2xx" (.getResponses2xx (.getHandler jetty))) + (statsd/gauge "requests.3xx" (.getResponses3xx (.getHandler jetty))) + (statsd/gauge "requests.4xx" (.getResponses4xx (.getHandler jetty))) + (statsd/gauge "requests.5xx" (.getResponses5xx (.getHandler jetty))) + (.statsReset (.getHandler jetty)) + (catch Exception e + (log/warn e)))) + +(mount/defstate jetty-stats + :start (scheduler/every (* 1000 30) (collect-jetty-stats)) + :stop (scheduler/stop jetty-stats)) + + + + (defn shutdown-mount [] (mount/stop)) (defn -main [& args] (let [without (cond-> [] - (not (env :run-web? )) (into [#'jetty]) + (not (env :run-web? )) (into [#'jetty + #'collect-jetty-stats]) (not (env :run-background?)) (into [#'square/square-loader #'vendor/refresh-vendor-usages-worker #'ledger/touch-broken-ledger-worker diff --git a/terraform/prod-background-worker-taskdef.json b/terraform/prod-background-worker-taskdef.json index f9c9300d..2aceee42 100644 --- a/terraform/prod-background-worker-taskdef.json +++ b/terraform/prod-background-worker-taskdef.json @@ -53,10 +53,6 @@ { "name": "ECS_FARGATE", "value": "true" - }, - { - "name": "DD_DOCKER_ENV_AS_TAGS", - "value": "{\"which_env\":\"env\"}" } ], "essential": true, diff --git a/terraform/prod-taskdef.json b/terraform/prod-taskdef.json index 20e14f62..1fceef8a 100644 --- a/terraform/prod-taskdef.json +++ b/terraform/prod-taskdef.json @@ -58,10 +58,6 @@ { "name": "ECS_FARGATE", "value": "true" - }, - { - "name": "DD_DOCKER_ENV_AS_TAGS", - "value": "{\"which_env\":\"env\"}" } ], "essential": true, diff --git a/terraform/staging-taskdef.json b/terraform/staging-taskdef.json index 95d3f01b..4e6dd5ff 100644 --- a/terraform/staging-taskdef.json +++ b/terraform/staging-taskdef.json @@ -53,10 +53,6 @@ { "name": "ECS_FARGATE", "value": "true" - }, - { - "name": "DD_DOCKER_ENV_AS_TAGS", - "value": "{\"which_env\":\"env\"}" } ], "essential": true, diff --git a/terraform/terraform.tfstate.d/staging/terraform.tfstate b/terraform/terraform.tfstate.d/staging/terraform.tfstate index 175075c4..a5867a32 100644 --- a/terraform/terraform.tfstate.d/staging/terraform.tfstate +++ b/terraform/terraform.tfstate.d/staging/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "0.14.4", - "serial": 99, + "serial": 107, "lineage": "91d10fe0-8033-8778-c202-78d5a81632e8", "outputs": { "aws_access_key_id": { @@ -153,7 +153,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:101", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:105", "timeouts": { "delete": null }, @@ -232,7 +232,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:21", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:25", "timeouts": { "delete": null }, @@ -256,8 +256,8 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:103", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"staging\",\"com.datadoghq.tags.service\":\"integreat-app\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"staging\"},{\"name\":\"DD_SERVICE\",\"value\":\"integreat-app\"},{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"DD_DOCKER_ENV_AS_TAGS\",\"value\":\"{\\\"which_env\\\":\\\"env\\\"}\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:106", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"staging\",\"com.datadoghq.tags.service\":\"integreat-app\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"staging\"},{\"name\":\"DD_SERVICE\",\"value\":\"integreat-app\"},{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "2048", "ephemeral_storage": [], "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", @@ -273,9 +273,9 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 103, + "revision": 106, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -294,7 +294,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:23", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:26", "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"staging\",\"com.datadoghq.tags.service\":\"integreat-background-worker\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"staging\"},{\"name\":\"DD_SERVICE\",\"value\":\"integreat-background-worker\"},{\"name\":\"config\",\"value\":\"/usr/local/config/staging-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "1024", "ephemeral_storage": [], @@ -311,7 +311,7 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 23, + "revision": 26, "runtime_platform": [], "tags": null, "tags_all": {}, diff --git a/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup b/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup index 7554dd19..32a917e4 100644 --- a/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "0.14.4", - "serial": 94, + "serial": 104, "lineage": "91d10fe0-8033-8778-c202-78d5a81632e8", "outputs": { "aws_access_key_id": { @@ -153,7 +153,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:101", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:105", "timeouts": { "delete": null }, @@ -232,7 +232,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:21", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:25", "timeouts": { "delete": null }, @@ -256,8 +256,8 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:102", - "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"},{\"name\":\"which_env\",\"value\":\"staging\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"DD_DOCKER_ENV_AS_TAGS\",\"value\":\"{\\\"which_env\\\":\\\"env\\\"}\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:106", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"staging\",\"com.datadoghq.tags.service\":\"integreat-app\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"staging\"},{\"name\":\"DD_SERVICE\",\"value\":\"integreat-app\"},{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "2048", "ephemeral_storage": [], "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", @@ -273,7 +273,7 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 102, + "revision": 106, "runtime_platform": [], "tags": null, "tags_all": {}, @@ -292,10 +292,11 @@ "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { + "status": "tainted", "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:22", - "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/staging-background-worker.edn\"},{\"name\":\"which_env\",\"value\":\"staging\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"DD_DOCKER_ENV_AS_TAGS\",\"value\":\"{\\\"which_env\\\":\\\"env\\\"}\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:23", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"staging\",\"com.datadoghq.tags.service\":\"integreat-background-worker\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"staging\"},{\"name\":\"DD_SERVICE\",\"value\":\"integreat-background-worker\"},{\"name\":\"config\",\"value\":\"/usr/local/config/staging-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:staging\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "1024", "ephemeral_storage": [], "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", @@ -311,9 +312,9 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 22, + "revision": 23, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": []