From 689cae6192ac02e50adfd736e0ab017a5defe085 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 8 Dec 2020 21:24:26 -0800 Subject: [PATCH] made web server not start background tasks. --- config/dev.edn | 2 + config/prod.edn | 2 + config/staging.edn | 2 + src/clj/auto_ap/server.clj | 24 +++++++---- terraform/deploy.tf | 42 ++++++++++++++++++- .../prod/terraform.tfstate | 42 +++++++++++++++++-- .../prod/terraform.tfstate.backup | 16 +++---- 7 files changed, 110 insertions(+), 20 deletions(-) diff --git a/config/dev.edn b/config/dev.edn index a0c9378b..cf524eb1 100644 --- a/config/dev.edn +++ b/config/dev.edn @@ -17,4 +17,6 @@ :yodlee-base-url "https://developer.api.yodlee.com/ysl" :yodlee-app "10003600" :yodlee-fastlink "https://node.developer.yodlee.com/authenticate/restserver/?channelAppName=restserver" + :run-web? true + :run-background? true } diff --git a/config/prod.edn b/config/prod.edn index 8b18ce86..4a8c3689 100644 --- a/config/prod.edn +++ b/config/prod.edn @@ -15,4 +15,6 @@ :yodlee-fastlink "https://quickstartus2node.yodleeinteractive.com/authenticate/qstartus12/?channelAppName=quickstartus2" :yodlee-proxy-host "172.31.10.83" :yodlee-proxy-port 8888 + :run-background? false + :run-web? true } diff --git a/config/staging.edn b/config/staging.edn index 86a3cb3b..f08ecce2 100644 --- a/config/staging.edn +++ b/config/staging.edn @@ -15,4 +15,6 @@ :yodlee-fastlink "https://quickstartus2node.yodleeinteractive.com/authenticate/qstartus12/?channelAppName=quickstartus2" :yodlee-proxy-host "172.31.10.83" :yodlee-proxy-port 8888 + :run-background? true + :run-web? true } diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index c51b99c6..7cb4ab99 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -1,12 +1,12 @@ (ns auto-ap.server (:require #_[auto-ap.background.mail :refer [always-process-sqs]] [auto-ap.handler :refer [app]] - [auto-ap.ledger] + [auto-ap.ledger :as ledger] [auto-ap.yodlee.core] [auto-ap.background.invoices] - [auto-ap.square.core] - [auto-ap.datomic.migrate] - [auto-ap.yodlee.import] + [auto-ap.square.core :as square] + [auto-ap.datomic.migrate :as migrate] + [auto-ap.yodlee.import :as yodlee] [nrepl.server :refer [start-server stop-server]] [config.core :refer [env]] [ring.adapter.jetty :refer [run-jetty]] @@ -28,6 +28,16 @@ (mount/stop)) (defn -main [& args] - (add-shutdown-hook! shutdown-mount) - (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler)) - (mount/start)) + (let [without (cond-> #{} + (not (env :run-web? )) (into [#'jetty]) + (not (env :run-background?)) (into [#'square/square-loader + #'square/square-settlement-loader + #'ledger/touch-broken-ledger-worker + #'ledger/process-txes-worker + #'ledger/ledger-reconciliation-worker + #'yodlee/import-transaction-worker + #'migrate/migrate-start]))] + + (add-shutdown-hook! shutdown-mount) + (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler)) + (mount/start-without without))) diff --git a/terraform/deploy.tf b/terraform/deploy.tf index 9ae3abde..a83b8f2d 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -22,11 +22,24 @@ resource "aws_ecs_task_definition" "integreat_app" { task_role_arn = "${var.task_role_arn}" } + +resource "aws_ecs_task_definition" "integreat_background_worker" { + + family = "integreat_background-worker_${var.stage}" + container_definitions = file("${var.stage}-background-worker-taskdef.json") + memory = 2048 + cpu = 1024 + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = "${var.execution_role_arn}" + task_role_arn = "${var.task_role_arn}" +} + resource "aws_ecs_service" "integreat_app" { name = "integreat_app_${var.stage}" cluster = "${var.ecs_cluster}" task_definition = aws_ecs_task_definition.integreat_app.arn - desired_count = 1 + desired_count = 2 health_check_grace_period_seconds = 600 deployment_controller { type = "ECS" @@ -55,6 +68,33 @@ resource "aws_ecs_service" "integreat_app" { timeouts {} } +resource "aws_ecs_service" "integreat_background_worker" { + name = "integreat_background_worker_${var.stage}" + cluster = "${var.ecs_cluster}" + task_definition = aws_ecs_task_definition.integreat_background_worker.arn + desired_count = 1 + deployment_controller { + type = "ECS" + } + scheduling_strategy = "REPLICA" + launch_type = "FARGATE" + platform_version = "LATEST" + + network_configuration { + assign_public_ip = true + security_groups = [ "sg-004e5855310c453a3", "sg-02d167406b1082698"] + subnets = [ "subnet-5e675761", "subnet-8519fde2", "subnet-89bab8d4" ] + } + + service_registries { + container_port = 0 + port = 0 + registry_arn = aws_service_discovery_service.service.arn + } + + timeouts {} +} + resource "aws_lb" "integreat_app" { name = "integreat-app-${var.stage}" internal = false diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 59c03105..2f250099 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "0.13.3", - "serial": 72, + "serial": 75, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -33,7 +33,7 @@ "attributes": { "account_id": "679918342773", "arn": "arn:aws:iam::679918342773:user/bryce", - "id": "2020-09-26 05:35:11.176798 +0000 UTC", + "id": "2020-12-09 05:22:32.641901 +0000 UTC", "user_id": "AIDAJPUJFTOKO4IRADMV4" } } @@ -180,6 +180,40 @@ "FARGATE" ], "revision": 4, + "tags": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "integreat_background_worker", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:1", + "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-background-worker\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", + "cpu": "1024", + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "integreat_background-worker_prod", + "id": "integreat_background-worker_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "2048", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, "tags": null, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -509,8 +543,8 @@ "path": "/api/health-check", "port": "traffic-port", "protocol": "HTTP", - "timeout": 5, - "unhealthy_threshold": 2 + "timeout": 14, + "unhealthy_threshold": 5 } ], "id": "arn:aws:elasticloadbalancing:us-east-1:679918342773:targetgroup/integreat-app-prod/d5b51736182d8407", diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index d019d050..59c03105 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "0.13.3", - "serial": 68, + "serial": 72, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -33,7 +33,7 @@ "attributes": { "account_id": "679918342773", "arn": "arn:aws:iam::679918342773:user/bryce", - "id": "2020-09-26 05:16:10.008633 +0000 UTC", + "id": "2020-09-26 05:35:11.176798 +0000 UTC", "user_id": "AIDAJPUJFTOKO4IRADMV4" } } @@ -139,8 +139,8 @@ "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-m232jsqnpvd5e2xb" } ], - "tags": null, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:1", + "tags": {}, + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:4", "timeouts": { "delete": null } @@ -163,8 +163,8 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:1", - "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/prod.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:4", + "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/prod.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", "cpu": "2048", "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", "family": "integreat_app_prod", @@ -179,8 +179,8 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 1, - "tags": {}, + "revision": 4, + "tags": null, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] },