From b03165ad283362fc84150dc5337dbbefe92b7877 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 29 Mar 2023 19:30:56 -0700 Subject: [PATCH] Should make restore backup work properly --- config/prod-cloud-background-worker.edn | 38 ++++++++++++++++ terraform/background-taskdef.json | 2 +- .../prod-cloud/terraform.tfstate | 10 ++--- .../prod-cloud/terraform.tfstate.backup | 43 ++++++++++++++++++- 4 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 config/prod-cloud-background-worker.edn diff --git a/config/prod-cloud-background-worker.edn b/config/prod-cloud-background-worker.edn new file mode 100644 index 00000000..c5c3e3b9 --- /dev/null +++ b/config/prod-cloud-background-worker.edn @@ -0,0 +1,38 @@ +{:scheme "https" + :db-name "prod-mirror2" + :client-config {:server-type :cloud + :region "us-east-1" + :system "iol-cloud" + :endpoint "https://53syis8n1m.execute-api.us-east-1.amazonaws.com"} + :dd-env "prod-cloud" + :dd-service "integreat-app-cloud" + :jwt-secret "auto ap invoices are awesome" + :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod-cloud" + :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod-cloud" + :invoice-email "invoices@mail.app.integreatconsult.com" + :import-failure-destination-email "ben@integreatconsult.com" + :data-bucket "data.prod.app.integreatconsult.com" + :yodlee-cobrand-name "qstartus12" + :yodlee-cobrand-login "qstartus12" + :yodlee-cobrand-password "MPD@mg78hd" + :yodlee-user-login "integreat" + :yodlee-user-password "Import3transactions!" + :yodlee-base-url "https://quickstart2.api.yodlee.com/ysl" + :yodlee-app "10003600" + :yodlee-fastlink "https://quickstartus2node.yodleeinteractive.com/authenticate/qstartus12/?channelAppName=quickstartus2" + :yodlee-proxy-host "172.31.10.83" + :yodlee-proxy-port 8888 + :run-background? false + :run-web? false + :yodlee2-admin-user "93398522-412b-470d-8400-3691392b12fb_ADMIN" + :yodlee2-integreat-user "integreat-main" + :yodlee2-client-id "3AATcwfPsWP1rP9oDoo4HvZhtaroGVcA" + :yodlee2-client-secret "cXTBmKbGfkaBFIpM" + :yodlee2-base-url "https://production.api.yodlee.com/ysl" + :yodlee2-fastlink "https://fl4.prod.yodlee.com/authenticate/USDevexProd2-319/fastlink/?channelAppName=usdevexprod2" + :yodlee2-proxy-host "172.31.10.83" + :yodlee2-proxy-port 8888 + :plaid {:base-url "https://production.plaid.com" + :client-id "61bfab05f7e762001b323f79" + :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"} + } diff --git a/terraform/background-taskdef.json b/terraform/background-taskdef.json index 3ba8a3c1..65b4dbd4 100644 --- a/terraform/background-taskdef.json +++ b/terraform/background-taskdef.json @@ -19,7 +19,7 @@ "value":"{\"INTEGREAT_JOB\":\"background_job\"}"} ], "essential": true, - "image": "679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:STAGE", + "image": "679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat-cloud:STAGE", "logConfiguration": { "logDriver": "awslogs", "options": { diff --git a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate index eabe3ae5..b581f823 100644 --- a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.3.8", - "serial": 66, + "serial": 72, "lineage": "cf731bb4-8fb3-47af-6e29-22030e089d96", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:15", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:16", "timeouts": { "delete": null }, @@ -1086,8 +1086,8 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/restore_from_backup_prod_cloud:1", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"restore-from-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"restore-from-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"restore-from-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-cloud-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod-cloud\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/restore_from_backup_prod_cloud:2", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"restore-from-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"restore-from-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"restore-from-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-cloud-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat-cloud:prod-cloud\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "4096", "ephemeral_storage": [], "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", @@ -1103,7 +1103,7 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 1, + "revision": 2, "runtime_platform": [], "tags": null, "tags_all": {}, diff --git a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup index 64c67671..eabe3ae5 100644 --- a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.3.8", - "serial": 60, + "serial": 66, "lineage": "cf731bb4-8fb3-47af-6e29-22030e089d96", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:2", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:15", "timeouts": { "delete": null }, @@ -1075,6 +1075,45 @@ "private": "bnVsbA==" } ] + }, + { + "module": "module.restore_from_backup_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/restore_from_backup_prod_cloud:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"restore-from-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"restore-from-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"restore-from-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-cloud-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod-cloud\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "4096", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "restore_from_backup_prod_cloud", + "id": "restore_from_backup_prod_cloud", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "8192", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] } ], "check_results": null