From 98f21bf3df2aa6370157f00778e57d897cf4ad34 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 16 Sep 2022 14:23:38 -0700 Subject: [PATCH] New form of square loading. --- src/clj/auto_ap/jobs/square.clj | 2 +- src/clj/auto_ap/server.clj | 3 +- src/clj/auto_ap/square/core.clj | 2 +- terraform/deploy.tf | 11 +- .../prod/terraform.tfstate | 139 +++++++++----- .../prod/terraform.tfstate.backup | 171 +++++++++++++----- 6 files changed, 237 insertions(+), 91 deletions(-) diff --git a/src/clj/auto_ap/jobs/square.clj b/src/clj/auto_ap/jobs/square.clj index 14dbd7c0..26dcf5db 100644 --- a/src/clj/auto_ap/jobs/square.clj +++ b/src/clj/auto_ap/jobs/square.clj @@ -8,7 +8,7 @@ [unilog.context :as lc])) (defn -main [& _] - (lc/with-context {:background-job "square-loading"} + (lc/with-context {:background-job "square-loading-new"} (mount/start (mount/only #{#'conn})) ((heartbeat square/upsert-all "square-loading")) (mount/stop (mount/only #{#'conn})) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index c671fcd2..24f19f03 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -103,8 +103,7 @@ (let [without (cond-> [] (not (env :run-web? )) (into [#'jetty #'jetty-stats]) - (not (env :run-background?)) (into [#'square/square-loader - #'vendor/refresh-vendor-usages-worker + (not (env :run-background?)) (into [#'vendor/refresh-vendor-usages-worker #'ledger/touch-broken-ledger-worker #'mail/import-invoices #'ledger/process-txes-worker diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index 244af25b..7291fdf7 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -530,7 +530,7 @@ (some-> (:object &throw-context) str) "Unknown error")})))))) -(mount/defstate square-loader +#_(mount/defstate square-loader :start (scheduler/every (* 4 59 60 1000) (heartbeat upsert-all "square-loading")) :stop (scheduler/stop square-loader)) diff --git a/terraform/deploy.tf b/terraform/deploy.tf index adb17733..d06e1164 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -330,14 +330,18 @@ resource "aws_cloudwatch_event_rule" "square_import_job_schedule" { } + resource "aws_cloudwatch_event_target" "square_import_job_target" { target_id = "square-import-job" arn = var.ecs_cluster rule = aws_cloudwatch_event_rule.square_import_job_schedule.name - role_arn = var.execution_role_arn + role_arn = "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733" ecs_target { task_count = 1 + launch_type = "FARGATE" + + # capacity_provider_strategy = [{capacity_provider = "FARGATE_SPOT"}] task_definition_arn = aws_ecs_task_definition.integreat_background_worker.arn network_configuration { assign_public_ip = true @@ -351,8 +355,9 @@ resource "aws_cloudwatch_event_target" "square_import_job_target" { "containerOverrides": [ { "name": "integreat-app", - "environment": [{"name": "INTEGREAT_JOB", "value": "hello-world"}, - {"name": "DD_SERVICE", "value": "hello-world"}] + "environment": [{"name": "INTEGREAT_JOB", "value": "square-import-job"}, + {"name": "DD_SERVICE", "value": "square-import-job"}, + {"name": "DD_CONTAINER_ENV_AS_TAGS", "value":"{\"INTEGREAT_JOB\":\"task_family\"}"}] } ] } diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index a63ac74d..0eb02943 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.7", - "serial": 105, + "serial": 121, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -83,6 +83,97 @@ } ] }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "square_import_job_schedule", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "id": "square-import-job-schedule", + "is_enabled": true, + "name": "square-import-job-schedule", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(4 hours)", + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "square_import_job_target", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [ + { + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "group": "", + "launch_type": "FARGATE", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "placement_constraint": [], + "platform_version": "", + "propagate_tags": "TASK_DEFINITION", + "tags": {}, + "task_count": 1, + "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165" + } + ], + "event_bus_name": "default", + "http_target": [], + "id": "square-import-job-schedule-square-import-job", + "input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"task_family\\\"}\"}] \n }\n ]\n}\n", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733", + "rule": "square-import-job-schedule", + "run_command_targets": [], + "sqs_target": [], + "target_id": "square-import-job" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.square_import_job_schedule", + "aws_ecs_task_definition.integreat_background_worker" + ] + } + ] + }, { "mode": "managed", "type": "aws_ecs_service", @@ -164,7 +255,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:353", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:354", "timeouts": { "delete": null }, @@ -243,7 +334,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:331", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:332", "timeouts": { "delete": null }, @@ -334,44 +425,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_ecs_task_definition", - "name": "square_import_job", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:2", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", - "cpu": "2048", - "ephemeral_storage": [], - "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", - "family": "square_import_job_prod", - "id": "square_import_job_prod", - "inference_accelerator": [], - "ipc_mode": "", - "memory": "8192", - "network_mode": "awsvpc", - "pid_mode": "", - "placement_constraints": [], - "proxy_configuration": [], - "requires_compatibilities": [ - "FARGATE" - ], - "revision": 2, - "runtime_platform": [], - "tags": {}, - "tags_all": {}, - "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", - "volume": [] - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, { "mode": "managed", "type": "aws_iam_access_key", @@ -1209,8 +1262,8 @@ "policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-scheduled-jobs-prod\"}],\"Version\":\"2012-10-17\"}", "receive_wait_time_seconds": 0, "redrive_policy": "", - "sqs_managed_sse_enabled": true, - "tags": null, + "sqs_managed_sse_enabled": false, + "tags": {}, "tags_all": {}, "url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod", "visibility_timeout_seconds": 30 diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index c8186d2f..ef979058 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.7", - "serial": 103, + "serial": 118, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -83,6 +83,97 @@ } ] }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "square_import_job_schedule", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "id": "square-import-job-schedule", + "is_enabled": true, + "name": "square-import-job-schedule", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(5 minutes)", + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "square_import_job_target", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [ + { + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "group": "", + "launch_type": "FARGATE", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "placement_constraint": [], + "platform_version": "", + "propagate_tags": "TASK_DEFINITION", + "tags": {}, + "task_count": 1, + "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165" + } + ], + "event_bus_name": "default", + "http_target": [], + "id": "square-import-job-schedule-square-import-job", + "input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"hello-world\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"hello-world\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"service\\\"}\"}] \n }\n ]\n}\n", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733", + "rule": "square-import-job-schedule", + "run_command_targets": [], + "sqs_target": [], + "target_id": "square-import-job" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.square_import_job_schedule", + "aws_ecs_task_definition.integreat_background_worker" + ] + } + ] + }, { "mode": "managed", "type": "aws_ecs_service", @@ -164,7 +255,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:351", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:354", "timeouts": { "delete": null }, @@ -243,7 +334,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:329", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:332", "timeouts": { "delete": null }, @@ -334,44 +425,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_ecs_task_definition", - "name": "square_import_job", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:2", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", - "cpu": "2048", - "ephemeral_storage": [], - "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", - "family": "square_import_job_prod", - "id": "square_import_job_prod", - "inference_accelerator": [], - "ipc_mode": "", - "memory": "8192", - "network_mode": "awsvpc", - "pid_mode": "", - "placement_constraints": [], - "proxy_configuration": [], - "requires_compatibilities": [ - "FARGATE" - ], - "revision": 2, - "runtime_platform": [], - "tags": null, - "tags_all": {}, - "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", - "volume": [] - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, { "mode": "managed", "type": "aws_iam_access_key", @@ -1183,6 +1236,42 @@ ] } ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "integreat-scheduled-jobs", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-east-1:679918342773:integreat-scheduled-jobs-prod", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "integreat-scheduled-jobs-prod", + "name_prefix": "", + "policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-scheduled-jobs-prod\"}],\"Version\":\"2012-10-17\"}", + "receive_wait_time_seconds": 0, + "redrive_policy": "", + "sqs_managed_sse_enabled": false, + "tags": {}, + "tags_all": {}, + "url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] } ] }