progress

src/clj/auto_ap/server.clj

@@ -93,7 +93,6 @@
      (cond (= job "square-import-job")
            (job-square/-main)
 
-
            (= job "reconcile-ledger")
            (job-reconcile-ledger/-main)
 

terraform/background-job/main.tf

@@ -4,6 +4,8 @@ variable "execution_role_arn" {}
 variable "stage" {}
 variable "job_name" {}
 variable "schedule" {}
+variable "cpu" { default = 2048 }
+variable "memory" { default = 8192 }
 
 
 
@@ -11,8 +13,8 @@ resource "aws_ecs_task_definition" "background_taskdef" {
 
   family                = replace("${var.job_name}_${var.stage}", "-", "_")
   container_definitions = replace(replace(file("./background-taskdef.json"), "STAGE", var.stage), "JOBX", var.job_name)
-  memory = 8192
+  memory = var.memory
-  cpu = 2048
+  cpu = var.cpu
   network_mode = "awsvpc"
   requires_compatibilities = ["FARGATE"]
   execution_role_arn = var.execution_role_arn

terraform/deploy.tf

@@ -332,4 +332,30 @@ module "square_import_job" {
        schedule = "rate(4 hours)"
        job_name = "square-import-job"
        execution_role_arn = var.execution_role_arn
+       memory = 4096
+       cpu = 1024
+}
+
+module "reconcile_ledger_job" {
+       source = "./background-job/"
+       ecs_cluster = var.ecs_cluster
+       task_role_arn = var.task_role_arn
+       stage = var.stage
+       schedule = "rate(1 hour)"
+       job_name = "reconcile-ledger"
+       execution_role_arn = var.execution_role_arn
+       memory = 8192
+       cpu = 2048
+}
+
+module "current_balance_cache" {
+       source = "./background-job/"
+       ecs_cluster = var.ecs_cluster
+       task_role_arn = var.task_role_arn
+       stage = var.stage
+       schedule = "rate(15 minutes)"
+       job_name = "current-balance-cache"
+       execution_role_arn = var.execution_role_arn
+       memory = 2048
+       cpu = 512
 }

terraform/terraform.tfstate.d/prod/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 158,
+  "serial": 168,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:358",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:359",
             "timeouts": {
               "delete": null
             },
@@ -243,7 +243,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:336",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:337",
             "timeouts": {
               "delete": null
             },
@@ -1182,6 +1182,270 @@
         }
       ]
     },
+    {
+      "module": "module.current_balance_cache",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_rule",
+      "name": "schedule",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:events:us-east-1:679918342773:rule/current-balance-cache-schedule",
+            "description": "",
+            "event_bus_name": "default",
+            "event_pattern": null,
+            "id": "current-balance-cache-schedule",
+            "is_enabled": true,
+            "name": "current-balance-cache-schedule",
+            "name_prefix": "",
+            "role_arn": "",
+            "schedule_expression": "rate(15 minutes)",
+            "tags": null,
+            "tags_all": {}
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
+    },
+    {
+      "module": "module.current_balance_cache",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_target",
+      "name": "job_target",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
+            "batch_target": [],
+            "dead_letter_config": [],
+            "ecs_target": [
+              {
+                "enable_ecs_managed_tags": false,
+                "enable_execute_command": false,
+                "group": "",
+                "launch_type": "",
+                "network_configuration": [
+                  {
+                    "assign_public_ip": true,
+                    "security_groups": [
+                      "sg-004e5855310c453a3",
+                      "sg-02d167406b1082698"
+                    ],
+                    "subnets": [
+                      "subnet-5e675761",
+                      "subnet-8519fde2",
+                      "subnet-89bab8d4"
+                    ]
+                  }
+                ],
+                "placement_constraint": [],
+                "platform_version": "",
+                "propagate_tags": "TASK_DEFINITION",
+                "tags": null,
+                "task_count": 1,
+                "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/current_balance_cache_prod:1"
+              }
+            ],
+            "event_bus_name": "default",
+            "http_target": [],
+            "id": "current-balance-cache-schedule-current-balance-cache",
+            "input": "",
+            "input_path": "",
+            "input_transformer": [],
+            "kinesis_target": [],
+            "redshift_target": [],
+            "retry_policy": [],
+            "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
+            "rule": "current-balance-cache-schedule",
+            "run_command_targets": [],
+            "sqs_target": [],
+            "target_id": "current-balance-cache"
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
+          "dependencies": [
+            "module.current_balance_cache.aws_cloudwatch_event_rule.schedule",
+            "module.current_balance_cache.aws_ecs_task_definition.background_taskdef"
+          ]
+        }
+      ]
+    },
+    {
+      "module": "module.current_balance_cache",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/current_balance_cache_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"current-balance-cache\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"current-balance-cache\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"current-balance-cache\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "512",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "current_balance_cache_prod",
+            "id": "current_balance_cache_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "2048",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
+    {
+      "module": "module.reconcile_ledger_job",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_rule",
+      "name": "schedule",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:events:us-east-1:679918342773:rule/reconcile-ledger-schedule",
+            "description": "",
+            "event_bus_name": "default",
+            "event_pattern": null,
+            "id": "reconcile-ledger-schedule",
+            "is_enabled": true,
+            "name": "reconcile-ledger-schedule",
+            "name_prefix": "",
+            "role_arn": "",
+            "schedule_expression": "rate(1 hour)",
+            "tags": null,
+            "tags_all": {}
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
+    },
+    {
+      "module": "module.reconcile_ledger_job",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_target",
+      "name": "job_target",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
+            "batch_target": [],
+            "dead_letter_config": [],
+            "ecs_target": [
+              {
+                "enable_ecs_managed_tags": false,
+                "enable_execute_command": false,
+                "group": "",
+                "launch_type": "",
+                "network_configuration": [
+                  {
+                    "assign_public_ip": true,
+                    "security_groups": [
+                      "sg-004e5855310c453a3",
+                      "sg-02d167406b1082698"
+                    ],
+                    "subnets": [
+                      "subnet-5e675761",
+                      "subnet-8519fde2",
+                      "subnet-89bab8d4"
+                    ]
+                  }
+                ],
+                "placement_constraint": [],
+                "platform_version": "",
+                "propagate_tags": "TASK_DEFINITION",
+                "tags": null,
+                "task_count": 1,
+                "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/reconcile_ledger_prod:1"
+              }
+            ],
+            "event_bus_name": "default",
+            "http_target": [],
+            "id": "reconcile-ledger-schedule-reconcile-ledger",
+            "input": "",
+            "input_path": "",
+            "input_transformer": [],
+            "kinesis_target": [],
+            "redshift_target": [],
+            "retry_policy": [],
+            "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
+            "rule": "reconcile-ledger-schedule",
+            "run_command_targets": [],
+            "sqs_target": [],
+            "target_id": "reconcile-ledger"
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
+          "dependencies": [
+            "module.reconcile_ledger_job.aws_cloudwatch_event_rule.schedule",
+            "module.reconcile_ledger_job.aws_ecs_task_definition.background_taskdef"
+          ]
+        }
+      ]
+    },
+    {
+      "module": "module.reconcile_ledger_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/reconcile_ledger_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"reconcile-ledger\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"reconcile-ledger\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"reconcile-ledger\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "2048",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "reconcile_ledger_prod",
+            "id": "reconcile_ledger_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "8192",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
     {
       "module": "module.square_import_job",
       "mode": "managed",
@@ -1248,7 +1512,7 @@
                 "propagate_tags": "TASK_DEFINITION",
                 "tags": {},
                 "task_count": 1,
-                "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3"
+                "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:4"
               }
             ],
             "event_bus_name": "default",
@@ -1285,16 +1549,16 @@
         {
           "schema_version": 1,
           "attributes": {
-            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3",
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:4",
             "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
-            "cpu": "2048",
+            "cpu": "1024",
             "ephemeral_storage": [],
             "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
             "family": "square_import_job_prod",
             "id": "square_import_job_prod",
             "inference_accelerator": [],
             "ipc_mode": "",
-            "memory": "8192",
+            "memory": "4096",
             "network_mode": "awsvpc",
             "pid_mode": "",
             "placement_constraints": [],
@@ -1302,9 +1566,9 @@
             "requires_compatibilities": [
               "FARGATE"
             ],
-            "revision": 3,
+            "revision": 4,
             "runtime_platform": [],
-            "tags": {},
+            "tags": null,
             "tags_all": {},
             "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
             "volume": []

terraform/terraform.tfstate.d/prod/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 156,
+  "serial": 158,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -1202,7 +1202,7 @@
             "name_prefix": "",
             "role_arn": "",
             "schedule_expression": "rate(4 hours)",
-            "tags": null,
+            "tags": {},
             "tags_all": {}
           },
           "sensitive_attributes": [],
@@ -1228,7 +1228,7 @@
                 "enable_ecs_managed_tags": false,
                 "enable_execute_command": false,
                 "group": "",
-                "launch_type": "FARGATE",
+                "launch_type": "",
                 "network_configuration": [
                   {
                     "assign_public_ip": true,
@@ -1246,7 +1246,7 @@
                 "placement_constraint": [],
                 "platform_version": "",
                 "propagate_tags": "TASK_DEFINITION",
-                "tags": null,
+                "tags": {},
                 "task_count": 1,
                 "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3"
               }