test

config/dev.edn

@@ -6,7 +6,9 @@
  :aws-region "us-east-1"
  :datomic-url "datomic:sql://invoices?jdbc:postgresql://database:5432/datomic?user=datomic&password=datomic"
  :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" 
+ :background-task-definition "integreat_background-worker_prod" 
  :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" 
+ :scheduled-jobs-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod" 
  :invoice-email "invoices-staging@mail.app.integreatconsult.com"
  :import-failure-destination-email "bryce@brycecovertoperations.com"
  :data-bucket "data.staging.app.integreatconsult.com"

project.clj

@@ -73,6 +73,10 @@
                  [com.amazonaws/aws-java-sdk-sqs "1.11.926"
                   :exclusions [commons-codec
                                org.apache.httpcomponents/httpclient]]
+
+                 [com.amazonaws/aws-java-sdk-ecs "1.11.926"
+                  :exclusions [commons-codec
+                               org.apache.httpcomponents/httpclient]]
                  [com.amazonaws/aws-java-sdk-ses "1.11.926"
                   :exclusions [commons-codec
                                org.apache.httpcomponents/httpclient]]

src/clj/auto_ap/server.clj

@@ -91,6 +91,14 @@
      (cond (= job "square-import-job")
            (job-square/-main)
 
+           (= job "hello-world")
+           (do
+             (log/info "HELLO WORLD")
+             (Thread/sleep 10000)
+             (log/info "GOODBYE WORLD")
+             (Thread/sleep 10000)
+             (System/exit 0))
+
            :else
            (let [without (cond-> []
                            (not (env :run-web? ))       (into [#'jetty

terraform/deploy.tf

@@ -37,17 +37,6 @@ resource "aws_ecs_task_definition" "integreat_background_worker" {
   task_role_arn = var.task_role_arn
 }
 
-resource "aws_ecs_task_definition" "square_import_job" {
-
-  family                = "square_import_job_${var.stage}"
-  container_definitions = file("${var.stage}-square-import-job-taskdef.json")
-  memory = 8192
-  cpu = 2048
-  network_mode = "awsvpc"
-  requires_compatibilities = ["FARGATE"]
-  execution_role_arn = var.execution_role_arn
-  task_role_arn = var.task_role_arn
-}
 
 resource "aws_ecs_service" "integreat_app" {
   name            = "integreat_app_${var.stage}"
@@ -334,3 +323,38 @@ resource "aws_service_discovery_service" "background_worker_service" {
     failure_threshold = 1
   }
 }
+
+resource "aws_cloudwatch_event_rule" "square_import_job_schedule" {
+  name        = "square-import-job-schedule"
+  schedule_expression = "rate(4 hours)"
+}
+
+
+resource "aws_cloudwatch_event_target" "square_import_job_target" {
+  target_id = "square-import-job"
+  arn       = var.ecs_cluster
+  rule      = aws_cloudwatch_event_rule.square_import_job_schedule.name
+  role_arn  = var.execution_role_arn
+
+  ecs_target {
+    task_count          = 1
+    task_definition_arn = aws_ecs_task_definition.integreat_background_worker.arn
+    network_configuration {
+      assign_public_ip = true
+      security_groups  = [ "sg-004e5855310c453a3", "sg-02d167406b1082698"]
+      subnets          = [ "subnet-5e675761", "subnet-8519fde2", "subnet-89bab8d4" ] 
+    }
+  }
+
+  input = <<DOC
+{
+  "containerOverrides": [
+    {
+      "name": "integreat-app",
+      "environment": [{"name": "INTEGREAT_JOB", "value": "hello-world"},
+                      {"name": "DD_SERVICE", "value": "hello-world"}] 
+    }
+  ]
+}
+DOC
+}

terraform/main.tf

@@ -112,6 +112,25 @@ resource "aws_sqs_queue" "integreat-mail" {
 POLICY
 }
 
+resource "aws_sqs_queue" "integreat-scheduled-jobs" {
+  name = "integreat-scheduled-jobs-${var.stage}"
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "${var.task_role_arn}"
+      },
+      "Action": "sqs:*",
+      "Resource": "arn:aws:sqs:*:*:integreat-scheduled-jobs-${var.stage}"
+    }
+  ]
+}
+POLICY
+}
+
 resource "aws_sqs_queue" "background-request" {
   name = "integreat-background-request-${var.stage}"
 

terraform/terraform.tfstate.d/prod/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 103,
+  "serial": 105,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:351",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:353",
             "timeouts": {
               "delete": null
             },
@@ -243,7 +243,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:329",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:331",
             "timeouts": {
               "delete": null
             },
@@ -362,7 +362,7 @@
             ],
             "revision": 2,
             "runtime_platform": [],
-            "tags": null,
+            "tags": {},
             "tags_all": {},
             "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
             "volume": []
@@ -1183,6 +1183,42 @@
           ]
         }
       ]
+    },
+    {
+      "mode": "managed",
+      "type": "aws_sqs_queue",
+      "name": "integreat-scheduled-jobs",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:sqs:us-east-1:679918342773:integreat-scheduled-jobs-prod",
+            "content_based_deduplication": false,
+            "deduplication_scope": "",
+            "delay_seconds": 0,
+            "fifo_queue": false,
+            "fifo_throughput_limit": "",
+            "id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod",
+            "kms_data_key_reuse_period_seconds": 300,
+            "kms_master_key_id": "",
+            "max_message_size": 262144,
+            "message_retention_seconds": 345600,
+            "name": "integreat-scheduled-jobs-prod",
+            "name_prefix": "",
+            "policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-scheduled-jobs-prod\"}],\"Version\":\"2012-10-17\"}",
+            "receive_wait_time_seconds": 0,
+            "redrive_policy": "",
+            "sqs_managed_sse_enabled": true,
+            "tags": null,
+            "tags_all": {},
+            "url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod",
+            "visibility_timeout_seconds": 30
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
     }
   ]
 }

terraform/terraform.tfstate.d/prod/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 100,
+  "serial": 103,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -343,8 +343,8 @@
         {
           "schema_version": 1,
           "attributes": {
-            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:1",
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:2",
-            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
             "cpu": "2048",
             "ephemeral_storage": [],
             "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
@@ -360,7 +360,7 @@
             "requires_compatibilities": [
               "FARGATE"
             ],
-            "revision": 1,
+            "revision": 2,
             "runtime_platform": [],
             "tags": null,
             "tags_all": {},