splitting out background tasks

This commit is contained in:
2022-09-18 06:51:17 -07:00
parent 8b271e9e2a
commit 5017d9bc67
6 changed files with 287 additions and 244 deletions

View File

@@ -16,7 +16,6 @@
[datomic.api :as d]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]
[clojure.set :as set])
(:import
(java.util UUID)
@@ -264,10 +263,6 @@
(defn refresh-current-balance []
(build-current-balance (bank-accounts-needing-refresh)))
(mount/defstate current-balance-worker
:start (scheduler/every (* 17 60 1000) (heartbeat refresh-current-balance "current-balance-cache"))
:stop (scheduler/stop current-balance-worker))
(defn get-client [context _ _]
(->graphql
(->> (d-clients/get-all)

View File

@@ -222,12 +222,6 @@
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs)))))
(mount/defstate reconciliation-frequency :start (* 1000 60 60))
(mount/defstate ledger-reconciliation-worker
:start (scheduler/every reconciliation-frequency (heartbeat reconcile-ledger "reconcile-ledger"))
:stop (scheduler/stop ledger-reconciliation-worker))
(defn touch-transaction [e]
@(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
@@ -465,7 +459,3 @@
:text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low}
nil))
(mount/defstate touch-broken-ledger-worker
:start (scheduler/every reconciliation-frequency (heartbeat touch-broken-ledger "touch-broken-ledger"))
:stop (scheduler/stop touch-broken-ledger-worker))

View File

@@ -16,6 +16,8 @@
[auto-ap.background.metrics :as metrics]
[auto-ap.background.mail :as mail]
[auto-ap.jobs.square :as job-square]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.current-balance-cache :as job-current-balance-cache]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[mount.core :as mount]
@@ -91,6 +93,13 @@
(cond (= job "square-import-job")
(job-square/-main)
(= job "reconcile-ledger")
(job-reconcile-ledger/-main)
(= job "current-balance-cache")
(job-current-balance-cache/-main)
(= job "hello-world")
(do
(log/info "HELLO WORLD")
@@ -104,14 +113,11 @@
(not (env :run-web? )) (into [#'jetty
#'jetty-stats])
(not (env :run-background?)) (into [#'vendor/refresh-vendor-usages-worker
#'ledger/touch-broken-ledger-worker
#'mail/import-invoices
#'ledger/process-txes-worker
#'ledger/ledger-reconciliation-worker
#'requests/request-listener
#'sysco/sysco-invoice-importer
#'auto-ap.background.invoices/close-auto-invoices-worker
#'gq-clients/current-balance-worker
#'yodlee2/import-worker
#'yodlee2/account-worker
#'intuit/import-worker

View File

@@ -324,42 +324,12 @@ resource "aws_service_discovery_service" "background_worker_service" {
}
}
resource "aws_cloudwatch_event_rule" "square_import_job_schedule" {
name = "square-import-job-schedule"
schedule_expression = "cron(45 * * * ? *)"
}
resource "aws_cloudwatch_event_target" "square_import_job_target" {
target_id = "square-import-job"
arn = var.ecs_cluster
rule = aws_cloudwatch_event_rule.square_import_job_schedule.name
role_arn = "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733"
ecs_target {
task_count = 1
launch_type = "FARGATE"
# capacity_provider_strategy = [{capacity_provider = "FARGATE_SPOT"}]
task_definition_arn = aws_ecs_task_definition.integreat_background_worker.arn
network_configuration {
assign_public_ip = true
security_groups = [ "sg-004e5855310c453a3", "sg-02d167406b1082698"]
subnets = [ "subnet-5e675761", "subnet-8519fde2", "subnet-89bab8d4" ]
}
}
input = <<DOC
{
"containerOverrides": [
{
"name": "integreat-app",
"environment": [{"name": "INTEGREAT_JOB", "value": "square-import-job"},
{"name": "DD_CONTAINER_ENV_AS_TAGS", "value":"{\"INTEGREAT_JOB\":\"background_job\"}"},
{"name": "DD_SERVICE", "value": "square-import-job"}]
}
]
}
DOC
module "square_import_job" {
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
schedule = "rate(4 hours)"
job_name = "square-import-job"
execution_role_arn = var.execution_role_arn
}

View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 145,
"serial": 158,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -83,97 +83,6 @@
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "square_import_job_schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "cron(45 * * * ? *)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "square_import_job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"square-import-job\"}]\n }\n ]\n}\n",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"aws_cloudwatch_event_rule.square_import_job_schedule",
"aws_ecs_task_definition.integreat_background_worker"
]
}
]
},
{
"mode": "managed",
"type": "aws_ecs_service",
@@ -255,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:356",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:358",
"timeouts": {
"delete": null
},
@@ -334,7 +243,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:334",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:336",
"timeouts": {
"delete": null
},
@@ -1272,6 +1181,138 @@
"private": "bnVsbA=="
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(4 hours)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"module.square_import_job.aws_cloudwatch_event_rule.schedule",
"module.square_import_job.aws_ecs_task_definition.background_taskdef"
]
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "2048",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "square_import_job_prod",
"id": "square_import_job_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 3,
"runtime_platform": [],
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
}
]
}

View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 143,
"serial": 156,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -83,97 +83,6 @@
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "square_import_job_schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "cron(30 * * * ? *)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "square_import_job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"square-import-job\"}]\n }\n ]\n}\n",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"aws_cloudwatch_event_rule.square_import_job_schedule",
"aws_ecs_task_definition.integreat_background_worker"
]
}
]
},
{
"mode": "managed",
"type": "aws_ecs_service",
@@ -255,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:356",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:358",
"timeouts": {
"delete": null
},
@@ -334,7 +243,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:334",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:336",
"timeouts": {
"delete": null
},
@@ -1272,6 +1181,138 @@
"private": "bnVsbA=="
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(4 hours)",
"tags": null,
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": null,
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"module.square_import_job.aws_cloudwatch_event_rule.schedule",
"module.square_import_job.aws_ecs_task_definition.background_taskdef"
]
}
]
},
{
"module": "module.square_import_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:3",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "2048",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "square_import_job_prod",
"id": "square_import_job_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 3,
"runtime_platform": [],
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
}
]
}