notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New form of square loading.

Browse Source
This commit is contained in:
Bryce Covert
2022-09-16 14:23:38 -07:00
parent f6758a850d
commit 98f21bf3df
6 changed files with 237 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/clj/auto_ap/jobs/square.clj
View File

@@ -8,7 +8,7 @@
[unilog.context :as lc]))
(defn -main [& _]
(lc/with-context {:background-job "square-loading"}
(lc/with-context {:background-job "square-loading-new"}
(mount/start (mount/only #{#'conn}))
((heartbeat square/upsert-all "square-loading"))
(mount/stop (mount/only #{#'conn}))

3
src/clj/auto_ap/server.clj
View File

@@ -103,8 +103,7 @@
(let [without (cond-> []
(not (env :run-web? )) (into [#'jetty
#'jetty-stats])
(not (env :run-background?)) (into [#'square/square-loader
#'vendor/refresh-vendor-usages-worker
(not (env :run-background?)) (into [#'vendor/refresh-vendor-usages-worker
#'ledger/touch-broken-ledger-worker
#'mail/import-invoices
#'ledger/process-txes-worker

2
src/clj/auto_ap/square/core.clj
View File

@@ -530,7 +530,7 @@
(some-> (:object &throw-context) str)
"Unknown error")}))))))
(mount/defstate square-loader
#_(mount/defstate square-loader
:start (scheduler/every (* 4 59 60 1000) (heartbeat upsert-all "square-loading"))
:stop (scheduler/stop square-loader))

11
terraform/deploy.tf
View File

@@ -330,14 +330,18 @@ resource "aws_cloudwatch_event_rule" "square_import_job_schedule" {
}
resource "aws_cloudwatch_event_target" "square_import_job_target" {
target_id = "square-import-job"
arn = var.ecs_cluster
rule = aws_cloudwatch_event_rule.square_import_job_schedule.name
role_arn = var.execution_role_arn
role_arn = "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733"
ecs_target {
task_count = 1
launch_type = "FARGATE"
# capacity_provider_strategy = [{capacity_provider = "FARGATE_SPOT"}]
task_definition_arn = aws_ecs_task_definition.integreat_background_worker.arn
network_configuration {
assign_public_ip = true
@@ -351,8 +355,9 @@ resource "aws_cloudwatch_event_target" "square_import_job_target" {
"containerOverrides": [
{
"name": "integreat-app",
"environment": [{"name": "INTEGREAT_JOB", "value": "hello-world"},
{"name": "DD_SERVICE", "value": "hello-world"}]
"environment": [{"name": "INTEGREAT_JOB", "value": "square-import-job"},
{"name": "DD_SERVICE", "value": "square-import-job"},
{"name": "DD_CONTAINER_ENV_AS_TAGS", "value":"{\"INTEGREAT_JOB\":\"task_family\"}"}]
}
]
}

139
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 105,
"serial": 121,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -83,6 +83,97 @@
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "square_import_job_schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(4 hours)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "square_import_job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"square-import-job\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"task_family\\\"}\"}] \n }\n ]\n}\n",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"aws_cloudwatch_event_rule.square_import_job_schedule",
"aws_ecs_task_definition.integreat_background_worker"
]
}
]
},
{
"mode": "managed",
"type": "aws_ecs_service",
@@ -164,7 +255,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:353",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:354",
"timeouts": {
"delete": null
},
@@ -243,7 +334,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:331",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:332",
"timeouts": {
"delete": null
},
@@ -334,44 +425,6 @@
}
]
},
{
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "square_import_job",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:2",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "2048",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "square_import_job_prod",
"id": "square_import_job_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 2,
"runtime_platform": [],
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"mode": "managed",
"type": "aws_iam_access_key",
@@ -1209,8 +1262,8 @@
"policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-scheduled-jobs-prod\"}],\"Version\":\"2012-10-17\"}",
"receive_wait_time_seconds": 0,
"redrive_policy": "",
"sqs_managed_sse_enabled": true,
"tags": null,
"sqs_managed_sse_enabled": false,
"tags": {},
"tags_all": {},
"url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod",
"visibility_timeout_seconds": 30

171
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 103,
"serial": 118,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -83,6 +83,97 @@
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "square_import_job_schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/square-import-job-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "square-import-job-schedule",
"is_enabled": true,
"name": "square-import-job-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(5 minutes)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "square_import_job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:165"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "square-import-job-schedule-square-import-job",
"input": "{\n \"containerOverrides\": [\n {\n \"name\": \"integreat-app\",\n \"environment\": [{\"name\": \"INTEGREAT_JOB\", \"value\": \"hello-world\"},\n {\"name\": \"DD_SERVICE\", \"value\": \"hello-world\"},\n {\"name\": \"DD_CONTAINER_ENV_AS_TAGS\", \"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"service\\\"}\"}] \n }\n ]\n}\n",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "square-import-job-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "square-import-job"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"aws_cloudwatch_event_rule.square_import_job_schedule",
"aws_ecs_task_definition.integreat_background_worker"
]
}
]
},
{
"mode": "managed",
"type": "aws_ecs_service",
@@ -164,7 +255,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:351",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:354",
"timeouts": {
"delete": null
},
@@ -243,7 +334,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:329",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_prod:332",
"timeouts": {
"delete": null
},
@@ -334,44 +425,6 @@
}
]
},
{
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "square_import_job",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/square_import_job_prod:2",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"square-import-job\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"square-import-job\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"square-import-job\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "2048",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "square_import_job_prod",
"id": "square_import_job_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 2,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"mode": "managed",
"type": "aws_iam_access_key",
@@ -1183,6 +1236,42 @@
]
}
]
},
{
"mode": "managed",
"type": "aws_sqs_queue",
"name": "integreat-scheduled-jobs",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:sqs:us-east-1:679918342773:integreat-scheduled-jobs-prod",
"content_based_deduplication": false,
"deduplication_scope": "",
"delay_seconds": 0,
"fifo_queue": false,
"fifo_throughput_limit": "",
"id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod",
"kms_data_key_reuse_period_seconds": 300,
"kms_master_key_id": "",
"max_message_size": 262144,
"message_retention_seconds": 345600,
"name": "integreat-scheduled-jobs-prod",
"name_prefix": "",
"policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-scheduled-jobs-prod\"}],\"Version\":\"2012-10-17\"}",
"receive_wait_time_seconds": 0,
"redrive_policy": "",
"sqs_managed_sse_enabled": false,
"tags": {},
"tags_all": {},
"url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-scheduled-jobs-prod",
"visibility_timeout_seconds": 30
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
}
]
}