From 6119e68cc8d253051a9f28273a5b7eb6bb00abb0 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 24 Dec 2021 08:53:23 -0800 Subject: [PATCH 1/4] updated terraform --- .../staging-background-worker-taskdef.json | 4 +- .../staging/terraform.tfstate | 370 ++++++++++++++++-- .../staging/terraform.tfstate.backup | 33 +- 3 files changed, 367 insertions(+), 40 deletions(-) diff --git a/terraform/staging-background-worker-taskdef.json b/terraform/staging-background-worker-taskdef.json index 18b8765a..9f69fe1c 100644 --- a/terraform/staging-background-worker-taskdef.json +++ b/terraform/staging-background-worker-taskdef.json @@ -20,7 +20,7 @@ "environment": [ { "name": "config", - "value": "/usr/local/config/prod-background-worker.edn" + "value": "/usr/local/config/staging-background-worker.edn" } ], "mountPoints": [], @@ -28,7 +28,7 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/integreat-background-worker-prod", + "awslogs-group": "/ecs/integreat-background-worker-staging", "awslogs-region": "us-east-1", "awslogs-stream-prefix": "ecs" } diff --git a/terraform/terraform.tfstate.d/staging/terraform.tfstate b/terraform/terraform.tfstate.d/staging/terraform.tfstate index 069c3b5d..520b1c4e 100644 --- a/terraform/terraform.tfstate.d/staging/terraform.tfstate +++ b/terraform/terraform.tfstate.d/staging/terraform.tfstate @@ -1,12 +1,13 @@ { "version": 4, - "terraform_version": "0.13.3", - "serial": 70, + "terraform_version": "0.14.4", + "serial": 84, "lineage": "91d10fe0-8033-8778-c202-78d5a81632e8", "outputs": { "aws_access_key_id": { "value": "AKIAJIS67OSJARD2E6VQ", - "type": "string" + "type": "string", + "sensitive": true }, "aws_default_region": { "value": "us-east-1", @@ -14,7 +15,8 @@ }, "aws_secret_access_key": { "value": "Z+AOjQU9M4SwKVU2meYtyNxXtz1Axu/9xohvteXf", - "type": "string" + "type": "string", + "sensitive": true }, "queue_url": { "value": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging", @@ -33,9 +35,10 @@ "attributes": { "account_id": "679918342773", "arn": "arn:aws:iam::679918342773:user/bryce", - "id": "2020-09-25 00:21:01.39516 +0000 UTC", + "id": "679918342773", "user_id": "AIDAJPUJFTOKO4IRADMV4" - } + }, + "sensitive_attributes": [] } ] }, @@ -71,9 +74,11 @@ "status": "ISSUED", "subject_alternative_names": [], "tags": {}, + "tags_all": {}, "validation_emails": [], "validation_method": "DNS" }, + "sensitive_attributes": [], "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjAifQ==" } ] @@ -89,6 +94,12 @@ "attributes": { "capacity_provider_strategy": [], "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "deployment_circuit_breaker": [ + { + "enable": false, + "rollback": false + } + ], "deployment_controller": [ { "type": "ECS" @@ -98,10 +109,11 @@ "deployment_minimum_healthy_percent": 100, "desired_count": 1, "enable_ecs_managed_tags": false, + "enable_execute_command": false, "force_new_deployment": null, "health_check_grace_period_seconds": 600, "iam_role": "aws-service-role", - "id": "arn:aws:ecs:us-east-1:679918342773:service/integreat_app_staging", + "id": "arn:aws:ecs:us-east-1:679918342773:service/default/integreat_app_staging", "launch_type": "FARGATE", "load_balancer": [ { @@ -136,19 +148,101 @@ "container_name": "", "container_port": 0, "port": 0, - "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-6auj2wqsh55k2nuj" + "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-nsn252bfk4r6bzpj" } ], - "tags": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:3", + "tags": null, + "tags_all": {}, + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:81", "timeouts": { "delete": null - } + }, + "wait_for_steady_state": false }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_ecs_task_definition.integreat_app", - "aws_lb_target_group.integreat_app" + "aws_lb_target_group.integreat_app", + "aws_service_discovery_service.service" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_ecs_service", + "name": "integreat_background_worker", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "capacity_provider_strategy": [], + "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "deployment_circuit_breaker": [ + { + "enable": false, + "rollback": false + } + ], + "deployment_controller": [ + { + "type": "ECS" + } + ], + "deployment_maximum_percent": 200, + "deployment_minimum_healthy_percent": 100, + "desired_count": 1, + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "force_new_deployment": null, + "health_check_grace_period_seconds": 0, + "iam_role": "aws-service-role", + "id": "arn:aws:ecs:us-east-1:679918342773:service/default/integreat_background_worker_staging", + "launch_type": "FARGATE", + "load_balancer": [], + "name": "integreat_background_worker_staging", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "ordered_placement_strategy": [], + "placement_constraints": [], + "platform_version": "LATEST", + "propagate_tags": "NONE", + "scheduling_strategy": "REPLICA", + "service_registries": [ + { + "container_name": "", + "container_port": 0, + "port": 0, + "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-4cuqgdwqo5acwqtq" + } + ], + "tags": null, + "tags_all": {}, + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:1", + "timeouts": { + "delete": null + }, + "wait_for_steady_state": false + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_ecs_task_definition.integreat_background_worker", + "aws_service_discovery_service.background_worker_service" ] } ] @@ -162,14 +256,53 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:3", - "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_staging:81", + "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/staging.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":3000,\"hostPort\":3000,\"protocol\":\"tcp\"},{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", "cpu": "2048", + "ephemeral_storage": [], "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", "family": "integreat_app_staging", "id": "integreat_app_staging", "inference_accelerator": [], "ipc_mode": "", + "memory": "8192", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 81, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "integreat_background_worker", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_background-worker_staging:1", + "container_definitions": "[{\"cpu\":0,\"environment\":[{\"name\":\"config\",\"value\":\"/usr/local/config/staging-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-staging\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "integreat_background-worker_staging", + "id": "integreat_background-worker_staging", + "inference_accelerator": [], + "ipc_mode": "", "memory": "4096", "network_mode": "awsvpc", "pid_mode": "", @@ -178,11 +311,14 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 3, - "tags": {}, + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] }, + "sensitive_attributes": [], "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" } ] @@ -196,7 +332,9 @@ { "schema_version": 0, "attributes": { + "create_date": "2018-04-08T22:09:45Z", "encrypted_secret": null, + "encrypted_ses_smtp_password_v4": null, "id": "AKIAJIS67OSJARD2E6VQ", "key_fingerprint": null, "pgp_key": null, @@ -205,6 +343,7 @@ "status": "Active", "user": "integreat-staging" }, + "sensitive_attributes": [], "dependencies": [ "aws_iam_user.app_user" ] @@ -227,8 +366,10 @@ "path": "/", "permissions_boundary": null, "tags": {}, + "tags_all": {}, "unique_id": "AIDAJ4KJ7STJZNLIN3M6S" - } + }, + "sensitive_attributes": [] } ] }, @@ -245,6 +386,7 @@ "policy_arn": "arn:aws:iam::aws:policy/AdministratorAccess", "user": "integreat-staging" }, + "sensitive_attributes": [], "dependencies": [ "aws_iam_user.app_user" ] @@ -269,13 +411,16 @@ ], "arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:loadbalancer/app/integreat-app-staging/dc040205e561419e", "arn_suffix": "app/integreat-app-staging/dc040205e561419e", + "customer_owned_ipv4_pool": "", + "desync_mitigation_mode": "defensive", "dns_name": "integreat-app-staging-229185591.us-east-1.elb.amazonaws.com", "drop_invalid_header_fields": false, "enable_cross_zone_load_balancing": null, "enable_deletion_protection": true, "enable_http2": true, + "enable_waf_fail_open": false, "id": "arn:aws:elasticloadbalancing:us-east-1:679918342773:loadbalancer/app/integreat-app-staging/dc040205e561419e", - "idle_timeout": 60, + "idle_timeout": 120, "internal": false, "ip_address_type": "ipv4", "load_balancer_type": "application", @@ -287,31 +432,43 @@ "subnet_mapping": [ { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-16161a39" }, { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-323deb78" }, { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-44c2774b" }, { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-5e675761" }, { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-8519fde2" }, { "allocation_id": "", + "ipv6_address": "", + "outpost_id": "", "private_ipv4_address": "", "subnet_id": "subnet-89bab8d4" } @@ -325,10 +482,12 @@ "subnet-89bab8d4" ], "tags": {}, + "tags_all": {}, "timeouts": null, "vpc_id": "vpc-b5b7d6ce", "zone_id": "Z35SXDOTRQ7X7K" }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19" } ] @@ -342,6 +501,7 @@ { "schema_version": 0, "attributes": { + "alpn_policy": null, "arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:listener/app/integreat-app-staging/dc040205e561419e/8e5542063f461bb4", "certificate_arn": null, "default_action": [ @@ -370,10 +530,13 @@ "port": 80, "protocol": "HTTP", "ssl_policy": "", + "tags": {}, + "tags_all": {}, "timeouts": { "read": null } }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsicmVhZCI6NjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_lb.integreat_app" @@ -390,6 +553,7 @@ { "schema_version": 0, "attributes": { + "alpn_policy": null, "arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:listener/app/integreat-app-staging/dc040205e561419e/b89075bc6a559a4f", "certificate_arn": "arn:aws:acm:us-east-1:679918342773:certificate/eec88d2f-93c4-4e7e-a167-e75d853f0759", "default_action": [ @@ -409,10 +573,13 @@ "port": 443, "protocol": "HTTPS", "ssl_policy": "ELBSecurityPolicy-2016-08", + "tags": {}, + "tags_all": {}, "timeouts": { "read": null } }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsicmVhZCI6NjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_acm_certificate.cert", @@ -464,7 +631,7 @@ "/css/*", "/finance-font/*", "/img/*", - "/js/compiled/app.js", + "/js/compiled/*", "index.html" ] } @@ -475,14 +642,14 @@ ], "id": "arn:aws:elasticloadbalancing:us-east-1:679918342773:listener-rule/app/integreat-app-staging/dc040205e561419e/b89075bc6a559a4f/8bc88fa379868c55", "listener_arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:listener/app/integreat-app-staging/dc040205e561419e/b89075bc6a559a4f", - "priority": 1 + "priority": 1, + "tags": {}, + "tags_all": {} }, + "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ - "aws_acm_certificate.cert", - "aws_lb.integreat_app", - "aws_lb_listener.https", - "aws_lb_target_group.integreat_app" + "aws_lb_listener.https" ] } ] @@ -498,7 +665,8 @@ "attributes": { "arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:targetgroup/integreat-app-staging/34af39c4c46e7ce1", "arn_suffix": "targetgroup/integreat-app-staging/34af39c4c46e7ce1", - "deregistration_delay": 120, + "connection_termination": false, + "deregistration_delay": "120", "health_check": [ { "enabled": true, @@ -508,8 +676,8 @@ "path": "/api/health-check", "port": "traffic-port", "protocol": "HTTP", - "timeout": 5, - "unhealthy_threshold": 2 + "timeout": 14, + "unhealthy_threshold": 5 } ], "id": "arn:aws:elasticloadbalancing:us-east-1:679918342773:targetgroup/integreat-app-staging/34af39c4c46e7ce1", @@ -518,20 +686,25 @@ "name": "integreat-app-staging", "name_prefix": null, "port": 80, + "preserve_client_ip": null, "protocol": "HTTP", + "protocol_version": "HTTP1", "proxy_protocol_v2": false, "slow_start": 0, "stickiness": [ { "cookie_duration": 86400, + "cookie_name": "", "enabled": false, "type": "lb_cookie" } ], "tags": {}, + "tags_all": {}, "target_type": "ip", "vpc_id": "vpc-b5b7d6ce" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -560,12 +733,13 @@ "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], - "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.staging.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"}],\"Version\":\"2012-10-17\"}", + "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.staging.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.staging.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", "server_side_encryption_configuration": [], "tags": {}, + "tags_all": {}, "versioning": [ { "enabled": false, @@ -583,6 +757,7 @@ "website_domain": "s3-website-us-east-1.amazonaws.com", "website_endpoint": "data.staging.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -617,6 +792,7 @@ "request_payer": "BucketOwner", "server_side_encryption_configuration": [], "tags": {}, + "tags_all": {}, "versioning": [ { "enabled": false, @@ -627,6 +803,7 @@ "website_domain": null, "website_endpoint": null }, + "sensitive_attributes": [], "dependencies": [ "data.aws_caller_identity.current" ] @@ -680,6 +857,7 @@ "request_payer": "BucketOwner", "server_side_encryption_configuration": [], "tags": {}, + "tags_all": {}, "versioning": [ { "enabled": false, @@ -697,6 +875,7 @@ "website_domain": "s3-website-us-east-1.amazonaws.com", "website_endpoint": "staging3.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -726,6 +905,7 @@ ], "topic": [] }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "aws_sqs_queue.integreat-mail", @@ -734,6 +914,88 @@ } ] }, + { + "mode": "managed", + "type": "aws_service_discovery_service", + "name": "background_worker_service", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-4cuqgdwqo5acwqtq", + "description": "", + "dns_config": [ + { + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "namespace_id": "ns-gv2z744em7myo2jp", + "routing_policy": "MULTIVALUE" + } + ], + "force_destroy": false, + "health_check_config": [], + "health_check_custom_config": [ + { + "failure_threshold": 1 + } + ], + "id": "srv-4cuqgdwqo5acwqtq", + "name": "integreat-background-worker-staging", + "namespace_id": "ns-gv2z744em7myo2jp", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_service_discovery_service", + "name": "service", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-nsn252bfk4r6bzpj", + "description": "", + "dns_config": [ + { + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "namespace_id": "ns-gv2z744em7myo2jp", + "routing_policy": "MULTIVALUE" + } + ], + "force_destroy": false, + "health_check_config": [], + "health_check_custom_config": [ + { + "failure_threshold": 1 + } + ], + "id": "srv-nsn252bfk4r6bzpj", + "name": "integreat-app-staging", + "namespace_id": "ns-gv2z744em7myo2jp", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_ses_receipt_rule", @@ -745,6 +1007,7 @@ "attributes": { "add_header_action": [], "after": null, + "arn": "arn:aws:ses:us-east-1:679918342773:receipt-rule-set/default-rule-set:receipt-rule/store-staging", "bounce_action": [], "enabled": true, "id": "store-staging", @@ -769,6 +1032,7 @@ "tls_policy": "Optional", "workmail_action": [] }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "aws_ses_receipt_rule_set.main", @@ -786,9 +1050,47 @@ { "schema_version": 0, "attributes": { + "arn": "arn:aws:ses:us-east-1:679918342773:receipt-rule-set/default-rule-set", "id": "default-rule-set", "rule_set_name": "default-rule-set" - } + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "background-request", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-east-1:679918342773:integreat-background-request-staging", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "integreat-background-request-staging", + "name_prefix": "", + "policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-background-request-staging\"}],\"Version\":\"2012-10-17\"}", + "receive_wait_time_seconds": 0, + "redrive_policy": "", + "sqs_managed_sse_enabled": false, + "tags": null, + "tags_all": {}, + "url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-staging", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "bnVsbA==" } ] }, @@ -803,21 +1105,27 @@ "attributes": { "arn": "arn:aws:sqs:us-east-1:679918342773:integreat-mail-staging", "content_based_deduplication": false, + "deduplication_scope": "", "delay_seconds": 0, "fifo_queue": false, + "fifo_throughput_limit": "", "id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging", "kms_data_key_reuse_period_seconds": 300, "kms_master_key_id": "", "max_message_size": 262144, "message_retention_seconds": 345600, "name": "integreat-mail-staging", - "name_prefix": null, - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":\"*\",\"Action\":\"sqs:SendMessage\",\"Resource\":\"arn:aws:sqs:*:*:integreat-mail-staging\",\"Condition\":{\"ArnEquals\":{\"aws:SourceArn\":\"arn:aws:s3:::integreat-mail-staging\"}}}]}", + "name_prefix": "", + "policy": "{\"Statement\":[{\"Action\":\"sqs:SendMessage\",\"Condition\":{\"ArnEquals\":{\"aws:SourceArn\":\"arn:aws:s3:::integreat-mail-staging\"}},\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:sqs:*:*:integreat-mail-staging\"}],\"Version\":\"2012-10-17\"}", "receive_wait_time_seconds": 0, "redrive_policy": "", + "sqs_managed_sse_enabled": false, "tags": {}, + "tags_all": {}, + "url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging", "visibility_timeout_seconds": 30 }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "data.aws_caller_identity.current" diff --git a/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup b/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup index 00c8af88..eb77bae5 100644 --- a/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/staging/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "0.13.3", - "serial": 68, + "terraform_version": "0.14.4", + "serial": 70, "lineage": "91d10fe0-8033-8778-c202-78d5a81632e8", "outputs": { "aws_access_key_id": { @@ -33,9 +33,10 @@ "attributes": { "account_id": "679918342773", "arn": "arn:aws:iam::679918342773:user/bryce", - "id": "2020-09-25 00:18:59.261808 +0000 UTC", + "id": "2020-09-25 00:21:01.39516 +0000 UTC", "user_id": "AIDAJPUJFTOKO4IRADMV4" - } + }, + "sensitive_attributes": [] } ] }, @@ -74,6 +75,7 @@ "validation_emails": [], "validation_method": "DNS" }, + "sensitive_attributes": [], "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjAifQ==" } ] @@ -145,6 +147,7 @@ "delete": null } }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_ecs_task_definition.integreat_app", @@ -183,6 +186,7 @@ "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] }, + "sensitive_attributes": [], "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" } ] @@ -205,6 +209,7 @@ "status": "Active", "user": "integreat-staging" }, + "sensitive_attributes": [], "dependencies": [ "aws_iam_user.app_user" ] @@ -228,7 +233,8 @@ "permissions_boundary": null, "tags": {}, "unique_id": "AIDAJ4KJ7STJZNLIN3M6S" - } + }, + "sensitive_attributes": [] } ] }, @@ -245,6 +251,7 @@ "policy_arn": "arn:aws:iam::aws:policy/AdministratorAccess", "user": "integreat-staging" }, + "sensitive_attributes": [], "dependencies": [ "aws_iam_user.app_user" ] @@ -329,6 +336,7 @@ "vpc_id": "vpc-b5b7d6ce", "zone_id": "Z35SXDOTRQ7X7K" }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19" } ] @@ -374,6 +382,7 @@ "read": null } }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsicmVhZCI6NjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_lb.integreat_app" @@ -413,6 +422,7 @@ "read": null } }, + "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsicmVhZCI6NjAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_acm_certificate.cert", @@ -477,6 +487,7 @@ "listener_arn": "arn:aws:elasticloadbalancing:us-east-1:679918342773:listener/app/integreat-app-staging/dc040205e561419e/b89075bc6a559a4f", "priority": 1 }, + "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_acm_certificate.cert", @@ -532,6 +543,7 @@ "target_type": "ip", "vpc_id": "vpc-b5b7d6ce" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -583,6 +595,7 @@ "website_domain": "s3-website-us-east-1.amazonaws.com", "website_endpoint": "data.staging.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -627,6 +640,7 @@ "website_domain": null, "website_endpoint": null }, + "sensitive_attributes": [], "dependencies": [ "data.aws_caller_identity.current" ] @@ -674,7 +688,7 @@ "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], - "policy": "{\n \"Id\": \"Policy1526084187222\",\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"Stmt1526084185514\",\n \"Action\": [\n \"s3:GetObject\"\n ],\n \"Effect\": \"Allow\",\n \"Resource\": \"arn:aws:s3:::staging.app.integreatconsult.com/*\",\n \"Principal\": \"*\"\n }\n ]\n}\n", + "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::staging3.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"}],\"Version\":\"2012-10-17\"}", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", @@ -697,6 +711,7 @@ "website_domain": "s3-website-us-east-1.amazonaws.com", "website_endpoint": "staging3.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" }, + "sensitive_attributes": [], "private": "bnVsbA==" } ] @@ -726,6 +741,7 @@ ], "topic": [] }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "aws_sqs_queue.integreat-mail", @@ -769,6 +785,7 @@ "tls_policy": "Optional", "workmail_action": [] }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "aws_ses_receipt_rule_set.main", @@ -788,7 +805,8 @@ "attributes": { "id": "default-rule-set", "rule_set_name": "default-rule-set" - } + }, + "sensitive_attributes": [] } ] }, @@ -818,6 +836,7 @@ "tags": {}, "visibility_timeout_seconds": 30 }, + "sensitive_attributes": [], "dependencies": [ "aws_s3_bucket.invoices", "data.aws_caller_identity.current" From f7bec7a86cf7a717582518765f2a50bfa86ca4d2 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 25 Dec 2021 07:07:41 -0800 Subject: [PATCH 2/4] making tests pass again --- src/clj/auto_ap/graphql.clj | 7 +- src/clj/auto_ap/import/transactions.clj | 83 +++++++++++-------- test/clj/auto_ap/functional/test.clj | 52 ------------ test/clj/auto_ap/import/transactions_test.clj | 17 +++- test/clj/auto_ap/integration/graphql.clj | 70 +++++++--------- 5 files changed, 99 insertions(+), 130 deletions(-) delete mode 100644 test/clj/auto_ap/functional/test.clj diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index ed6aacc3..1b2d12bb 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -40,7 +40,12 @@ (def integreat-schema { - :scalars {:id {:parse #(when % (Long/parseLong %)) + :scalars {:id {:parse #(cond (number? %) + % + + % + (Long/parseLong %)) + :serialize #(.toString %)} :ident {:parse (fn [x] {:db/ident x}) :serialize #(or (:ident %) (:db/ident %) %)} diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 55449b19..69fcc5c4 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -196,46 +196,57 @@ :else :import))) +(defn maybe-assoc-check-number [transaction] + (if-let [check-number (or (:transaction/check-number transaction) + (extract-check-number transaction))] + (assoc transaction :transaction/check-number transaction) + transaction)) + + +(defn maybe-clear-payment [{:transaction/keys [check-number client bank-account amount id] :as transaction}] + (when-let [existing-payment (transaction->existing-payment transaction check-number client bank-account amount id)] + (assoc transaction + :transaction/approval-status :transaction-approval-status/approved + :transaction/payment {:db/id (:db/id existing-payment) + :payment/status :payment-status/cleared} + :transaction/vendor (:db/id (:payment/vendor existing-payment)) + :transaction/location "A" + :transaction/accounts [#:transaction-account + {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :location "A" + :amount (Math/abs (double amount))}]))) + +(defn maybe-autopay-invoices [{:transaction/keys [amount client bank-account] :as transaction}] + (when-let [autopay-invoices-matches (seq (match-transaction-to-unfulfilled-autopayments amount client))] + (add-new-payment autopay-invoices-matches bank-account client))) + +(defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}] + (when (>= amount 0.0) + (when-let [expected-deposit (find-expected-deposit client amount (coerce/to-date-time date))] + (assoc transaction :transaction/expected-deposit {:db/id expected-deposit + :expected-deposit/status :expected-deposit-status/cleared})))) + +(defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations] + (if-let [unpaid-invoices (seq (match-transaction-to-unpaid-invoices amount client))] + nil + (apply-rules transaction valid-locations))) + (defn transaction->txs [transaction bank-account apply-rules] (let [bank-account-id (:db/id bank-account) client (:client/_bank-accounts bank-account) client-id (:db/id client) - valid-locations (or (:bank-account/locations bank-account) (:client/locations client))] - (into [] - (let [{:transaction/keys [amount id date]} transaction - check-number (extract-check-number transaction) - existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id) - autopay-invoices-matches (when-not existing-check - (match-transaction-to-unfulfilled-autopayments amount client-id)) - unpaid-invoices-matches (when-not existing-check - (match-transaction-to-unpaid-invoices amount client-id )) - expected-deposit (when (and (> amount 0.0) - (not existing-check)) - (find-expected-deposit (:db/id client) amount (coerce/to-date-time date)))] - (cond-> - [(assoc transaction :transaction/approval-status :transaction-approval-status/unapproved)] - check-number (update 0 #(assoc % :transaction/check-number check-number)) - existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved - :transaction/payment {:db/id (:db/id existing-check) - :payment/status :payment-status/cleared} - :transaction/vendor (:db/id (:payment/vendor existing-check)) - :transaction/location "A" - :transaction/accounts [#:transaction-account - {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :location "A" - :amount (Math/abs (double amount))}])) - - ;; temporarily removed to automatically match autopaid invoices - #_(and (not existing-check) - (seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id) - expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit - :expected-deposit/status :expected-deposit-status/cleared})) - - - (and (not (seq autopay-invoices-matches)) - (not (seq unpaid-invoices-matches)) - (not expected-deposit)) (update 0 #(apply-rules % valid-locations)) - true (update 0 remove-nils)))))) + valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) + code-fn (some-fn maybe-clear-payment + maybe-clear-expected-deposit + #_maybe-autopay-invoices + #(maybe-code % apply-rules valid-locations) + identity)] + [(-> transaction + (assoc :transaction/client client-id) + (assoc :transaction/bank-account bank-account-id) + (assoc :transaction/approval-status :transaction-approval-status/unapproved) + maybe-assoc-check-number + code-fn)])) diff --git a/test/clj/auto_ap/functional/test.clj b/test/clj/auto_ap/functional/test.clj deleted file mode 100644 index 21ce7b91..00000000 --- a/test/clj/auto_ap/functional/test.clj +++ /dev/null @@ -1,52 +0,0 @@ -(ns auto-ap.functional.test - (:require [clojure.test :as t :refer :all] - [etaoin.api :as e] - [etaoin.keys :as k] - [config.core :refer [env]] - [clj-time.core :as time] - [buddy.sign.jwt :as jwt])) - -(def base-url "https://staging3.app.integreatconsult.com") -(defn login-admin [driver] - (let [jwt-token (jwt/sign {:user "Automated Tests" - :exp (time/plus (time/now) (time/days 30)) - :user/role "admin" - :user/name "Automated Tests"} - (:jwt-secret env) - {:alg :hs512})] - (e/go driver (str base-url "/?jwt=" jwt-token)) - (e/wait-visible driver {:tag :h1 - :class "title"}))) - -(deftest create-invoice - (testing "Creating a new invoice" - (e/with-wait-timeout 10 - (e/with-firefox {} driver - (login-admin driver) - (e/click driver {:tag :a :fn/text "Invoices"}) - (e/wait-visible driver {:tag :h1 - :class "title" - :fn/text "Unpaid Invoices"}))))) - - -(deftest edit-client - (testing "Editing a client" - (e/with-wait-timeout 10 - (e/with-firefox {} driver - (login-admin driver) - - (e/click driver {:tag :a :class "navbar-link login"}) - (e/click driver {:tag :a :fn/text "Administration"}) - - (e/wait-visible driver {:tag :h1 - :class "title" - :fn/text "Admin"}) - (e/click driver {:tag :span :fn/text "Clients"}) - (e/wait-visible driver {:tag :h1 - :class "title" - :fn/text "Clients"}) - (e/click driver {:tag :i :class "fa fa-pencil"}) - (e/wait-visible driver {:tag :button - :fn/text "Save"}) - (e/click driver {:tag :button :fn/text "Save"}) - (e/wait-invisible driver {:tag :form}))))) diff --git a/test/clj/auto_ap/import/transactions_test.clj b/test/clj/auto_ap/import/transactions_test.clj index 20e7cb7a..247abafe 100644 --- a/test/clj/auto_ap/import/transactions_test.clj +++ b/test/clj/auto_ap/import/transactions_test.clj @@ -83,7 +83,10 @@ result (sut/transaction->txs base-transaction (d/entity (d/db conn) bank-account-id) noop-rule)] - (t/is (= [(assoc base-transaction :transaction/approval-status :transaction-approval-status/unapproved)] + (t/is (= [(assoc base-transaction + :transaction/approval-status :transaction-approval-status/unapproved + :transaction/bank-account bank-account-id + :transaction/client client-id)] result)))) (t/testing "Should match an uncleared check" @@ -113,6 +116,18 @@ :payment/status :payment-status/cleared} (:transaction/payment transaction-result)))) + + (t/testing "Should match a check that matches on amount if check number does not match" + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/description-original "CHECK 12301" + :transaction/amount -30.0) + (d/entity (d/db conn ) bank-account-id) + noop-rule)] + + (t/is (= {:db/id payment-id + :payment/status :payment-status/cleared} + (:transaction/payment transaction-result))))) + (t/testing "Should not match an already matched check" @(d/transact (d/connect uri) [{:db/id payment-id :payment/status :payment-status/cleared}]) (let [[result] (sut/transaction->txs (assoc base-transaction diff --git a/test/clj/auto_ap/integration/graphql.clj b/test/clj/auto_ap/integration/graphql.clj index 2c388d9b..5a330f20 100644 --- a/test/clj/auto_ap/integration/graphql.clj +++ b/test/clj/auto_ap/integration/graphql.clj @@ -39,11 +39,13 @@ (defn new-transaction [args] (merge {:transaction/amount 100.0 + :transaction/date #inst "2021-01-01" :transaction/id (.toString (java.util.UUID/randomUUID))} args)) (defn new-invoice [args] (merge {:invoice/total 100.0 + :invoice/date #inst "2021-01-01" :invoice/invoice-number (.toString (java.util.UUID/randomUUID))} args)) @@ -88,7 +90,7 @@ (deftest ledger-page (testing "ledger" (testing "it should find ledger entries" - (let [result (:ledger-page (:data (sut/query (admin-token) "{ ledger_page(client_id: null) { count, start, journal_entries { id } }}")))] + (let [result (:ledger-page (:data (sut/query (admin-token) "{ ledger_page(filters: {client_id: null}) { count, start, journal_entries { id } }}")))] (is (int? (:count result))) (is (int? (:start result))) (is (seqable? (:journal-entries result))))))) @@ -97,7 +99,7 @@ (deftest vendors (testing "vendors" (testing "it should find vendors" - (let [result (:ledger-page (:data (sut/query (admin-token) "{ ledger_page(client_id: null) { count, start, journal_entries { id } }}")))] + (let [result (:ledger-page (:data (sut/query (admin-token) "{ ledger_page(filters: {client_id: null}) { count, start, journal_entries { id } }}")))] (is (int? (:count result))) (is (int? (:start result))) (is (seqable? (:journal-entries result))))))) @@ -128,7 +130,7 @@ :venia/queries [{:query/data (sut/->graphql [:upsert-transaction-rule {:transaction-rule {:accounts [{:account-id account-id :percentage "0.25" - :location "B"}]}} + :location "Shared"}]}} [:id ]])}]})] (is (thrown? clojure.lang.ExceptionInfo (sut/query (admin-token) q))))) @@ -139,7 +141,7 @@ :venia/queries [{:query/data (sut/->graphql [:upsert-transaction-rule {:transaction-rule {:accounts [{:account-id account-id :percentage "1.0" - :location "B"}]}} + :location "Shared"}]}} [:id ]])}]})] (is (thrown? clojure.lang.ExceptionInfo (sut/query (admin-token) q))))) (testing "it should add rules" @@ -152,10 +154,10 @@ :transaction-approval-status :approved :accounts [{:account-id account-id :percentage "0.5" - :location "B"} + :location "Shared"} {:account-id account-id :percentage "0.5" - :location "A"}]}} + :location "Shared"}]}} [:id :description :transaction-approval-status [:vendor [:name]] @@ -182,7 +184,7 @@ :accounts [{:id (-> result :accounts (get 0) :id) :account-id account-id :percentage "1.0" - :location "B"}]}} + :location "Shared"}]}} [[:vendor [:name]]]])}]}) result (-> (sut/query (admin-token) q) :data @@ -200,7 +202,7 @@ :accounts [{:id (-> result :accounts (get 0) :id) :account-id account-id :percentage "1.0" - :location "B"}]}} + :location "Shared"}]}} [[:accounts [:id :percentage [:account [:name]]]]]])}]}) result (-> (sut/query (admin-token) q) :data @@ -208,22 +210,6 @@ (is (= 1 (count (:accounts result)))))))))) -(deftest test-get-yodlee-merchants - (testing "it should find yodlee merchants" - @(d/transact (d/connect uri) - [{:yodlee-merchant/name "Merchant 1" - :yodlee-merchant/yodlee-id "123"} - {:yodlee-merchant/name "Merchant 2" - :yodlee-merchant/yodlee-id "456"}]) - - (is (= [{:name "Merchant 1" :yodlee-id "123"} {:name "Merchant 2" :yodlee-id "456"}] - (-> (sut/query (admin-token) (v/graphql-query {:venia/operation {:operation/type :query - :operation/name "GetYodleeMerchants"} - :venia/queries [{:query/data (sut/->graphql [:yodlee-merchants - [:yodlee-id :name]])}]})) - :data - :yodlee-merchants))))) - (deftest test-transaction-rule (testing "it should match rules" @@ -236,7 +222,8 @@ :bank-account/name "1"} :transaction/amount 1.00 - :transaction/id "2019-01-05 matching-desc 1"} + :transaction/id "2019-01-05 matching-desc 1" + :db/id "a"} {:transaction/description-original "nonmatching-desc" :transaction/client {:client/name "2" @@ -245,8 +232,11 @@ :bank-account/name "2"} :transaction/date #inst "2019-01-15T23:23:00.000-08:00" :transaction/amount 2.00 - :transaction/id "2019-01-15 nonmatching-desc 2"}]) - {:strs [client-1 client-2 bank-account-1 bank-account-2]} (get-in matching-transaction [:tempids]) + :transaction/id "2019-01-15 nonmatching-desc 2" + :db/id "b"}]) + {:strs [a b client-1 client-2 bank-account-1 bank-account-2]} (get-in matching-transaction [:tempids]) + a (str a) + b (str b) rule-test (fn [rule] (-> (sut/query (admin-token) (v/graphql-query {:venia/operation {:operation/type :query @@ -257,27 +247,27 @@ :data :test-transaction-rule))] (testing "based on date " - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:dom-gte 14 :dom-lte 16}))) - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:dom-gte 14}))) - (is (= (set [{:id "2019-01-05 matching-desc 1"} {:id "2019-01-15 nonmatching-desc 2"}]) (set (rule-test {:dom-lte 15})))) - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:dom-gte 15}))) - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:dom-gte 15 :dom-lte 15})))) + (is (= [{:id b}] (rule-test {:dom-gte 14 :dom-lte 16}))) + (is (= [{:id b}] (rule-test {:dom-gte 14}))) + (is (= (set [{:id a} {:id b}]) (set (rule-test {:dom-lte 15})))) + (is (= [{:id b}] (rule-test {:dom-gte 15}))) + (is (= [{:id b}] (rule-test {:dom-gte 15 :dom-lte 15})))) (testing "based on description" - (is (= [{:id "2019-01-05 matching-desc 1"}] (rule-test {:description "^match"})))) + (is (= [{:id a}] (rule-test {:description "^match"})))) (testing "based on amount" - (is (= [{:id "2019-01-05 matching-desc 1"}] (rule-test {:amount-gte 1.0 :amount-lte 1.0}))) - (is (= (set [{:id "2019-01-05 matching-desc 1"} {:id "2019-01-15 nonmatching-desc 2"}]) (set (rule-test {:amount-gte 1.0 }))) ) - (is (= (set [{:id "2019-01-05 matching-desc 1"} {:id "2019-01-15 nonmatching-desc 2"}]) (set (rule-test {:amount-lte 2.0 }))) )) + (is (= [{:id a}] (rule-test {:amount-gte 1.0 :amount-lte 1.0}))) + (is (= (set [{:id a} {:id b}]) (set (rule-test {:amount-gte 1.0 }))) ) + (is (= (set [{:id a} {:id b}]) (set (rule-test {:amount-lte 2.0 }))) )) (testing "based on client" - (is (= [{:id "2019-01-05 matching-desc 1"}] (rule-test {:client-id client-1}))) - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:client-id client-2})))) + (is (= [{:id a}] (rule-test {:client-id (str client-1)}))) + (is (= [{:id b}] (rule-test {:client-id (str client-2)})))) (testing "based on bank account" - (is (= [{:id "2019-01-05 matching-desc 1"}] (rule-test {:bank-account-id bank-account-1}))) - (is (= [{:id "2019-01-15 nonmatching-desc 2"}] (rule-test {:bank-account-id bank-account-2}))))))) + (is (= [{:id a}] (rule-test {:bank-account-id (str bank-account-1)}))) + (is (= [{:id b}] (rule-test {:bank-account-id (str bank-account-2)}))))))) (deftest test-match-transaction-rule (testing "it should apply a rules" From 97fe81c773489dbd5ccf2a06dd727e1b75df8200 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 26 Dec 2021 09:24:11 -0800 Subject: [PATCH 3/4] First step in coding sales. --- src/clj/auto_ap/datomic/migrate.clj | 30 +++++++++- src/clj/auto_ap/import/transactions.clj | 14 +++-- src/clj/auto_ap/ledger.clj | 25 ++++++++- test/clj/auto_ap/import/transactions_test.clj | 11 ++++ test/clj/auto_ap/ledger_test.clj | 56 +++++++++++++++++++ 5 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 test/clj/auto_ap/ledger_test.clj diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 824f07f7..03094a03 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -115,6 +115,31 @@ []))] [existing-accounts])) + +(defn apply-idents-to-well-known [conn] + (let [[ccp-square] (d/q '[:find [?v] + :where [?v :vendor/name "CCP Square"]] + (d/db conn)) + [receipts-split] (d/q '[:find [?a] + :where [?a :account/numeric-code 12990]] + (d/db conn)) + [ccp] (d/q '[:find [?a] + :where [?a :account/numeric-code 12100]] + (d/db conn)) + [accounts-payable] (d/q '[:find [?a] + :where [?a :account/numeric-code 21000]] + (d/db conn))] + [[{:db/id ccp :db/ident :account/ccp}] + [{:db/id ccp-square + :db/ident :vendor/ccp-square + :vendor/name "CCP Square" + :vendor/default-account :account/ccp} + {:db/id receipts-split + :db/ident :account/receipts-split} + + {:db/id accounts-payable + :db/ident :account/accounts-payable}]])) + (defn migrate [conn] (let [ norms-map (merge {:auto-ap/base-schema {:txes auto-ap.datomic/base-schema} @@ -443,7 +468,10 @@ :db/doc "error message for a failed job" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]] - :requires [:auto-ap/add-transaction-import2]}} + :requires [:auto-ap/add-transaction-import2]} + :auto-ap/apply-idents-to-well-known {:txes-fn `apply-idents-to-well-known + :requires [:auto-ap/add-general-ledger6 + :auto-ap/add-account-to-vendor]}} diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 69fcc5c4..3101928d 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -223,8 +223,15 @@ (defn maybe-clear-expected-deposit [{:transaction/keys [amount client date] :as transaction}] (when (>= amount 0.0) (when-let [expected-deposit (find-expected-deposit client amount (coerce/to-date-time date))] - (assoc transaction :transaction/expected-deposit {:db/id expected-deposit - :expected-deposit/status :expected-deposit-status/cleared})))) + (assoc transaction + :transaction/expected-deposit {:db/id expected-deposit + :expected-deposit/status :expected-deposit-status/cleared} + :transaction/accounts [{:transaction-account/account :account/ccp + :transaction-account/amount amount + :transaction-account/location "A"}] + :transaction/approval-status :transaction-approval-status/approved + :transaction/vendor :vendor/ccp-square + )))) (defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations] (if-let [unpaid-invoices (seq (match-transaction-to-unpaid-invoices amount client))] @@ -249,9 +256,6 @@ code-fn)])) - - - (defn get-existing [bank-account] (log/info "looking up bank account data for" bank-account) (into {} diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 67a5cf82..de8a0a17 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -17,10 +17,12 @@ (map :a) (map namespace) set)] + (println namespaces changes) (cond (namespaces "invoice" ) [[:invoice e]] (namespaces "invoice-expense-account" ) [[:invoice (:db/id (:invoice/_expense-accounts entity))]] (namespaces "transaction-account" ) [[:transaction (:db/id (:transaction/_accounts entity))]] (namespaces "transaction" ) [[:transaction e]] + (namespaces "expected-deposit" ) [[:expected-deposit e]] :else nil))) @@ -33,6 +35,7 @@ (cond (namespaces "invoice" ) :invoice (namespaces "invoice-expense-account" ) :invoice-expense-account (namespaces "transaction-account" ) :transaction-account + (namespaces "expected-deposit" ) :expected-deposit :else nil))) (defmulti entity-change->ledger (fn [_ [type]] @@ -57,7 +60,7 @@ :journal-entry/vendor (:db/id (:invoice/vendor entity)) :journal-entry/amount (Math/abs (:invoice/total entity)) - :journal-entry/line-items (into [(cond-> {:journal-entry-line/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable :journal-entry-line/location "A" } credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity))) @@ -120,6 +123,26 @@ :journal-entry/cleared true})))) +(defmethod entity-change->ledger :expected-deposit + [db [type id]] + (let [{:expected-deposit/keys [total client date]} (d/pull db '[:expected-deposit/total :expected-deposit/client :expected-deposit/date] id)] + #:journal-entry + {:source "expected-deposit" + :original-entity id + + :client client + :date date + :amount total + :vendor :vendor/ccp-square + :line-items [#:journal-entry-line + {:credit total + :location "A" + :account :account/receipts-split} + #:journal-entry-line + {:debit total + :location "A" + :account :account/ccp}]})) + (defmethod entity-change->ledger :invoice-expense-account [db [entity changes]] nil diff --git a/test/clj/auto_ap/import/transactions_test.clj b/test/clj/auto_ap/import/transactions_test.clj index 247abafe..47e5eba5 100644 --- a/test/clj/auto_ap/import/transactions_test.clj +++ b/test/clj/auto_ap/import/transactions_test.clj @@ -171,6 +171,17 @@ :expected-deposit/status :expected-deposit-status/cleared} (:transaction/expected-deposit transaction-result))))) + (t/testing "Should credit CCP" + (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + :transaction/date #inst "2021-07-03T00:00:00-08:00" + :transaction/amount 100.0) + (d/entity (d/db conn) bank-account-id) + noop-rule)] + (t/is (= [{:transaction-account/account :account/ccp + :transaction-account/amount 100.0 + :transaction-account/location "A"}] + (:transaction/accounts transaction-result))))) + (t/testing "Should not match old expected deposits" (let [[transaction-result] (sut/transaction->txs (assoc base-transaction :transaction/date #inst "2021-07-13" diff --git a/test/clj/auto_ap/ledger_test.clj b/test/clj/auto_ap/ledger_test.clj new file mode 100644 index 00000000..cbeed14d --- /dev/null +++ b/test/clj/auto_ap/ledger_test.clj @@ -0,0 +1,56 @@ +(ns auto-ap.ledger-test + (:require [auto-ap.datomic :refer [conn uri]] + [auto-ap.datomic.migrate :as m] + [auto-ap.ledger :as sut] + [clojure.test :as t] + [datomic.api :as d])) + +(defn wrap-setup + [f] + (with-redefs [auto-ap.datomic/uri "datomic:mem://datomic-transactor:4334/invoice"] + (d/create-database uri) + (with-redefs [auto-ap.datomic/conn (d/connect uri)] + (m/migrate auto-ap.datomic/conn) + (f) + (d/release auto-ap.datomic/conn) + (d/delete-database uri)))) + +(t/use-fixtures :each wrap-setup) + + +(t/deftest entity-change->ledger + (t/testing "Should code an expected deposit" + (let [{:strs [ed ccp receipts-split client]} + (:tempids @(d/transact conn [#:expected-deposit {:status :expected-deposit-status/pending + :client {:db/id "client" + :client/code "BRYCE" + :client/locations ["M"]} + :total 4.0 + :fee 1.0 + :date #inst "2021-01-01T00:00:00-08:00" + :location "M" + :db/id "ed"}])) + result (sut/entity-change->ledger (d/db conn) [:expected-deposit ed])] + (t/is (= #:journal-entry + {:source "expected-deposit" + :client {:db/id client} + :date #inst "2021-01-01T00:00:00-08:00" + :original-entity ed + :vendor :vendor/ccp-square + :amount 4.0 + } + (dissoc result :journal-entry/line-items))) + + (t/testing "should debit ccp" + (t/is (= [#:journal-entry-line + {:debit 4.0 + :location "A" + :account :account/ccp}] + (filter :journal-entry-line/debit (:journal-entry/line-items result)))) + ) + (t/testing "should credit receipts split ccp" + (t/is (= [#:journal-entry-line + {:credit 4.0 + :location "A" + :account :account/receipts-split}] + (filter :journal-entry-line/credit (:journal-entry/line-items result)))))))) From c5a65f008aa2dd1187ee4feb8a85f01616762858 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 26 Dec 2021 09:29:13 -0800 Subject: [PATCH 4/4] More forgiving --- src/clj/auto_ap/square/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index 0ff22732..cfc297ad 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -452,7 +452,8 @@ (map (fn [x] [:db/retractEntity x])))) (defn upsert-all [] - (doseq [[client-code] (get-in env [:square-config])] + (doseq [[client-code] (get-in env [:square-config]) + :when (d/entid (d/db conn) [:client/code client-code])] (lc/with-context {:source (str "Square loading for " client-code) :client client-code} (log/info "Loading Orders")