From 632d4bb709c6c20baa12d71412b9a9132d1cdda4 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sun, 4 Dec 2022 14:43:50 -0800 Subject: [PATCH] minor tweak to make it go better --- .../auto_ap/jobs/register_invoice_import.clj | 154 +++++++++--------- src/cljs/auto_ap/views/pages/admin/jobs.cljs | 13 +- .../prod/terraform.tfstate | 49 +++++- .../prod/terraform.tfstate.backup | 144 +++++++++++++++- 4 files changed, 264 insertions(+), 96 deletions(-) diff --git a/src/clj/auto_ap/jobs/register_invoice_import.clj b/src/clj/auto_ap/jobs/register_invoice_import.clj index 7a1f203d..219d44e9 100644 --- a/src/clj/auto_ap/jobs/register_invoice_import.clj +++ b/src/clj/auto_ap/jobs/register_invoice_import.clj @@ -23,16 +23,14 @@ io/reader csv/read-csv))) -(defn register-invoice-import [args] - (let [{:keys [ledger-url]} args - data (s3->csv ledger-url) - db (d/db conn) - i->invoice-id (fn [i] +(defn register-invoice-import* [data] + (let [db (d/db conn) + i->invoice-id (fn [i] (try (Long/parseLong i) (catch Exception e (:db/id (d/pull db '[:db/id] [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) - invoice-totals (->> data + invoice-totals (->> data (drop 1) (group-by first) (map (fn [[k values]] @@ -42,92 +40,96 @@ (map (fn [[_ _ _ _ amount]] (- (Double/parseDouble amount)))))) ])) - (into {})) - changes (->> - (for [[i - invoice-expense-account-id - target-account - target-date - amount - _ - location] (drop 1 data) - :let [invoice-id (i->invoice-id i) + (into {}))] + (->> + (for [[i + invoice-expense-account-id + target-account + target-date + amount + target-account-code + location] (drop 1 data) + :let [invoice-id (i->invoice-id i) - invoice (d/entity db invoice-id) - current-total (:invoice/total invoice) - target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible - new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) - (:db/id (first (:invoice/expense-accounts invoice)))))) + invoice (d/entity db invoice-id) + current-total (:invoice/total invoice) + target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible + new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) + (:db/id (first (:invoice/expense-accounts invoice)))))) - invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) - (:db/id (first (:invoice/expense-accounts invoice))) - (d/tempid :db.part/user)) - invoice-expense-account (when-not new-account? - (or (d/entity db invoice-expense-account-id) - (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) - current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) - target-account-id (Long/parseLong (str/trim target-account)) + invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) + (:db/id (first (:invoice/expense-accounts invoice))) + (d/tempid :db.part/user)) + invoice-expense-account (when-not new-account? + (or (d/entity db invoice-expense-account-id) + (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) + current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) + target-account-id (Long/parseLong (str/trim target-account)) - target-date (coerce/to-date (atime/parse target-date atime/normal-date)) - current-date (:invoice/date invoice) - + target-date (coerce/to-date (atime/parse target-date atime/normal-date)) + current-date (:invoice/date invoice) + - current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0) - target-expense-account-amount (- (Double/parseDouble amount)) + current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0) + target-expense-account-amount (- (Double/parseDouble amount)) - current-expense-account-location (:invoice-expense-account/location invoice-expense-account) - target-expense-account-location location + current-expense-account-location (:invoice-expense-account/location invoice-expense-account) + target-expense-account-location location - [[_ _ invoice-payment]] (vec (d/q - '[:find ?p ?a ?ip - :in $ ?i - :where [?ip :invoice-payment/invoice ?i] - [?ip :invoice-payment/amount ?a] - [?ip :invoice-payment/payment ?p] - ] - db invoice-id))] - :when current-total] + [[_ _ invoice-payment]] (vec (d/q + '[:find ?p ?a ?ip + :in $ ?i + :where [?ip :invoice-payment/invoice ?i] + [?ip :invoice-payment/amount ?a] + [?ip :invoice-payment/payment ?p] + ] + db invoice-id))] + :when current-total] - [ - (when (not (dollars= current-total target-total)) - {:db/id invoice-id - :invoice/total target-total}) + [ + (when (not (dollars= current-total target-total)) + {:db/id invoice-id + :invoice/total target-total}) - (when new-account? - {:db/id invoice-id - :invoice/expense-accounts invoice-expense-account-id}) + (when new-account? + {:db/id invoice-id + :invoice/expense-accounts invoice-expense-account-id}) - (when (and target-date (not= current-date target-date)) - {:db/id invoice-id - :invoice/date target-date}) + (when (and target-date (not= current-date target-date)) + {:db/id invoice-id + :invoice/date target-date}) - (when (and - (not (dollars= current-total target-total)) - invoice-payment) - [:db/retractEntity invoice-payment]) + (when (and + (not (dollars= current-total target-total)) + invoice-payment) + [:db/retractEntity invoice-payment]) - (when (or new-account? - (not (dollars= current-expense-account-amount target-expense-account-amount))) - {:db/id invoice-expense-account-id - :invoice-expense-account/amount target-expense-account-amount}) + (when (or new-account? + (not (dollars= current-expense-account-amount target-expense-account-amount))) + {:db/id invoice-expense-account-id + :invoice-expense-account/amount target-expense-account-amount}) - (when (not= current-expense-account-location - target-expense-account-location) - {:db/id invoice-expense-account-id - :invoice-expense-account/location target-expense-account-location}) + (when (not= current-expense-account-location + target-expense-account-location) + {:db/id invoice-expense-account-id + :invoice-expense-account/location target-expense-account-location}) - (when (not= current-account-id target-account-id ) - {:db/id invoice-expense-account-id - :invoice-expense-account/account target-account-id})]) - (mapcat identity) - (filter identity) - vec)] - (doseq [n (partition-all 50 changes)] + (when (not= current-account-id target-account-id ) + {:db/id invoice-expense-account-id + :invoice-expense-account/account target-account-id})]) + (mapcat identity) + (filter identity) + vec))) + +(defn register-invoice-import [args] + (let [{:keys [ledger-url]} args + data (s3->csv ledger-url)] + (log/info "contains " (count data) " rows") + (doseq [n (partition-all 50 (register-invoice-import* data))] (log/info "transacting" n) - @(d/transact conn n)) - )) + @(d/transact conn n)))) (defn -main [& _] (execute "register-invoice-import" #(register-invoice-import (:args env)))) diff --git a/src/cljs/auto_ap/views/pages/admin/jobs.cljs b/src/cljs/auto_ap/views/pages/admin/jobs.cljs index bdfc84cb..57c5e953 100644 --- a/src/cljs/auto_ap/views/pages/admin/jobs.cljs +++ b/src/cljs/auto_ap/views/pages/admin/jobs.cljs @@ -131,16 +131,9 @@ (defn register-invoice-import-button [] [form-builder/builder {:submit-event [::request :register-invoice-import] :id ::register-invoice-import-form} - [:p.field.has-addons - [:p.control - [:a.button.is-static.is-small - "s3://data.prod.app.integreatconsult.com/bulk-import/"]] - [:p.control - [form-builder/raw-field-v2 {:field :ledger-url} - [:input.input.is-small {:placeholder "invoices.csv"}]]] - [:p.control - [form-builder/submit-button {:class "is-small"} "Register Invoice Import"] - ]]]) + [:p.control + [form-builder/submit-button {:class "is-small"} "Register Invoice Import"] + ]]) diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 37664b9c..68a9dcad 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.3.2", - "serial": 281, + "serial": 343, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:413", "timeouts": { "delete": null }, @@ -1391,7 +1391,7 @@ "name_prefix": "", "role_arn": "", "schedule_expression": "rate(8 hours)", - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -1436,7 +1436,7 @@ "placement_constraint": [], "platform_version": "", "propagate_tags": "TASK_DEFINITION", - "tags": null, + "tags": {}, "task_count": 1, "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1" } @@ -1494,7 +1494,7 @@ ], "revision": 1, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -2040,6 +2040,45 @@ } ] }, + { + "module": "module.register_invoice_import_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/register_invoice_import_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"register-invoice-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"register-invoice-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"register-invoice-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "2048", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "register_invoice_import_prod", + "id": "register_invoice_import_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "8192", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.square2_import_job", "mode": "managed", diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index 66a9d287..37664b9c 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.3.2", - "serial": 220, + "serial": 281, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412", "timeouts": { "delete": null }, @@ -348,7 +348,7 @@ "desync_mitigation_mode": "defensive", "dns_name": "integreat-app-prod-1104326262.us-east-1.elb.amazonaws.com", "drop_invalid_header_fields": false, - "enable_cross_zone_load_balancing": null, + "enable_cross_zone_load_balancing": true, "enable_deletion_protection": true, "enable_http2": true, "enable_waf_fail_open": false, @@ -1092,7 +1092,7 @@ ], "revision": 1, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -1370,6 +1370,140 @@ } ] }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "schedule", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:events:us-east-1:679918342773:rule/ezcater-upsert-schedule", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "id": "ezcater-upsert-schedule", + "is_enabled": true, + "name": "ezcater-upsert-schedule", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(8 hours)", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "job_target", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [ + { + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "group": "", + "launch_type": "", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "placement_constraint": [], + "platform_version": "", + "propagate_tags": "TASK_DEFINITION", + "tags": null, + "task_count": 1, + "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1" + } + ], + "event_bus_name": "default", + "http_target": [], + "id": "ezcater-upsert-schedule-ezcater-upsert", + "input": "", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733", + "rule": "ezcater-upsert-schedule", + "run_command_targets": [], + "sqs_target": [], + "target_id": "ezcater-upsert" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ezcater_upsert_job.aws_cloudwatch_event_rule.schedule", + "module.ezcater_upsert_job.aws_ecs_task_definition.background_taskdef" + ] + } + ] + }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ezcater-upsert\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ezcater-upsert\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ezcater-upsert\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "ezcater_upsert_prod", + "id": "ezcater_upsert_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "2048", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.import_uploaded_invoices_job", "mode": "managed", @@ -2616,5 +2750,5 @@ ] } ], - "check_results": null + "check_results": [] }