minor tweak to make it go better

src/clj/auto_ap/jobs/register_invoice_import.clj

@@ -23,10 +23,8 @@
            io/reader
            csv/read-csv)))
 
-(defn register-invoice-import [args]
+(defn register-invoice-import* [data]
-  (let [{:keys [ledger-url]} args
+  (let [db             (d/db conn)
-        data                 (s3->csv ledger-url)
-        db                   (d/db conn)
         i->invoice-id  (fn [i]
                          (try (Long/parseLong i)
                               (catch Exception e
@@ -42,14 +40,14 @@
                                                  (map (fn [[_ _ _ _ amount]]
                                                         (- (Double/parseDouble amount))))))
                                     ]))
-                            (into {}))
+                            (into {}))]
-        changes              (->> 
+    (->> 
       (for [[i
              invoice-expense-account-id
              target-account
              target-date
              amount
-                                      _
+             target-account-code
              location] (drop 1 data)
             :let       [invoice-id (i->invoice-id i)
 
@@ -123,11 +121,15 @@
             :invoice-expense-account/account target-account-id})])
       (mapcat identity)
       (filter identity)
-                               vec)]
+      vec)))
-    (doseq [n (partition-all 50 changes)]
+
+(defn register-invoice-import [args]
+  (let [{:keys [ledger-url]} args
+        data                 (s3->csv ledger-url)]
+    (log/info "contains " (count data) " rows")
+    (doseq [n (partition-all 50 (register-invoice-import* data))]
       (log/info "transacting" n)
-      @(d/transact conn n))
+      @(d/transact conn n))))
-    ))
 
 (defn -main [& _]
   (execute "register-invoice-import" #(register-invoice-import (:args env))))

src/cljs/auto_ap/views/pages/admin/jobs.cljs

@@ -131,16 +131,9 @@
 (defn register-invoice-import-button []
   [form-builder/builder {:submit-event [::request :register-invoice-import]
                          :id           ::register-invoice-import-form}
-   [:p.field.has-addons
-    [:p.control
-     [:a.button.is-static.is-small
-      "s3://data.prod.app.integreatconsult.com/bulk-import/"]]
-    [:p.control
-     [form-builder/raw-field-v2 {:field :ledger-url}
-      [:input.input.is-small {:placeholder "invoices.csv"}]]]
    [:p.control
     [form-builder/submit-button {:class "is-small"} "Register Invoice Import"]
-     ]]])
+    ]])
 
 
 

terraform/terraform.tfstate.d/prod/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.3.2",
-  "serial": 281,
+  "serial": 343,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:413",
             "timeouts": {
               "delete": null
             },
@@ -1391,7 +1391,7 @@
             "name_prefix": "",
             "role_arn": "",
             "schedule_expression": "rate(8 hours)",
-            "tags": null,
+            "tags": {},
             "tags_all": {}
           },
           "sensitive_attributes": [],
@@ -1436,7 +1436,7 @@
                 "placement_constraint": [],
                 "platform_version": "",
                 "propagate_tags": "TASK_DEFINITION",
-                "tags": null,
+                "tags": {},
                 "task_count": 1,
                 "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1"
               }
@@ -1494,7 +1494,7 @@
             ],
             "revision": 1,
             "runtime_platform": [],
-            "tags": null,
+            "tags": {},
             "tags_all": {},
             "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
             "volume": []
@@ -2040,6 +2040,45 @@
         }
       ]
     },
+    {
+      "module": "module.register_invoice_import_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/register_invoice_import_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"register-invoice-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"register-invoice-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"register-invoice-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "2048",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "register_invoice_import_prod",
+            "id": "register_invoice_import_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "8192",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
     {
       "module": "module.square2_import_job",
       "mode": "managed",

terraform/terraform.tfstate.d/prod/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.3.2",
-  "serial": 220,
+  "serial": 281,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412",
             "timeouts": {
               "delete": null
             },
@@ -348,7 +348,7 @@
             "desync_mitigation_mode": "defensive",
             "dns_name": "integreat-app-prod-1104326262.us-east-1.elb.amazonaws.com",
             "drop_invalid_header_fields": false,
-            "enable_cross_zone_load_balancing": null,
+            "enable_cross_zone_load_balancing": true,
             "enable_deletion_protection": true,
             "enable_http2": true,
             "enable_waf_fail_open": false,
@@ -1092,7 +1092,7 @@
             ],
             "revision": 1,
             "runtime_platform": [],
-            "tags": null,
+            "tags": {},
             "tags_all": {},
             "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
             "volume": []
@@ -1370,6 +1370,140 @@
         }
       ]
     },
+    {
+      "module": "module.ezcater_upsert_job",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_rule",
+      "name": "schedule",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "index_key": 0,
+          "schema_version": 0,
+          "attributes": {
+            "arn": "arn:aws:events:us-east-1:679918342773:rule/ezcater-upsert-schedule",
+            "description": "",
+            "event_bus_name": "default",
+            "event_pattern": null,
+            "id": "ezcater-upsert-schedule",
+            "is_enabled": true,
+            "name": "ezcater-upsert-schedule",
+            "name_prefix": "",
+            "role_arn": "",
+            "schedule_expression": "rate(8 hours)",
+            "tags": null,
+            "tags_all": {}
+          },
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
+        }
+      ]
+    },
+    {
+      "module": "module.ezcater_upsert_job",
+      "mode": "managed",
+      "type": "aws_cloudwatch_event_target",
+      "name": "job_target",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "index_key": 0,
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
+            "batch_target": [],
+            "dead_letter_config": [],
+            "ecs_target": [
+              {
+                "enable_ecs_managed_tags": false,
+                "enable_execute_command": false,
+                "group": "",
+                "launch_type": "",
+                "network_configuration": [
+                  {
+                    "assign_public_ip": true,
+                    "security_groups": [
+                      "sg-004e5855310c453a3",
+                      "sg-02d167406b1082698"
+                    ],
+                    "subnets": [
+                      "subnet-5e675761",
+                      "subnet-8519fde2",
+                      "subnet-89bab8d4"
+                    ]
+                  }
+                ],
+                "placement_constraint": [],
+                "platform_version": "",
+                "propagate_tags": "TASK_DEFINITION",
+                "tags": null,
+                "task_count": 1,
+                "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1"
+              }
+            ],
+            "event_bus_name": "default",
+            "http_target": [],
+            "id": "ezcater-upsert-schedule-ezcater-upsert",
+            "input": "",
+            "input_path": "",
+            "input_transformer": [],
+            "kinesis_target": [],
+            "redshift_target": [],
+            "retry_policy": [],
+            "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
+            "rule": "ezcater-upsert-schedule",
+            "run_command_targets": [],
+            "sqs_target": [],
+            "target_id": "ezcater-upsert"
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
+          "dependencies": [
+            "module.ezcater_upsert_job.aws_cloudwatch_event_rule.schedule",
+            "module.ezcater_upsert_job.aws_ecs_task_definition.background_taskdef"
+          ]
+        }
+      ]
+    },
+    {
+      "module": "module.ezcater_upsert_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ezcater-upsert\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ezcater-upsert\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ezcater-upsert\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "1024",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "ezcater_upsert_prod",
+            "id": "ezcater_upsert_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "2048",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
     {
       "module": "module.import_uploaded_invoices_job",
       "mode": "managed",
@@ -2616,5 +2750,5 @@
       ]
     }
   ],
-  "check_results": null
+  "check_results": []
 }