Allows a background change for bulk ledger

src/clj/auto_ap/graphql/jobs.clj

@@ -91,7 +91,7 @@
                          :network-configuration      {:aws-vpc-configuration {:subnets          ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
                                                                               :security-groups  ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
                                                                               :assign-public-ip AssignPublicIp/ENABLED}}}
-                  args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str task)}]}]))))
+                  args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
 
 (defn request-job [context value _]
   (assert-admin (:id context))

src/clj/auto_ap/jobs/bulk_journal_import.clj

@@ -1,13 +1,79 @@
 (ns auto-ap.jobs.bulk-journal-import
   (:gen-class)
   (:require
+   [amazonica.aws.s3 :as s3]
+   [auto-ap.graphql.ledger :refer [import-ledger]]
    [auto-ap.jobs.core :refer [execute]]
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [clojure.tools.logging :as log]
    [config.core :refer [env]]))
 
-(defn bulk-journal-import []
+(defn line->id [{:keys [source id client-code]}]
-  (log/info "importing ledger from" (:args env)))
+  (str client-code "-"  source "-" id))
+
+
+(defn csv->graphql-rows [lines]
+  (for [lines (partition-by line->id (drop 1 lines))
+        :let      [{:keys [source client-code date vendor-name note cleared-against] :as line} (first lines)]] 
+    {:source          source
+     :external_id     (line->id line) 
+     :client_code     client-code
+     :date            date
+     :note            note
+     :cleared_against cleared-against
+     :vendor_name     vendor-name
+     :amount          (reduce + 0
+                              (->> lines
+                                   (map :debit)
+                                   (map #(if (str/blank? %)
+                                           0.0
+                                           (Double/parseDouble %)))))
+     :line_items      (map (fn [{:keys [debit credit account-identifier location]}]
+                             {:account_identifier account-identifier
+                              :location           (some-> location str/trim)
+                              :debit              (if (str/blank? debit)
+                                                     0.0
+                                                     (Double/parseDouble debit))
+                              :credit             (if (str/blank? credit)
+                                                    0.0
+                                                    (Double/parseDouble credit))})
+                           lines)}))
+
+(def bucket (:data-bucket env))
+
+(defn s3->csv [ledger-url]
+  (->> (-> (s3/get-object {:bucket-name bucket
+                           :key         (str "bulk-import/" ledger-url)})
+           :input-stream
+           io/reader
+           csv/read-csv)
+       (sequence (comp
+                  (filter (fn [x]
+                            (seq (filter (complement str/blank?) x))))
+                  (map (fn [[id client-code source vendor-name date account-identifier location debit credit note cleared-against]]
+                         {:id id
+                          :client-code client-code
+                          :source source
+                          :vendor-name vendor-name
+                          :date date
+                          :account-identifier account-identifier
+                          :location location
+                          :debit debit
+                          :credit credit
+                          :note note
+                          :cleared-against cleared-against}))))))
+
+(defn bulk-journal-import [args]
+  (let [{:keys [ledger-url]} args
+        _ (log/info "importing ledger from" ledger-url)
+        csv-stream (s3->csv ledger-url)
+        import-rows (csv->graphql-rows csv-stream)]
+    (import-ledger {:id {:user/name "Bulk-import" :user/role "admin"}}
+                   {:entries import-rows}
+                   nil)))
 
 (defn -main [& _]
-  (execute "bulk-journal-import" bulk-journal-import))
+  (execute "bulk-journal-import" #(bulk-journal-import (:args env))))
 

terraform/terraform.tfstate.d/prod/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 217,
+  "serial": 220,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373",
             "timeouts": {
               "delete": null
             },
@@ -1063,6 +1063,45 @@
         }
       ]
     },
+    {
+      "module": "module.bulk_journal_import_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/bulk_journal_import_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"bulk-journal-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"bulk-journal-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"bulk-journal-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "1024",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "bulk_journal_import_prod",
+            "id": "bulk_journal_import_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "4096",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
     {
       "module": "module.close_auto_invoices_job",
       "mode": "managed",
@@ -2155,7 +2194,7 @@
             "name": "sysco-schedule",
             "name_prefix": "",
             "role_arn": "",
-            "schedule_expression": "rate(1 hour)",
+            "schedule_expression": "rate(3 hours)",
             "tags": {},
             "tags_all": {}
           },
@@ -2432,7 +2471,7 @@
             ],
             "revision": 2,
             "runtime_platform": [],
-            "tags": null,
+            "tags": {},
             "tags_all": {},
             "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
             "volume": []

terraform/terraform.tfstate.d/prod/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
   "terraform_version": "1.2.7",
-  "serial": 214,
+  "serial": 217,
   "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
   "outputs": {
     "aws_access_key_id": {
@@ -164,7 +164,7 @@
             ],
             "tags": {},
             "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:370",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372",
             "timeouts": {
               "delete": null
             },
@@ -2413,7 +2413,7 @@
         {
           "schema_version": 1,
           "attributes": {
-            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/yodlee2_accounts_prod:1",
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/yodlee2_accounts_prod:2",
             "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"yodlee2-accounts\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"yodlee2-accounts\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"yodlee2-accounts\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
             "cpu": "512",
             "ephemeral_storage": [],
@@ -2422,7 +2422,7 @@
             "id": "yodlee2_accounts_prod",
             "inference_accelerator": [],
             "ipc_mode": "",
-            "memory": "1024",
+            "memory": "2048",
             "network_mode": "awsvpc",
             "pid_mode": "",
             "placement_constraints": [],
@@ -2430,7 +2430,7 @@
             "requires_compatibilities": [
               "FARGATE"
             ],
-            "revision": 1,
+            "revision": 2,
             "runtime_platform": [],
             "tags": null,
             "tags_all": {},