diff --git a/src/clj/auto_ap/graphql/jobs.clj b/src/clj/auto_ap/graphql/jobs.clj index 2090e2bd..6b8292b3 100644 --- a/src/clj/auto_ap/graphql/jobs.clj +++ b/src/clj/auto_ap/graphql/jobs.clj @@ -91,7 +91,7 @@ :network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"] :security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"] :assign-public-ip AssignPublicIp/ENABLED}}} - args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str task)}]}])))) + args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}])))) (defn request-job [context value _] (assert-admin (:id context)) diff --git a/src/clj/auto_ap/jobs/bulk_journal_import.clj b/src/clj/auto_ap/jobs/bulk_journal_import.clj index 5a1edd5e..47952d85 100644 --- a/src/clj/auto_ap/jobs/bulk_journal_import.clj +++ b/src/clj/auto_ap/jobs/bulk_journal_import.clj @@ -1,13 +1,79 @@ (ns auto-ap.jobs.bulk-journal-import (:gen-class) (:require + [amazonica.aws.s3 :as s3] + [auto-ap.graphql.ledger :refer [import-ledger]] [auto-ap.jobs.core :refer [execute]] + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [clojure.string :as str] [clojure.tools.logging :as log] [config.core :refer [env]])) -(defn bulk-journal-import [] - (log/info "importing ledger from" (:args env))) +(defn line->id [{:keys [source id client-code]}] + (str client-code "-" source "-" id)) + + +(defn csv->graphql-rows [lines] + (for [lines (partition-by line->id (drop 1 lines)) + :let [{:keys [source client-code date vendor-name note cleared-against] :as line} (first lines)]] + {:source source + :external_id (line->id line) + :client_code client-code + :date date + :note note + :cleared_against cleared-against + :vendor_name vendor-name + :amount (reduce + 0 + (->> lines + (map :debit) + (map #(if (str/blank? %) + 0.0 + (Double/parseDouble %))))) + :line_items (map (fn [{:keys [debit credit account-identifier location]}] + {:account_identifier account-identifier + :location (some-> location str/trim) + :debit (if (str/blank? debit) + 0.0 + (Double/parseDouble debit)) + :credit (if (str/blank? credit) + 0.0 + (Double/parseDouble credit))}) + lines)})) + +(def bucket (:data-bucket env)) + +(defn s3->csv [ledger-url] + (->> (-> (s3/get-object {:bucket-name bucket + :key (str "bulk-import/" ledger-url)}) + :input-stream + io/reader + csv/read-csv) + (sequence (comp + (filter (fn [x] + (seq (filter (complement str/blank?) x)))) + (map (fn [[id client-code source vendor-name date account-identifier location debit credit note cleared-against]] + {:id id + :client-code client-code + :source source + :vendor-name vendor-name + :date date + :account-identifier account-identifier + :location location + :debit debit + :credit credit + :note note + :cleared-against cleared-against})))))) + +(defn bulk-journal-import [args] + (let [{:keys [ledger-url]} args + _ (log/info "importing ledger from" ledger-url) + csv-stream (s3->csv ledger-url) + import-rows (csv->graphql-rows csv-stream)] + (import-ledger {:id {:user/name "Bulk-import" :user/role "admin"}} + {:entries import-rows} + nil))) (defn -main [& _] - (execute "bulk-journal-import" bulk-journal-import)) + (execute "bulk-journal-import" #(bulk-journal-import (:args env)))) diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 3ae5ec71..49fa66fe 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.7", - "serial": 217, + "serial": 220, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373", "timeouts": { "delete": null }, @@ -1063,6 +1063,45 @@ } ] }, + { + "module": "module.bulk_journal_import_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/bulk_journal_import_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"bulk-journal-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"bulk-journal-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"bulk-journal-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "bulk_journal_import_prod", + "id": "bulk_journal_import_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "4096", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.close_auto_invoices_job", "mode": "managed", @@ -2155,7 +2194,7 @@ "name": "sysco-schedule", "name_prefix": "", "role_arn": "", - "schedule_expression": "rate(1 hour)", + "schedule_expression": "rate(3 hours)", "tags": {}, "tags_all": {} }, @@ -2432,7 +2471,7 @@ ], "revision": 2, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index 1bee9e79..3ae5ec71 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.7", - "serial": 214, + "serial": 217, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:370", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372", "timeouts": { "delete": null }, @@ -2413,7 +2413,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/yodlee2_accounts_prod:1", + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/yodlee2_accounts_prod:2", "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"yodlee2-accounts\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"yodlee2-accounts\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"yodlee2-accounts\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", "cpu": "512", "ephemeral_storage": [], @@ -2422,7 +2422,7 @@ "id": "yodlee2_accounts_prod", "inference_accelerator": [], "ipc_mode": "", - "memory": "1024", + "memory": "2048", "network_mode": "awsvpc", "pid_mode": "", "placement_constraints": [], @@ -2430,7 +2430,7 @@ "requires_compatibilities": [ "FARGATE" ], - "revision": 1, + "revision": 2, "runtime_platform": [], "tags": null, "tags_all": {},