diff --git a/src/clj/auto_ap/jobs/register_invoice_import.clj b/src/clj/auto_ap/jobs/register_invoice_import.clj new file mode 100644 index 00000000..7a1f203d --- /dev/null +++ b/src/clj/auto_ap/jobs/register_invoice_import.clj @@ -0,0 +1,133 @@ +(ns auto-ap.jobs.register-invoice-import + (:gen-class) + (:require + [auto-ap.jobs.core :refer [execute]] + [amazonica.aws.s3 :as s3] + [datomic.api :as d] + [auto-ap.utils :refer [dollars=]] + [clojure.string :as str] + [clj-time.coerce :as coerce] + [auto-ap.time :as atime] + [auto-ap.datomic :refer [conn]] + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [config.core :refer [env]] + [clojure.tools.logging :as log])) + +(def bucket (:data-bucket env)) + +(defn s3->csv [url] + (->> (-> (s3/get-object {:bucket-name bucket + :key (str "bulk-import/" url)}) + :input-stream + io/reader + csv/read-csv))) + +(defn register-invoice-import [args] + (let [{:keys [ledger-url]} args + data (s3->csv ledger-url) + db (d/db conn) + i->invoice-id (fn [i] + (try (Long/parseLong i) + (catch Exception e + (:db/id (d/pull db '[:db/id] + [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) + invoice-totals (->> data + (drop 1) + (group-by first) + (map (fn [[k values]] + [(i->invoice-id k) + (reduce + 0.0 + (->> values + (map (fn [[_ _ _ _ amount]] + (- (Double/parseDouble amount)))))) + ])) + (into {})) + changes (->> + (for [[i + invoice-expense-account-id + target-account + target-date + amount + _ + location] (drop 1 data) + :let [invoice-id (i->invoice-id i) + + invoice (d/entity db invoice-id) + current-total (:invoice/total invoice) + target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible + new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) + (:db/id (first (:invoice/expense-accounts invoice)))))) + + invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) + (:db/id (first (:invoice/expense-accounts invoice))) + (d/tempid :db.part/user)) + invoice-expense-account (when-not new-account? + (or (d/entity db invoice-expense-account-id) + (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) + current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) + target-account-id (Long/parseLong (str/trim target-account)) + + target-date (coerce/to-date (atime/parse target-date atime/normal-date)) + current-date (:invoice/date invoice) + + + current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0) + target-expense-account-amount (- (Double/parseDouble amount)) + + + current-expense-account-location (:invoice-expense-account/location invoice-expense-account) + target-expense-account-location location + + + [[_ _ invoice-payment]] (vec (d/q + '[:find ?p ?a ?ip + :in $ ?i + :where [?ip :invoice-payment/invoice ?i] + [?ip :invoice-payment/amount ?a] + [?ip :invoice-payment/payment ?p] + ] + db invoice-id))] + :when current-total] + + [ + (when (not (dollars= current-total target-total)) + {:db/id invoice-id + :invoice/total target-total}) + + (when new-account? + {:db/id invoice-id + :invoice/expense-accounts invoice-expense-account-id}) + + (when (and target-date (not= current-date target-date)) + {:db/id invoice-id + :invoice/date target-date}) + + (when (and + (not (dollars= current-total target-total)) + invoice-payment) + [:db/retractEntity invoice-payment]) + + (when (or new-account? + (not (dollars= current-expense-account-amount target-expense-account-amount))) + {:db/id invoice-expense-account-id + :invoice-expense-account/amount target-expense-account-amount}) + + (when (not= current-expense-account-location + target-expense-account-location) + {:db/id invoice-expense-account-id + :invoice-expense-account/location target-expense-account-location}) + + (when (not= current-account-id target-account-id ) + {:db/id invoice-expense-account-id + :invoice-expense-account/account target-account-id})]) + (mapcat identity) + (filter identity) + vec)] + (doseq [n (partition-all 50 changes)] + (log/info "transacting" n) + @(d/transact conn n)) + )) + +(defn -main [& _] + (execute "register-invoice-import" #(register-invoice-import (:args env)))) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 027ba685..80619072 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -2,16 +2,17 @@ (:gen-class) (:require [auto-ap.handler :refer [app]] + [auto-ap.jobs.bulk-journal-import :as job-bulk-journal-import] [auto-ap.jobs.close-auto-invoices :as job-close-auto-invoices] [auto-ap.jobs.current-balance-cache :as job-current-balance-cache] + [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.intuit :as job-intuit] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] [auto-ap.jobs.plaid :as job-plaid] - [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] + [auto-ap.jobs.register-invoice-import :as job-register-invoice-import] [auto-ap.jobs.square :as job-square] [auto-ap.jobs.square2 :as job-square2] - [auto-ap.jobs.bulk-journal-import :as job-bulk-journal-import] [auto-ap.jobs.sysco :as job-sysco] [auto-ap.jobs.vendor-usages :as job-vendor-usages] [auto-ap.jobs.yodlee2 :as job-yodlee2] @@ -125,6 +126,10 @@ (= job "ezcater-upsert") (job-ezcater-upsert/-main) + + (= job "register-invoice-import") + (job-register-invoice-import/-main) + (= job "bulk-journal-import") (job-bulk-journal-import/-main) diff --git a/src/clj/user.clj b/src/clj/user.clj index 288bef70..302b131c 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -5,6 +5,7 @@ [auto-ap.ledger :as l :refer [transact-with-ledger]] [auto-ap.server] [auto-ap.square.core :as square] + [auto-ap.square.core2 :as square2] [auto-ap.time :as atime] [auto-ap.utils :refer [by]] [clj-time.coerce :as c] @@ -466,11 +467,43 @@ (square/upsert-settlements client square-location))))) + +(defn historical-load-sales2 [client-code days] + (let [client (d/pull (d/db auto-ap.datomic/conn) + square/square-read + [:client/code client-code])] + (doseq [square-location (:client/square-locations client) + :when (:square-location/client-location square-location)] + + (println "orders") + (lc/with-context {:source "Historical loading data"} + (doseq [d (per/periodic-seq (t/plus (t/today) (t/days (- days))) + (t/today) + (t/days 1))] + (println d) + (square2/upsert client square-location (c/to-date-time d) (c/to-date-time (t/plus d (t/days 1)))))) + + (println "refunds") + (square2/upsert-refunds client square-location) + + + (println "settlements") + (with-redefs [square2/lookup-dates (fn lookup-dates [] + (->> (per/periodic-seq (t/plus (t/today) (t/days (- days))) + (t/today) + (t/days 2)) + (map (fn [d] + [(atime/unparse (t/plus d (t/days 1)) atime/iso-date) + + (atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))] + + (square2/upsert-settlements client square-location))))) + #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-sales-for-day [date] (doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...] - :where ['?e :client/square-locations ]] - (d/db auto-ap.datomic/conn)) + :where ['?e :client/square-locations ]] + (d/db auto-ap.datomic/conn)) square-location (:client/square-locations client) :when (:square-location/client-location square-location)] (println client) diff --git a/src/cljs/auto_ap/views/pages/admin/jobs.cljs b/src/cljs/auto_ap/views/pages/admin/jobs.cljs index 3cbc32d7..bdfc84cb 100644 --- a/src/cljs/auto_ap/views/pages/admin/jobs.cljs +++ b/src/cljs/auto_ap/views/pages/admin/jobs.cljs @@ -17,7 +17,7 @@ (def default-read [:name :start-date :end-date :status]) -(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :ezcater-upsert]) +(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :register-invoice-import :ezcater-upsert]) (re-frame/reg-event-fx ::params-change @@ -64,7 +64,7 @@ :venia/queries [{:query/data [:request-job {:which (name which) - :args (when (= which :bulk-journal-import) + :args (when (#{:bulk-journal-import :register-invoice-import} which) (str/escape (pr-str (:data form)) {\" "\\\""}))} [:message]]}]} :on-success [::success]}})) @@ -128,6 +128,22 @@ [form-builder/submit-button {:class "is-small"} "Ledger Import"] ]]]) +(defn register-invoice-import-button [] + [form-builder/builder {:submit-event [::request :register-invoice-import] + :id ::register-invoice-import-form} + [:p.field.has-addons + [:p.control + [:a.button.is-static.is-small + "s3://data.prod.app.integreatconsult.com/bulk-import/"]] + [:p.control + [form-builder/raw-field-v2 {:field :ledger-url} + [:input.input.is-small {:placeholder "invoices.csv"}]]] + [:p.control + [form-builder/submit-button {:class "is-small"} "Register Invoice Import"] + ]]]) + + + ;; VIEWS (def jobs-content (with-meta @@ -147,7 +163,8 @@ [job-button {:which :intuit} "Start Intuit"] [job-button {:which :plaid} "Start Plaid"] [job-button {:which :ezcater-upsert} "EZCater Sync"] - [bulk-journal-import-button]]] + [bulk-journal-import-button] + [register-invoice-import-button]]] [table/table {:id :jobs :data-page ::page}]])])) {:component-did-mount (dispatch-event [::mounted ]) diff --git a/terraform/deploy.tf b/terraform/deploy.tf index 99aa3b98..ec67e636 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -444,4 +444,16 @@ module "bulk_journal_import_job" { use_schedule = false memory = 4096 cpu = 1024 +} + +module "register_invoice_import_job" { + source = "./background-job/" + ecs_cluster = var.ecs_cluster + task_role_arn = var.task_role_arn + stage = var.stage + job_name = "register-invoice-import" + execution_role_arn = var.execution_role_arn + use_schedule = false + memory = 8192 + cpu = 2048 } \ No newline at end of file diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 49fa66fe..37664b9c 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.2.7", - "serial": 220, + "terraform_version": "1.3.2", + "serial": 281, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412", "timeouts": { "delete": null }, @@ -348,7 +348,7 @@ "desync_mitigation_mode": "defensive", "dns_name": "integreat-app-prod-1104326262.us-east-1.elb.amazonaws.com", "drop_invalid_header_fields": false, - "enable_cross_zone_load_balancing": null, + "enable_cross_zone_load_balancing": true, "enable_deletion_protection": true, "enable_http2": true, "enable_waf_fail_open": false, @@ -1092,7 +1092,7 @@ ], "revision": 1, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -1370,6 +1370,140 @@ } ] }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "schedule", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:events:us-east-1:679918342773:rule/ezcater-upsert-schedule", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "id": "ezcater-upsert-schedule", + "is_enabled": true, + "name": "ezcater-upsert-schedule", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(8 hours)", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "job_target", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [ + { + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "group": "", + "launch_type": "", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "placement_constraint": [], + "platform_version": "", + "propagate_tags": "TASK_DEFINITION", + "tags": null, + "task_count": 1, + "task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1" + } + ], + "event_bus_name": "default", + "http_target": [], + "id": "ezcater-upsert-schedule-ezcater-upsert", + "input": "", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733", + "rule": "ezcater-upsert-schedule", + "run_command_targets": [], + "sqs_target": [], + "target_id": "ezcater-upsert" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ezcater_upsert_job.aws_cloudwatch_event_rule.schedule", + "module.ezcater_upsert_job.aws_ecs_task_definition.background_taskdef" + ] + } + ] + }, + { + "module": "module.ezcater_upsert_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ezcater-upsert\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ezcater-upsert\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ezcater-upsert\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "ezcater_upsert_prod", + "id": "ezcater_upsert_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "2048", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.import_uploaded_invoices_job", "mode": "managed", @@ -2615,5 +2749,6 @@ } ] } - ] + ], + "check_results": [] } diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index 3ae5ec71..66a9d287 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.2.7", - "serial": 217, + "terraform_version": "1.3.2", + "serial": 220, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373", "timeouts": { "delete": null }, @@ -1063,6 +1063,45 @@ } ] }, + { + "module": "module.bulk_journal_import_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/bulk_journal_import_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"bulk-journal-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"bulk-journal-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"bulk-journal-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "bulk_journal_import_prod", + "id": "bulk_journal_import_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "4096", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.close_auto_invoices_job", "mode": "managed", @@ -2155,7 +2194,7 @@ "name": "sysco-schedule", "name_prefix": "", "role_arn": "", - "schedule_expression": "rate(1 hour)", + "schedule_expression": "rate(3 hours)", "tags": {}, "tags_all": {} }, @@ -2432,7 +2471,7 @@ ], "revision": 2, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -2576,5 +2615,6 @@ } ] } - ] + ], + "check_results": null }