notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adds register invoice import

Browse Source
This commit is contained in:
Bryce Covert
2022-12-04 11:09:51 -08:00
parent 58fad1b4f1
commit 3e512a52ac
7 changed files with 394 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
src/clj/auto_ap/jobs/register_invoice_import.clj Normal file
View File

@@ -0,0 +1,133 @@
(ns auto-ap.jobs.register-invoice-import
(:gen-class)
(:require
[auto-ap.jobs.core :refer [execute]]
[amazonica.aws.s3 :as s3]
[datomic.api :as d]
[auto-ap.utils :refer [dollars=]]
[clojure.string :as str]
[clj-time.coerce :as coerce]
[auto-ap.time :as atime]
[auto-ap.datomic :refer [conn]]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[config.core :refer [env]]
[clojure.tools.logging :as log]))
(def bucket (:data-bucket env))
(defn s3->csv [url]
(->> (-> (s3/get-object {:bucket-name bucket
:key (str "bulk-import/" url)})
:input-stream
io/reader
csv/read-csv)))
(defn register-invoice-import [args]
(let [{:keys [ledger-url]} args
data (s3->csv ledger-url)
db (d/db conn)
i->invoice-id (fn [i]
(try (Long/parseLong i)
(catch Exception e
(:db/id (d/pull db '[:db/id]
[:invoice/original-id (Long/parseLong (first (str/split i #"-")))])))))
invoice-totals (->> data
(drop 1)
(group-by first)
(map (fn [[k values]]
[(i->invoice-id k)
(reduce + 0.0
(->> values
(map (fn [[_ _ _ _ amount]]
(- (Double/parseDouble amount))))))
]))
(into {}))
changes (->>
(for [[i
invoice-expense-account-id
target-account
target-date
amount
_
location] (drop 1 data)
:let [invoice-id (i->invoice-id i)
invoice (d/entity db invoice-id)
current-total (:invoice/total invoice)
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
(:db/id (first (:invoice/expense-accounts invoice))))))
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
(:db/id (first (:invoice/expense-accounts invoice)))
(d/tempid :db.part/user))
invoice-expense-account (when-not new-account?
(or (d/entity db invoice-expense-account-id)
(d/entity db [:invoice-expense-account/original-id invoice-expense-account-id])))
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
target-account-id (Long/parseLong (str/trim target-account))
target-date (coerce/to-date (atime/parse target-date atime/normal-date))
current-date (:invoice/date invoice)
current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0)
target-expense-account-amount (- (Double/parseDouble amount))
current-expense-account-location (:invoice-expense-account/location invoice-expense-account)
target-expense-account-location location
[[_ _ invoice-payment]] (vec (d/q
'[:find ?p ?a ?ip
:in $ ?i
:where [?ip :invoice-payment/invoice ?i]
[?ip :invoice-payment/amount ?a]
[?ip :invoice-payment/payment ?p]
]
db invoice-id))]
:when current-total]
[
(when (not (dollars= current-total target-total))
{:db/id invoice-id
:invoice/total target-total})
(when new-account?
{:db/id invoice-id
:invoice/expense-accounts invoice-expense-account-id})
(when (and target-date (not= current-date target-date))
{:db/id invoice-id
:invoice/date target-date})
(when (and
(not (dollars= current-total target-total))
invoice-payment)
[:db/retractEntity invoice-payment])
(when (or new-account?
(not (dollars= current-expense-account-amount target-expense-account-amount)))
{:db/id invoice-expense-account-id
:invoice-expense-account/amount target-expense-account-amount})
(when (not= current-expense-account-location
target-expense-account-location)
{:db/id invoice-expense-account-id
:invoice-expense-account/location target-expense-account-location})
(when (not= current-account-id target-account-id )
{:db/id invoice-expense-account-id
:invoice-expense-account/account target-account-id})])
(mapcat identity)
(filter identity)
vec)]
(doseq [n (partition-all 50 changes)]
(log/info "transacting" n)
@(d/transact conn n))
))
(defn -main [& _]
(execute "register-invoice-import" #(register-invoice-import (:args env))))

9
src/clj/auto_ap/server.clj
View File

@@ -2,16 +2,17 @@
(:gen-class)
(:require
[auto-ap.handler :refer [app]]
[auto-ap.jobs.bulk-journal-import :as job-bulk-journal-import]
[auto-ap.jobs.close-auto-invoices :as job-close-auto-invoices]
[auto-ap.jobs.current-balance-cache :as job-current-balance-cache]
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
[auto-ap.jobs.intuit :as job-intuit]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.plaid :as job-plaid]
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
[auto-ap.jobs.register-invoice-import :as job-register-invoice-import]
[auto-ap.jobs.square :as job-square]
[auto-ap.jobs.square2 :as job-square2]
[auto-ap.jobs.bulk-journal-import :as job-bulk-journal-import]
[auto-ap.jobs.sysco :as job-sysco]
[auto-ap.jobs.vendor-usages :as job-vendor-usages]
[auto-ap.jobs.yodlee2 :as job-yodlee2]
@@ -125,6 +126,10 @@
(= job "ezcater-upsert")
(job-ezcater-upsert/-main)
(= job "register-invoice-import")
(job-register-invoice-import/-main)
(= job "bulk-journal-import")
(job-bulk-journal-import/-main)

37
src/clj/user.clj
View File

@@ -5,6 +5,7 @@
[auto-ap.ledger :as l :refer [transact-with-ledger]]
[auto-ap.server]
[auto-ap.square.core :as square]
[auto-ap.square.core2 :as square2]
[auto-ap.time :as atime]
[auto-ap.utils :refer [by]]
[clj-time.coerce :as c]
@@ -466,11 +467,43 @@
(square/upsert-settlements client square-location)))))
(defn historical-load-sales2 [client-code days]
(let [client (d/pull (d/db auto-ap.datomic/conn)
square/square-read
[:client/code client-code])]
(doseq [square-location (:client/square-locations client)
:when (:square-location/client-location square-location)]
(println "orders")
(lc/with-context {:source "Historical loading data"}
(doseq [d (per/periodic-seq (t/plus (t/today) (t/days (- days)))
(t/today)
(t/days 1))]
(println d)
(square2/upsert client square-location (c/to-date-time d) (c/to-date-time (t/plus d (t/days 1))))))
(println "refunds")
(square2/upsert-refunds client square-location)
(println "settlements")
(with-redefs [square2/lookup-dates (fn lookup-dates []
(->> (per/periodic-seq (t/plus (t/today) (t/days (- days)))
(t/today)
(t/days 2))
(map (fn [d]
[(atime/unparse (t/plus d (t/days 1)) atime/iso-date)
(atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))]
(square2/upsert-settlements client square-location)))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn load-sales-for-day [date]
(doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...]
:where ['?e :client/square-locations ]]
(d/db auto-ap.datomic/conn))
:where ['?e :client/square-locations ]]
(d/db auto-ap.datomic/conn))
square-location (:client/square-locations client)
:when (:square-location/client-location square-location)]
(println client)

23
src/cljs/auto_ap/views/pages/admin/jobs.cljs
View File

@@ -17,7 +17,7 @@
(def default-read [:name :start-date :end-date :status])
(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :ezcater-upsert])
(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :register-invoice-import :ezcater-upsert])
(re-frame/reg-event-fx
::params-change
@@ -64,7 +64,7 @@
:venia/queries [{:query/data
[:request-job
{:which (name which)
:args (when (= which :bulk-journal-import)
:args (when (#{:bulk-journal-import :register-invoice-import} which)
(str/escape (pr-str (:data form)) {\" "\\\""}))}
[:message]]}]}
:on-success [::success]}}))
@@ -128,6 +128,22 @@
[form-builder/submit-button {:class "is-small"} "Ledger Import"]
]]])
(defn register-invoice-import-button []
[form-builder/builder {:submit-event [::request :register-invoice-import]
:id ::register-invoice-import-form}
[:p.field.has-addons
[:p.control
[:a.button.is-static.is-small
"s3://data.prod.app.integreatconsult.com/bulk-import/"]]
[:p.control
[form-builder/raw-field-v2 {:field :ledger-url}
[:input.input.is-small {:placeholder "invoices.csv"}]]]
[:p.control
[form-builder/submit-button {:class "is-small"} "Register Invoice Import"]
]]])
;; VIEWS
(def jobs-content
(with-meta
@@ -147,7 +163,8 @@
[job-button {:which :intuit} "Start Intuit"]
[job-button {:which :plaid} "Start Plaid"]
[job-button {:which :ezcater-upsert} "EZCater Sync"]
[bulk-journal-import-button]]]
[bulk-journal-import-button]
[register-invoice-import-button]]]
[table/table {:id :jobs
:data-page ::page}]])]))
{:component-did-mount (dispatch-event [::mounted ])

12
terraform/deploy.tf
View File

@@ -444,4 +444,16 @@ module "bulk_journal_import_job" {
use_schedule = false
memory = 4096
cpu = 1024
}
module "register_invoice_import_job" {
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
job_name = "register-invoice-import"
execution_role_arn = var.execution_role_arn
use_schedule = false
memory = 8192
cpu = 2048
}

147
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 220,
"terraform_version": "1.3.2",
"serial": 281,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -164,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:412",
"timeouts": {
"delete": null
},
@@ -348,7 +348,7 @@
"desync_mitigation_mode": "defensive",
"dns_name": "integreat-app-prod-1104326262.us-east-1.elb.amazonaws.com",
"drop_invalid_header_fields": false,
"enable_cross_zone_load_balancing": null,
"enable_cross_zone_load_balancing": true,
"enable_deletion_protection": true,
"enable_http2": true,
"enable_waf_fail_open": false,
@@ -1092,7 +1092,7 @@
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
@@ -1370,6 +1370,140 @@
}
]
},
{
"module": "module.ezcater_upsert_job",
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"index_key": 0,
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/ezcater-upsert-schedule",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "ezcater-upsert-schedule",
"is_enabled": true,
"name": "ezcater-upsert-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(8 hours)",
"tags": null,
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"module": "module.ezcater_upsert_job",
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"index_key": 0,
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": null,
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "ezcater-upsert-schedule-ezcater-upsert",
"input": "",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "ezcater-upsert-schedule",
"run_command_targets": [],
"sqs_target": [],
"target_id": "ezcater-upsert"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"module.ezcater_upsert_job.aws_cloudwatch_event_rule.schedule",
"module.ezcater_upsert_job.aws_ecs_task_definition.background_taskdef"
]
}
]
},
{
"module": "module.ezcater_upsert_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ezcater_upsert_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ezcater-upsert\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ezcater-upsert\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ezcater-upsert\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "1024",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "ezcater_upsert_prod",
"id": "ezcater_upsert_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "2048",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"module": "module.import_uploaded_invoices_job",
"mode": "managed",
@@ -2615,5 +2749,6 @@
}
]
}
]
],
"check_results": []
}

52
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.2.7",
"serial": 217,
"terraform_version": "1.3.2",
"serial": 220,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -164,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:372",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:373",
"timeouts": {
"delete": null
},
@@ -1063,6 +1063,45 @@
}
]
},
{
"module": "module.bulk_journal_import_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/bulk_journal_import_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"bulk-journal-import\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"bulk-journal-import\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"bulk-journal-import\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "1024",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "bulk_journal_import_prod",
"id": "bulk_journal_import_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "4096",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"module": "module.close_auto_invoices_job",
"mode": "managed",
@@ -2155,7 +2194,7 @@
"name": "sysco-schedule",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(1 hour)",
"schedule_expression": "rate(3 hours)",
"tags": {},
"tags_all": {}
},
@@ -2432,7 +2471,7 @@
],
"revision": 2,
"runtime_platform": [],
"tags": null,
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
@@ -2576,5 +2615,6 @@
}
]
}
]
],
"check_results": null
}