notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logging updates, makes invoice voiding trigger refresh of running balances.

Browse Source
This commit is contained in:
Bryce
2024-09-27 16:59:35 -07:00
parent 4c14e43734
commit e25e2bfc46
5 changed files with 124 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
iol_ion/src/iol_ion/tx/upsert_invoice.clj
View File

@@ -1,4 +1,5 @@
(ns iol-ion.tx.upsert-invoice
(:import [java.util Date])
(:require [datomic.api :as dc]))
(defn -remove-nils [m]
@@ -74,8 +75,15 @@
(:db/id invoice))
journal-entry (invoice->journal-entry (:db-after with-invoice)
invoice-id
(:db/id invoice))]
(:db/id invoice))
client-id (-> (dc/pull (:db-after with-invoice)
[{:invoice/client [:db/id]}]
invoice-id)
:invoice/client
:db/id)]
(into upserted-entity
(if journal-entry
[[:upsert-ledger journal-entry]]
[[:db/retractEntity [:journal-entry/original-entity (:db/id invoice)]]]))))
[[:db/retractEntity [:journal-entry/original-entity (:db/id invoice)]]
{:db/id client-id
:client/ledger-last-change (Date.)}]))))

2
resources/functions.edn
View File

File diff suppressed because one or more lines are too long

136
src/clj/auto_ap/ledger.clj
View File

@@ -382,66 +382,100 @@
:in $
:where [?c :client/code]]
(dc/db conn))]
{:db/id c :client/ledger-last-change (c/to-date (t/now))})
{:db/id c :client/ledger-last-change (c/to-date (t/now))
:client/last-running-balance #inst "2000-01-01"})
{:user/name "backfill-client and dates"}))
(defn refresh-bank-account-balances [client-ids]
(mu/with-context {:source "current-balance-refresh"}
(let [db (dc/db conn)
clients (map first (dc/q '[:find (pull ?c [:db/id :client/code {:client/bank-accounts [:db/id :bank-account/code]}])
:in $ [?c ...]
:where [?c :client/code]]
db
client-ids))]
(dorun
(for [{client :db/id code :client/code bank-accounts :client/bank-accounts} clients
{bank-account :db/id bac :bank-account/code} bank-accounts]
(let [{[_ _ _ _ _ running-balance] :v} (->> (dc/index-range db :journal-entry-line/running-balance-tuple [client bank-account "A"] [client bank-account "A" #inst "2050-01-01"])
seq
(sort-by (fn [{id :e [_ _ _ current-date] :v}]
[current-date id]))
(last))
running-balance (or running-balance 0.0)]
(alog/info ::updating-bank-account-balance
:bank-account bac
:balance running-balance)
@(dc/transact conn [{:db/id bank-account
:bank-account/current-balance-synced (c/to-date (t/now))
:bank-account/current-balance running-balance}])))))))
(defn upsert-running-balance []
(let [db (dc/db conn)
starting-at (c/to-date (t/now))
clients (clients-needing-refresh db)
_ (alog/info ::clients-needing-update :clients clients :count (count clients))
client-change-stats (atom {})
changes (for [c clients
:let [client-id (:db/id c)
account-lookup (build-account-lookup client-id)]
running-balance-set (->> (dc/index-range db :journal-entry-line/running-balance-tuple [client-id] [(inc client-id)])
(seq)
(partition-by (fn [{[current-client current-account current-location current-date debit credit running-balance]
:v}]
[current-client current-account current-location])))
running-balance-change (->> running-balance-set
(sort-by (fn [{id :e [_ _ _ current-date] :v}]
[current-date id]))
(reduce
(fn [{:keys [changes last-running-balance]}
{id :e
[current-client current-account current-location current-date debit credit running-balance]
:v}]
(let [delta (if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account_type (account-lookup current-account)))
(- (or debit 0.0) (or credit 0.0))
(- (or credit 0.0) (or debit 0.0)))
correct-running-balance (+ last-running-balance delta)
running-balance-changed? (not (dollars= correct-running-balance (or running-balance 0.0)))]
(when running-balance-changed?
(swap! client-change-stats update (:client/code c) (fnil inc 0)))
(cond-> {:last-account-lookup account-lookup
:last-running-balance correct-running-balance
:changes changes}
running-balance-changed?
(update :changes conj {:db/id id
:journal-entry-line/running-balance correct-running-balance}))))
{:last-running-balance 0.0})
:changes)]
running-balance-change)]
(alog/info ::change-stats :stats @client-change-stats)
(mu/trace ::update-running-balance []
(auto-ap.datomic/audit-transact-batch changes
{:user/name "running balance updater"}))
(auto-ap.datomic/audit-transact (mapv (fn [c]
{:db/id (:db/id c)
:client/last-running-balance starting-at})
clients)
{:user/name "running balance updater"})
(count changes)))
(mu/with-context {:service "upsert-running-balance"
:source "upsert-running-balance" }
(mu/trace ::updating-balance
[:service "upsert-running-balance"
:source "upsert-running-balance" ]
(let [db (dc/db conn)
starting-at (c/to-date (t/now))
clients (clients-needing-refresh db)
_ (alog/info ::clients-needing-update :clients clients :count (count clients))
client-change-stats (atom {})
changes (for [c clients
:let [client-id (:db/id c)
account-lookup (build-account-lookup client-id)]
running-balance-set (->> (dc/index-range db :journal-entry-line/running-balance-tuple [client-id] [(inc client-id)])
(seq)
(partition-by (fn [{[current-client current-account current-location current-date debit credit running-balance]
:v}]
[current-client current-account current-location])))
running-balance-change (->> running-balance-set
(sort-by (fn [{id :e [_ _ _ current-date] :v}]
[current-date id]))
(reduce
(fn [{:keys [changes last-running-balance]}
{id :e
[current-client current-account current-location current-date debit credit running-balance]
:v}]
(let [delta (if (#{:account-type/asset
:account-type/dividend
:account-type/expense} (:account_type (account-lookup current-account)))
(- (or debit 0.0) (or credit 0.0))
(- (or credit 0.0) (or debit 0.0)))
correct-running-balance (+ last-running-balance delta)
running-balance-changed? (not (dollars= correct-running-balance (or running-balance 0.0)))]
(when running-balance-changed?
(swap! client-change-stats update (:client/code c) (fnil inc 0)))
(cond-> {:last-account-lookup account-lookup
:last-running-balance correct-running-balance
:changes changes}
running-balance-changed?
(update :changes conj {:db/id id
:journal-entry-line/running-balance correct-running-balance}))))
{:last-running-balance 0.0})
:changes)]
running-balance-change)]
(mu/trace ::update-running-balance []
(auto-ap.datomic/audit-transact-batch changes
{:user/name "running balance updater"}))
(auto-ap.datomic/audit-transact (mapv (fn [c]
{:db/id (:db/id c)
:client/last-running-balance starting-at})
clients)
{:user/name "running balance updater"})
(alog/info ::change-stats :stats @client-change-stats)
(refresh-bank-account-balances (map :db/id clients))
(count changes)))))
(comment
(mark-all-clients-dirty)
(clients-needing-refresh (dc/db conn))
(upsert-running-balance)
;; SETUP running-balance-tuple

146
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.9.2",
"serial": 714,
"serial": 718,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -435,7 +435,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:837",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:841",
"timeouts": {
"create": null,
"delete": null,
@@ -667,9 +667,9 @@
"provisioned_throughput_in_mibps": 0,
"size_in_bytes": [
{
"value": 1432420352,
"value": 1434062848,
"value_in_ia": 0,
"value_in_standard": 1432420352
"value_in_standard": 1434062848
}
],
"tags": {
@@ -2156,144 +2156,6 @@
}
]
},
{
"module": "module.current_balance_cache[0]",
"mode": "managed",
"type": "aws_cloudwatch_event_rule",
"name": "schedule",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"index_key": 0,
"schema_version": 0,
"attributes": {
"arn": "arn:aws:events:us-east-1:679918342773:rule/current-balance-cache-schedule-prod",
"description": "",
"event_bus_name": "default",
"event_pattern": null,
"id": "current-balance-cache-schedule-prod",
"is_enabled": true,
"name": "current-balance-cache-schedule-prod",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(30 minutes)",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"module": "module.current_balance_cache[0]",
"mode": "managed",
"type": "aws_cloudwatch_event_target",
"name": "job_target",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"index_key": 0,
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:cluster/default",
"batch_target": [],
"dead_letter_config": [],
"ecs_target": [
{
"capacity_provider_strategy": [],
"enable_ecs_managed_tags": false,
"enable_execute_command": false,
"group": "",
"launch_type": "FARGATE",
"network_configuration": [
{
"assign_public_ip": true,
"security_groups": [
"sg-004e5855310c453a3",
"sg-02d167406b1082698"
],
"subnets": [
"subnet-5e675761",
"subnet-8519fde2",
"subnet-89bab8d4"
]
}
],
"ordered_placement_strategy": [],
"placement_constraint": [],
"platform_version": "",
"propagate_tags": "TASK_DEFINITION",
"tags": {},
"task_count": 1,
"task_definition_arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/current_balance_cache_prod:3"
}
],
"event_bus_name": "default",
"http_target": [],
"id": "current-balance-cache-schedule-prod-current-balance-cache",
"input": "",
"input_path": "",
"input_transformer": [],
"kinesis_target": [],
"redshift_target": [],
"retry_policy": [],
"role_arn": "arn:aws:iam::679918342773:role/service-role/Amazon_EventBridge_Invoke_ECS_1758992733",
"rule": "current-balance-cache-schedule-prod",
"run_command_targets": [],
"sqs_target": [],
"target_id": "current-balance-cache"
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
"dependencies": [
"module.current_balance_cache.aws_cloudwatch_event_rule.schedule",
"module.current_balance_cache.aws_ecs_task_definition.background_taskdef"
]
}
]
},
{
"module": "module.current_balance_cache[0]",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/current_balance_cache_prod:3",
"arn_without_revision": "arn:aws:ecs:us-east-1:679918342773:task-definition/current_balance_cache_prod",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"current-balance-cache\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"current-balance-cache\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"current-balance-cache\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat-cloud:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"systemControls\":[],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"systemControls\":[],\"volumesFrom\":[]}]",
"cpu": "512",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "current_balance_cache_prod",
"id": "current_balance_cache_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "2048",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 3,
"runtime_platform": [],
"skip_destroy": false,
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"module": "module.import_uploaded_invoices_job[0]",
"mode": "managed",

32
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.9.2",
"serial": 712,
"serial": 714,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -435,7 +435,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:834",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:837",
"timeouts": {
"create": null,
"delete": null,
@@ -667,9 +667,9 @@
"provisioned_throughput_in_mibps": 0,
"size_in_bytes": [
{
"value": 1429075968,
"value": 1432420352,
"value_in_ia": 0,
"value_in_standard": 1429075968
"value_in_standard": 1432420352
}
],
"tags": {
@@ -799,7 +799,23 @@
"credit_specification": [],
"disable_api_stop": false,
"disable_api_termination": false,
"ebs_block_device": [],
"ebs_block_device": [
{
"delete_on_termination": false,
"device_name": "/dev/xvdf",
"encrypted": false,
"iops": 100,
"kms_key_id": "",
"snapshot_id": "",
"tags": {
"Name": "solr_storage_prod"
},
"throughput": 0,
"volume_id": "vol-0069283d41ff6c010",
"volume_size": 30,
"volume_type": "gp2"
}
],
"ebs_optimized": false,
"enclave_options": [
{
@@ -811,7 +827,7 @@
"hibernation": false,
"host_id": "",
"host_resource_group_arn": null,
"iam_instance_profile": "",
"iam_instance_profile": "datomic-ddb",
"id": "i-0ce809d16781daada",
"instance_initiated_shutdown_behavior": "stop",
"instance_state": "running",
@@ -858,7 +874,7 @@
"encrypted": false,
"iops": 3000,
"kms_key_id": "",
"tags": null,
"tags": {},
"throughput": 125,
"volume_id": "vol-0ec6fb488e712ba93",
"volume_size": 30,
@@ -2160,7 +2176,7 @@
"name": "current-balance-cache-schedule-prod",
"name_prefix": "",
"role_arn": "",
"schedule_expression": "rate(60 minutes)",
"schedule_expression": "rate(30 minutes)",
"tags": {},
"tags_all": {}
},