diff --git a/resources/schema.edn b/resources/schema.edn index 158f241f..a48f1f53 100644 --- a/resources/schema.edn +++ b/resources/schema.edn @@ -1650,11 +1650,15 @@ :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "Current status", :db/ident :yodlee-account/status} + {:db/valueType #:db{:ident :db.type/double}, :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "Available Balance", :db/ident :yodlee-account/available-balance} - +{:db/valueType :db.type/double, + :db/cardinality :db.cardinality/one, + :db/doc "Pending Balance", + :db/ident :yodlee-account/pending-balance} {:db/valueType #:db{:ident :db.type/string}, :db/cardinality #:db{:ident :db.cardinality/one}, :db/doc "The yodlee merchant name", diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 06eeec6a..f567ca99 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -318,6 +318,7 @@ (defprotocol ImportBatch (import-transaction! [this transaction]) (get-stats [this ]) + (get-pending-balance [this]) (finish! [this]) (fail! [this error])) @@ -333,6 +334,7 @@ :import-batch/error 0 :import-batch/not-ready 0 :import-batch/extant 0}) + pending-balance (atom {}) extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 )) import-id (get (:tempids @(dc/transact-async conn [{:db/id "import-batch" :import-batch/date (coerce/to-date (t/now)) @@ -351,6 +353,17 @@ extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing) (:transaction/bank-account transaction)) action (categorize-transaction transaction bank-account extant)] + (try + (when (not= "POSTED" (:transaction/status transaction)) + + (swap! pending-balance (fn [pb] + (update pb + (:transaction/bank-account transaction) + (fnil + 0.0) + (:transaction/amount transaction))))) + (catch Exception e + (alog/warn ::cant-capture-pending + :error e))) (swap! stats #(update % (condp = action :import :import-batch/imported @@ -376,6 +389,8 @@ (get-stats [_] @stats) + (get-pending-balance [_ bank-account] + (get @pending-balance bank-account)) (fail! [_ error] (alog/error ::cant-complete-import diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj index 3840437d..7f51f895 100644 --- a/src/clj/auto_ap/import/yodlee2.clj +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -71,7 +71,15 @@ :transaction/client [:client/code client-code]))) (alog/info ::finished-import) - (t/finish! import-batch)) + (t/finish! import-batch) + (doseq [[yodlee-account bank-account] account-lookup] + (try + @(dc/transact auto-ap.datomic/conn + [{:db/id yodlee-account + :yodlee-account/pending-balance (t/get-pending-balance import-batch bank-account)}]) + (catch Exception e + (alog/error ::cant-persist-yodlee-account-pending-balance + :error e))))) (statsd/event {:title "Yodlee2 import Finished" :text (pr-str (t/get-stats import-batch)) :priority :low} diff --git a/src/clj/auto_ap/ssr/dashboard.clj b/src/clj/auto_ap/ssr/dashboard.clj index 7f7fd626..5b23cadf 100644 --- a/src/clj/auto_ap/ssr/dashboard.clj +++ b/src/clj/auto_ap/ssr/dashboard.clj @@ -42,6 +42,7 @@ {:bank-account/intuit-bank-account [:intuit-bank-account/current-balance [:intuit-bank-account/last-synced :xform clj-time.coerce/from-date]]} {:bank-account/yodlee-account [:yodlee-account/available-balance + :yodlee-account/pending-balance [:yodlee-account/last-synced :xform clj-time.coerce/from-date]]} {:bank-account/plaid-account [:plaid-account/balance [:plaid-account/last-synced :xform clj-time.coerce/from-date]]}]}] @@ -71,6 +72,11 @@ (-> b :bank-account/yodlee-account :yodlee-account/available-balance) (-> b :bank-account/plaid-account :plaid-account/balance) 0.0))] + (when-let [pending-balance (-> b :bank-account/yodlee-account :yodlee-account/available-balance)] + (list + [:div (str n " Pending Balance")] + [:div.text-right (format "$%,.2f" pending-balance)])) + [:div.text-xs.text-gray-400.text-right (or (some-> (:bank-account/intuit-bank-account b) (:intuit-bank-account/last-synced) (atime/unparse-local atime/standard-time) @@ -215,7 +221,6 @@ "Income: " (format "$%,.2f" sales)] [:div "Expenses: " (format "$%,.2f" expenses)]))))) - (defn tasks-card [request] (html-response (com/card {:class "w-full h-full p-4"} diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 1def819b..8139110e 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 712, + "serial": 714, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -435,7 +435,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:834", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:837", "timeouts": { "create": null, "delete": null, @@ -667,9 +667,9 @@ "provisioned_throughput_in_mibps": 0, "size_in_bytes": [ { - "value": 1429075968, + "value": 1432420352, "value_in_ia": 0, - "value_in_standard": 1429075968 + "value_in_standard": 1432420352 } ], "tags": { @@ -799,7 +799,23 @@ "credit_specification": [], "disable_api_stop": false, "disable_api_termination": false, - "ebs_block_device": [], + "ebs_block_device": [ + { + "delete_on_termination": false, + "device_name": "/dev/xvdf", + "encrypted": false, + "iops": 100, + "kms_key_id": "", + "snapshot_id": "", + "tags": { + "Name": "solr_storage_prod" + }, + "throughput": 0, + "volume_id": "vol-0069283d41ff6c010", + "volume_size": 30, + "volume_type": "gp2" + } + ], "ebs_optimized": false, "enclave_options": [ { @@ -811,7 +827,7 @@ "hibernation": false, "host_id": "", "host_resource_group_arn": null, - "iam_instance_profile": "", + "iam_instance_profile": "datomic-ddb", "id": "i-0ce809d16781daada", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", @@ -858,7 +874,7 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, "volume_id": "vol-0ec6fb488e712ba93", "volume_size": 30, @@ -2160,7 +2176,7 @@ "name": "current-balance-cache-schedule-prod", "name_prefix": "", "role_arn": "", - "schedule_expression": "rate(60 minutes)", + "schedule_expression": "rate(30 minutes)", "tags": {}, "tags_all": {} }, diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index 938664ec..1def819b 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 704, + "serial": 712, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -19,7 +19,7 @@ "sensitive": true }, "ec2_public_ip": { - "value": "54.162.172.142", + "value": "54.175.3.15", "type": "string" }, "queue_url": { @@ -31,34 +31,34 @@ { "mode": "data", "type": "aws_ami", - "name": "amazon_linux_2", + "name": "amazon_linux_2023", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-1::image/ami-0c32fe44cfb616fdd", + "arn": "arn:aws:ec2:us-east-1::image/ami-0fea1219385942eb5", "block_device_mappings": [ { "device_name": "/dev/xvda", "ebs": { "delete_on_termination": "true", "encrypted": "false", - "iops": "0", - "snapshot_id": "snap-0d41f1cffffa2ecba", - "throughput": "0", - "volume_size": "8", - "volume_type": "gp2" + "iops": "3000", + "snapshot_id": "snap-0631ed3b265acda34", + "throughput": "125", + "volume_size": "30", + "volume_type": "gp3" }, "no_device": "", "virtual_name": "" } ], - "boot_mode": "", - "creation_date": "2024-09-16T18:28:30.000Z", - "deprecation_time": "2025-07-01T00:00:00.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20240916.0 arm64 HVM gp2", + "boot_mode": "uefi", + "creation_date": "2024-09-17T21:17:56.000Z", + "deprecation_time": "2026-09-17T21:17:56.000Z", + "description": "Amazon Linux AMI 2023.0.20240917 arm64 ECS HVM EBS", "ena_support": true, "executable_users": null, "filter": [ @@ -71,7 +71,7 @@ { "name": "name", "values": [ - "amzn2-ami-hvm-*" + "al2023-ami-*" ] }, { @@ -82,18 +82,18 @@ } ], "hypervisor": "xen", - "id": "ami-0c32fe44cfb616fdd", - "image_id": "ami-0c32fe44cfb616fdd", - "image_location": "amazon/amzn2-ami-hvm-2.0.20240916.0-arm64-gp2", + "id": "ami-0fea1219385942eb5", + "image_id": "ami-0fea1219385942eb5", + "image_location": "amazon/al2023-ami-ecs-hvm-2023.0.20240917-kernel-6.1-arm64", "image_owner_alias": "amazon", "image_type": "machine", - "imds_support": "", + "imds_support": "v2.0", "include_deprecated": false, "kernel_id": "", "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20240916.0-arm64-gp2", + "name": "al2023-ami-ecs-hvm-2023.0.20240917-kernel-6.1-arm64", "name_regex": null, - "owner_id": "137112412989", + "owner_id": "591542846629", "owners": null, "platform": "", "platform_details": "Linux/UNIX", @@ -102,7 +102,7 @@ "ramdisk_id": "", "root_device_name": "/dev/xvda", "root_device_type": "ebs", - "root_snapshot_id": "snap-0d41f1cffffa2ecba", + "root_snapshot_id": "snap-0631ed3b265acda34", "sriov_net_support": "simple", "state": "available", "state_reason": { @@ -324,11 +324,11 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:ec2:us-east-1:679918342773:volume/vol-010fe54b640fb8a99", + "arn": "arn:aws:ec2:us-east-1:679918342773:volume/vol-0069283d41ff6c010", "availability_zone": "us-east-1a", "encrypted": false, "final_snapshot": false, - "id": "vol-010fe54b640fb8a99", + "id": "vol-0069283d41ff6c010", "iops": 100, "kms_key_id": "", "multi_attach_enabled": false, @@ -349,7 +349,7 @@ "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjMwMDAwMDAwMDAwMH19", "dependencies": [ "aws_instance.solr_ec2", - "data.aws_ami.amazon_linux_2" + "data.aws_ami.amazon_linux_2023" ] } ] @@ -777,8 +777,8 @@ { "schema_version": 1, "attributes": { - "ami": "ami-0c32fe44cfb616fdd", - "arn": "arn:aws:ec2:us-east-1:679918342773:instance/i-0bbdf9c800a03cba7", + "ami": "ami-0fea1219385942eb5", + "arn": "arn:aws:ec2:us-east-1:679918342773:instance/i-0ce809d16781daada", "associate_public_ip_address": true, "availability_zone": "us-east-1a", "capacity_reservation_specification": [ @@ -799,23 +799,7 @@ "credit_specification": [], "disable_api_stop": false, "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": false, - "device_name": "/dev/xvdf", - "encrypted": false, - "iops": 100, - "kms_key_id": "", - "snapshot_id": "", - "tags": { - "Name": "solr_storage_prod" - }, - "throughput": 0, - "volume_id": "vol-010fe54b640fb8a99", - "volume_size": 30, - "volume_type": "gp2" - } - ], + "ebs_block_device": [], "ebs_optimized": false, "enclave_options": [ { @@ -828,7 +812,7 @@ "host_id": "", "host_resource_group_arn": null, "iam_instance_profile": "", - "id": "i-0bbdf9c800a03cba7", + "id": "i-0ce809d16781daada", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m7g.large", @@ -844,8 +828,8 @@ "metadata_options": [ { "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional", + "http_put_response_hop_limit": 2, + "http_tokens": "required", "instance_metadata_tags": "disabled" } ], @@ -855,8 +839,8 @@ "password_data": "", "placement_group": "", "placement_partition_number": 0, - "primary_network_interface_id": "eni-0c7562b5274a4b60d", - "private_dns": "ip-172-31-46-148.ec2.internal", + "primary_network_interface_id": "eni-07dec2725fa4c70ed", + "private_dns": "ip-172-31-38-103.ec2.internal", "private_dns_name_options": [ { "enable_resource_name_dns_a_record": false, @@ -864,26 +848,27 @@ "hostname_type": "ip-name" } ], - "private_ip": "172.31.46.148", - "public_dns": "ec2-54-162-172-142.compute-1.amazonaws.com", - "public_ip": "54.162.172.142", + "private_ip": "172.31.38.103", + "public_dns": "ec2-54-175-3-15.compute-1.amazonaws.com", + "public_ip": "54.175.3.15", "root_block_device": [ { "delete_on_termination": true, "device_name": "/dev/xvda", "encrypted": false, - "iops": 100, + "iops": 3000, "kms_key_id": "", - "tags": {}, - "throughput": 0, - "volume_id": "vol-0fea5a3f3ca20ab76", + "tags": null, + "throughput": 125, + "volume_id": "vol-0ec6fb488e712ba93", "volume_size": 30, - "volume_type": "gp2" + "volume_type": "gp3" } ], "secondary_private_ips": [], "security_groups": [ "datomic-access", + "http-proxy-2", "integreat-app" ], "source_dest_check": true, @@ -909,7 +894,7 @@ "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "data.aws_ami.amazon_linux_2" + "data.aws_ami.amazon_linux_2023" ] } ] @@ -1630,7 +1615,7 @@ "schema_version": 0, "attributes": { "attributes": { - "AWS_INSTANCE_IPV4": "172.31.46.148" + "AWS_INSTANCE_IPV4": "172.31.38.103" }, "id": "solr-ec2", "instance_id": "solr-ec2", @@ -1641,7 +1626,7 @@ "dependencies": [ "aws_instance.solr_ec2", "aws_service_discovery_service.solr_ec2", - "data.aws_ami.amazon_linux_2" + "data.aws_ami.amazon_linux_2023" ] } ] @@ -1959,19 +1944,19 @@ "attributes": { "device_name": "/dev/xvdf", "force_detach": null, - "id": "vai-172440334", - "instance_id": "i-0bbdf9c800a03cba7", + "id": "vai-3578365595", + "instance_id": "i-0ce809d16781daada", "skip_destroy": null, "stop_instance_before_detaching": null, "timeouts": null, - "volume_id": "vol-010fe54b640fb8a99" + "volume_id": "vol-0069283d41ff6c010" }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_ebs_volume.solr_ec2_storage", "aws_instance.solr_ec2", - "data.aws_ami.amazon_linux_2" + "data.aws_ami.amazon_linux_2023" ] } ]