notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merged.

Browse Source
This commit is contained in:
Bryce Covert
2023-04-27 11:33:54 -07:00
parent 94ae76aa3a 8ebc158081
commit aca8f7c92c
7 changed files with 246 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docker-compose.dev.yml
View File

@@ -18,7 +18,11 @@ services:
restart: "always"
environment:
- DATOMIC_PORT=4334
- XMS=-Xms4g
- XMX=-Xmx8g
- XMS=-Xms3g
- XMX=-Xmx6g
ulimits:
nofile:
soft: 65536
hard: 65536
ports:
- 4334:4334

8
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -65,15 +65,17 @@
({:bank-account-type/check :account-type/asset
:bank-account-type/cash :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (:account li)))))]
(:db/ident (:bank-account/type (:journal-entry-line/account li)))))]
[(-> j :journal-entry/client :client/code)
(-> j :journal-entry/vendor :vendor/name)
(atime/unparse (coerce/to-date-time (-> j :journal-entry/date))
atime/normal-date)
(-> j :db/id)
(-> li :db/id)
(-> li :journal-entry-line/account :account/numeric-code)
(-> li :journal-entry-line/account :account/name)
(or (-> li :journal-entry-line/account :account/numeric-code)
(-> li :journal-entry-line/account :bank-account/numeric-code))
(or (-> li :journal-entry-line/account :account/name)
(-> li :journal-entry-line/account :bank-account/name))
(some-> account-type name )
(-> li :journal-entry-line/debit)
(-> li :journal-entry-line/credit)

156
src/clj/auto_ap/jobs/ntg.clj
View File

@@ -43,14 +43,16 @@
:key k}))
(defn is-csv-file? [x]
(= "csv" (last (str/split x #"[\\.]"))))
(= "dat" (last (str/split x #"[\\.]"))))
(defn decipher-source [k]
(cond
(str/includes? k "Cintas")
(and (str/includes? k "Cintas")
(str/includes? k "zcic"))
:cintas
(and (str/includes? k "GeneralProduce")
(str/includes? k "FRANCHISEE")
(is-csv-file? k))
:general-produce
@@ -63,25 +65,34 @@
(defmethod extract-invoice-details :general-produce
[k input-stream clients]
(log/info ::parsing-general-produce :key k)
(->> (read-csv input-stream)
(drop 1)
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
(= "Y" break-flag)))
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
(let [matching-client (and location-hint
(parse/exact-match clients location-hint))]
{:invoice/vendor vendor
:invoice/location (parse/best-location-match matching-client location-hint location-hint )
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble invoice-total)
:invoice/outstanding-balance (Double/parseDouble invoice-total)
:invoice/client matching-client
:invoice/import-status :import-status/imported
:invoice/status :invoice-status/unpaid
:invoice/client-identifier location-hint
})))
(filter :invoice/client)))
(try
(->> (read-csv input-stream)
(drop 1)
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
(= "Y" break-flag)))
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
(let [matching-client (and location-hint
(parse/exact-match clients location-hint))]
(when-not matching-client
(log/warn ::missing-client
:client-hint location-hint))
{:invoice/location (parse/best-location-match matching-client location-hint location-hint )
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble invoice-total)
:invoice/vendor :vendor/general-produce
:invoice/outstanding-balance (Double/parseDouble invoice-total)
:invoice/client matching-client
:invoice/import-status :import-status/imported
:invoice/status :invoice-status/unpaid
:invoice/client-identifier location-hint
})))
(filter :invoice/client)
(into []))
(catch Exception e
(log/error ::cant-import-general-produce
:error e)
[])))
(defmethod extract-invoice-details :unknown
[k input-stream clients]
@@ -115,9 +126,7 @@
first
:content
first)
matching-client (and location-hint (parse/exact-match clients location-hint))
]
matching-client (and location-hint (parse/exact-match clients location-hint))]
(if matching-client
(let [invoice-date (->> node-seq
(map zip/node)
@@ -182,7 +191,7 @@
:invoice invoice)
[invoice])
(do
(log/info ::missing-client
(log/warn ::missing-client
:client-hint location-hint)
[]))))
@@ -204,55 +213,56 @@
:destination-key invoice-key })
invoice-key))
(defn import-ntg-invoices []
(let [clients (map first (d/q '[:find (pull ?c [:client/code
:db/id
:client/feature-flags
{:client/location-matches [:location-match/matches :location-match/location]}
:client/name
:client/matches
:client/locations])
:where [?c :client/code]]
(d/db conn)))
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "ntg-invoices/pending"})
:object-summaries
(map :key))]
(defn import-ntg-invoices
([] (import-ntg-invoices (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "ntg-invoices/pending"})
:object-summaries
(map :key))))
([keys]
(let [clients (map first (d/q '[:find (pull ?c [:client/code
:db/id
:client/feature-flags
{:client/location-matches [:location-match/matches :location-match/location]}
:client/name
:client/matches
:client/locations])
:where [?c :client/code]]
(d/db conn)))]
(log/info ::found-invoice-keys
:keys keys )
(let [transaction (->> keys
(mapcat (fn [k]
(try
(let [invoice-key (copy-readable-version k)
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
(with-open [is (-> (s3/get-object {:bucket-name bucket-name
:key k})
:input-stream)]
(->> (extract-invoice-details k
is
clients)
(map (fn [i]
(log/info ::importing-invoice
:invoice i)
i))
(mapv (fn [i]
`(propose-invoice ~(assoc i :invoice/source-url invoice-url))))))) (catch Exception e
(log/error ::cant-load-file
:key k
:exception e)
(mark-error k)
[]))))
(into []))]
(audit-transact transaction {:user/name "sysco importer" :user/role "admin"})
(log/info ::success
:count (count transaction)
:sample (take 3 transaction)))
(doseq [k keys]
(mark-key k))))
(log/info ::found-invoice-keys
:keys keys )
(let [transaction (->> keys
(mapcat (fn [k]
(try
(let [invoice-key (copy-readable-version k)
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
(with-open [is (-> (s3/get-object {:bucket-name bucket-name
:key k})
:input-stream)]
(->> (extract-invoice-details k
is
clients)
(map (fn [i]
(log/info ::importing-invoice
:invoice i)
i))
(mapv (fn [i]
`(propose-invoice ~(assoc i :invoice/source-url invoice-url))))))) (catch Exception e
(log/error ::cant-load-file
:key k
:exception e)
(mark-error k)
[]))))
(into []))]
(audit-transact transaction {:user/name "sysco importer" :user/role "admin"})
(log/info ::success
:count (count transaction)
:sample (take 3 transaction)))
(doseq [k keys]
(mark-key k)))))
(defn -main [& _]
(execute "ntg-invoices" import-ntg-invoices))
(execute "ntg" import-ntg-invoices))

4
src/clj/auto_ap/server.clj
View File

@@ -9,6 +9,7 @@
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
[auto-ap.jobs.intuit :as job-intuit]
[auto-ap.jobs.ntg :as job-ntg]
#_[auto-ap.backup :as backup]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
@@ -145,6 +146,9 @@
;; (= job "export-backup")
;; (backup/-main)
(= job "ntg")
(job-ntg/-main)
:else
(do
(add-shutdown-hook! shutdown-mount)

13
terraform/deploy.tf
View File

@@ -470,3 +470,16 @@ module "restore_from_backup_job" {
memory = 8192
cpu = 4096
}
module "ntg_job" {
schedule = "rate(6 hours)"
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
job_name = "ntg"
execution_role_arn = var.execution_role_arn
use_schedule = false
memory = 4096
cpu = 1024
}

47
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.3.8",
"serial": 411,
"terraform_version": "1.4.2",
"serial": 416,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -164,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:535",
"timeouts": {
"delete": null
},
@@ -1443,7 +1443,7 @@
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
@@ -1894,6 +1894,45 @@
}
]
},
{
"module": "module.ntg_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ntg_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ntg\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ntg\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ntg\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "1024",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "ntg_prod",
"id": "ntg_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "4096",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"module": "module.plaid_job",
"mode": "managed",

101
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.3.8",
"serial": 406,
"terraform_version": "1.4.2",
"serial": 411,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -164,7 +164,7 @@
],
"tags": {},
"tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415",
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520",
"timeouts": {
"delete": null
},
@@ -669,11 +669,25 @@
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}",
"policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [],
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {},
"tags_all": {},
"versioning": [
@@ -693,7 +707,8 @@
"website_domain": "s3-website-us-east-1.amazonaws.com",
"website_endpoint": "data.prod.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com"
},
"sensitive_attributes": []
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
@@ -725,7 +740,21 @@
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [],
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {},
"tags_all": {},
"versioning": [
@@ -739,6 +768,7 @@
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"data.aws_caller_identity.current"
]
@@ -790,7 +820,21 @@
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [],
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {},
"tags_all": {},
"versioning": [
@@ -1370,6 +1414,45 @@
}
]
},
{
"module": "module.export_backup_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/export_backup_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"export-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"export-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"export-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "4096",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "export_backup_prod",
"id": "export_backup_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{
"module": "module.ezcater_upsert_job",
"mode": "managed",
@@ -1801,7 +1884,7 @@
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags": {},
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []