notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merged.

Browse Source
This commit is contained in:
Bryce Covert
2023-04-27 11:33:54 -07:00
parent 94ae76aa3a 8ebc158081
commit aca8f7c92c
7 changed files with 246 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docker-compose.dev.yml
View File

@@ -18,7 +18,11 @@ services:
restart: "always" restart: "always"
environment: environment:
- DATOMIC_PORT=4334 - DATOMIC_PORT=4334
- XMS=-Xms4g - XMS=-Xms3g
- XMX=-Xmx8g - XMX=-Xmx6g
ulimits:
nofile:
soft: 65536
hard: 65536
ports: ports:
- 4334:4334 - 4334:4334

8
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -65,15 +65,17 @@
({:bank-account-type/check :account-type/asset ({:bank-account-type/check :account-type/asset
:bank-account-type/cash :account-type/asset :bank-account-type/cash :account-type/asset
:bank-account-type/credit :account-type/liability} :bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (:account li)))))] (:db/ident (:bank-account/type (:journal-entry-line/account li)))))]
[(-> j :journal-entry/client :client/code) [(-> j :journal-entry/client :client/code)
(-> j :journal-entry/vendor :vendor/name) (-> j :journal-entry/vendor :vendor/name)
(atime/unparse (coerce/to-date-time (-> j :journal-entry/date)) (atime/unparse (coerce/to-date-time (-> j :journal-entry/date))
atime/normal-date) atime/normal-date)
(-> j :db/id) (-> j :db/id)
(-> li :db/id) (-> li :db/id)
(-> li :journal-entry-line/account :account/numeric-code) (or (-> li :journal-entry-line/account :account/numeric-code)
(-> li :journal-entry-line/account :account/name) (-> li :journal-entry-line/account :bank-account/numeric-code))
(or (-> li :journal-entry-line/account :account/name)
(-> li :journal-entry-line/account :bank-account/name))
(some-> account-type name ) (some-> account-type name )
(-> li :journal-entry-line/debit) (-> li :journal-entry-line/debit)
(-> li :journal-entry-line/credit) (-> li :journal-entry-line/credit)

156
src/clj/auto_ap/jobs/ntg.clj
View File

@@ -43,14 +43,16 @@
:key k})) :key k}))
(defn is-csv-file? [x] (defn is-csv-file? [x]
(= "csv" (last (str/split x #"[\\.]")))) (= "dat" (last (str/split x #"[\\.]"))))
(defn decipher-source [k] (defn decipher-source [k]
(cond (cond
(str/includes? k "Cintas") (and (str/includes? k "Cintas")
(str/includes? k "zcic"))
:cintas :cintas
(and (str/includes? k "GeneralProduce") (and (str/includes? k "GeneralProduce")
(str/includes? k "FRANCHISEE")
(is-csv-file? k)) (is-csv-file? k))
:general-produce :general-produce
@@ -63,25 +65,34 @@
(defmethod extract-invoice-details :general-produce (defmethod extract-invoice-details :general-produce
[k input-stream clients] [k input-stream clients]
(log/info ::parsing-general-produce :key k) (log/info ::parsing-general-produce :key k)
(->> (read-csv input-stream) (try
(drop 1) (->> (read-csv input-stream)
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]] (drop 1)
(= "Y" break-flag))) (filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]] (= "Y" break-flag)))
(let [matching-client (and location-hint (map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
(parse/exact-match clients location-hint))] (let [matching-client (and location-hint
{:invoice/vendor vendor (parse/exact-match clients location-hint))]
:invoice/location (parse/best-location-match matching-client location-hint location-hint ) (when-not matching-client
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date)) (log/warn ::missing-client
:invoice/invoice-number invoice-number :client-hint location-hint))
:invoice/total (Double/parseDouble invoice-total) {:invoice/location (parse/best-location-match matching-client location-hint location-hint )
:invoice/outstanding-balance (Double/parseDouble invoice-total) :invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
:invoice/client matching-client :invoice/invoice-number invoice-number
:invoice/import-status :import-status/imported :invoice/total (Double/parseDouble invoice-total)
:invoice/status :invoice-status/unpaid :invoice/vendor :vendor/general-produce
:invoice/client-identifier location-hint :invoice/outstanding-balance (Double/parseDouble invoice-total)
}))) :invoice/client matching-client
(filter :invoice/client))) :invoice/import-status :import-status/imported
:invoice/status :invoice-status/unpaid
:invoice/client-identifier location-hint
})))
(filter :invoice/client)
(into []))
(catch Exception e
(log/error ::cant-import-general-produce
:error e)
[])))
(defmethod extract-invoice-details :unknown (defmethod extract-invoice-details :unknown
[k input-stream clients] [k input-stream clients]
@@ -115,9 +126,7 @@
first first
:content :content
first) first)
matching-client (and location-hint (parse/exact-match clients location-hint)) matching-client (and location-hint (parse/exact-match clients location-hint))]
]
(if matching-client (if matching-client
(let [invoice-date (->> node-seq (let [invoice-date (->> node-seq
(map zip/node) (map zip/node)
@@ -182,7 +191,7 @@
:invoice invoice) :invoice invoice)
[invoice]) [invoice])
(do (do
(log/info ::missing-client (log/warn ::missing-client
:client-hint location-hint) :client-hint location-hint)
[])))) []))))
@@ -204,55 +213,56 @@
:destination-key invoice-key }) :destination-key invoice-key })
invoice-key)) invoice-key))
(defn import-ntg-invoices [] (defn import-ntg-invoices
(let [clients (map first (d/q '[:find (pull ?c [:client/code ([] (import-ntg-invoices (->> (s3/list-objects-v2 {:bucket-name bucket-name
:db/id :prefix "ntg-invoices/pending"})
:client/feature-flags :object-summaries
{:client/location-matches [:location-match/matches :location-match/location]} (map :key))))
:client/name ([keys]
:client/matches (let [clients (map first (d/q '[:find (pull ?c [:client/code
:client/locations]) :db/id
:where [?c :client/code]] :client/feature-flags
(d/db conn))) {:client/location-matches [:location-match/matches :location-match/location]}
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name :client/name
:prefix "ntg-invoices/pending"}) :client/matches
:object-summaries :client/locations])
(map :key))] :where [?c :client/code]]
(d/db conn)))]
(log/info ::found-invoice-keys (log/info ::found-invoice-keys
:keys keys ) :keys keys )
(let [transaction (->> keys (let [transaction (->> keys
(mapcat (fn [k] (mapcat (fn [k]
(try (try
(let [invoice-key (copy-readable-version k) (let [invoice-key (copy-readable-version k)
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)] invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
(with-open [is (-> (s3/get-object {:bucket-name bucket-name (with-open [is (-> (s3/get-object {:bucket-name bucket-name
:key k}) :key k})
:input-stream)] :input-stream)]
(->> (extract-invoice-details k (->> (extract-invoice-details k
is is
clients) clients)
(map (fn [i] (map (fn [i]
(log/info ::importing-invoice (log/info ::importing-invoice
:invoice i) :invoice i)
i)) i))
(mapv (fn [i] (mapv (fn [i]
`(propose-invoice ~(assoc i :invoice/source-url invoice-url))))))) (catch Exception e `(propose-invoice ~(assoc i :invoice/source-url invoice-url))))))) (catch Exception e
(log/error ::cant-load-file (log/error ::cant-load-file
:key k :key k
:exception e) :exception e)
(mark-error k) (mark-error k)
[])))) []))))
(into []))] (into []))]
(audit-transact transaction {:user/name "sysco importer" :user/role "admin"}) (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})
(log/info ::success (log/info ::success
:count (count transaction) :count (count transaction)
:sample (take 3 transaction))) :sample (take 3 transaction)))
(doseq [k keys] (doseq [k keys]
(mark-key k)))) (mark-key k)))))
(defn -main [& _] (defn -main [& _]
(execute "ntg-invoices" import-ntg-invoices)) (execute "ntg" import-ntg-invoices))

4
src/clj/auto_ap/server.clj
View File

@@ -9,6 +9,7 @@
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
[auto-ap.jobs.intuit :as job-intuit] [auto-ap.jobs.intuit :as job-intuit]
[auto-ap.jobs.ntg :as job-ntg]
#_[auto-ap.backup :as backup] #_[auto-ap.backup :as backup]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales] [auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
@@ -145,6 +146,9 @@
;; (= job "export-backup") ;; (= job "export-backup")
;; (backup/-main) ;; (backup/-main)
(= job "ntg")
(job-ntg/-main)
:else :else
(do (do
(add-shutdown-hook! shutdown-mount) (add-shutdown-hook! shutdown-mount)

13
terraform/deploy.tf
View File

@@ -470,3 +470,16 @@ module "restore_from_backup_job" {
memory = 8192 memory = 8192
cpu = 4096 cpu = 4096
} }
module "ntg_job" {
schedule = "rate(6 hours)"
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
job_name = "ntg"
execution_role_arn = var.execution_role_arn
use_schedule = false
memory = 4096
cpu = 1024
}

47
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.3.8", "terraform_version": "1.4.2",
"serial": 411, "serial": 416,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": { "outputs": {
"aws_access_key_id": { "aws_access_key_id": {
@@ -164,7 +164,7 @@
], ],
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520", "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:535",
"timeouts": { "timeouts": {
"delete": null "delete": null
}, },
@@ -1443,7 +1443,7 @@
], ],
"revision": 1, "revision": 1,
"runtime_platform": [], "runtime_platform": [],
"tags": null, "tags": {},
"tags_all": {}, "tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": [] "volume": []
@@ -1894,6 +1894,45 @@
} }
] ]
}, },
{
"module": "module.ntg_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/ntg_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"ntg\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"ntg\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"ntg\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "1024",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "ntg_prod",
"id": "ntg_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "4096",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{ {
"module": "module.plaid_job", "module": "module.plaid_job",
"mode": "managed", "mode": "managed",

101
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.3.8", "terraform_version": "1.4.2",
"serial": 406, "serial": 411,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": { "outputs": {
"aws_access_key_id": { "aws_access_key_id": {
@@ -164,7 +164,7 @@
], ],
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415", "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520",
"timeouts": { "timeouts": {
"delete": null "delete": null
}, },
@@ -669,11 +669,25 @@
"lifecycle_rule": [], "lifecycle_rule": [],
"logging": [], "logging": [],
"object_lock_configuration": [], "object_lock_configuration": [],
"policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}", "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}",
"region": "us-east-1", "region": "us-east-1",
"replication_configuration": [], "replication_configuration": [],
"request_payer": "BucketOwner", "request_payer": "BucketOwner",
"server_side_encryption_configuration": [], "server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"versioning": [ "versioning": [
@@ -693,7 +707,8 @@
"website_domain": "s3-website-us-east-1.amazonaws.com", "website_domain": "s3-website-us-east-1.amazonaws.com",
"website_endpoint": "data.prod.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" "website_endpoint": "data.prod.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com"
}, },
"sensitive_attributes": [] "sensitive_attributes": [],
"private": "bnVsbA=="
} }
] ]
}, },
@@ -725,7 +740,21 @@
"region": "us-east-1", "region": "us-east-1",
"replication_configuration": [], "replication_configuration": [],
"request_payer": "BucketOwner", "request_payer": "BucketOwner",
"server_side_encryption_configuration": [], "server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"versioning": [ "versioning": [
@@ -739,6 +768,7 @@
"website_endpoint": null "website_endpoint": null
}, },
"sensitive_attributes": [], "sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [ "dependencies": [
"data.aws_caller_identity.current" "data.aws_caller_identity.current"
] ]
@@ -790,7 +820,21 @@
"region": "us-east-1", "region": "us-east-1",
"replication_configuration": [], "replication_configuration": [],
"request_payer": "BucketOwner", "request_payer": "BucketOwner",
"server_side_encryption_configuration": [], "server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {}, "tags": {},
"tags_all": {}, "tags_all": {},
"versioning": [ "versioning": [
@@ -1370,6 +1414,45 @@
} }
] ]
}, },
{
"module": "module.export_backup_job",
"mode": "managed",
"type": "aws_ecs_task_definition",
"name": "background_taskdef",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/export_backup_prod:1",
"container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"export-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"export-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"export-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
"cpu": "4096",
"ephemeral_storage": [],
"execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
"family": "export_backup_prod",
"id": "export_backup_prod",
"inference_accelerator": [],
"ipc_mode": "",
"memory": "8192",
"network_mode": "awsvpc",
"pid_mode": "",
"placement_constraints": [],
"proxy_configuration": [],
"requires_compatibilities": [
"FARGATE"
],
"revision": 1,
"runtime_platform": [],
"tags": null,
"tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": []
},
"sensitive_attributes": [],
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
}
]
},
{ {
"module": "module.ezcater_upsert_job", "module": "module.ezcater_upsert_job",
"mode": "managed", "mode": "managed",
@@ -1801,7 +1884,7 @@
], ],
"revision": 1, "revision": 1,
"runtime_platform": [], "runtime_platform": [],
"tags": null, "tags": {},
"tags_all": {}, "tags_all": {},
"task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
"volume": [] "volume": []