notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

readiness for ntg automatic invoice importation

Browse Source
This commit is contained in:
Bryce Covert
2023-04-26 14:26:49 -07:00
parent 9ab89b2952
commit 19d83f655f
3 changed files with 75 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
src/clj/auto_ap/jobs/ntg.clj
View File

@@ -43,14 +43,16 @@
:key k})) :key k}))
(defn is-csv-file? [x] (defn is-csv-file? [x]
(= "csv" (last (str/split x #"[\\.]")))) (= "dat" (last (str/split x #"[\\.]"))))
(defn decipher-source [k] (defn decipher-source [k]
(cond (cond
(str/includes? k "Cintas") (and (str/includes? k "Cintas")
(str/includes? k "zcic"))
:cintas :cintas
(and (str/includes? k "GeneralProduce") (and (str/includes? k "GeneralProduce")
(str/includes? k "FRANCHISEE")
(is-csv-file? k)) (is-csv-file? k))
:general-produce :general-produce
@@ -70,6 +72,9 @@
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]] (map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
(let [matching-client (and location-hint (let [matching-client (and location-hint
(parse/exact-match clients location-hint))] (parse/exact-match clients location-hint))]
(when-not matching-client
(log/warn ::missing-client
:client-hint location-hint))
{:invoice/vendor vendor {:invoice/vendor vendor
:invoice/location (parse/best-location-match matching-client location-hint location-hint ) :invoice/location (parse/best-location-match matching-client location-hint location-hint )
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date)) :invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
@@ -115,9 +120,7 @@
first first
:content :content
first) first)
matching-client (and location-hint (parse/exact-match clients location-hint)) matching-client (and location-hint (parse/exact-match clients location-hint))]
]
(if matching-client (if matching-client
(let [invoice-date (->> node-seq (let [invoice-date (->> node-seq
(map zip/node) (map zip/node)
@@ -180,7 +183,7 @@
:invoice invoice) :invoice invoice)
[invoice]) [invoice])
(do (do
(log/info ::missing-client (log/warn ::missing-client
:client-hint location-hint) :client-hint location-hint)
[])))) []))))
@@ -202,58 +205,56 @@
:destination-key invoice-key }) :destination-key invoice-key })
invoice-key)) invoice-key))
(defn import-ntg-invoices [] (defn import-ntg-invoices
(let [clients (map first (d/q '[:find (pull ?c [:client/code ([] (import-ntg-invoices (->> (s3/list-objects-v2 {:bucket-name bucket-name
:db/id :prefix "ntg-invoices/pending"})
:client/feature-flags :object-summaries
{:client/location-matches [:location-match/matches :location-match/location]} (map :key))))
:client/name ([keys ]
:client/matches (let [clients (map first (d/q '[:find (pull ?c [:client/code
:client/locations]) :db/id
:where [?c :client/code]] :client/feature-flags
(d/db conn))) {:client/location-matches [:location-match/matches :location-match/location]}
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name :client/name
:prefix "ntg-invoices/pending"}) :client/matches
:object-summaries :client/locations])
(map :key))] :where [?c :client/code]]
(d/db conn)))]
(log/info ::found-invoice-keys
(log/info ::found-invoice-keys :keys keys )
:keys keys ) (let [transaction (->> keys
(mapcat (fn [k]
(let [transaction (->> keys (try
(mapcat (fn [k] (let [invoice-key (copy-readable-version k)
(try invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
(let [invoice-key (copy-readable-version k) (with-open [is (-> (s3/get-object {:bucket-name bucket-name
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)] :key k})
(with-open [is (-> (s3/get-object {:bucket-name bucket-name :input-stream)]
:key k}) (doto
:input-stream)] (->> (extract-invoice-details k
(doto is
(->> (extract-invoice-details k clients)
is (map (fn [i]
clients) (log/info ::importing-invoice
(map (fn [i] :invoice i)
(log/info ::importing-invoice i))
:invoice i) (mapv (fn [i]
i)) [:propose-invoice (assoc i :invoice/source-url invoice-url)])))
(mapv (fn [i] println)))
[:propose-invoice (assoc i :invoice/source-url invoice-url)]))) (catch Exception e
println))) (log/error ::cant-load-file
(catch Exception e :key k
(log/error ::cant-load-file :exception e)
:key k (mark-error k)
:exception e) []))))
(mark-error k) (into []))]
[])))) (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})
(into []))] (log/info ::success
(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"}) :count (count transaction)
(log/info ::success :sample (take 3 transaction)))
:count (count transaction) (doseq [k keys]
:sample (take 3 transaction))) (mark-key k)))))
(doseq [k keys]
(mark-key k))))
(defn -main [& _] (defn -main [& _]
(execute "ntg-invoices" import-ntg-invoices)) (execute "ntg" import-ntg-invoices))

4
src/clj/auto_ap/server.clj
View File

@@ -8,6 +8,7 @@
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
[auto-ap.jobs.intuit :as job-intuit] [auto-ap.jobs.intuit :as job-intuit]
[auto-ap.jobs.ntg :as job-ntg]
[auto-ap.backup :as backup] [auto-ap.backup :as backup]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales] [auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
@@ -141,6 +142,9 @@
(= job "export-backup") (= job "export-backup")
(backup/-main) (backup/-main)
(= job "ntg")
(job-ntg/-main)
:else :else
(do (do

12
terraform/deploy.tf
View File

@@ -481,3 +481,15 @@ module "export_backup_job" {
memory = 8192 memory = 8192
cpu = 4096 cpu = 4096
} }
module "ntg_job" {
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
job_name = "ntg"
execution_role_arn = var.execution_role_arn
use_schedule = false
memory = 4096
cpu = 1024
}