diff --git a/src/clj/auto_ap/jobs/ntg.clj b/src/clj/auto_ap/jobs/ntg.clj index b13e507d..340eff66 100644 --- a/src/clj/auto_ap/jobs/ntg.clj +++ b/src/clj/auto_ap/jobs/ntg.clj @@ -43,14 +43,16 @@ :key k})) (defn is-csv-file? [x] - (= "csv" (last (str/split x #"[\\.]")))) + (= "dat" (last (str/split x #"[\\.]")))) (defn decipher-source [k] (cond - (str/includes? k "Cintas") + (and (str/includes? k "Cintas") + (str/includes? k "zcic")) :cintas (and (str/includes? k "GeneralProduce") + (str/includes? k "FRANCHISEE") (is-csv-file? k)) :general-produce @@ -70,6 +72,9 @@ (map (fn [[vendor location-hint invoice-number ship-date invoice-total ]] (let [matching-client (and location-hint (parse/exact-match clients location-hint))] + (when-not matching-client + (log/warn ::missing-client + :client-hint location-hint)) {:invoice/vendor vendor :invoice/location (parse/best-location-match matching-client location-hint location-hint ) :invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date)) @@ -115,9 +120,7 @@ first :content first) - matching-client (and location-hint (parse/exact-match clients location-hint)) - - ] + matching-client (and location-hint (parse/exact-match clients location-hint))] (if matching-client (let [invoice-date (->> node-seq (map zip/node) @@ -180,7 +183,7 @@ :invoice invoice) [invoice]) (do - (log/info ::missing-client + (log/warn ::missing-client :client-hint location-hint) [])))) @@ -202,58 +205,56 @@ :destination-key invoice-key }) invoice-key)) -(defn import-ntg-invoices [] - (let [clients (map first (d/q '[:find (pull ?c [:client/code - :db/id - :client/feature-flags - {:client/location-matches [:location-match/matches :location-match/location]} - :client/name - :client/matches - :client/locations]) - :where [?c :client/code]] - (d/db conn))) - keys (->> (s3/list-objects-v2 {:bucket-name bucket-name - :prefix "ntg-invoices/pending"}) - :object-summaries - (map :key))] - - - (log/info ::found-invoice-keys - :keys keys ) - - (let [transaction (->> keys - (mapcat (fn [k] - (try - (let [invoice-key (copy-readable-version k) - invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)] - (with-open [is (-> (s3/get-object {:bucket-name bucket-name - :key k}) - :input-stream)] - (doto - (->> (extract-invoice-details k - is - clients) - (map (fn [i] - (log/info ::importing-invoice - :invoice i) - i)) - (mapv (fn [i] - [:propose-invoice (assoc i :invoice/source-url invoice-url)]))) - println))) - (catch Exception e - (log/error ::cant-load-file - :key k - :exception e) - (mark-error k) - [])))) - (into []))] - (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"}) - (log/info ::success - :count (count transaction) - :sample (take 3 transaction))) - (doseq [k keys] - (mark-key k)))) +(defn import-ntg-invoices + ([] (import-ntg-invoices (->> (s3/list-objects-v2 {:bucket-name bucket-name + :prefix "ntg-invoices/pending"}) + :object-summaries + (map :key)))) + ([keys ] + (let [clients (map first (d/q '[:find (pull ?c [:client/code + :db/id + :client/feature-flags + {:client/location-matches [:location-match/matches :location-match/location]} + :client/name + :client/matches + :client/locations]) + :where [?c :client/code]] + (d/db conn)))] + (log/info ::found-invoice-keys + :keys keys ) + (let [transaction (->> keys + (mapcat (fn [k] + (try + (let [invoice-key (copy-readable-version k) + invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)] + (with-open [is (-> (s3/get-object {:bucket-name bucket-name + :key k}) + :input-stream)] + (doto + (->> (extract-invoice-details k + is + clients) + (map (fn [i] + (log/info ::importing-invoice + :invoice i) + i)) + (mapv (fn [i] + [:propose-invoice (assoc i :invoice/source-url invoice-url)]))) + println))) + (catch Exception e + (log/error ::cant-load-file + :key k + :exception e) + (mark-error k) + [])))) + (into []))] + (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"}) + (log/info ::success + :count (count transaction) + :sample (take 3 transaction))) + (doseq [k keys] + (mark-key k))))) (defn -main [& _] - (execute "ntg-invoices" import-ntg-invoices)) + (execute "ntg" import-ntg-invoices)) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index b549bf2e..42b0b3bb 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -8,6 +8,7 @@ [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.intuit :as job-intuit] + [auto-ap.jobs.ntg :as job-ntg] [auto-ap.backup :as backup] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] [auto-ap.jobs.load-historical-sales :as job-load-historical-sales] @@ -141,6 +142,9 @@ (= job "export-backup") (backup/-main) + (= job "ntg") + (job-ntg/-main) + :else (do diff --git a/terraform/deploy.tf b/terraform/deploy.tf index 8f5dda8b..70bd996e 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -481,3 +481,15 @@ module "export_backup_job" { memory = 8192 cpu = 4096 } + +module "ntg_job" { + source = "./background-job/" + ecs_cluster = var.ecs_cluster + task_role_arn = var.task_role_arn + stage = var.stage + job_name = "ntg" + execution_role_arn = var.execution_role_arn + use_schedule = false + memory = 4096 + cpu = 1024 +} \ No newline at end of file