diff --git a/src/clj/auto_ap/jobs/ntg.clj b/src/clj/auto_ap/jobs/ntg.clj new file mode 100644 index 00000000..a9e65dab --- /dev/null +++ b/src/clj/auto_ap/jobs/ntg.clj @@ -0,0 +1,117 @@ +(ns auto-ap.jobs.ntg + (:require + [amazonica.aws.s3 :as s3] + [auto-ap.datomic :refer [conn]] + [auto-ap.datomic.clients :as d-clients] + [auto-ap.datomic.invoices :refer [code-invoice]] + [auto-ap.jobs.core :refer [execute]] + [auto-ap.ledger :refer [transact-with-ledger]] + [auto-ap.parse :as parse] + [auto-ap.time :as t] + [clj-time.coerce :as coerce] + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [clojure.string :as str] + [auto-ap.logging :as log :refer [capture-context->lc with-context-as]] + [com.unbounce.dogstatsd.core :as statsd] + [config.core :refer [env]] + [datomic.api :as d] + [com.brunobonacci.mulog :as mu] + + [auto-ap.time :as atime]) + (:import + (java.util UUID))) + + +(def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env)) + +(defn read-csv [k] + (log/info ::reading-csv :key k) + (-> (s3/get-object {:bucket-name bucket-name + :key k}) + :input-stream + io/reader + csv/read-csv)) + + +(defn extract-invoice-details [csv-rows clients] + (clojure.pprint/pprint (take 4 csv-rows)) + (->> csv-rows + (drop 1) + (filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]] + + (= "Y" break-flag))) + (map (fn [[vendor location-hint invoice-number ship-date invoice-total ]] + + (let [[matching-client similarity] (and location-hint + (parse/best-match clients location-hint 0.0))] + (clojure.pprint/pprint {:invoice/vendor vendor + :invoice/location (parse/best-location-match matching-client location-hint location-hint ) + :invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date)) + :invoice/invoice-number invoice-number + :invoice/total (Double/parseDouble invoice-total) + :invoice/outstanding-balance (Double/parseDouble invoice-total) + :invoice/client matching-client + :invoice/import-status :import-status/completed + :invoice/status :invoice-status/unpaid + :invoice/client-identifier location-hint + })))) + (filter :invoice/client))) + +(defn mark-key [k] + (s3/copy-object {:source-bucket-name bucket-name + :destination-bucket-name bucket-name + :destination-key (str/replace-first k "pending" "imported") + :source-key k}) + #_(s3/delete-object {:bucket-name bucket-name + :key k})) + +(defn is-csv-file? [x] + (= "csv" (last (str/split x #"[\\.]")))) + +(defn import-ntg-invoices [] + (let [clients (d-clients/get-all) + keys (->> (s3/list-objects-v2 {:bucket-name bucket-name + :prefix "ntg-invoices/pending"}) + :object-summaries + (map :key))] + + + (log/info ::found-invoice-keys + :keys keys ) + + (let [transaction (->> keys + (filter is-csv-file?) + (mapcat (fn [k] + (try + (log/info ::trying-csv :key k) + (let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ; + invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)] + (s3/copy-object {:source-bucket-name bucket-name + :destination-bucket-name bucket-name + :source-key k + :destination-key invoice-key}) + (->> (extract-invoice-details (read-csv k) clients) + (map (fn [i] + [:propose-invoice (assoc i :invoice/source-url invoice-url)])) + )) + (catch Exception e + (log/error ::cant-load-file + :key k + :exception e) + (log/info + (s3/copy-object {:source-bucket-name bucket-name + :destination-bucket-name bucket-name + :source-key k + :destination-key (str "ntg-invoices/error/" + (.getName (io/file k)))})) + []))))) + #_result #_(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})] + (clojure.pprint/pprint transaction) + #_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2))) + (doseq [k keys] + (mark-key k)))) + + +(defn -main [& _] + (execute "ntg-invoices" import-ntg-invoices))