Adds outline for NTG invoice importing

2023-02-23 21:27:33 -08:00
parent 3a27e6b4e6
commit 590265f4a9
1 changed files with 117 additions and 0 deletions
--- a/src/clj/auto_ap/jobs/ntg.clj
+++ b/src/clj/auto_ap/jobs/ntg.clj
@@ -0,0 +1,117 @@
+(ns auto-ap.jobs.ntg
+  (:require
+   [amazonica.aws.s3 :as s3]
+   [auto-ap.datomic :refer [conn]]
+   [auto-ap.datomic.clients :as d-clients]
+   [auto-ap.datomic.invoices :refer [code-invoice]]
+   [auto-ap.jobs.core :refer [execute]]
+   [auto-ap.ledger :refer [transact-with-ledger]]
+   [auto-ap.parse :as parse]
+   [auto-ap.time :as t]
+   [clj-time.coerce :as coerce]
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [auto-ap.logging :as log :refer [capture-context->lc with-context-as]]
+   [com.unbounce.dogstatsd.core :as statsd]
+   [config.core :refer [env]]
+   [datomic.api :as d]
+   [com.brunobonacci.mulog :as mu]
+
+   [auto-ap.time :as atime])
+  (:import
+   (java.util UUID)))
+
+
+(def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env))
+
+(defn read-csv [k]
+  (log/info ::reading-csv :key k)
+  (-> (s3/get-object {:bucket-name bucket-name
+                      :key         k})
+      :input-stream
+      io/reader
+      csv/read-csv))
+
+
+(defn extract-invoice-details [csv-rows clients]
+  (clojure.pprint/pprint (take 4 csv-rows))
+  (->> csv-rows
+       (drop 1)
+       (filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
+                 
+                 (= "Y" break-flag)))
+       (map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
+              
+              (let [[matching-client similarity] (and location-hint
+                                                      (parse/best-match clients location-hint 0.0))]
+                (clojure.pprint/pprint {:invoice/vendor              vendor
+                                        :invoice/location            (parse/best-location-match matching-client location-hint location-hint )
+                                        :invoice/date                (coerce/to-date (atime/parse ship-date atime/normal-date))
+                                        :invoice/invoice-number      invoice-number
+                                        :invoice/total               (Double/parseDouble invoice-total)
+                                        :invoice/outstanding-balance (Double/parseDouble invoice-total)
+                                        :invoice/client              matching-client
+                                        :invoice/import-status       :import-status/completed
+                                        :invoice/status              :invoice-status/unpaid
+                                        :invoice/client-identifier   location-hint
+                                        }))))
+       (filter :invoice/client)))
+
+(defn mark-key [k]
+  (s3/copy-object {:source-bucket-name      bucket-name
+                   :destination-bucket-name bucket-name
+                   :destination-key         (str/replace-first k "pending" "imported")
+                   :source-key              k})
+  #_(s3/delete-object {:bucket-name bucket-name
+                     :key         k}))
+
+(defn is-csv-file? [x]
+  (= "csv" (last (str/split x #"[\\.]"))))
+
+(defn import-ntg-invoices []
+  (let [clients      (d-clients/get-all)
+        keys         (->> (s3/list-objects-v2 {:bucket-name bucket-name 
+                                               :prefix      "ntg-invoices/pending"})
+                          :object-summaries
+                          (map :key))]
+    
+
+    (log/info ::found-invoice-keys
+              :keys keys )
+    
+    (let [transaction (->> keys
+                           (filter is-csv-file?)
+                           (mapcat (fn [k]
+                                     (try 
+                                       (log/info ::trying-csv :key k)
+                                       (let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
+                                             invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
+                                         (s3/copy-object {:source-bucket-name      bucket-name
+                                                          :destination-bucket-name bucket-name
+                                                          :source-key              k
+                                                          :destination-key         invoice-key})
+                                         (->> (extract-invoice-details (read-csv k) clients)
+                                             (map (fn [i]
+                                                    [:propose-invoice (assoc i :invoice/source-url invoice-url)]))
+                                             ))
+                                       (catch Exception e
+                                         (log/error ::cant-load-file
+                                                    :key k
+                                                    :exception e)
+                                         (log/info 
+                                           (s3/copy-object {:source-bucket-name      bucket-name 
+                                                            :destination-bucket-name bucket-name
+                                                            :source-key              k
+                                                            :destination-key         (str "ntg-invoices/error/"
+                                                                                          (.getName (io/file k)))}))
+                                         [])))))
+          #_result      #_(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]
+      (clojure.pprint/pprint transaction)
+      #_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
+    (doseq [k keys]
+      (mark-key k))))
+
+
+(defn -main [& _]
+  (execute "ntg-invoices" import-ntg-invoices))