notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Adds outline for NTG invoice importing

Browse Source
This commit is contained in:
Bryce Covert
2023-02-23 21:27:33 -08:00
parent 3a27e6b4e6
commit 590265f4a9
1 changed files with 117 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
src/clj/auto_ap/jobs/ntg.clj Normal file
View File

@@ -0,0 +1,117 @@
(ns auto-ap.jobs.ntg
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.parse :as parse]
[auto-ap.time :as t]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]
[auto-ap.logging :as log :refer [capture-context->lc with-context-as]]
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[datomic.api :as d]
[com.brunobonacci.mulog :as mu]
[auto-ap.time :as atime])
(:import
(java.util UUID)))
(def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env))
(defn read-csv [k]
(log/info ::reading-csv :key k)
(-> (s3/get-object {:bucket-name bucket-name
:key k})
:input-stream
io/reader
csv/read-csv))
(defn extract-invoice-details [csv-rows clients]
(clojure.pprint/pprint (take 4 csv-rows))
(->> csv-rows
(drop 1)
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
(= "Y" break-flag)))
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
(let [[matching-client similarity] (and location-hint
(parse/best-match clients location-hint 0.0))]
(clojure.pprint/pprint {:invoice/vendor vendor
:invoice/location (parse/best-location-match matching-client location-hint location-hint )
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble invoice-total)
:invoice/outstanding-balance (Double/parseDouble invoice-total)
:invoice/client matching-client
:invoice/import-status :import-status/completed
:invoice/status :invoice-status/unpaid
:invoice/client-identifier location-hint
}))))
(filter :invoice/client)))
(defn mark-key [k]
(s3/copy-object {:source-bucket-name bucket-name
:destination-bucket-name bucket-name
:destination-key (str/replace-first k "pending" "imported")
:source-key k})
#_(s3/delete-object {:bucket-name bucket-name
:key k}))
(defn is-csv-file? [x]
(= "csv" (last (str/split x #"[\\.]"))))
(defn import-ntg-invoices []
(let [clients (d-clients/get-all)
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "ntg-invoices/pending"})
:object-summaries
(map :key))]
(log/info ::found-invoice-keys
:keys keys )
(let [transaction (->> keys
(filter is-csv-file?)
(mapcat (fn [k]
(try
(log/info ::trying-csv :key k)
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
(s3/copy-object {:source-bucket-name bucket-name
:destination-bucket-name bucket-name
:source-key k
:destination-key invoice-key})
(->> (extract-invoice-details (read-csv k) clients)
(map (fn [i]
[:propose-invoice (assoc i :invoice/source-url invoice-url)]))
))
(catch Exception e
(log/error ::cant-load-file
:key k
:exception e)
(log/info
(s3/copy-object {:source-bucket-name bucket-name
:destination-bucket-name bucket-name
:source-key k
:destination-key (str "ntg-invoices/error/"
(.getName (io/file k)))}))
[])))))
#_result #_(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]
(clojure.pprint/pprint transaction)
#_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
(doseq [k keys]
(mark-key k))))
(defn -main [& _]
(execute "ntg-invoices" import-ntg-invoices))