fixes invoice import issues, fixes issue with too many keys in s3 for general produce.

src/clj/auto_ap/jobs/ntg.clj

@@ -225,10 +225,27 @@
                      :destination-key         invoice-key })
     invoice-key))
 
+(defn get-all-keys
+  ([]
+   (let [first-page-result (s3/list-objects-v2 {:bucket-name bucket-name 
+                                                :prefix      "ntg-invoices/pending"})]
+     (lazy-seq (concat (:object-summaries first-page-result) (get-all-keys (:next-continuation-token first-page-result))))))
+  ([next-token ]
+   (when next-token 
+     (let [page-result (s3/list-objects-v2 {:bucket-name        bucket-name 
+                                            :prefix             "ntg-invoices/pending"
+                                            :continuation-token next-token})]
+       (println "getting next page " next-token)
+       (when (seq (:object-summaries page-result))
+         (lazy-seq (concat (:object-summaries page-result) (get-all-keys (:next-continuation-token page-result)))))))))
+
+(defn recent? [k]
+  (time/after? (:last-modified k) (time/plus (time/now) (time/days -15)))
+  )
+
 (defn import-ntg-invoices
-  ([] (import-ntg-invoices (->> (s3/list-objects-v2 {:bucket-name bucket-name 
-                                                     :prefix      "ntg-invoices/pending"})
-                                :object-summaries
+  ([] (import-ntg-invoices (->> (get-all-keys)
+                                (filter recent?)
                                 (map :key))))
   ([keys]
                            (let [clients (map first (d/q '[:find (pull ?c [:client/code

src/clj/auto_ap/parse/csv.clj

@@ -27,6 +27,9 @@
                        (str/includes? (str header) "DISCOUNT_MESSAGE")
                        :wismettac
 
+                       (str/includes? (str header) "Status")
+                       :ledyard
+                       
                        :else
                        nil)]
     (log/info "csv type was determined to be" csv-type)
@@ -162,6 +165,23 @@
    []
    (drop 1 rows)))
 
+(defmethod parse-csv :ledyard
+  [rows]
+  (transduce
+    (comp 
+      (map (fn [[invoice-number date due amount standard :as row]]
+             {:vendor-code    "Performance Food Group - LEDYARD"
+              :invoice-number invoice-number
+              :date           (some-> date not-empty (parse-date-fallover ["MM/dd/yyyy"]))
+              :due (some-> due not-empty (parse-date-fallover ["MM/dd/yyyy"]))
+              :total          (str/replace amount #"[\$,]" "")
+              :text           (str/join " " row)
+              :full-text      (str/join " " row)}))
+      )
+    conj
+    []
+    (drop 1 rows)))
+
 #_{:clj-kondo/ignore [:unused-binding]}
 (defmethod parse-csv nil
   [rows]

src/clj/auto_ap/parse/templates.clj

@@ -555,6 +555,7 @@
    {:vendor "JFC International"
     :keywords [#"48490 MILMONT DRIVE"]
     :extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
+              :account-number #"CUSTOMER #.*?\n\n\s+(.*?)\s{2,}"
               :customer-identifier #"SOLD\s+(?:TO\s+)?([\S ]+?)(?=(\s{2,}|\n))"
               :invoice-number #"(\S+)\s+(?:[0-9]+/[0-9]+/[0-9]+)"
               :total #"(?:INVOICE|TOTAL|CREDIT)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"}