From 8d122571817bec68a1d0a306d7957082b11caf7d Mon Sep 17 00:00:00 2001
From: Bryce Covert <bryce@brycecovertoperations.com>
Date: Tue, 28 Mar 2023 13:10:12 -0700
Subject: [PATCH] fixing logging.

---
 src/clj/auto_ap/backup.clj | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/clj/auto_ap/backup.clj b/src/clj/auto_ap/backup.clj
index 88bce087..b86e616b 100644
--- a/src/clj/auto_ap/backup.clj
+++ b/src/clj/auto_ap/backup.clj
@@ -439,6 +439,9 @@
 
 (def in-flight (atom 0))
 
+(def so-far (atom 0))
+(def total (atom 0))
+
 (defn dump-all
   ([] (dump-all nil))
   ([item-list]
@@ -453,6 +456,7 @@
      (mu/log ::schema-dumped)
      (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
              :let   [_ (swap! dumped conj entity)
+                     _ (reset! so-far 0)
                      _ (mu/log ::querying :entity entity)
                      entities (d/q '[:find [?e ...]
                                      :in $ [?a ...]
@@ -461,28 +465,29 @@
                                    (cond-> (entity->best-key entity)
                                      (not (vector? (entity->best-key entity))) vector))
 
+                     _ (reset! total (count entities))
                      _ (mu/log ::entity-total-found :count (count entities) :entity entity)]]
        (mu/trace ::single-entity
          [:entity entity]
-         (mu/with-context {:entity entity}
+         (mu/with-context {:entity entity :total @total}
            (mu/log ::starting)
            (mu/log ::deleting)
            (io/delete-file (io/file "/tmp/tmp-ednl") true)
            (mu/log ::pulling)
            (ednl/with-append [append "/tmp/tmp-ednl" ]
              @(s/consume (fn [batch]
-                           (mu/with-context {:entity entity}
+                           (mu/with-context {:entity entity :total @total}
                              (doseq [a batch]
-                               (mu/log ::appending :count (count batch))
                                (try 
                                  (append a) 
                                  (catch Exception e
                                    (mu/log ::error
                                            :exception e)
                                    (throw e)))
-                               (mu/log ::appended :count (count batch))))
-                           
-                           )
+                               )
+                             (swap! so-far #(+ % (count batch)))
+                             (mu/log ::appended :count (count batch)
+                                     :so-far @so-far)))
                          (->> (partition-all 100 entities)
                               (into [])
                               (s/->source)