notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fixing logging.

Browse Source
This commit is contained in:
Bryce Covert
2023-03-28 13:10:12 -07:00
parent b71095e44b
commit 8d12257181
1 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/clj/auto_ap/backup.clj
View File

@@ -439,6 +439,9 @@
(def in-flight (atom 0)) (def in-flight (atom 0))
(def so-far (atom 0))
(def total (atom 0))
(defn dump-all (defn dump-all
([] (dump-all nil)) ([] (dump-all nil))
([item-list] ([item-list]
@@ -453,6 +456,7 @@
(mu/log ::schema-dumped) (mu/log ::schema-dumped)
(doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies))) (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
:let [_ (swap! dumped conj entity) :let [_ (swap! dumped conj entity)
_ (reset! so-far 0)
_ (mu/log ::querying :entity entity) _ (mu/log ::querying :entity entity)
entities (d/q '[:find [?e ...] entities (d/q '[:find [?e ...]
:in $ [?a ...] :in $ [?a ...]
@@ -461,28 +465,29 @@
(cond-> (entity->best-key entity) (cond-> (entity->best-key entity)
(not (vector? (entity->best-key entity))) vector)) (not (vector? (entity->best-key entity))) vector))
_ (reset! total (count entities))
_ (mu/log ::entity-total-found :count (count entities) :entity entity)]] _ (mu/log ::entity-total-found :count (count entities) :entity entity)]]
(mu/trace ::single-entity (mu/trace ::single-entity
[:entity entity] [:entity entity]
(mu/with-context {:entity entity} (mu/with-context {:entity entity :total @total}
(mu/log ::starting) (mu/log ::starting)
(mu/log ::deleting) (mu/log ::deleting)
(io/delete-file (io/file "/tmp/tmp-ednl") true) (io/delete-file (io/file "/tmp/tmp-ednl") true)
(mu/log ::pulling) (mu/log ::pulling)
(ednl/with-append [append "/tmp/tmp-ednl" ] (ednl/with-append [append "/tmp/tmp-ednl" ]
@(s/consume (fn [batch] @(s/consume (fn [batch]
(mu/with-context {:entity entity} (mu/with-context {:entity entity :total @total}
(doseq [a batch] (doseq [a batch]
(mu/log ::appending :count (count batch))
(try (try
(append a) (append a)
(catch Exception e (catch Exception e
(mu/log ::error (mu/log ::error
:exception e) :exception e)
(throw e))) (throw e)))
(mu/log ::appended :count (count batch))))
) )
(swap! so-far #(+ % (count batch)))
(mu/log ::appended :count (count batch)
:so-far @so-far)))
(->> (partition-all 100 entities) (->> (partition-all 100 entities)
(into []) (into [])
(s/->source) (s/->source)