Adds the ability to launch a job that restores a database

src/clj/auto_ap/jobs/restore_from_backup.clj

@@ -1,16 +1,23 @@
 (ns auto-ap.jobs.restore-from-backup
-  (:require [clojure.java.io :as io]
-            [amazonica.aws.s3 :as s3]
-            [config.core :refer [env]]
-            [clojure.core.async :as a]
-            [datomic.client.api :as dc]
-            [auto-ap.datomic :refer [client]]
-            [lambdaisland.edn-lines :as ednl]
-            [datomic.client.api.async :as dca]
-            [datomic.dev-local :as dl]
-            [clojure.set :as set]
-            [clojure.string :as str]
-            [clj-http.client :as client]))
+  (:require
+   [amazonica.aws.s3 :as s3]
+   [auto-ap.datomic]
+   [auto-ap.ledger]
+   [auto-ap.jobs.core :refer [execute]]
+   [clojure.edn :as edn]
+   [clojure.java.io :as io]
+   [clojure.set :as set]
+   [com.brunobonacci.mulog :as mu]
+   [config.core :refer [env]]
+   [datomic.client.api :as dc]
+   [lambdaisland.edn-lines :as ednl]
+   [manifold.deferred :as de]
+   [manifold.executor :as ex]
+   [manifold.stream :as s]))
+
+(def request-pool (ex/fixed-thread-executor 100))
+
+(def buffered (ex/fixed-thread-executor 100))
 
 (defn order-of-insert [entity-dependencies]
   (loop [entity-dependencies entity-dependencies 
@@ -27,7 +34,6 @@
                                 entity-dependencies)))
                       (apply dissoc entity-dependencies next-order)
                       next-order)]
-      (println order next-deps)
       (if (seq next-deps)
         (recur next-deps (into order next-order))
         (into order next-order)))))
@@ -35,53 +41,14 @@
 
 
 
-(defn tx-pipeline
-  "Transacts data from from-ch. Returns a map with:
-     :result, a return channel getting {:error t} or {:completed n}
-     :stop, a fn you can use to terminate early."
-  [conn conc from-ch f]
-  (let [to-ch (a/chan 400)
-        done-ch (a/chan)
-        transact-data (fn [data result]
-                        (a/go 
-                          (try
-                            
-                            (let [tx-r (a/<! (dca/transact conn {:tx-data (f data)}))]
-                              (when (:db/error tx-r)
-                                (throw (ex-info "Invalid transaction" tx-r)))
-                              (a/>! result tx-r)
-                              (a/close! result))
-                                        ; if exception in a transaction
-                                        ; will close channels and put error
-                                        ; on done channel.
-                            (catch Throwable t
-                              (.printStackTrace t)
-                              (a/close! from-ch)
-                              (a/close! to-ch)
-                              (a/>! done-ch {:error t})))))]
-
-   ; go block prints a '.' after every 1000 transactions, puts completed
-   ; report on done channel when no value left to be taken.
-   (a/go-loop [total 0]
-     (if (= (mod total 5) 0)
-       (do  
-         (print ".")
-         (flush)))
-     (if-let [c (a/<! to-ch)]
-       (recur (inc total))
-       (a/>! done-ch {:completed total})))
-
-   ; pipeline that uses transducer form of map to transact data taken from
-   ; from-ch and puts results on to-ch
-   (a/pipeline-async conc to-ch transact-data from-ch)
-
-   ; returns done channel and a function that you can use
-   ; for early termination.
-   {:result done-ch
-    :stop (fn [] (a/close! to-ch))}))
-
 (def loaded (atom #{}))
 
+(defn upsert-batch [batch]
+  (de/future-with request-pool 
+                  (do 
+                    (dc/transact auto-ap.datomic/conn {:tx-data batch})
+                    batch)))
+
 
 
 (defn pull-file [backup which]
@@ -90,14 +57,43 @@
                                             :key         (str "/datomic-backup/" backup "/" which)}))
              w))
   "/tmp/tmp-edn")
-  
+
+(def so-far (atom 0))
+(def die? (atom false))
+
+(defn load-entity [entity entities]
+  (mu/trace ::restore-entity
+            [:entity entity]
+            (mu/with-context {:entity entity}
+              (mu/log ::starting)
+              @(s/consume (fn [batch]
+                           (mu/with-context {:entity entity}
+                             (try 
+                               (swap! so-far #(+ % (count batch)))
+                               (mu/log ::loaded :count (count batch)
+                                       :so-far @so-far)
+                               (catch Exception e
+                                 (mu/log ::error
+                                         :exception e)
+                                 (throw e)))))
+                         (->> (partition-all 1000 entities)
+                              (s/->source)
+                              (s/onto buffered)
+                              (s/map (fn [entities]
+                                       (when @die?
+                                         (reset! die? false)
+                                         (throw (Exception. "dead")))
+                                       (upsert-batch entities)))
+                              (s/buffer 50)
+                              (s/realize-each)))
+              (swap! loaded conj entity))))
 
 (defn load-from-backup 
   ([backup-id connection] (load-from-backup backup-id connection nil))
   ([backup-id connection item-list]
-   (let [schema              (clojure.edn/read-string (slurp (pull-file backup-id "schema.edn")))
-         full-dependencies   (clojure.edn/read-string (slurp (pull-file backup-id "full-dependencies.edn")))
-         entity-dependencies (clojure.edn/read-string (slurp (pull-file backup-id "entity-dependencies.edn")))]
+   (let [schema              (edn/read-string (slurp (pull-file backup-id "schema.edn")))
+         full-dependencies   (edn/read-string (slurp (pull-file backup-id "full-dependencies.edn")))
+         entity-dependencies (edn/read-string (slurp (pull-file backup-id "entity-dependencies.edn")))]
      (dc/transact connection {:tx-data [{:db/ident       :entity/migration-key
                                          :db/unique      :db.unique/identity
                                          :db/cardinality :db.cardinality/one
@@ -108,47 +104,54 @@
                                          schema)})
 
      ;; TEMP - this has been fixed in current export (ezcater-olaciotn)
-     (dc/transact auto-ap.datomic/conn {:tx-data [{:entity/migration-key 17592291325318}]})
      (dc/transact connection {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
                                         {:entity/migration-key 17592232621701}
                                         {:entity/migration-key 17592263907739}
-                                        {:entity/migration-key 17592271516922}]
-
-                              })
-
+                                        {:entity/migration-key 17592271516922}]})
 
 
      (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
-             :let   [_ (swap! loaded conj entity)
-                     _ (println "querying for " entity)
-                     entities (ednl/slurp (pull-file backup-id (str entity ".ednl")))
+             :let   [_ (reset! so-far 0)
+                     _ (mu/log ::querying :entity entity)
+                     entities (mu/trace ::pulling-file
+                                        [:file (str backup-id "/" entity ".ednl")]
+                                        (ednl/slurp (pull-file backup-id (str entity ".ednl"))))]]
+       (load-entity entity entities)))))
 
-                     _ (println "Found some! here's a few: " (take 3 entities))
-                     tx-chan (a/chan 50)
-                     entities->transaction (fn [entities]
-                                             entities)
-                     pipeline (tx-pipeline connection
-                                           10
-                                           tx-chan
-                                           entities->transaction)]]
-       (doseq [batch (partition-all 200 entities)]
-         (try 
-           (a/>!! tx-chan batch)
-           (catch Exception e
-             (println e)
-             ((:stop pipeline)))))
+(defn restore-fresh-from-backup [{:keys [backup]}]
+  (mu/log ::beginning-restore)
+  (load-from-backup backup auto-ap.datomic/conn)
+  (mu/log ::restore-complete)
+  (mu/log ::beginning-index-build)
+  (auto-ap.datomic/transact-schema auto-ap.datomic/conn)
+  (auto-ap.ledger/reset-client+account+location+date)
+  (mu/log ::index-build-complete)
+  (mu/log ::refresh-running-balance-cache)
+  (auto-ap.ledger/rebuild-running-balance-cache)
+  (mu/log ::refresh-running-balance-cache-complete)
+  (mu/log ::done))
 
-       (println "waiting for done from" pipeline)
-       (flush)
-       (a/close! tx-chan)
-       (println (a/<!! (:result pipeline)))
-       ((:stop pipeline))
-       (println)
-       (println "Done")))))
+
+(defn -main [& _]
+  (try 
+    (execute "restore-from-backup" #(restore-fresh-from-backup (:args env)))
+    (catch Exception e
+      (println e)
+      (mu/log ::quit-error
+              :exception e
+              :background-job "restore-from-backup"
+              :service "restore-from-backup")
+      (Thread/sleep 5000)
+      (throw e))))
 
 ;; cloud load
-(comment 
-  (load-from-backup "a1975512-9091-49d1-a348-ee445363ba34" auto-ap.datomic/conn )
+#_(comment 
+
+  ;; /datomic-backup/079df203-eae0-4acf-94d5-8608ba8b8a9a
+  (load-from-backup "079df203-eae0-4acf-94d5-8608ba8b8a9a" auto-ap.datomic/conn ["charge"])
+
+  (load-entity "charge" (ednl/slurp "/tmp/tmp-edn"))
+
 
   )
 ;; => nil