From c8ada656c8ec2e1ff4acf2cd07745e0ec893d010 Mon Sep 17 00:00:00 2001
From: Bryce Covert <bryce@brycecovertoperations.com>
Date: Thu, 30 Mar 2023 11:11:17 -0700
Subject: [PATCH] starts logging in the event of an issue.

---
 src/clj/auto_ap/datomic.clj                  | 22 ++++++++++++++++
 src/clj/auto_ap/jobs/restore_from_backup.clj | 27 +++-----------------
 src/clj/auto_ap/ledger.clj                   | 17 +++---------
 3 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj
index bff703d2..ed6f3eae 100644
--- a/src/clj/auto_ap/datomic.clj
+++ b/src/clj/auto_ap/datomic.clj
@@ -831,3 +831,25 @@
   ;; this is temporary for any  new stuff that needs to be asserted for cloud migration.
   (dc/transact conn
                {:tx-data (edn/read-string (slurp (io/resource "cloud-migration-schema.edn")))}))
+
+(defn backoff [n]
+  (let [base-timeout 500 
+        max-timeout  300000 ; 5 minutes
+        max-retries  10
+        backoff-time (* base-timeout (Math/pow 2 (min n max-retries)))]
+    (min (+ backoff-time (rand-int base-timeout)) max-timeout)))
+
+(defn transact-with-backoff
+  ([tx ] (transact-with-backoff tx 0))
+  ([tx attempt]
+   (try 
+     (dc/transact conn {:tx-data tx})
+     (catch Exception e
+       (if (< attempt 10)
+         (do 
+           (Thread/sleep (backoff attempt))
+           (mu/log ::transact-failed
+                   :exception e
+                   :status "WARN")
+           (transact-with-backoff tx (inc attempt)))
+         (throw e))))))
diff --git a/src/clj/auto_ap/jobs/restore_from_backup.clj b/src/clj/auto_ap/jobs/restore_from_backup.clj
index 005639e1..fae9d756 100644
--- a/src/clj/auto_ap/jobs/restore_from_backup.clj
+++ b/src/clj/auto_ap/jobs/restore_from_backup.clj
@@ -1,7 +1,7 @@
 (ns auto-ap.jobs.restore-from-backup
   (:require
    [amazonica.aws.s3 :as s3]
-   [auto-ap.datomic]
+   [auto-ap.datomic :refer [transact-with-backoff]]
    [auto-ap.ledger]
    [auto-ap.jobs.core :refer [execute]]
    [clojure.edn :as edn]
@@ -43,32 +43,11 @@
 
 (def loaded (atom #{}))
 
-(defn backoff [n]
-  (let [base-timeout 500 
-        max-timeout  300000 ; 5 minutes
-        max-retries  10
-        backoff-time (* base-timeout (Math/pow 2 (min n max-retries)))]
-    (min (+ backoff-time (rand-int base-timeout)) max-timeout)))
-
-(defn upsert-batch-impl
-  ([batch ] (upsert-batch-impl batch 0))
-  ([batch attempt]
-   (try 
-     (dc/transact auto-ap.datomic/conn {:tx-data batch})
-     batch
-     (catch Exception e
-       (if (< attempt 10)
-         (do 
-           (Thread/sleep (backoff attempt))
-           (upsert-batch-impl batch (inc attempt)))
-         (throw e))))))
-
 (defn upsert-batch
   [batch]
   (de/future-with request-pool 
-                  (upsert-batch-impl batch)))
-
-
+                  (transact-with-backoff batch)
+                  batch))
 
 (defn pull-file [backup which]
   (mu/log ::pulling-file
diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj
index 17bff040..d6b96054 100644
--- a/src/clj/auto_ap/ledger.clj
+++ b/src/clj/auto_ap/ledger.clj
@@ -3,6 +3,7 @@
    [auto-ap.datomic
     :refer [audit-transact
             audit-transact-batch
+            transact-with-backoff
             conn
             pull-id
             pull-ref
@@ -504,18 +505,6 @@
                          (:bank-account/numeric-code (bank-accounts a)))
        :client_id client-id})))
 
-(defn upsert-batch-impl
-  ([batch ] (upsert-batch-impl batch 0))
-  ([batch attempt]
-   (try 
-     (dc/transact auto-ap.datomic/conn {:tx-data batch})
-     batch
-     (catch Exception e
-       (if (< attempt 10)
-         (do 
-           (Thread/sleep (* attempt 1000))
-           (upsert-batch-impl batch (inc attempt)))
-         (throw e))))))
 
 (defn reset-client+account+location+date
   ([] (reset-client+account+location+date (map first (dc/q  '[:find ?c :where [?c :client/code]] (dc/db conn)))))
@@ -549,7 +538,7 @@
                 (s/->source)
                 (s/map (fn [batch]
                          (de/future
-                           (upsert-batch-impl batch)
+                           (transact-with-backoff batch)
                            (count batch))))
                 (s/buffer 50)
                 (s/realize-each)
@@ -711,7 +700,7 @@
           (s/transform (partition-all 500))
           (s/map (fn [batch]
                    (de/future
-                     (upsert-batch-impl batch)
+                     (transact-with-backoff batch)
                      (count batch))))
           (s/buffer 50)
           (s/realize-each)