(cloud) more aggressive exponential backoff.

src/clj/auto_ap/jobs/restore_from_backup.clj

@@ -43,6 +43,13 @@
 
 (def loaded (atom #{}))
 
+(defn backoff [n]
+  (let [base-timeout 500 
+        max-timeout  300000 ; 5 minutes
+        max-retries  10
+        backoff-time (* base-timeout (Math/pow 2 (min n max-retries)))]
+    (min (+ backoff-time (rand-int base-timeout)) max-timeout)))
+
 (defn upsert-batch-impl
   ([batch ] (upsert-batch-impl batch 0))
   ([batch attempt]
@@ -52,7 +59,7 @@
      (catch Exception e
        (if (< attempt 10)
          (do 
-           (Thread/sleep (* attempt 1000))
+           (Thread/sleep (backoff attempt))
            (upsert-batch-impl batch (inc attempt)))
          (throw e))))))