From 744c4b60f7ff5248a2606172b83295428ac918db Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 30 Mar 2023 10:33:04 -0700 Subject: [PATCH] (cloud) more aggressive exponential backoff. --- src/clj/auto_ap/jobs/restore_from_backup.clj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/jobs/restore_from_backup.clj b/src/clj/auto_ap/jobs/restore_from_backup.clj index 18debcb5..005639e1 100644 --- a/src/clj/auto_ap/jobs/restore_from_backup.clj +++ b/src/clj/auto_ap/jobs/restore_from_backup.clj @@ -43,6 +43,13 @@ (def loaded (atom #{})) +(defn backoff [n] + (let [base-timeout 500 + max-timeout 300000 ; 5 minutes + max-retries 10 + backoff-time (* base-timeout (Math/pow 2 (min n max-retries)))] + (min (+ backoff-time (rand-int base-timeout)) max-timeout))) + (defn upsert-batch-impl ([batch ] (upsert-batch-impl batch 0)) ([batch attempt] @@ -52,7 +59,7 @@ (catch Exception e (if (< attempt 10) (do - (Thread/sleep (* attempt 1000)) + (Thread/sleep (backoff attempt)) (upsert-batch-impl batch (inc attempt))) (throw e))))))