Sets up ability to do backups from production

2023-03-28 12:40:23 -07:00
parent 24366058cf
commit b71095e44b
25 changed files with 1476 additions and 19 deletions
--- a/project.clj
+++ b/project.clj
@@ -89,6 +89,7 @@
                 #_[org.eclipse.jetty.websocket/websocket-servlet "11.0.0"]
                 [org.clojure/data.json "1.0.0"]
                 [org.clojure/data.csv "1.0.0"]
+                 [lambdaisland/edn-lines "1.0.10"]
                 [io.rkn/conformity "0.5.4"]
                 [com.brunobonacci/mulog "0.9.0"]
                 [com.brunobonacci/mulog-adv-console "0.9.0"]
@@ -101,7 +102,8 @@
                 
                 ;; needed for java 11
                 [javax.xml.bind/jaxb-api "2.4.0-b180830.0359"]
-                 [io.forward/clojure-mail "1.0.8"]]
+                 [io.forward/clojure-mail "1.0.8"]
+                 [lambdaisland/edn-lines "1.0.10"]]
  :managed-dependencies [;; explicit dependencies to get to latest versions for above
                         [com.fasterxml.jackson.core/jackson-core "2.12.0"]
                         [com.fasterxml.jackson.core/jackson-databind "2.12.0"]
--- a/src/clj/auto_ap/backup.clj
+++ b/src/clj/auto_ap/backup.clj
@@ -0,0 +1,503 @@
+;; This buffer is for Clojure experiments and evaluation.
+
+;; Press C-j to evaluate the last expression.
+
+;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
+
+(ns auto-ap.backup
+  (:require [datomic.api :as d]
+            [manifold.deferred :as de]
+            [manifold.executor :as ex]
+            [manifold.stream :as s]
+            [manifold.time :as mt]
+            [auto-ap.jobs.core :refer [execute]]
+            [clojure.java.io :as io]
+            [amazonica.aws.s3 :as s3]
+            [config.core :refer [env]]
+            [clojure.core.async :as a]
+            [lambdaisland.edn-lines :as ednl]
+            [clojure.set :as set]
+            [com.brunobonacci.mulog :as mu]))
+
+
+
+
+(def request-pool (ex/fixed-thread-executor 30))
+
+(def buffered (ex/fixed-thread-executor 30))
+
+(defn get-schema [remote-db]
+  (let [everything         (->> (d/q '[:find [(pull ?e [:db/ident
+                                                        {:db/valueType [:db/ident]}
+                                                        {:db/cardinality [:db/ident]}
+                                                        :db.attr/preds
+                                                        {:db/unique [:db/ident]}
+                                                        :db/isComponent
+                                                        :db/id
+                                                        :db/noHistory
+                                                        :db/tupleAttrs
+                                                        :db.entity/attrs
+                                                        :db.entity/preds
+                                                        :db/doc]) ...]
+                                       :where [?e :db/ident]]
+                                     remote-db))
+        schema-attrs       (->> everything
+                                (filter :db/ident)
+                                (filter (fn [{:db/keys [ident]}]
+                                          (if (namespace ident)
+                                            (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident))
+                                            true
+                                            ))))
+        meta-schema-schema (filter #(-> % :db/ident not) everything)]
+    schema-attrs))
+
+
+(def entity->best-key
+  {"transaction-rule"
+   [:transaction-rule/description, :transaction-rule/note :transaction-rule/vendor]
+   "square-location"
+   :square-location/square-id,
+   "expected-deposit"
+   :expected-deposit/date,
+   "journal-entry-line"
+   [:journal-entry-line/account, :journal-entry-line/debit :journal-entry-line/credit]
+   "vendor"
+   [:vendor/name,]
+   "transaction"
+   :transaction/amount,
+   "yodlee-provider-account"
+   :yodlee-provider-account/id,
+   "journal-entry"
+   :journal-entry/source,
+   "yodlee-merchant"  :yodlee-merchant/yodlee-id,
+   "invoice"
+   :invoice/invoice-number,
+   "vendor-terms-override"
+   :vendor-terms-override/client,
+   "integration-status"
+   :integration-status/state,
+   "conformity"       :conformity/conformed-norms-index,
+   "user"
+   :user/provider-id,
+   "sales-refund"
+   :sales-refund/total,
+   "plaid-account"
+   :plaid-account/name,
+   "charge"
+   [:charge/total, :charge/external-id]
+   "location-match"   :location-match/location,
+   "vendor-schedule-payment-dom"
+   :vendor-schedule-payment-dom/dom,
+   "account-client-override"
+   :account-client-override/client,
+   "plaid-item"
+   :plaid-item/client,
+   "transaction-account"
+   :transaction-account/account,
+   "address"
+   [:address/street1, :address/city :address/state :address/zip]
+   "order-line-item"
+   :order-line-item/total,
+   "ezcater-location" :ezcater-location/location,
+   "account"
+   [:account/numeric-code, :account/code :account/name :account/type]
+   "intuit-bank-account"
+   :intuit-bank-account/name,
+   "saved-query"
+   :saved-query/guid,
+   "ezcater-caterer"
+   :ezcater-caterer/uuid,
+   "forecasted-transaction"
+   :forecasted-transaction/day-of-month,
+   "audit"            :audit/user,
+   "yodlee-account"
+   :yodlee-account/id,
+   "transaction-rule-account"
+   [:transaction-rule-account/account, :transaction-rule-account/location]
+   "ezcater-integration"
+   :ezcater-integration/subscriber-uuid,
+   "report"
+   :report/created,
+   "bank-account"
+   :bank-account/code,
+   "vendor-usage"
+   :vendor-usage/key,
+   "invoice-expense-account"
+   [:invoice-expense-account/expense-account-id, :invoice-expense-account/account :invoice-expense-account/location :invoice-expense-account/amount]
+   "sales-order"
+   :sales-order/date,
+   "client"
+   :client/code,
+   "email-contact"    :email-contact/email,
+   "invoice-payment"
+   :invoice-payment/amount,
+   "contact"
+   [:contact/name, :contact/phone :contact/email]
+   "import-batch"
+   :import-batch/date,
+   "payment"
+   [:payment/date, :payment/bank-account]
+   "vendor-account-override"
+   :vendor-account-override/client})
+
+#_(defn references [schema]
+  (filter (comp #{:db.type/ref} :db/ident :db/valueType) schema ))
+
+
+
+
+
+#_(defn reference->entity  [remote-db]
+  (->> (d/q '[:find ?a ?v3
+              :in $ $$ [?a ...]
+              :where [$$ _ ?a ?e]
+              [$ ?e ?v _ _]
+              [$ ?v :db/ident ?v2 _ _]
+              [(namespace ?v2) ?v3]
+              [(namespace ?v2) ?v3]]
+            remote-db
+            (d/since remote-db #inst "2022-06-01")
+            (map :db/ident references)
+            )
+       (group-by first)
+       (map (fn [[k v]]
+              [k (disj (set (map second v)) "db")]))
+       (into {})))
+
+#_(def manual-dependencies
+  {:client/location-matches #{"location-match"}
+   :transaction/yodlee-merchant #{"yodlee-merchant"}
+   :vendor-account-override/account #{"account"}
+   :vendor-account-override/client #{"client"}
+   :vendor/secondary-contact #{"contact"}
+   :vendor/account-overrides #{"vendor-account-override"}
+   :client/bank-accounts #{"bank-account"}
+   :transaction-rule/yodlee-merchant #{"yodlee-merchant"}
+   :client/forecasted-transactions #{"forecasted-transaction"}
+   :transaction/forecast-match #{"forecasted-transaction"}
+   :vendor/automatically-paid-when-due #{"client"}
+   :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}
+   :vendor/terms-overrides #{"vendor-terms-override"}
+   :vendor-schedule-payment-dom/client #{"client"}})
+
+#_(defn full-dependencies [remote-db]
+  (update (merge-with into (reference->entity remote-db) manual-dependencies)
+          :journal-entry/original-entity
+          #(disj % "journal-entry")))
+
+#_(defn entity-dependencies  [schema]
+  (let [base-dependencies
+        (into 
+          {}
+          (map (fn [i]
+                 [i #{}])
+               (set (map (comp namespace :db/ident) 
+                         (filter :db/valueType
+                                 schema)))) 
+          ) 
+        ]
+    (into base-dependencies (reduce
+                              (fn [acc [ref deps]]
+                                (update acc (namespace ref) (fnil #(into % deps) #{})))
+                              {}
+                              (full-dependencies remote-db)))))
+
+(def full-dependencies
+  {:invoice/client #{"client"},
+ :sales-order/client #{"client"},
+ :transaction-rule/transaction-approval-status #{},
+ :transaction/forecast-match #{"forecasted-transaction"},
+ :user/role #{},
+ :vendor-schedule-payment-dom/client #{"client"},
+ :invoice-payment/payment #{"payment"},
+ :transaction-rule/client #{"client"},
+ :invoice/status #{},
+ :payment/type #{},
+ :expected-deposit/client #{"client"},
+ :transaction/bank-account #{"bank-account"},
+ :transaction-rule-account/account #{"account"},
+ :import-batch/status #{},
+ :user/clients #{"client"},
+ :payment/client #{"client"},
+ :expected-deposit/charges #{"charge"},
+ :vendor/automatically-paid-when-due #{"client"},
+ :payment/invoices #{"invoice"},
+ :client/forecasted-transactions #{"forecasted-transaction"},
+ :transaction/matched-rule #{"transaction-rule"},
+ :invoice/import-status #{},
+ :charge/processor #{},
+ :expected-deposit/vendor #{"vendor"},
+ :client/square-locations #{"square-location"},
+ :payment/status #{},
+ :client/location-matches #{"location-match"},
+ :saved-query/client #{"client"},
+ :transaction/payment #{"payment"},
+ :transaction-rule/vendor #{"vendor"},
+ :plaid-item/client #{"client"},
+ :account/applicability #{},
+ :journal-entry-line/account #{"account" "bank-account"},
+ :client/bank-accounts #{"bank-account"},
+ :yodlee-provider-account/client #{"client"},
+ :account/vendor-allowance #{},
+ :payment/bank-account #{"bank-account"},
+ :account/default-allowance #{},
+ :transaction-rule/yodlee-merchant #{"yodlee-merchant"},
+ :vendor/account-overrides #{"vendor-account-override"},
+ :transaction/client #{"client"},
+ :invoice/vendor #{"vendor"},
+ :sales-order/vendor #{"vendor"},
+ :expected-deposit/status #{},
+ :journal-entry/original-entity #{"transaction" "invoice"},
+ :vendor-usage/client #{"client"},
+ :transaction/expected-deposit #{"expected-deposit"},
+ :client/ezcater-locations #{"ezcater-location"},
+ :journal-entry/client #{"client"},
+ :vendor/secondary-contact #{"contact"},
+ :journal-entry/line-items #{"journal-entry-line"},
+ :vendor/legal-entity-1099-type #{},
+ :transaction-rule/bank-account #{"bank-account"},
+ :transaction-account/account #{"account"},
+ :vendor/terms-overrides #{"vendor-terms-override"},
+ :vendor/default-account #{"account"},
+ :transaction/yodlee-merchant #{"yodlee-merchant"},
+ :sales-refund/client #{"client"},
+ :client/emails #{"email-contact"},
+ :payment/vendor #{"vendor"},
+ :invoice-payment/invoice #{"invoice"},
+ :report/client #{"client"},
+ :transaction-rule/accounts #{"transaction-rule-account"},
+ :charge/client #{"client"},
+ :bank-account/type #{},
+ :invoice-expense-account/account #{"account"},
+ :vendor/legal-entity-tin-type #{},
+ :transaction/approval-status #{},
+ :import-batch/entry #{"transaction"},
+ :bank-account/intuit-bank-account #{"intuit-bank-account"},
+ :account/type #{},
+ :sales-refund/vendor #{"vendor"},
+ :bank-account/yodlee-account #{"yodlee-account"},
+ :vendor/address #{"address"},
+ :integration-status/state #{},
+ :transaction/accounts #{"transaction-account"},
+ :sales-order/charges #{"charge"},
+ :client/address #{"address"},
+ :ezcater-location/caterer #{"ezcater-caterer"},
+ :vendor-account-override/client #{"client"},
+ :bank-account/integration-status #{"integration-status"},
+ :yodlee-provider-account/accounts #{"yodlee-account"},
+ :account/invoice-allowance #{},
+ :journal-entry/vendor #{"vendor"},
+ :plaid-item/accounts #{"plaid-account"},
+ :vendor-usage/vendor #{"vendor"},
+ :sales-order/line-items #{"order-line-item"},
+ :invoice/expense-accounts #{"invoice-expense-account"},
+ :account-client-override/client #{"client"},
+ :vendor/primary-contact #{"contact"},
+ :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"},
+ :account/client-overrides #{"account-client-override"},
+ :transaction/vendor #{"vendor"},
+ :client/square-integration-status #{"integration-status"},
+ :ezcater-integration/caterers #{"ezcater-caterer"},
+ :vendor-account-override/account #{"account"},
+ :import-batch/source #{}})
+
+(def entity-dependencies
+  {"transaction-rule"
+   #{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account"
+     "client"},
+   "square-location"             #{},
+   "expected-deposit"            #{"vendor" "charge" "client"},
+   "journal-entry-line"          #{"account" "bank-account"},
+   "vendor"
+   #{"vendor-schedule-payment-dom" "address" "account" "client" "contact"
+     "vendor-account-override"},
+   "transaction"
+   #{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant"
+     "transaction-account" "forecasted-transaction" "bank-account" "client"
+     "payment"},
+   "yodlee-provider-account"     #{"yodlee-account" "client"},
+   "journal-entry"
+   #{"journal-entry-line" "vendor" "transaction" "invoice" "client"},
+   "yodlee-merchant"             #{},
+   "invoice"                     #{"vendor" "invoice-expense-account" "client"},
+   "vendor-terms-override"       #{},
+   "integration-status"          #{},
+   "conformity"                  #{},
+   "user"                        #{"client"},
+   "sales-refund"                #{"vendor" "client"},
+   "plaid-account"               #{},
+   "charge"                      #{"client"},
+   "location-match"              #{},
+   "vendor-schedule-payment-dom" #{"client"},
+   "account-client-override"     #{"client"},
+   "plaid-item"                  #{"plaid-account" "client"},
+   "transaction-account"         #{"account"},
+   "address"                     #{},
+   "order-line-item"             #{},
+   "ezcater-location"            #{"ezcater-caterer"},
+   "account"                     #{"account-client-override"},
+   "intuit-bank-account"         #{},
+   "saved-query"                 #{"client"},
+   "ezcater-caterer"             #{},
+   "forecasted-transaction"      #{},
+   "audit"                       #{},
+   "yodlee-account"              #{},
+   "transaction-rule-account"    #{"account"},
+   "ezcater-integration"         #{"ezcater-caterer"},
+   "report"                      #{"client"},
+   "bank-account"                #{"integration-status" "intuit-bank-account" "yodlee-account"},
+   "vendor-usage"                #{"vendor" "client"},
+   "invoice-expense-account"     #{"account"},
+   "sales-order"                 #{"vendor" "charge" "order-line-item" "client"},
+   "client"
+   #{"square-location" "integration-status" "location-match" "address"
+     "ezcater-location" "forecasted-transaction" "bank-account" "email-contact"},
+   "email-contact"               #{},
+   "invoice-payment"             #{"invoice" "payment"},
+   "contact"                     #{},
+   "import-batch"                #{"transaction"},
+   "payment"                     #{"vendor" "invoice" "bank-account" "client"},
+   "vendor-account-override"     #{"account" "client"}})
+
+(defn order-of-insert [entity-dependencies]
+  (loop [entity-dependencies entity-dependencies 
+         order []]
+    (let [next-order (for [[entity deps] entity-dependencies
+                           :when         (not (seq deps))]
+                       entity)
+          next-deps (reduce
+                      (fn [entity-dependencies next-entity]
+                        (into {}
+                              (map
+                                (fn [[k v]]
+                                  [k (disj v next-entity)])
+                                entity-dependencies)))
+                      (apply dissoc entity-dependencies next-order)
+                      next-order)]
+      (if (seq next-deps)
+        (recur next-deps (into order next-order))
+        (into order next-order)))))
+
+
+(def loaded (atom #{}))
+
+
+(def dumped (atom #{}))
+
+(defn write-s3 [data location]
+  (spit (io/file "/tmp/temp-edn")
+        (with-out-str (clojure.pprint/pprint data)))
+  (s3/put-object :bucket-name (:data-bucket env)
+                 :key location
+                 :input-stream (io/make-input-stream (io/file "/tmp/temp-edn") {})))
+
+(defn dump-schema [schema backup]
+  (write-s3 (map
+              (fn [s]
+                (set/rename-keys s {:db/id :entity/migration-key}))
+              schema)
+            (str backup "/schema.edn"))
+  (write-s3 full-dependencies
+            (str backup "/full-dependencies.edn"))
+  (write-s3 entity-dependencies
+            (str backup "/entity-dependencies.edn")))
+
+(defn pull-batch [remote-db schema entity entities]
+  (de/future-with request-pool
+                  (mu/with-context {:entity entity}
+                    (try 
+                      (mu/log ::pulling :count (count entities))
+                      (->> (d/pull-many remote-db
+                                        (->> schema
+                                             (filter :db/valueType)
+                                             (mapv :db/ident)
+                                             (filter #(= entity (namespace %)))
+                                             (into [:db/id]))
+                                        entities)
+                           (mapv (fn [m ]
+                                   (reduce
+                                     (fn [m [k v]]
+                                       (cond
+                                         (= k :db/id)
+                                         (-> m
+                                             (assoc :entity/migration-key v)
+                                             (dissoc :db/id))
+                                         (full-dependencies k)
+                                         (if (vector? v)
+                                           (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
+                                           (assoc m k [:entity/migration-key (:db/id v)]))
+                                         :else
+                                         (dissoc m :payment/pdf-data
+                                                 :payment/memo)))
+                                     m
+                                     m))))
+                      (catch Throwable e
+                        (mu/log ::pull-error
+                                :exception e)
+                        
+                        (throw e))))))
+
+(def in-flight (atom 0))
+
+(defn dump-all
+  ([] (dump-all nil))
+  ([item-list]
+   (let [backup-id (str "/datomic-backup/" (java.util.UUID/randomUUID))
+         _ (mu/log ::starting-backup :backup backup-id)
+         remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod"))
+         _         (mu/log ::fetching-schema)
+         schema    (get-schema remote-db)
+         ]
+     (mu/log ::dumping-schema)
+     (dump-schema schema backup-id)
+     (mu/log ::schema-dumped)
+     (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
+             :let   [_ (swap! dumped conj entity)
+                     _ (mu/log ::querying :entity entity)
+                     entities (d/q '[:find [?e ...]
+                                     :in $ [?a ...]
+                                     :where [?e ?a]]
+                                   remote-db
+                                   (cond-> (entity->best-key entity)
+                                     (not (vector? (entity->best-key entity))) vector))
+
+                     _ (mu/log ::entity-total-found :count (count entities) :entity entity)]]
+       (mu/trace ::single-entity
+         [:entity entity]
+         (mu/with-context {:entity entity}
+           (mu/log ::starting)
+           (mu/log ::deleting)
+           (io/delete-file (io/file "/tmp/tmp-ednl") true)
+           (mu/log ::pulling)
+           (ednl/with-append [append "/tmp/tmp-ednl" ]
+             @(s/consume (fn [batch]
+                           (mu/with-context {:entity entity}
+                             (doseq [a batch]
+                               (mu/log ::appending :count (count batch))
+                               (try 
+                                 (append a) 
+                                 (catch Exception e
+                                   (mu/log ::error
+                                           :exception e)
+                                   (throw e)))
+                               (mu/log ::appended :count (count batch))))
+                           
+                           )
+                         (->> (partition-all 100 entities)
+                              (into [])
+                              (s/->source)
+                              (s/onto buffered)
+                              (s/buffer 20)
+                              (s/map (fn [entities]
+                                       (pull-batch remote-db schema entity entities)))
+                              (s/buffer 20)
+                              (s/realize-each)))
+             )
+           (mu/log ::copying)
+           (s3/put-object :bucket-name (:data-bucket env)
+                          :key (str backup-id "/" entity ".ednl")
+                          :input-stream (io/make-input-stream (io/file "/tmp/tmp-ednl") {}))
+           (mu/log ::copied)))))))
+
+(defn -main [& _]
+  (execute "export-backup" #(dump-all)))
--- a/src/clj/auto_ap/server.clj
+++ b/src/clj/auto_ap/server.clj
@@ -8,6 +8,7 @@
   [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
   [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
   [auto-ap.jobs.intuit :as job-intuit]
+   [auto-ap.backup :as backup]
   [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
   [auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
   [auto-ap.jobs.plaid :as job-plaid]
@@ -137,6 +138,9 @@
           (= job "bulk-journal-import")
           (job-bulk-journal-import/-main)

+           (= job "export-backup")
+           (backup/-main)
+
           :else
           
           (do 
--- a/src/cljs/auto_ap/views/pages/admin/jobs.cljs
+++ b/src/cljs/auto_ap/views/pages/admin/jobs.cljs
@@ -144,7 +144,8 @@
                                        [:square2-import-job "Square2 Import"]
                                        [:register-invoice-import "Register Invoice Import "]
                                        [:ezcater-upsert "Upsert recent ezcater orders"]
-                                        [:load-historical-sales "Load Historical Square Sales"]]
+                                        [:load-historical-sales "Load Historical Square Sales"]
+                                        [:export-backup "Export Backup"]]
                          :allow-nil?  true
                          :keywordize? true}]]
      (cond (= :bulk-journal-import (:job data))
--- a/terraform/.gitx/HEAD
+++ b/terraform/.gitx/HEAD
@@ -0,0 +1 @@
+ref: refs/heads/master
--- a/terraform/.gitx/config
+++ b/terraform/.gitx/config
@@ -0,0 +1,5 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
--- a/terraform/.gitx/description
+++ b/terraform/.gitx/description
@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.
--- a/terraform/.gitx/hooks/applypatch-msg.sample
+++ b/terraform/.gitx/hooks/applypatch-msg.sample
@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message taken by
+# applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.  The hook is
+# allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "applypatch-msg".
+
+. git-sh-setup
+commitmsg="$(git rev-parse --git-path hooks/commit-msg)"
+test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"}
+:
--- a/terraform/.gitx/hooks/commit-msg.sample
+++ b/terraform/.gitx/hooks/commit-msg.sample
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message.
+# Called by "git commit" with one argument, the name of the file
+# that has the commit message.  The hook should exit with non-zero
+# status after issuing an appropriate message if it wants to stop the
+# commit.  The hook is allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "commit-msg".
+
+# Uncomment the below to add a Signed-off-by line to the message.
+# Doing this in a hook is a bad idea in general, but the prepare-commit-msg
+# hook is more suited to it.
+#
+# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
+
+# This example catches duplicate Signed-off-by lines.
+
+test "" = "$(grep '^Signed-off-by: ' "$1" |
+	 sort | uniq -c | sed -e '/^[ 	]*1[ 	]/d')" || {
+	echo >&2 Duplicate Signed-off-by lines.
+	exit 1
+}
--- a/terraform/.gitx/hooks/fsmonitor-watchman.sample
+++ b/terraform/.gitx/hooks/fsmonitor-watchman.sample
@@ -0,0 +1,174 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use IPC::Open2;
+
+# An example hook script to integrate Watchman
+# (https://facebook.github.io/watchman/) with git to speed up detecting
+# new and modified files.
+#
+# The hook is passed a version (currently 2) and last update token
+# formatted as a string and outputs to stdout a new update token and
+# all files that have been modified since the update token. Paths must
+# be relative to the root of the working tree and separated by a single NUL.
+#
+# To enable this hook, rename this file to "query-watchman" and set
+# 'git config core.fsmonitor .git/hooks/query-watchman'
+#
+my ($version, $last_update_token) = @ARGV;
+
+# Uncomment for debugging
+# print STDERR "$0 $version $last_update_token\n";
+
+# Check the hook interface version
+if ($version ne 2) {
+	die "Unsupported query-fsmonitor hook version '$version'.\n" .
+	    "Falling back to scanning...\n";
+}
+
+my $git_work_tree = get_working_dir();
+
+my $retry = 1;
+
+my $json_pkg;
+eval {
+	require JSON::XS;
+	$json_pkg = "JSON::XS";
+	1;
+} or do {
+	require JSON::PP;
+	$json_pkg = "JSON::PP";
+};
+
+launch_watchman();
+
+sub launch_watchman {
+	my $o = watchman_query();
+	if (is_work_tree_watched($o)) {
+		output_result($o->{clock}, @{$o->{files}});
+	}
+}
+
+sub output_result {
+	my ($clockid, @files) = @_;
+
+	# Uncomment for debugging watchman output
+	# open (my $fh, ">", ".git/watchman-output.out");
+	# binmode $fh, ":utf8";
+	# print $fh "$clockid\n@files\n";
+	# close $fh;
+
+	binmode STDOUT, ":utf8";
+	print $clockid;
+	print "\0";
+	local $, = "\0";
+	print @files;
+}
+
+sub watchman_clock {
+	my $response = qx/watchman clock "$git_work_tree"/;
+	die "Failed to get clock id on '$git_work_tree'.\n" .
+		"Falling back to scanning...\n" if $? != 0;
+
+	return $json_pkg->new->utf8->decode($response);
+}
+
+sub watchman_query {
+	my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty')
+	or die "open2() failed: $!\n" .
+	"Falling back to scanning...\n";
+
+	# In the query expression below we're asking for names of files that
+	# changed since $last_update_token but not from the .git folder.
+	#
+	# To accomplish this, we're using the "since" generator to use the
+	# recency index to select candidate nodes and "fields" to limit the
+	# output to file names only. Then we're using the "expression" term to
+	# further constrain the results.
+	my $last_update_line = "";
+	if (substr($last_update_token, 0, 1) eq "c") {
+		$last_update_token = "\"$last_update_token\"";
+		$last_update_line = qq[\n"since": $last_update_token,];
+	}
+	my $query = <<"	END";
+		["query", "$git_work_tree", {$last_update_line
+			"fields": ["name"],
+			"expression": ["not", ["dirname", ".git"]]
+		}]
+	END
+
+	# Uncomment for debugging the watchman query
+	# open (my $fh, ">", ".git/watchman-query.json");
+	# print $fh $query;
+	# close $fh;
+
+	print CHLD_IN $query;
+	close CHLD_IN;
+	my $response = do {local $/; <CHLD_OUT>};
+
+	# Uncomment for debugging the watch response
+	# open ($fh, ">", ".git/watchman-response.json");
+	# print $fh $response;
+	# close $fh;
+
+	die "Watchman: command returned no output.\n" .
+	"Falling back to scanning...\n" if $response eq "";
+	die "Watchman: command returned invalid output: $response\n" .
+	"Falling back to scanning...\n" unless $response =~ /^\{/;
+
+	return $json_pkg->new->utf8->decode($response);
+}
+
+sub is_work_tree_watched {
+	my ($output) = @_;
+	my $error = $output->{error};
+	if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) {
+		$retry--;
+		my $response = qx/watchman watch "$git_work_tree"/;
+		die "Failed to make watchman watch '$git_work_tree'.\n" .
+		    "Falling back to scanning...\n" if $? != 0;
+		$output = $json_pkg->new->utf8->decode($response);
+		$error = $output->{error};
+		die "Watchman: $error.\n" .
+		"Falling back to scanning...\n" if $error;
+
+		# Uncomment for debugging watchman output
+		# open (my $fh, ">", ".git/watchman-output.out");
+		# close $fh;
+
+		# Watchman will always return all files on the first query so
+		# return the fast "everything is dirty" flag to git and do the
+		# Watchman query just to get it over with now so we won't pay
+		# the cost in git to look up each individual file.
+		my $o = watchman_clock();
+		$error = $output->{error};
+
+		die "Watchman: $error.\n" .
+		"Falling back to scanning...\n" if $error;
+
+		output_result($o->{clock}, ("/"));
+		$last_update_token = $o->{clock};
+
+		eval { launch_watchman() };
+		return 0;
+	}
+
+	die "Watchman: $error.\n" .
+	"Falling back to scanning...\n" if $error;
+
+	return 1;
+}
+
+sub get_working_dir {
+	my $working_dir;
+	if ($^O =~ 'msys' || $^O =~ 'cygwin') {
+		$working_dir = Win32::GetCwd();
+		$working_dir =~ tr/\\/\//;
+	} else {
+		require Cwd;
+		$working_dir = Cwd::cwd();
+	}
+
+	return $working_dir;
+}
--- a/terraform/.gitx/hooks/post-update.sample
+++ b/terraform/.gitx/hooks/post-update.sample
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# An example hook script to prepare a packed repository for use over
+# dumb transports.
+#
+# To enable this hook, rename this file to "post-update".
+
+exec git update-server-info
--- a/terraform/.gitx/hooks/pre-applypatch.sample
+++ b/terraform/.gitx/hooks/pre-applypatch.sample
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed
+# by applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.
+#
+# To enable this hook, rename this file to "pre-applypatch".
+
+. git-sh-setup
+precommit="$(git rev-parse --git-path hooks/pre-commit)"
+test -x "$precommit" && exec "$precommit" ${1+"$@"}
+:
--- a/terraform/.gitx/hooks/pre-commit.sample
+++ b/terraform/.gitx/hooks/pre-commit.sample
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed.
+# Called by "git commit" with no arguments.  The hook should
+# exit with non-zero status after issuing an appropriate message if
+# it wants to stop the commit.
+#
+# To enable this hook, rename this file to "pre-commit".
+
+if git rev-parse --verify HEAD >/dev/null 2>&1
+then
+	against=HEAD
+else
+	# Initial commit: diff against an empty tree object
+	against=$(git hash-object -t tree /dev/null)
+fi
+
+# If you want to allow non-ASCII filenames set this variable to true.
+allownonascii=$(git config --type=bool hooks.allownonascii)
+
+# Redirect output to stderr.
+exec 1>&2
+
+# Cross platform projects tend to avoid non-ASCII filenames; prevent
+# them from being added to the repository. We exploit the fact that the
+# printable range starts at the space character and ends with tilde.
+if [ "$allownonascii" != "true" ] &&
+	# Note that the use of brackets around a tr range is ok here, (it's
+	# even required, for portability to Solaris 10's /usr/bin/tr), since
+	# the square bracket bytes happen to fall in the designated range.
+	test $(git diff --cached --name-only --diff-filter=A -z $against |
+	  LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0
+then
+	cat <<\EOF
+Error: Attempt to add a non-ASCII file name.
+
+This can cause problems if you want to work with people on other platforms.
+
+To be portable it is advisable to rename the file.
+
+If you know what you are doing you can disable this check using:
+
+  git config hooks.allownonascii true
+EOF
+	exit 1
+fi
+
+# If there are whitespace errors, print the offending file names and fail.
+exec git diff-index --check --cached $against --
--- a/terraform/.gitx/hooks/pre-merge-commit.sample
+++ b/terraform/.gitx/hooks/pre-merge-commit.sample
@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed.
+# Called by "git merge" with no arguments.  The hook should
+# exit with non-zero status after issuing an appropriate message to
+# stderr if it wants to stop the merge commit.
+#
+# To enable this hook, rename this file to "pre-merge-commit".
+
+. git-sh-setup
+test -x "$GIT_DIR/hooks/pre-commit" &&
+        exec "$GIT_DIR/hooks/pre-commit"
+:
--- a/terraform/.gitx/hooks/pre-push.sample
+++ b/terraform/.gitx/hooks/pre-push.sample
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# An example hook script to verify what is about to be pushed.  Called by "git
+# push" after it has checked the remote status, but before anything has been
+# pushed.  If this script exits with a non-zero status nothing will be pushed.
+#
+# This hook is called with the following parameters:
+#
+# $1 -- Name of the remote to which the push is being done
+# $2 -- URL to which the push is being done
+#
+# If pushing without using a named remote those arguments will be equal.
+#
+# Information about the commits which are being pushed is supplied as lines to
+# the standard input in the form:
+#
+#   <local ref> <local oid> <remote ref> <remote oid>
+#
+# This sample shows how to prevent push of commits where the log message starts
+# with "WIP" (work in progress).
+
+remote="$1"
+url="$2"
+
+zero=$(git hash-object --stdin </dev/null | tr '[0-9a-f]' '0')
+
+while read local_ref local_oid remote_ref remote_oid
+do
+	if test "$local_oid" = "$zero"
+	then
+		# Handle delete
+		:
+	else
+		if test "$remote_oid" = "$zero"
+		then
+			# New branch, examine all commits
+			range="$local_oid"
+		else
+			# Update to existing branch, examine new commits
+			range="$remote_oid..$local_oid"
+		fi
+
+		# Check for WIP commit
+		commit=$(git rev-list -n 1 --grep '^WIP' "$range")
+		if test -n "$commit"
+		then
+			echo >&2 "Found WIP commit in $local_ref, not pushing"
+			exit 1
+		fi
+	fi
+done
+
+exit 0
--- a/terraform/.gitx/hooks/pre-rebase.sample
+++ b/terraform/.gitx/hooks/pre-rebase.sample
@@ -0,0 +1,169 @@
+#!/bin/sh
+#
+# Copyright (c) 2006, 2008 Junio C Hamano
+#
+# The "pre-rebase" hook is run just before "git rebase" starts doing
+# its job, and can prevent the command from running by exiting with
+# non-zero status.
+#
+# The hook is called with the following parameters:
+#
+# $1 -- the upstream the series was forked from.
+# $2 -- the branch being rebased (or empty when rebasing the current branch).
+#
+# This sample shows how to prevent topic branches that are already
+# merged to 'next' branch from getting rebased, because allowing it
+# would result in rebasing already published history.
+
+publish=next
+basebranch="$1"
+if test "$#" = 2
+then
+	topic="refs/heads/$2"
+else
+	topic=`git symbolic-ref HEAD` ||
+	exit 0 ;# we do not interrupt rebasing detached HEAD
+fi
+
+case "$topic" in
+refs/heads/??/*)
+	;;
+*)
+	exit 0 ;# we do not interrupt others.
+	;;
+esac
+
+# Now we are dealing with a topic branch being rebased
+# on top of master.  Is it OK to rebase it?
+
+# Does the topic really exist?
+git show-ref -q "$topic" || {
+	echo >&2 "No such branch $topic"
+	exit 1
+}
+
+# Is topic fully merged to master?
+not_in_master=`git rev-list --pretty=oneline ^master "$topic"`
+if test -z "$not_in_master"
+then
+	echo >&2 "$topic is fully merged to master; better remove it."
+	exit 1 ;# we could allow it, but there is no point.
+fi
+
+# Is topic ever merged to next?  If so you should not be rebasing it.
+only_next_1=`git rev-list ^master "^$topic" ${publish} | sort`
+only_next_2=`git rev-list ^master           ${publish} | sort`
+if test "$only_next_1" = "$only_next_2"
+then
+	not_in_topic=`git rev-list "^$topic" master`
+	if test -z "$not_in_topic"
+	then
+		echo >&2 "$topic is already up to date with master"
+		exit 1 ;# we could allow it, but there is no point.
+	else
+		exit 0
+	fi
+else
+	not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"`
+	/usr/bin/perl -e '
+		my $topic = $ARGV[0];
+		my $msg = "* $topic has commits already merged to public branch:\n";
+		my (%not_in_next) = map {
+			/^([0-9a-f]+) /;
+			($1 => 1);
+		} split(/\n/, $ARGV[1]);
+		for my $elem (map {
+				/^([0-9a-f]+) (.*)$/;
+				[$1 => $2];
+			} split(/\n/, $ARGV[2])) {
+			if (!exists $not_in_next{$elem->[0]}) {
+				if ($msg) {
+					print STDERR $msg;
+					undef $msg;
+				}
+				print STDERR " $elem->[1]\n";
+			}
+		}
+	' "$topic" "$not_in_next" "$not_in_master"
+	exit 1
+fi
+
+<<\DOC_END
+
+This sample hook safeguards topic branches that have been
+published from being rewound.
+
+The workflow assumed here is:
+
+ * Once a topic branch forks from "master", "master" is never
+   merged into it again (either directly or indirectly).
+
+ * Once a topic branch is fully cooked and merged into "master",
+   it is deleted.  If you need to build on top of it to correct
+   earlier mistakes, a new topic branch is created by forking at
+   the tip of the "master".  This is not strictly necessary, but
+   it makes it easier to keep your history simple.
+
+ * Whenever you need to test or publish your changes to topic
+   branches, merge them into "next" branch.
+
+The script, being an example, hardcodes the publish branch name
+to be "next", but it is trivial to make it configurable via
+$GIT_DIR/config mechanism.
+
+With this workflow, you would want to know:
+
+(1) ... if a topic branch has ever been merged to "next".  Young
+    topic branches can have stupid mistakes you would rather
+    clean up before publishing, and things that have not been
+    merged into other branches can be easily rebased without
+    affecting other people.  But once it is published, you would
+    not want to rewind it.
+
+(2) ... if a topic branch has been fully merged to "master".
+    Then you can delete it.  More importantly, you should not
+    build on top of it -- other people may already want to
+    change things related to the topic as patches against your
+    "master", so if you need further changes, it is better to
+    fork the topic (perhaps with the same name) afresh from the
+    tip of "master".
+
+Let's look at this example:
+
+		   o---o---o---o---o---o---o---o---o---o "next"
+		  /       /           /           /
+		 /   a---a---b A     /           /
+		/   /               /           /
+	       /   /   c---c---c---c B         /
+	      /   /   /             \         /
+	     /   /   /   b---b C     \       /
+	    /   /   /   /             \     /
+    ---o---o---o---o---o---o---o---o---o---o---o "master"
+
+
+A, B and C are topic branches.
+
+ * A has one fix since it was merged up to "next".
+
+ * B has finished.  It has been fully merged up to "master" and "next",
+   and is ready to be deleted.
+
+ * C has not merged to "next" at all.
+
+We would want to allow C to be rebased, refuse A, and encourage
+B to be deleted.
+
+To compute (1):
+
+	git rev-list ^master ^topic next
+	git rev-list ^master        next
+
+	if these match, topic has not merged in next at all.
+
+To compute (2):
+
+	git rev-list master..topic
+
+	if this is empty, it is fully merged to "master".
+
+DOC_END
--- a/terraform/.gitx/hooks/pre-receive.sample
+++ b/terraform/.gitx/hooks/pre-receive.sample
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An example hook script to make use of push options.
+# The example simply echoes all push options that start with 'echoback='
+# and rejects all pushes when the "reject" push option is used.
+#
+# To enable this hook, rename this file to "pre-receive".
+
+if test -n "$GIT_PUSH_OPTION_COUNT"
+then
+	i=0
+	while test "$i" -lt "$GIT_PUSH_OPTION_COUNT"
+	do
+		eval "value=\$GIT_PUSH_OPTION_$i"
+		case "$value" in
+		echoback=*)
+			echo "echo from the pre-receive-hook: ${value#*=}" >&2
+			;;
+		reject)
+			exit 1
+		esac
+		i=$((i + 1))
+	done
+fi
--- a/terraform/.gitx/hooks/prepare-commit-msg.sample
+++ b/terraform/.gitx/hooks/prepare-commit-msg.sample
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# An example hook script to prepare the commit log message.
+# Called by "git commit" with the name of the file that has the
+# commit message, followed by the description of the commit
+# message's source.  The hook's purpose is to edit the commit
+# message file.  If the hook fails with a non-zero status,
+# the commit is aborted.
+#
+# To enable this hook, rename this file to "prepare-commit-msg".
+
+# This hook includes three examples. The first one removes the
+# "# Please enter the commit message..." help message.
+#
+# The second includes the output of "git diff --name-status -r"
+# into the message, just before the "git status" output.  It is
+# commented because it doesn't cope with --amend or with squashed
+# commits.
+#
+# The third example adds a Signed-off-by line to the message, that can
+# still be edited.  This is rarely a good idea.
+
+COMMIT_MSG_FILE=$1
+COMMIT_SOURCE=$2
+SHA1=$3
+
+/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE"
+
+# case "$COMMIT_SOURCE,$SHA1" in
+#  ,|template,)
+#    /usr/bin/perl -i.bak -pe '
+#       print "\n" . `git diff --cached --name-status -r`
+# 	 if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;;
+#  *) ;;
+# esac
+
+# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE"
+# if test -z "$COMMIT_SOURCE"
+# then
+#   /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE"
+# fi
--- a/terraform/.gitx/hooks/push-to-checkout.sample
+++ b/terraform/.gitx/hooks/push-to-checkout.sample
@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# An example hook script to update a checked-out tree on a git push.
+#
+# This hook is invoked by git-receive-pack(1) when it reacts to git
+# push and updates reference(s) in its repository, and when the push
+# tries to update the branch that is currently checked out and the
+# receive.denyCurrentBranch configuration variable is set to
+# updateInstead.
+#
+# By default, such a push is refused if the working tree and the index
+# of the remote repository has any difference from the currently
+# checked out commit; when both the working tree and the index match
+# the current commit, they are updated to match the newly pushed tip
+# of the branch. This hook is to be used to override the default
+# behaviour; however the code below reimplements the default behaviour
+# as a starting point for convenient modification.
+#
+# The hook receives the commit with which the tip of the current
+# branch is going to be updated:
+commit=$1
+
+# It can exit with a non-zero status to refuse the push (when it does
+# so, it must not modify the index or the working tree).
+die () {
+	echo >&2 "$*"
+	exit 1
+}
+
+# Or it can make any necessary changes to the working tree and to the
+# index to bring them to the desired state when the tip of the current
+# branch is updated to the new commit, and exit with a zero status.
+#
+# For example, the hook can simply run git read-tree -u -m HEAD "$1"
+# in order to emulate git fetch that is run in the reverse direction
+# with git push, as the two-tree form of git read-tree -u -m is
+# essentially the same as git switch or git checkout that switches
+# branches while keeping the local changes in the working tree that do
+# not interfere with the difference between the branches.
+
+# The below is a more-or-less exact translation to shell of the C code
+# for the default behaviour for git's push-to-checkout hook defined in
+# the push_to_deploy() function in builtin/receive-pack.c.
+#
+# Note that the hook will be executed from the repository directory,
+# not from the working tree, so if you want to perform operations on
+# the working tree, you will have to adapt your code accordingly, e.g.
+# by adding "cd .." or using relative paths.
+
+if ! git update-index -q --ignore-submodules --refresh
+then
+	die "Up-to-date check failed"
+fi
+
+if ! git diff-files --quiet --ignore-submodules --
+then
+	die "Working directory has unstaged changes"
+fi
+
+# This is a rough translation of:
+#
+#   head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX
+if git cat-file -e HEAD 2>/dev/null
+then
+	head=HEAD
+else
+	head=$(git hash-object -t tree --stdin </dev/null)
+fi
+
+if ! git diff-index --quiet --cached --ignore-submodules $head --
+then
+	die "Working directory has staged changes"
+fi
+
+if ! git read-tree -u -m "$commit"
+then
+	die "Could not update working tree to new HEAD"
+fi
--- a/terraform/.gitx/hooks/update.sample
+++ b/terraform/.gitx/hooks/update.sample
@@ -0,0 +1,128 @@
+#!/bin/sh
+#
+# An example hook script to block unannotated tags from entering.
+# Called by "git receive-pack" with arguments: refname sha1-old sha1-new
+#
+# To enable this hook, rename this file to "update".
+#
+# Config
+# ------
+# hooks.allowunannotated
+#   This boolean sets whether unannotated tags will be allowed into the
+#   repository.  By default they won't be.
+# hooks.allowdeletetag
+#   This boolean sets whether deleting tags will be allowed in the
+#   repository.  By default they won't be.
+# hooks.allowmodifytag
+#   This boolean sets whether a tag may be modified after creation. By default
+#   it won't be.
+# hooks.allowdeletebranch
+#   This boolean sets whether deleting branches will be allowed in the
+#   repository.  By default they won't be.
+# hooks.denycreatebranch
+#   This boolean sets whether remotely creating branches will be denied
+#   in the repository.  By default this is allowed.
+#
+
+# --- Command line
+refname="$1"
+oldrev="$2"
+newrev="$3"
+
+# --- Safety check
+if [ -z "$GIT_DIR" ]; then
+	echo "Don't run this script from the command line." >&2
+	echo " (if you want, you could supply GIT_DIR then run" >&2
+	echo "  $0 <ref> <oldrev> <newrev>)" >&2
+	exit 1
+fi
+
+if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then
+	echo "usage: $0 <ref> <oldrev> <newrev>" >&2
+	exit 1
+fi
+
+# --- Config
+allowunannotated=$(git config --type=bool hooks.allowunannotated)
+allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch)
+denycreatebranch=$(git config --type=bool hooks.denycreatebranch)
+allowdeletetag=$(git config --type=bool hooks.allowdeletetag)
+allowmodifytag=$(git config --type=bool hooks.allowmodifytag)
+
+# check for no description
+projectdesc=$(sed -e '1q' "$GIT_DIR/description")
+case "$projectdesc" in
+"Unnamed repository"* | "")
+	echo "*** Project description file hasn't been set" >&2
+	exit 1
+	;;
+esac
+
+# --- Check types
+# if $newrev is 0000...0000, it's a commit to delete a ref.
+zero=$(git hash-object --stdin </dev/null | tr '[0-9a-f]' '0')
+if [ "$newrev" = "$zero" ]; then
+	newrev_type=delete
+else
+	newrev_type=$(git cat-file -t $newrev)
+fi
+
+case "$refname","$newrev_type" in
+	refs/tags/*,commit)
+		# un-annotated tag
+		short_refname=${refname##refs/tags/}
+		if [ "$allowunannotated" != "true" ]; then
+			echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2
+			echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2
+			exit 1
+		fi
+		;;
+	refs/tags/*,delete)
+		# delete tag
+		if [ "$allowdeletetag" != "true" ]; then
+			echo "*** Deleting a tag is not allowed in this repository" >&2
+			exit 1
+		fi
+		;;
+	refs/tags/*,tag)
+		# annotated tag
+		if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1
+		then
+			echo "*** Tag '$refname' already exists." >&2
+			echo "*** Modifying a tag is not allowed in this repository." >&2
+			exit 1
+		fi
+		;;
+	refs/heads/*,commit)
+		# branch
+		if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then
+			echo "*** Creating a branch is not allowed in this repository" >&2
+			exit 1
+		fi
+		;;
+	refs/heads/*,delete)
+		# delete branch
+		if [ "$allowdeletebranch" != "true" ]; then
+			echo "*** Deleting a branch is not allowed in this repository" >&2
+			exit 1
+		fi
+		;;
+	refs/remotes/*,commit)
+		# tracking branch
+		;;
+	refs/remotes/*,delete)
+		# delete tracking branch
+		if [ "$allowdeletebranch" != "true" ]; then
+			echo "*** Deleting a tracking branch is not allowed in this repository" >&2
+			exit 1
+		fi
+		;;
+	*)
+		# Anything else (is there anything else?)
+		echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2
+		exit 1
+		;;
+esac
+
+# --- Finished
+exit 0
--- a/terraform/.gitx/info/exclude
+++ b/terraform/.gitx/info/exclude
@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~
--- a/terraform/deploy.tf
+++ b/terraform/deploy.tf
@@ -469,3 +469,15 @@ module "load_historical_sales_job" {
       memory = 4096
       cpu = 1024
 }
+
+module "export_backup_job" {
+       source = "./background-job/"
+       ecs_cluster = var.ecs_cluster
+       task_role_arn = var.task_role_arn
+       stage = var.stage
+       job_name = "export-backup"
+       execution_role_arn = var.execution_role_arn
+       use_schedule = false
+       memory = 8192
+       cpu = 4096
+}
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -82,6 +82,15 @@ resource "aws_s3_bucket" "data" {
            },
            "Resource":  "arn:aws:s3:::data.${var.stage}.app.integreatconsult.com",
            "Sid":       "AllowReadForProd"
+        },
+	{
+		"Action": "s3:*",
+		"Effect": "Allow",
+		"Principal": {
+			"AWS": "arn:aws:iam::679918342773:role/http-proxy"
+		},
+		"Resource": "arn:aws:s3:::data.${var.stage}.app.integreatconsult.com",
+		"Sid":  "AllowReadForProdProxy"
 	}
    ]
 }
--- a/terraform/terraform.tfstate.d/prod/terraform.tfstate
+++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate
@@ -1,7 +1,7 @@
 {
  "version": 4,
-  "terraform_version": "1.3.2",
-  "serial": 406,
+  "terraform_version": "1.3.8",
+  "serial": 411,
  "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
  "outputs": {
    "aws_access_key_id": {
@@ -164,7 +164,7 @@
            ],
            "tags": {},
            "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520",
            "timeouts": {
              "delete": null
            },
@@ -669,11 +669,25 @@
            "lifecycle_rule": [],
            "logging": [],
            "object_lock_configuration": [],
-            "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}",
+            "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}",
            "region": "us-east-1",
            "replication_configuration": [],
            "request_payer": "BucketOwner",
-            "server_side_encryption_configuration": [],
+            "server_side_encryption_configuration": [
+              {
+                "rule": [
+                  {
+                    "apply_server_side_encryption_by_default": [
+                      {
+                        "kms_master_key_id": "",
+                        "sse_algorithm": "AES256"
+                      }
+                    ],
+                    "bucket_key_enabled": false
+                  }
+                ]
+              }
+            ],
            "tags": {},
            "tags_all": {},
            "versioning": [
@@ -693,7 +707,8 @@
            "website_domain": "s3-website-us-east-1.amazonaws.com",
            "website_endpoint": "data.prod.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com"
          },
-          "sensitive_attributes": []
+          "sensitive_attributes": [],
+          "private": "bnVsbA=="
        }
      ]
    },
@@ -725,7 +740,21 @@
            "region": "us-east-1",
            "replication_configuration": [],
            "request_payer": "BucketOwner",
-            "server_side_encryption_configuration": [],
+            "server_side_encryption_configuration": [
+              {
+                "rule": [
+                  {
+                    "apply_server_side_encryption_by_default": [
+                      {
+                        "kms_master_key_id": "",
+                        "sse_algorithm": "AES256"
+                      }
+                    ],
+                    "bucket_key_enabled": false
+                  }
+                ]
+              }
+            ],
            "tags": {},
            "tags_all": {},
            "versioning": [
@@ -739,6 +768,7 @@
            "website_endpoint": null
          },
          "sensitive_attributes": [],
+          "private": "bnVsbA==",
          "dependencies": [
            "data.aws_caller_identity.current"
          ]
@@ -790,7 +820,21 @@
            "region": "us-east-1",
            "replication_configuration": [],
            "request_payer": "BucketOwner",
-            "server_side_encryption_configuration": [],
+            "server_side_encryption_configuration": [
+              {
+                "rule": [
+                  {
+                    "apply_server_side_encryption_by_default": [
+                      {
+                        "kms_master_key_id": "",
+                        "sse_algorithm": "AES256"
+                      }
+                    ],
+                    "bucket_key_enabled": false
+                  }
+                ]
+              }
+            ],
            "tags": {},
            "tags_all": {},
            "versioning": [
@@ -1370,6 +1414,45 @@
        }
      ]
    },
+    {
+      "module": "module.export_backup_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/export_backup_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"export-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"export-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"export-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "4096",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "export_backup_prod",
+            "id": "export_backup_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "8192",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
    {
      "module": "module.ezcater_upsert_job",
      "mode": "managed",
@@ -1801,7 +1884,7 @@
            ],
            "revision": 1,
            "runtime_platform": [],
-            "tags": null,
+            "tags": {},
            "tags_all": {},
            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
            "volume": []
@@ -2828,5 +2911,5 @@
      ]
    }
  ],
-  "check_results": []
+  "check_results": null
 }
--- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
+++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
@@ -1,7 +1,7 @@
 {
  "version": 4,
-  "terraform_version": "1.3.2",
-  "serial": 343,
+  "terraform_version": "1.3.8",
+  "serial": 406,
  "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
  "outputs": {
    "aws_access_key_id": {
@@ -164,7 +164,7 @@
            ],
            "tags": {},
            "tags_all": {},
-            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:413",
+            "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415",
            "timeouts": {
              "delete": null
            },
@@ -1772,6 +1772,45 @@
        }
      ]
    },
+    {
+      "module": "module.load_historical_sales_job",
+      "mode": "managed",
+      "type": "aws_ecs_task_definition",
+      "name": "background_taskdef",
+      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
+      "instances": [
+        {
+          "schema_version": 1,
+          "attributes": {
+            "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/load_historical_sales_prod:1",
+            "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"load-historical-sales\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"load-historical-sales\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"load-historical-sales\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]",
+            "cpu": "1024",
+            "ephemeral_storage": [],
+            "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole",
+            "family": "load_historical_sales_prod",
+            "id": "load_historical_sales_prod",
+            "inference_accelerator": [],
+            "ipc_mode": "",
+            "memory": "4096",
+            "network_mode": "awsvpc",
+            "pid_mode": "",
+            "placement_constraints": [],
+            "proxy_configuration": [],
+            "requires_compatibilities": [
+              "FARGATE"
+            ],
+            "revision": 1,
+            "runtime_platform": [],
+            "tags": null,
+            "tags_all": {},
+            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
+            "volume": []
+          },
+          "sensitive_attributes": [],
+          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
+        }
+      ]
+    },
    {
      "module": "module.plaid_job",
      "mode": "managed",
@@ -2069,7 +2108,7 @@
            ],
            "revision": 1,
            "runtime_platform": [],
-            "tags": null,
+            "tags": {},
            "tags_all": {},
            "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb",
            "volume": []
@@ -2789,5 +2828,5 @@
      ]
    }
  ],
-  "check_results": []
+  "check_results": null
 }
				`@@ -0,0 +1 @@`
				`Unnamed repository; edit this file 'description' to name the repository.`