From b71095e44bcf6696eac7df6bf3ad391e4c582abb Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 28 Mar 2023 12:40:23 -0700 Subject: [PATCH] Sets up ability to do backups from production --- project.clj | 4 +- src/clj/auto_ap/backup.clj | 503 ++++++++++++++++++ src/clj/auto_ap/server.clj | 4 + src/cljs/auto_ap/views/pages/admin/jobs.cljs | 3 +- terraform/.gitx/HEAD | 1 + terraform/.gitx/config | 5 + terraform/.gitx/description | 1 + terraform/.gitx/hooks/applypatch-msg.sample | 15 + terraform/.gitx/hooks/commit-msg.sample | 24 + .../.gitx/hooks/fsmonitor-watchman.sample | 174 ++++++ terraform/.gitx/hooks/post-update.sample | 8 + terraform/.gitx/hooks/pre-applypatch.sample | 14 + terraform/.gitx/hooks/pre-commit.sample | 49 ++ terraform/.gitx/hooks/pre-merge-commit.sample | 13 + terraform/.gitx/hooks/pre-push.sample | 53 ++ terraform/.gitx/hooks/pre-rebase.sample | 169 ++++++ terraform/.gitx/hooks/pre-receive.sample | 24 + .../.gitx/hooks/prepare-commit-msg.sample | 42 ++ terraform/.gitx/hooks/push-to-checkout.sample | 78 +++ terraform/.gitx/hooks/update.sample | 128 +++++ terraform/.gitx/info/exclude | 6 + terraform/deploy.tf | 14 +- terraform/main.tf | 11 +- .../prod/terraform.tfstate | 103 +++- .../prod/terraform.tfstate.backup | 49 +- 25 files changed, 1476 insertions(+), 19 deletions(-) create mode 100644 src/clj/auto_ap/backup.clj create mode 100644 terraform/.gitx/HEAD create mode 100644 terraform/.gitx/config create mode 100644 terraform/.gitx/description create mode 100755 terraform/.gitx/hooks/applypatch-msg.sample create mode 100755 terraform/.gitx/hooks/commit-msg.sample create mode 100755 terraform/.gitx/hooks/fsmonitor-watchman.sample create mode 100755 terraform/.gitx/hooks/post-update.sample create mode 100755 terraform/.gitx/hooks/pre-applypatch.sample create mode 100755 terraform/.gitx/hooks/pre-commit.sample create mode 100755 terraform/.gitx/hooks/pre-merge-commit.sample create mode 100755 terraform/.gitx/hooks/pre-push.sample create mode 100755 terraform/.gitx/hooks/pre-rebase.sample create mode 100755 terraform/.gitx/hooks/pre-receive.sample create mode 100755 terraform/.gitx/hooks/prepare-commit-msg.sample create mode 100755 terraform/.gitx/hooks/push-to-checkout.sample create mode 100755 terraform/.gitx/hooks/update.sample create mode 100644 terraform/.gitx/info/exclude diff --git a/project.clj b/project.clj index 2fea659e..35bf7ef0 100644 --- a/project.clj +++ b/project.clj @@ -89,6 +89,7 @@ #_[org.eclipse.jetty.websocket/websocket-servlet "11.0.0"] [org.clojure/data.json "1.0.0"] [org.clojure/data.csv "1.0.0"] + [lambdaisland/edn-lines "1.0.10"] [io.rkn/conformity "0.5.4"] [com.brunobonacci/mulog "0.9.0"] [com.brunobonacci/mulog-adv-console "0.9.0"] @@ -101,7 +102,8 @@ ;; needed for java 11 [javax.xml.bind/jaxb-api "2.4.0-b180830.0359"] - [io.forward/clojure-mail "1.0.8"]] + [io.forward/clojure-mail "1.0.8"] + [lambdaisland/edn-lines "1.0.10"]] :managed-dependencies [;; explicit dependencies to get to latest versions for above [com.fasterxml.jackson.core/jackson-core "2.12.0"] [com.fasterxml.jackson.core/jackson-databind "2.12.0"] diff --git a/src/clj/auto_ap/backup.clj b/src/clj/auto_ap/backup.clj new file mode 100644 index 00000000..88bce087 --- /dev/null +++ b/src/clj/auto_ap/backup.clj @@ -0,0 +1,503 @@ +;; This buffer is for Clojure experiments and evaluation. + +;; Press C-j to evaluate the last expression. + +;; You can also press C-u C-j to evaluate the expression and pretty-print its result. + +(ns auto-ap.backup + (:require [datomic.api :as d] + [manifold.deferred :as de] + [manifold.executor :as ex] + [manifold.stream :as s] + [manifold.time :as mt] + [auto-ap.jobs.core :refer [execute]] + [clojure.java.io :as io] + [amazonica.aws.s3 :as s3] + [config.core :refer [env]] + [clojure.core.async :as a] + [lambdaisland.edn-lines :as ednl] + [clojure.set :as set] + [com.brunobonacci.mulog :as mu])) + + + + +(def request-pool (ex/fixed-thread-executor 30)) + +(def buffered (ex/fixed-thread-executor 30)) + +(defn get-schema [remote-db] + (let [everything (->> (d/q '[:find [(pull ?e [:db/ident + {:db/valueType [:db/ident]} + {:db/cardinality [:db/ident]} + :db.attr/preds + {:db/unique [:db/ident]} + :db/isComponent + :db/id + :db/noHistory + :db/tupleAttrs + :db.entity/attrs + :db.entity/preds + :db/doc]) ...] + :where [?e :db/ident]] + remote-db)) + schema-attrs (->> everything + (filter :db/ident) + (filter (fn [{:db/keys [ident]}] + (if (namespace ident) + (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident)) + true + )))) + meta-schema-schema (filter #(-> % :db/ident not) everything)] + schema-attrs)) + + +(def entity->best-key + {"transaction-rule" + [:transaction-rule/description, :transaction-rule/note :transaction-rule/vendor] + "square-location" + :square-location/square-id, + "expected-deposit" + :expected-deposit/date, + "journal-entry-line" + [:journal-entry-line/account, :journal-entry-line/debit :journal-entry-line/credit] + "vendor" + [:vendor/name,] + "transaction" + :transaction/amount, + "yodlee-provider-account" + :yodlee-provider-account/id, + "journal-entry" + :journal-entry/source, + "yodlee-merchant" :yodlee-merchant/yodlee-id, + "invoice" + :invoice/invoice-number, + "vendor-terms-override" + :vendor-terms-override/client, + "integration-status" + :integration-status/state, + "conformity" :conformity/conformed-norms-index, + "user" + :user/provider-id, + "sales-refund" + :sales-refund/total, + "plaid-account" + :plaid-account/name, + "charge" + [:charge/total, :charge/external-id] + "location-match" :location-match/location, + "vendor-schedule-payment-dom" + :vendor-schedule-payment-dom/dom, + "account-client-override" + :account-client-override/client, + "plaid-item" + :plaid-item/client, + "transaction-account" + :transaction-account/account, + "address" + [:address/street1, :address/city :address/state :address/zip] + "order-line-item" + :order-line-item/total, + "ezcater-location" :ezcater-location/location, + "account" + [:account/numeric-code, :account/code :account/name :account/type] + "intuit-bank-account" + :intuit-bank-account/name, + "saved-query" + :saved-query/guid, + "ezcater-caterer" + :ezcater-caterer/uuid, + "forecasted-transaction" + :forecasted-transaction/day-of-month, + "audit" :audit/user, + "yodlee-account" + :yodlee-account/id, + "transaction-rule-account" + [:transaction-rule-account/account, :transaction-rule-account/location] + "ezcater-integration" + :ezcater-integration/subscriber-uuid, + "report" + :report/created, + "bank-account" + :bank-account/code, + "vendor-usage" + :vendor-usage/key, + "invoice-expense-account" + [:invoice-expense-account/expense-account-id, :invoice-expense-account/account :invoice-expense-account/location :invoice-expense-account/amount] + "sales-order" + :sales-order/date, + "client" + :client/code, + "email-contact" :email-contact/email, + "invoice-payment" + :invoice-payment/amount, + "contact" + [:contact/name, :contact/phone :contact/email] + "import-batch" + :import-batch/date, + "payment" + [:payment/date, :payment/bank-account] + "vendor-account-override" + :vendor-account-override/client}) + +#_(defn references [schema] + (filter (comp #{:db.type/ref} :db/ident :db/valueType) schema )) + + + + + +#_(defn reference->entity [remote-db] + (->> (d/q '[:find ?a ?v3 + :in $ $$ [?a ...] + :where [$$ _ ?a ?e] + [$ ?e ?v _ _] + [$ ?v :db/ident ?v2 _ _] + [(namespace ?v2) ?v3] + [(namespace ?v2) ?v3]] + remote-db + (d/since remote-db #inst "2022-06-01") + (map :db/ident references) + ) + (group-by first) + (map (fn [[k v]] + [k (disj (set (map second v)) "db")])) + (into {}))) + +#_(def manual-dependencies + {:client/location-matches #{"location-match"} + :transaction/yodlee-merchant #{"yodlee-merchant"} + :vendor-account-override/account #{"account"} + :vendor-account-override/client #{"client"} + :vendor/secondary-contact #{"contact"} + :vendor/account-overrides #{"vendor-account-override"} + :client/bank-accounts #{"bank-account"} + :transaction-rule/yodlee-merchant #{"yodlee-merchant"} + :client/forecasted-transactions #{"forecasted-transaction"} + :transaction/forecast-match #{"forecasted-transaction"} + :vendor/automatically-paid-when-due #{"client"} + :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"} + :vendor/terms-overrides #{"vendor-terms-override"} + :vendor-schedule-payment-dom/client #{"client"}}) + +#_(defn full-dependencies [remote-db] + (update (merge-with into (reference->entity remote-db) manual-dependencies) + :journal-entry/original-entity + #(disj % "journal-entry"))) + +#_(defn entity-dependencies [schema] + (let [base-dependencies + (into + {} + (map (fn [i] + [i #{}]) + (set (map (comp namespace :db/ident) + (filter :db/valueType + schema)))) + ) + ] + (into base-dependencies (reduce + (fn [acc [ref deps]] + (update acc (namespace ref) (fnil #(into % deps) #{}))) + {} + (full-dependencies remote-db))))) + +(def full-dependencies + {:invoice/client #{"client"}, + :sales-order/client #{"client"}, + :transaction-rule/transaction-approval-status #{}, + :transaction/forecast-match #{"forecasted-transaction"}, + :user/role #{}, + :vendor-schedule-payment-dom/client #{"client"}, + :invoice-payment/payment #{"payment"}, + :transaction-rule/client #{"client"}, + :invoice/status #{}, + :payment/type #{}, + :expected-deposit/client #{"client"}, + :transaction/bank-account #{"bank-account"}, + :transaction-rule-account/account #{"account"}, + :import-batch/status #{}, + :user/clients #{"client"}, + :payment/client #{"client"}, + :expected-deposit/charges #{"charge"}, + :vendor/automatically-paid-when-due #{"client"}, + :payment/invoices #{"invoice"}, + :client/forecasted-transactions #{"forecasted-transaction"}, + :transaction/matched-rule #{"transaction-rule"}, + :invoice/import-status #{}, + :charge/processor #{}, + :expected-deposit/vendor #{"vendor"}, + :client/square-locations #{"square-location"}, + :payment/status #{}, + :client/location-matches #{"location-match"}, + :saved-query/client #{"client"}, + :transaction/payment #{"payment"}, + :transaction-rule/vendor #{"vendor"}, + :plaid-item/client #{"client"}, + :account/applicability #{}, + :journal-entry-line/account #{"account" "bank-account"}, + :client/bank-accounts #{"bank-account"}, + :yodlee-provider-account/client #{"client"}, + :account/vendor-allowance #{}, + :payment/bank-account #{"bank-account"}, + :account/default-allowance #{}, + :transaction-rule/yodlee-merchant #{"yodlee-merchant"}, + :vendor/account-overrides #{"vendor-account-override"}, + :transaction/client #{"client"}, + :invoice/vendor #{"vendor"}, + :sales-order/vendor #{"vendor"}, + :expected-deposit/status #{}, + :journal-entry/original-entity #{"transaction" "invoice"}, + :vendor-usage/client #{"client"}, + :transaction/expected-deposit #{"expected-deposit"}, + :client/ezcater-locations #{"ezcater-location"}, + :journal-entry/client #{"client"}, + :vendor/secondary-contact #{"contact"}, + :journal-entry/line-items #{"journal-entry-line"}, + :vendor/legal-entity-1099-type #{}, + :transaction-rule/bank-account #{"bank-account"}, + :transaction-account/account #{"account"}, + :vendor/terms-overrides #{"vendor-terms-override"}, + :vendor/default-account #{"account"}, + :transaction/yodlee-merchant #{"yodlee-merchant"}, + :sales-refund/client #{"client"}, + :client/emails #{"email-contact"}, + :payment/vendor #{"vendor"}, + :invoice-payment/invoice #{"invoice"}, + :report/client #{"client"}, + :transaction-rule/accounts #{"transaction-rule-account"}, + :charge/client #{"client"}, + :bank-account/type #{}, + :invoice-expense-account/account #{"account"}, + :vendor/legal-entity-tin-type #{}, + :transaction/approval-status #{}, + :import-batch/entry #{"transaction"}, + :bank-account/intuit-bank-account #{"intuit-bank-account"}, + :account/type #{}, + :sales-refund/vendor #{"vendor"}, + :bank-account/yodlee-account #{"yodlee-account"}, + :vendor/address #{"address"}, + :integration-status/state #{}, + :transaction/accounts #{"transaction-account"}, + :sales-order/charges #{"charge"}, + :client/address #{"address"}, + :ezcater-location/caterer #{"ezcater-caterer"}, + :vendor-account-override/client #{"client"}, + :bank-account/integration-status #{"integration-status"}, + :yodlee-provider-account/accounts #{"yodlee-account"}, + :account/invoice-allowance #{}, + :journal-entry/vendor #{"vendor"}, + :plaid-item/accounts #{"plaid-account"}, + :vendor-usage/vendor #{"vendor"}, + :sales-order/line-items #{"order-line-item"}, + :invoice/expense-accounts #{"invoice-expense-account"}, + :account-client-override/client #{"client"}, + :vendor/primary-contact #{"contact"}, + :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}, + :account/client-overrides #{"account-client-override"}, + :transaction/vendor #{"vendor"}, + :client/square-integration-status #{"integration-status"}, + :ezcater-integration/caterers #{"ezcater-caterer"}, + :vendor-account-override/account #{"account"}, + :import-batch/source #{}}) + +(def entity-dependencies + {"transaction-rule" + #{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account" + "client"}, + "square-location" #{}, + "expected-deposit" #{"vendor" "charge" "client"}, + "journal-entry-line" #{"account" "bank-account"}, + "vendor" + #{"vendor-schedule-payment-dom" "address" "account" "client" "contact" + "vendor-account-override"}, + "transaction" + #{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant" + "transaction-account" "forecasted-transaction" "bank-account" "client" + "payment"}, + "yodlee-provider-account" #{"yodlee-account" "client"}, + "journal-entry" + #{"journal-entry-line" "vendor" "transaction" "invoice" "client"}, + "yodlee-merchant" #{}, + "invoice" #{"vendor" "invoice-expense-account" "client"}, + "vendor-terms-override" #{}, + "integration-status" #{}, + "conformity" #{}, + "user" #{"client"}, + "sales-refund" #{"vendor" "client"}, + "plaid-account" #{}, + "charge" #{"client"}, + "location-match" #{}, + "vendor-schedule-payment-dom" #{"client"}, + "account-client-override" #{"client"}, + "plaid-item" #{"plaid-account" "client"}, + "transaction-account" #{"account"}, + "address" #{}, + "order-line-item" #{}, + "ezcater-location" #{"ezcater-caterer"}, + "account" #{"account-client-override"}, + "intuit-bank-account" #{}, + "saved-query" #{"client"}, + "ezcater-caterer" #{}, + "forecasted-transaction" #{}, + "audit" #{}, + "yodlee-account" #{}, + "transaction-rule-account" #{"account"}, + "ezcater-integration" #{"ezcater-caterer"}, + "report" #{"client"}, + "bank-account" #{"integration-status" "intuit-bank-account" "yodlee-account"}, + "vendor-usage" #{"vendor" "client"}, + "invoice-expense-account" #{"account"}, + "sales-order" #{"vendor" "charge" "order-line-item" "client"}, + "client" + #{"square-location" "integration-status" "location-match" "address" + "ezcater-location" "forecasted-transaction" "bank-account" "email-contact"}, + "email-contact" #{}, + "invoice-payment" #{"invoice" "payment"}, + "contact" #{}, + "import-batch" #{"transaction"}, + "payment" #{"vendor" "invoice" "bank-account" "client"}, + "vendor-account-override" #{"account" "client"}}) + +(defn order-of-insert [entity-dependencies] + (loop [entity-dependencies entity-dependencies + order []] + (let [next-order (for [[entity deps] entity-dependencies + :when (not (seq deps))] + entity) + next-deps (reduce + (fn [entity-dependencies next-entity] + (into {} + (map + (fn [[k v]] + [k (disj v next-entity)]) + entity-dependencies))) + (apply dissoc entity-dependencies next-order) + next-order)] + (if (seq next-deps) + (recur next-deps (into order next-order)) + (into order next-order))))) + + +(def loaded (atom #{})) + + +(def dumped (atom #{})) + +(defn write-s3 [data location] + (spit (io/file "/tmp/temp-edn") + (with-out-str (clojure.pprint/pprint data))) + (s3/put-object :bucket-name (:data-bucket env) + :key location + :input-stream (io/make-input-stream (io/file "/tmp/temp-edn") {}))) + +(defn dump-schema [schema backup] + (write-s3 (map + (fn [s] + (set/rename-keys s {:db/id :entity/migration-key})) + schema) + (str backup "/schema.edn")) + (write-s3 full-dependencies + (str backup "/full-dependencies.edn")) + (write-s3 entity-dependencies + (str backup "/entity-dependencies.edn"))) + +(defn pull-batch [remote-db schema entity entities] + (de/future-with request-pool + (mu/with-context {:entity entity} + (try + (mu/log ::pulling :count (count entities)) + (->> (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= entity (namespace %))) + (into [:db/id])) + entities) + (mapv (fn [m ] + (reduce + (fn [m [k v]] + (cond + (= k :db/id) + (-> m + (assoc :entity/migration-key v) + (dissoc :db/id)) + (full-dependencies k) + (if (vector? v) + (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) + (assoc m k [:entity/migration-key (:db/id v)])) + :else + (dissoc m :payment/pdf-data + :payment/memo))) + m + m)))) + (catch Throwable e + (mu/log ::pull-error + :exception e) + + (throw e)))))) + +(def in-flight (atom 0)) + +(defn dump-all + ([] (dump-all nil)) + ([item-list] + (let [backup-id (str "/datomic-backup/" (java.util.UUID/randomUUID)) + _ (mu/log ::starting-backup :backup backup-id) + remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod")) + _ (mu/log ::fetching-schema) + schema (get-schema remote-db) + ] + (mu/log ::dumping-schema) + (dump-schema schema backup-id) + (mu/log ::schema-dumped) + (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies))) + :let [_ (swap! dumped conj entity) + _ (mu/log ::querying :entity entity) + entities (d/q '[:find [?e ...] + :in $ [?a ...] + :where [?e ?a]] + remote-db + (cond-> (entity->best-key entity) + (not (vector? (entity->best-key entity))) vector)) + + _ (mu/log ::entity-total-found :count (count entities) :entity entity)]] + (mu/trace ::single-entity + [:entity entity] + (mu/with-context {:entity entity} + (mu/log ::starting) + (mu/log ::deleting) + (io/delete-file (io/file "/tmp/tmp-ednl") true) + (mu/log ::pulling) + (ednl/with-append [append "/tmp/tmp-ednl" ] + @(s/consume (fn [batch] + (mu/with-context {:entity entity} + (doseq [a batch] + (mu/log ::appending :count (count batch)) + (try + (append a) + (catch Exception e + (mu/log ::error + :exception e) + (throw e))) + (mu/log ::appended :count (count batch)))) + + ) + (->> (partition-all 100 entities) + (into []) + (s/->source) + (s/onto buffered) + (s/buffer 20) + (s/map (fn [entities] + (pull-batch remote-db schema entity entities))) + (s/buffer 20) + (s/realize-each))) + ) + (mu/log ::copying) + (s3/put-object :bucket-name (:data-bucket env) + :key (str backup-id "/" entity ".ednl") + :input-stream (io/make-input-stream (io/file "/tmp/tmp-ednl") {})) + (mu/log ::copied))))))) + +(defn -main [& _] + (execute "export-backup" #(dump-all))) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 84f79fff..b549bf2e 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -8,6 +8,7 @@ [auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert] [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.intuit :as job-intuit] + [auto-ap.backup :as backup] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] [auto-ap.jobs.load-historical-sales :as job-load-historical-sales] [auto-ap.jobs.plaid :as job-plaid] @@ -137,6 +138,9 @@ (= job "bulk-journal-import") (job-bulk-journal-import/-main) + (= job "export-backup") + (backup/-main) + :else (do diff --git a/src/cljs/auto_ap/views/pages/admin/jobs.cljs b/src/cljs/auto_ap/views/pages/admin/jobs.cljs index dafe6836..cbb5c274 100644 --- a/src/cljs/auto_ap/views/pages/admin/jobs.cljs +++ b/src/cljs/auto_ap/views/pages/admin/jobs.cljs @@ -144,7 +144,8 @@ [:square2-import-job "Square2 Import"] [:register-invoice-import "Register Invoice Import "] [:ezcater-upsert "Upsert recent ezcater orders"] - [:load-historical-sales "Load Historical Square Sales"]] + [:load-historical-sales "Load Historical Square Sales"] + [:export-backup "Export Backup"]] :allow-nil? true :keywordize? true}]] (cond (= :bulk-journal-import (:job data)) diff --git a/terraform/.gitx/HEAD b/terraform/.gitx/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/terraform/.gitx/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/terraform/.gitx/config b/terraform/.gitx/config new file mode 100644 index 00000000..515f4836 --- /dev/null +++ b/terraform/.gitx/config @@ -0,0 +1,5 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true diff --git a/terraform/.gitx/description b/terraform/.gitx/description new file mode 100644 index 00000000..498b267a --- /dev/null +++ b/terraform/.gitx/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/terraform/.gitx/hooks/applypatch-msg.sample b/terraform/.gitx/hooks/applypatch-msg.sample new file mode 100755 index 00000000..a5d7b84a --- /dev/null +++ b/terraform/.gitx/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/terraform/.gitx/hooks/commit-msg.sample b/terraform/.gitx/hooks/commit-msg.sample new file mode 100755 index 00000000..b58d1184 --- /dev/null +++ b/terraform/.gitx/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/terraform/.gitx/hooks/fsmonitor-watchman.sample b/terraform/.gitx/hooks/fsmonitor-watchman.sample new file mode 100755 index 00000000..23e856f5 --- /dev/null +++ b/terraform/.gitx/hooks/fsmonitor-watchman.sample @@ -0,0 +1,174 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 2) and last update token +# formatted as a string and outputs to stdout a new update token and +# all files that have been modified since the update token. Paths must +# be relative to the root of the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $last_update_token) = @ARGV; + +# Uncomment for debugging +# print STDERR "$0 $version $last_update_token\n"; + +# Check the hook interface version +if ($version ne 2) { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree = get_working_dir(); + +my $retry = 1; + +my $json_pkg; +eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; +} or do { + require JSON::PP; + $json_pkg = "JSON::PP"; +}; + +launch_watchman(); + +sub launch_watchman { + my $o = watchman_query(); + if (is_work_tree_watched($o)) { + output_result($o->{clock}, @{$o->{files}}); + } +} + +sub output_result { + my ($clockid, @files) = @_; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # binmode $fh, ":utf8"; + # print $fh "$clockid\n@files\n"; + # close $fh; + + binmode STDOUT, ":utf8"; + print $clockid; + print "\0"; + local $, = "\0"; + print @files; +} + +sub watchman_clock { + my $response = qx/watchman clock "$git_work_tree"/; + die "Failed to get clock id on '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + return $json_pkg->new->utf8->decode($response); +} + +sub watchman_query { + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $last_update_token but not from the .git folder. + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + my $last_update_line = ""; + if (substr($last_update_token, 0, 1) eq "c") { + $last_update_token = "\"$last_update_token\""; + $last_update_line = qq[\n"since": $last_update_token,]; + } + my $query = <<" END"; + ["query", "$git_work_tree", {$last_update_line + "fields": ["name"], + "expression": ["not", ["dirname", ".git"]] + }] + END + + # Uncomment for debugging the watchman query + # open (my $fh, ">", ".git/watchman-query.json"); + # print $fh $query; + # close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + # Uncomment for debugging the watch response + # open ($fh, ">", ".git/watchman-response.json"); + # print $fh $response; + # close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + return $json_pkg->new->utf8->decode($response); +} + +sub is_work_tree_watched { + my ($output) = @_; + my $error = $output->{error}; + if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { + $retry--; + my $response = qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + $output = $json_pkg->new->utf8->decode($response); + $error = $output->{error}; + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # close $fh; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + my $o = watchman_clock(); + $error = $output->{error}; + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + output_result($o->{clock}, ("/")); + $last_update_token = $o->{clock}; + + eval { launch_watchman() }; + return 0; + } + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + return 1; +} + +sub get_working_dir { + my $working_dir; + if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $working_dir = Win32::GetCwd(); + $working_dir =~ tr/\\/\//; + } else { + require Cwd; + $working_dir = Cwd::cwd(); + } + + return $working_dir; +} diff --git a/terraform/.gitx/hooks/post-update.sample b/terraform/.gitx/hooks/post-update.sample new file mode 100755 index 00000000..ec17ec19 --- /dev/null +++ b/terraform/.gitx/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/terraform/.gitx/hooks/pre-applypatch.sample b/terraform/.gitx/hooks/pre-applypatch.sample new file mode 100755 index 00000000..4142082b --- /dev/null +++ b/terraform/.gitx/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/terraform/.gitx/hooks/pre-commit.sample b/terraform/.gitx/hooks/pre-commit.sample new file mode 100755 index 00000000..e144712c --- /dev/null +++ b/terraform/.gitx/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=$(git hash-object -t tree /dev/null) +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --type=bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/terraform/.gitx/hooks/pre-merge-commit.sample b/terraform/.gitx/hooks/pre-merge-commit.sample new file mode 100755 index 00000000..399eab19 --- /dev/null +++ b/terraform/.gitx/hooks/pre-merge-commit.sample @@ -0,0 +1,13 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git merge" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message to +# stderr if it wants to stop the merge commit. +# +# To enable this hook, rename this file to "pre-merge-commit". + +. git-sh-setup +test -x "$GIT_DIR/hooks/pre-commit" && + exec "$GIT_DIR/hooks/pre-commit" +: diff --git a/terraform/.gitx/hooks/pre-push.sample b/terraform/.gitx/hooks/pre-push.sample new file mode 100755 index 00000000..4ce688d3 --- /dev/null +++ b/terraform/.gitx/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/terraform/.gitx/hooks/pre-rebase.sample b/terraform/.gitx/hooks/pre-rebase.sample new file mode 100755 index 00000000..6cbef5c3 --- /dev/null +++ b/terraform/.gitx/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up to date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +<<\DOC_END + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". + +DOC_END diff --git a/terraform/.gitx/hooks/pre-receive.sample b/terraform/.gitx/hooks/pre-receive.sample new file mode 100755 index 00000000..a1fd29ec --- /dev/null +++ b/terraform/.gitx/hooks/pre-receive.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to make use of push options. +# The example simply echoes all push options that start with 'echoback=' +# and rejects all pushes when the "reject" push option is used. +# +# To enable this hook, rename this file to "pre-receive". + +if test -n "$GIT_PUSH_OPTION_COUNT" +then + i=0 + while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" + do + eval "value=\$GIT_PUSH_OPTION_$i" + case "$value" in + echoback=*) + echo "echo from the pre-receive-hook: ${value#*=}" >&2 + ;; + reject) + exit 1 + esac + i=$((i + 1)) + done +fi diff --git a/terraform/.gitx/hooks/prepare-commit-msg.sample b/terraform/.gitx/hooks/prepare-commit-msg.sample new file mode 100755 index 00000000..10fa14c5 --- /dev/null +++ b/terraform/.gitx/hooks/prepare-commit-msg.sample @@ -0,0 +1,42 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first one removes the +# "# Please enter the commit message..." help message. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +COMMIT_MSG_FILE=$1 +COMMIT_SOURCE=$2 +SHA1=$3 + +/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" + +# case "$COMMIT_SOURCE,$SHA1" in +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; +# *) ;; +# esac + +# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" +# if test -z "$COMMIT_SOURCE" +# then +# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" +# fi diff --git a/terraform/.gitx/hooks/push-to-checkout.sample b/terraform/.gitx/hooks/push-to-checkout.sample new file mode 100755 index 00000000..af5a0c00 --- /dev/null +++ b/terraform/.gitx/hooks/push-to-checkout.sample @@ -0,0 +1,78 @@ +#!/bin/sh + +# An example hook script to update a checked-out tree on a git push. +# +# This hook is invoked by git-receive-pack(1) when it reacts to git +# push and updates reference(s) in its repository, and when the push +# tries to update the branch that is currently checked out and the +# receive.denyCurrentBranch configuration variable is set to +# updateInstead. +# +# By default, such a push is refused if the working tree and the index +# of the remote repository has any difference from the currently +# checked out commit; when both the working tree and the index match +# the current commit, they are updated to match the newly pushed tip +# of the branch. This hook is to be used to override the default +# behaviour; however the code below reimplements the default behaviour +# as a starting point for convenient modification. +# +# The hook receives the commit with which the tip of the current +# branch is going to be updated: +commit=$1 + +# It can exit with a non-zero status to refuse the push (when it does +# so, it must not modify the index or the working tree). +die () { + echo >&2 "$*" + exit 1 +} + +# Or it can make any necessary changes to the working tree and to the +# index to bring them to the desired state when the tip of the current +# branch is updated to the new commit, and exit with a zero status. +# +# For example, the hook can simply run git read-tree -u -m HEAD "$1" +# in order to emulate git fetch that is run in the reverse direction +# with git push, as the two-tree form of git read-tree -u -m is +# essentially the same as git switch or git checkout that switches +# branches while keeping the local changes in the working tree that do +# not interfere with the difference between the branches. + +# The below is a more-or-less exact translation to shell of the C code +# for the default behaviour for git's push-to-checkout hook defined in +# the push_to_deploy() function in builtin/receive-pack.c. +# +# Note that the hook will be executed from the repository directory, +# not from the working tree, so if you want to perform operations on +# the working tree, you will have to adapt your code accordingly, e.g. +# by adding "cd .." or using relative paths. + +if ! git update-index -q --ignore-submodules --refresh +then + die "Up-to-date check failed" +fi + +if ! git diff-files --quiet --ignore-submodules -- +then + die "Working directory has unstaged changes" +fi + +# This is a rough translation of: +# +# head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX +if git cat-file -e HEAD 2>/dev/null +then + head=HEAD +else + head=$(git hash-object -t tree --stdin &2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --type=bool hooks.allowunannotated) +allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) +denycreatebranch=$(git config --type=bool hooks.denycreatebranch) +allowdeletetag=$(git config --type=bool hooks.allowdeletetag) +allowmodifytag=$(git config --type=bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero=$(git hash-object --stdin &2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/terraform/.gitx/info/exclude b/terraform/.gitx/info/exclude new file mode 100644 index 00000000..a5196d1b --- /dev/null +++ b/terraform/.gitx/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/terraform/deploy.tf b/terraform/deploy.tf index 7c8dd398..8f5dda8b 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -468,4 +468,16 @@ module "load_historical_sales_job" { use_schedule = false memory = 4096 cpu = 1024 -} \ No newline at end of file +} + +module "export_backup_job" { + source = "./background-job/" + ecs_cluster = var.ecs_cluster + task_role_arn = var.task_role_arn + stage = var.stage + job_name = "export-backup" + execution_role_arn = var.execution_role_arn + use_schedule = false + memory = 8192 + cpu = 4096 +} diff --git a/terraform/main.tf b/terraform/main.tf index 2e030d06..a37676ec 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -82,7 +82,16 @@ resource "aws_s3_bucket" "data" { }, "Resource": "arn:aws:s3:::data.${var.stage}.app.integreatconsult.com", "Sid": "AllowReadForProd" - } + }, + { + "Action": "s3:*", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::679918342773:role/http-proxy" + }, + "Resource": "arn:aws:s3:::data.${var.stage}.app.integreatconsult.com", + "Sid": "AllowReadForProdProxy" + } ] } POLICY diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate b/terraform/terraform.tfstate.d/prod/terraform.tfstate index 7704fb84..7886b6d8 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.3.2", - "serial": 406, + "terraform_version": "1.3.8", + "serial": 411, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:520", "timeouts": { "delete": null }, @@ -669,11 +669,25 @@ "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], - "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}", + "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", - "server_side_encryption_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], "tags": {}, "tags_all": {}, "versioning": [ @@ -693,7 +707,8 @@ "website_domain": "s3-website-us-east-1.amazonaws.com", "website_endpoint": "data.prod.app.integreatconsult.com.s3-website-us-east-1.amazonaws.com" }, - "sensitive_attributes": [] + "sensitive_attributes": [], + "private": "bnVsbA==" } ] }, @@ -725,7 +740,21 @@ "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", - "server_side_encryption_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], "tags": {}, "tags_all": {}, "versioning": [ @@ -739,6 +768,7 @@ "website_endpoint": null }, "sensitive_attributes": [], + "private": "bnVsbA==", "dependencies": [ "data.aws_caller_identity.current" ] @@ -790,7 +820,21 @@ "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", - "server_side_encryption_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], "tags": {}, "tags_all": {}, "versioning": [ @@ -1370,6 +1414,45 @@ } ] }, + { + "module": "module.export_backup_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/export_backup_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"export-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"export-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"export-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "4096", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "export_backup_prod", + "id": "export_backup_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "8192", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.ezcater_upsert_job", "mode": "managed", @@ -1801,7 +1884,7 @@ ], "revision": 1, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -2828,5 +2911,5 @@ ] } ], - "check_results": [] + "check_results": null } diff --git a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup index 68a9dcad..13e8f90c 100644 --- a/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.3.2", - "serial": 343, + "terraform_version": "1.3.8", + "serial": 406, "lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51", "outputs": { "aws_access_key_id": { @@ -164,7 +164,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:413", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod:415", "timeouts": { "delete": null }, @@ -1772,6 +1772,45 @@ } ] }, + { + "module": "module.load_historical_sales_job", + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "background_taskdef", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/load_historical_sales_prod:1", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod\",\"com.datadoghq.tags.service\":\"load-historical-sales\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod\"},{\"name\":\"DD_SERVICE\",\"value\":\"load-historical-sales\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"load-historical-sales\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat:prod\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "load_historical_sales_prod", + "id": "load_historical_sales_prod", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "4096", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 1, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, { "module": "module.plaid_job", "mode": "managed", @@ -2069,7 +2108,7 @@ ], "revision": 1, "runtime_platform": [], - "tags": null, + "tags": {}, "tags_all": {}, "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", "volume": [] @@ -2789,5 +2828,5 @@ ] } ], - "check_results": [] + "check_results": null }