275 lines
14 KiB
Clojure
275 lines
14 KiB
Clojure
(ns auto-ap.ssr.admin.background-jobs
|
|
(:require
|
|
[amazonica.aws.ecs :as ecs]
|
|
[auto-ap.logging :as alog]
|
|
[auto-ap.routes.utils
|
|
:refer [wrap-admin wrap-client-redirect-unauthenticated]]
|
|
[auto-ap.ssr-routes :as ssr-routes]
|
|
[auto-ap.ssr.components :as com]
|
|
[auto-ap.ssr.form-cursor :as fc]
|
|
[auto-ap.ssr.grid-page-helper :as helper]
|
|
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
|
|
[auto-ap.ssr.utils
|
|
:refer [apply-middleware-to-all-handlers
|
|
entity-id
|
|
form-validation-error
|
|
html-response
|
|
modal-response
|
|
wrap-form-4xx-2
|
|
wrap-schema-enforce]]
|
|
[auto-ap.time :as atime]
|
|
[bidi.bidi :as bidi]
|
|
[clj-time.coerce :as coerce]
|
|
[clj-time.core :as time]
|
|
[clojure.string :as str]
|
|
[config.core :refer [env]]
|
|
[malli.core :as mc])
|
|
(:import
|
|
(com.amazonaws.services.ecs.model AssignPublicIp)))
|
|
|
|
(defn get-ecs-tasks []
|
|
(->>
|
|
(concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50)))
|
|
(ecs/describe-tasks :include [] :tasks)
|
|
:tasks
|
|
(map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %)))))
|
|
(sort-by :created-at)
|
|
reverse))
|
|
|
|
|
|
(defn is-background-job?
|
|
"This function checks whether a given task is a background job.
|
|
It does this by checking the environment of the task's container definitions for an environment variable
|
|
with the name 'INTEGREAT_JOB'. If such a variable exists, the function returns true, otherwise, it returns false.
|
|
Parameters: task - a map representing the task to be checked.
|
|
Returns: true if the task is a background job, false otherwise."
|
|
[task]
|
|
(->> task
|
|
:task-definition
|
|
:container-definitions
|
|
(mapcat :environment)
|
|
(filter (comp #{"INTEGREAT_JOB"} :name))
|
|
seq))
|
|
(defn task-definition->job-name [task-definition]
|
|
(->> (:container-definitions task-definition)
|
|
(mapcat :environment)
|
|
(filter (comp #{"INTEGREAT_JOB"} :name))
|
|
(map :value)
|
|
first))
|
|
|
|
(defn job-exited-successfully? [task]
|
|
(if (= 0 (->> task
|
|
:containers
|
|
(filter (comp #{"integreat-app" } :name))
|
|
(first)
|
|
:exit-code))
|
|
true
|
|
false))
|
|
|
|
(defn ecs-task->job [task]
|
|
{:arn (:task-arn task)
|
|
:status (condp = (:last-status task)
|
|
"RUNNING" :running
|
|
"PENDING" :pending
|
|
"PROVISIONING" :pending
|
|
"DEPROVISIONING" :running
|
|
"STOPPED" (if (job-exited-successfully? task)
|
|
:succeeded
|
|
:failed))
|
|
:name (task-definition->job-name (:task-definition task))
|
|
:end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))
|
|
:start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))})
|
|
|
|
(defn fetch-page [request]
|
|
(let [jobs (->> (get-ecs-tasks)
|
|
(filter is-background-job?)
|
|
(map ecs-task->job))]
|
|
[jobs (count jobs)]))
|
|
|
|
(def grid-page
|
|
(helper/build {:id "job-table"
|
|
:id-fn :arn
|
|
:nav com/admin-aside-nav
|
|
:fetch-page fetch-page
|
|
:action-buttons (fn [request]
|
|
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes :admin-job-start-dialog))
|
|
:color :primary}
|
|
"Run job")])
|
|
:breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes
|
|
:admin)}
|
|
"Admin"]
|
|
|
|
[:a {:href (bidi/path-for ssr-routes/only-routes
|
|
:admin-jobs)}
|
|
"Background Jobs"]]
|
|
:title "Jobs"
|
|
:entity-name "Job"
|
|
:route :admin-job-table
|
|
:headers [
|
|
{:key "start"
|
|
:name "Start"
|
|
:render #(some-> % :start-date (atime/unparse-local atime/standard-time))}
|
|
{:key "end"
|
|
:name "End"
|
|
:render #(some-> % :end-date (atime/unparse-local atime/standard-time))}
|
|
{:key "duration"
|
|
:name "Duration"
|
|
:render (fn [e]
|
|
(when (and (:start-date e)
|
|
(:end-date e))
|
|
(str (time/in-minutes (time/interval
|
|
(:start-date e)
|
|
(:end-date e))) " minutes")))}
|
|
{:key "name"
|
|
:name "Name"
|
|
:render :name}
|
|
{:key "status"
|
|
:name "Status"
|
|
:render :status}]}))
|
|
|
|
(def row* (partial helper/row* grid-page))
|
|
(def table* (partial helper/table* grid-page))
|
|
|
|
(defn currently-running-jobs []
|
|
(->> (get-ecs-tasks)
|
|
(filter is-background-job?)
|
|
(map ecs-task->job)
|
|
(filter (comp #{:pending :running} :status))
|
|
(map :name)
|
|
set))
|
|
|
|
(defn run-task [task args]
|
|
(alog/info ::starting :task task)
|
|
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
|
|
:count 1
|
|
:cluster "default"
|
|
:enable-ecs-managed-tags true
|
|
:task-definition task
|
|
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
|
|
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
|
|
:assign-public-ip AssignPublicIp/ENABLED}}}
|
|
args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
|
|
|
|
(defn job-start [{:keys [form-params]}]
|
|
(if (not (get (currently-running-jobs) (:name form-params)))
|
|
(let [new-job (run-task
|
|
(-> (:name form-params)
|
|
(str/replace #"-" "_")
|
|
(str/replace #":" "")
|
|
(str "_" (:dd-env env)))
|
|
(dissoc form-params :name))]
|
|
{:message (str "task " (str new-job) " started.")})
|
|
(form-validation-error "This job is already running"
|
|
:form form-params)))
|
|
|
|
(defn subform* [{:keys [name]}]
|
|
(into [:div {:class "fade-in-settle transition"}]
|
|
(cond (= "bulk-journal-import" name)
|
|
[(fc/with-field :ledger-url
|
|
(com/validated-field {:label "Url"
|
|
:errors (fc/field-errors)}
|
|
[:div.flex.place-items-center.gap-2
|
|
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
|
|
(com/text-input {:placeholder "ledger-data.csv"
|
|
:name (fc/field-name)
|
|
:value (fc/field-value)} )]))]
|
|
(= "register-invoice-import" name)
|
|
[
|
|
(fc/with-field :invoice-url
|
|
(com/validated-field {:label "Url"
|
|
:errors (fc/field-errors)}
|
|
[:div.flex.place-items-center.gap-2
|
|
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
|
|
(com/text-input {:placeholder "invoice-data.csv"
|
|
:name (fc/field-name)
|
|
:value (fc/field-value)} )]))]
|
|
(= "load-historical-sales" name)
|
|
[
|
|
(fc/with-field :client
|
|
(com/validated-field {:label "Client"
|
|
:errors (fc/field-errors)}
|
|
(com/typeahead {:name (fc/field-name)
|
|
:value (fc/field-value)
|
|
:placeholder "Search..."
|
|
:url (bidi/path-for ssr-routes/only-routes
|
|
:company-search)})))
|
|
(fc/with-field :days
|
|
(com/validated-field {:label "Days to load"
|
|
:errors (fc/field-errors)}
|
|
(com/text-input {:placeholder "60"
|
|
:name (fc/field-name)
|
|
:value (fc/field-value)} )))]
|
|
:else nil))
|
|
|
|
|
|
)
|
|
|
|
(defn subform [{{:keys [name]} :query-params }]
|
|
(html-response
|
|
(fc/start-form {} nil
|
|
(subform* {:name name}))))
|
|
|
|
(defn job-start-dialog [{:keys [form-errors form-params] :as request}]
|
|
(fc/start-form (or form-params {}) form-errors
|
|
(modal-response
|
|
(com/modal ;; TODO we need a cleaner way to have forms that wrap the whole. In this cas
|
|
{}
|
|
[:form {:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start)
|
|
:class "h-full w-full"}
|
|
[:fieldset {:class "hx-disable h-full w-full"}
|
|
(com/modal-card {}
|
|
[:div.m-2 "New job"]
|
|
[:div.space-y-6
|
|
|
|
(fc/with-field :name
|
|
(com/validated-field {:label "Job"
|
|
:errors (fc/field-errors)}
|
|
(com/select {:name (fc/field-name)
|
|
:value (fc/field-value)
|
|
:class "w-64"
|
|
:options [["" ""]
|
|
["yodlee2" "Yodlee Import"]
|
|
["yodlee2-accounts" "Yodlee Account Import"]
|
|
["intuit" "Intuit import"]
|
|
["plaid" "Plaid import"]
|
|
["bulk-journal-import" "Bulk Journal Import"]
|
|
["square2-import-job" "Square2 Import"]
|
|
["register-invoice-import" "Register Invoice Import "]
|
|
["ezcater-upsert" "Upsert recent ezcater orders"]
|
|
["load-historical-sales" "Load Historical Square Sales"]
|
|
["export-backup" "Export Backup"]]
|
|
:hx-get (bidi/path-for ssr-routes/only-routes
|
|
:admin-job-subform)
|
|
:hx-target "#sub-form"
|
|
:hx-swap "innerHTML"})))
|
|
|
|
[:div#sub-form (subform* {:name (fc/with-field :name (fc/field-value))}) ]]
|
|
[:div
|
|
|
|
(com/form-errors {:errors (:errors fc/*form-errors*)})
|
|
(com/validated-save-button {:errors form-errors} "Run job")])]]))))
|
|
|
|
(def form-schema (mc/schema [:map
|
|
[:name [:string {:min 1}]]
|
|
[:ledger-url {:optional true} [:string {:min 1}]]
|
|
[:invoice-url {:optional true} [:string {:min 1}]]
|
|
[:client {:optional true} entity-id]
|
|
[:days {:optional true} [:int {:min 1 :max 120}]]
|
|
]))
|
|
|
|
(def key->handler
|
|
(apply-middleware-to-all-handlers
|
|
(->>
|
|
{:admin-jobs (helper/page-route grid-page)
|
|
:admin-job-table (helper/table-route grid-page)
|
|
:admin-job-subform (-> subform (wrap-schema-enforce :query-schema [:map [:name {:optional true} [:maybe :string]]]))
|
|
:admin-job-start (-> job-start
|
|
(wrap-schema-enforce :form-schema form-schema)
|
|
(wrap-nested-form-params)
|
|
(wrap-form-4xx-2 job-start-dialog))
|
|
:admin-job-start-dialog job-start-dialog})
|
|
(fn [h]
|
|
(-> h
|
|
(wrap-admin)
|
|
(wrap-client-redirect-unauthenticated)))))
|