Makes pnl work for companies with parenthesis
This commit is contained in:
237
src/clj/auto_ap/ssr/admin/background_jobs.clj
Normal file
237
src/clj/auto_ap/ssr/admin/background_jobs.clj
Normal file
@@ -0,0 +1,237 @@
|
||||
(ns auto-ap.ssr.admin.background-jobs
|
||||
(:require
|
||||
[amazonica.aws.ecs :as ecs]
|
||||
[auto-ap.logging :as alog]
|
||||
[auto-ap.routes.utils
|
||||
:refer [wrap-admin wrap-client-redirect-unauthenticated]]
|
||||
[auto-ap.ssr-routes :as ssr-routes]
|
||||
[auto-ap.ssr.components :as com]
|
||||
[auto-ap.ssr.grid-page-helper :as helper]
|
||||
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
|
||||
[auto-ap.ssr.utils
|
||||
:refer [apply-middleware-to-all-handlers
|
||||
html-response
|
||||
validation-error
|
||||
wrap-form-4xx
|
||||
wrap-schema-decode]]
|
||||
[auto-ap.time :as atime]
|
||||
[bidi.bidi :as bidi]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clojure.string :as str]
|
||||
[config.core :refer [env]])
|
||||
(:import
|
||||
(com.amazonaws.services.ecs.model AssignPublicIp)))
|
||||
|
||||
(defn get-ecs-tasks []
|
||||
(->>
|
||||
(concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50)))
|
||||
(ecs/describe-tasks :include [] :tasks)
|
||||
:tasks
|
||||
(map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %)))))
|
||||
(sort-by :created-at)
|
||||
reverse))
|
||||
|
||||
(defn is-background-job? [task]
|
||||
(->> task
|
||||
:task-definition
|
||||
:container-definitions
|
||||
(mapcat :environment)
|
||||
(filter (comp #{"INTEGREAT_JOB"} :name))
|
||||
seq))
|
||||
|
||||
(defn task-definition->job-name [task-definition]
|
||||
(->> (:container-definitions task-definition)
|
||||
(mapcat :environment)
|
||||
(filter (comp #{"INTEGREAT_JOB"} :name))
|
||||
(map :value)
|
||||
first))
|
||||
|
||||
(defn job-exited-successfully? [task]
|
||||
(if (= 0 (->> task
|
||||
:containers
|
||||
(filter (comp #{"integreat-app" } :name))
|
||||
(first)
|
||||
:exit-code))
|
||||
true
|
||||
false))
|
||||
|
||||
(defn ecs-task->job [task]
|
||||
|
||||
{:status (condp = (:last-status task)
|
||||
"RUNNING" :running
|
||||
"PENDING" :pending
|
||||
"PROVISIONING" :pending
|
||||
"DEPROVISIONING" :running
|
||||
"STOPPED" (if (job-exited-successfully? task)
|
||||
:succeeded
|
||||
:failed))
|
||||
:name (task-definition->job-name (:task-definition task))
|
||||
:end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))
|
||||
:start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))})
|
||||
|
||||
(defn fetch-page [request]
|
||||
(let [jobs (->> (get-ecs-tasks)
|
||||
(filter is-background-job?)
|
||||
(map ecs-task->job))]
|
||||
[jobs (count jobs)]))
|
||||
|
||||
(def grid-page
|
||||
(helper/build {:id "job-table"
|
||||
:nav (com/admin-aside-nav)
|
||||
:fetch-page fetch-page
|
||||
:action-buttons (fn [request]
|
||||
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes
|
||||
:admin-job-start-dialog))
|
||||
:hx-target "#modal-holder"
|
||||
:hx-swap "outerHTML"
|
||||
:color :primary}
|
||||
"Run job")])
|
||||
:breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes
|
||||
:admin)}
|
||||
"Admin"]
|
||||
|
||||
[:a {:href (bidi/path-for ssr-routes/only-routes
|
||||
:admin-jobs)}
|
||||
"Background Jobs"]]
|
||||
:title "Jobs"
|
||||
:entity-name "Job"
|
||||
:route :admin-job-table
|
||||
:headers [
|
||||
{:key "start"
|
||||
:name "Start"
|
||||
:render #(some-> % :start-date (atime/unparse-local atime/standard-time))}
|
||||
{:key "end"
|
||||
:name "End"
|
||||
:render #(some-> % :end-date (atime/unparse-local atime/standard-time))}
|
||||
{:key "duration"
|
||||
:name "Duration"
|
||||
:render (fn [e]
|
||||
(when (and (:start-date e)
|
||||
(:end-date e))
|
||||
(str (time/in-minutes (time/interval
|
||||
(:start-date e)
|
||||
(:end-date e))) " minutes")))}
|
||||
{:key "name"
|
||||
:name "Name"
|
||||
:render :name}
|
||||
{:key "status"
|
||||
:name "Status"
|
||||
:render :status}]}))
|
||||
|
||||
(def row* (partial helper/row* grid-page))
|
||||
(def table* (partial helper/table* grid-page))
|
||||
|
||||
(defn currently-running-jobs []
|
||||
(->> (get-ecs-tasks)
|
||||
(filter is-background-job?)
|
||||
(map ecs-task->job)
|
||||
(filter (comp #{:pending :running} :status))
|
||||
(map :name)
|
||||
set))
|
||||
|
||||
(defn run-task [task args]
|
||||
(alog/info ::starting :task task)
|
||||
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
|
||||
:count 1
|
||||
:cluster "default"
|
||||
:enable-ecs-managed-tags true
|
||||
:task-definition task
|
||||
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
|
||||
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
|
||||
:assign-public-ip AssignPublicIp/ENABLED}}}
|
||||
args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
|
||||
|
||||
(defn job-start [{:keys [form-params]}]
|
||||
(if (not (get (currently-running-jobs) (:name form-params)))
|
||||
(let [new-job (run-task
|
||||
(-> (:name form-params)
|
||||
(str/replace #"-" "_")
|
||||
(str/replace #":" "")
|
||||
(str "_" (:dd-env env)))
|
||||
(dissoc form-params :name))]
|
||||
{:message (str "task " (str new-job) " started.")})
|
||||
(validation-error "This job is already running")))
|
||||
|
||||
(defn subform [{{:strs [name]} :query-params }]
|
||||
(html-response (cond (= "bulk-journal-import" name)
|
||||
[:div (com/field {:label "Url"}
|
||||
[:div.flex.place-items-center.gap-2
|
||||
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
|
||||
(com/text-input {:placeholder "ledger-data.csv"
|
||||
:name "ledger-url"} )])]
|
||||
(= "register-invoice-import" name)
|
||||
[:div (com/field {:label "Url"}
|
||||
[:div.flex.place-items-center.gap-2
|
||||
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
|
||||
(com/text-input {:placeholder "invoice-data.csv"
|
||||
:name "invoice-url"} )])]
|
||||
(= "load-historical-sales" name)
|
||||
[:div
|
||||
(com/field {:label "Client"}
|
||||
(com/typeahead {:name "client"
|
||||
:placeholder "Search..."
|
||||
:url (bidi/path-for ssr-routes/only-routes
|
||||
:company-search)
|
||||
:id (str "client-search")}))
|
||||
(com/field {:label "Days to load"}
|
||||
(com/text-input {:placeholder "60"
|
||||
:name "days"} ))]
|
||||
:else [:div]))
|
||||
|
||||
)
|
||||
|
||||
(defn job-start-dialog [_]
|
||||
(html-response (com/modal
|
||||
{:modal-class "max-w-4xl"}
|
||||
[:form#edit-form {:hx-ext "response-targets"
|
||||
:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start
|
||||
)
|
||||
:hx-swap "outerHTML swap:300ms"
|
||||
:hx-target-400 "#form-errors .error-content"}
|
||||
[:fieldset {:class "hx-disable"}
|
||||
(com/modal-card
|
||||
{}
|
||||
[:div.flex [:div.p-2 "New job"] ]
|
||||
[:div.space-y-6
|
||||
|
||||
(com/field {:label "Job"}
|
||||
(com/select {:name "name"
|
||||
:class "w-64"
|
||||
:options [["" ""]
|
||||
["yodlee2" "Yodlee Import"]
|
||||
["yodlee2-accounts" "Yodlee Account Import"]
|
||||
["intuit" "Intuit import"]
|
||||
["plaid" "Plaid import"]
|
||||
["bulk-journal-import" "Bulk Journal Import"]
|
||||
["square2-import-job" "Square2 Import"]
|
||||
["register-invoice-import" "Register Invoice Import "]
|
||||
["ezcater-upsert" "Upsert recent ezcater orders"]
|
||||
["load-historical-sales" "Load Historical Square Sales"]
|
||||
["export-backup" "Export Backup"]]
|
||||
:hx-get (bidi/path-for ssr-routes/only-routes
|
||||
:admin-job-subform)
|
||||
:hx-target "#sub-form"
|
||||
:hx-swap "innerHTML"}))
|
||||
|
||||
[:div#sub-form]
|
||||
[:div#form-errors [:span.error-content]]
|
||||
(com/button {:color :primary :form "edit-form" :type "submit"}
|
||||
"Run")]
|
||||
[:div])]])))
|
||||
|
||||
(def key->handler
|
||||
(apply-middleware-to-all-handlers
|
||||
(->>
|
||||
{:admin-jobs (helper/page-route grid-page)
|
||||
:admin-job-table (helper/table-route grid-page)
|
||||
:admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name :string]]))
|
||||
:admin-job-start (-> job-start
|
||||
(wrap-schema-decode :form-schema [:map [:name :string]])
|
||||
(wrap-nested-form-params)
|
||||
(wrap-form-4xx))
|
||||
:admin-job-start-dialog job-start-dialog})
|
||||
(fn [h]
|
||||
(-> h
|
||||
(wrap-admin)
|
||||
(wrap-client-redirect-unauthenticated)))))
|
||||
Reference in New Issue
Block a user