notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Makes background jobs work again

Browse Source
This commit is contained in:
Bryce
2023-10-25 15:35:33 -07:00
parent a2d834794f
commit 506d2ac782
10 changed files with 138 additions and 3860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3757
resources/public/output.css
View File

File diff suppressed because one or more lines are too long

189
src/clj/auto_ap/ssr/admin/background_jobs.clj
View File

@@ -2,24 +2,28 @@
(:require
[amazonica.aws.ecs :as ecs]
[auto-ap.logging :as alog]
[clojure.string :as str]
[auto-ap.routes.utils
:refer [wrap-admin wrap-client-redirect-unauthenticated]]
[auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com]
[auto-ap.ssr.form-cursor :as fc]
[auto-ap.ssr.grid-page-helper :as helper]
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
[auto-ap.ssr.utils
:refer [apply-middleware-to-all-handlers
entity-id
form-validation-error
html-response
validation-error
wrap-form-4xx
modal-response
wrap-form-4xx-2
wrap-schema-decode]]
[auto-ap.time :as atime]
[bidi.bidi :as bidi]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.string :as str]
[config.core :refer [env]])
[malli.core :as mc]
[auto-ap.ssr.hx :as hx])
(:import
(com.amazonaws.services.ecs.model AssignPublicIp)))
@@ -57,8 +61,8 @@
false))
(defn ecs-task->job [task]
{:status (condp = (:last-status task)
{:arn (:task-arn task)
:status (condp = (:last-status task)
"RUNNING" :running
"PENDING" :pending
"PROVISIONING" :pending
@@ -78,13 +82,11 @@
(def grid-page
(helper/build {:id "job-table"
:id-fn :arn
:nav (com/admin-aside-nav)
:fetch-page fetch-page
:action-buttons (fn [request]
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes
:admin-job-start-dialog))
:hx-target "#modal-holder"
:hx-swap "outerHTML"
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes :admin-job-start-dialog))
:color :primary}
"Run job")])
:breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes
@@ -151,86 +153,119 @@
(str "_" (:dd-env env)))
(dissoc form-params :name))]
{:message (str "task " (str new-job) " started.")})
(validation-error "This job is already running")))
(form-validation-error "This job is already running"
:form form-params)))
(defn subform [{{:strs [name]} :query-params }]
(html-response (cond (= "bulk-journal-import" name)
[:div (com/field {:label "Url"}
[:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "ledger-data.csv"
:name "ledger-url"} )])]
(= "register-invoice-import" name)
[:div (com/field {:label "Url"}
(defn subform* [{:keys [name]}]
(into [:div {:class "fade-in-settle transition"}]
(cond (= "bulk-journal-import" name)
[(fc/with-field :ledger-url
(com/validated-field {:label "Url"
:errors (fc/field-errors)}
[:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "ledger-data.csv"
:name (fc/field-name)
:value (fc/field-value)} )]))]
(= "register-invoice-import" name)
[
(fc/with-field :invoice-url
(com/validated-field {:label "Url"
:errors (fc/field-errors)}
[:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "invoice-data.csv"
:name "invoice-url"} )])]
(= "load-historical-sales" name)
[:div
(com/field {:label "Client"}
(com/typeahead {:name "client"
:placeholder "Search..."
:url (bidi/path-for ssr-routes/only-routes
:company-search)
:id (str "client-search")}))
(com/field {:label "Days to load"}
(com/text-input {:placeholder "60"
:name "days"} ))]
:else [:div]))
:name (fc/field-name)
:value (fc/field-value)} )]))]
(= "load-historical-sales" name)
[
(fc/with-field :client
(com/validated-field {:label "Client"
:errors (fc/field-errors)}
(com/typeahead-2 {:name (fc/field-name)
:value (fc/field-value)
:placeholder "Search..."
:url (bidi/path-for ssr-routes/only-routes
:company-search)})))
(fc/with-field :days
(com/validated-field {:label "Days to load"
:errors (fc/field-errors)}
(com/text-input {:placeholder "60"
:name (fc/field-name)
:value (fc/field-value)} )))]
:else nil))
)
(defn job-start-dialog [_]
(html-response (com/modal
{:modal-class "max-w-4xl"}
[:form {:hx-ext "response-targets"
:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start
)
:hx-swap "outerHTML swap:300ms"
:hx-target-400 "#form-errors .error-content"}
[:fieldset {:class "hx-disable"}
(com/modal-card
{}
[:div.flex [:div.p-2 "New job"] ]
[:div.space-y-6
(defn subform [{{:keys [name]} :query-params }]
(html-response
(fc/start-form {} nil
(subform* {:name name}))))
(com/field {:label "Job"}
(com/select {:name "name"
:class "w-64"
:options [["" ""]
["yodlee2" "Yodlee Import"]
["yodlee2-accounts" "Yodlee Account Import"]
["intuit" "Intuit import"]
["plaid" "Plaid import"]
["bulk-journal-import" "Bulk Journal Import"]
["square2-import-job" "Square2 Import"]
["register-invoice-import" "Register Invoice Import "]
["ezcater-upsert" "Upsert recent ezcater orders"]
["load-historical-sales" "Load Historical Square Sales"]
["export-backup" "Export Backup"]]
:hx-get (bidi/path-for ssr-routes/only-routes
:admin-job-subform)
:hx-target "#sub-form"
:hx-swap "innerHTML"}))
(defn job-start-dialog [{:keys [form-errors form-params] :as request}]
(fc/start-form (or form-params {}) form-errors
(modal-response
(com/modal ;; TODO we need a cleaner way to have forms that wrap the whole. In this cas
{}
[:form {:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start)
:class "h-full w-full"
"x-on:htmx:response-error" "unexpectedError=true"
"x-on:htmx:before-request" "unexpectedError=false"
:x-data (hx/json {:unexpectedError false})}
[:fieldset {:class "hx-disable h-full w-full"}
(com/modal-card {}
[:div.m-2 "New job"]
[:div.space-y-6
[:div#sub-form]
[:div#form-errors [:span.error-content]]
(com/button {:color :primary :form "edit-form" :type "submit"}
"Run")]
[:div])]])))
(fc/with-field :name
(com/validated-field {:label "Job"
:errors (fc/field-errors)}
(com/select {:name (fc/field-name)
:value (fc/field-value)
:class "w-64"
:options [["" ""]
["yodlee2" "Yodlee Import"]
["yodlee2-accounts" "Yodlee Account Import"]
["intuit" "Intuit import"]
["plaid" "Plaid import"]
["bulk-journal-import" "Bulk Journal Import"]
["square2-import-job" "Square2 Import"]
["register-invoice-import" "Register Invoice Import "]
["ezcater-upsert" "Upsert recent ezcater orders"]
["load-historical-sales" "Load Historical Square Sales"]
["export-backup" "Export Backup"]]
:hx-get (bidi/path-for ssr-routes/only-routes
:admin-job-subform)
:hx-target "#sub-form"
:hx-swap "innerHTML"})))
[:div#sub-form (subform* {:name (fc/with-field :name (fc/field-value))}) ]]
[:div
[:div#5xx-error.bg-red-100.mb-2.p-1 {:x-show "unexpectedError"}
"An unexpected error has occured."]
(com/form-errors {:errors (:errors fc/*form-errors*)})
(com/validated-save-button {:errors form-errors} "Run job")])]]))))
(def form-schema (mc/schema [:map
[:name [:string {:min 1}]]
[:ledger-url {:optional true} [:string {:min 1}]]
[:invoice-url {:optional true} [:string {:min 1}]]
[:client {:optional true} entity-id]
[:days {:optional true} [:int {:min 1 :max 120}]]
]))
(def key->handler
(apply-middleware-to-all-handlers
(->>
{:admin-jobs (helper/page-route grid-page)
:admin-job-table (helper/table-route grid-page)
:admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name :string]]))
:admin-job-start (-> job-start
(wrap-schema-decode :form-schema [:map [:name :string]])
(wrap-nested-form-params)
(wrap-form-4xx))
:admin-job-start-dialog job-start-dialog})
{:admin-jobs (helper/page-route grid-page)
:admin-job-table (helper/table-route grid-page)
:admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name {:optional true} [:maybe :string]]]))
:admin-job-start (-> job-start
(wrap-schema-decode :form-schema form-schema)
(wrap-nested-form-params)
(wrap-form-4xx-2 job-start-dialog))
:admin-job-start-dialog job-start-dialog})
(fn [h]
(-> h
(wrap-admin)

5
src/clj/auto_ap/ssr/admin/transaction_rules.clj
View File

@@ -397,10 +397,9 @@
:hx-target "this"}
[:form {:hx-ext "response-targets"
:hx-swap "outerHTML swap:300ms"
:hx-target-400 "#form-errors .error-content"
:x-trap "true"
:class "group/form w-full h-full"
:class "w-full h-full"
(if (:db/id entity)
:hx-put
:hx-post) (str (bidi/path-for ssr-routes/only-routes :admin-transaction-rule-edit-save))}
@@ -556,7 +555,7 @@
]]
[:div
(com/form-errors {:errors (:errors fc/*form-errors*)})
(com/validated-save-button {:errors (:errors form-errors)} "Save rule")])])))
(com/validated-save-button {:errors form-errors} "Save rule")])])))
(defn new-account [{{:keys [client-id index]} :query-params}]

9
src/clj/auto_ap/ssr/components/dialog.clj
View File

@@ -1,7 +1,6 @@
(ns auto-ap.ssr.components.dialog
(:require [hiccup2.core :as hiccup]
[auto-ap.ssr.hx :as hx]
[auto-ap.ssr.hiccup-helper :as hh]))
(:require
[auto-ap.ssr.hiccup-helper :as hh]))
(defn modal- [params & children]
[:div (-> params
@@ -12,7 +11,8 @@
(defn modal-card- [params header content footer]
[:div#modal-card (update params
:class (fn [c] (-> c
(or "w-full p-4 h-full")
(or "")
(hh/add-class "w-full p-4 h-full")
)))
[:div {:class "bg-white rounded-lg shadow dark:bg-gray-700 dark:text-white modal-content w-full flex flex-col h-full"}
[:div {:class "flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600 shrink-0"} header]
@@ -22,4 +22,3 @@
content]
(when footer [:div {:class "p-4 shrink-0"} footer])]])
;; fade-in-settle slide-up-settle duration-300 transition-all

10
src/clj/auto_ap/ssr/components/inputs.clj
View File

@@ -84,7 +84,7 @@ c.clearChoices();
:popper nil})
:x-modelable "value.value"
:x-model (:x-model params)
:x-init "popper = Popper.createPopper($refs.input, $refs.dropdown, {placement: 'bottom-start', strategy: 'fixed', modifiers: {name: 'offset', options: {offset: [0, 20]}}})"
:x-init "popper = Popper.createPopper($refs.input, $refs.dropdown, {placement: 'bottom-start', strategy: 'fixed', modifiers: {name: 'offset', options: {offset: [0, 10]}}})"
}
[:a {:class (-> (hh/add-class (or (:class params) "") default-input-classes)
(hh/add-class "cursor-pointer"))
@@ -117,11 +117,11 @@ c.clearChoices();
"x-ref" "dropdown"
"@keydown.escape" "open = false; value = {value: '', label: '' }"
"x-transition:enter" "ease-[cubic-bezier(.3,2.3,.6,1)] duration-200"
"x-transition:enter-start" "!opacity-0 !mt-0"
"x-transition:enter-end" "!opacity-1 !mt-1"
"x-transition:enter-start" "!opacity-0"
"x-transition:enter-end" "!opacity-1"
"x-transition:leave" "ease-out duration-200"
"x-transition:leave-start" "!opacity-1 !mt-1"
"x-transition:leave-end" "!opacity-0 !mt-0"
"x-transition:leave-start" "!opacity-1"
"x-transition:leave-end" "!opacity-0"
"x-show " "open"
"x-trap" "open"
"@click.outside" "open=false;"}

5
src/clj/auto_ap/ssr/components/navbar.clj
View File

@@ -24,10 +24,7 @@
(when (is-admin? identity)
[:button.mt-1.lg:w-96.relative.hidden.lg:block {:class "bg-gray-50 hover:bg-gray-200 dark:hover:bg-gray-700 border border-gray-300 text-gray-900 sm:text-sm rounded-lg focus:ring-primary-500 focus:border-primary-500 w-full pl-10 py-4 pr-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 gap-4 "
:hx-get (bidi/path-for ssr-routes/only-routes
:search)
:hx-target "#modal-holder"
:hx-swap "outerHTML"}
:hx-get (bidi/path-for ssr-routes/only-routes :search)}
[:div {:class "absolute inset-y-0 left-0 flex items-center pl-3 pointer-events-none text-gray-500"}
[:div.w-4.h-4 svg/search]
[:span.ml-2 "Search"]]])

1
src/clj/auto_ap/ssr/components/page.clj
View File

@@ -49,4 +49,5 @@
(into
[:div.p-4]
children)]]
])

8
src/clj/auto_ap/ssr/search.clj
View File

@@ -2,7 +2,7 @@
(:require
[auto-ap.graphql.utils :refer [can-see-client?]]
[auto-ap.solr :as solr]
[auto-ap.ssr.utils :refer [html-response]]
[auto-ap.ssr.utils :refer [html-response modal-response]]
[auto-ap.time :as atime]
[clojure.string :as str]
[com.brunobonacci.mulog :as mu]
@@ -130,11 +130,11 @@
:form (:form-params request))
(if-let [q (get (:form-params request) "q")]
(html-response (search-results* q (:identity request)))
(html-response
(modal-response
(com/modal {}
(com/modal-card {}
(com/modal-card {:class "w-full h-full"}
[:div.p-2 "Search"]
[:div#search.overflow-auto.space-y-6.p-2.h-96
[:div#search.overflow-auto.space-y-6.p-2.w-full
(com/text-input {:id "search-input"
:type "search"

10
src/clj/auto_ap/ssr/transaction/insights.clj
View File

@@ -1,21 +1,21 @@
(ns auto-ap.ssr.transaction.insights
(:require
[auto-ap.client-routes :as client-routes]
[auto-ap.datomic :refer [conn pull-attr visible-clients]]
[auto-ap.datomic :refer [conn visible-clients]]
[auto-ap.rule-matching :refer [spread-cents]]
[auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com]
[auto-ap.ssr.svg :as svg]
[auto-ap.ssr.ui :refer [base-page]]
[auto-ap.ssr.utils :refer [html-response]]
[auto-ap.ssr.utils :refer [html-response modal-response]]
[auto-ap.time :as atime]
[bidi.bidi :as bidi]
[cemerick.url :as url]
[clj-http.client :as http]
[clj-time.coerce :as coerce]
[datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]]
[hiccup2.core :as hiccup]))
[hiccup2.core :as hiccup]
[iol-ion.tx :refer [random-tempid]]))
(def pull-expr [:transaction/description-original
:db/id
@@ -238,7 +238,7 @@
pull-expr
(Long/parseLong transaction-id))
similar (pinecone-similarity-list transaction-id)]
(html-response
(modal-response
(com/modal {}
(com/modal-card {:style {:width "900px"}}
[:div.flex [:div.p-2 "Similar Transactions"]]

4
src/clj/auto_ap/ssr/utils.clj
View File

@@ -322,7 +322,9 @@
(defn path->name2 [k & rest]
(let [k->n (fn [k]
(if (keyword? k)
(str (namespace k) "/" (name k))
(str (when (namespace k)
(str (namespace k) "/"))
(name k))
k))]
(str (k->n k)
(str/join ""