notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Makes background jobs work again

Browse Source
This commit is contained in:
Bryce
2023-10-25 15:35:33 -07:00
parent a2d834794f
commit 506d2ac782
10 changed files with 138 additions and 3860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3757
resources/public/output.css
View File

File diff suppressed because one or more lines are too long

189
src/clj/auto_ap/ssr/admin/background_jobs.clj
View File

@@ -2,24 +2,28 @@
(:require (:require
[amazonica.aws.ecs :as ecs] [amazonica.aws.ecs :as ecs]
[auto-ap.logging :as alog] [auto-ap.logging :as alog]
[clojure.string :as str]
[auto-ap.routes.utils [auto-ap.routes.utils
:refer [wrap-admin wrap-client-redirect-unauthenticated]] :refer [wrap-admin wrap-client-redirect-unauthenticated]]
[auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com] [auto-ap.ssr.components :as com]
[auto-ap.ssr.form-cursor :as fc]
[auto-ap.ssr.grid-page-helper :as helper] [auto-ap.ssr.grid-page-helper :as helper]
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]] [auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
[auto-ap.ssr.utils [auto-ap.ssr.utils
:refer [apply-middleware-to-all-handlers :refer [apply-middleware-to-all-handlers
entity-id
form-validation-error
html-response html-response
validation-error modal-response
wrap-form-4xx wrap-form-4xx-2
wrap-schema-decode]] wrap-schema-decode]]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.string :as str] [malli.core :as mc]
[config.core :refer [env]]) [auto-ap.ssr.hx :as hx])
(:import (:import
(com.amazonaws.services.ecs.model AssignPublicIp))) (com.amazonaws.services.ecs.model AssignPublicIp)))
@@ -57,8 +61,8 @@
false)) false))
(defn ecs-task->job [task] (defn ecs-task->job [task]
{:arn (:task-arn task)
{:status (condp = (:last-status task) :status (condp = (:last-status task)
"RUNNING" :running "RUNNING" :running
"PENDING" :pending "PENDING" :pending
"PROVISIONING" :pending "PROVISIONING" :pending
@@ -78,13 +82,11 @@
(def grid-page (def grid-page
(helper/build {:id "job-table" (helper/build {:id "job-table"
:id-fn :arn
:nav (com/admin-aside-nav) :nav (com/admin-aside-nav)
:fetch-page fetch-page :fetch-page fetch-page
:action-buttons (fn [request] :action-buttons (fn [request]
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes [(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes :admin-job-start-dialog))
:admin-job-start-dialog))
:hx-target "#modal-holder"
:hx-swap "outerHTML"
:color :primary} :color :primary}
"Run job")]) "Run job")])
:breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes :breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes
@@ -151,86 +153,119 @@
(str "_" (:dd-env env))) (str "_" (:dd-env env)))
(dissoc form-params :name))] (dissoc form-params :name))]
{:message (str "task " (str new-job) " started.")}) {:message (str "task " (str new-job) " started.")})
(validation-error "This job is already running"))) (form-validation-error "This job is already running"
:form form-params)))
(defn subform [{{:strs [name]} :query-params }] (defn subform* [{:keys [name]}]
(html-response (cond (= "bulk-journal-import" name) (into [:div {:class "fade-in-settle transition"}]
[:div (com/field {:label "Url"} (cond (= "bulk-journal-import" name)
[:div.flex.place-items-center.gap-2 [(fc/with-field :ledger-url
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"] (com/validated-field {:label "Url"
(com/text-input {:placeholder "ledger-data.csv" :errors (fc/field-errors)}
:name "ledger-url"} )])] [:div.flex.place-items-center.gap-2
(= "register-invoice-import" name) [:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
[:div (com/field {:label "Url"} (com/text-input {:placeholder "ledger-data.csv"
:name (fc/field-name)
:value (fc/field-value)} )]))]
(= "register-invoice-import" name)
[
(fc/with-field :invoice-url
(com/validated-field {:label "Url"
:errors (fc/field-errors)}
[:div.flex.place-items-center.gap-2 [:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"] [:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "invoice-data.csv" (com/text-input {:placeholder "invoice-data.csv"
:name "invoice-url"} )])] :name (fc/field-name)
(= "load-historical-sales" name) :value (fc/field-value)} )]))]
[:div (= "load-historical-sales" name)
(com/field {:label "Client"} [
(com/typeahead {:name "client" (fc/with-field :client
:placeholder "Search..." (com/validated-field {:label "Client"
:url (bidi/path-for ssr-routes/only-routes :errors (fc/field-errors)}
:company-search) (com/typeahead-2 {:name (fc/field-name)
:id (str "client-search")})) :value (fc/field-value)
(com/field {:label "Days to load"} :placeholder "Search..."
(com/text-input {:placeholder "60" :url (bidi/path-for ssr-routes/only-routes
:name "days"} ))] :company-search)})))
:else [:div])) (fc/with-field :days
(com/validated-field {:label "Days to load"
:errors (fc/field-errors)}
(com/text-input {:placeholder "60"
:name (fc/field-name)
:value (fc/field-value)} )))]
:else nil))
) )
(defn job-start-dialog [_] (defn subform [{{:keys [name]} :query-params }]
(html-response (com/modal (html-response
{:modal-class "max-w-4xl"} (fc/start-form {} nil
[:form {:hx-ext "response-targets" (subform* {:name name}))))
:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start
)
:hx-swap "outerHTML swap:300ms"
:hx-target-400 "#form-errors .error-content"}
[:fieldset {:class "hx-disable"}
(com/modal-card
{}
[:div.flex [:div.p-2 "New job"] ]
[:div.space-y-6
(com/field {:label "Job"} (defn job-start-dialog [{:keys [form-errors form-params] :as request}]
(com/select {:name "name" (fc/start-form (or form-params {}) form-errors
:class "w-64" (modal-response
:options [["" ""] (com/modal ;; TODO we need a cleaner way to have forms that wrap the whole. In this cas
["yodlee2" "Yodlee Import"] {}
["yodlee2-accounts" "Yodlee Account Import"] [:form {:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start)
["intuit" "Intuit import"] :class "h-full w-full"
["plaid" "Plaid import"] "x-on:htmx:response-error" "unexpectedError=true"
["bulk-journal-import" "Bulk Journal Import"] "x-on:htmx:before-request" "unexpectedError=false"
["square2-import-job" "Square2 Import"] :x-data (hx/json {:unexpectedError false})}
["register-invoice-import" "Register Invoice Import "] [:fieldset {:class "hx-disable h-full w-full"}
["ezcater-upsert" "Upsert recent ezcater orders"] (com/modal-card {}
["load-historical-sales" "Load Historical Square Sales"] [:div.m-2 "New job"]
["export-backup" "Export Backup"]] [:div.space-y-6
:hx-get (bidi/path-for ssr-routes/only-routes
:admin-job-subform)
:hx-target "#sub-form"
:hx-swap "innerHTML"}))
[:div#sub-form] (fc/with-field :name
[:div#form-errors [:span.error-content]] (com/validated-field {:label "Job"
(com/button {:color :primary :form "edit-form" :type "submit"} :errors (fc/field-errors)}
"Run")] (com/select {:name (fc/field-name)
[:div])]]))) :value (fc/field-value)
:class "w-64"
:options [["" ""]
["yodlee2" "Yodlee Import"]
["yodlee2-accounts" "Yodlee Account Import"]
["intuit" "Intuit import"]
["plaid" "Plaid import"]
["bulk-journal-import" "Bulk Journal Import"]
["square2-import-job" "Square2 Import"]
["register-invoice-import" "Register Invoice Import "]
["ezcater-upsert" "Upsert recent ezcater orders"]
["load-historical-sales" "Load Historical Square Sales"]
["export-backup" "Export Backup"]]
:hx-get (bidi/path-for ssr-routes/only-routes
:admin-job-subform)
:hx-target "#sub-form"
:hx-swap "innerHTML"})))
[:div#sub-form (subform* {:name (fc/with-field :name (fc/field-value))}) ]]
[:div
[:div#5xx-error.bg-red-100.mb-2.p-1 {:x-show "unexpectedError"}
"An unexpected error has occured."]
(com/form-errors {:errors (:errors fc/*form-errors*)})
(com/validated-save-button {:errors form-errors} "Run job")])]]))))
(def form-schema (mc/schema [:map
[:name [:string {:min 1}]]
[:ledger-url {:optional true} [:string {:min 1}]]
[:invoice-url {:optional true} [:string {:min 1}]]
[:client {:optional true} entity-id]
[:days {:optional true} [:int {:min 1 :max 120}]]
]))
(def key->handler (def key->handler
(apply-middleware-to-all-handlers (apply-middleware-to-all-handlers
(->> (->>
{:admin-jobs (helper/page-route grid-page) {:admin-jobs (helper/page-route grid-page)
:admin-job-table (helper/table-route grid-page) :admin-job-table (helper/table-route grid-page)
:admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name :string]])) :admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name {:optional true} [:maybe :string]]]))
:admin-job-start (-> job-start :admin-job-start (-> job-start
(wrap-schema-decode :form-schema [:map [:name :string]]) (wrap-schema-decode :form-schema form-schema)
(wrap-nested-form-params) (wrap-nested-form-params)
(wrap-form-4xx)) (wrap-form-4xx-2 job-start-dialog))
:admin-job-start-dialog job-start-dialog}) :admin-job-start-dialog job-start-dialog})
(fn [h] (fn [h]
(-> h (-> h
(wrap-admin) (wrap-admin)

5
src/clj/auto_ap/ssr/admin/transaction_rules.clj
View File

@@ -397,10 +397,9 @@
:hx-target "this"} :hx-target "this"}
[:form {:hx-ext "response-targets" [:form {:hx-ext "response-targets"
:hx-swap "outerHTML swap:300ms"
:hx-target-400 "#form-errors .error-content" :hx-target-400 "#form-errors .error-content"
:x-trap "true" :x-trap "true"
:class "group/form w-full h-full" :class "w-full h-full"
(if (:db/id entity) (if (:db/id entity)
:hx-put :hx-put
:hx-post) (str (bidi/path-for ssr-routes/only-routes :admin-transaction-rule-edit-save))} :hx-post) (str (bidi/path-for ssr-routes/only-routes :admin-transaction-rule-edit-save))}
@@ -556,7 +555,7 @@
]] ]]
[:div [:div
(com/form-errors {:errors (:errors fc/*form-errors*)}) (com/form-errors {:errors (:errors fc/*form-errors*)})
(com/validated-save-button {:errors (:errors form-errors)} "Save rule")])]))) (com/validated-save-button {:errors form-errors} "Save rule")])])))
(defn new-account [{{:keys [client-id index]} :query-params}] (defn new-account [{{:keys [client-id index]} :query-params}]

9
src/clj/auto_ap/ssr/components/dialog.clj
View File

@@ -1,7 +1,6 @@
(ns auto-ap.ssr.components.dialog (ns auto-ap.ssr.components.dialog
(:require [hiccup2.core :as hiccup] (:require
[auto-ap.ssr.hx :as hx] [auto-ap.ssr.hiccup-helper :as hh]))
[auto-ap.ssr.hiccup-helper :as hh]))
(defn modal- [params & children] (defn modal- [params & children]
[:div (-> params [:div (-> params
@@ -12,7 +11,8 @@
(defn modal-card- [params header content footer] (defn modal-card- [params header content footer]
[:div#modal-card (update params [:div#modal-card (update params
:class (fn [c] (-> c :class (fn [c] (-> c
(or "w-full p-4 h-full") (or "")
(hh/add-class "w-full p-4 h-full")
))) )))
[:div {:class "bg-white rounded-lg shadow dark:bg-gray-700 dark:text-white modal-content w-full flex flex-col h-full"} [:div {:class "bg-white rounded-lg shadow dark:bg-gray-700 dark:text-white modal-content w-full flex flex-col h-full"}
[:div {:class "flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600 shrink-0"} header] [:div {:class "flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600 shrink-0"} header]
@@ -22,4 +22,3 @@
content] content]
(when footer [:div {:class "p-4 shrink-0"} footer])]]) (when footer [:div {:class "p-4 shrink-0"} footer])]])
;; fade-in-settle slide-up-settle duration-300 transition-all

10
src/clj/auto_ap/ssr/components/inputs.clj
View File

@@ -84,7 +84,7 @@ c.clearChoices();
:popper nil}) :popper nil})
:x-modelable "value.value" :x-modelable "value.value"
:x-model (:x-model params) :x-model (:x-model params)
:x-init "popper = Popper.createPopper($refs.input, $refs.dropdown, {placement: 'bottom-start', strategy: 'fixed', modifiers: {name: 'offset', options: {offset: [0, 20]}}})" :x-init "popper = Popper.createPopper($refs.input, $refs.dropdown, {placement: 'bottom-start', strategy: 'fixed', modifiers: {name: 'offset', options: {offset: [0, 10]}}})"
} }
[:a {:class (-> (hh/add-class (or (:class params) "") default-input-classes) [:a {:class (-> (hh/add-class (or (:class params) "") default-input-classes)
(hh/add-class "cursor-pointer")) (hh/add-class "cursor-pointer"))
@@ -117,11 +117,11 @@ c.clearChoices();
"x-ref" "dropdown" "x-ref" "dropdown"
"@keydown.escape" "open = false; value = {value: '', label: '' }" "@keydown.escape" "open = false; value = {value: '', label: '' }"
"x-transition:enter" "ease-[cubic-bezier(.3,2.3,.6,1)] duration-200" "x-transition:enter" "ease-[cubic-bezier(.3,2.3,.6,1)] duration-200"
"x-transition:enter-start" "!opacity-0 !mt-0" "x-transition:enter-start" "!opacity-0"
"x-transition:enter-end" "!opacity-1 !mt-1" "x-transition:enter-end" "!opacity-1"
"x-transition:leave" "ease-out duration-200" "x-transition:leave" "ease-out duration-200"
"x-transition:leave-start" "!opacity-1 !mt-1" "x-transition:leave-start" "!opacity-1"
"x-transition:leave-end" "!opacity-0 !mt-0" "x-transition:leave-end" "!opacity-0"
"x-show " "open" "x-show " "open"
"x-trap" "open" "x-trap" "open"
"@click.outside" "open=false;"} "@click.outside" "open=false;"}

5
src/clj/auto_ap/ssr/components/navbar.clj
View File

@@ -24,10 +24,7 @@
(when (is-admin? identity) (when (is-admin? identity)
[:button.mt-1.lg:w-96.relative.hidden.lg:block {:class "bg-gray-50 hover:bg-gray-200 dark:hover:bg-gray-700 border border-gray-300 text-gray-900 sm:text-sm rounded-lg focus:ring-primary-500 focus:border-primary-500 w-full pl-10 py-4 pr-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 gap-4 " [:button.mt-1.lg:w-96.relative.hidden.lg:block {:class "bg-gray-50 hover:bg-gray-200 dark:hover:bg-gray-700 border border-gray-300 text-gray-900 sm:text-sm rounded-lg focus:ring-primary-500 focus:border-primary-500 w-full pl-10 py-4 pr-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 gap-4 "
:hx-get (bidi/path-for ssr-routes/only-routes :hx-get (bidi/path-for ssr-routes/only-routes :search)}
:search)
:hx-target "#modal-holder"
:hx-swap "outerHTML"}
[:div {:class "absolute inset-y-0 left-0 flex items-center pl-3 pointer-events-none text-gray-500"} [:div {:class "absolute inset-y-0 left-0 flex items-center pl-3 pointer-events-none text-gray-500"}
[:div.w-4.h-4 svg/search] [:div.w-4.h-4 svg/search]
[:span.ml-2 "Search"]]]) [:span.ml-2 "Search"]]])

1
src/clj/auto_ap/ssr/components/page.clj
View File

@@ -49,4 +49,5 @@
(into (into
[:div.p-4] [:div.p-4]
children)]] children)]]
]) ])

8
src/clj/auto_ap/ssr/search.clj
View File

@@ -2,7 +2,7 @@
(:require (:require
[auto-ap.graphql.utils :refer [can-see-client?]] [auto-ap.graphql.utils :refer [can-see-client?]]
[auto-ap.solr :as solr] [auto-ap.solr :as solr]
[auto-ap.ssr.utils :refer [html-response]] [auto-ap.ssr.utils :refer [html-response modal-response]]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[clojure.string :as str] [clojure.string :as str]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
@@ -130,11 +130,11 @@
:form (:form-params request)) :form (:form-params request))
(if-let [q (get (:form-params request) "q")] (if-let [q (get (:form-params request) "q")]
(html-response (search-results* q (:identity request))) (html-response (search-results* q (:identity request)))
(html-response (modal-response
(com/modal {} (com/modal {}
(com/modal-card {} (com/modal-card {:class "w-full h-full"}
[:div.p-2 "Search"] [:div.p-2 "Search"]
[:div#search.overflow-auto.space-y-6.p-2.h-96 [:div#search.overflow-auto.space-y-6.p-2.w-full
(com/text-input {:id "search-input" (com/text-input {:id "search-input"
:type "search" :type "search"

10
src/clj/auto_ap/ssr/transaction/insights.clj
View File

@@ -1,21 +1,21 @@
(ns auto-ap.ssr.transaction.insights (ns auto-ap.ssr.transaction.insights
(:require (:require
[auto-ap.client-routes :as client-routes] [auto-ap.client-routes :as client-routes]
[auto-ap.datomic :refer [conn pull-attr visible-clients]] [auto-ap.datomic :refer [conn visible-clients]]
[auto-ap.rule-matching :refer [spread-cents]] [auto-ap.rule-matching :refer [spread-cents]]
[auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com] [auto-ap.ssr.components :as com]
[auto-ap.ssr.svg :as svg] [auto-ap.ssr.svg :as svg]
[auto-ap.ssr.ui :refer [base-page]] [auto-ap.ssr.ui :refer [base-page]]
[auto-ap.ssr.utils :refer [html-response]] [auto-ap.ssr.utils :refer [html-response modal-response]]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[cemerick.url :as url] [cemerick.url :as url]
[clj-http.client :as http] [clj-http.client :as http]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[datomic.api :as dc] [datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]] [hiccup2.core :as hiccup]
[hiccup2.core :as hiccup])) [iol-ion.tx :refer [random-tempid]]))
(def pull-expr [:transaction/description-original (def pull-expr [:transaction/description-original
:db/id :db/id
@@ -238,7 +238,7 @@
pull-expr pull-expr
(Long/parseLong transaction-id)) (Long/parseLong transaction-id))
similar (pinecone-similarity-list transaction-id)] similar (pinecone-similarity-list transaction-id)]
(html-response (modal-response
(com/modal {} (com/modal {}
(com/modal-card {:style {:width "900px"}} (com/modal-card {:style {:width "900px"}}
[:div.flex [:div.p-2 "Similar Transactions"]] [:div.flex [:div.p-2 "Similar Transactions"]]

4
src/clj/auto_ap/ssr/utils.clj
View File

@@ -322,7 +322,9 @@
(defn path->name2 [k & rest] (defn path->name2 [k & rest]
(let [k->n (fn [k] (let [k->n (fn [k]
(if (keyword? k) (if (keyword? k)
(str (namespace k) "/" (name k)) (str (when (namespace k)
(str (namespace k) "/"))
(name k))
k))] k))]
(str (k->n k) (str (k->n k)
(str/join "" (str/join ""