notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Makes pnl work for companies with parenthesis

Browse Source
This commit is contained in:
Bryce
2023-10-17 13:03:48 -07:00
parent fc8eefb8cb
commit bd9476544c
13 changed files with 269 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/clj/auto_ap/datomic/accounts.clj
View File

@@ -52,16 +52,6 @@
(map first)
(map <-datomic)))))
(defn get-by-id [id]
(->>
(dc/q {:find [(list 'pull '?e default-read)]
:in ['$ '?e]}
(dc/db conn )
id)
(map first)
(map <-datomic)
first))
(defn get-for-vendor [vendor-id client-id]
(if client-id
(->>

10
src/clj/auto_ap/graphql.clj
View File

@@ -9,13 +9,12 @@
[auto-ap.graphql.import-batch :as gq-import-batches]
[auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts]
[auto-ap.graphql.invoices :as gq-invoices]
[auto-ap.graphql.jobs :as gq-jobs]
[auto-ap.graphql.ledger :as gq-ledger]
[auto-ap.graphql.plaid :as gq-plaid]
[auto-ap.graphql.sales-orders :as gq-sales-orders]
[auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.graphql.transactions :as gq-transactions]
[auto-ap.graphql.utils :refer [assert-admin attach-tracing-resolvers]]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]
[auto-ap.graphql.vendors :as gq-vendors]
[auto-ap.graphql.yodlee-merchants :as ym]
[auto-ap.logging :as alog :refer [error-event info-event warn-event]]
@@ -23,8 +22,6 @@
[clj-time.coerce :as coerce]
[clj-time.core :as t]
[clojure.string :as str]
[config.core :refer [env]]
[clojure.tools.logging :as log]
[clojure.walk :as walk]
[com.brunobonacci.mulog :as mu]
[com.unbounce.dogstatsd.core :as statsd]
@@ -33,9 +30,7 @@
[com.walmartlabs.lacinia.schema :as schema]
[datomic.api :as dc]
[unilog.context :as lc]
[yang.time :refer [time-it]]
[auto-ap.routes.auth :as auth]
[buddy.sign.jwt :as jwt])
[yang.time :refer [time-it]])
(:import
(clojure.lang IPersistentMap)))
@@ -782,7 +777,6 @@
gq-invoices/attach
gq-clients/attach
gq-sales-orders/attach
gq-jobs/attach
schema/compile))

161
src/clj/auto_ap/graphql/jobs.clj
View File

@@ -1,161 +0,0 @@
(ns auto-ap.graphql.jobs
(:require
[amazonica.aws.ecs :as ecs]
[auto-ap.graphql.utils
:refer [assert-admin assert-failure result->page]]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.string :as str]
[config.core :refer [env]]
[clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[clojure.edn :as edn]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]])
(:import
(com.amazonaws.services.ecs.model AssignPublicIp)))
(defn get-ecs-tasks []
(->>
(concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50)))
(ecs/describe-tasks :include [] :tasks)
:tasks
(map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %)))))
(sort-by :created-at)
reverse))
(defn is-background-job? [task]
(->> task
:task-definition
:container-definitions
(mapcat :environment)
(filter (comp #{"INTEGREAT_JOB"} :name))
seq))
(defn task-definition->job-name [task-definition]
(->> (:container-definitions task-definition)
(mapcat :environment)
(filter (comp #{"INTEGREAT_JOB"} :name))
(map :value)
first))
(defn job-exited-successfully? [task]
(if (= 0 (->> task
:containers
(filter (comp #{"integreat-app" } :name))
(first)
:exit-code))
true
false))
(defn ecs-task->job [task]
{:status (condp = (:last-status task)
"RUNNING" :running
"PENDING" :pending
"PROVISIONING" :pending
"DEPROVISIONING" :running
"STOPPED" (if (job-exited-successfully? task)
:succeeded
:failed))
:name (task-definition->job-name (:task-definition task))
:end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))
:start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))})
(defn get-jobs-page [context args _]
(assert-admin (:id context))
(let [args (assoc (:filters args) :id (:id context))
jobs (->> (get-ecs-tasks)
(filter is-background-job?)
(map ecs-task->job))]
(result->page
jobs
(count jobs)
:data
args)))
(defn currently-running-jobs []
(->> (get-ecs-tasks)
(filter is-background-job?)
(map ecs-task->job)
(filter (comp #{:pending :running} :status))
(map :name)
set))
(defn run-task [task args]
(log/info "running job" task)
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition task
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
(defn request-job [context value _]
(assert-admin (:id context))
(if (not (get (currently-running-jobs) (:which value)))
(let [new-job (run-task
(-> (:which value)
(str/replace #"-" "_")
(str/replace #":" "")
(str "_" (:dd-env env)))
(some-> (:args value) edn/read-string))]
{:message (str "task " (str new-job) " started.")})
(assert-failure "This job is already running")))
(def objects
{:job {:fields {:name {:type 'String}
:start_date {:type :iso_date_time}
:end_date {:type :iso_date_time}
:status {:type :job_status}}}
:jobs_page {:fields {:data {:type '(list :job)}
:count {:type 'Int}
:total {:type 'Int}
:start {:type 'Int}
:end {:type 'Int}}}})
(def input-objects
{:jobs_filters {:fields {:sort {:type '(list :sort_item)}
:start {:type 'Int}
:per_page {:type 'Int}}}})
(def queries
{:jobs_page {:type :jobs_page
:args {:filters {:type :jobs_filters}
:sort {:type '(list :sort_item)}
:start {:type 'Int}
:per_page {:type 'Int}}
:resolve :get-jobs-page}})
(def enums
{:job_status {:values [{:enum-value :pending}
{:enum-value :running}
{:enum-value :succeeded}
{:enum-value :failed}]} })
(def resolvers
{:get-jobs-page get-jobs-page
:mutation/request-job request-job})
(def mutations
{:request_job
{:type :message
:args {:which {:type 'String}
:args {:type 'String}}
:resolve :mutation/request-job}})
(defn attach [schema]
(->
(merge-with merge schema
{:objects objects
:input-objects input-objects
:queries queries
:enums enums
:mutations mutations})
(attach-tracing-resolvers resolvers)))

11
src/clj/auto_ap/pdf/ledger.clj
View File

@@ -293,6 +293,9 @@
output-stream)
(.toByteArray output-stream)))
(defn join-names [client-ids]
(str/replace (->> client-ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #"[^\w]" "_" ))
(defn pnl-args->name [args]
(let [min-date (atime/unparse-local
(->> args :periods (map :start) first)
@@ -300,7 +303,7 @@
max-date (atime/unparse-local
(->> args :periods (map :end) last)
atime/iso-date)
names (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
names (->> args :client_ids join-names)]
(format "Profit-and-loss-%s-to-%s-for-%s" min-date max-date names)))
(defn cash-flows-args->name [args]
@@ -310,7 +313,7 @@
max-date (atime/unparse-local
(->> args :periods (map :end) last)
atime/iso-date)
names (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
names (->> args :client_ids join-names)]
(format "Cash-flows-%s-to-%s-for-%s" min-date max-date names)))
(defn journal-detail-args->name [args]
@@ -320,14 +323,14 @@
max-date (atime/unparse-local
(->> args :date_range :end)
atime/iso-date)
names (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
names (->> args :client_ids join-names)]
(format "Profit-and-loss-%s-to-%s-for-%s" min-date max-date names)))
(defn balance-sheet-args->name [args]
(let [date (atime/unparse-local
(:date args)
atime/iso-date)
name (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
name (->> args :client_ids join-names)]
(format "Balance-sheet-%s-for-%s" date name)))
(defn print-pnl [user args data]

237
src/clj/auto_ap/ssr/admin/background_jobs.clj Normal file
View File

@@ -0,0 +1,237 @@
(ns auto-ap.ssr.admin.background-jobs
(:require
[amazonica.aws.ecs :as ecs]
[auto-ap.logging :as alog]
[auto-ap.routes.utils
:refer [wrap-admin wrap-client-redirect-unauthenticated]]
[auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com]
[auto-ap.ssr.grid-page-helper :as helper]
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
[auto-ap.ssr.utils
:refer [apply-middleware-to-all-handlers
html-response
validation-error
wrap-form-4xx
wrap-schema-decode]]
[auto-ap.time :as atime]
[bidi.bidi :as bidi]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.string :as str]
[config.core :refer [env]])
(:import
(com.amazonaws.services.ecs.model AssignPublicIp)))
(defn get-ecs-tasks []
(->>
(concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50)))
(ecs/describe-tasks :include [] :tasks)
:tasks
(map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %)))))
(sort-by :created-at)
reverse))
(defn is-background-job? [task]
(->> task
:task-definition
:container-definitions
(mapcat :environment)
(filter (comp #{"INTEGREAT_JOB"} :name))
seq))
(defn task-definition->job-name [task-definition]
(->> (:container-definitions task-definition)
(mapcat :environment)
(filter (comp #{"INTEGREAT_JOB"} :name))
(map :value)
first))
(defn job-exited-successfully? [task]
(if (= 0 (->> task
:containers
(filter (comp #{"integreat-app" } :name))
(first)
:exit-code))
true
false))
(defn ecs-task->job [task]
{:status (condp = (:last-status task)
"RUNNING" :running
"PENDING" :pending
"PROVISIONING" :pending
"DEPROVISIONING" :running
"STOPPED" (if (job-exited-successfully? task)
:succeeded
:failed))
:name (task-definition->job-name (:task-definition task))
:end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))
:start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))})
(defn fetch-page [request]
(let [jobs (->> (get-ecs-tasks)
(filter is-background-job?)
(map ecs-task->job))]
[jobs (count jobs)]))
(def grid-page
(helper/build {:id "job-table"
:nav (com/admin-aside-nav)
:fetch-page fetch-page
:action-buttons (fn [request]
[(com/button {:hx-get (str (bidi/path-for ssr-routes/only-routes
:admin-job-start-dialog))
:hx-target "#modal-holder"
:hx-swap "outerHTML"
:color :primary}
"Run job")])
:breadcrumbs [[:a {:href (bidi/path-for ssr-routes/only-routes
:admin)}
"Admin"]
[:a {:href (bidi/path-for ssr-routes/only-routes
:admin-jobs)}
"Background Jobs"]]
:title "Jobs"
:entity-name "Job"
:route :admin-job-table
:headers [
{:key "start"
:name "Start"
:render #(some-> % :start-date (atime/unparse-local atime/standard-time))}
{:key "end"
:name "End"
:render #(some-> % :end-date (atime/unparse-local atime/standard-time))}
{:key "duration"
:name "Duration"
:render (fn [e]
(when (and (:start-date e)
(:end-date e))
(str (time/in-minutes (time/interval
(:start-date e)
(:end-date e))) " minutes")))}
{:key "name"
:name "Name"
:render :name}
{:key "status"
:name "Status"
:render :status}]}))
(def row* (partial helper/row* grid-page))
(def table* (partial helper/table* grid-page))
(defn currently-running-jobs []
(->> (get-ecs-tasks)
(filter is-background-job?)
(map ecs-task->job)
(filter (comp #{:pending :running} :status))
(map :name)
set))
(defn run-task [task args]
(alog/info ::starting :task task)
(ecs/run-task (cond-> {:capacity-provider-strategy [{:base 1 :weight 1 :capacity-provider "FARGATE_SPOT"}]
:count 1
:cluster "default"
:enable-ecs-managed-tags true
:task-definition task
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}}
args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
(defn job-start [{:keys [form-params]}]
(if (not (get (currently-running-jobs) (:name form-params)))
(let [new-job (run-task
(-> (:name form-params)
(str/replace #"-" "_")
(str/replace #":" "")
(str "_" (:dd-env env)))
(dissoc form-params :name))]
{:message (str "task " (str new-job) " started.")})
(validation-error "This job is already running")))
(defn subform [{{:strs [name]} :query-params }]
(html-response (cond (= "bulk-journal-import" name)
[:div (com/field {:label "Url"}
[:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "ledger-data.csv"
:name "ledger-url"} )])]
(= "register-invoice-import" name)
[:div (com/field {:label "Url"}
[:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "invoice-data.csv"
:name "invoice-url"} )])]
(= "load-historical-sales" name)
[:div
(com/field {:label "Client"}
(com/typeahead {:name "client"
:placeholder "Search..."
:url (bidi/path-for ssr-routes/only-routes
:company-search)
:id (str "client-search")}))
(com/field {:label "Days to load"}
(com/text-input {:placeholder "60"
:name "days"} ))]
:else [:div]))
)
(defn job-start-dialog [_]
(html-response (com/modal
{:modal-class "max-w-4xl"}
[:form#edit-form {:hx-ext "response-targets"
:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start
)
:hx-swap "outerHTML swap:300ms"
:hx-target-400 "#form-errors .error-content"}
[:fieldset {:class "hx-disable"}
(com/modal-card
{}
[:div.flex [:div.p-2 "New job"] ]
[:div.space-y-6
(com/field {:label "Job"}
(com/select {:name "name"
:class "w-64"
:options [["" ""]
["yodlee2" "Yodlee Import"]
["yodlee2-accounts" "Yodlee Account Import"]
["intuit" "Intuit import"]
["plaid" "Plaid import"]
["bulk-journal-import" "Bulk Journal Import"]
["square2-import-job" "Square2 Import"]
["register-invoice-import" "Register Invoice Import "]
["ezcater-upsert" "Upsert recent ezcater orders"]
["load-historical-sales" "Load Historical Square Sales"]
["export-backup" "Export Backup"]]
:hx-get (bidi/path-for ssr-routes/only-routes
:admin-job-subform)
:hx-target "#sub-form"
:hx-swap "innerHTML"}))
[:div#sub-form]
[:div#form-errors [:span.error-content]]
(com/button {:color :primary :form "edit-form" :type "submit"}
"Run")]
[:div])]])))
(def key->handler
(apply-middleware-to-all-handlers
(->>
{:admin-jobs (helper/page-route grid-page)
:admin-job-table (helper/table-route grid-page)
:admin-job-subform (-> subform (wrap-schema-decode :query-schema [:map [:name :string]]))
:admin-job-start (-> job-start
(wrap-schema-decode :form-schema [:map [:name :string]])
(wrap-nested-form-params)
(wrap-form-4xx))
:admin-job-start-dialog job-start-dialog})
(fn [h]
(-> h
(wrap-admin)
(wrap-client-redirect-unauthenticated)))))

2
src/clj/auto_ap/ssr/components/aside.clj
View File

@@ -323,7 +323,7 @@
[:li
(menu-button- {:icon svg/rabbit
:href (bidi/path-for client-routes/routes
:href (bidi/path-for ssr-routes/only-routes
:admin-jobs)}
"Background Jobs")]
[:li

4
src/clj/auto_ap/ssr/core.clj
View File

@@ -4,6 +4,7 @@
[auto-ap.routes.utils
:refer [wrap-admin wrap-client-redirect-unauthenticated wrap-secure]]
[auto-ap.ssr.admin.history :as history]
[auto-ap.ssr.admin.background-jobs :as admin-jobs]
[auto-ap.ssr.auth :as auth]
[auto-ap.ssr.company :as company]
[auto-ap.ssr.company-dropdown :as company-dropdown]
@@ -73,5 +74,6 @@
(into pos-cash-drawer-shifts/key->handler)
(into pos-refunds/key->handler)
(into users/key->handler)
(into admin-accounts/key->handler)))
(into admin-accounts/key->handler)
(into admin-jobs/key->handler)))