notid/integreat
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(sales-summaries): stop days falling out of balance #17

Closed
notid wants to merge 4 commits from sales-summary-balancing into master
pull from: sales-summary-balancing
merge into: notid:master
Conversation 0 Commits 4 Files Changed 19 +418 -285
6 changed files with 418 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 95d0537c69 - Show all commits

55
src/clj/auto_ap/datomic/sales_summaries.clj Normal file
View File

@@ -0,0 +1,55 @@
(ns auto-ap.datomic.sales-summaries
(:require
[iol-ion.query :refer [dollars=]]))
(defn- ledger-side
"The ledger side of an item as a keyword, whether it arrived as a plain keyword (a
transaction map, or a pull using `:xform iol-ion.query/ident`) or as the `{:db/ident ...}`
map a plain `pull` returns. Resolving both shapes here matters: items whose side does not
compare equal are counted on neither side, which would leave a summary looking balanced at
zero and therefore silently accepted."
[item]
(let [side (:ledger-mapped/ledger-side item)]
(if (map? side) (:db/ident side) side)))
(defn- side-total [side items]
(->> items
(filter #(= side (ledger-side %)))
(map #(:ledger-mapped/amount % 0.0))
(reduce + 0.0)))
(defn total-debits [items]
(side-total :ledger-side/debit items))
(defn total-credits [items]
(side-total :ledger-side/credit items))
(defn fully-mapped? [items]
(every? :ledger-mapped/account items))
(defn fully-sided?
"Every item says which side of the ledger it belongs on. Guards `accepted?` against
reading a collection of sideless items as balanced at zero."
[items]
(every? #(#{:ledger-side/debit :ledger-side/credit} (ledger-side %)) items))
(defn balanced? [items]
(dollars= (total-debits items) (total-credits items)))
(defn accepted?
"True once a summary is finished: every line is mapped to an account and debits equal
credits. This is the same condition the sales summaries grid renders as \"Balanced\", and
the condition the scheduled refresh treats as \"leave this alone\"."
[items]
(boolean (and (seq items)
(fully-mapped? items)
(fully-sided? items)
(balanced? items))))
(defn <-pulled-item
"Flattens the ref values on a pulled sales summary item back to the scalars a transaction
expects. `accepted?` reads either shape, so this is only needed on the write path."
[item]
(cond-> item
(map? (:ledger-mapped/ledger-side item)) (update :ledger-mapped/ledger-side :db/ident)
(map? (:ledger-mapped/account item)) (update :ledger-mapped/account :db/id)))

308
src/clj/auto_ap/jobs/sales_summaries.clj
View File

@@ -1,5 +1,6 @@
(ns auto-ap.jobs.sales-summaries (ns auto-ap.jobs.sales-summaries
(:require [auto-ap.datomic :refer [conn]] (:require [auto-ap.datomic :refer [conn]]
[auto-ap.datomic.sales-summaries :as d-ss]
[auto-ap.jobs.core :refer [execute]] [auto-ap.jobs.core :refer [execute]]
[auto-ap.logging :as alog] [auto-ap.logging :as alog]
[auto-ap.time :as atime] [auto-ap.time :as atime]
@@ -8,6 +9,7 @@
[clj-time.periodic :as per] [clj-time.periodic :as per]
[clojure.string :as str] [clojure.string :as str]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
[config.core :refer [env]]
[datomic.api :as dc])) [datomic.api :as dc]))
(defn mark-dirty [client start end] (defn mark-dirty [client start end]
@@ -39,27 +41,100 @@
(dc/db conn) (dc/db conn)
number))) number)))
(defn delete-all [] (defn delete-all []
@(dc/transact-async conn @(dc/transact-async conn
(->> (->>
(dc/q '[:find ?ss (dc/q '[:find ?ss
:where [?ss :sales-summary/date]] :where [?ss :sales-summary/date]]
(dc/db conn)) (dc/db conn))
(map (fn [[ ss]] (map (fn [[ss]]
[:db/retractEntity ss]))))) [:db/retractEntity ss])))))
(def item-read
"Enough of a summary item to both evaluate `d-ss/accepted?` and transact the item back
unchanged. `:db/id` matters: `:sales-summary/items` is a component attribute upserted via
`[:reset-rels ...]`, so an item re-transacted without its id is deleted and recreated."
'[:db/id
:sales-summary-item/category
:sales-summary-item/sort-order
:sales-summary-item/manual?
:ledger-mapped/amount
{:ledger-mapped/ledger-side [:db/ident]}
{:ledger-mapped/account [:db/id]}])
(defn dirty-sales-summaries [c] (defn dirty-sales-summaries [c]
(let [client-id (dc/entid (dc/db conn) c)] (let [client-id (dc/entid (dc/db conn) c)]
(->> (dc/index-pull (dc/db conn) (->> (dc/index-pull (dc/db conn)
{:index :avet {:index :avet
:selector '[:sales-summary/date :sales-summary/client :db/id] :selector (conj '[:sales-summary/date :sales-summary/client :db/id]
{:sales-summary/items item-read})
:start [:sales-summary/client+dirty [client-id true]]}) :start [:sales-summary/client+dirty [client-id true]]})
(filter (fn [sales-summary] (filter (fn [sales-summary]
(= client-id (:db/id (:sales-summary/client sales-summary)))))))) (= client-id (:db/id (:sales-summary/client sales-summary))))))))
(def default-days
"How far back the scheduled refresh looks for summaries that still need recomputing."
7)
(defn trailing-window
"`[start end)` covering the last `days` business days, ending with today. `end` is
exclusive, matching both `periodic-seq`'s 3-arity and the grid's date filters, so
`(trailing-window 7)` is day -6 through today inclusive."
[days]
[(.toDateMidnight (atime/localize (time/minus (time/now) (time/days (dec days)))))
(.toDateMidnight (atime/localize (time/plus (time/now) (time/days 1))))])
(defn accepted-client+dates
"Set of `[client-id date]` pairs in `[start end)` whose summary is already accepted, and
so should be left alone rather than re-marked. Accepted means balanced with every line
mapped to an account — the condition the grid renders as \"Balanced\"."
[db start end]
(->> (dc/q '[:find (pull ?ss selector)
:in $ ?start ?end selector
:where
[?ss :sales-summary/date ?d]
[(>= ?d ?start)]
[(< ?d ?end)]]
db
(c/to-date start)
(c/to-date end)
(conj '[:sales-summary/date {:sales-summary/client [:db/id]}]
{:sales-summary/items item-read}))
(map first)
(filter #(d-ss/accepted? (map d-ss/<-pulled-item (:sales-summary/items %))))
(map (juxt (comp :db/id :sales-summary/client) :sales-summary/date))
set))
(defn mark-stale-dirty
"Marks every client/day in the trailing `days` window dirty so `sales-summaries-v2` will
recompute it, skipping days whose summary is already accepted. Because accepted is derived
rather than stored, a summary that later falls out of balance is picked up again on the
next run. Returns the number of client/days marked."
[days]
(let [db (dc/db conn)
[start end] (trailing-window days)
accepted (accepted-client+dates db start end)
clients (map first (dc/q '[:find ?c
:in $
:where [_ :sales-order/client ?c]]
db))
dates (map c/to-date (per/periodic-seq start end (time/days 1)))
tx-data (for [client clients
date dates
:when (not (accepted [client date]))]
{:sales-summary/client client
:sales-summary/date date
:sales-summary/dirty true
:sales-summary/client+date [client date]})]
(alog/info ::marking-dirty
:days days
:client-count (count clients)
:accepted-count (count accepted)
:marked (count tx-data))
(doseq [batch (partition-all 500 tx-data)]
@(dc/transact conn batch))
(count tx-data)))
(defn- get-fee [c date] (defn- get-fee [c date]
(- (or (ffirst (dc/q '[:find ?f (- (or (ffirst (dc/q '[:find ?f
:in $ ?client ?d :in $ ?client ?d
@@ -99,53 +174,53 @@
"food app refunds" 41400}) "food app refunds" 41400})
(defn get-payment-items [c date] (defn get-payment-items [c date]
(->> (->>
(dc/q '[:find ?processor ?type-name (sum ?total) (dc/q '[:find ?processor ?type-name (sum ?total)
:with ?c :with ?c
:in $ [?clients ?start-date ?end-date] :in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]] :where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/charges ?c] [?e :sales-order/charges ?c]
[?c :charge/type-name ?type-name] [?c :charge/type-name ?type-name]
(or-join [?c ?processor] (or-join [?c ?processor]
(and [?c :charge/processor ?p] (and [?c :charge/processor ?p]
[?p :db/ident ?processor]) [?p :db/ident ?processor])
(and (and
(not [?c :charge/processor]) (not [?c :charge/processor])
[(ground :ccp-processor/na) ?processor])) [(ground :ccp-processor/na) ?processor]))
[?c :charge/total ?total]] [?c :charge/total ?total]]
(dc/db conn) (dc/db conn)
[[c] date date]) [[c] date date])
(reduce (reduce
(fn [acc [processor type-name total]] (fn [acc [processor type-name total]]
(update (update
acc acc
(cond (= type-name "CARD") (cond (= type-name "CARD")
"Card Payments" "Card Payments"
(= type-name "CASH") (= type-name "CASH")
"Cash Payments" "Cash Payments"
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name) (#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
"Gift Card Payments" "Gift Card Payments"
(#{:ccp-processor/toast (#{:ccp-processor/toast
#_:ccp-processor/ezcater #_:ccp-processor/ezcater
#_:ccp-processor/koala #_:ccp-processor/koala
:ccp-processor/doordash :ccp-processor/doordash
:ccp-processor/grubhub :ccp-processor/grubhub
:ccp-processor/uber-eats} processor) :ccp-processor/uber-eats} processor)
"Food App Payments" "Food App Payments"
:else :else
"Unknown") "Unknown")
(fnil + 0.0) (fnil + 0.0)
total)) total))
{}) {})
(map (fn [[k v]] (map (fn [[k v]]
{:db/id (str (java.util.UUID/randomUUID)) {:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 0 :sales-summary-item/sort-order 0
:sales-summary-item/category k :sales-summary-item/category k
:ledger-mapped/amount (if (= "Card Payments" k) :ledger-mapped/amount (if (= "Card Payments" k)
(- v (get-fee c date)) (- v (get-fee c date))
v) v)
:ledger-mapped/ledger-side :ledger-side/debit})))) :ledger-mapped/ledger-side :ledger-side/debit}))))
(defn get-discounts [c date] (defn get-discounts [c date]
(when-let [discount (ffirst (dc/q '[:find (sum ?discount) (when-let [discount (ffirst (dc/q '[:find (sum ?discount)
@@ -162,7 +237,7 @@
:ledger-mapped/ledger-side :ledger-side/debit})) :ledger-mapped/ledger-side :ledger-side/debit}))
(defn get-refund-items [c date] (defn get-refund-items [c date]
(->> (->>
(dc/q '[:find ?type-name (sum ?t) (dc/q '[:find ?type-name (sum ?t)
:with ?e :with ?e
:in $ [?clients ?start-date ?end-date] :in $ [?clients ?start-date ?end-date]
@@ -173,26 +248,24 @@
(dc/db conn) (dc/db conn)
[[c] date date]) [[c] date date])
(reduce (reduce
(fn [acc [type-name total]] (fn [acc [type-name total]]
(update (update
acc acc
(cond (= type-name "CARD") (cond (= type-name "CARD")
"Card Refunds" "Card Refunds"
(= type-name "CASH") (= type-name "CASH")
"Cash Refunds" "Cash Refunds"
:else :else
"Food App Refunds") "Food App Refunds")
(fnil + 0.0) (fnil + 0.0)
total)) total))
{}) {})
(map (fn [[k v]] (map (fn [[k v]]
{:db/id (str (java.util.UUID/randomUUID)) {:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 3 :sales-summary-item/sort-order 3
:sales-summary-item/category k :sales-summary-item/category k
:ledger-mapped/amount v :ledger-mapped/amount v
:ledger-mapped/ledger-side :ledger-side/credit})))) :ledger-mapped/ledger-side :ledger-side/credit}))))
(defn get-fees [c date] (defn get-fees [c date]
(when-let [fee (get-fee c date)] (when-let [fee (get-fee c date)]
@@ -278,17 +351,18 @@
(defn sales-summaries-v2 [] (defn sales-summaries-v2 []
(doseq [[c client-code] (dc/q '[:find ?c ?client-code (doseq [[c client-code] (dc/q '[:find ?c ?client-code
:in $ :in $
:where [?c :client/code ?client-code]] :where [?c :client/code ?client-code]]
(dc/db conn)) (dc/db conn))
{:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)] {:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)]
(mu/with-context {:client-code client-code (mu/with-context {:client-code client-code
:date date} :date date}
(alog/info ::updating) (alog/info ::updating)
(let [manual-items (->> existing-summary (let [manual-items (->> existing-summary
:sales-summary/items :sales-summary/items
(filter :sales-summary-item/manual?)) (filter :sales-summary-item/manual?)
calculated-items (->> (map d-ss/<-pulled-item))
calculated-items (->>
(get-sales c date) (get-sales c date)
(concat (get-payment-items c date)) (concat (get-payment-items c date))
(concat (get-refund-items c date)) (concat (get-refund-items c date))
@@ -301,20 +375,19 @@
(map (fn [z] (map (fn [z]
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account) (assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
:sales-summary-item/manual? false)))) :sales-summary-item/manual? false))))
all-items (concat calculated-items manual-items) all-items (concat calculated-items manual-items)
result {:db/id id result {:db/id id
:sales-summary/client c :sales-summary/client c
:sales-summary/date date :sales-summary/date date
:sales-summary/dirty false :sales-summary/dirty false
:sales-summary/client+date [c date] :sales-summary/client+date [c date]
:sales-summary/items all-items}] :sales-summary/items all-items}]
(if (seq (:sales-summary/items result)) (if (seq (:sales-summary/items result))
(do (do
(alog/info ::upserting-summaries (alog/info ::upserting-summaries
:category-count (count (:sales-summary/items result))) :category-count (count (:sales-summary/items result)))
@(dc/transact conn [[:upsert-sales-summary result]])) @(dc/transact conn [[:upsert-sales-summary result]]))
@(dc/transact conn [{:db/id id :sales-summary/dirty false}])))))) @(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
(defn reset-summaries [] (defn reset-summaries []
@(dc/transact conn (->> (dc/q '[:find ?sos @(dc/transact conn (->> (dc/q '[:find ?sos
@@ -324,9 +397,6 @@
(map (fn [[sos]] (map (fn [[sos]]
[:db/retractEntity sos]))))) [:db/retractEntity sos])))))
(comment (comment
(auto-ap.datomic/transact-schema conn) (auto-ap.datomic/transact-schema conn)
@@ -336,26 +406,19 @@
(dirty-sales-summaries [:client/code "NGWH"]) (dirty-sales-summaries [:client/code "NGWH"])
(apply mark-dirty [:client/code "NGWH"] (last-n-days 5)) (apply mark-dirty [:client/code "NGWH"] (last-n-days 5))
(iol-ion.tx.upsert-sales-summary-ledger/summary->journal-entry (dc/db conn) 17592314245819) (iol-ion.tx.upsert-sales-summary-ledger/summary->journal-entry (dc/db conn) 17592314245819)
(iol-ion.tx.upsert-sales-summary-ledger/upsert-sales-summary (dc/db conn) {:db/id 17592314241429}) (iol-ion.tx.upsert-sales-summary-ledger/upsert-sales-summary (dc/db conn) {:db/id 17592314241429})
(mark-all-dirty 5) (mark-all-dirty 5)
(delete-all) (delete-all)
(sales-summaries-v2) (sales-summaries-v2)
1 1
(dc/q '[:find (pull ?sos [* {:sales-summary/sales-items [*]}]) (dc/q '[:find (pull ?sos [* {:sales-summary/sales-items [*]}])
:in $ :in $
:where [?sos :sales-summary/client [:client/code "NGHW"]] :where [?sos :sales-summary/client [:client/code "NGHW"]]
@@ -386,15 +449,24 @@
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy} @(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}]) {:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
(auto-ap.datomic/transact-schema conn) (auto-ap.datomic/transact-schema conn))
)
(defn days-arg
"Trailing-window size from the job's `args`, e.g. `{:days 30}` set as a container override
from the admin Background Jobs page for an ad-hoc wider backfill. Values arrive as EDN but
may still be strings, so coerce defensively the way load-historical-sales does."
[args]
(let [days (:days args)]
(cond-> (or days default-days)
(string? days) (#(Long/parseLong %)))))
(defn refresh-sales-summaries
"Marks the trailing `days` window dirty, skipping accepted summaries, then recomputes
everything left dirty."
([] (refresh-sales-summaries default-days))
([days]
(mark-stale-dirty days)
(sales-summaries-v2)))
(defn -main [& _] (defn -main [& _]
(execute "sales-summaries" sales-summaries-v2)) (execute "sales-summaries" #(refresh-sales-summaries (days-arg (:args env)))))

29
src/clj/auto_ap/server.clj
View File

@@ -13,6 +13,7 @@
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales] [auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
[auto-ap.jobs.plaid :as job-plaid] [auto-ap.jobs.plaid :as job-plaid]
[auto-ap.jobs.register-invoice-import :as job-register-invoice-import] [auto-ap.jobs.register-invoice-import :as job-register-invoice-import]
[auto-ap.jobs.sales-summaries :as job-sales-summaries]
[auto-ap.jobs.square :as job-square] [auto-ap.jobs.square :as job-square]
[auto-ap.jobs.sysco :as job-sysco] [auto-ap.jobs.sysco :as job-sysco]
[auto-ap.jobs.vendor-usages :as job-vendor-usages] [auto-ap.jobs.vendor-usages :as job-vendor-usages]
@@ -33,23 +34,22 @@
(.addShutdownHook (Runtime/getRuntime) (.addShutdownHook (Runtime/getRuntime)
(Thread. f))) (Thread. f)))
(defn gzip-handler [] (defn gzip-handler []
(let [gz (GzipHandler.)] (let [gz (GzipHandler.)]
(doto gz (doto gz
(.setIncludedMethods (into-array ["GET" "POST" "PUT" "DELETE" "PATCH"])) (.setIncludedMethods (into-array ["GET" "POST" "PUT" "DELETE" "PATCH"]))
(.setIncludedMimeTypes (into-array ["text/css" (.setIncludedMimeTypes (into-array ["text/css"
"text/*" "text/*"
"text/plain" "text/plain"
"text/javascript" "text/javascript"
"text/csv" "text/csv"
"text/html" "text/html"
"text/html;charset=utf-8" "text/html;charset=utf-8"
"application/javascript" "application/javascript"
"application/csv" "application/csv"
"application/edn" "application/edn"
"application/json" "application/json"
"image/svg+xml"])) "image/svg+xml"]))
(.setMinGzipSize 1024)) (.setMinGzipSize 1024))
gz)) gz))
@@ -126,6 +126,9 @@
(= job "close-auto-invoices") (= job "close-auto-invoices")
(job-close-auto-invoices/-main) (job-close-auto-invoices/-main)
(= job "sales-summaries")
(job-sales-summaries/-main)
(= job "ezcater-upsert") (= job "ezcater-upsert")
(job-ezcater-upsert/-main) (job-ezcater-upsert/-main)

206
src/clj/auto_ap/ssr/admin/background_jobs.clj
View File

@@ -28,14 +28,13 @@
(com.amazonaws.services.ecs.model AssignPublicIp))) (com.amazonaws.services.ecs.model AssignPublicIp)))
(defn get-ecs-tasks [] (defn get-ecs-tasks []
(->> (->>
(concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50))) (concat (:task-arns (ecs/list-tasks :max-results 50)) (:task-arns (ecs/list-tasks :desired-status "STOPPED" :max-results 50)))
(ecs/describe-tasks :include [] :tasks) (ecs/describe-tasks :include [] :tasks)
:tasks :tasks
(map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %))))) (map #(assoc % :task-definition (:task-definition (ecs/describe-task-definition :task-definition (:task-definition-arn %)))))
(sort-by :created-at) (sort-by :created-at)
reverse)) reverse))
(defn is-background-job? (defn is-background-job?
"This function checks whether a given task is a background job. "This function checks whether a given task is a background job.
@@ -60,7 +59,7 @@
(defn job-exited-successfully? [task] (defn job-exited-successfully? [task]
(if (= 0 (->> task (if (= 0 (->> task
:containers :containers
(filter (comp #{"integreat-app" } :name)) (filter (comp #{"integreat-app"} :name))
(first) (first)
:exit-code)) :exit-code))
true true
@@ -77,7 +76,7 @@
:succeeded :succeeded
:failed)) :failed))
:name (task-definition->job-name (:task-definition task)) :name (task-definition->job-name (:task-definition task))
:end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0))) :end-date (some-> (:stopped-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))
:start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))}) :start-date (some-> (:created-at task) coerce/to-date-time (time/to-time-zone (time/time-zone-for-offset 0)))})
(defn fetch-page [request] (defn fetch-page [request]
@@ -85,7 +84,7 @@
(filter is-background-job?) (filter is-background-job?)
(map ecs-task->job))] (map ecs-task->job))]
[jobs (count jobs)])) [jobs (count jobs)]))
(def query-schema (mc/schema [:map ])) (def query-schema (mc/schema [:map]))
(def grid-page (def grid-page
(helper/build {:id "job-table" (helper/build {:id "job-table"
@@ -107,8 +106,7 @@
:entity-name "Job" :entity-name "Job"
:query-schema query-schema :query-schema query-schema
:route :admin-job-table :route :admin-job-table
:headers [ :headers [{:key "start"
{:key "start"
:name "Start" :name "Start"
:render #(some-> % :start-date (atime/unparse-local atime/standard-time))} :render #(some-> % :start-date (atime/unparse-local atime/standard-time))}
{:key "end" {:key "end"
@@ -119,7 +117,7 @@
:render (fn [e] :render (fn [e]
(when (and (:start-date e) (when (and (:start-date e)
(:end-date e)) (:end-date e))
(str (time/in-minutes (time/interval (str (time/in-minutes (time/interval
(:start-date e) (:start-date e)
(:end-date e))) " minutes")))} (:end-date e))) " minutes")))}
{:key "name" {:key "name"
@@ -150,16 +148,16 @@
:network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"] :network-configuration {:aws-vpc-configuration {:subnets ["subnet-5e675761" "subnet-8519fde2" "subnet-89bab8d4"]
:security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"] :security-groups ["sg-004e5855310c453a3" "sg-02d167406b1082698"]
:assign-public-ip AssignPublicIp/ENABLED}}} :assign-public-ip AssignPublicIp/ENABLED}}}
args (assoc-in [:overrides :container-overrides ] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}])))) args (assoc-in [:overrides :container-overrides] [{:name "integreat-app" :environment [{:name "args" :value (pr-str args)}]}]))))
(defn job-start [{:keys [form-params]}] (defn job-start [{:keys [form-params]}]
(if (not (get (currently-running-jobs) (:name form-params))) (if (not (get (currently-running-jobs) (:name form-params)))
(let [new-job (run-task (let [new-job (run-task
(-> (:name form-params) (-> (:name form-params)
(str/replace #"-" "_") (str/replace #"-" "_")
(str/replace #":" "") (str/replace #":" "")
(str "_" (:dd-env env))) (str "_" (:dd-env env)))
(dissoc form-params :name))] (dissoc form-params :name))]
{:message (str "task " (str new-job) " started.")}) {:message (str "task " (str new-job) " started.")})
(form-validation-error "This job is already running" (form-validation-error "This job is already running"
:form-params form-params))) :form-params form-params)))
@@ -170,107 +168,109 @@
[(fc/with-field :ledger-url [(fc/with-field :ledger-url
(com/validated-field {:label "Url" (com/validated-field {:label "Url"
:errors (fc/field-errors)} :errors (fc/field-errors)}
[:div.flex.place-items-center.gap-2 [:div.flex.place-items-center.gap-2
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"] [:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
(com/text-input {:placeholder "ledger-data.csv" (com/text-input {:placeholder "ledger-data.csv"
:name (fc/field-name) :name (fc/field-name)
:value (fc/field-value)} )]))] :value (fc/field-value)})]))]
(= "register-invoice-import" name) (= "register-invoice-import" name)
[ [(fc/with-field :invoice-url
(fc/with-field :invoice-url (com/validated-field {:label "Url"
(com/validated-field {:label "Url" :errors (fc/field-errors)}
:errors (fc/field-errors)} [:div.flex.place-items-center.gap-2
[:div.flex.place-items-center.gap-2 [:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"]
[:pre.text-xs.mr-1 "s3://data.prod.app.integreatconsult.com/bulk-import/"] (com/text-input {:placeholder "invoice-data.csv"
(com/text-input {:placeholder "invoice-data.csv" :name (fc/field-name)
:name (fc/field-name) :value (fc/field-value)})]))]
:value (fc/field-value)} )]))]
(= "load-historical-sales" name) (= "load-historical-sales" name)
[ [(fc/with-field :client
(fc/with-field :client (com/validated-field {:label "Client"
(com/validated-field {:label "Client" :errors (fc/field-errors)}
:errors (fc/field-errors)} (com/typeahead {:name (fc/field-name)
(com/typeahead {:name (fc/field-name) :value (fc/field-value)
:value (fc/field-value) :placeholder "Search..."
:placeholder "Search..." :url (bidi/path-for ssr-routes/only-routes
:url (bidi/path-for ssr-routes/only-routes :company-search)})))
:company-search)}))) (fc/with-field :days
(fc/with-field :days
(com/validated-field {:label "Days to load" (com/validated-field {:label "Days to load"
:errors (fc/field-errors)} :errors (fc/field-errors)}
(com/text-input {:placeholder "60" (com/text-input {:placeholder "60"
:name (fc/field-name) :name (fc/field-name)
:value (fc/field-value)} )))] :value (fc/field-value)})))]
:else nil)) (= "sales-summaries" name)
[(fc/with-field :days
(com/validated-field {:label "Days to refresh"
:errors (fc/field-errors)}
(com/text-input {:placeholder "7"
:name (fc/field-name)
:value (fc/field-value)})))]
:else nil)))
(defn subform [{{:keys [name]} :query-params}]
)
(defn subform [{{:keys [name]} :query-params }]
(html-response (html-response
(fc/start-form {} nil (fc/start-form {} nil
(subform* {:name name})))) (subform* {:name name}))))
(defn job-start-dialog [{:keys [form-errors form-params] :as request}] (defn job-start-dialog [{:keys [form-errors form-params] :as request}]
(fc/start-form (or form-params {}) form-errors (fc/start-form (or form-params {}) form-errors
(modal-response (modal-response
(com/modal ;; TODO we need a cleaner way to have forms that wrap the whole. In this cas (com/modal ;; TODO we need a cleaner way to have forms that wrap the whole. In this cas
{} {}
[:form {:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start) [:form {:hx-post (bidi/path-for ssr-routes/only-routes :admin-job-start)
:class "h-full w-full"} :class "h-full w-full"}
[:fieldset {:class "hx-disable h-full w-full"} [:fieldset {:class "hx-disable h-full w-full"}
(com/modal-card {} (com/modal-card {}
[:div.m-2 "New job"] [:div.m-2 "New job"]
[:div.space-y-6 [:div.space-y-6
(fc/with-field :name (fc/with-field :name
(com/validated-field {:label "Job" (com/validated-field {:label "Job"
:errors (fc/field-errors)} :errors (fc/field-errors)}
(com/select {:name (fc/field-name) (com/select {:name (fc/field-name)
:value (fc/field-value) :value (fc/field-value)
:class "w-64" :class "w-64"
:options [["" ""] :options [["" ""]
["yodlee2" "Yodlee Import"] ["yodlee2" "Yodlee Import"]
["yodlee2-accounts" "Yodlee Account Import"] ["yodlee2-accounts" "Yodlee Account Import"]
["intuit" "Intuit import"] ["intuit" "Intuit import"]
["plaid" "Plaid import"] ["plaid" "Plaid import"]
["bulk-journal-import" "Bulk Journal Import"] ["bulk-journal-import" "Bulk Journal Import"]
["square-import-job" "Square Import"] ["square-import-job" "Square Import"]
["register-invoice-import" "Register Invoice Import "] ["register-invoice-import" "Register Invoice Import "]
["ezcater-upsert" "Upsert recent ezcater orders"] ["ezcater-upsert" "Upsert recent ezcater orders"]
["load-historical-sales" "Load Historical Square Sales"] ["load-historical-sales" "Load Historical Square Sales"]
["export-backup" "Export Backup"]] ["sales-summaries" "Refresh Sales Summaries"]
:hx-get (bidi/path-for ssr-routes/only-routes ["export-backup" "Export Backup"]]
:admin-job-subform) :hx-get (bidi/path-for ssr-routes/only-routes
:hx-target "#sub-form" :admin-job-subform)
:hx-swap "innerHTML"}))) :hx-target "#sub-form"
:hx-swap "innerHTML"})))
[:div#sub-form (subform* {:name (fc/with-field :name (fc/field-value))}) ]] [:div#sub-form (subform* {:name (fc/with-field :name (fc/field-value))})]]
[:div [:div
(com/form-errors {:errors (:errors fc/*form-errors*)}) (com/form-errors {:errors (:errors fc/*form-errors*)})
(com/validated-save-button {:errors form-errors} "Run job")])]])))) (com/validated-save-button {:errors form-errors} "Run job")])]]))))
(def form-schema (mc/schema [:map (def form-schema (mc/schema [:map
[:name [:string {:min 1}]] [:name [:string {:min 1}]]
[:ledger-url {:optional true} [:string {:min 1}]] [:ledger-url {:optional true} [:string {:min 1}]]
[:invoice-url {:optional true} [:string {:min 1}]] [:invoice-url {:optional true} [:string {:min 1}]]
[:client {:optional true} entity-id] [:client {:optional true} entity-id]
[:days {:optional true} [:int {:min 1 :max 120}]] [:days {:optional true} [:int {:min 1 :max 120}]]]))
]))
(def key->handler (def key->handler
(apply-middleware-to-all-handlers (apply-middleware-to-all-handlers
(->> (->>
{:admin-jobs (helper/page-route grid-page) {:admin-jobs (helper/page-route grid-page)
:admin-job-table (helper/table-route grid-page) :admin-job-table (helper/table-route grid-page)
:admin-job-subform (-> subform (wrap-schema-enforce :query-schema [:map [:name {:optional true} [:maybe :string]]])) :admin-job-subform (-> subform (wrap-schema-enforce :query-schema [:map [:name {:optional true} [:maybe :string]]]))
:admin-job-start (-> job-start :admin-job-start (-> job-start
(wrap-schema-enforce :form-schema form-schema) (wrap-schema-enforce :form-schema form-schema)
(wrap-nested-form-params) (wrap-nested-form-params)
(wrap-form-4xx-2 job-start-dialog)) (wrap-form-4xx-2 job-start-dialog))
:admin-job-start-dialog job-start-dialog}) :admin-job-start-dialog job-start-dialog})
(fn [h] (fn [h]
(-> h (-> h
(wrap-admin) (wrap-admin)
(wrap-client-redirect-unauthenticated))))) (wrap-client-redirect-unauthenticated)))))

91
src/clj/auto_ap/ssr/pos/sales_summaries.clj
View File

@@ -4,6 +4,7 @@
:refer [apply-pagination apply-sort-3 conn merge-query pull-many :refer [apply-pagination apply-sort-3 conn merge-query pull-many
query2]] query2]]
[auto-ap.datomic.accounts :as d-accounts] [auto-ap.datomic.accounts :as d-accounts]
[auto-ap.datomic.sales-summaries :refer [total-credits total-debits]]
[auto-ap.graphql.utils :refer [extract-client-ids]] [auto-ap.graphql.utils :refer [extract-client-ids]]
[auto-ap.query-params :refer [wrap-copy-qp-pqp]] [auto-ap.query-params :refer [wrap-copy-qp-pqp]]
[auto-ap.client-routes :as client-routes] [auto-ap.client-routes :as client-routes]
@@ -116,18 +117,6 @@
(defn sort-items [ss] (defn sort-items [ss]
(sort-by (juxt :ledger-mapped/ledger-side :sales-summary-item/sort-order :sales-summary-item/category) ss)) (sort-by (juxt :ledger-mapped/ledger-side :sales-summary-item/sort-order :sales-summary-item/category) ss))
(defn total-debits [items]
(->> items
(filter #(= :ledger-side/debit (:ledger-mapped/ledger-side %)))
(map #(:ledger-mapped/amount % 0.0))
(reduce + 0.0)))
(defn total-credits [items]
(->> items
(filter #(= :ledger-side/credit (:ledger-mapped/ledger-side %)))
(map #(:ledger-mapped/amount % 0.0))
(reduce + 0.0)))
(defn truncate [s max-len] (defn truncate [s max-len]
(if (> (count s) max-len) (if (> (count s) max-len)
(str (subs s 0 (- max-len 3)) "...") (str (subs s 0 (- max-len 3)) "...")
@@ -158,13 +147,13 @@
[:span.text-sm account-name] [:span.text-sm account-name]
(com/pill {:color :red} "Missing acct")) (com/pill {:color :red} "Missing acct"))
(com/a-icon-button {:class "p-1" (com/a-icon-button {:class "p-1"
:hx-get (bidi/path-for ssr-routes/only-routes ::route/edit-item-account) :hx-get (bidi/path-for ssr-routes/only-routes ::route/edit-item-account)
:hx-target "closest .account-cell" :hx-target "closest .account-cell"
:hx-swap "outerHTML" :hx-swap "outerHTML"
:hx-vals (hx/json {:item-index (or (:item-index item) 0) :hx-vals (hx/json {:item-index (or (:item-index item) 0)
:client-id client-id :client-id client-id
:current-account-id (or account-id "")})} :current-account-id (or account-id "")})}
svg/pencil)])) svg/pencil)]))
(defn account-edit-cell [{:keys [field-name-prefix client-id current-account-id]}] (defn account-edit-cell [{:keys [field-name-prefix client-id current-account-id]}]
(let [account-input-name (str field-name-prefix "[ledger-mapped/account]")] (let [account-input-name (str field-name-prefix "[ledger-mapped/account]")]
@@ -172,23 +161,23 @@
(account-typeahead* {:name account-input-name (account-typeahead* {:name account-input-name
:value current-account-id :value current-account-id
:client-id client-id}) :client-id client-id})
[:div.flex.gap-1 [:div.flex.gap-1
(com/a-icon-button {:class "p-1" (com/a-icon-button {:class "p-1"
:hx-put (bidi/path-for ssr-routes/only-routes ::route/save-item-account) :hx-put (bidi/path-for ssr-routes/only-routes ::route/save-item-account)
:hx-target "closest .account-cell" :hx-target "closest .account-cell"
:hx-swap "outerHTML" :hx-swap "outerHTML"
:hx-include "closest .account-cell" :hx-include "closest .account-cell"
:hx-vals (hx/json {:field-name-prefix field-name-prefix :hx-vals (hx/json {:field-name-prefix field-name-prefix
:client-id client-id})} :client-id client-id})}
svg/check) svg/check)
(com/a-icon-button {:class "p-1" (com/a-icon-button {:class "p-1"
:hx-get (bidi/path-for ssr-routes/only-routes ::route/cancel-item-account) :hx-get (bidi/path-for ssr-routes/only-routes ::route/cancel-item-account)
:hx-target "closest .account-cell" :hx-target "closest .account-cell"
:hx-swap "outerHTML" :hx-swap "outerHTML"
:hx-vals (hx/json {:field-name-prefix field-name-prefix :hx-vals (hx/json {:field-name-prefix field-name-prefix
:client-id client-id :client-id client-id
:current-account-id (or current-account-id "")})} :current-account-id (or current-account-id "")})}
svg/x)]])) svg/x)]]))
(def grid-page (def grid-page
(helper/build {:id "entity-table" (helper/build {:id "entity-table"
@@ -576,8 +565,8 @@
[:span.text-gray-500 (truncate (:sales-summary-item/category item) 30)] [:span.text-gray-500 (truncate (:sales-summary-item/category item) 30)]
(account-display-cell {:item (assoc item :item-index actual-idx) (account-display-cell {:item (assoc item :item-index actual-idx)
:field-name-prefix (str "step-params[sales-summary/items][" actual-idx "]") :field-name-prefix (str "step-params[sales-summary/items][" actual-idx "]")
:client-id client-id}) :client-id client-id})
[:span.ml-auto.font-mono.tabular-nums.text-gray-900 (format "$%,.2f" (:ledger-mapped/amount item))]])) [:span.ml-auto.font-mono.tabular-nums.text-gray-900 (format "$%,.2f" (:ledger-mapped/amount item))]]))
[:div.h-6]))] [:div.h-6]))]
[:div.mt-2.border-t.pt-1 [:div.mt-2.border-t.pt-1
(summary-total-display request) (summary-total-display request)
@@ -619,13 +608,13 @@
[:span.text-gray-500 (truncate (:sales-summary-item/category item) 30)] [:span.text-gray-500 (truncate (:sales-summary-item/category item) 30)]
(account-display-cell {:item (assoc item :item-index actual-idx) (account-display-cell {:item (assoc item :item-index actual-idx)
:field-name-prefix (str "step-params[sales-summary/items][" actual-idx "]") :field-name-prefix (str "step-params[sales-summary/items][" actual-idx "]")
:client-id client-id}) :client-id client-id})
[:span.ml-auto.font-mono.tabular-nums.text-gray-900 (format "$%,.2f" (:ledger-mapped/amount item))]])) [:span.ml-auto.font-mono.tabular-nums.text-gray-900 (format "$%,.2f" (:ledger-mapped/amount item))]]))
[:div.h-6]))] [:div.h-6]))]
[:div.mt-2.border-t.pt-1 [:div.mt-2.border-t.pt-1
(summary-total-display request) (summary-total-display request)
(unbalanced-display request)]]] (unbalanced-display request)]]]
[:div.mt-4.border-t.pt-2 [:div.mt-4.border-t.pt-2
(fc/with-field :sales-summary/items (fc/with-field :sales-summary/items
(com/data-grid-new-row {:colspan 2 (com/data-grid-new-row {:colspan 2
:hx-get (bidi/path-for ssr-routes/only-routes ::route/new-summary-item) :hx-get (bidi/path-for ssr-routes/only-routes ::route/new-summary-item)
@@ -761,16 +750,16 @@
::route/edit-wizard-navigate (-> mm/next-handler ::route/edit-wizard-navigate (-> mm/next-handler
(mm/wrap-wizard edit-wizard) (mm/wrap-wizard edit-wizard)
(mm/wrap-decode-multi-form-state)) (mm/wrap-decode-multi-form-state))
::route/new-summary-item (-> (add-new-entity-handler [:step-params :sales-summary/items] ::route/new-summary-item (-> (add-new-entity-handler [:step-params :sales-summary/items]
(fn render [cursor request] (fn render [cursor request]
(sales-summary-item-row* (sales-summary-item-row*
{:value cursor {:value cursor
:client-id (:client-id (:query-params request))})) :client-id (:client-id (:query-params request))}))
(fn build-new-row [base _] (fn build-new-row [base _]
(assoc base :sales-summary-item/manual? true))) (assoc base :sales-summary-item/manual? true)))
(wrap-schema-enforce :query-schema [:map (wrap-schema-enforce :query-schema [:map
[:client-id {:optional true} [:client-id {:optional true}
[:maybe entity-id]]])) [:maybe entity-id]]]))
::route/edit-item-account (-> edit-item-account ::route/edit-item-account (-> edit-item-account
(wrap-schema-enforce :query-schema [:map (wrap-schema-enforce :query-schema [:map
[:item-index nat-int?] [:item-index nat-int?]

14
terraform/deploy.tf
View File

@@ -386,6 +386,20 @@ module "close_auto_invoices_job" {
cpu = 512 cpu = 512
} }
module "sales_summaries_job" {
count = var.enable_schedules ? 1 : 0
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
schedule = "rate(1 day)"
job_name = "sales-summaries"
execution_role_arn = var.execution_role_arn
use_schedule = true
memory = 4096
cpu = 2048
}
module "yodlee2_accounts_job" { module "yodlee2_accounts_job" {
count = var.enable_schedules ? 1 : 0 count = var.enable_schedules ? 1 : 0
source = "./background-job/" source = "./background-job/"