notid/integreat
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat(sales-summaries): refresh on a schedule, skipping accepted days

Browse Source 
sales-summaries-v2 recomputes every dirty summary, but nothing set the dirty
flag on a schedule: mark-all-dirty was only ever called by hand from the
comment block, and the job was registered in neither server.clj's
INTEGREAT_JOB dispatch nor terraform/deploy.tf, so -main was dead code that
could never run in production. Summaries were only recalculated when someone
remembered to do it in the REPL, and POS data keeps arriving after a business
day closes, so a summary computed once on the day was routinely wrong and
stayed wrong.

Add a daily job that marks the trailing 7 days dirty and recomputes them,
leaving finished work alone. "Finished" is the condition the app already calls
Balanced -- debits equal credits and every line is mapped to an account. Since
that is derived rather than stored, a summary that later falls out of balance
is picked up again on the next run.

Extract the Balanced predicate into auto-ap.datomic.sales-summaries so the
grid's pill and the job share one definition, rather than a background job
requiring an SSR namespace. total-debits/total-credits resolve the ledger side
from either a plain keyword or the {:db/ident ...} map a pull returns, and
accepted? requires every item to declare a side: un-normalized pulled items
otherwise sum to 0.0 on both sides, read as balanced, and get skipped
silently and permanently.

Also fix sales-summaries-v2 destroying user-entered line items. It filtered
for :sales-summary-item/manual? to preserve them, but dirty-sales-summaries'
index-pull selector never fetched :sales-summary/items, so manual-items was
always empty. Because items is a component attribute upserted via
[:reset-rels ...], every recompute deleted the hand-entered lines -- often the
very lines that make a summary balance. Harmless while nothing ran on a
schedule; destructive the moment this does.

Register the job in the admin Background Jobs dropdown too, with a days
field: schedules are prod-only, so the admin page is the only way to run it
on staging.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Bryce
2026-08-12 15:01:10 -07:00
parent fc54b92ddb
commit 95d0537c69
6 changed files with 418 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
src/clj/auto_ap/jobs/sales_summaries.clj
View File

@@ -1,5 +1,6 @@
(ns auto-ap.jobs.sales-summaries
(:require [auto-ap.datomic :refer [conn]]
[auto-ap.datomic.sales-summaries :as d-ss]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.logging :as alog]
[auto-ap.time :as atime]
@@ -8,6 +9,7 @@
[clj-time.periodic :as per]
[clojure.string :as str]
[com.brunobonacci.mulog :as mu]
[config.core :refer [env]]
[datomic.api :as dc]))
(defn mark-dirty [client start end]
@@ -39,27 +41,100 @@
(dc/db conn)
number)))
(defn delete-all []
@(dc/transact-async conn
(->>
(dc/q '[:find ?ss
:where [?ss :sales-summary/date]]
(dc/db conn))
(map (fn [[ ss]]
[:db/retractEntity ss])))))
(->>
(dc/q '[:find ?ss
:where [?ss :sales-summary/date]]
(dc/db conn))
(map (fn [[ss]]
[:db/retractEntity ss])))))
(def item-read
"Enough of a summary item to both evaluate `d-ss/accepted?` and transact the item back
unchanged. `:db/id` matters: `:sales-summary/items` is a component attribute upserted via
`[:reset-rels ...]`, so an item re-transacted without its id is deleted and recreated."
'[:db/id
:sales-summary-item/category
:sales-summary-item/sort-order
:sales-summary-item/manual?
:ledger-mapped/amount
{:ledger-mapped/ledger-side [:db/ident]}
{:ledger-mapped/account [:db/id]}])
(defn dirty-sales-summaries [c]
(let [client-id (dc/entid (dc/db conn) c)]
(->> (dc/index-pull (dc/db conn)
{:index :avet
:selector '[:sales-summary/date :sales-summary/client :db/id]
:selector (conj '[:sales-summary/date :sales-summary/client :db/id]
{:sales-summary/items item-read})
:start [:sales-summary/client+dirty [client-id true]]})
(filter (fn [sales-summary]
(= client-id (:db/id (:sales-summary/client sales-summary))))))))
(def default-days
"How far back the scheduled refresh looks for summaries that still need recomputing."
7)
(defn trailing-window
"`[start end)` covering the last `days` business days, ending with today. `end` is
exclusive, matching both `periodic-seq`'s 3-arity and the grid's date filters, so
`(trailing-window 7)` is day -6 through today inclusive."
[days]
[(.toDateMidnight (atime/localize (time/minus (time/now) (time/days (dec days)))))
(.toDateMidnight (atime/localize (time/plus (time/now) (time/days 1))))])
(defn accepted-client+dates
"Set of `[client-id date]` pairs in `[start end)` whose summary is already accepted, and
so should be left alone rather than re-marked. Accepted means balanced with every line
mapped to an account — the condition the grid renders as \"Balanced\"."
[db start end]
(->> (dc/q '[:find (pull ?ss selector)
:in $ ?start ?end selector
:where
[?ss :sales-summary/date ?d]
[(>= ?d ?start)]
[(< ?d ?end)]]
db
(c/to-date start)
(c/to-date end)
(conj '[:sales-summary/date {:sales-summary/client [:db/id]}]
{:sales-summary/items item-read}))
(map first)
(filter #(d-ss/accepted? (map d-ss/<-pulled-item (:sales-summary/items %))))
(map (juxt (comp :db/id :sales-summary/client) :sales-summary/date))
set))
(defn mark-stale-dirty
"Marks every client/day in the trailing `days` window dirty so `sales-summaries-v2` will
recompute it, skipping days whose summary is already accepted. Because accepted is derived
rather than stored, a summary that later falls out of balance is picked up again on the
next run. Returns the number of client/days marked."
[days]
(let [db (dc/db conn)
[start end] (trailing-window days)
accepted (accepted-client+dates db start end)
clients (map first (dc/q '[:find ?c
:in $
:where [_ :sales-order/client ?c]]
db))
dates (map c/to-date (per/periodic-seq start end (time/days 1)))
tx-data (for [client clients
date dates
:when (not (accepted [client date]))]
{:sales-summary/client client
:sales-summary/date date
:sales-summary/dirty true
:sales-summary/client+date [client date]})]
(alog/info ::marking-dirty
:days days
:client-count (count clients)
:accepted-count (count accepted)
:marked (count tx-data))
(doseq [batch (partition-all 500 tx-data)]
@(dc/transact conn batch))
(count tx-data)))
(defn- get-fee [c date]
(- (or (ffirst (dc/q '[:find ?f
:in $ ?client ?d
@@ -99,53 +174,53 @@
"food app refunds" 41400})
(defn get-payment-items [c date]
(->>
(dc/q '[:find ?processor ?type-name (sum ?total)
:with ?c
:in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/charges ?c]
[?c :charge/type-name ?type-name]
(or-join [?c ?processor]
(and [?c :charge/processor ?p]
[?p :db/ident ?processor])
(and
(not [?c :charge/processor])
[(ground :ccp-processor/na) ?processor]))
[?c :charge/total ?total]]
(dc/db conn)
[[c] date date])
(reduce
(fn [acc [processor type-name total]]
(update
acc
(cond (= type-name "CARD")
"Card Payments"
(= type-name "CASH")
"Cash Payments"
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
"Gift Card Payments"
(#{:ccp-processor/toast
#_:ccp-processor/ezcater
#_:ccp-processor/koala
:ccp-processor/doordash
:ccp-processor/grubhub
:ccp-processor/uber-eats} processor)
"Food App Payments"
:else
"Unknown")
(fnil + 0.0)
total))
{})
(map (fn [[k v]]
{:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 0
:sales-summary-item/category k
:ledger-mapped/amount (if (= "Card Payments" k)
(- v (get-fee c date))
v)
:ledger-mapped/ledger-side :ledger-side/debit}))))
(->>
(dc/q '[:find ?processor ?type-name (sum ?total)
:with ?c
:in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/charges ?c]
[?c :charge/type-name ?type-name]
(or-join [?c ?processor]
(and [?c :charge/processor ?p]
[?p :db/ident ?processor])
(and
(not [?c :charge/processor])
[(ground :ccp-processor/na) ?processor]))
[?c :charge/total ?total]]
(dc/db conn)
[[c] date date])
(reduce
(fn [acc [processor type-name total]]
(update
acc
(cond (= type-name "CARD")
"Card Payments"
(= type-name "CASH")
"Cash Payments"
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
"Gift Card Payments"
(#{:ccp-processor/toast
#_:ccp-processor/ezcater
#_:ccp-processor/koala
:ccp-processor/doordash
:ccp-processor/grubhub
:ccp-processor/uber-eats} processor)
"Food App Payments"
:else
"Unknown")
(fnil + 0.0)
total))
{})
(map (fn [[k v]]
{:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 0
:sales-summary-item/category k
:ledger-mapped/amount (if (= "Card Payments" k)
(- v (get-fee c date))
v)
:ledger-mapped/ledger-side :ledger-side/debit}))))
(defn get-discounts [c date]
(when-let [discount (ffirst (dc/q '[:find (sum ?discount)
@@ -162,7 +237,7 @@
:ledger-mapped/ledger-side :ledger-side/debit}))
(defn get-refund-items [c date]
(->>
(->>
(dc/q '[:find ?type-name (sum ?t)
:with ?e
:in $ [?clients ?start-date ?end-date]
@@ -173,26 +248,24 @@
(dc/db conn)
[[c] date date])
(reduce
(fn [acc [type-name total]]
(update
acc
(cond (= type-name "CARD")
"Card Refunds"
(= type-name "CASH")
"Cash Refunds"
:else
"Food App Refunds")
(fnil + 0.0)
total))
{})
(fn [acc [type-name total]]
(update
acc
(cond (= type-name "CARD")
"Card Refunds"
(= type-name "CASH")
"Cash Refunds"
:else
"Food App Refunds")
(fnil + 0.0)
total))
{})
(map (fn [[k v]]
{:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 3
:sales-summary-item/category k
:ledger-mapped/amount v
:ledger-mapped/ledger-side :ledger-side/credit}))))
{:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 3
:sales-summary-item/category k
:ledger-mapped/amount v
:ledger-mapped/ledger-side :ledger-side/credit}))))
(defn get-fees [c date]
(when-let [fee (get-fee c date)]
@@ -278,17 +351,18 @@
(defn sales-summaries-v2 []
(doseq [[c client-code] (dc/q '[:find ?c ?client-code
:in $
:where [?c :client/code ?client-code]]
(dc/db conn))
{:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)]
:in $
:where [?c :client/code ?client-code]]
(dc/db conn))
{:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)]
(mu/with-context {:client-code client-code
:date date}
(alog/info ::updating)
(let [manual-items (->> existing-summary
:sales-summary/items
(filter :sales-summary-item/manual?))
calculated-items (->>
:date date}
(alog/info ::updating)
(let [manual-items (->> existing-summary
:sales-summary/items
(filter :sales-summary-item/manual?)
(map d-ss/<-pulled-item))
calculated-items (->>
(get-sales c date)
(concat (get-payment-items c date))
(concat (get-refund-items c date))
@@ -301,20 +375,19 @@
(map (fn [z]
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
:sales-summary-item/manual? false))))
all-items (concat calculated-items manual-items)
result {:db/id id
:sales-summary/client c
:sales-summary/date date
:sales-summary/dirty false
:sales-summary/client+date [c date]
:sales-summary/items all-items}]
(if (seq (:sales-summary/items result))
(do
(alog/info ::upserting-summaries
:category-count (count (:sales-summary/items result)))
@(dc/transact conn [[:upsert-sales-summary result]]))
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
all-items (concat calculated-items manual-items)
result {:db/id id
:sales-summary/client c
:sales-summary/date date
:sales-summary/dirty false
:sales-summary/client+date [c date]
:sales-summary/items all-items}]
(if (seq (:sales-summary/items result))
(do
(alog/info ::upserting-summaries
:category-count (count (:sales-summary/items result)))
@(dc/transact conn [[:upsert-sales-summary result]]))
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
(defn reset-summaries []
@(dc/transact conn (->> (dc/q '[:find ?sos
@@ -324,9 +397,6 @@
(map (fn [[sos]]
[:db/retractEntity sos])))))
(comment
(auto-ap.datomic/transact-schema conn)
@@ -336,26 +406,19 @@
(dirty-sales-summaries [:client/code "NGWH"])
(apply mark-dirty [:client/code "NGWH"] (last-n-days 5))
(iol-ion.tx.upsert-sales-summary-ledger/summary->journal-entry (dc/db conn) 17592314245819)
(iol-ion.tx.upsert-sales-summary-ledger/upsert-sales-summary (dc/db conn) {:db/id 17592314241429})
(mark-all-dirty 5)
(delete-all)
(sales-summaries-v2)
1
(dc/q '[:find (pull ?sos [* {:sales-summary/sales-items [*]}])
:in $
:where [?sos :sales-summary/client [:client/code "NGHW"]]
@@ -386,15 +449,24 @@
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
(auto-ap.datomic/transact-schema conn)
)
(auto-ap.datomic/transact-schema conn))
(defn days-arg
"Trailing-window size from the job's `args`, e.g. `{:days 30}` set as a container override
from the admin Background Jobs page for an ad-hoc wider backfill. Values arrive as EDN but
may still be strings, so coerce defensively the way load-historical-sales does."
[args]
(let [days (:days args)]
(cond-> (or days default-days)
(string? days) (#(Long/parseLong %)))))
(defn refresh-sales-summaries
"Marks the trailing `days` window dirty, skipping accepted summaries, then recomputes
everything left dirty."
([] (refresh-sales-summaries default-days))
([days]
(mark-stale-dirty days)
(sales-summaries-v2)))
(defn -main [& _]
(execute "sales-summaries" sales-summaries-v2))
(execute "sales-summaries" #(refresh-sales-summaries (days-arg (:args env)))))