feat(sales): initial Parquet migration infrastructure
- Add DuckDB/S3 parquet storage layer (auto-ap.storage.parquet) - Add sales_to_parquet migration script for historical data - Add cleanup_sales for post-migration Datomic cleanup - Add sales_orders_new.clj with DuckDB read layer for SSR views - Add test scaffolding for parquet storage - Add plan document for move-detailed-sales-to-parquet feat(sales): redirect production and read flows to Parquet/DuckDB - U3: Square production (upsert) now buffers to parquet via flatten-order-to-parquet! - U3: EzCater core import-order now buffers to parquet instead of Datomic transact - U3: EzCater XLS upload-xls now buffers to parquet instead of audit-transact - U4: Rewrite sales_orders.clj to read from DuckDB via pq/get-sales-orders - U5: Rewrite sales_summaries to use parquet aggregation functions - get-payment-items-parquet, get-discounts-parquet, get-refund-items-parquet - get-tax-parquet, get-tip-parquet, get-sales-parquet - Add sum-* aggregation functions to storage/sales_summaries.clj - sum-discounts, sum-refunds-by-type, sum-taxes, sum-tips, sum-sales-by-category
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
[auto-ap.jobs.core :refer [execute]]
|
||||
[auto-ap.logging :as alog]
|
||||
[auto-ap.time :as atime]
|
||||
[auto-ap.storage.parquet :as pq]
|
||||
[clj-time.coerce :as c]
|
||||
[clj-time.core :as time]
|
||||
[clj-time.periodic :as per]
|
||||
@@ -98,99 +99,94 @@
|
||||
"card refunds" 41400
|
||||
"food app refunds" 41400})
|
||||
|
||||
(defn get-payment-items [c date]
|
||||
(->>
|
||||
(dc/q '[:find ?processor ?type-name (sum ?total)
|
||||
:with ?c
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-order/charges ?c]
|
||||
[?c :charge/type-name ?type-name]
|
||||
(or-join [?c ?processor]
|
||||
(and [?c :charge/processor ?p]
|
||||
[?p :db/ident ?processor])
|
||||
(and
|
||||
(not [?c :charge/processor])
|
||||
[(ground :ccp-processor/na) ?processor]))
|
||||
[?c :charge/total ?total]]
|
||||
(dc/db conn)
|
||||
[[c] date date])
|
||||
(reduce
|
||||
(fn [acc [processor type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Payments"
|
||||
(= type-name "CASH")
|
||||
"Cash Payments"
|
||||
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
|
||||
"Gift Card Payments"
|
||||
(#{:ccp-processor/toast
|
||||
#_:ccp-processor/ezcater
|
||||
#_:ccp-processor/koala
|
||||
:ccp-processor/doordash
|
||||
:ccp-processor/grubhub
|
||||
:ccp-processor/uber-eats} processor)
|
||||
"Food App Payments"
|
||||
:else
|
||||
"Unknown")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(map (fn [[k v]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 0
|
||||
:sales-summary-item/category k
|
||||
|
||||
:ledger-mapped/amount (if (= "Card Payments" k)
|
||||
(- v (get-fee c date))
|
||||
v)
|
||||
:ledger-mapped/ledger-side :ledger-side/debit}))))
|
||||
(defn- get-payment-items-parquet [c date]
|
||||
(let [date-str (.toString date)]
|
||||
(when-let [rows (seq (pq/query-deduped "charge" date-str date-str))]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
filtered (filter #(= client-code (:client_code %)) rows)]
|
||||
(reduce
|
||||
(fn [acc {:keys [processor type-name total]}]
|
||||
(update acc
|
||||
(cond
|
||||
(= type-name "CARD") "Card Payments"
|
||||
(= type-name "CASH") "Cash Payments"
|
||||
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name) "Gift Card Payments"
|
||||
(#{"doordash" "grubhub" "uber-eats"} processor) "Food App Payments"
|
||||
:else "Unknown")
|
||||
(fnil + 0.0)
|
||||
(or total 0.0)))
|
||||
{}
|
||||
filtered)))))
|
||||
|
||||
(defn get-discounts [c date]
|
||||
(when-let [discount (ffirst (dc/q '[:find (sum ?discount)
|
||||
:with ?e
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-order/discount ?discount]]
|
||||
(dc/db conn)
|
||||
[[c] date date]))]
|
||||
(defn- get-discounts-parquet [c date]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
date-str (.toString date)
|
||||
discount (auto-ap.storage.sales-summaries/sum-discounts client-code date-str date-str)]
|
||||
(when (and discount (pos? discount))
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 1
|
||||
:sales-summary-item/category "Discounts"
|
||||
:ledger-mapped/amount discount
|
||||
:ledger-mapped/ledger-side :ledger-side/debit})))
|
||||
|
||||
(defn- get-refund-items-parquet [c date]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
date-str (.toString date)
|
||||
refunds (auto-ap.storage.sales-summaries/sum-refunds-by-type client-code date-str date-str)]
|
||||
(when (seq refunds)
|
||||
(map (fn [[type-name total]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 3
|
||||
:sales-summary-item/category (cond
|
||||
(= type-name "CARD") "Card Refunds"
|
||||
(= type-name "CASH") "Cash Refunds"
|
||||
:else "Food App Refunds")
|
||||
:ledger-mapped/amount total
|
||||
:ledger-mapped/ledger-side :ledger-side/credit})
|
||||
refunds))))
|
||||
|
||||
(defn- get-fees [c date]
|
||||
(when-let [fee (get-fee c date)]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 1
|
||||
:sales-summary-item/category "Discounts"
|
||||
:ledger-mapped/amount discount
|
||||
:sales-summary-item/sort-order 2
|
||||
:sales-summary-item/category "Fees"
|
||||
:ledger-mapped/amount fee
|
||||
:ledger-mapped/ledger-side :ledger-side/debit}))
|
||||
|
||||
(defn get-refund-items [c date]
|
||||
(->>
|
||||
(dc/q '[:find ?type-name (sum ?t)
|
||||
:with ?e
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where
|
||||
:where [(iol-ion.query/scan-sales-refunds $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-refund/type ?type-name]
|
||||
[?e :sales-refund/total ?t]]
|
||||
(dc/db conn)
|
||||
[[c] date date])
|
||||
(reduce
|
||||
(fn [acc [type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Refunds"
|
||||
(= type-name "CASH")
|
||||
"Cash Refunds"
|
||||
:else
|
||||
"Food App Refunds")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(map (fn [[k v]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 3
|
||||
:sales-summary-item/category k
|
||||
:ledger-mapped/amount v
|
||||
:ledger-mapped/ledger-side :ledger-side/credit}))))
|
||||
(defn- get-tax-parquet [c date]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
date-str (.toString date)
|
||||
tax (auto-ap.storage.sales-summaries/sum-taxes client-code date-str date-str)]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/category "Tax"
|
||||
:sales-summary-item/sort-order 1
|
||||
:ledger-mapped/ledger-side :ledger-side/credit
|
||||
:ledger-mapped/amount (or tax 0.0)}))
|
||||
|
||||
(defn- get-tip-parquet [c date]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
date-str (.toString date)
|
||||
tip (auto-ap.storage.sales-summaries/sum-tips client-code date-str date-str)]
|
||||
{:ledger-mapped/ledger-side :ledger-side/credit
|
||||
:sales-summary-item/sort-order 2
|
||||
:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/category "Tip"
|
||||
:ledger-mapped/amount (or tip 0.0)}))
|
||||
|
||||
(defn- get-sales-parquet [c date]
|
||||
(let [client-code (if (map? c) (:client/code c) c)
|
||||
date-str (.toString date)
|
||||
sales (auto-ap.storage.sales-summaries/sum-sales-by-category client-code date-str date-str)]
|
||||
(for [{:keys [category total tax discount]} sales]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/category (or category "Unknown")
|
||||
:sales-summary-item/sort-order 0
|
||||
:sales-summary-item/total total
|
||||
:sales-summary-item/net (- (+ total discount) tax)
|
||||
:sales-summary-item/tax tax
|
||||
:sales-summary-item/discount discount
|
||||
:ledger-mapped/ledger-side :ledger-side/credit
|
||||
:ledger-mapped/amount (- (+ total discount) tax)})))
|
||||
|
||||
|
||||
|
||||
@@ -293,14 +289,13 @@
|
||||
|
||||
:sales-summary/items
|
||||
(->>
|
||||
(get-sales c date)
|
||||
(concat (get-payment-items c date))
|
||||
(concat (get-refund-items c date))
|
||||
(cons (get-discounts c date))
|
||||
(get-sales-parquet c date)
|
||||
(concat (get-payment-items-parquet c date))
|
||||
(concat (get-refund-items-parquet c date))
|
||||
(cons (get-discounts-parquet c date))
|
||||
(cons (get-fees c date))
|
||||
(cons (get-tax c date))
|
||||
(cons (get-tip c date))
|
||||
(cons (get-returns c date))
|
||||
(cons (get-tax-parquet c date))
|
||||
(cons (get-tip-parquet c date))
|
||||
(filter identity)
|
||||
(map (fn [z]
|
||||
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
|
||||
@@ -311,14 +306,13 @@
|
||||
(alog/info ::upserting-summaries
|
||||
:category-count (count (:sales-summary/items result)))
|
||||
@(dc/transact conn [[:upsert-entity result]]))
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
|
||||
(let [c (auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL" ])
|
||||
date #inst "2024-04-14T00:00:00-07:00"]
|
||||
(get-payment-items c date)
|
||||
|
||||
)
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
|
||||
(comment
|
||||
;; TODO: Move to test file or proper location
|
||||
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ])
|
||||
date #inst "2024-04-14T00:00:00-07:00"]
|
||||
(get-payment-items c date)))
|
||||
|
||||
(defn reset-summaries []
|
||||
@(dc/transact conn (->> (dc/q '[:find ?sos
|
||||
|
||||
Reference in New Issue
Block a user