Bryce
97e051db81
- Add DuckDB/S3 parquet storage layer (auto-ap.storage.parquet) - Add sales_to_parquet migration script for historical data - Add cleanup_sales for post-migration Datomic cleanup - Add sales_orders_new.clj with DuckDB read layer for SSR views - Add test scaffolding for parquet storage - Add plan document for move-detailed-sales-to-parquet feat(sales): redirect production and read flows to Parquet/DuckDB - U3: Square production (upsert) now buffers to parquet via flatten-order-to-parquet! - U3: EzCater core import-order now buffers to parquet instead of Datomic transact - U3: EzCater XLS upload-xls now buffers to parquet instead of audit-transact - U4: Rewrite sales_orders.clj to read from DuckDB via pq/get-sales-orders - U5: Rewrite sales_summaries to use parquet aggregation functions - get-payment-items-parquet, get-discounts-parquet, get-refund-items-parquet - get-tax-parquet, get-tip-parquet, get-sales-parquet - Add sum-* aggregation functions to storage/sales_summaries.clj - sum-discounts, sum-refunds-by-type, sum-taxes, sum-tips, sum-sales-by-category
270 lines
12 KiB
Clojure
270 lines
12 KiB
Clojure
(ns auto-ap.routes.ezcater-xls
|
|
(:require
|
|
[auto-ap.datomic :refer [conn]]
|
|
[auto-ap.logging :as alog]
|
|
[clojure.data.json :as json]
|
|
[auto-ap.parse.excel :as excel]
|
|
[auto-ap.parse :as parse]
|
|
[amazonica.aws.lambda :as lambda]
|
|
[config.core :refer [env]]
|
|
[auto-ap.shared-views.admin.side-bar :refer [admin-side-bar]]
|
|
[auto-ap.ssr-routes :as ssr-routes]
|
|
[auto-ap.ssr.ui :refer [base-page]]
|
|
[auto-ap.ssr.utils :refer [html-response]]
|
|
[auto-ap.time :as atime]
|
|
[auto-ap.storage.parquet :as parquet]
|
|
[bidi.bidi :as bidi]
|
|
[clj-time.coerce :as coerce]
|
|
[clojure.java.io :as io]
|
|
[com.brunobonacci.mulog :as mu]
|
|
[datomic.api :as dc]
|
|
[hiccup2.core :as hiccup]
|
|
[amazonica.aws.s3 :as s3]
|
|
[auto-ap.ssr.components :as com]))
|
|
|
|
(defn fmt-amount [a]
|
|
(with-precision 2
|
|
(some-> a
|
|
bigdec
|
|
(.setScale 2 java.math.RoundingMode/HALF_UP)
|
|
(double))))
|
|
|
|
(defn rows->maps [rows]
|
|
(let [[headers & rows] rows]
|
|
(for [r rows]
|
|
(into {}
|
|
(map vector headers r)))))
|
|
|
|
(defn map->sales-order [r clients]
|
|
(println r)
|
|
(let [order-number (get r "Order Number")
|
|
event-date (get r "Event Date")
|
|
store-name (get r "Store Name")
|
|
adjustments (some-> (get r "Adjustments") not-empty (Double/parseDouble))
|
|
tax (some-> (get r "Sales Tax") not-empty (Double/parseDouble))
|
|
food-total (some-> (get r "Food Total") not-empty (Double/parseDouble))
|
|
commission (some-> (get r "Commission") not-empty (Double/parseDouble))
|
|
fee (some-> (get r "Payment Transaction Fee") not-empty (Double/parseDouble))
|
|
tip (some-> (get r "Tip") not-empty (Double/parseDouble))
|
|
caterer-name (get r "Caterer Name")
|
|
client (some->> caterer-name
|
|
not-empty
|
|
(parse/exact-match clients))
|
|
client-id (:db/id client)
|
|
location (first (:client/locations client))
|
|
event-date (some-> (excel/xls-date->date event-date)
|
|
coerce/to-date-time
|
|
atime/as-local-time
|
|
coerce/to-date )]
|
|
(cond (and event-date client-id location )
|
|
[:order #:sales-order
|
|
{:date event-date
|
|
:external-id (str "ezcater/order/" client-id "-" location "-" order-number)
|
|
:client client-id
|
|
:location location
|
|
:reference-link (str order-number)
|
|
:line-items [#:order-line-item
|
|
{:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0)
|
|
:item-name "EZCater Catering"
|
|
:category "EZCater Catering"
|
|
:discount (fmt-amount (or adjustments 0.0))
|
|
:tax (fmt-amount tax)
|
|
:total (fmt-amount (+ food-total
|
|
tax))}]
|
|
|
|
:charges [#:charge
|
|
{:type-name "CARD"
|
|
:date event-date
|
|
:client client-id
|
|
:location location
|
|
:external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
|
|
:processor :ccp-processor/ezcater
|
|
:total (fmt-amount (+ food-total
|
|
tax
|
|
tip))
|
|
:tip (fmt-amount tip)}]
|
|
:total (fmt-amount (+ food-total
|
|
tax
|
|
(or adjustments 0.0)))
|
|
:discount (fmt-amount (or adjustments 0.0))
|
|
:service-charge (fmt-amount (+ fee commission))
|
|
:tax (fmt-amount tax)
|
|
:tip (fmt-amount tip)
|
|
:returns 0.0
|
|
:vendor :vendor/ccp-ezcater}]
|
|
|
|
caterer-name
|
|
(do
|
|
(alog/warn ::missing-client
|
|
:order order-number
|
|
:store-name store-name
|
|
:caterer-name caterer-name)
|
|
[:missing caterer-name])
|
|
|
|
:else
|
|
nil)))
|
|
|
|
(defn- flatten-order-to-parquet! [order]
|
|
"Flatten a sales-order into entity-type tagged maps and buffer to parquet."
|
|
(let [so-ext-id (:sales-order/external-id order)
|
|
so-date (some-> (:sales-order/date order) .toString)
|
|
client (:sales-order/client order)
|
|
client-code (if (map? client) (:client/code client) client)]
|
|
(parquet/buffer! "sales-order"
|
|
{:entity-type "sales-order"
|
|
:external-id so-ext-id
|
|
:client-code client-code
|
|
:location (:sales-order/location order)
|
|
:vendor (:sales-order/vendor order)
|
|
:total (:sales-order/total order)
|
|
:tax (:sales-order/tax order)
|
|
:tip (:sales-order/tip order)
|
|
:discount (:sales-order/discount order)
|
|
:service-charge (:sales-order/service-charge order)
|
|
:date so-date})
|
|
(when-let [charges (:sales-order/charges order)]
|
|
(doseq [chg charges]
|
|
(parquet/buffer! "charge"
|
|
{:entity-type "charge"
|
|
:external-id (:charge/external-id chg)
|
|
:type-name (:charge/type-name chg)
|
|
:total (:charge/total chg)
|
|
:tax (:charge/tax chg)
|
|
:tip (:charge/tip chg)
|
|
:date so-date
|
|
:processor (some-> (:charge/processor chg) name)
|
|
:sales-order-external-id so-ext-id})))
|
|
(when-let [items (:sales-order/line-items order)]
|
|
(doseq [li items]
|
|
(parquet/buffer! "line-item"
|
|
{:entity-type "line-item"
|
|
:item-name (:order-line-item/item-name li)
|
|
:category (:order-line-item/category li)
|
|
:total (:order-line-item/total li)
|
|
:tax (:order-line-item/tax li)
|
|
:discount (:order-line-item/discount li)
|
|
:sales-order-external-id so-ext-id})))))
|
|
|
|
(defn stream->sales-orders [s]
|
|
(let [clients (map first (dc/q '[:find (pull ?c [:client/code
|
|
:db/id
|
|
:client/feature-flags
|
|
{:client/location-matches [:location-match/matches :location-match/location]}
|
|
:client/name
|
|
:client/matches
|
|
:client/locations])
|
|
:where [?c :client/code]]
|
|
(dc/db conn)))
|
|
object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))]
|
|
(mu/log ::writing-temp-xls
|
|
:location object)
|
|
(s3/put-object {:bucket-name (:data-bucket env)
|
|
:key object
|
|
:input-stream s})
|
|
(into []
|
|
(->> (excel/extract-sheet-details (:data-bucket env) object)
|
|
rows->maps
|
|
(map #(map->sales-order % clients))
|
|
(filter identity)))))
|
|
|
|
(defn import-stuff []
|
|
(with-open [s (io/input-stream "/home/brycecovert/Downloads/test_2023-04-01_2023-04-30_2023-05-04.xlsx")]
|
|
(stream->sales-orders s)))
|
|
|
|
(defn page* []
|
|
[:div.mt-4
|
|
(com/card {}
|
|
[:div.px-4.py-3.space-y-4.flex.flex-col
|
|
[:h1.text-2xl.mb-3.font-bold "EZCater XLS Import"]
|
|
[:p.text-sm.italic
|
|
"Please go to "
|
|
(com/link {:href "https://www.ezcater.com/ez_manage/reports/new" :target "_blank"} "EZCater's report page")
|
|
" to generate a new report. Then drop it below."]
|
|
[:div#page-notification.notification.block {:style {:display "none"}}]
|
|
[:form.bg-blue-100.border-2.border-dashed.rounded-lg.border-blue-300.p-4.max-w-md.w-md.text-center.cursor-pointer
|
|
{:action (bidi/path-for ssr-routes/only-routes
|
|
:admin-ezcater-xls)
|
|
:method "POST"
|
|
:id "ezcater"}
|
|
"Drop the ezcater xls here."]
|
|
[:script
|
|
(hiccup/raw
|
|
"
|
|
ezcater_dropzone = new Dropzone (\"#ezcater\", {
|
|
success: function (file, response) {
|
|
document.getElementById(\"page-notification\").innerHTML = response;
|
|
document.getElementById(\"page-notification\").style[\"display\"] = \"block\";
|
|
},
|
|
acceptedFiles: '.xls,.xlsx',
|
|
disablePreviews: true
|
|
});")]])])
|
|
|
|
(defn upload-xls [{:keys [identity] :as request}]
|
|
|
|
(let [file (or (get (:params request) :file)
|
|
(get (:params request) "file"))]
|
|
(mu/log ::uploading-file
|
|
:file file)
|
|
(with-open [s (io/input-stream (:tempfile file))]
|
|
(try
|
|
(let [parse-results (stream->sales-orders s)
|
|
new-orders (->> parse-results
|
|
(filter (comp #{:order} first))
|
|
(map last))
|
|
|
|
missing-location (->> parse-results
|
|
(filter (comp #{:missing} first))
|
|
(map last))
|
|
buffered-count (loop [orders new-orders
|
|
count 0]
|
|
(if-let [o (first orders)]
|
|
(do
|
|
(try
|
|
(flatten-order-to-parquet! o)
|
|
(catch Exception e
|
|
(alog/error ::buffer-failed
|
|
:exception e
|
|
:order (:sales-order/external-id o))))
|
|
(recur (rest orders) (inc count)))
|
|
count))]
|
|
(html-response [:div (format "Successfully imported %d orders." buffered-count)
|
|
(when (seq missing-location)
|
|
[:div "Missing the following locations"
|
|
[:ul.ul
|
|
(for [ml (into #{} missing-location)]
|
|
[:li ml])]])]))
|
|
(catch Exception e
|
|
(alog/error ::import-error
|
|
:error e)
|
|
(html-response [:div (.getMessage e)]))))))
|
|
|
|
(defn page [{:keys [matched-route request-method] :as request}]
|
|
(mu/log ::method
|
|
:method request-method)
|
|
(if (= :post request-method)
|
|
(upload-xls request)
|
|
(base-page
|
|
request
|
|
(com/page {:nav com/admin-aside-nav
|
|
:client-selection (:client-selection request)
|
|
:client (:client request)
|
|
:clients (:clients request)
|
|
:identity (:identity request)
|
|
:request request
|
|
:app-params {:hx-get (bidi/path-for ssr-routes/only-routes
|
|
:admin-ezcater-xls)
|
|
:hx-trigger "clientSelected from:body"
|
|
:hx-select "#app-contents"
|
|
:hx-swap "outerHTML swap:300ms"}}
|
|
(com/breadcrumbs {}
|
|
[:a {:href (bidi/path-for ssr-routes/only-routes
|
|
:admin)}
|
|
"Admin"]
|
|
[:a {:href (bidi/path-for ssr-routes/only-routes
|
|
:admin-ezcater-xls)}
|
|
"EZCater XLS Import"])
|
|
(page*))
|
|
|
|
"Invoice Glimpse")))
|
|
|