notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat(sales): wire SSR page to parquet/DuckDB layer with full 7.9M-record support

Browse Source 
- Add fetch-page-ssr and summarize-page-ssr to read from parquet via DuckDB
- Add get-sales-orders-summary for cross-page totals (SUM across all rows)
- Optimize parquet-query for large ranges (>60 days) with year-level globs
- Add default-date-range with fallback to data's actual range
- Fix migration: flatten-order-to-pieces! vswap!, pull specs, date handling
- Add denormalized columns: payment-methods, processors, categories, source
- Handle schema-enforce middleware stripping dates via raw query-string parsing
- Add graceful fallback for missing parquet files (catch Exception)
- Fix load-unflushed! with .exists check on WAL files
This commit is contained in:
Bryce
2026-04-27 20:05:13 -07:00
parent ea7f46ea8a
commit 9153494ed7
4 changed files with 329 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
src/clj/auto_ap/datomic/sales_orders.clj
View File

@@ -1,30 +1,42 @@
(ns auto-ap.datomic.sales-orders (ns auto-ap.datomic.sales-orders
(:require (:require
[auto-ap.storage.parquet :as pq] [auto-ap.storage.parquet :as pq]
[auto-ap.time :as atime]
[clj-time.coerce :as coerce]
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[com.brunobonacci.mulog :as mu])) [com.brunobonacci.mulog :as mu]
[ring.util.codec :as ring-codec]))
(defn- payment-methods->charges [pm-str]
(when (not-empty pm-str)
(mapv (fn [pm] {:charge/type-name pm})
(str/split pm-str #","))))
(defn <-row (defn <-row
"Convert a flat parquet row into the shape consumers expect. "Convert a flat parquet row into the shape consumers expect."
Parquet produces maps of the form:
{\"external-id\" \"square/order/123\", ...}
which we transform to:
{:sales-order/external-id \"square/order/123\", ...}"
[row] [row]
(-> row (let [pm (:payment-methods row)]
(set/rename-keys (-> row
{"external-id" :sales-order/external-id (set/rename-keys
"location" :sales-order/location {:external-id :sales-order/external-id
"total" :sales-order/total :location :sales-order/location
"tax" :sales-order/tax :total :sales-order/total
"tip" :sales-order/tip :tax :sales-order/tax
"discount" :sales-order/discount :tip :sales-order/tip
"service-charge" :sales-order/service-charge :discount :sales-order/discount
"vendor" :sales-order/vendor :service-charge :sales-order/service-charge
"client-code" :sales-order/client-code :vendor :sales-order/vendor
"date" :sales-order/date}) :client-code :sales-order/client-code
(update :sales-order/date #(some-> % str)))) :date :sales-order/date
:source :sales-order/source
:reference-link :sales-order/reference-link
:payment-methods :sales-order/payment-methods
:processors :sales-order/processors
:categories :sales-order/categories})
(update :sales-order/date #(some-> % str))
(dissoc :entity-type :_seq-no)
(assoc :sales-order/charges (payment-methods->charges pm)))))
(defn build-where-clause [args] (defn build-where-clause [args]
(let [clauses (keep identity (let [clauses (keep identity
@@ -58,17 +70,100 @@
:order "DESC" :order "DESC"
:limit limit :limit limit
:offset offset})] :offset offset})]
{:ids (mapv #(str (:external_id %)) (:rows result)) {:ids (mapv #(str (:external-id %)) (:rows result))
:rows (:rows result) :rows (:rows result)
:count (:count result)})))) :count (:count result)}))))
(defn graphql-results [rows _ids _args] (defn graphql-results [rows _ids _args]
(mapv <-row rows)) (mapv <-row rows))
(defn- extract-date-str [v]
(when v
(cond
(string? v) (if (> (count v) 10) (.substring v 0 10) v)
(instance? org.joda.time.DateTime v) (atime/unparse-local v atime/normal-date)
(instance? org.joda.time.LocalDate v) (atime/unparse-local v atime/normal-date)
(instance? java.util.Date v) (atime/unparse-local (coerce/to-date-time v) atime/normal-date)
(instance? java.time.LocalDate v) (.toString v)
:else (str v))))
(defn- get-date [qp k]
(or (extract-date-str (get qp k))
(extract-date-str (get qp (name k)))))
(defn- kw->str [v]
(when (some? v)
(if (keyword? v) (name v) (str v))))
(defn- qp->opts [qp]
(let [sort-params (:sort qp)
sort-key (when (seq sort-params) (-> sort-params first :name))
sort-dir (when (seq sort-params) (-> sort-params first :dir))]
(cond-> {}
(some? (:client-code qp)) (assoc :client (kw->str (:client-code qp)))
(some? (:location qp)) (assoc :location (kw->str (:location qp)))
(not-empty (:payment-method qp)) (assoc :payment-method (:payment-method qp))
(some? (:processor qp)) (assoc :processor (kw->str (:processor qp)))
(not-empty (:category qp)) (assoc :category (:category qp))
(:total-gte qp) (assoc :total-gte (:total-gte qp))
(:total-lte qp) (assoc :total-lte (:total-lte qp))
sort-key (assoc :sort sort-key)
sort-dir (assoc :order (or sort-dir "DESC"))
true (assoc :limit (or (:per-page qp) 25)
:offset (or (:start qp) 0)))))
(defn- last-week-range []
(let [today (java.time.LocalDate/now)
end (.toString (.minusDays today 1))
start (.toString (.minusDays today 8))]
[start end]))
(defn- default-date-range []
(let [[s e] (last-week-range)
result (try (pq/get-sales-orders-summary s e) (catch Exception _ nil))]
(if (and result (> (:total result) 0))
[s e]
(let [yesterday (.toString (.minusDays (java.time.LocalDate/of 2024 4 24) 1))
week-before (.toString (.minusDays (java.time.LocalDate/of 2024 4 24) 8))]
[week-before yesterday]))))
(defn- qp->date-range [qp]
(let [[default-start default-end] (default-date-range)]
[(or (get-date qp :start-date)
(extract-date-str (get-in qp [:date-range :start]))
default-start)
(or (get-date qp :end-date)
(extract-date-str (get-in qp [:date-range :end]))
default-end)]))
(defn fetch-page-ssr
"Fetch sales orders from parquet for the SSR page."
[request]
(let [qp (:query-params request)
raw-qp (some-> (:query-string request)
ring-codec/form-decode
(->> (into {} (remove (fn [[_ v]] (str/blank? v))))))
[start end] (qp->date-range (merge raw-qp qp))
opts (qp->opts qp)
result (pq/get-sales-orders start end opts)
rows (mapv <-row (:rows result))]
{:rows rows :count (:count result)}))
(defn summarize-page-ssr
"Summarize all matching sales orders via parquet."
[request]
(let [qp (:query-params request)
raw-qp (some-> (:query-string request)
ring-codec/form-decode
(->> (into {} (remove (fn [[_ v]] (str/blank? v))))))
[start end] (qp->date-range (merge raw-qp qp))
opts (dissoc (qp->opts qp) :limit :offset :sort :order)]
(pq/get-sales-orders-summary start end opts)))
(defn summarize-orders [rows] (defn summarize-orders [rows]
(when (seq rows) (when (seq rows)
(let [total (reduce + 0.0 (map #(or (:total %) 0.0) rows)) (let [total (reduce + 0.0 (map #(or (:sales-order/total %) 0.0) rows))
tax (reduce + 0.0 (map #(or (:tax %) 0.0) rows))] tax (reduce + 0.0 (map #(or (:sales-order/tax %) 0.0) rows))]
{:total total {:total total
:tax tax}))) :tax tax})))

138
src/clj/auto_ap/migration/sales_to_parquet.clj
View File

@@ -10,14 +10,14 @@
(write-dead-letter [flat]) ; write orphaned records" (write-dead-letter [flat]) ; write orphaned records"
(:require [auto-ap.datomic :refer [conn]] (:require [auto-ap.datomic :refer [conn]]
[auto-ap.storage.parquet :as p] [auto-ap.storage.parquet :as p]
[datomic.api :as dc] [clojure.string :as str]
[clj-time.core :as time])) [datomic.api :as dc]))
(defn- fetch-all-sales-order-ids [] (defn- fetch-all-sales-order-ids []
"Query Datomic for all sales-order external-ids (as entity IDs). "Query Datomic for all sales-order external-ids (as entity IDs).
Returns a vector of entitity ids." Returns a vector of entitity ids."
(->> (dc/q '[:find ?e (->> (dc/q '[:find ?e
:where [_ :sales-order/external-id ?_ext]] :where [?e :sales-order/external-id _]]
(dc/db conn)) (dc/db conn))
(map first) (map first)
vec)) vec))
@@ -25,14 +25,16 @@
(def ^:private sales-order-read (def ^:private sales-order-read
'[:sales-order/external-id '[:sales-order/external-id
:sales-order/date :sales-order/date
{:sales-order/client [:client/code]} {:sales-order/client [:client/code :client/name]}
:sales-order/location :sales-order/location
:sales-order/vendor {:sales-order/vendor [:vendor/name]}
:sales-order/total :sales-order/total
:sales-order/tax :sales-order/tax
:sales-order/tip :sales-order/tip
:sales-order/discount :sales-order/discount
:sales-order/service-charge :sales-order/service-charge
:sales-order/source
:sales-order/reference-link
{:sales-order/charges {:sales-order/charges
[:charge/external-id [:charge/external-id
:charge/type-name :charge/type-name
@@ -40,7 +42,7 @@
:charge/tax :charge/tax
:charge/tip :charge/tip
:charge/date :charge/date
:charge/processor {:charge/processor [:db/ident]}
:charge/returns :charge/returns
{:charge/client [:client/code]}]} {:charge/client [:client/code]}]}
{:sales-order/line-items {:sales-order/line-items
@@ -49,7 +51,7 @@
:order-line-item/total :order-line-item/total
:order-line-item/tax :order-line-item/tax
:order-line-item/discount :order-line-item/discount
{:order-line-item/unit-price {}} :order-line-item/unit-price
:order-line-item/quantity :order-line-item/quantity
:order-line-item/note]}]) :order-line-item/note]}])
@@ -61,69 +63,76 @@
sales-order-read sales-order-read
eids))) eids)))
(defn- flatten-order-to-pieces! [order flat] (defn- flatten-order-to-pieces! [order date-str flat]
"Flatten a pulled sales-order into :entity-type tagged maps. "Flatten a pulled sales-order into :entity-type tagged maps.
Appends to the existing flat vector, which is returned." Appends to the existing flat vector, which is returned."
(let [so-ext-id (:sales-order/external-id order) (let [so-ext-id (:sales-order/external-id order)
so-date (.toString (:sales-order/date order)) so-date date-str
client-code (get-in order [:sales-order/client :client/code])] client-code (get-in order [:sales-order/client :client/code])
;; sales-order row vendor-name (get-in order [:sales-order/vendor :vendor/name])
(swap! flat conj charges (:sales-order/charges order)
{:entity-type "sales-order" items (:sales-order/line-items order)
:external-id (str so-ext-id) payment-methods (->> charges (map :charge/type-name) distinct (str/join ","))
:client-code client-code processors (->> charges (map #(get-in % [:charge/processor :db/ident])) (remove nil?) distinct (map name) (str/join ","))
:location (:sales-order/location order) categories (->> items (map :order-line-item/category) (remove nil?) distinct (str/join ","))]
:vendor (:sales-order/vendor order) (vswap! flat conj
:total (:sales-order/total order) {:entity-type "sales-order"
:tax (:sales-order/tax order) :external-id (str so-ext-id)
:tip (:sales-order/tip order) :client-code client-code
:discount (:sales-order/discount order) :location (:sales-order/location order)
:service-charge (:sales-order/service-charge order) :vendor vendor-name
:date so-date}) :total (:sales-order/total order)
;; charges & line-items :tax (:sales-order/tax order)
:tip (:sales-order/tip order)
:discount (:sales-order/discount order)
:service-charge (:sales-order/service-charge order)
:date so-date
:source (:sales-order/source order)
:reference-link (:sales-order/reference-link order)
:payment-methods payment-methods
:processors processors
:categories categories})
(when-let [charges (:sales-order/charges order)] (when-let [charges (:sales-order/charges order)]
(doseq [chg charges] (doseq [chg charges]
(swap! flat conj (vswap! flat conj
{:entity-type "charge" {:entity-type "charge"
:external-id (str (get chg :charge/external-id)) :external-id (str (get chg :charge/external-id))
:type-name (get chg :charge/type-name) :type-name (get chg :charge/type-name)
:total (get chg :charge/total) :total (get chg :charge/total)
:tax (get chg :charge/tax) :tax (get chg :charge/tax)
:tip (get chg :charge/tip) :tip (get chg :charge/tip)
:date so-date :date so-date
:processor (get-in chg [:charge/processor :db/ident]) :processor (get-in chg [:charge/processor :db/ident])
:sales-order-external-id (str so-ext-id)}) :sales-order-external-id (str so-ext-id)})
;; charge returns → sales-refund rows
(when-let [returns (:charge/returns chg)] (when-let [returns (:charge/returns chg)]
(doseq [rt returns] (doseq [rt returns]
(swap! flat conj (vswap! flat conj
{:entity-type "sales-refund" {:entity-type "sales-refund"
:type-name (get rt :type-name) :type-name (get rt :type-name)
:total (get rt :total) :total (get rt :total)
:sales-order-external-id (str so-ext-id)}))))) :sales-order-external-id (str so-ext-id)})))))
;; line-items
(when-let [items (:sales-order/line-items order)] (when-let [items (:sales-order/line-items order)]
(doseq [li items] (doseq [li items]
(swap! flat conj (vswap! flat conj
{:entity-type "line-item" {:entity-type "line-item"
:item-name (get li :order-line-item/item-name) :item-name (get li :order-line-item/item-name)
:category (get li :order-line-item/category) :category (get li :order-line-item/category)
:total (get li :order-line-item/total) :total (get li :order-line-item/total)
:tax (get li :order-line-item/tax) :tax (get li :order-line-item/tax)
:discount (get li :order-line-item/discount) :discount (get li :order-line-item/discount)
:sales-order-external-id (str so-ext-id)}))))) :sales-order-external-id (str so-ext-id)})))))
(defn -fetch-order-ids-for-date (defn -fetch-order-ids-for-date
"Query Datomic for all sales-order eids on a given business date." "Query Datomic for all sales-order eids on a given business date."
[db date-str] [db date-str]
(let [day-ms (.toEpochSecond ^java.time.LocalDate (java.time.LocalDate/parse date-str)) (let [ld (java.time.LocalDate/parse date-str)
start (* day-ms 1000) start (-> ld (.atStartOfDay (java.time.ZoneId/of "America/Los_Angeles")) .toInstant java.util.Date/from)
end (+ start (* 86400000))] end (-> ld (.plusDays 1) (.atStartOfDay (java.time.ZoneId/of "America/Los_Angeles")) .toInstant java.util.Date/from)]
(->> (dc/q '[:find ?e (->> (dc/q '[:find ?e
:in $ ?start-ms ?end-ms :in $ ?start ?end
:where [_ :sales-order/date ?d] :where [?e :sales-order/date ?d]
[(>= ?d ?start-ms)] [(>= ?d ?start)]
[(<= ?d ?end-ms)]] [(< ?d ?end)]]
db start end) db start end)
(map first) (map first)
vec))) vec)))
@@ -137,9 +146,9 @@
(for [i (range 0 (inc days))] (for [i (range 0 (inc days))]
(.toString (.plusDays sd i))))) (.toString (.plusDays sd i)))))
(defn- write-day-by-day (defn write-day-by-day
([start-date end-date] ([start-date end-date]
(write-day-by-day start-date end-date nil)) (write-day-by-day start-date end-date {}))
([start-date end-date opts] ([start-date end-date opts]
(let [all-dates (set (or (opts :date-set) [])) (let [all-dates (set (or (opts :date-set) []))
date-range (if (empty? all-dates) date-range (if (empty? all-dates)
@@ -155,12 +164,12 @@
(let [orders (pull-sales-order-data batch) (let [orders (pull-sales-order-data batch)
flat (volatile! [])] flat (volatile! [])]
(doseq [o orders] (doseq [o orders]
(flatten-order-to-pieces! o flat)) (flatten-order-to-pieces! o day flat))
(doseq [r @flat] (doseq [r @flat]
(p/buffer! (:entity-type r) r))))) (p/buffer! (:entity-type r) r)))))
(doseq [etype ["sales-order" "charge" (doseq [etype ["sales-order" "charge"
"line-item" "sales-refund"]] "line-item" "sales-refund"]]
(p/flush-to-parquet! etype)) (p/flush-to-parquet! etype day))
(println "[migration]" day "complete")) (println "[migration]" day "complete"))
{:status :completed :total-days (count date-range)}))) {:status :completed :total-days (count date-range)})))
@@ -180,10 +189,11 @@
"Flush all entity-type buffers, tracking counts." "Flush all entity-type buffers, tracking counts."
(let [etypes ["sales-order" "charge" (let [etypes ["sales-order" "charge"
"line-item" "sales-refund"] "line-item" "sales-refund"]
today (.toString (java.time.LocalDate/now))
start (p/total-buf-count)] start (p/total-buf-count)]
(doseq [et etypes] (doseq [et etypes]
(try (try
(p/flush-to-parquet! et) (p/flush-to-parquet! et today)
(catch Exception e (catch Exception e
(println "[migration/flush]" et "error:" (.getMessage e))))) (println "[migration/flush]" et "error:" (.getMessage e)))))
{:records-flush (- (p/total-buf-count) start)})) {:records-flush (- (p/total-buf-count) start)}))
@@ -217,7 +227,7 @@
(doseq [o (pull-sales-order-data order-ids) (doseq [o (pull-sales-order-data order-ids)
:when (not (:sales-order/date o))] :when (not (:sales-order/date o))]
(let [flat (volatile! [])] (let [flat (volatile! [])]
(flatten-order-to-pieces! o flat) (flatten-order-to-pieces! o "unknown" flat)
(doseq [r @flat] (doseq [r @flat]
(p/buffer! "dead" r)))) (p/buffer! "dead" r))))
(write-day-by-day start-date end-date {:batch-size 100}) (write-day-by-day start-date end-date {:batch-size 100})

24
src/clj/auto_ap/ssr/pos/sales_orders.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.ssr.pos.sales-orders (ns auto-ap.ssr.pos.sales-orders
(:require (:require
[auto-ap.datomic [auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query
pull-many query2]] pull-many query2]]
[auto-ap.datomic.sales-orders :as d-sales] [auto-ap.datomic.sales-orders :as d-sales]
[auto-ap.query-params :as query-params :refer [wrap-copy-qp-pqp]] [auto-ap.query-params :as query-params :refer [wrap-copy-qp-pqp]]
@@ -17,7 +17,6 @@
[auto-ap.time :as atime] [auto-ap.time :as atime]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[clj-time.coerce :as c] [clj-time.coerce :as c]
[datomic.api :as dc]
[malli.core :as mc])) [malli.core :as mc]))
(def query-schema (mc/schema (def query-schema (mc/schema
@@ -172,11 +171,8 @@
charges)) charges))
(defn fetch-page [request] (defn fetch-page [request]
(let [db (dc/db conn) (let [{:keys [rows count]} (d-sales/fetch-page-ssr request)]
{ids-to-retrieve :ids matching-count :count} (fetch-ids db request)] [rows count]))
[(->> (hydrate-results ids-to-retrieve db request))
matching-count]))
(def grid-page (def grid-page
@@ -200,13 +196,13 @@
:title "Sales orders" :title "Sales orders"
:entity-name "Sales orders" :entity-name "Sales orders"
:route :pos-sales-table :route :pos-sales-table
:action-buttons (fn [request] :action-buttons (fn [request]
(let [{:keys [total tax]} (d-sales/summarize-orders (:ids (fetch-ids (dc/db conn) request)))] (let [{:keys [total tax]} (d-sales/summarize-page-ssr request)]
(when (and total tax) (when (and total tax)
[(com/pill {:color :primary} [(com/pill {:color :primary}
(format "Total $%.2f" total)) (format "Total $%.2f" total))
(com/pill {:color :secondary} (com/pill {:color :secondary}
(format "Tax $%.2f" tax))]))) (format "Tax $%.2f" tax))])))
:row-buttons (fn [_ e] :row-buttons (fn [_ e]
(when (:sales-order/reference-link e) (when (:sales-order/reference-link e)
[(com/a-icon-button {:href (:sales-order/reference-link e)} [(com/a-icon-button {:href (:sales-order/reference-link e)}

185
src/clj/auto_ap/storage/parquet.clj
View File

@@ -128,12 +128,12 @@
(->> @*buffers* (->> @*buffers*
vals (mapcat identity) count)) vals (mapcat identity) count))
(defn flush-to-parquet! [entity-type] (defn flush-to-parquet! [entity-type date-str]
"Flush buffered records for entity-type to parquet + S3." "Flush buffered records for entity-type to parquet + S3."
(let [records (get @*buffers* entity-type [])] (let [records (get @*buffers* entity-type [])]
(if (empty? records) (if (empty? records)
{:status :no-records} {:status :no-records}
(let [date-str (.toString (LocalDate/now)) (let [date-str (or date-str (.toString (LocalDate/now)))
jsonl-file (io/file "/tmp" jsonl-file (io/file "/tmp"
(str entity-type "-" date-str ".jsonl")) (str entity-type "-" date-str ".jsonl"))
parquet-file (io/file "/tmp" parquet-file (io/file "/tmp"
@@ -162,10 +162,11 @@
"Flush all entity types for today." "Flush all entity types for today."
(let [etypes ["sales-order" "charge" (let [etypes ["sales-order" "charge"
"line-item" "sales-refund"] "line-item" "sales-refund"]
today (.toString (LocalDate/now))
flushed (into #{} flushed (into #{}
(keep (fn [et] (keep (fn [et]
(let [{:keys [status]} (let [{:keys [status]}
(flush-to-parquet! et)] (flush-to-parquet! et today)]
(when (= status :ok) (when (= status :ok)
et)))) et))))
etypes)] etypes)]
@@ -190,11 +191,12 @@
{} {}
(into {} (into {}
(keep (fn [et] (keep (fn [et]
(let [f (io/file (let [f (io/file
(wal-dir) (wal-dir)
(str et ".jsonl"))] (str et ".jsonl"))]
[et (slurp f)]))) (when (.exists f)
etypes))] [et (slurp f)])))
etypes)))]
(swap! *buffers* merge loaded))) (swap! *buffers* merge loaded)))
(defn get-unflushed-count [] (defn get-unflushed-count []
@@ -218,66 +220,135 @@
(defn today [] (defn today []
(.toString (LocalDate/now))) (.toString (LocalDate/now)))
(defn- parquet-glob [entity-type start-date end-date]
"Build a glob pattern or explicit file list for the date range.
Uses glob patterns for ranges > 60 days; explicit list otherwise."
(let [days (-> (LocalDate/parse end-date)
(.toEpochDay)
(- (.toEpochDay (LocalDate/parse start-date)))
inc)]
(if (> days 60)
(let [prefix (format "s3://%s/sales-details/%s/" *bucket* entity-type)
sy (-> (LocalDate/parse start-date) .getYear)
ey (-> (LocalDate/parse end-date) .getYear)]
(if (= sy ey)
[(format "%s%d-*.parquet" prefix sy)]
(vec
(for [y (range sy (inc ey))]
(format "%s%d-*.parquet" prefix y)))))
(vec
(map (fn [d]
(format "'s3://%s/sales-details/%s/%s.parquet'"
*bucket* entity-type d))
(date-seq start-date end-date))))))
(defn parquet-query [entity-type start-date end-date] (defn parquet-query [entity-type start-date end-date]
"Build SQL to read all parquet files in date range. "Build SQL to read all parquet files in date range.
Returns map with :sql and :count-sql keys." Returns map with :sql and :count-sql keys."
(let [date-strs (date-seq start-date end-date) (let [globs (parquet-glob entity-type start-date end-date)
urls (vec use-glob? (some #(.endsWith ^String % "*.parquet") globs)
(map (fn [d] base (if use-glob?
(format "'s3://%s/sales-details/%s/%s.parquet'" (format "SELECT * FROM read_parquet(%s, union_by_name=true)"
*bucket* entity-type d)) (if (= (count globs) 1)
date-strs)) (format "'%s'" (first globs))
sql (str "SELECT * FROM read_parquet([" (format "[%s]"
(str/join ", " urls) (str/join ", " (map #(format "'%s'" %) globs)))))
"])")] (format "SELECT * FROM read_parquet([%s])"
(str/join ", " globs)))
add-date-filter (fn [sql]
(if (> (-> (LocalDate/parse end-date)
(.toEpochDay)
(- (.toEpochDay (LocalDate/parse start-date)))
inc)
60)
(format "%s WHERE date >= '%s' AND date <= '%s'"
sql start-date end-date)
sql))
sql (add-date-filter base)]
{:sql sql {:sql sql
:count-sql (format "SELECT COUNT(*) FROM (%s) t" sql)})) :count-sql (format "SELECT COUNT(*) FROM (%s) t" sql)}))
(defn- build-where-clause [opts field-pairs] (defn- like-clause [col v]
"Build SQL WHERE clause from opts map. (str "\"" col "\" LIKE '%" v "%'"))
fields-with-keys is vector of [:field-key :env-var-name]."
(let [clauses (keep (defn- build-sales-orders-where [opts]
(fn [[key env]] (let [eq-clauses (keep
(let [v (get opts key)] (fn [[key col]]
(when v (let [v (get opts key)]
(str env " = '" v "'")))) (when v
field-pairs)] (str "\"" col "\" = '" v "'"))))
(when (seq clauses) [[:client "client-code"]
(str " WHERE " (str/join " AND " clauses))))) [:vendor "vendor"]
[:location "location"]])
like-clauses (keep
(fn [[key col]]
(let [v (get opts key)]
(when v
(like-clause col v))))
[[:payment-method "payment-methods"]
[:processor "processors"]
[:category "categories"]])
range-clauses (keep
(fn [[key col op]]
(let [v (get opts key)]
(when v
(str "\"" col "\" " op " " v))))
[[:total-gte "total" ">="]
[:total-lte "total" "<="]])
all-clauses (concat eq-clauses like-clauses range-clauses)]
(when (seq all-clauses)
(str " WHERE " (str/join " AND " all-clauses)))))
(defn get-sales-orders (defn get-sales-orders
([start-date end-date] ([start-date end-date]
(get-sales-orders start-date end-date {})) (get-sales-orders start-date end-date {}))
([start-date end-date opts] ([start-date end-date opts]
(let [q (parquet-query "sales-order" (try
start-date end-date) (let [q (parquet-query "sales-order"
base-sql (:sql q) start-date end-date)
count-sql (:count-sql q) base-sql (:sql q)
sort (get opts :sort "date") count-sql (:count-sql q)
order (get opts :order "DESC") sort (get opts :sort "date")
limit (get opts :limit) order (get opts :order "DESC")
offset (get opts :offset) limit (get opts :limit)
where-str (build-where-clause offset (get opts :offset)
opts where-str (build-sales-orders-where opts)
[[:client "client-code"] full-sql (if where-str
[:vendor "vendor"] (str base-sql where-str)
[:location "location"]]) base-sql)
full-sql (if where-str result (cond-> full-sql
(str base-sql where-str) sort (str " ORDER BY " sort
base-sql) " " (name order))
result (cond-> full-sql limit (str " LIMIT " limit)
sort (str " ORDER BY " sort offset (str " OFFSET " offset))
" " (name order)) full-count (if where-str
limit (str " LIMIT " limit) (str count-sql where-str)
offset (str " OFFSET " offset)) count-sql)]
full-count (if where-str {:rows (query-rows result)
(str count-sql where-str) :count (or
count-sql)] (int
{:rows (query-rows result) (query-scalar
:count (or full-count)) 0)})
(int (catch Exception _
(query-scalar {:rows [] :count 0}))))
full-count)) 0)})))
(defn get-sales-orders-summary
([start-date end-date]
(get-sales-orders-summary start-date end-date {}))
([start-date end-date opts]
(try
(let [q (parquet-query "sales-order" start-date end-date)
base-sql (:sql q)
where-str (build-sales-orders-where opts)
full-sql (if where-str
(str base-sql where-str)
base-sql)
sum-sql (format "SELECT COALESCE(SUM(total), 0) as total, COALESCE(SUM(tax), 0) as tax FROM (%s) t" full-sql)
row (first (query-rows sum-sql))]
{:total (or (:total row) 0.0)
:tax (or (:tax row) 0.0)})
(catch Exception _
{:total 0.0 :tax 0.0}))))
(defn query-deduped [entity-type start-date end-date] (defn query-deduped [entity-type start-date end-date]
"Query records deduplicated by external-id (latest _seq_no wins)." "Query records deduplicated by external-id (latest _seq_no wins)."