Bryce
ea7f46ea8a
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full ORDER BY clause from build-sort-clause into get-sales-orders which prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC') - Fix WHERE clause column names in parquet build-where-clause: external_id.client -> client-code, external_id.vendor -> vendor - Fix parquet-query format string (%%s -> %s with proper format call) - Fix ex-info call signature in flush! (was passing :error as third arg instead of inside the data map) - Add S3 credentials to DuckDB connect! so httpfs can read from S3 - Fix parquet buffer indentation and alignment across square/core3, ezcater/core, ezcater_xls, payments, sales_summaries, migrations - Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions, upsert-recent find/where clauses mangled by paren-repair) - Uncomment accidentally commented code block in square/core3 - Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
185 lines
6.9 KiB
Clojure
185 lines
6.9 KiB
Clojure
(ns auto-ap.storage.sales-summaries
|
|
"Aggregation functions querying Parquet files on S3 via DuckDB.
|
|
Entity types: sales-order | charge | line-item | sales-refund
|
|
S3 pattern: s3://<bucket>/sales-details/<entity-type>/<YYYY-MM-DD>.parquet"
|
|
(:require [auto-ap.storage.parquet :as p]
|
|
[clojure.string :as str]))
|
|
|
|
(defn- dq [name]
|
|
(str "\"" name "\""))
|
|
|
|
(defn- sum-dbl [val]
|
|
(try
|
|
(if val (double val) 0.0)
|
|
(catch Exception _e
|
|
0.0)))
|
|
|
|
(defn- pq-files [entity-type start-date end-date]
|
|
"Vector of S3 parquet file paths for date range."
|
|
(let [dates (p/date-seq start-date end-date)]
|
|
(vec
|
|
(map #(str "'s3://" p/*bucket*
|
|
"/sales-details/" entity-type "/"
|
|
% ".parquet") dates))))
|
|
|
|
(defn sum-payments-by-type [client-id start-date end-date]
|
|
"Return {processor-key -> {type-name-string -> total-double}}."
|
|
(let [files (pq-files "charge" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT "
|
|
(dq "processor")
|
|
" AS proc, "
|
|
(dq "type-name")
|
|
" AS type_name, "
|
|
"SUM("
|
|
(dq "total")
|
|
")::DOUBLE AS total_amount "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "' "
|
|
"GROUP BY "
|
|
(dq "processor") ", "
|
|
(dq "type-name"))]
|
|
(let [rows (p/query-rows sql)]
|
|
(reduce (fn [acc row]
|
|
(let [proc (:proc row)
|
|
tname (str/trim (name (:type_name row)))
|
|
total (sum-dbl (:total_amount row))]
|
|
(update acc proc
|
|
(fn [inner]
|
|
(let [b (or inner {})]
|
|
(assoc b
|
|
tname
|
|
(+ (get b tname 0.0) total)))))))
|
|
{}
|
|
rows)))
|
|
(catch Exception e
|
|
(println "[sales-summaries]" (.getMessage e))
|
|
{}))))
|
|
|
|
(defn sum-discounts [client-id start-date end-date]
|
|
(let [files (pq-files "sales-order" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT SUM("
|
|
(dq "discount")
|
|
")::DOUBLE AS discount_total "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "'")]
|
|
(or (some-> (first (p/query-rows sql)) :discount_total sum-dbl) 0.0))
|
|
(catch Exception e
|
|
(println "[sales-summaries/discounts]" (.getMessage e))
|
|
0.0))))
|
|
|
|
(defn sum-refunds-by-type [client-id start-date end-date]
|
|
(let [files (pq-files "sales-refund" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT "
|
|
(dq "type-name")
|
|
" AS type_name, "
|
|
"SUM("
|
|
(dq "total")
|
|
")::DOUBLE AS total_amount "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "sales-order-external-id")
|
|
" IN (SELECT "
|
|
(dq "external-id")
|
|
" FROM read_parquet(["
|
|
(str/join ", " (pq-files "sales-order" start-date end-date))
|
|
"]) WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "') "
|
|
"GROUP BY " (dq "type-name"))]
|
|
(let [rows (p/query-rows sql)]
|
|
(reduce (fn [acc row]
|
|
(let [tname (str/trim (name (:type_name row)))
|
|
total (sum-dbl (:total_amount row))]
|
|
(assoc acc tname (+ (get acc tname 0.0) total))))
|
|
{}
|
|
rows)))
|
|
(catch Exception e
|
|
(println "[sales-summaries/refunds]" (.getMessage e))
|
|
{}))))
|
|
|
|
(defn sum-taxes [client-id start-date end-date]
|
|
(let [files (pq-files "sales-order" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT SUM("
|
|
(dq "tax")
|
|
")::DOUBLE AS tax_total "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "'")]
|
|
(or (some-> (first (p/query-rows sql)) :tax_total sum-dbl) 0.0))
|
|
(catch Exception e
|
|
(println "[sales-summaries/tax]" (.getMessage e))
|
|
0.0))))
|
|
|
|
(defn sum-tips [client-id start-date end-date]
|
|
(let [files (pq-files "sales-order" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT SUM("
|
|
(dq "tip")
|
|
")::DOUBLE AS tip_total "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "'")]
|
|
(or (some-> (first (p/query-rows sql)) :tip_total sum-dbl) 0.0))
|
|
(catch Exception e
|
|
(println "[sales-summaries/tip]" (.getMessage e))
|
|
0.0))))
|
|
|
|
(defn sum-sales-by-category [client-id start-date end-date]
|
|
(let [files (pq-files "line-item" start-date end-date)]
|
|
(try
|
|
(let [sql (str "SELECT "
|
|
(dq "category")
|
|
" AS category, "
|
|
"SUM("
|
|
(dq "total")
|
|
")::DOUBLE AS total_amount, "
|
|
"SUM("
|
|
(dq "tax")
|
|
")::DOUBLE AS tax_amount, "
|
|
"SUM("
|
|
(dq "discount")
|
|
")::DOUBLE AS discount_amount "
|
|
"FROM read_parquet(["
|
|
(str/join ", " files)
|
|
"]) "
|
|
"WHERE "
|
|
(dq "sales-order-external-id")
|
|
" IN (SELECT "
|
|
(dq "external-id")
|
|
" FROM read_parquet(["
|
|
(str/join ", " (pq-files "sales-order" start-date end-date))
|
|
"]) WHERE "
|
|
(dq "client-code")
|
|
" = '" client-id "') "
|
|
"GROUP BY " (dq "category"))]
|
|
(let [rows (p/query-rows sql)]
|
|
(mapv (fn [row]
|
|
{:category (or (:category row) "Unknown")
|
|
:total (sum-dbl (:total_amount row))
|
|
:tax (sum-dbl (:tax_amount row))
|
|
:discount (sum-dbl (:discount_amount row))})
|
|
rows)))
|
|
(catch Exception e
|
|
(println "[sales-summaries/sales]" (.getMessage e))
|
|
[]))))
|