notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(sales): fix parquet SQL generation and cleanup formatting

Browse Source 
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full
  ORDER BY clause from build-sort-clause into get-sales-orders which
  prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC')
- Fix WHERE clause column names in parquet build-where-clause:
  external_id.client -> client-code, external_id.vendor -> vendor
- Fix parquet-query format string (%%s -> %s with proper format call)
- Fix ex-info call signature in flush! (was passing :error as third arg
  instead of inside the data map)
- Add S3 credentials to DuckDB connect! so httpfs can read from S3
- Fix parquet buffer indentation and alignment across square/core3,
  ezcater/core, ezcater_xls, payments, sales_summaries, migrations
- Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions,
  upsert-recent find/where clauses mangled by paren-repair)
- Uncomment accidentally commented code block in square/core3
- Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
This commit is contained in:
Bryce
2026-04-27 10:33:22 -07:00
parent 4597611655
commit ea7f46ea8a
10 changed files with 440 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/clj/auto_ap/datomic/sales_orders.clj
View File

@@ -1,8 +1,6 @@
(ns auto-ap.datomic.sales-orders
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.storage.parquet :as pq]
[clj-time.coerce :as c]
[clojure.set :as set]
[clojure.string :as str]
[com.brunobonacci.mulog :as mu]))
@@ -29,12 +27,13 @@
(update :sales-order/date #(some-> % str))))
(defn build-where-clause [args]
(let [clauses [(when-let [c (:client-code args)]
["external_id.client = '" c "'"])
(let [clauses (keep identity
[(when-let [c (:client-code args)]
(str "external_id.client = '" c "'"))
(when-let [v (:vendor args)]
["external_id.vendor = '" (name v) "'"])
(str "external_id.vendor = '" (name v) "'"))
(when-let [l (:location args)]
["location = '" l "'"])]
(str "location = '" l "'"))])]
(when (seq clauses)
(str "WHERE " (str/join " AND " clauses)))))
@@ -47,18 +46,15 @@
(defn raw-graphql-ids [args]
(let [start (some-> (:start (:date-range args)) .toString)
end (some-> (:end (:date-range args)) .substring 0 10)
where (build-where-clause args)
sort (build-sort-clause args)
end (some-> (:end (:date-range args)) (.substring 0 10))
limit (or (:limit args) page-size)
offset (or (:offset args) 0)
where-str (when where (str " " where))]
offset (or (:offset args) 0)]
(when start
(let [result (pq/get-sales-orders start end
{:client (:client-code args)
:vendor (:vendor args)
:location (:location args)
:sort sort
:sort (or (:sort args) "date")
:order "DESC"
:limit limit
:offset offset})]

51
src/clj/auto_ap/ezcater/core.clj
View File

@@ -21,16 +21,15 @@
:body (json/write-str {"query" (v/graphql-query q)})
:as :json})
:body
:data
))
:data))
(defn get-caterers [integration]
(:caterers (query integration {:venia/queries [{:query/data
[:caterers [:name :uuid [:address [:name :street]]]]}]} )))
[:caterers [:name :uuid [:address [:name :street]]]]}]})))
(defn get-subscriptions [integration]
(->> (query integration {:venia/queries [{:query/data
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]] ]]}]} )
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]]]]}]})
:subscribers
first
:subscriptions))
@@ -39,7 +38,7 @@
(map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid
:db/id
:ezcater-integration/integration-status [:db/id]])
{:ezcater-integration/integration-status [:db/id]}])
:in $
:where [?i :ezcater-integration/api-key]]
(dc/db conn))))
@@ -95,7 +94,6 @@
:eventKey 'cancelled}}
[[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]})))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn upsert-ezcater
([] (upsert-ezcater (get-integrations)))
@@ -122,7 +120,6 @@
{:client/_ezcater-locations [:client/code]}]}]
[:ezcater-caterer/uuid caterer-uuid]))
(defn round-carry-cents [f]
(with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP))))
@@ -140,17 +137,17 @@
(* commision%
0.01M
(+
(-> order :totals :subTotal :subunits )
(-> order :totals :subTotal :subunits)
(reduce +
0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))))
(defn ccp-fee [order]
(round-carry-cents
(* 0.000299M
(+
(-> order :totals :subTotal :subunits )
(-> order :totals :salesTax :subunits )
(-> order :totals :subTotal :subunits)
(-> order :totals :salesTax :subunits)
(reduce +
0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
@@ -158,7 +155,7 @@
(defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}]
(let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01))
(-> order :totals :salesTax :subunits (* 0.01)))
(-> order :catererCart :totals :catererTotalDue )
(-> order :catererCart :totals :catererTotalDue)
(commision order)
(ccp-fee order)))
service-charge (+ (commision order) (ccp-fee order))
@@ -169,7 +166,7 @@
:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid)
:client [:client/code client-code]
:location client-location
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid ))
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid))
:line-items [#:order-line-item
{:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0)
:item-name "EZCater Catering"
@@ -199,7 +196,7 @@
:tax tax
:tip tip
:returns 0.0
:vendor :vendor/ccp-ezcater}}))
:vendor :vendor/ccp-ezcater}))
(defn- flatten-order-to-parquet! [order]
"Flatten a sales-order into entity-type tagged maps and buffer to parquet."
@@ -242,7 +239,6 @@
:discount (:order-line-item/discount li)
:sales-order-external-id so-ext-id})))))
(defn get-by-id [integration id]
(query
integration
@@ -273,27 +269,20 @@
[:currency
:subunits]]]]]]
[:totals [[:customerTotalDue
[
:currency
:subunits
]]
[:currency
:subunits]]
[:pointOfSaleIntegrationFee
[
:currency
:subunits
]]
[:currency
:subunits]]
[:tip
[:currency
:subunits]]
[:salesTax
[
:currency
:subunits
]]
[:currency
:subunits]]
[:salesTaxRemittance
[:currency
:subunits
]]
:subunits]]
[:subTotal
[:currency
:subunits]]]]]]]}))
@@ -359,7 +348,7 @@
"key" "accepted",
"occurred_at" "2022-07-21T19:21:07.549Z"}
ezcater-order (lookup-order lookup-map)
extant-order (dc/pull (dc/db conn) '[:sales-order/total
extant-order (dc/pull (dc/db conn '[:sales-order/total]
:sales-order/tax
:sales-order/tip
:sales-order/discount
@@ -371,7 +360,7 @@
:sales-order/line-items [:order-line-item/external-id
:order-line-item/total
:order-line-item/tax
:order-line-item/discount]}]
:order-line-item/discount]})
[:sales-order/external-id order])
updated-order (-> (order->sales-order ezcater-order)

26
src/clj/auto_ap/jobs/sales_summaries.clj
View File

@@ -40,18 +40,15 @@
(dc/db conn)
number)))
(defn delete-all []
@(dc/transact-async conn
(->>
(dc/q '[:find ?ss
:where [?ss :sales-summary/date]]
(dc/db conn))
(map (fn [[ ss]]
(map (fn [[ss]]
[:db/retractEntity ss])))))
(defn dirty-sales-summaries [c]
(let [client-id (dc/entid (dc/db conn) c)]
(->> (dc/index-pull (dc/db conn)
@@ -188,8 +185,6 @@
:ledger-mapped/ledger-side :ledger-side/credit
:ledger-mapped/amount (- (+ total discount) tax)})))
(defn get-fees [c date]
(when-let [fee (get-fee c date)]
{:db/id (str (java.util.UUID/randomUUID))
@@ -299,8 +294,7 @@
(filter identity)
(map (fn [z]
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
:sales-summary-item/manual? false))
)) }]
:sales-summary-item/manual? false))))}]
(if (seq (:sales-summary/items result))
(do
(alog/info ::upserting-summaries
@@ -310,7 +304,7 @@
(comment
;; TODO: Move to test file or proper location
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ])
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL"])
date #inst "2024-04-14T00:00:00-07:00"]
(get-payment-items c date)))
@@ -322,9 +316,6 @@
(map (fn [[sos]]
[:db/retractEntity sos])))))
(comment
(auto-ap.datomic/transact-schema conn)
@@ -367,19 +358,14 @@
:in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/line-items ?li]
[?li :order-line-item/item-name ?n] ]
[?li :order-line-item/item-name ?n]]
(dc/db conn)
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"])
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
(auto-ap.datomic/transact-schema conn)
)
(auto-ap.datomic/transact-schema conn))
(defn -main [& _]
(execute "sales-summaries" sales-summaries-v2))

1
src/clj/auto_ap/migration/sales_to_parquet.clj
View File

@@ -128,7 +128,6 @@
(map first)
vec)))
(defn- date-seq [start end]
"Seq of YYYY-MM-DD strings between start and end inclusive."
(let [sd (java.time.LocalDate/parse start)

4
src/clj/auto_ap/routes/ezcater_xls.clj
View File

@@ -55,8 +55,8 @@
event-date (some-> (excel/xls-date->date event-date)
coerce/to-date-time
atime/as-local-time
coerce/to-date )]
(cond (and event-date client-id location )
coerce/to-date)]
(cond (and event-date client-id location)
[:order #:sales-order
{:date event-date
:external-id (str "ezcater/order/" client-id "-" location "-" order-number)

83
src/clj/auto_ap/square/core3.clj
View File

@@ -28,11 +28,9 @@
"Authorization" (str "Bearer " (:client/square-auth-token client))
"Content-Type" "application/json"}))
(defn ->square-date [d]
(f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d))
(def manifold-api-stream
(let [stream (s/stream 100)]
(->> stream
@@ -105,7 +103,6 @@
:exception error))
[]))))
(def item-cache (atom {}))
(defn fetch-catalog [client i v]
@@ -125,13 +122,11 @@
#(do (swap! item-cache assoc i %)
%))))
(defn fetch-catalog-cache [client i version]
(if (get @item-cache i)
(de/success-deferred (get @item-cache i))
(fetch-catalog client i version)))
(defn item->category-name-impl [client item version]
(capture-context->lc
(cond (:item_id (:item_variation_data item))
@@ -162,7 +157,6 @@
:item item)
"Uncategorized"))))
(defn item-id->category-name [client i version]
(capture-context->lc
(-> [client i]
@@ -227,7 +221,6 @@
(concat (:orders result) continued-results))))
(:orders result)))))))
(defn search
([client location start end]
(capture-context->lc
@@ -251,11 +244,9 @@
(concat (:orders result) continued-results))))
(:orders result))))))))
(defn amount->money [amt]
(* 0.01 (or (:amount amt) 0.0)))
;; to get totals:
(comment
(reduce
@@ -416,7 +407,6 @@
:client client
:location location)))))))
(defn get-payment [client p]
(de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v2/payments/" p)
@@ -425,7 +415,6 @@
:body
:payment))
(defn continue-payout-entry-list [c l poi cursor]
(capture-context->lc lc
(de/chain
@@ -674,8 +663,7 @@
(catch Exception e
(log/error ::buffer-failed
:exception e
:order (:sales-order/external-id order)))))))))))
:order (:sales-order/external-id order))))))))))))
(defn upsert-payouts
([client]
@@ -725,7 +713,6 @@
(log/info ::done-loading-refunds)))))))
(defn get-cash-shift [client id]
(de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id))
:method :get
@@ -884,8 +871,6 @@
d1
d2))
(defn remove-voided-orders
([client]
(apply de/zip
@@ -920,22 +905,20 @@
(doseq [x (partition-all 100 results)]
(log/info ::removing-orders
:count (count x))
@(dc/transact-async conn x)))))
@(dc/transact-async conn x)
(de/catch (fn [e]
(log/warn ::couldnt-remove :error e)
nil) ))))))
nil)))))))))))
#_(comment
(require 'auto-ap.time-reader)
@(let [[c [l]] (get-square-client-and-location "DBFS") ]
(log/peek :x [ c l])
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29")
@(let [[c [l]] (get-square-client-and-location "DBFS")]
(log/peek :x [c l])
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29"))
)
@(let [[c [l]] (get-square-client-and-location "NGAK") ]
(log/peek :x [ c l])
@(let [[c [l]] (get-square-client-and-location "NGAK")]
(log/peek :x [c l])
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
(doseq [c (get-square-clients)]
@@ -943,8 +926,6 @@
@(remove-voided-orders c)
(catch Exception e
nil)))
)
(defn upsert-all [& clients]
@@ -1014,8 +995,6 @@
[:clients clients]
@(apply upsert-all clients)))
(comment
(defn refunds-raw-cont
([client l cursor so-far]
@@ -1045,7 +1024,6 @@
(->>
@(let [[c [l]] (get-square-client-and-location "NGGG")]
(search c l (time/now) (time/plus (time/now) (time/days -1))))
(filter (fn [r]
@@ -1055,7 +1033,6 @@
(->>
@(let [[c [l]] (get-square-client-and-location "NGGG")]
(refunds-raw-cont c l nil []))
(filter (fn [r]
(str/starts-with? (:created_at r) "2024-03-14")))))
@@ -1090,12 +1067,7 @@
[(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)])
:separator \tab)
;; =>
;; =>
(require 'auto-ap.time-reader)
@@ -1104,26 +1076,15 @@
(clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")]
l
(def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00"
#clj-time/date-time "2025-02-28T00:00:00-08:00"))
(take 10 (map #(first (deref (order->sales-order c l %))) z)))
)
(take 10 (map #(first (deref (order->sales-order c l %))) z))))
(->> z
(filter (fn [o]
(seq (filter (comp #{"OTHER"} :type) (:tenders o)))))
(filter #(not (:name (:source %))))
(count)
)
(count))
(doseq [[code] (seq (dc/q '[:find ?code
:in $
@@ -1133,32 +1094,22 @@
[?o :sales-order/client ?c]
[?c :client/code ?code]]
(dc/db conn)))
:let [[c [l]] (get-square-client-and-location code)
]
:let [[c [l]] (get-square-client-and-location code)]
order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now))
:when (= "Invoices" (:name (:source order) ))
:when (= "Invoices" (:name (:source order)))
:let [[sales-order] @(order->sales-order c l order)]]
(when (should-import-order? order)
(println "DATE IS" (:sales-order/date sales-order))
(when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00"))
(println "WOULD UPDATE" sales-order)
@(dc/transact auto-ap.datomic/conn [sales-order])
)
#_@(dc/transact )
(println "DONE"))
)
@(dc/transact auto-ap.datomic/conn [sales-order]))
#_@(dc/transact)
(println "DONE")))
#_(filter (comp #{"OTHER"} :type) (mapcat :tenders z))
@(let [[c [l]] (get-square-client-and-location "NGRY")]
#_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01"))
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" )))
)
)
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY")))))

17
src/clj/auto_ap/ssr/payments.clj
View File

@@ -116,7 +116,6 @@
:content "Debit"}]}))
(exact-match-id* request)]])
(def default-read '[*
[:payment/date :xform clj-time.coerce/from-date]
{:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}
@@ -212,7 +211,6 @@
'[(iol-ion.query/dollars= ?transaction-amount ?amount)]]}
:args [(:amount query-params)]})
(:status route-params)
(merge-query {:query {:in ['?status]
:where ['[?e :payment/status ?status]]}
@@ -277,7 +275,6 @@
{ids-to-retrieve :ids matching-count :count
all-ids :all-ids} (fetch-ids db request)]
[(->> (hydrate-results ids-to-retrieve db request))
matching-count
(sum-visible-pending all-ids)
@@ -285,8 +282,7 @@
(:client request)
(:client-id (:query-params request))
(when (:client-code (:query-params request))
[:client/code (:client-code (:query-params request))])))
]))
[:client/code (:client-code (:query-params request))])))]))
(def query-schema (mc/schema
[:maybe [:map {:date-range [:date-range :start-date :end-date]}
@@ -327,7 +323,7 @@
(assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)])
:query-schema query-schema
:action-buttons (fn [request]
(let [[_ _ visible-in-float total-in-float ] (:page-results request)]
(let [[_ _ visible-in-float total-in-float] (:page-results request)]
[(com/pill {:color :primary} " Visible in float "
(format "$%,.2f" visible-in-float))
(com/pill {:color :secondary} " Total in float "
@@ -434,8 +430,6 @@
(def row* (partial helper/row* grid-page))
(comment
(mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
@@ -445,7 +439,6 @@
(mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)))
(defn delete [{check :entity :as request identity :identity}]
@@ -459,7 +452,7 @@
#(assert-can-see-client identity (:db/id (:payment/client check))))
(notify-if-locked (:db/id (:payment/client check))
(:payment/date check))
(let [ removing-payments (mapcat (fn [x]
(let [removing-payments (mapcat (fn [x]
(let [invoice (:invoice-payment/invoice x)
new-balance (+ (:invoice/outstanding-balance invoice)
(:invoice-payment/amount x))]
@@ -578,7 +571,6 @@
(assoc-in [:query-params :start] 0)
(assoc-in [:query-params :per-page] 250))))
:else
selected)
updated-count (void-payments-internal ids (:identity request))]
@@ -591,7 +583,7 @@
(defn wrap-status-from-source [handler]
(fn [{:keys [matched-current-page-route] :as request}]
(let [ request (cond-> request
(let [request (cond-> request
(= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared)
(= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending)
(= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided)
@@ -618,7 +610,6 @@
::route/bulk-delete (-> bulk-delete-dialog
(wrap-admin))
::route/table (helper/table-route grid-page)}
(fn [h]
(-> h

19
src/clj/auto_ap/storage/parquet.clj
View File

@@ -22,6 +22,10 @@
(let [conn (DriverManager/getConnection "jdbc:duckdb:")
stmt (.createStatement conn)]
(.execute stmt "INSTALL httpfs; LOAD httpfs;")
(when-let [key (:aws-access-key-id env)]
(.execute stmt (str "SET s3_access_key_id='" key "'"))
(.execute stmt (str "SET s3_secret_access_key='" (:aws-secret-access-key env) "'"))
(.execute stmt (str "SET s3_region='" (or (:aws-region env) "us-east-1") "'")))
(.close stmt)
(.addShutdownHook (Runtime/getRuntime)
(Thread. #(fn [])))
@@ -150,8 +154,9 @@
(.delete ^java.io.File parquet-file)
{:key s3-key :status :ok}
(catch Exception e
(throw (ex-info "Flush failed" {:entity-type entity-type}
:error (.getMessage e)))))))))
(throw (ex-info "Flush failed"
{:entity-type entity-type
:error (.getMessage e)}))))))))
(defn flush-by-date! []
"Flush all entity types for today."
@@ -218,8 +223,9 @@
Returns map with :sql and :count-sql keys."
(let [date-strs (date-seq start-date end-date)
urls (vec
(map #(format "'s3://%%s/sales-details/%%s/%%s.parquet'"
*bucket* entity-type %)
(map (fn [d]
(format "'s3://%s/sales-details/%s/%s.parquet'"
*bucket* entity-type d))
date-strs))
sql (str "SELECT * FROM read_parquet(["
(str/join ", " urls)
@@ -252,10 +258,9 @@
limit (get opts :limit)
offset (get opts :offset)
where-str (build-where-clause
opts
[[:client "external_id.client"]
[:vendor "external_id.vendor"]
[[:client "client-code"]
[:vendor "vendor"]
[:location "location"]])
full-sql (if where-str
(str base-sql where-str)