notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(sales): fix parquet SQL generation and cleanup formatting

Browse Source 
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full
  ORDER BY clause from build-sort-clause into get-sales-orders which
  prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC')
- Fix WHERE clause column names in parquet build-where-clause:
  external_id.client -> client-code, external_id.vendor -> vendor
- Fix parquet-query format string (%%s -> %s with proper format call)
- Fix ex-info call signature in flush! (was passing :error as third arg
  instead of inside the data map)
- Add S3 credentials to DuckDB connect! so httpfs can read from S3
- Fix parquet buffer indentation and alignment across square/core3,
  ezcater/core, ezcater_xls, payments, sales_summaries, migrations
- Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions,
  upsert-recent find/where clauses mangled by paren-repair)
- Uncomment accidentally commented code block in square/core3
- Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
This commit is contained in:
Bryce
2026-04-27 10:33:22 -07:00
parent 4597611655
commit ea7f46ea8a
10 changed files with 440 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/clj/auto_ap/datomic/sales_orders.clj
View File

@@ -1,8 +1,6 @@
(ns auto-ap.datomic.sales-orders (ns auto-ap.datomic.sales-orders
(:require (:require
[auto-ap.datomic :refer [conn]]
[auto-ap.storage.parquet :as pq] [auto-ap.storage.parquet :as pq]
[clj-time.coerce :as c]
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[com.brunobonacci.mulog :as mu])) [com.brunobonacci.mulog :as mu]))
@@ -29,12 +27,13 @@
(update :sales-order/date #(some-> % str)))) (update :sales-order/date #(some-> % str))))
(defn build-where-clause [args] (defn build-where-clause [args]
(let [clauses [(when-let [c (:client-code args)] (let [clauses (keep identity
["external_id.client = '" c "'"]) [(when-let [c (:client-code args)]
(str "external_id.client = '" c "'"))
(when-let [v (:vendor args)] (when-let [v (:vendor args)]
["external_id.vendor = '" (name v) "'"]) (str "external_id.vendor = '" (name v) "'"))
(when-let [l (:location args)] (when-let [l (:location args)]
["location = '" l "'"])] (str "location = '" l "'"))])]
(when (seq clauses) (when (seq clauses)
(str "WHERE " (str/join " AND " clauses))))) (str "WHERE " (str/join " AND " clauses)))))
@@ -47,18 +46,15 @@
(defn raw-graphql-ids [args] (defn raw-graphql-ids [args]
(let [start (some-> (:start (:date-range args)) .toString) (let [start (some-> (:start (:date-range args)) .toString)
end (some-> (:end (:date-range args)) .substring 0 10) end (some-> (:end (:date-range args)) (.substring 0 10))
where (build-where-clause args)
sort (build-sort-clause args)
limit (or (:limit args) page-size) limit (or (:limit args) page-size)
offset (or (:offset args) 0) offset (or (:offset args) 0)]
where-str (when where (str " " where))]
(when start (when start
(let [result (pq/get-sales-orders start end (let [result (pq/get-sales-orders start end
{:client (:client-code args) {:client (:client-code args)
:vendor (:vendor args) :vendor (:vendor args)
:location (:location args) :location (:location args)
:sort sort :sort (or (:sort args) "date")
:order "DESC" :order "DESC"
:limit limit :limit limit
:offset offset})] :offset offset})]

51
src/clj/auto_ap/ezcater/core.clj
View File

@@ -21,16 +21,15 @@
:body (json/write-str {"query" (v/graphql-query q)}) :body (json/write-str {"query" (v/graphql-query q)})
:as :json}) :as :json})
:body :body
:data :data))
))
(defn get-caterers [integration] (defn get-caterers [integration]
(:caterers (query integration {:venia/queries [{:query/data (:caterers (query integration {:venia/queries [{:query/data
[:caterers [:name :uuid [:address [:name :street]]]]}]} ))) [:caterers [:name :uuid [:address [:name :street]]]]}]})))
(defn get-subscriptions [integration] (defn get-subscriptions [integration]
(->> (query integration {:venia/queries [{:query/data (->> (query integration {:venia/queries [{:query/data
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]] ]]}]} ) [:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]]]]}]})
:subscribers :subscribers
first first
:subscriptions)) :subscriptions))
@@ -39,7 +38,7 @@
(map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key (map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid :ezcater-integration/subscriber-uuid
:db/id :db/id
:ezcater-integration/integration-status [:db/id]]) {:ezcater-integration/integration-status [:db/id]}])
:in $ :in $
:where [?i :ezcater-integration/api-key]] :where [?i :ezcater-integration/api-key]]
(dc/db conn)))) (dc/db conn))))
@@ -95,7 +94,6 @@
:eventKey 'cancelled}} :eventKey 'cancelled}}
[[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]}))))) [[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]})))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn upsert-ezcater (defn upsert-ezcater
([] (upsert-ezcater (get-integrations))) ([] (upsert-ezcater (get-integrations)))
@@ -122,7 +120,6 @@
{:client/_ezcater-locations [:client/code]}]}] {:client/_ezcater-locations [:client/code]}]}]
[:ezcater-caterer/uuid caterer-uuid])) [:ezcater-caterer/uuid caterer-uuid]))
(defn round-carry-cents [f] (defn round-carry-cents [f]
(with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP)))) (with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP))))
@@ -140,17 +137,17 @@
(* commision% (* commision%
0.01M 0.01M
(+ (+
(-> order :totals :subTotal :subunits ) (-> order :totals :subTotal :subunits)
(reduce + (reduce +
0 0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))))
(defn ccp-fee [order] (defn ccp-fee [order]
(round-carry-cents (round-carry-cents
(* 0.000299M (* 0.000299M
(+ (+
(-> order :totals :subTotal :subunits ) (-> order :totals :subTotal :subunits)
(-> order :totals :salesTax :subunits ) (-> order :totals :salesTax :subunits)
(reduce + (reduce +
0 0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
@@ -158,7 +155,7 @@
(defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}] (defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}]
(let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01)) (let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01))
(-> order :totals :salesTax :subunits (* 0.01))) (-> order :totals :salesTax :subunits (* 0.01)))
(-> order :catererCart :totals :catererTotalDue ) (-> order :catererCart :totals :catererTotalDue)
(commision order) (commision order)
(ccp-fee order))) (ccp-fee order)))
service-charge (+ (commision order) (ccp-fee order)) service-charge (+ (commision order) (ccp-fee order))
@@ -169,7 +166,7 @@
:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid) :external-id (str "ezcater/order/" client-code "-" client-location "-" uuid)
:client [:client/code client-code] :client [:client/code client-code]
:location client-location :location client-location
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid )) :reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid))
:line-items [#:order-line-item :line-items [#:order-line-item
{:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0) {:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0)
:item-name "EZCater Catering" :item-name "EZCater Catering"
@@ -199,7 +196,7 @@
:tax tax :tax tax
:tip tip :tip tip
:returns 0.0 :returns 0.0
:vendor :vendor/ccp-ezcater}})) :vendor :vendor/ccp-ezcater}))
(defn- flatten-order-to-parquet! [order] (defn- flatten-order-to-parquet! [order]
"Flatten a sales-order into entity-type tagged maps and buffer to parquet." "Flatten a sales-order into entity-type tagged maps and buffer to parquet."
@@ -242,7 +239,6 @@
:discount (:order-line-item/discount li) :discount (:order-line-item/discount li)
:sales-order-external-id so-ext-id}))))) :sales-order-external-id so-ext-id})))))
(defn get-by-id [integration id] (defn get-by-id [integration id]
(query (query
integration integration
@@ -273,27 +269,20 @@
[:currency [:currency
:subunits]]]]]] :subunits]]]]]]
[:totals [[:customerTotalDue [:totals [[:customerTotalDue
[ [:currency
:currency :subunits]]
:subunits
]]
[:pointOfSaleIntegrationFee [:pointOfSaleIntegrationFee
[ [:currency
:currency :subunits]]
:subunits
]]
[:tip [:tip
[:currency [:currency
:subunits]] :subunits]]
[:salesTax [:salesTax
[ [:currency
:currency :subunits]]
:subunits
]]
[:salesTaxRemittance [:salesTaxRemittance
[:currency [:currency
:subunits :subunits]]
]]
[:subTotal [:subTotal
[:currency [:currency
:subunits]]]]]]]})) :subunits]]]]]]]}))
@@ -359,7 +348,7 @@
"key" "accepted", "key" "accepted",
"occurred_at" "2022-07-21T19:21:07.549Z"} "occurred_at" "2022-07-21T19:21:07.549Z"}
ezcater-order (lookup-order lookup-map) ezcater-order (lookup-order lookup-map)
extant-order (dc/pull (dc/db conn) '[:sales-order/total extant-order (dc/pull (dc/db conn '[:sales-order/total]
:sales-order/tax :sales-order/tax
:sales-order/tip :sales-order/tip
:sales-order/discount :sales-order/discount
@@ -371,7 +360,7 @@
:sales-order/line-items [:order-line-item/external-id :sales-order/line-items [:order-line-item/external-id
:order-line-item/total :order-line-item/total
:order-line-item/tax :order-line-item/tax
:order-line-item/discount]}] :order-line-item/discount]})
[:sales-order/external-id order]) [:sales-order/external-id order])
updated-order (-> (order->sales-order ezcater-order) updated-order (-> (order->sales-order ezcater-order)

26
src/clj/auto_ap/jobs/sales_summaries.clj
View File

@@ -40,18 +40,15 @@
(dc/db conn) (dc/db conn)
number))) number)))
(defn delete-all [] (defn delete-all []
@(dc/transact-async conn @(dc/transact-async conn
(->> (->>
(dc/q '[:find ?ss (dc/q '[:find ?ss
:where [?ss :sales-summary/date]] :where [?ss :sales-summary/date]]
(dc/db conn)) (dc/db conn))
(map (fn [[ ss]] (map (fn [[ss]]
[:db/retractEntity ss]))))) [:db/retractEntity ss])))))
(defn dirty-sales-summaries [c] (defn dirty-sales-summaries [c]
(let [client-id (dc/entid (dc/db conn) c)] (let [client-id (dc/entid (dc/db conn) c)]
(->> (dc/index-pull (dc/db conn) (->> (dc/index-pull (dc/db conn)
@@ -188,8 +185,6 @@
:ledger-mapped/ledger-side :ledger-side/credit :ledger-mapped/ledger-side :ledger-side/credit
:ledger-mapped/amount (- (+ total discount) tax)}))) :ledger-mapped/amount (- (+ total discount) tax)})))
(defn get-fees [c date] (defn get-fees [c date]
(when-let [fee (get-fee c date)] (when-let [fee (get-fee c date)]
{:db/id (str (java.util.UUID/randomUUID)) {:db/id (str (java.util.UUID/randomUUID))
@@ -299,8 +294,7 @@
(filter identity) (filter identity)
(map (fn [z] (map (fn [z]
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account) (assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
:sales-summary-item/manual? false)) :sales-summary-item/manual? false))))}]
)) }]
(if (seq (:sales-summary/items result)) (if (seq (:sales-summary/items result))
(do (do
(alog/info ::upserting-summaries (alog/info ::upserting-summaries
@@ -310,7 +304,7 @@
(comment (comment
;; TODO: Move to test file or proper location ;; TODO: Move to test file or proper location
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ]) (let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL"])
date #inst "2024-04-14T00:00:00-07:00"] date #inst "2024-04-14T00:00:00-07:00"]
(get-payment-items c date))) (get-payment-items c date)))
@@ -322,9 +316,6 @@
(map (fn [[sos]] (map (fn [[sos]]
[:db/retractEntity sos]))))) [:db/retractEntity sos])))))
(comment (comment
(auto-ap.datomic/transact-schema conn) (auto-ap.datomic/transact-schema conn)
@@ -367,19 +358,14 @@
:in $ [?clients ?start-date ?end-date] :in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]] :where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/line-items ?li] [?e :sales-order/line-items ?li]
[?li :order-line-item/item-name ?n] ] [?li :order-line-item/item-name ?n]]
(dc/db conn) (dc/db conn)
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"]) [[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"])
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy} @(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}]) {:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
(auto-ap.datomic/transact-schema conn) (auto-ap.datomic/transact-schema conn))
)
(defn -main [& _] (defn -main [& _]
(execute "sales-summaries" sales-summaries-v2)) (execute "sales-summaries" sales-summaries-v2))

1
src/clj/auto_ap/migration/sales_to_parquet.clj
View File

@@ -128,7 +128,6 @@
(map first) (map first)
vec))) vec)))
(defn- date-seq [start end] (defn- date-seq [start end]
"Seq of YYYY-MM-DD strings between start and end inclusive." "Seq of YYYY-MM-DD strings between start and end inclusive."
(let [sd (java.time.LocalDate/parse start) (let [sd (java.time.LocalDate/parse start)

4
src/clj/auto_ap/routes/ezcater_xls.clj
View File

@@ -55,8 +55,8 @@
event-date (some-> (excel/xls-date->date event-date) event-date (some-> (excel/xls-date->date event-date)
coerce/to-date-time coerce/to-date-time
atime/as-local-time atime/as-local-time
coerce/to-date )] coerce/to-date)]
(cond (and event-date client-id location ) (cond (and event-date client-id location)
[:order #:sales-order [:order #:sales-order
{:date event-date {:date event-date
:external-id (str "ezcater/order/" client-id "-" location "-" order-number) :external-id (str "ezcater/order/" client-id "-" location "-" order-number)

83
src/clj/auto_ap/square/core3.clj
View File

@@ -28,11 +28,9 @@
"Authorization" (str "Bearer " (:client/square-auth-token client)) "Authorization" (str "Bearer " (:client/square-auth-token client))
"Content-Type" "application/json"})) "Content-Type" "application/json"}))
(defn ->square-date [d] (defn ->square-date [d]
(f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d)) (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d))
(def manifold-api-stream (def manifold-api-stream
(let [stream (s/stream 100)] (let [stream (s/stream 100)]
(->> stream (->> stream
@@ -105,7 +103,6 @@
:exception error)) :exception error))
[])))) []))))
(def item-cache (atom {})) (def item-cache (atom {}))
(defn fetch-catalog [client i v] (defn fetch-catalog [client i v]
@@ -125,13 +122,11 @@
#(do (swap! item-cache assoc i %) #(do (swap! item-cache assoc i %)
%)))) %))))
(defn fetch-catalog-cache [client i version] (defn fetch-catalog-cache [client i version]
(if (get @item-cache i) (if (get @item-cache i)
(de/success-deferred (get @item-cache i)) (de/success-deferred (get @item-cache i))
(fetch-catalog client i version))) (fetch-catalog client i version)))
(defn item->category-name-impl [client item version] (defn item->category-name-impl [client item version]
(capture-context->lc (capture-context->lc
(cond (:item_id (:item_variation_data item)) (cond (:item_id (:item_variation_data item))
@@ -162,7 +157,6 @@
:item item) :item item)
"Uncategorized")))) "Uncategorized"))))
(defn item-id->category-name [client i version] (defn item-id->category-name [client i version]
(capture-context->lc (capture-context->lc
(-> [client i] (-> [client i]
@@ -227,7 +221,6 @@
(concat (:orders result) continued-results)))) (concat (:orders result) continued-results))))
(:orders result))))))) (:orders result)))))))
(defn search (defn search
([client location start end] ([client location start end]
(capture-context->lc (capture-context->lc
@@ -251,11 +244,9 @@
(concat (:orders result) continued-results)))) (concat (:orders result) continued-results))))
(:orders result)))))))) (:orders result))))))))
(defn amount->money [amt] (defn amount->money [amt]
(* 0.01 (or (:amount amt) 0.0))) (* 0.01 (or (:amount amt) 0.0)))
;; to get totals: ;; to get totals:
(comment (comment
(reduce (reduce
@@ -416,7 +407,6 @@
:client client :client client
:location location))))))) :location location)))))))
(defn get-payment [client p] (defn get-payment [client p]
(de/chain (manifold-api-call (de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v2/payments/" p) {:url (str "https://connect.squareup.com/v2/payments/" p)
@@ -425,7 +415,6 @@
:body :body
:payment)) :payment))
(defn continue-payout-entry-list [c l poi cursor] (defn continue-payout-entry-list [c l poi cursor]
(capture-context->lc lc (capture-context->lc lc
(de/chain (de/chain
@@ -674,8 +663,7 @@
(catch Exception e (catch Exception e
(log/error ::buffer-failed (log/error ::buffer-failed
:exception e :exception e
:order (:sales-order/external-id order))))))))))) :order (:sales-order/external-id order))))))))))))
(defn upsert-payouts (defn upsert-payouts
([client] ([client]
@@ -725,7 +713,6 @@
(log/info ::done-loading-refunds))))))) (log/info ::done-loading-refunds)))))))
(defn get-cash-shift [client id] (defn get-cash-shift [client id]
(de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id)) (de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id))
:method :get :method :get
@@ -884,8 +871,6 @@
d1 d1
d2)) d2))
(defn remove-voided-orders (defn remove-voided-orders
([client] ([client]
(apply de/zip (apply de/zip
@@ -920,22 +905,20 @@
(doseq [x (partition-all 100 results)] (doseq [x (partition-all 100 results)]
(log/info ::removing-orders (log/info ::removing-orders
:count (count x)) :count (count x))
@(dc/transact-async conn x))))) @(dc/transact-async conn x)
(de/catch (fn [e] (de/catch (fn [e]
(log/warn ::couldnt-remove :error e) (log/warn ::couldnt-remove :error e)
nil) )))))) nil)))))))))))
#_(comment #_(comment
(require 'auto-ap.time-reader) (require 'auto-ap.time-reader)
@(let [[c [l]] (get-square-client-and-location "DBFS") ] @(let [[c [l]] (get-square-client-and-location "DBFS")]
(log/peek :x [ c l]) (log/peek :x [c l])
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29") (search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29"))
) @(let [[c [l]] (get-square-client-and-location "NGAK")]
(log/peek :x [c l])
@(let [[c [l]] (get-square-client-and-location "NGAK") ]
(log/peek :x [ c l])
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15")) (remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
(doseq [c (get-square-clients)] (doseq [c (get-square-clients)]
@@ -943,8 +926,6 @@
@(remove-voided-orders c) @(remove-voided-orders c)
(catch Exception e (catch Exception e
nil))) nil)))
) )
(defn upsert-all [& clients] (defn upsert-all [& clients]
@@ -1014,8 +995,6 @@
[:clients clients] [:clients clients]
@(apply upsert-all clients))) @(apply upsert-all clients)))
(comment (comment
(defn refunds-raw-cont (defn refunds-raw-cont
([client l cursor so-far] ([client l cursor so-far]
@@ -1045,7 +1024,6 @@
(->> (->>
@(let [[c [l]] (get-square-client-and-location "NGGG")] @(let [[c [l]] (get-square-client-and-location "NGGG")]
(search c l (time/now) (time/plus (time/now) (time/days -1)))) (search c l (time/now) (time/plus (time/now) (time/days -1))))
(filter (fn [r] (filter (fn [r]
@@ -1055,7 +1033,6 @@
(->> (->>
@(let [[c [l]] (get-square-client-and-location "NGGG")] @(let [[c [l]] (get-square-client-and-location "NGGG")]
(refunds-raw-cont c l nil [])) (refunds-raw-cont c l nil []))
(filter (fn [r] (filter (fn [r]
(str/starts-with? (:created_at r) "2024-03-14"))))) (str/starts-with? (:created_at r) "2024-03-14")))))
@@ -1090,12 +1067,7 @@
[(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)]) [(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)])
:separator \tab) :separator \tab)
;; =>
;; =>
(require 'auto-ap.time-reader) (require 'auto-ap.time-reader)
@@ -1104,26 +1076,15 @@
(clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")] (clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")]
l l
(def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00" (def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00"
#clj-time/date-time "2025-02-28T00:00:00-08:00")) #clj-time/date-time "2025-02-28T00:00:00-08:00"))
(take 10 (map #(first (deref (order->sales-order c l %))) z))) (take 10 (map #(first (deref (order->sales-order c l %))) z))))
)
(->> z (->> z
(filter (fn [o] (filter (fn [o]
(seq (filter (comp #{"OTHER"} :type) (:tenders o))))) (seq (filter (comp #{"OTHER"} :type) (:tenders o)))))
(filter #(not (:name (:source %)))) (filter #(not (:name (:source %))))
(count) (count))
)
(doseq [[code] (seq (dc/q '[:find ?code (doseq [[code] (seq (dc/q '[:find ?code
:in $ :in $
@@ -1133,32 +1094,22 @@
[?o :sales-order/client ?c] [?o :sales-order/client ?c]
[?c :client/code ?code]] [?c :client/code ?code]]
(dc/db conn))) (dc/db conn)))
:let [[c [l]] (get-square-client-and-location code) :let [[c [l]] (get-square-client-and-location code)]
]
order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now)) order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now))
:when (= "Invoices" (:name (:source order) )) :when (= "Invoices" (:name (:source order)))
:let [[sales-order] @(order->sales-order c l order)]] :let [[sales-order] @(order->sales-order c l order)]]
(when (should-import-order? order) (when (should-import-order? order)
(println "DATE IS" (:sales-order/date sales-order)) (println "DATE IS" (:sales-order/date sales-order))
(when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00")) (when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00"))
(println "WOULD UPDATE" sales-order) (println "WOULD UPDATE" sales-order)
@(dc/transact auto-ap.datomic/conn [sales-order]) @(dc/transact auto-ap.datomic/conn [sales-order]))
) #_@(dc/transact)
#_@(dc/transact ) (println "DONE")))
(println "DONE"))
)
#_(filter (comp #{"OTHER"} :type) (mapcat :tenders z)) #_(filter (comp #{"OTHER"} :type) (mapcat :tenders z))
@(let [[c [l]] (get-square-client-and-location "NGRY")] @(let [[c [l]] (get-square-client-and-location "NGRY")]
#_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01")) #_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01"))
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" ))) (order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY")))))
)
)

17
src/clj/auto_ap/ssr/payments.clj
View File

@@ -116,7 +116,6 @@
:content "Debit"}]})) :content "Debit"}]}))
(exact-match-id* request)]]) (exact-match-id* request)]])
(def default-read '[* (def default-read '[*
[:payment/date :xform clj-time.coerce/from-date] [:payment/date :xform clj-time.coerce/from-date]
{:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]} {:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}
@@ -212,7 +211,6 @@
'[(iol-ion.query/dollars= ?transaction-amount ?amount)]]} '[(iol-ion.query/dollars= ?transaction-amount ?amount)]]}
:args [(:amount query-params)]}) :args [(:amount query-params)]})
(:status route-params) (:status route-params)
(merge-query {:query {:in ['?status] (merge-query {:query {:in ['?status]
:where ['[?e :payment/status ?status]]} :where ['[?e :payment/status ?status]]}
@@ -277,7 +275,6 @@
{ids-to-retrieve :ids matching-count :count {ids-to-retrieve :ids matching-count :count
all-ids :all-ids} (fetch-ids db request)] all-ids :all-ids} (fetch-ids db request)]
[(->> (hydrate-results ids-to-retrieve db request)) [(->> (hydrate-results ids-to-retrieve db request))
matching-count matching-count
(sum-visible-pending all-ids) (sum-visible-pending all-ids)
@@ -285,8 +282,7 @@
(:client request) (:client request)
(:client-id (:query-params request)) (:client-id (:query-params request))
(when (:client-code (:query-params request)) (when (:client-code (:query-params request))
[:client/code (:client-code (:query-params request))]))) [:client/code (:client-code (:query-params request))])))]))
]))
(def query-schema (mc/schema (def query-schema (mc/schema
[:maybe [:map {:date-range [:date-range :start-date :end-date]} [:maybe [:map {:date-range [:date-range :start-date :end-date]}
@@ -327,7 +323,7 @@
(assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)]) (assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)])
:query-schema query-schema :query-schema query-schema
:action-buttons (fn [request] :action-buttons (fn [request]
(let [[_ _ visible-in-float total-in-float ] (:page-results request)] (let [[_ _ visible-in-float total-in-float] (:page-results request)]
[(com/pill {:color :primary} " Visible in float " [(com/pill {:color :primary} " Visible in float "
(format "$%,.2f" visible-in-float)) (format "$%,.2f" visible-in-float))
(com/pill {:color :secondary} " Total in float " (com/pill {:color :secondary} " Total in float "
@@ -434,8 +430,6 @@
(def row* (partial helper/row* grid-page)) (def row* (partial helper/row* grid-page))
(comment (comment
(mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
@@ -445,7 +439,6 @@
(mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))) (mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)))
(defn delete [{check :entity :as request identity :identity}] (defn delete [{check :entity :as request identity :identity}]
@@ -459,7 +452,7 @@
#(assert-can-see-client identity (:db/id (:payment/client check)))) #(assert-can-see-client identity (:db/id (:payment/client check))))
(notify-if-locked (:db/id (:payment/client check)) (notify-if-locked (:db/id (:payment/client check))
(:payment/date check)) (:payment/date check))
(let [ removing-payments (mapcat (fn [x] (let [removing-payments (mapcat (fn [x]
(let [invoice (:invoice-payment/invoice x) (let [invoice (:invoice-payment/invoice x)
new-balance (+ (:invoice/outstanding-balance invoice) new-balance (+ (:invoice/outstanding-balance invoice)
(:invoice-payment/amount x))] (:invoice-payment/amount x))]
@@ -578,7 +571,6 @@
(assoc-in [:query-params :start] 0) (assoc-in [:query-params :start] 0)
(assoc-in [:query-params :per-page] 250)))) (assoc-in [:query-params :per-page] 250))))
:else :else
selected) selected)
updated-count (void-payments-internal ids (:identity request))] updated-count (void-payments-internal ids (:identity request))]
@@ -591,7 +583,7 @@
(defn wrap-status-from-source [handler] (defn wrap-status-from-source [handler]
(fn [{:keys [matched-current-page-route] :as request}] (fn [{:keys [matched-current-page-route] :as request}]
(let [ request (cond-> request (let [request (cond-> request
(= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared) (= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared)
(= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending) (= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending)
(= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided) (= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided)
@@ -618,7 +610,6 @@
::route/bulk-delete (-> bulk-delete-dialog ::route/bulk-delete (-> bulk-delete-dialog
(wrap-admin)) (wrap-admin))
::route/table (helper/table-route grid-page)} ::route/table (helper/table-route grid-page)}
(fn [h] (fn [h]
(-> h (-> h

19
src/clj/auto_ap/storage/parquet.clj
View File

@@ -22,6 +22,10 @@
(let [conn (DriverManager/getConnection "jdbc:duckdb:") (let [conn (DriverManager/getConnection "jdbc:duckdb:")
stmt (.createStatement conn)] stmt (.createStatement conn)]
(.execute stmt "INSTALL httpfs; LOAD httpfs;") (.execute stmt "INSTALL httpfs; LOAD httpfs;")
(when-let [key (:aws-access-key-id env)]
(.execute stmt (str "SET s3_access_key_id='" key "'"))
(.execute stmt (str "SET s3_secret_access_key='" (:aws-secret-access-key env) "'"))
(.execute stmt (str "SET s3_region='" (or (:aws-region env) "us-east-1") "'")))
(.close stmt) (.close stmt)
(.addShutdownHook (Runtime/getRuntime) (.addShutdownHook (Runtime/getRuntime)
(Thread. #(fn []))) (Thread. #(fn [])))
@@ -150,8 +154,9 @@
(.delete ^java.io.File parquet-file) (.delete ^java.io.File parquet-file)
{:key s3-key :status :ok} {:key s3-key :status :ok}
(catch Exception e (catch Exception e
(throw (ex-info "Flush failed" {:entity-type entity-type} (throw (ex-info "Flush failed"
:error (.getMessage e))))))))) {:entity-type entity-type
:error (.getMessage e)}))))))))
(defn flush-by-date! [] (defn flush-by-date! []
"Flush all entity types for today." "Flush all entity types for today."
@@ -218,8 +223,9 @@
Returns map with :sql and :count-sql keys." Returns map with :sql and :count-sql keys."
(let [date-strs (date-seq start-date end-date) (let [date-strs (date-seq start-date end-date)
urls (vec urls (vec
(map #(format "'s3://%%s/sales-details/%%s/%%s.parquet'" (map (fn [d]
*bucket* entity-type %) (format "'s3://%s/sales-details/%s/%s.parquet'"
*bucket* entity-type d))
date-strs)) date-strs))
sql (str "SELECT * FROM read_parquet([" sql (str "SELECT * FROM read_parquet(["
(str/join ", " urls) (str/join ", " urls)
@@ -252,10 +258,9 @@
limit (get opts :limit) limit (get opts :limit)
offset (get opts :offset) offset (get opts :offset)
where-str (build-where-clause where-str (build-where-clause
opts opts
[[:client "external_id.client"] [[:client "client-code"]
[:vendor "external_id.vendor"] [:vendor "vendor"]
[:location "location"]]) [:location "location"]])
full-sql (if where-str full-sql (if where-str
(str base-sql where-str) (str base-sql where-str)