notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(sales): fix parquet SQL generation and cleanup formatting

Browse Source 
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full
  ORDER BY clause from build-sort-clause into get-sales-orders which
  prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC')
- Fix WHERE clause column names in parquet build-where-clause:
  external_id.client -> client-code, external_id.vendor -> vendor
- Fix parquet-query format string (%%s -> %s with proper format call)
- Fix ex-info call signature in flush! (was passing :error as third arg
  instead of inside the data map)
- Add S3 credentials to DuckDB connect! so httpfs can read from S3
- Fix parquet buffer indentation and alignment across square/core3,
  ezcater/core, ezcater_xls, payments, sales_summaries, migrations
- Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions,
  upsert-recent find/where clauses mangled by paren-repair)
- Uncomment accidentally commented code block in square/core3
- Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
This commit is contained in:
Bryce
2026-04-27 10:33:22 -07:00
parent 4597611655
commit ea7f46ea8a
10 changed files with 440 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/clj/auto_ap/datomic/sales_orders.clj
View File

@@ -1,11 +1,9 @@
(ns auto-ap.datomic.sales-orders (ns auto-ap.datomic.sales-orders
(:require (:require
[auto-ap.datomic :refer [conn]]
[auto-ap.storage.parquet :as pq] [auto-ap.storage.parquet :as pq]
[clj-time.coerce :as c] [clojure.set :as set]
[clojure.set :as set] [clojure.string :as str]
[clojure.string :as str] [com.brunobonacci.mulog :as mu]))
[com.brunobonacci.mulog :as mu]))
(defn <-row (defn <-row
"Convert a flat parquet row into the shape consumers expect. "Convert a flat parquet row into the shape consumers expect.
@@ -29,12 +27,13 @@
(update :sales-order/date #(some-> % str)))) (update :sales-order/date #(some-> % str))))
(defn build-where-clause [args] (defn build-where-clause [args]
(let [clauses [(when-let [c (:client-code args)] (let [clauses (keep identity
["external_id.client = '" c "'"]) [(when-let [c (:client-code args)]
(when-let [v (:vendor args)] (str "external_id.client = '" c "'"))
["external_id.vendor = '" (name v) "'"]) (when-let [v (:vendor args)]
(when-let [l (:location args)] (str "external_id.vendor = '" (name v) "'"))
["location = '" l "'"])] (when-let [l (:location args)]
(str "location = '" l "'"))])]
(when (seq clauses) (when (seq clauses)
(str "WHERE " (str/join " AND " clauses))))) (str "WHERE " (str/join " AND " clauses)))))
@@ -47,18 +46,15 @@
(defn raw-graphql-ids [args] (defn raw-graphql-ids [args]
(let [start (some-> (:start (:date-range args)) .toString) (let [start (some-> (:start (:date-range args)) .toString)
end (some-> (:end (:date-range args)) .substring 0 10) end (some-> (:end (:date-range args)) (.substring 0 10))
where (build-where-clause args)
sort (build-sort-clause args)
limit (or (:limit args) page-size) limit (or (:limit args) page-size)
offset (or (:offset args) 0) offset (or (:offset args) 0)]
where-str (when where (str " " where))]
(when start (when start
(let [result (pq/get-sales-orders start end (let [result (pq/get-sales-orders start end
{:client (:client-code args) {:client (:client-code args)
:vendor (:vendor args) :vendor (:vendor args)
:location (:location args) :location (:location args)
:sort sort :sort (or (:sort args) "date")
:order "DESC" :order "DESC"
:limit limit :limit limit
:offset offset})] :offset offset})]

337
src/clj/auto_ap/ezcater/core.clj
View File

@@ -21,35 +21,34 @@
:body (json/write-str {"query" (v/graphql-query q)}) :body (json/write-str {"query" (v/graphql-query q)})
:as :json}) :as :json})
:body :body
:data :data))
))
(defn get-caterers [integration] (defn get-caterers [integration]
(:caterers (query integration {:venia/queries [{:query/data (:caterers (query integration {:venia/queries [{:query/data
[:caterers [:name :uuid [:address [:name :street]]]]}]} ))) [:caterers [:name :uuid [:address [:name :street]]]]}]})))
(defn get-subscriptions [integration] (defn get-subscriptions [integration]
(->> (query integration {:venia/queries [{:query/data (->> (query integration {:venia/queries [{:query/data
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]] ]]}]} ) [:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]]]]}]})
:subscribers :subscribers
first first
:subscriptions)) :subscriptions))
(defn get-integrations [] (defn get-integrations []
(map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key (map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid :ezcater-integration/subscriber-uuid
:db/id :db/id
:ezcater-integration/integration-status [:db/id]]) {:ezcater-integration/integration-status [:db/id]}])
:in $ :in $
:where [?i :ezcater-integration/api-key]] :where [?i :ezcater-integration/api-key]]
(dc/db conn)))) (dc/db conn))))
(defn mark-integration-status [integration integration-status] (defn mark-integration-status [integration integration-status]
@(dc/transact conn @(dc/transact conn
[{:db/id (:db/id integration) [{:db/id (:db/id integration)
:ezcater-integration/integration-status (assoc integration-status :ezcater-integration/integration-status (assoc integration-status
:db/id (or (-> integration :ezcater-integration/integration-status :db/id) :db/id (or (-> integration :ezcater-integration/integration-status :db/id)
(random-tempid)))}])) (random-tempid)))}]))
(defn upsert-caterers (defn upsert-caterers
([integration] ([integration]
@@ -65,14 +64,14 @@
([integration] ([integration]
(let [extant (get-subscriptions integration) (let [extant (get-subscriptions integration)
to-ensure (set (map first (dc/q '[:find ?cu to-ensure (set (map first (dc/q '[:find ?cu
:in $ :in $
:where [_ :client/ezcater-locations ?el] :where [_ :client/ezcater-locations ?el]
[?el :ezcater-location/caterer ?c] [?el :ezcater-location/caterer ?c]
[?c :ezcater-caterer/uuid ?cu]] [?c :ezcater-caterer/uuid ?cu]]
(dc/db conn)))) (dc/db conn))))
to-create (set/difference to-create (set/difference
to-ensure to-ensure
(set (map :parentId extant)))] (set (map :parentId extant)))]
(doseq [parentId to-create] (doseq [parentId to-create]
(query integration (query integration
{:venia/operation {:operation/type :mutation {:venia/operation {:operation/type :mutation
@@ -95,7 +94,6 @@
:eventKey 'cancelled}} :eventKey 'cancelled}}
[[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]}))))) [[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]})))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn upsert-ezcater (defn upsert-ezcater
([] (upsert-ezcater (get-integrations))) ([] (upsert-ezcater (get-integrations)))
@@ -116,12 +114,11 @@
(defn get-caterer [caterer-uuid] (defn get-caterer [caterer-uuid]
(dc/pull (dc/db conn) (dc/pull (dc/db conn)
'[:ezcater-caterer/name '[:ezcater-caterer/name
{:ezcater-integration/_caterers [:ezcater-integration/api-key]} {:ezcater-integration/_caterers [:ezcater-integration/api-key]}
{:ezcater-location/_caterer [:ezcater-location/location {:ezcater-location/_caterer [:ezcater-location/location
{:client/_ezcater-locations [:client/code]}]}] {:client/_ezcater-locations [:client/code]}]}]
[:ezcater-caterer/uuid caterer-uuid])) [:ezcater-caterer/uuid caterer-uuid]))
(defn round-carry-cents [f] (defn round-carry-cents [f]
(with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP)))) (with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP))))
@@ -137,69 +134,69 @@
:else :else
0.07M)] 0.07M)]
(round-carry-cents (round-carry-cents
(* commision% (* commision%
0.01M 0.01M
(+ (+
(-> order :totals :subTotal :subunits ) (-> order :totals :subTotal :subunits)
(reduce + (reduce +
0 0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))))
(defn ccp-fee [order] (defn ccp-fee [order]
(round-carry-cents (round-carry-cents
(* 0.000299M (* 0.000299M
(+ (+
(-> order :totals :subTotal :subunits ) (-> order :totals :subTotal :subunits)
(-> order :totals :salesTax :subunits ) (-> order :totals :salesTax :subunits)
(reduce + (reduce +
0 0
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
(defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}] (defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}]
(let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01)) (let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01))
(-> order :totals :salesTax :subunits (* 0.01))) (-> order :totals :salesTax :subunits (* 0.01)))
(-> order :catererCart :totals :catererTotalDue ) (-> order :catererCart :totals :catererTotalDue)
(commision order) (commision order)
(ccp-fee order))) (ccp-fee order)))
service-charge (+ (commision order) (ccp-fee order)) service-charge (+ (commision order) (ccp-fee order))
tax (-> order :totals :salesTax :subunits (* 0.01)) tax (-> order :totals :salesTax :subunits (* 0.01))
tip (-> order :totals :tip :subunits (* 0.01))] tip (-> order :totals :tip :subunits (* 0.01))]
#:sales-order #:sales-order
{:date (atime/localize (coerce/to-date-time timestamp)) {:date (atime/localize (coerce/to-date-time timestamp))
:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid) :external-id (str "ezcater/order/" client-code "-" client-location "-" uuid)
:client [:client/code client-code] :client [:client/code client-code]
:location client-location :location client-location
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid )) :reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid))
:line-items [#:order-line-item :line-items [#:order-line-item
{:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0) {:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0)
:item-name "EZCater Catering" :item-name "EZCater Catering"
:category "EZCater Catering" :category "EZCater Catering"
:discount adjustment :discount adjustment
:tax tax :tax tax
:total (+ (-> order :totals :subTotal :subunits (* 0.01)) :total (+ (-> order :totals :subTotal :subunits (* 0.01))
tax tax
tip)}] tip)}]
:charges [#:charge :charges [#:charge
{:type-name "CARD" {:type-name "CARD"
:date (atime/localize (coerce/to-date-time timestamp)) :date (atime/localize (coerce/to-date-time timestamp))
:client [:client/code client-code] :client [:client/code client-code]
:location client-location :location client-location
:external-id (str "ezcater/charge/" uuid) :external-id (str "ezcater/charge/" uuid)
:processor :ccp-processor/ezcater :processor :ccp-processor/ezcater
:total (+ (-> order :totals :subTotal :subunits (* 0.01)) :total (+ (-> order :totals :subTotal :subunits (* 0.01))
tax tax
tip) tip)
:tip tip}] :tip tip}]
:total (+ (-> order :totals :subTotal :subunits (* 0.01)) :total (+ (-> order :totals :subTotal :subunits (* 0.01))
tax tax
tip) tip)
:discount adjustment :discount adjustment
:service-charge service-charge :service-charge service-charge
:tax tax :tax tax
:tip tip :tip tip
:returns 0.0 :returns 0.0
:vendor :vendor/ccp-ezcater}})) :vendor :vendor/ccp-ezcater}))
(defn- flatten-order-to-parquet! [order] (defn- flatten-order-to-parquet! [order]
"Flatten a sales-order into entity-type tagged maps and buffer to parquet." "Flatten a sales-order into entity-type tagged maps and buffer to parquet."
@@ -222,81 +219,73 @@
(when-let [charges (:sales-order/charges order)] (when-let [charges (:sales-order/charges order)]
(doseq [chg charges] (doseq [chg charges]
(parquet/buffer! "charge" (parquet/buffer! "charge"
{:entity-type "charge" {:entity-type "charge"
:external-id (:charge/external-id chg) :external-id (:charge/external-id chg)
:type-name (:charge/type-name chg) :type-name (:charge/type-name chg)
:total (:charge/total chg) :total (:charge/total chg)
:tax (:charge/tax chg) :tax (:charge/tax chg)
:tip (:charge/tip chg) :tip (:charge/tip chg)
:date so-date :date so-date
:processor (some-> (:charge/processor chg) name) :processor (some-> (:charge/processor chg) name)
:sales-order-external-id so-ext-id}))) :sales-order-external-id so-ext-id})))
(when-let [items (:sales-order/line-items order)] (when-let [items (:sales-order/line-items order)]
(doseq [li items] (doseq [li items]
(parquet/buffer! "line-item" (parquet/buffer! "line-item"
{:entity-type "line-item" {:entity-type "line-item"
:item-name (:order-line-item/item-name li) :item-name (:order-line-item/item-name li)
:category (:order-line-item/category li) :category (:order-line-item/category li)
:total (:order-line-item/total li) :total (:order-line-item/total li)
:tax (:order-line-item/tax li) :tax (:order-line-item/tax li)
:discount (:order-line-item/discount li) :discount (:order-line-item/discount li)
:sales-order-external-id so-ext-id}))))) :sales-order-external-id so-ext-id})))))
(defn get-by-id [integration id] (defn get-by-id [integration id]
(query (query
integration integration
{:venia/queries [[:order {:id id} {:venia/queries [[:order {:id id}
[:uuid [:uuid
:orderNumber :orderNumber
:orderSourceType :orderSourceType
[:caterer [:caterer
[:name [:name
:uuid :uuid
[:address [:street]]]] [:address [:street]]]]
[:event [:event
[:timestamp [:timestamp
:catererHandoffFoodTime :catererHandoffFoodTime
:orderType]] :orderType]]
[:catererCart [[:orderItems [:catererCart [[:orderItems
[:name [:name
:quantity :quantity
:posItemId :posItemId
[:totalInSubunits [:totalInSubunits
[:currency [:currency
:subunits]]]] :subunits]]]]
[:totals [:totals
[:catererTotalDue]] [:catererTotalDue]]
[:feesAndDiscounts [:feesAndDiscounts
{:type 'DELIVERY_FEE} {:type 'DELIVERY_FEE}
[[:cost [[:cost
[:currency [:currency
:subunits]]]]]] :subunits]]]]]]
[:totals [[:customerTotalDue [:totals [[:customerTotalDue
[ [:currency
:currency :subunits]]
:subunits [:pointOfSaleIntegrationFee
]] [:currency
[:pointOfSaleIntegrationFee :subunits]]
[ [:tip
:currency [:currency
:subunits :subunits]]
]] [:salesTax
[:tip [:currency
[:currency :subunits]]
:subunits]] [:salesTaxRemittance
[:salesTax [:currency
[ :subunits]]
:currency [:subTotal
:subunits [:currency
]] :subunits]]]]]]]}))
[:salesTaxRemittance
[:currency
:subunits
]]
[:subTotal
[:currency
:subunits]]]]]]]}))
(defn lookup-order [json] (defn lookup-order [json]
(let [caterer (get-caterer (get json "parent_id")) (let [caterer (get-caterer (get json "parent_id"))
@@ -305,17 +294,17 @@
location (-> caterer :ezcater-location/_caterer first :ezcater-location/location)] location (-> caterer :ezcater-location/_caterer first :ezcater-location/location)]
(if (and client location) (if (and client location)
(doto (doto
(-> (get-by-id integration (get json "entity_id")) (-> (get-by-id integration (get json "entity_id"))
(:order) (:order)
(assoc :client-code client (assoc :client-code client
:client-location location)) :client-location location))
(#(alog/info ::order-details :detail %))) (#(alog/info ::order-details :detail %)))
(alog/warn ::caterer-no-longer-has-location :json json)))) (alog/warn ::caterer-no-longer-has-location :json json))))
(defn import-order [json] (defn import-order [json]
(alog/info (alog/info
::try-import-order ::try-import-order
:json json) :json json)
(when-let [order (some-> json (when-let [order (some-> json
(lookup-order) (lookup-order)
(order->sales-order) (order->sales-order)
@@ -324,7 +313,7 @@
(try (try
(flatten-order-to-parquet! order) (flatten-order-to-parquet! order)
(alog/info ::order-buffered (alog/info ::order-buffered
:external-id (:sales-order/external-id order)) :external-id (:sales-order/external-id order))
(catch Exception e (catch Exception e
(alog/error ::buffer-failed (alog/error ::buffer-failed
:exception e :exception e
@@ -359,30 +348,30 @@
"key" "accepted", "key" "accepted",
"occurred_at" "2022-07-21T19:21:07.549Z"} "occurred_at" "2022-07-21T19:21:07.549Z"}
ezcater-order (lookup-order lookup-map) ezcater-order (lookup-order lookup-map)
extant-order (dc/pull (dc/db conn) '[:sales-order/total extant-order (dc/pull (dc/db conn '[:sales-order/total]
:sales-order/tax :sales-order/tax
:sales-order/tip :sales-order/tip
:sales-order/discount :sales-order/discount
:sales-order/external-id :sales-order/external-id
{:sales-order/charges [:charge/tax {:sales-order/charges [:charge/tax
:charge/tip :charge/tip
:charge/total :charge/total
:charge/external-id] :charge/external-id]
:sales-order/line-items [:order-line-item/external-id :sales-order/line-items [:order-line-item/external-id
:order-line-item/total :order-line-item/total
:order-line-item/tax :order-line-item/tax
:order-line-item/discount]}] :order-line-item/discount]})
[:sales-order/external-id order]) [:sales-order/external-id order])
updated-order (-> (order->sales-order ezcater-order) updated-order (-> (order->sales-order ezcater-order)
(select-keys (select-keys
#{:sales-order/total #{:sales-order/total
:sales-order/tax :sales-order/tax
:sales-order/tip :sales-order/tip
:sales-order/discount :sales-order/discount
:sales-order/charges :sales-order/charges
:sales-order/external-id :sales-order/external-id
:sales-order/line-items}) :sales-order/line-items})
(update :sales-order/line-items (update :sales-order/line-items
(fn [c] (fn [c]
(map #(select-keys % #{:order-line-item/external-id (map #(select-keys % #{:order-line-item/external-id

54
src/clj/auto_ap/jobs/sales_summaries.clj
View File

@@ -40,17 +40,14 @@
(dc/db conn) (dc/db conn)
number))) number)))
(defn delete-all [] (defn delete-all []
@(dc/transact-async conn @(dc/transact-async conn
(->> (->>
(dc/q '[:find ?ss (dc/q '[:find ?ss
:where [?ss :sales-summary/date]] :where [?ss :sales-summary/date]]
(dc/db conn)) (dc/db conn))
(map (fn [[ ss]] (map (fn [[ss]]
[:db/retractEntity ss]))))) [:db/retractEntity ss])))))
(defn dirty-sales-summaries [c] (defn dirty-sales-summaries [c]
(let [client-id (dc/entid (dc/db conn) c)] (let [client-id (dc/entid (dc/db conn) c)]
@@ -138,9 +135,9 @@
{:db/id (str (java.util.UUID/randomUUID)) {:db/id (str (java.util.UUID/randomUUID))
:sales-summary-item/sort-order 3 :sales-summary-item/sort-order 3
:sales-summary-item/category (cond :sales-summary-item/category (cond
(= type-name "CARD") "Card Refunds" (= type-name "CARD") "Card Refunds"
(= type-name "CASH") "Cash Refunds" (= type-name "CASH") "Cash Refunds"
:else "Food App Refunds") :else "Food App Refunds")
:ledger-mapped/amount total :ledger-mapped/amount total
:ledger-mapped/ledger-side :ledger-side/credit}) :ledger-mapped/ledger-side :ledger-side/credit})
refunds)))) refunds))))
@@ -188,8 +185,6 @@
:ledger-mapped/ledger-side :ledger-side/credit :ledger-mapped/ledger-side :ledger-side/credit
:ledger-mapped/amount (- (+ total discount) tax)}))) :ledger-mapped/amount (- (+ total discount) tax)})))
(defn get-fees [c date] (defn get-fees [c date]
(when-let [fee (get-fee c date)] (when-let [fee (get-fee c date)]
{:db/id (str (java.util.UUID/randomUUID)) {:db/id (str (java.util.UUID/randomUUID))
@@ -299,18 +294,17 @@
(filter identity) (filter identity)
(map (fn [z] (map (fn [z]
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account) (assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
:sales-summary-item/manual? false)) :sales-summary-item/manual? false))))}]
)) }]
(if (seq (:sales-summary/items result)) (if (seq (:sales-summary/items result))
(do (do
(alog/info ::upserting-summaries (alog/info ::upserting-summaries
:category-count (count (:sales-summary/items result))) :category-count (count (:sales-summary/items result)))
@(dc/transact conn [[:upsert-entity result]])) @(dc/transact conn [[:upsert-entity result]]))
@(dc/transact conn [{:db/id id :sales-summary/dirty false}])))))) @(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
(comment (comment
;; TODO: Move to test file or proper location ;; TODO: Move to test file or proper location
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ]) (let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL"])
date #inst "2024-04-14T00:00:00-07:00"] date #inst "2024-04-14T00:00:00-07:00"]
(get-payment-items c date))) (get-payment-items c date)))
@@ -322,16 +316,13 @@
(map (fn [[sos]] (map (fn [[sos]]
[:db/retractEntity sos]))))) [:db/retractEntity sos])))))
(comment (comment
(auto-ap.datomic/transact-schema conn) (auto-ap.datomic/transact-schema conn)
@(dc/transact conn [{:db/ident :sales-summary/total-unknown-processor-payments @(dc/transact conn [{:db/ident :sales-summary/total-unknown-processor-payments
:db/noHistory true, :db/noHistory true,
:db/valueType :db.type/double :db/valueType :db.type/double
:db/cardinality :db.cardinality/one}]) :db/cardinality :db.cardinality/one}])
(apply mark-dirty [:client/code "NGCL"] (last-n-days 30)) (apply mark-dirty [:client/code "NGCL"] (last-n-days 30))
@@ -363,23 +354,18 @@
(dc/db conn) (dc/db conn)
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGHW"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-11T00:00:00-07:00"]) [[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGHW"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-11T00:00:00-07:00"])
(dc/q '[:find ?n (dc/q '[:find ?n
:in $ [?clients ?start-date ?end-date] :in $ [?clients ?start-date ?end-date]
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]] :where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
[?e :sales-order/line-items ?li] [?e :sales-order/line-items ?li]
[?li :order-line-item/item-name ?n] ] [?li :order-line-item/item-name ?n]]
(dc/db conn) (dc/db conn)
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"]) [[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"])
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy} @(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}]) {:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
(auto-ap.datomic/transact-schema conn)
)
(auto-ap.datomic/transact-schema conn))
(defn -main [& _] (defn -main [& _]
(execute "sales-summaries" sales-summaries-v2)) (execute "sales-summaries" sales-summaries-v2))

44
src/clj/auto_ap/migration/cleanup_sales.clj
View File

@@ -17,8 +17,8 @@
"Return all entity IDs that have :sales-order/external-id." "Return all entity IDs that have :sales-order/external-id."
[db] [db]
(->> (d-api/q '[:find ?e (->> (d-api/q '[:find ?e
:where [?e :sales-order/external-id]] :where [?e :sales-order/external-id]]
db) db)
(map first))) (map first)))
(defn- collect-child-ids (defn- collect-child-ids
@@ -28,19 +28,19 @@
[db order-ids] [db order-ids]
(let [order-set (set order-ids) (let [order-set (set order-ids)
charges (->> (d-api/q '[:find ?c charges (->> (d-api/q '[:find ?c
:in $ [?o ...] :in $ [?o ...]
:where [$ ?o :sales-order/charges ?c]] :where [$ ?o :sales-order/charges ?c]]
db order-set) db order-set)
(map second)) (map second))
refunds (->> (d-api/q '[:find ?r refunds (->> (d-api/q '[:find ?r
:in $ [?o ...] :in $ [?o ...]
:where [$ ?o :sales-order/refunds ?r]] :where [$ ?o :sales-order/refunds ?r]]
db order-set) db order-set)
(map second)) (map second))
line-items (->> (d-api/q '[:find ?li line-items (->> (d-api/q '[:find ?li
:in $ [?c ...] :in $ [?c ...]
:where [$ ?c :charge/line-items ?li]] :where [$ ?c :charge/line-items ?li]]
db charges) db charges)
(map second))] (map second))]
{:orders order-ids {:orders order-ids
:charges (vec charges) :charges (vec charges)
@@ -149,7 +149,7 @@
all-ids (query-sales-order-ids db) all-ids (query-sales-order-ids db)
group (group-orders-by-month db all-ids) group (group-orders-by-month db all-ids)
target-keys (get group [year month] [])] target-keys (get group [year month] [])]
(if (zero? (count target-keys)) (if (zero? (count target-keys))
(do (println " no orders found for" year "-" month) (do (println " no orders found for" year "-" month)
:skipped) :skipped)
(do (do
@@ -202,18 +202,18 @@
missing (:missing result)] missing (:missing result)]
(cond (cond
(:ok result) (:ok result)
(do (println "verified" y "-" m "S3 OK, deleting...") (do (println "verified" y "-" m "S3 OK, deleting...")
(delete-by-month conn$ nil y m)) (delete-by-month conn$ nil y m))
(> (count missing) 0) (> (count missing) 0)
(do (println "ERROR" y "-" m "missing in S3:" (do (println "ERROR" y "-" m "missing in S3:"
(str/join ", " missing)) (str/join ", " missing))
(throw (throw
(ex-info (ex-info
"Missing S3 data — aborting!" "Missing S3 data — aborting!"
{:year y :month m {:year y :month m
:missing missing}))) :missing missing})))
:else :else
(println "SKIPPING" y "-" m "no parquet files"))))) (println "SKIPPING" y "-" m "no parquet files")))))
(println "safe-cleanup-all complete"))) (println "safe-cleanup-all complete")))

21
src/clj/auto_ap/migration/sales_to_parquet.clj
View File

@@ -42,7 +42,7 @@
:charge/date :charge/date
:charge/processor :charge/processor
:charge/returns :charge/returns
{:charge/client [:client/code]}]} {:charge/client [:client/code]}]}
{:sales-order/line-items {:sales-order/line-items
[:order-line-item/item-name [:order-line-item/item-name
:order-line-item/category :order-line-item/category
@@ -122,13 +122,12 @@
(->> (dc/q '[:find ?e (->> (dc/q '[:find ?e
:in $ ?start-ms ?end-ms :in $ ?start-ms ?end-ms
:where [_ :sales-order/date ?d] :where [_ :sales-order/date ?d]
[(>= ?d ?start-ms)] [(>= ?d ?start-ms)]
[(<= ?d ?end-ms)]] [(<= ?d ?end-ms)]]
db start end) db start end)
(map first) (map first)
vec))) vec)))
(defn- date-seq [start end] (defn- date-seq [start end]
"Seq of YYYY-MM-DD strings between start and end inclusive." "Seq of YYYY-MM-DD strings between start and end inclusive."
(let [sd (java.time.LocalDate/parse start) (let [sd (java.time.LocalDate/parse start)
@@ -163,7 +162,7 @@
"line-item" "sales-refund"]] "line-item" "sales-refund"]]
(p/flush-to-parquet! etype)) (p/flush-to-parquet! etype))
(println "[migration]" day "complete")) (println "[migration]" day "complete"))
{:status :completed :total-days (count date-range)}))) {:status :completed :total-days (count date-range)})))
(defn- write-dead-letter (defn- write-dead-letter
([flat] ([flat]
@@ -192,11 +191,11 @@
(defn- get-date-range [] (defn- get-date-range []
"Get the earliest and latest business dates from Datomic." "Get the earliest and latest business dates from Datomic."
(let [dates (->> (dc/q '[:find ?d (let [dates (->> (dc/q '[:find ?d
:where [_ :sales-order/date ?d]] :where [_ :sales-order/date ?d]]
(dc/db conn)) (dc/db conn))
(map first) (map first)
distinct distinct
sort)] sort)]
[(when (seq dates) (.toString (first dates))) [(when (seq dates) (.toString (first dates)))
(when (seq dates) (.toString (last dates)))])) (when (seq dates) (.toString (last dates)))]))
@@ -220,7 +219,7 @@
(let [flat (volatile! [])] (let [flat (volatile! [])]
(flatten-order-to-pieces! o flat) (flatten-order-to-pieces! o flat)
(doseq [r @flat] (doseq [r @flat]
(p/buffer! "dead" r)))) (p/buffer! "dead" r))))
(write-day-by-day start-date end-date {:batch-size 100}) (write-day-by-day start-date end-date {:batch-size 100})
(flush-all-types) (flush-all-types)
(println "[migration] done") (println "[migration] done")

122
src/clj/auto_ap/routes/ezcater_xls.clj
View File

@@ -55,54 +55,54 @@
event-date (some-> (excel/xls-date->date event-date) event-date (some-> (excel/xls-date->date event-date)
coerce/to-date-time coerce/to-date-time
atime/as-local-time atime/as-local-time
coerce/to-date )] coerce/to-date)]
(cond (and event-date client-id location ) (cond (and event-date client-id location)
[:order #:sales-order [:order #:sales-order
{:date event-date {:date event-date
:external-id (str "ezcater/order/" client-id "-" location "-" order-number) :external-id (str "ezcater/order/" client-id "-" location "-" order-number)
:client client-id :client client-id
:location location :location location
:reference-link (str order-number) :reference-link (str order-number)
:line-items [#:order-line-item :line-items [#:order-line-item
{:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0) {:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0)
:item-name "EZCater Catering" :item-name "EZCater Catering"
:category "EZCater Catering" :category "EZCater Catering"
:discount (fmt-amount (or adjustments 0.0)) :discount (fmt-amount (or adjustments 0.0))
:tax (fmt-amount tax) :tax (fmt-amount tax)
:total (fmt-amount (+ food-total :total (fmt-amount (+ food-total
tax))}] tax))}]
:charges [#:charge :charges [#:charge
{:type-name "CARD" {:type-name "CARD"
:date event-date :date event-date
:client client-id :client client-id
:location location :location location
:external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0) :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
:processor :ccp-processor/ezcater :processor :ccp-processor/ezcater
:total (fmt-amount (+ food-total :total (fmt-amount (+ food-total
tax tax
tip)) tip))
:tip (fmt-amount tip)}] :tip (fmt-amount tip)}]
:total (fmt-amount (+ food-total :total (fmt-amount (+ food-total
tax tax
(or adjustments 0.0))) (or adjustments 0.0)))
:discount (fmt-amount (or adjustments 0.0)) :discount (fmt-amount (or adjustments 0.0))
:service-charge (fmt-amount (+ fee commission)) :service-charge (fmt-amount (+ fee commission))
:tax (fmt-amount tax) :tax (fmt-amount tax)
:tip (fmt-amount tip) :tip (fmt-amount tip)
:returns 0.0 :returns 0.0
:vendor :vendor/ccp-ezcater}] :vendor :vendor/ccp-ezcater}]
caterer-name caterer-name
(do (do
(alog/warn ::missing-client (alog/warn ::missing-client
:order order-number :order order-number
:store-name store-name :store-name store-name
:caterer-name caterer-name) :caterer-name caterer-name)
[:missing caterer-name]) [:missing caterer-name])
:else :else
nil))) nil)))
(defn- flatten-order-to-parquet! [order] (defn- flatten-order-to-parquet! [order]
"Flatten a sales-order into entity-type tagged maps and buffer to parquet." "Flatten a sales-order into entity-type tagged maps and buffer to parquet."
@@ -125,25 +125,25 @@
(when-let [charges (:sales-order/charges order)] (when-let [charges (:sales-order/charges order)]
(doseq [chg charges] (doseq [chg charges]
(parquet/buffer! "charge" (parquet/buffer! "charge"
{:entity-type "charge" {:entity-type "charge"
:external-id (:charge/external-id chg) :external-id (:charge/external-id chg)
:type-name (:charge/type-name chg) :type-name (:charge/type-name chg)
:total (:charge/total chg) :total (:charge/total chg)
:tax (:charge/tax chg) :tax (:charge/tax chg)
:tip (:charge/tip chg) :tip (:charge/tip chg)
:date so-date :date so-date
:processor (some-> (:charge/processor chg) name) :processor (some-> (:charge/processor chg) name)
:sales-order-external-id so-ext-id}))) :sales-order-external-id so-ext-id})))
(when-let [items (:sales-order/line-items order)] (when-let [items (:sales-order/line-items order)]
(doseq [li items] (doseq [li items]
(parquet/buffer! "line-item" (parquet/buffer! "line-item"
{:entity-type "line-item" {:entity-type "line-item"
:item-name (:order-line-item/item-name li) :item-name (:order-line-item/item-name li)
:category (:order-line-item/category li) :category (:order-line-item/category li)
:total (:order-line-item/total li) :total (:order-line-item/total li)
:tax (:order-line-item/tax li) :tax (:order-line-item/tax li)
:discount (:order-line-item/discount li) :discount (:order-line-item/discount li)
:sales-order-external-id so-ext-id}))))) :sales-order-external-id so-ext-id})))))
(defn stream->sales-orders [s] (defn stream->sales-orders [s]
(let [clients (map first (dc/q '[:find (pull ?c [:client/code (let [clients (map first (dc/q '[:find (pull ?c [:client/code
@@ -235,7 +235,7 @@
[:li ml])]])])) [:li ml])]])]))
(catch Exception e (catch Exception e
(alog/error ::import-error (alog/error ::import-error
:error e) :error e)
(html-response [:div (.getMessage e)])))))) (html-response [:div (.getMessage e)]))))))
(defn page [{:keys [matched-route request-method] :as request}] (defn page [{:keys [matched-route request-method] :as request}]

151
src/clj/auto_ap/square/core3.clj
View File

@@ -28,11 +28,9 @@
"Authorization" (str "Bearer " (:client/square-auth-token client)) "Authorization" (str "Bearer " (:client/square-auth-token client))
"Content-Type" "application/json"})) "Content-Type" "application/json"}))
(defn ->square-date [d] (defn ->square-date [d]
(f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d)) (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d))
(def manifold-api-stream (def manifold-api-stream
(let [stream (s/stream 100)] (let [stream (s/stream 100)]
(->> stream (->> stream
@@ -43,10 +41,10 @@
(de/loop [attempt 0] (de/loop [attempt 0]
(-> (de/chain (de/future-with (ex/execute-pool) (-> (de/chain (de/future-with (ex/execute-pool)
#_(log/info ::request-started #_(log/info ::request-started
:url (:url request) :url (:url request)
:attempt attempt :attempt attempt
:source "Square 3" :source "Square 3"
:background-job "Square 3") :background-job "Square 3")
(try (try
(client/request (assoc request (client/request (assoc request
:socket-timeout 10000 :socket-timeout 10000
@@ -105,7 +103,6 @@
:exception error)) :exception error))
[])))) []))))
(def item-cache (atom {})) (def item-cache (atom {}))
(defn fetch-catalog [client i v] (defn fetch-catalog [client i v]
@@ -125,13 +122,11 @@
#(do (swap! item-cache assoc i %) #(do (swap! item-cache assoc i %)
%)))) %))))
(defn fetch-catalog-cache [client i version] (defn fetch-catalog-cache [client i version]
(if (get @item-cache i) (if (get @item-cache i)
(de/success-deferred (get @item-cache i)) (de/success-deferred (get @item-cache i))
(fetch-catalog client i version))) (fetch-catalog client i version)))
(defn item->category-name-impl [client item version] (defn item->category-name-impl [client item version]
(capture-context->lc (capture-context->lc
(cond (:item_id (:item_variation_data item)) (cond (:item_id (:item_variation_data item))
@@ -162,7 +157,6 @@
:item item) :item item)
"Uncategorized")))) "Uncategorized"))))
(defn item-id->category-name [client i version] (defn item-id->category-name [client i version]
(capture-context->lc (capture-context->lc
(-> [client i] (-> [client i]
@@ -227,7 +221,6 @@
(concat (:orders result) continued-results)))) (concat (:orders result) continued-results))))
(:orders result))))))) (:orders result)))))))
(defn search (defn search
([client location start end] ([client location start end]
(capture-context->lc (capture-context->lc
@@ -251,11 +244,9 @@
(concat (:orders result) continued-results)))) (concat (:orders result) continued-results))))
(:orders result)))))))) (:orders result))))))))
(defn amount->money [amt] (defn amount->money [amt]
(* 0.01 (or (:amount amt) 0.0))) (* 0.01 (or (:amount amt) 0.0)))
;; to get totals: ;; to get totals:
(comment (comment
(reduce (reduce
@@ -416,7 +407,6 @@
:client client :client client
:location location))))))) :location location)))))))
(defn get-payment [client p] (defn get-payment [client p]
(de/chain (manifold-api-call (de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v2/payments/" p) {:url (str "https://connect.squareup.com/v2/payments/" p)
@@ -425,7 +415,6 @@
:body :body
:payment)) :payment))
(defn continue-payout-entry-list [c l poi cursor] (defn continue-payout-entry-list [c l poi cursor]
(capture-context->lc lc (capture-context->lc lc
(de/chain (de/chain
@@ -610,8 +599,8 @@
so-date (some-> (:sales-order/date order) .toString) so-date (some-> (:sales-order/date order) .toString)
client (:sales-order/client order) client (:sales-order/client order)
client-code (when client (if (map? client) client-code (when client (if (map? client)
(:client/code client) (:client/code client)
client))] client))]
(parquet/buffer! "sales-order" (parquet/buffer! "sales-order"
{:entity-type "sales-order" {:entity-type "sales-order"
:external-id so-ext-id :external-id so-ext-id
@@ -627,32 +616,32 @@
(when-let [charges (:sales-order/charges order)] (when-let [charges (:sales-order/charges order)]
(doseq [chg charges] (doseq [chg charges]
(parquet/buffer! "charge" (parquet/buffer! "charge"
{:entity-type "charge" {:entity-type "charge"
:external-id (:charge/external-id chg) :external-id (:charge/external-id chg)
:type-name (:charge/type-name chg) :type-name (:charge/type-name chg)
:total (:charge/total chg) :total (:charge/total chg)
:tax (:charge/tax chg) :tax (:charge/tax chg)
:tip (:charge/tip chg) :tip (:charge/tip chg)
:date so-date :date so-date
:processor (some-> (:charge/processor chg) name) :processor (some-> (:charge/processor chg) name)
:sales-order-external-id so-ext-id}) :sales-order-external-id so-ext-id})
(when-let [returns (:charge/returns chg)] (when-let [returns (:charge/returns chg)]
(doseq [rt returns] (doseq [rt returns]
(parquet/buffer! "sales-refund" (parquet/buffer! "sales-refund"
{:entity-type "sales-refund" {:entity-type "sales-refund"
:type-name (:type-name rt) :type-name (:type-name rt)
:total (:total rt) :total (:total rt)
:sales-order-external-id so-ext-id}))))) :sales-order-external-id so-ext-id})))))
(when-let [items (:sales-order/line-items order)] (when-let [items (:sales-order/line-items order)]
(doseq [li items] (doseq [li items]
(parquet/buffer! "line-item" (parquet/buffer! "line-item"
{:entity-type "line-item" {:entity-type "line-item"
:item-name (:order-line-item/item-name li) :item-name (:order-line-item/item-name li)
:category (:order-line-item/category li) :category (:order-line-item/category li)
:total (:order-line-item/total li) :total (:order-line-item/total li)
:tax (:order-line-item/tax li) :tax (:order-line-item/tax li)
:discount (:order-line-item/discount li) :discount (:order-line-item/discount li)
:sales-order-external-id so-ext-id}))))) :sales-order-external-id so-ext-id})))))
(defn upsert (defn upsert
([client] ([client]
@@ -674,8 +663,7 @@
(catch Exception e (catch Exception e
(log/error ::buffer-failed (log/error ::buffer-failed
:exception e :exception e
:order (:sales-order/external-id order))))))))))) :order (:sales-order/external-id order))))))))))))
(defn upsert-payouts (defn upsert-payouts
([client] ([client]
@@ -725,7 +713,6 @@
(log/info ::done-loading-refunds))))))) (log/info ::done-loading-refunds)))))))
(defn get-cash-shift [client id] (defn get-cash-shift [client id]
(de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id)) (de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id))
:method :get :method :get
@@ -884,8 +871,6 @@
d1 d1
d2)) d2))
(defn remove-voided-orders (defn remove-voided-orders
([client] ([client]
(apply de/zip (apply de/zip
@@ -920,32 +905,28 @@
(doseq [x (partition-all 100 results)] (doseq [x (partition-all 100 results)]
(log/info ::removing-orders (log/info ::removing-orders
:count (count x)) :count (count x))
@(dc/transact-async conn x))))) @(dc/transact-async conn x)
(de/catch (fn [e] (de/catch (fn [e]
(log/warn ::couldnt-remove :error e) (log/warn ::couldnt-remove :error e)
nil) )))))) nil)))))))))))
#_(comment #_(comment
(require 'auto-ap.time-reader) (require 'auto-ap.time-reader)
@(let [[c [l]] (get-square-client-and-location "DBFS") ] @(let [[c [l]] (get-square-client-and-location "DBFS")]
(log/peek :x [ c l]) (log/peek :x [c l])
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29") (search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29"))
) @(let [[c [l]] (get-square-client-and-location "NGAK")]
(log/peek :x [c l])
@(let [[c [l]] (get-square-client-and-location "NGAK") ] (remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
(log/peek :x [ c l]) (doseq [c (get-square-clients)]
(try
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15")) @(remove-voided-orders c)
(doseq [c (get-square-clients)] (catch Exception e
(try nil)))
@(remove-voided-orders c) )
(catch Exception e
nil)))
)
(defn upsert-all [& clients] (defn upsert-all [& clients]
(capture-context->lc (capture-context->lc
@@ -1014,8 +995,6 @@
[:clients clients] [:clients clients]
@(apply upsert-all clients))) @(apply upsert-all clients)))
(comment (comment
(defn refunds-raw-cont (defn refunds-raw-cont
([client l cursor so-far] ([client l cursor so-far]
@@ -1045,7 +1024,6 @@
(->> (->>
@(let [[c [l]] (get-square-client-and-location "NGGG")] @(let [[c [l]] (get-square-client-and-location "NGGG")]
(search c l (time/now) (time/plus (time/now) (time/days -1)))) (search c l (time/now) (time/plus (time/now) (time/days -1))))
(filter (fn [r] (filter (fn [r]
@@ -1055,7 +1033,6 @@
(->> (->>
@(let [[c [l]] (get-square-client-and-location "NGGG")] @(let [[c [l]] (get-square-client-and-location "NGGG")]
(refunds-raw-cont c l nil [])) (refunds-raw-cont c l nil []))
(filter (fn [r] (filter (fn [r]
(str/starts-with? (:created_at r) "2024-03-14"))))) (str/starts-with? (:created_at r) "2024-03-14")))))
@@ -1090,12 +1067,7 @@
[(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)]) [(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)])
:separator \tab) :separator \tab)
;; =>
;; =>
(require 'auto-ap.time-reader) (require 'auto-ap.time-reader)
@@ -1104,26 +1076,15 @@
(clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")] (clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")]
l l
(def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00" (def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00"
#clj-time/date-time "2025-02-28T00:00:00-08:00")) #clj-time/date-time "2025-02-28T00:00:00-08:00"))
(take 10 (map #(first (deref (order->sales-order c l %))) z))) (take 10 (map #(first (deref (order->sales-order c l %))) z))))
)
(->> z (->> z
(filter (fn [o] (filter (fn [o]
(seq (filter (comp #{"OTHER"} :type) (:tenders o))))) (seq (filter (comp #{"OTHER"} :type) (:tenders o)))))
(filter #(not (:name (:source %)))) (filter #(not (:name (:source %))))
(count) (count))
)
(doseq [[code] (seq (dc/q '[:find ?code (doseq [[code] (seq (dc/q '[:find ?code
:in $ :in $
@@ -1133,32 +1094,22 @@
[?o :sales-order/client ?c] [?o :sales-order/client ?c]
[?c :client/code ?code]] [?c :client/code ?code]]
(dc/db conn))) (dc/db conn)))
:let [[c [l]] (get-square-client-and-location code) :let [[c [l]] (get-square-client-and-location code)]
]
order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now)) order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now))
:when (= "Invoices" (:name (:source order) )) :when (= "Invoices" (:name (:source order)))
:let [[sales-order] @(order->sales-order c l order)]] :let [[sales-order] @(order->sales-order c l order)]]
(when (should-import-order? order) (when (should-import-order? order)
(println "DATE IS" (:sales-order/date sales-order)) (println "DATE IS" (:sales-order/date sales-order))
(when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00")) (when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00"))
(println "WOULD UPDATE" sales-order) (println "WOULD UPDATE" sales-order)
@(dc/transact auto-ap.datomic/conn [sales-order]) @(dc/transact auto-ap.datomic/conn [sales-order]))
) #_@(dc/transact)
#_@(dc/transact ) (println "DONE")))
(println "DONE"))
)
#_(filter (comp #{"OTHER"} :type) (mapcat :tenders z)) #_(filter (comp #{"OTHER"} :type) (mapcat :tenders z))
@(let [[c [l]] (get-square-client-and-location "NGRY")] @(let [[c [l]] (get-square-client-and-location "NGRY")]
#_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01")) #_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01"))
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" ))) (order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY")))))
)
)

85
src/clj/auto_ap/ssr/payments.clj
View File

@@ -104,19 +104,18 @@
:size :small})]) :size :small})])
(com/field {:label "Payment Type"} (com/field {:label "Payment Type"}
(com/radio-card {:size :small (com/radio-card {:size :small
:name "payment-type" :name "payment-type"
:value (:payment-type (:query-params request)) :value (:payment-type (:query-params request))
:options [{:value "" :options [{:value ""
:content "All"} :content "All"}
{:value "cash" {:value "cash"
:content "Cash"} :content "Cash"}
{:value "check" {:value "check"
:content "Check"} :content "Check"}
{:value "debit" {:value "debit"
:content "Debit"}]})) :content "Debit"}]}))
(exact-match-id* request)]]) (exact-match-id* request)]])
(def default-read '[* (def default-read '[*
[:payment/date :xform clj-time.coerce/from-date] [:payment/date :xform clj-time.coerce/from-date]
{:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]} {:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}
@@ -212,7 +211,6 @@
'[(iol-ion.query/dollars= ?transaction-amount ?amount)]]} '[(iol-ion.query/dollars= ?transaction-amount ?amount)]]}
:args [(:amount query-params)]}) :args [(:amount query-params)]})
(:status route-params) (:status route-params)
(merge-query {:query {:in ['?status] (merge-query {:query {:in ['?status]
:where ['[?e :payment/status ?status]]} :where ['[?e :payment/status ?status]]}
@@ -244,12 +242,12 @@
(defn sum-visible-pending [ids] (defn sum-visible-pending [ids]
(->> (->>
(dc/q {:find ['?id '?o] (dc/q {:find ['?id '?o]
:in ['$ '[?id ...]] :in ['$ '[?id ...]]
:where ['[?id :payment/amount ?o] :where ['[?id :payment/amount ?o]
'[?id :payment/status :payment-status/pending]]} '[?id :payment/status :payment-status/pending]]}
(dc/db conn) (dc/db conn)
ids) ids)
(map last) (map last)
(reduce (reduce
+ +
@@ -257,15 +255,15 @@
(defn sum-client-pending [clients] (defn sum-client-pending [clients]
(->> (->>
(dc/q {:find '[?e ?a] (dc/q {:find '[?e ?a]
:in '[$ [?clients ?start ?end]] :in '[$ [?clients ?start ?end]]
:where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]] :where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]
[?e :payment/status :payment-status/pending] [?e :payment/status :payment-status/pending]
[?e :payment/amount ?a]]} [?e :payment/amount ?a]]}
(dc/db conn) (dc/db conn)
[clients [clients
nil nil
nil]) nil])
(map last) (map last)
(reduce (reduce
@@ -277,16 +275,14 @@
{ids-to-retrieve :ids matching-count :count {ids-to-retrieve :ids matching-count :count
all-ids :all-ids} (fetch-ids db request)] all-ids :all-ids} (fetch-ids db request)]
[(->> (hydrate-results ids-to-retrieve db request)) [(->> (hydrate-results ids-to-retrieve db request))
matching-count matching-count
(sum-visible-pending all-ids) (sum-visible-pending all-ids)
(sum-client-pending (extract-client-ids (:clients request) (sum-client-pending (extract-client-ids (:clients request)
(:client request) (:client request)
(:client-id (:query-params request)) (:client-id (:query-params request))
(when (:client-code (:query-params request)) (when (:client-code (:query-params request))
[:client/code (:client-code (:query-params request))]))) [:client/code (:client-code (:query-params request))])))]))
]))
(def query-schema (mc/schema (def query-schema (mc/schema
[:maybe [:map {:date-range [:date-range :start-date :end-date]} [:maybe [:map {:date-range [:date-range :start-date :end-date]}
@@ -327,7 +323,7 @@
(assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)]) (assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)])
:query-schema query-schema :query-schema query-schema
:action-buttons (fn [request] :action-buttons (fn [request]
(let [[_ _ visible-in-float total-in-float ] (:page-results request)] (let [[_ _ visible-in-float total-in-float] (:page-results request)]
[(com/pill {:color :primary} " Visible in float " [(com/pill {:color :primary} " Visible in float "
(format "$%,.2f" visible-in-float)) (format "$%,.2f" visible-in-float))
(com/pill {:color :secondary} " Total in float " (com/pill {:color :secondary} " Total in float "
@@ -409,7 +405,7 @@
:render (fn [{:payment/keys [date]}] :render (fn [{:payment/keys [date]}]
(some-> date (atime/unparse-local atime/normal-date)))} (some-> date (atime/unparse-local atime/normal-date)))}
{:key "amount" {:key "amount"
:sort-key "amount" :sort-key "amount"
:name "Amount" :name "Amount"
:render (fn [{:payment/keys [amount]}] :render (fn [{:payment/keys [amount]}]
(some->> amount (format "$%.2f")))} (some->> amount (format "$%.2f")))}
@@ -421,10 +417,10 @@
(map :invoice-payment/invoice) (map :invoice-payment/invoice)
(filter identity) (filter identity)
(map (fn [invoice] (map (fn [invoice]
{:link (hu/url (bidi/path-for ssr-routes/only-routes {:link (hu/url (bidi/path-for ssr-routes/only-routes
::invoice-route/all-page) ::invoice-route/all-page)
{:exact-match-id (:db/id invoice)}) {:exact-match-id (:db/id invoice)})
:content (str "Inv. " (:invoice/invoice-number invoice))}))) :content (str "Inv. " (:invoice/invoice-number invoice))})))
(some-> p :transaction/_payment ((fn [t] (some-> p :transaction/_payment ((fn [t]
[{:link (hu/url (bidi/path-for client-routes/routes [{:link (hu/url (bidi/path-for client-routes/routes
:transactions) :transactions)
@@ -434,8 +430,6 @@
(def row* (partial helper/row* grid-page)) (def row* (partial helper/row* grid-page))
(comment (comment
(mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
@@ -445,7 +439,6 @@
(mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
(mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))) (mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)))
(defn delete [{check :entity :as request identity :identity}] (defn delete [{check :entity :as request identity :identity}]
@@ -459,7 +452,7 @@
#(assert-can-see-client identity (:db/id (:payment/client check)))) #(assert-can-see-client identity (:db/id (:payment/client check))))
(notify-if-locked (:db/id (:payment/client check)) (notify-if-locked (:db/id (:payment/client check))
(:payment/date check)) (:payment/date check))
(let [ removing-payments (mapcat (fn [x] (let [removing-payments (mapcat (fn [x]
(let [invoice (:invoice-payment/invoice x) (let [invoice (:invoice-payment/invoice x)
new-balance (+ (:invoice/outstanding-balance invoice) new-balance (+ (:invoice/outstanding-balance invoice)
(:invoice-payment/amount x))] (:invoice-payment/amount x))]
@@ -578,7 +571,6 @@
(assoc-in [:query-params :start] 0) (assoc-in [:query-params :start] 0)
(assoc-in [:query-params :per-page] 250)))) (assoc-in [:query-params :per-page] 250))))
:else :else
selected) selected)
updated-count (void-payments-internal ids (:identity request))] updated-count (void-payments-internal ids (:identity request))]
@@ -591,7 +583,7 @@
(defn wrap-status-from-source [handler] (defn wrap-status-from-source [handler]
(fn [{:keys [matched-current-page-route] :as request}] (fn [{:keys [matched-current-page-route] :as request}]
(let [ request (cond-> request (let [request (cond-> request
(= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared) (= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared)
(= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending) (= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending)
(= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided) (= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided)
@@ -605,7 +597,7 @@
::route/pending-page (-> (helper/page-route grid-page) ::route/pending-page (-> (helper/page-route grid-page)
(wrap-implied-route-param :status :payment-status/pending)) (wrap-implied-route-param :status :payment-status/pending))
::route/voided-page (-> (helper/page-route grid-page) ::route/voided-page (-> (helper/page-route grid-page)
(wrap-implied-route-param :status :payment-status/voided)) (wrap-implied-route-param :status :payment-status/voided))
::route/all-page (-> (helper/page-route grid-page) ::route/all-page (-> (helper/page-route grid-page)
(wrap-implied-route-param :status nil)) (wrap-implied-route-param :status nil))
@@ -618,7 +610,6 @@
::route/bulk-delete (-> bulk-delete-dialog ::route/bulk-delete (-> bulk-delete-dialog
(wrap-admin)) (wrap-admin))
::route/table (helper/table-route grid-page)} ::route/table (helper/table-route grid-page)}
(fn [h] (fn [h]
(-> h (-> h

75
src/clj/auto_ap/storage/parquet.clj
View File

@@ -22,6 +22,10 @@
(let [conn (DriverManager/getConnection "jdbc:duckdb:") (let [conn (DriverManager/getConnection "jdbc:duckdb:")
stmt (.createStatement conn)] stmt (.createStatement conn)]
(.execute stmt "INSTALL httpfs; LOAD httpfs;") (.execute stmt "INSTALL httpfs; LOAD httpfs;")
(when-let [key (:aws-access-key-id env)]
(.execute stmt (str "SET s3_access_key_id='" key "'"))
(.execute stmt (str "SET s3_secret_access_key='" (:aws-secret-access-key env) "'"))
(.execute stmt (str "SET s3_region='" (or (:aws-region env) "us-east-1") "'")))
(.close stmt) (.close stmt)
(.addShutdownHook (Runtime/getRuntime) (.addShutdownHook (Runtime/getRuntime)
(Thread. #(fn []))) (Thread. #(fn [])))
@@ -107,9 +111,9 @@
(str entity-type ".jsonl"))] (str entity-type ".jsonl"))]
(io/make-parents wal-file) (io/make-parents wal-file)
(with-open [w (io/writer wal-file :append true)] (with-open [w (io/writer wal-file :append true)]
(.write w ^String (json/write-str {:seq-no seq-no (.write w ^String (json/write-str {:seq-no seq-no
:record record})) :record record}))
(.write w (int \newline)))) (.write w (int \newline))))
(catch Exception e (catch Exception e
(println "[parquet/wal]" (.getMessage e)))) (println "[parquet/wal]" (.getMessage e))))
entry)) entry))
@@ -138,8 +142,8 @@
(try (try
(with-open [w (io/writer jsonl-file :append true)] (with-open [w (io/writer jsonl-file :append true)]
(doseq [r records] (doseq [r records]
(.write w ^String (json/write-str (dissoc r :_seq-no))) (.write w ^String (json/write-str (dissoc r :_seq-no)))
(.write w (int \newline)))) (.write w (int \newline))))
(execute-to-parquet! (execute-to-parquet!
(format "SELECT * FROM read_json_auto('%s')" (format "SELECT * FROM read_json_auto('%s')"
(.getAbsolutePath jsonl-file)) (.getAbsolutePath jsonl-file))
@@ -150,8 +154,9 @@
(.delete ^java.io.File parquet-file) (.delete ^java.io.File parquet-file)
{:key s3-key :status :ok} {:key s3-key :status :ok}
(catch Exception e (catch Exception e
(throw (ex-info "Flush failed" {:entity-type entity-type} (throw (ex-info "Flush failed"
:error (.getMessage e))))))))) {:entity-type entity-type
:error (.getMessage e)}))))))))
(defn flush-by-date! [] (defn flush-by-date! []
"Flush all entity types for today." "Flush all entity types for today."
@@ -218,8 +223,9 @@
Returns map with :sql and :count-sql keys." Returns map with :sql and :count-sql keys."
(let [date-strs (date-seq start-date end-date) (let [date-strs (date-seq start-date end-date)
urls (vec urls (vec
(map #(format "'s3://%%s/sales-details/%%s/%%s.parquet'" (map (fn [d]
*bucket* entity-type %) (format "'s3://%s/sales-details/%s/%s.parquet'"
*bucket* entity-type d))
date-strs)) date-strs))
sql (str "SELECT * FROM read_parquet([" sql (str "SELECT * FROM read_parquet(["
(str/join ", " urls) (str/join ", " urls)
@@ -244,35 +250,34 @@
(get-sales-orders start-date end-date {})) (get-sales-orders start-date end-date {}))
([start-date end-date opts] ([start-date end-date opts]
(let [q (parquet-query "sales-order" (let [q (parquet-query "sales-order"
start-date end-date) start-date end-date)
base-sql (:sql q) base-sql (:sql q)
count-sql (:count-sql q) count-sql (:count-sql q)
sort (get opts :sort "date") sort (get opts :sort "date")
order (get opts :order "DESC") order (get opts :order "DESC")
limit (get opts :limit) limit (get opts :limit)
offset (get opts :offset) offset (get opts :offset)
where-str (build-where-clause where-str (build-where-clause
opts opts
[[:client "external_id.client"] [[:client "client-code"]
[:vendor "external_id.vendor"] [:vendor "vendor"]
[:location "location"]]) [:location "location"]])
full-sql (if where-str full-sql (if where-str
(str base-sql where-str) (str base-sql where-str)
base-sql) base-sql)
result (cond-> full-sql result (cond-> full-sql
sort (str " ORDER BY " sort sort (str " ORDER BY " sort
" " (name order)) " " (name order))
limit (str " LIMIT " limit) limit (str " LIMIT " limit)
offset (str " OFFSET " offset)) offset (str " OFFSET " offset))
full-count (if where-str full-count (if where-str
(str count-sql where-str) (str count-sql where-str)
count-sql)] count-sql)]
{:rows (query-rows result) {:rows (query-rows result)
:count (or :count (or
(int (int
(query-scalar (query-scalar
full-count)) 0)}))) full-count)) 0)})))
(defn query-deduped [entity-type start-date end-date] (defn query-deduped [entity-type start-date end-date]
"Query records deduplicated by external-id (latest _seq_no wins)." "Query records deduplicated by external-id (latest _seq_no wins)."

2
src/clj/auto_ap/storage/sales_summaries.clj
View File

@@ -55,7 +55,7 @@
tname tname
(+ (get b tname 0.0) total))))))) (+ (get b tname 0.0) total)))))))
{} {}
rows))) rows)))
(catch Exception e (catch Exception e
(println "[sales-summaries]" (.getMessage e)) (println "[sales-summaries]" (.getMessage e))
{})))) {}))))