fix(sales): fix parquet SQL generation and cleanup formatting
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full ORDER BY clause from build-sort-clause into get-sales-orders which prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC') - Fix WHERE clause column names in parquet build-where-clause: external_id.client -> client-code, external_id.vendor -> vendor - Fix parquet-query format string (%%s -> %s with proper format call) - Fix ex-info call signature in flush! (was passing :error as third arg instead of inside the data map) - Add S3 credentials to DuckDB connect! so httpfs can read from S3 - Fix parquet buffer indentation and alignment across square/core3, ezcater/core, ezcater_xls, payments, sales_summaries, migrations - Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions, upsert-recent find/where clauses mangled by paren-repair) - Uncomment accidentally commented code block in square/core3 - Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
This commit is contained in:
@@ -1,11 +1,9 @@
|
||||
(ns auto-ap.datomic.sales-orders
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.storage.parquet :as pq]
|
||||
[clj-time.coerce :as c]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as str]
|
||||
[com.brunobonacci.mulog :as mu]))
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as str]
|
||||
[com.brunobonacci.mulog :as mu]))
|
||||
|
||||
(defn <-row
|
||||
"Convert a flat parquet row into the shape consumers expect.
|
||||
@@ -29,12 +27,13 @@
|
||||
(update :sales-order/date #(some-> % str))))
|
||||
|
||||
(defn build-where-clause [args]
|
||||
(let [clauses [(when-let [c (:client-code args)]
|
||||
["external_id.client = '" c "'"])
|
||||
(when-let [v (:vendor args)]
|
||||
["external_id.vendor = '" (name v) "'"])
|
||||
(when-let [l (:location args)]
|
||||
["location = '" l "'"])]
|
||||
(let [clauses (keep identity
|
||||
[(when-let [c (:client-code args)]
|
||||
(str "external_id.client = '" c "'"))
|
||||
(when-let [v (:vendor args)]
|
||||
(str "external_id.vendor = '" (name v) "'"))
|
||||
(when-let [l (:location args)]
|
||||
(str "location = '" l "'"))])]
|
||||
(when (seq clauses)
|
||||
(str "WHERE " (str/join " AND " clauses)))))
|
||||
|
||||
@@ -47,18 +46,15 @@
|
||||
|
||||
(defn raw-graphql-ids [args]
|
||||
(let [start (some-> (:start (:date-range args)) .toString)
|
||||
end (some-> (:end (:date-range args)) .substring 0 10)
|
||||
where (build-where-clause args)
|
||||
sort (build-sort-clause args)
|
||||
end (some-> (:end (:date-range args)) (.substring 0 10))
|
||||
limit (or (:limit args) page-size)
|
||||
offset (or (:offset args) 0)
|
||||
where-str (when where (str " " where))]
|
||||
offset (or (:offset args) 0)]
|
||||
(when start
|
||||
(let [result (pq/get-sales-orders start end
|
||||
{:client (:client-code args)
|
||||
:vendor (:vendor args)
|
||||
:location (:location args)
|
||||
:sort sort
|
||||
:sort (or (:sort args) "date")
|
||||
:order "DESC"
|
||||
:limit limit
|
||||
:offset offset})]
|
||||
|
||||
@@ -21,42 +21,41 @@
|
||||
:body (json/write-str {"query" (v/graphql-query q)})
|
||||
:as :json})
|
||||
:body
|
||||
:data
|
||||
))
|
||||
:data))
|
||||
|
||||
(defn get-caterers [integration]
|
||||
(:caterers (query integration {:venia/queries [{:query/data
|
||||
[:caterers [:name :uuid [:address [:name :street]]]]}]} )))
|
||||
[:caterers [:name :uuid [:address [:name :street]]]]}]})))
|
||||
|
||||
(defn get-subscriptions [integration]
|
||||
(->> (query integration {:venia/queries [{:query/data
|
||||
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]] ]]}]} )
|
||||
[:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]]]]}]})
|
||||
:subscribers
|
||||
first
|
||||
:subscriptions))
|
||||
|
||||
(defn get-integrations []
|
||||
(map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
|
||||
:ezcater-integration/subscriber-uuid
|
||||
:db/id
|
||||
:ezcater-integration/integration-status [:db/id]])
|
||||
:in $
|
||||
:where [?i :ezcater-integration/api-key]]
|
||||
(dc/db conn))))
|
||||
:ezcater-integration/subscriber-uuid
|
||||
:db/id
|
||||
{:ezcater-integration/integration-status [:db/id]}])
|
||||
:in $
|
||||
:where [?i :ezcater-integration/api-key]]
|
||||
(dc/db conn))))
|
||||
|
||||
(defn mark-integration-status [integration integration-status]
|
||||
@(dc/transact conn
|
||||
[{:db/id (:db/id integration)
|
||||
:ezcater-integration/integration-status (assoc integration-status
|
||||
:db/id (or (-> integration :ezcater-integration/integration-status :db/id)
|
||||
(random-tempid)))}]))
|
||||
[{:db/id (:db/id integration)
|
||||
:ezcater-integration/integration-status (assoc integration-status
|
||||
:db/id (or (-> integration :ezcater-integration/integration-status :db/id)
|
||||
(random-tempid)))}]))
|
||||
|
||||
(defn upsert-caterers
|
||||
([integration]
|
||||
@(dc/transact
|
||||
conn
|
||||
(for [caterer (get-caterers integration)]
|
||||
{:db/id (:db/id integration)
|
||||
{:db/id (:db/id integration)
|
||||
:ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")")
|
||||
:ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer)))
|
||||
:ezcater-caterer/uuid (:uuid caterer)}]}))))
|
||||
@@ -65,14 +64,14 @@
|
||||
([integration]
|
||||
(let [extant (get-subscriptions integration)
|
||||
to-ensure (set (map first (dc/q '[:find ?cu
|
||||
:in $
|
||||
:where [_ :client/ezcater-locations ?el]
|
||||
[?el :ezcater-location/caterer ?c]
|
||||
[?c :ezcater-caterer/uuid ?cu]]
|
||||
(dc/db conn))))
|
||||
:in $
|
||||
:where [_ :client/ezcater-locations ?el]
|
||||
[?el :ezcater-location/caterer ?c]
|
||||
[?c :ezcater-caterer/uuid ?cu]]
|
||||
(dc/db conn))))
|
||||
to-create (set/difference
|
||||
to-ensure
|
||||
(set (map :parentId extant)))]
|
||||
to-ensure
|
||||
(set (map :parentId extant)))]
|
||||
(doseq [parentId to-create]
|
||||
(query integration
|
||||
{:venia/operation {:operation/type :mutation
|
||||
@@ -95,7 +94,6 @@
|
||||
:eventKey 'cancelled}}
|
||||
[[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]})))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn upsert-ezcater
|
||||
([] (upsert-ezcater (get-integrations)))
|
||||
@@ -116,12 +114,11 @@
|
||||
|
||||
(defn get-caterer [caterer-uuid]
|
||||
(dc/pull (dc/db conn)
|
||||
'[:ezcater-caterer/name
|
||||
{:ezcater-integration/_caterers [:ezcater-integration/api-key]}
|
||||
{:ezcater-location/_caterer [:ezcater-location/location
|
||||
{:client/_ezcater-locations [:client/code]}]}]
|
||||
[:ezcater-caterer/uuid caterer-uuid]))
|
||||
|
||||
'[:ezcater-caterer/name
|
||||
{:ezcater-integration/_caterers [:ezcater-integration/api-key]}
|
||||
{:ezcater-location/_caterer [:ezcater-location/location
|
||||
{:client/_ezcater-locations [:client/code]}]}]
|
||||
[:ezcater-caterer/uuid caterer-uuid]))
|
||||
|
||||
(defn round-carry-cents [f]
|
||||
(with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP))))
|
||||
@@ -137,69 +134,69 @@
|
||||
:else
|
||||
0.07M)]
|
||||
(round-carry-cents
|
||||
(* commision%
|
||||
0.01M
|
||||
(+
|
||||
(-> order :totals :subTotal :subunits )
|
||||
(reduce +
|
||||
0
|
||||
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
|
||||
(* commision%
|
||||
0.01M
|
||||
(+
|
||||
(-> order :totals :subTotal :subunits)
|
||||
(reduce +
|
||||
0
|
||||
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))))
|
||||
|
||||
(defn ccp-fee [order]
|
||||
(round-carry-cents
|
||||
(* 0.000299M
|
||||
(+
|
||||
(-> order :totals :subTotal :subunits )
|
||||
(-> order :totals :salesTax :subunits )
|
||||
(reduce +
|
||||
0
|
||||
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
|
||||
(* 0.000299M
|
||||
(+
|
||||
(-> order :totals :subTotal :subunits)
|
||||
(-> order :totals :salesTax :subunits)
|
||||
(reduce +
|
||||
0
|
||||
(map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))
|
||||
|
||||
(defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}]
|
||||
(let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
(-> order :totals :salesTax :subunits (* 0.01)))
|
||||
(-> order :catererCart :totals :catererTotalDue )
|
||||
(-> order :catererCart :totals :catererTotalDue)
|
||||
(commision order)
|
||||
(ccp-fee order)))
|
||||
service-charge (+ (commision order) (ccp-fee order))
|
||||
tax (-> order :totals :salesTax :subunits (* 0.01))
|
||||
tip (-> order :totals :tip :subunits (* 0.01))]
|
||||
#:sales-order
|
||||
{:date (atime/localize (coerce/to-date-time timestamp))
|
||||
:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid)
|
||||
:client [:client/code client-code]
|
||||
:location client-location
|
||||
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid ))
|
||||
:line-items [#:order-line-item
|
||||
{:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0)
|
||||
:item-name "EZCater Catering"
|
||||
:category "EZCater Catering"
|
||||
:discount adjustment
|
||||
:tax tax
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)}]
|
||||
:charges [#:charge
|
||||
{:type-name "CARD"
|
||||
:date (atime/localize (coerce/to-date-time timestamp))
|
||||
:client [:client/code client-code]
|
||||
:location client-location
|
||||
:external-id (str "ezcater/charge/" uuid)
|
||||
:processor :ccp-processor/ezcater
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)
|
||||
:tip tip}]
|
||||
{:date (atime/localize (coerce/to-date-time timestamp))
|
||||
:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid)
|
||||
:client [:client/code client-code]
|
||||
:location client-location
|
||||
:reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid))
|
||||
:line-items [#:order-line-item
|
||||
{:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0)
|
||||
:item-name "EZCater Catering"
|
||||
:category "EZCater Catering"
|
||||
:discount adjustment
|
||||
:tax tax
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)}]
|
||||
:charges [#:charge
|
||||
{:type-name "CARD"
|
||||
:date (atime/localize (coerce/to-date-time timestamp))
|
||||
:client [:client/code client-code]
|
||||
:location client-location
|
||||
:external-id (str "ezcater/charge/" uuid)
|
||||
:processor :ccp-processor/ezcater
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)
|
||||
:tip tip}]
|
||||
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)
|
||||
:discount adjustment
|
||||
:service-charge service-charge
|
||||
:tax tax
|
||||
:tip tip
|
||||
:returns 0.0
|
||||
:vendor :vendor/ccp-ezcater}}))
|
||||
:total (+ (-> order :totals :subTotal :subunits (* 0.01))
|
||||
tax
|
||||
tip)
|
||||
:discount adjustment
|
||||
:service-charge service-charge
|
||||
:tax tax
|
||||
:tip tip
|
||||
:returns 0.0
|
||||
:vendor :vendor/ccp-ezcater}))
|
||||
|
||||
(defn- flatten-order-to-parquet! [order]
|
||||
"Flatten a sales-order into entity-type tagged maps and buffer to parquet."
|
||||
@@ -222,81 +219,73 @@
|
||||
(when-let [charges (:sales-order/charges order)]
|
||||
(doseq [chg charges]
|
||||
(parquet/buffer! "charge"
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})))
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})))
|
||||
(when-let [items (:sales-order/line-items order)]
|
||||
(doseq [li items]
|
||||
(parquet/buffer! "line-item"
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
|
||||
(defn get-by-id [integration id]
|
||||
(query
|
||||
integration
|
||||
{:venia/queries [[:order {:id id}
|
||||
[:uuid
|
||||
:orderNumber
|
||||
:orderSourceType
|
||||
[:caterer
|
||||
[:name
|
||||
:uuid
|
||||
[:address [:street]]]]
|
||||
[:event
|
||||
[:timestamp
|
||||
:catererHandoffFoodTime
|
||||
:orderType]]
|
||||
[:catererCart [[:orderItems
|
||||
[:name
|
||||
:quantity
|
||||
:posItemId
|
||||
[:totalInSubunits
|
||||
[:currency
|
||||
:subunits]]]]
|
||||
[:totals
|
||||
[:catererTotalDue]]
|
||||
[:feesAndDiscounts
|
||||
{:type 'DELIVERY_FEE}
|
||||
[[:cost
|
||||
[:currency
|
||||
:subunits]]]]]]
|
||||
[:totals [[:customerTotalDue
|
||||
[
|
||||
:currency
|
||||
:subunits
|
||||
]]
|
||||
[:pointOfSaleIntegrationFee
|
||||
[
|
||||
:currency
|
||||
:subunits
|
||||
]]
|
||||
[:tip
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:salesTax
|
||||
[
|
||||
:currency
|
||||
:subunits
|
||||
]]
|
||||
[:salesTaxRemittance
|
||||
[:currency
|
||||
:subunits
|
||||
]]
|
||||
[:subTotal
|
||||
[:currency
|
||||
:subunits]]]]]]]}))
|
||||
(query
|
||||
integration
|
||||
{:venia/queries [[:order {:id id}
|
||||
[:uuid
|
||||
:orderNumber
|
||||
:orderSourceType
|
||||
[:caterer
|
||||
[:name
|
||||
:uuid
|
||||
[:address [:street]]]]
|
||||
[:event
|
||||
[:timestamp
|
||||
:catererHandoffFoodTime
|
||||
:orderType]]
|
||||
[:catererCart [[:orderItems
|
||||
[:name
|
||||
:quantity
|
||||
:posItemId
|
||||
[:totalInSubunits
|
||||
[:currency
|
||||
:subunits]]]]
|
||||
[:totals
|
||||
[:catererTotalDue]]
|
||||
[:feesAndDiscounts
|
||||
{:type 'DELIVERY_FEE}
|
||||
[[:cost
|
||||
[:currency
|
||||
:subunits]]]]]]
|
||||
[:totals [[:customerTotalDue
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:pointOfSaleIntegrationFee
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:tip
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:salesTax
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:salesTaxRemittance
|
||||
[:currency
|
||||
:subunits]]
|
||||
[:subTotal
|
||||
[:currency
|
||||
:subunits]]]]]]]}))
|
||||
|
||||
(defn lookup-order [json]
|
||||
(let [caterer (get-caterer (get json "parent_id"))
|
||||
@@ -304,18 +293,18 @@
|
||||
client (-> caterer :ezcater-location/_caterer first :client/_ezcater-locations :client/code)
|
||||
location (-> caterer :ezcater-location/_caterer first :ezcater-location/location)]
|
||||
(if (and client location)
|
||||
(doto
|
||||
(-> (get-by-id integration (get json "entity_id"))
|
||||
(:order)
|
||||
(assoc :client-code client
|
||||
:client-location location))
|
||||
(doto
|
||||
(-> (get-by-id integration (get json "entity_id"))
|
||||
(:order)
|
||||
(assoc :client-code client
|
||||
:client-location location))
|
||||
(#(alog/info ::order-details :detail %)))
|
||||
(alog/warn ::caterer-no-longer-has-location :json json))))
|
||||
|
||||
(defn import-order [json]
|
||||
(alog/info
|
||||
::try-import-order
|
||||
:json json)
|
||||
::try-import-order
|
||||
:json json)
|
||||
(when-let [order (some-> json
|
||||
(lookup-order)
|
||||
(order->sales-order)
|
||||
@@ -324,7 +313,7 @@
|
||||
(try
|
||||
(flatten-order-to-parquet! order)
|
||||
(alog/info ::order-buffered
|
||||
:external-id (:sales-order/external-id order))
|
||||
:external-id (:sales-order/external-id order))
|
||||
(catch Exception e
|
||||
(alog/error ::buffer-failed
|
||||
:exception e
|
||||
@@ -359,30 +348,30 @@
|
||||
"key" "accepted",
|
||||
"occurred_at" "2022-07-21T19:21:07.549Z"}
|
||||
ezcater-order (lookup-order lookup-map)
|
||||
extant-order (dc/pull (dc/db conn) '[:sales-order/total
|
||||
:sales-order/tax
|
||||
:sales-order/tip
|
||||
:sales-order/discount
|
||||
:sales-order/external-id
|
||||
{:sales-order/charges [:charge/tax
|
||||
:charge/tip
|
||||
:charge/total
|
||||
:charge/external-id]
|
||||
:sales-order/line-items [:order-line-item/external-id
|
||||
:order-line-item/total
|
||||
:order-line-item/tax
|
||||
:order-line-item/discount]}]
|
||||
[:sales-order/external-id order])
|
||||
extant-order (dc/pull (dc/db conn '[:sales-order/total]
|
||||
:sales-order/tax
|
||||
:sales-order/tip
|
||||
:sales-order/discount
|
||||
:sales-order/external-id
|
||||
{:sales-order/charges [:charge/tax
|
||||
:charge/tip
|
||||
:charge/total
|
||||
:charge/external-id]
|
||||
:sales-order/line-items [:order-line-item/external-id
|
||||
:order-line-item/total
|
||||
:order-line-item/tax
|
||||
:order-line-item/discount]})
|
||||
[:sales-order/external-id order])
|
||||
|
||||
updated-order (-> (order->sales-order ezcater-order)
|
||||
(select-keys
|
||||
#{:sales-order/total
|
||||
:sales-order/tax
|
||||
:sales-order/tip
|
||||
:sales-order/discount
|
||||
:sales-order/charges
|
||||
:sales-order/external-id
|
||||
:sales-order/line-items})
|
||||
#{:sales-order/total
|
||||
:sales-order/tax
|
||||
:sales-order/tip
|
||||
:sales-order/discount
|
||||
:sales-order/charges
|
||||
:sales-order/external-id
|
||||
:sales-order/line-items})
|
||||
(update :sales-order/line-items
|
||||
(fn [c]
|
||||
(map #(select-keys % #{:order-line-item/external-id
|
||||
|
||||
@@ -40,17 +40,14 @@
|
||||
(dc/db conn)
|
||||
number)))
|
||||
|
||||
|
||||
(defn delete-all []
|
||||
@(dc/transact-async conn
|
||||
(->>
|
||||
(dc/q '[:find ?ss
|
||||
:where [?ss :sales-summary/date]]
|
||||
(dc/db conn))
|
||||
(map (fn [[ ss]]
|
||||
[:db/retractEntity ss])))))
|
||||
|
||||
|
||||
(->>
|
||||
(dc/q '[:find ?ss
|
||||
:where [?ss :sales-summary/date]]
|
||||
(dc/db conn))
|
||||
(map (fn [[ss]]
|
||||
[:db/retractEntity ss])))))
|
||||
|
||||
(defn dirty-sales-summaries [c]
|
||||
(let [client-id (dc/entid (dc/db conn) c)]
|
||||
@@ -138,9 +135,9 @@
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 3
|
||||
:sales-summary-item/category (cond
|
||||
(= type-name "CARD") "Card Refunds"
|
||||
(= type-name "CASH") "Cash Refunds"
|
||||
:else "Food App Refunds")
|
||||
(= type-name "CARD") "Card Refunds"
|
||||
(= type-name "CASH") "Cash Refunds"
|
||||
:else "Food App Refunds")
|
||||
:ledger-mapped/amount total
|
||||
:ledger-mapped/ledger-side :ledger-side/credit})
|
||||
refunds))))
|
||||
@@ -188,8 +185,6 @@
|
||||
:ledger-mapped/ledger-side :ledger-side/credit
|
||||
:ledger-mapped/amount (- (+ total discount) tax)})))
|
||||
|
||||
|
||||
|
||||
(defn get-fees [c date]
|
||||
(when-let [fee (get-fee c date)]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
@@ -299,18 +294,17 @@
|
||||
(filter identity)
|
||||
(map (fn [z]
|
||||
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
|
||||
:sales-summary-item/manual? false))
|
||||
)) }]
|
||||
:sales-summary-item/manual? false))))}]
|
||||
(if (seq (:sales-summary/items result))
|
||||
(do
|
||||
(alog/info ::upserting-summaries
|
||||
:category-count (count (:sales-summary/items result)))
|
||||
@(dc/transact conn [[:upsert-entity result]]))
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
|
||||
(comment
|
||||
;; TODO: Move to test file or proper location
|
||||
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ])
|
||||
(let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL"])
|
||||
date #inst "2024-04-14T00:00:00-07:00"]
|
||||
(get-payment-items c date)))
|
||||
|
||||
@@ -322,16 +316,13 @@
|
||||
(map (fn [[sos]]
|
||||
[:db/retractEntity sos])))))
|
||||
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(auto-ap.datomic/transact-schema conn)
|
||||
|
||||
@(dc/transact conn [{:db/ident :sales-summary/total-unknown-processor-payments
|
||||
:db/noHistory true,
|
||||
:db/valueType :db.type/double
|
||||
:db/cardinality :db.cardinality/one}])
|
||||
:db/noHistory true,
|
||||
:db/valueType :db.type/double
|
||||
:db/cardinality :db.cardinality/one}])
|
||||
|
||||
(apply mark-dirty [:client/code "NGCL"] (last-n-days 30))
|
||||
|
||||
@@ -350,7 +341,7 @@
|
||||
[?sos :sales-summary/date ?d]
|
||||
[(= ?d #inst "2024-04-10T00:00:00-07:00")]]
|
||||
(dc/db conn))
|
||||
|
||||
|
||||
(dc/q '[:find ?n ?p2 (sum ?total)
|
||||
:with ?c
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
@@ -363,23 +354,18 @@
|
||||
(dc/db conn)
|
||||
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGHW"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-11T00:00:00-07:00"])
|
||||
|
||||
(dc/q '[:find ?n
|
||||
(dc/q '[:find ?n
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-order/line-items ?li]
|
||||
[?li :order-line-item/item-name ?n] ]
|
||||
[?li :order-line-item/item-name ?n]]
|
||||
(dc/db conn)
|
||||
[[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"])
|
||||
|
||||
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
|
||||
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
|
||||
|
||||
(auto-ap.datomic/transact-schema conn)
|
||||
|
||||
)
|
||||
|
||||
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
|
||||
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
|
||||
|
||||
(auto-ap.datomic/transact-schema conn))
|
||||
|
||||
(defn -main [& _]
|
||||
(execute "sales-summaries" sales-summaries-v2))
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
"Return all entity IDs that have :sales-order/external-id."
|
||||
[db]
|
||||
(->> (d-api/q '[:find ?e
|
||||
:where [?e :sales-order/external-id]]
|
||||
db)
|
||||
:where [?e :sales-order/external-id]]
|
||||
db)
|
||||
(map first)))
|
||||
|
||||
(defn- collect-child-ids
|
||||
@@ -28,19 +28,19 @@
|
||||
[db order-ids]
|
||||
(let [order-set (set order-ids)
|
||||
charges (->> (d-api/q '[:find ?c
|
||||
:in $ [?o ...]
|
||||
:where [$ ?o :sales-order/charges ?c]]
|
||||
db order-set)
|
||||
:in $ [?o ...]
|
||||
:where [$ ?o :sales-order/charges ?c]]
|
||||
db order-set)
|
||||
(map second))
|
||||
refunds (->> (d-api/q '[:find ?r
|
||||
:in $ [?o ...]
|
||||
:where [$ ?o :sales-order/refunds ?r]]
|
||||
db order-set)
|
||||
:in $ [?o ...]
|
||||
:where [$ ?o :sales-order/refunds ?r]]
|
||||
db order-set)
|
||||
(map second))
|
||||
line-items (->> (d-api/q '[:find ?li
|
||||
:in $ [?c ...]
|
||||
:where [$ ?c :charge/line-items ?li]]
|
||||
db charges)
|
||||
:in $ [?c ...]
|
||||
:where [$ ?c :charge/line-items ?li]]
|
||||
db charges)
|
||||
(map second))]
|
||||
{:orders order-ids
|
||||
:charges (vec charges)
|
||||
@@ -149,7 +149,7 @@
|
||||
all-ids (query-sales-order-ids db)
|
||||
group (group-orders-by-month db all-ids)
|
||||
target-keys (get group [year month] [])]
|
||||
(if (zero? (count target-keys))
|
||||
(if (zero? (count target-keys))
|
||||
(do (println " no orders found for" year "-" month)
|
||||
:skipped)
|
||||
(do
|
||||
@@ -202,18 +202,18 @@
|
||||
missing (:missing result)]
|
||||
(cond
|
||||
(:ok result)
|
||||
(do (println "verified" y "-" m "S3 OK, deleting...")
|
||||
(delete-by-month conn$ nil y m))
|
||||
(do (println "verified" y "-" m "S3 OK, deleting...")
|
||||
(delete-by-month conn$ nil y m))
|
||||
|
||||
(> (count missing) 0)
|
||||
(do (println "ERROR" y "-" m "missing in S3:"
|
||||
(str/join ", " missing))
|
||||
(throw
|
||||
(ex-info
|
||||
"Missing S3 data — aborting!"
|
||||
{:year y :month m
|
||||
:missing missing})))
|
||||
(do (println "ERROR" y "-" m "missing in S3:"
|
||||
(str/join ", " missing))
|
||||
(throw
|
||||
(ex-info
|
||||
"Missing S3 data — aborting!"
|
||||
{:year y :month m
|
||||
:missing missing})))
|
||||
|
||||
:else
|
||||
(println "SKIPPING" y "-" m "no parquet files")))))
|
||||
(println "SKIPPING" y "-" m "no parquet files")))))
|
||||
(println "safe-cleanup-all complete")))
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
:charge/date
|
||||
:charge/processor
|
||||
:charge/returns
|
||||
{:charge/client [:client/code]}]}
|
||||
{:charge/client [:client/code]}]}
|
||||
{:sales-order/line-items
|
||||
[:order-line-item/item-name
|
||||
:order-line-item/category
|
||||
@@ -122,13 +122,12 @@
|
||||
(->> (dc/q '[:find ?e
|
||||
:in $ ?start-ms ?end-ms
|
||||
:where [_ :sales-order/date ?d]
|
||||
[(>= ?d ?start-ms)]
|
||||
[(<= ?d ?end-ms)]]
|
||||
[(>= ?d ?start-ms)]
|
||||
[(<= ?d ?end-ms)]]
|
||||
db start end)
|
||||
(map first)
|
||||
vec)))
|
||||
|
||||
|
||||
(defn- date-seq [start end]
|
||||
"Seq of YYYY-MM-DD strings between start and end inclusive."
|
||||
(let [sd (java.time.LocalDate/parse start)
|
||||
@@ -163,7 +162,7 @@
|
||||
"line-item" "sales-refund"]]
|
||||
(p/flush-to-parquet! etype))
|
||||
(println "[migration]" day "complete"))
|
||||
{:status :completed :total-days (count date-range)})))
|
||||
{:status :completed :total-days (count date-range)})))
|
||||
|
||||
(defn- write-dead-letter
|
||||
([flat]
|
||||
@@ -192,11 +191,11 @@
|
||||
(defn- get-date-range []
|
||||
"Get the earliest and latest business dates from Datomic."
|
||||
(let [dates (->> (dc/q '[:find ?d
|
||||
:where [_ :sales-order/date ?d]]
|
||||
(dc/db conn))
|
||||
(map first)
|
||||
distinct
|
||||
sort)]
|
||||
:where [_ :sales-order/date ?d]]
|
||||
(dc/db conn))
|
||||
(map first)
|
||||
distinct
|
||||
sort)]
|
||||
[(when (seq dates) (.toString (first dates)))
|
||||
(when (seq dates) (.toString (last dates)))]))
|
||||
|
||||
@@ -220,7 +219,7 @@
|
||||
(let [flat (volatile! [])]
|
||||
(flatten-order-to-pieces! o flat)
|
||||
(doseq [r @flat]
|
||||
(p/buffer! "dead" r))))
|
||||
(p/buffer! "dead" r))))
|
||||
(write-day-by-day start-date end-date {:batch-size 100})
|
||||
(flush-all-types)
|
||||
(println "[migration] done")
|
||||
|
||||
@@ -55,54 +55,54 @@
|
||||
event-date (some-> (excel/xls-date->date event-date)
|
||||
coerce/to-date-time
|
||||
atime/as-local-time
|
||||
coerce/to-date )]
|
||||
(cond (and event-date client-id location )
|
||||
coerce/to-date)]
|
||||
(cond (and event-date client-id location)
|
||||
[:order #:sales-order
|
||||
{:date event-date
|
||||
:external-id (str "ezcater/order/" client-id "-" location "-" order-number)
|
||||
:client client-id
|
||||
:location location
|
||||
:reference-link (str order-number)
|
||||
:line-items [#:order-line-item
|
||||
{:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0)
|
||||
:item-name "EZCater Catering"
|
||||
:category "EZCater Catering"
|
||||
:discount (fmt-amount (or adjustments 0.0))
|
||||
:tax (fmt-amount tax)
|
||||
:total (fmt-amount (+ food-total
|
||||
tax))}]
|
||||
{:date event-date
|
||||
:external-id (str "ezcater/order/" client-id "-" location "-" order-number)
|
||||
:client client-id
|
||||
:location location
|
||||
:reference-link (str order-number)
|
||||
:line-items [#:order-line-item
|
||||
{:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0)
|
||||
:item-name "EZCater Catering"
|
||||
:category "EZCater Catering"
|
||||
:discount (fmt-amount (or adjustments 0.0))
|
||||
:tax (fmt-amount tax)
|
||||
:total (fmt-amount (+ food-total
|
||||
tax))}]
|
||||
|
||||
:charges [#:charge
|
||||
{:type-name "CARD"
|
||||
:date event-date
|
||||
:client client-id
|
||||
:location location
|
||||
:external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
|
||||
:processor :ccp-processor/ezcater
|
||||
:total (fmt-amount (+ food-total
|
||||
tax
|
||||
tip))
|
||||
:tip (fmt-amount tip)}]
|
||||
:total (fmt-amount (+ food-total
|
||||
tax
|
||||
(or adjustments 0.0)))
|
||||
:discount (fmt-amount (or adjustments 0.0))
|
||||
:service-charge (fmt-amount (+ fee commission))
|
||||
:tax (fmt-amount tax)
|
||||
:tip (fmt-amount tip)
|
||||
:returns 0.0
|
||||
:vendor :vendor/ccp-ezcater}]
|
||||
:charges [#:charge
|
||||
{:type-name "CARD"
|
||||
:date event-date
|
||||
:client client-id
|
||||
:location location
|
||||
:external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
|
||||
:processor :ccp-processor/ezcater
|
||||
:total (fmt-amount (+ food-total
|
||||
tax
|
||||
tip))
|
||||
:tip (fmt-amount tip)}]
|
||||
:total (fmt-amount (+ food-total
|
||||
tax
|
||||
(or adjustments 0.0)))
|
||||
:discount (fmt-amount (or adjustments 0.0))
|
||||
:service-charge (fmt-amount (+ fee commission))
|
||||
:tax (fmt-amount tax)
|
||||
:tip (fmt-amount tip)
|
||||
:returns 0.0
|
||||
:vendor :vendor/ccp-ezcater}]
|
||||
|
||||
caterer-name
|
||||
(do
|
||||
(alog/warn ::missing-client
|
||||
:order order-number
|
||||
:store-name store-name
|
||||
:caterer-name caterer-name)
|
||||
[:missing caterer-name])
|
||||
caterer-name
|
||||
(do
|
||||
(alog/warn ::missing-client
|
||||
:order order-number
|
||||
:store-name store-name
|
||||
:caterer-name caterer-name)
|
||||
[:missing caterer-name])
|
||||
|
||||
:else
|
||||
nil)))
|
||||
:else
|
||||
nil)))
|
||||
|
||||
(defn- flatten-order-to-parquet! [order]
|
||||
"Flatten a sales-order into entity-type tagged maps and buffer to parquet."
|
||||
@@ -125,25 +125,25 @@
|
||||
(when-let [charges (:sales-order/charges order)]
|
||||
(doseq [chg charges]
|
||||
(parquet/buffer! "charge"
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})))
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})))
|
||||
(when-let [items (:sales-order/line-items order)]
|
||||
(doseq [li items]
|
||||
(parquet/buffer! "line-item"
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
|
||||
(defn stream->sales-orders [s]
|
||||
(let [clients (map first (dc/q '[:find (pull ?c [:client/code
|
||||
@@ -158,7 +158,7 @@
|
||||
object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))]
|
||||
(mu/log ::writing-temp-xls
|
||||
:location object)
|
||||
(s3/put-object {:bucket-name (:data-bucket env)
|
||||
(s3/put-object {:bucket-name (:data-bucket env)
|
||||
:key object
|
||||
:input-stream s})
|
||||
(into []
|
||||
@@ -200,13 +200,13 @@
|
||||
});")]])])
|
||||
|
||||
(defn upload-xls [{:keys [identity] :as request}]
|
||||
|
||||
|
||||
(let [file (or (get (:params request) :file)
|
||||
(get (:params request) "file"))]
|
||||
(mu/log ::uploading-file
|
||||
:file file)
|
||||
(with-open [s (io/input-stream (:tempfile file))]
|
||||
(try
|
||||
(try
|
||||
(let [parse-results (stream->sales-orders s)
|
||||
new-orders (->> parse-results
|
||||
(filter (comp #{:order} first))
|
||||
@@ -235,7 +235,7 @@
|
||||
[:li ml])]])]))
|
||||
(catch Exception e
|
||||
(alog/error ::import-error
|
||||
:error e)
|
||||
:error e)
|
||||
(html-response [:div (.getMessage e)]))))))
|
||||
|
||||
(defn page [{:keys [matched-route request-method] :as request}]
|
||||
|
||||
@@ -28,11 +28,9 @@
|
||||
"Authorization" (str "Bearer " (:client/square-auth-token client))
|
||||
"Content-Type" "application/json"}))
|
||||
|
||||
|
||||
(defn ->square-date [d]
|
||||
(f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d))
|
||||
|
||||
|
||||
(def manifold-api-stream
|
||||
(let [stream (s/stream 100)]
|
||||
(->> stream
|
||||
@@ -43,10 +41,10 @@
|
||||
(de/loop [attempt 0]
|
||||
(-> (de/chain (de/future-with (ex/execute-pool)
|
||||
#_(log/info ::request-started
|
||||
:url (:url request)
|
||||
:attempt attempt
|
||||
:source "Square 3"
|
||||
:background-job "Square 3")
|
||||
:url (:url request)
|
||||
:attempt attempt
|
||||
:source "Square 3"
|
||||
:background-job "Square 3")
|
||||
(try
|
||||
(client/request (assoc request
|
||||
:socket-timeout 10000
|
||||
@@ -105,7 +103,6 @@
|
||||
:exception error))
|
||||
[]))))
|
||||
|
||||
|
||||
(def item-cache (atom {}))
|
||||
|
||||
(defn fetch-catalog [client i v]
|
||||
@@ -125,13 +122,11 @@
|
||||
#(do (swap! item-cache assoc i %)
|
||||
%))))
|
||||
|
||||
|
||||
(defn fetch-catalog-cache [client i version]
|
||||
(if (get @item-cache i)
|
||||
(de/success-deferred (get @item-cache i))
|
||||
(fetch-catalog client i version)))
|
||||
|
||||
|
||||
(defn item->category-name-impl [client item version]
|
||||
(capture-context->lc
|
||||
(cond (:item_id (:item_variation_data item))
|
||||
@@ -162,7 +157,6 @@
|
||||
:item item)
|
||||
"Uncategorized"))))
|
||||
|
||||
|
||||
(defn item-id->category-name [client i version]
|
||||
(capture-context->lc
|
||||
(-> [client i]
|
||||
@@ -227,7 +221,6 @@
|
||||
(concat (:orders result) continued-results))))
|
||||
(:orders result)))))))
|
||||
|
||||
|
||||
(defn search
|
||||
([client location start end]
|
||||
(capture-context->lc
|
||||
@@ -251,11 +244,9 @@
|
||||
(concat (:orders result) continued-results))))
|
||||
(:orders result))))))))
|
||||
|
||||
|
||||
(defn amount->money [amt]
|
||||
(* 0.01 (or (:amount amt) 0.0)))
|
||||
|
||||
|
||||
;; to get totals:
|
||||
(comment
|
||||
(reduce
|
||||
@@ -281,7 +272,7 @@
|
||||
:reference-link (str (url/url "https://squareup.com/receipt/preview" (:id t)))
|
||||
:external-id (when (:id t)
|
||||
(str "square/charge/" (:id t)))
|
||||
:processor (cond
|
||||
:processor (cond
|
||||
(#{"OTHER" "THIRD_PARTY_CARD"} (:type t))
|
||||
(condp = (some-> (:note t) str/lower-case)
|
||||
"doordash" :ccp-processor/doordash
|
||||
@@ -354,7 +345,7 @@
|
||||
#:sales-order
|
||||
{:date (if (= "Invoices" (:name (:source order)))
|
||||
(when (:closed_at order)
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
:client (:db/id client)
|
||||
:location (:square-location/client-location location)
|
||||
@@ -416,7 +407,6 @@
|
||||
:client client
|
||||
:location location)))))))
|
||||
|
||||
|
||||
(defn get-payment [client p]
|
||||
(de/chain (manifold-api-call
|
||||
{:url (str "https://connect.squareup.com/v2/payments/" p)
|
||||
@@ -425,7 +415,6 @@
|
||||
:body
|
||||
:payment))
|
||||
|
||||
|
||||
(defn continue-payout-entry-list [c l poi cursor]
|
||||
(capture-context->lc lc
|
||||
(de/chain
|
||||
@@ -610,8 +599,8 @@
|
||||
so-date (some-> (:sales-order/date order) .toString)
|
||||
client (:sales-order/client order)
|
||||
client-code (when client (if (map? client)
|
||||
(:client/code client)
|
||||
client))]
|
||||
(:client/code client)
|
||||
client))]
|
||||
(parquet/buffer! "sales-order"
|
||||
{:entity-type "sales-order"
|
||||
:external-id so-ext-id
|
||||
@@ -627,32 +616,32 @@
|
||||
(when-let [charges (:sales-order/charges order)]
|
||||
(doseq [chg charges]
|
||||
(parquet/buffer! "charge"
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})
|
||||
(when-let [returns (:charge/returns chg)]
|
||||
(doseq [rt returns]
|
||||
(parquet/buffer! "sales-refund"
|
||||
{:entity-type "sales-refund"
|
||||
:type-name (:type-name rt)
|
||||
:total (:total rt)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
{:entity-type "sales-refund"
|
||||
:type-name (:type-name rt)
|
||||
:total (:total rt)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
(when-let [items (:sales-order/line-items order)]
|
||||
(doseq [li items]
|
||||
(parquet/buffer! "line-item"
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
|
||||
(defn upsert
|
||||
([client]
|
||||
@@ -674,8 +663,7 @@
|
||||
(catch Exception e
|
||||
(log/error ::buffer-failed
|
||||
:exception e
|
||||
:order (:sales-order/external-id order)))))))))))
|
||||
|
||||
:order (:sales-order/external-id order))))))))))))
|
||||
|
||||
(defn upsert-payouts
|
||||
([client]
|
||||
@@ -725,7 +713,6 @@
|
||||
|
||||
(log/info ::done-loading-refunds)))))))
|
||||
|
||||
|
||||
(defn get-cash-shift [client id]
|
||||
(de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id))
|
||||
:method :get
|
||||
@@ -884,8 +871,6 @@
|
||||
d1
|
||||
d2))
|
||||
|
||||
|
||||
|
||||
(defn remove-voided-orders
|
||||
([client]
|
||||
(apply de/zip
|
||||
@@ -912,7 +897,7 @@
|
||||
(:sales-order/external-id o))))))
|
||||
(s/map (fn [[o]]
|
||||
[[:db/retractEntity [:sales-order/external-id (:sales-order/external-id o)]]]))
|
||||
|
||||
|
||||
(s/reduce into [])))
|
||||
|
||||
(fn [results]
|
||||
@@ -920,32 +905,28 @@
|
||||
(doseq [x (partition-all 100 results)]
|
||||
(log/info ::removing-orders
|
||||
:count (count x))
|
||||
@(dc/transact-async conn x)))))
|
||||
@(dc/transact-async conn x)
|
||||
(de/catch (fn [e]
|
||||
(log/warn ::couldnt-remove :error e)
|
||||
nil) ))))))
|
||||
nil)))))))))))
|
||||
|
||||
#_(comment
|
||||
(require 'auto-ap.time-reader)
|
||||
#_(comment
|
||||
(require 'auto-ap.time-reader)
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "DBFS") ]
|
||||
(log/peek :x [ c l])
|
||||
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29")
|
||||
@(let [[c [l]] (get-square-client-and-location "DBFS")]
|
||||
(log/peek :x [c l])
|
||||
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29"))
|
||||
|
||||
)
|
||||
@(let [[c [l]] (get-square-client-and-location "NGAK")]
|
||||
(log/peek :x [c l])
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "NGAK") ]
|
||||
(log/peek :x [ c l])
|
||||
|
||||
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
|
||||
(doseq [c (get-square-clients)]
|
||||
(try
|
||||
@(remove-voided-orders c)
|
||||
(catch Exception e
|
||||
nil)))
|
||||
|
||||
|
||||
)
|
||||
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
|
||||
(doseq [c (get-square-clients)]
|
||||
(try
|
||||
@(remove-voided-orders c)
|
||||
(catch Exception e
|
||||
nil)))
|
||||
)
|
||||
|
||||
(defn upsert-all [& clients]
|
||||
(capture-context->lc
|
||||
@@ -1014,8 +995,6 @@
|
||||
[:clients clients]
|
||||
@(apply upsert-all clients)))
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(defn refunds-raw-cont
|
||||
([client l cursor so-far]
|
||||
@@ -1045,9 +1024,8 @@
|
||||
(->>
|
||||
@(let [[c [l]] (get-square-client-and-location "NGGG")]
|
||||
|
||||
|
||||
(search c l (time/now) (time/plus (time/now) (time/days -1))))
|
||||
|
||||
|
||||
(filter (fn [r]
|
||||
(str/starts-with? (:created_at r) "2024-03-14"))))
|
||||
|
||||
@@ -1055,7 +1033,6 @@
|
||||
(->>
|
||||
@(let [[c [l]] (get-square-client-and-location "NGGG")]
|
||||
|
||||
|
||||
(refunds-raw-cont c l nil []))
|
||||
(filter (fn [r]
|
||||
(str/starts-with? (:created_at r) "2024-03-14")))))
|
||||
@@ -1089,13 +1066,8 @@
|
||||
[]))]
|
||||
[(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)])
|
||||
:separator \tab)
|
||||
|
||||
|
||||
|
||||
|
||||
;; =>
|
||||
|
||||
|
||||
;; =>
|
||||
|
||||
(require 'auto-ap.time-reader)
|
||||
|
||||
@@ -1104,27 +1076,16 @@
|
||||
(clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")]
|
||||
l
|
||||
|
||||
|
||||
(def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00"
|
||||
#clj-time/date-time "2025-02-28T00:00:00-08:00"))
|
||||
(take 10 (map #(first (deref (order->sales-order c l %))) z)))
|
||||
(take 10 (map #(first (deref (order->sales-order c l %))) z))))
|
||||
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
(->> z
|
||||
(->> z
|
||||
(filter (fn [o]
|
||||
(seq (filter (comp #{"OTHER"} :type) (:tenders o)))))
|
||||
(filter #(not (:name (:source %))))
|
||||
(count)
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
(count))
|
||||
|
||||
(doseq [[code] (seq (dc/q '[:find ?code
|
||||
:in $
|
||||
:where [?o :sales-order/date ?d]
|
||||
@@ -1133,32 +1094,22 @@
|
||||
[?o :sales-order/client ?c]
|
||||
[?c :client/code ?code]]
|
||||
(dc/db conn)))
|
||||
:let [[c [l]] (get-square-client-and-location code)
|
||||
]
|
||||
:let [[c [l]] (get-square-client-and-location code)]
|
||||
order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now))
|
||||
:when (= "Invoices" (:name (:source order) ))
|
||||
:when (= "Invoices" (:name (:source order)))
|
||||
:let [[sales-order] @(order->sales-order c l order)]]
|
||||
|
||||
|
||||
(when (should-import-order? order)
|
||||
(println "DATE IS" (:sales-order/date sales-order))
|
||||
(when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00"))
|
||||
(println "WOULD UPDATE" sales-order)
|
||||
@(dc/transact auto-ap.datomic/conn [sales-order])
|
||||
)
|
||||
#_@(dc/transact )
|
||||
(println "DONE"))
|
||||
|
||||
|
||||
)
|
||||
@(dc/transact auto-ap.datomic/conn [sales-order]))
|
||||
#_@(dc/transact)
|
||||
(println "DONE")))
|
||||
|
||||
#_(filter (comp #{"OTHER"} :type) (mapcat :tenders z))
|
||||
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "NGRY")]
|
||||
#_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01"))
|
||||
|
||||
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" )))
|
||||
|
||||
)
|
||||
|
||||
)
|
||||
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY")))))
|
||||
|
||||
@@ -104,19 +104,18 @@
|
||||
:size :small})])
|
||||
(com/field {:label "Payment Type"}
|
||||
(com/radio-card {:size :small
|
||||
:name "payment-type"
|
||||
:value (:payment-type (:query-params request))
|
||||
:options [{:value ""
|
||||
:content "All"}
|
||||
{:value "cash"
|
||||
:content "Cash"}
|
||||
{:value "check"
|
||||
:content "Check"}
|
||||
{:value "debit"
|
||||
:content "Debit"}]}))
|
||||
:name "payment-type"
|
||||
:value (:payment-type (:query-params request))
|
||||
:options [{:value ""
|
||||
:content "All"}
|
||||
{:value "cash"
|
||||
:content "Cash"}
|
||||
{:value "check"
|
||||
:content "Check"}
|
||||
{:value "debit"
|
||||
:content "Debit"}]}))
|
||||
(exact-match-id* request)]])
|
||||
|
||||
|
||||
(def default-read '[*
|
||||
[:payment/date :xform clj-time.coerce/from-date]
|
||||
{:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}
|
||||
@@ -212,7 +211,6 @@
|
||||
'[(iol-ion.query/dollars= ?transaction-amount ?amount)]]}
|
||||
:args [(:amount query-params)]})
|
||||
|
||||
|
||||
(:status route-params)
|
||||
(merge-query {:query {:in ['?status]
|
||||
:where ['[?e :payment/status ?status]]}
|
||||
@@ -243,30 +241,30 @@
|
||||
refunds))
|
||||
|
||||
(defn sum-visible-pending [ids]
|
||||
(->>
|
||||
(dc/q {:find ['?id '?o]
|
||||
:in ['$ '[?id ...]]
|
||||
:where ['[?id :payment/amount ?o]
|
||||
'[?id :payment/status :payment-status/pending]]}
|
||||
(dc/db conn)
|
||||
ids)
|
||||
(->>
|
||||
(dc/q {:find ['?id '?o]
|
||||
:in ['$ '[?id ...]]
|
||||
:where ['[?id :payment/amount ?o]
|
||||
'[?id :payment/status :payment-status/pending]]}
|
||||
(dc/db conn)
|
||||
ids)
|
||||
(map last)
|
||||
(reduce
|
||||
+
|
||||
0.0)))
|
||||
|
||||
(defn sum-client-pending [clients]
|
||||
(->>
|
||||
(dc/q {:find '[?e ?a]
|
||||
:in '[$ [?clients ?start ?end]]
|
||||
:where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]
|
||||
[?e :payment/status :payment-status/pending]
|
||||
[?e :payment/amount ?a]]}
|
||||
(dc/db conn)
|
||||
[clients
|
||||
nil
|
||||
nil])
|
||||
|
||||
(->>
|
||||
(dc/q {:find '[?e ?a]
|
||||
:in '[$ [?clients ?start ?end]]
|
||||
:where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]
|
||||
[?e :payment/status :payment-status/pending]
|
||||
[?e :payment/amount ?a]]}
|
||||
(dc/db conn)
|
||||
[clients
|
||||
nil
|
||||
nil])
|
||||
|
||||
(map last)
|
||||
(reduce
|
||||
+
|
||||
@@ -277,16 +275,14 @@
|
||||
{ids-to-retrieve :ids matching-count :count
|
||||
all-ids :all-ids} (fetch-ids db request)]
|
||||
|
||||
|
||||
[(->> (hydrate-results ids-to-retrieve db request))
|
||||
matching-count
|
||||
(sum-visible-pending all-ids)
|
||||
(sum-client-pending (extract-client-ids (:clients request)
|
||||
(:client request)
|
||||
(:client-id (:query-params request))
|
||||
(when (:client-code (:query-params request))
|
||||
[:client/code (:client-code (:query-params request))])))
|
||||
]))
|
||||
(:client request)
|
||||
(:client-id (:query-params request))
|
||||
(when (:client-code (:query-params request))
|
||||
[:client/code (:client-code (:query-params request))])))]))
|
||||
|
||||
(def query-schema (mc/schema
|
||||
[:maybe [:map {:date-range [:date-range :start-date :end-date]}
|
||||
@@ -327,7 +323,7 @@
|
||||
(assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)])
|
||||
:query-schema query-schema
|
||||
:action-buttons (fn [request]
|
||||
(let [[_ _ visible-in-float total-in-float ] (:page-results request)]
|
||||
(let [[_ _ visible-in-float total-in-float] (:page-results request)]
|
||||
[(com/pill {:color :primary} " Visible in float "
|
||||
(format "$%,.2f" visible-in-float))
|
||||
(com/pill {:color :secondary} " Total in float "
|
||||
@@ -354,7 +350,7 @@
|
||||
|
||||
(= (-> request :query-params :sort first :name) "Bank account")
|
||||
(-> entity :payment/bank-account :bank-account/name)
|
||||
|
||||
|
||||
:else nil))
|
||||
:title (fn [r]
|
||||
(str
|
||||
@@ -409,7 +405,7 @@
|
||||
:render (fn [{:payment/keys [date]}]
|
||||
(some-> date (atime/unparse-local atime/normal-date)))}
|
||||
{:key "amount"
|
||||
:sort-key "amount"
|
||||
:sort-key "amount"
|
||||
:name "Amount"
|
||||
:render (fn [{:payment/keys [amount]}]
|
||||
(some->> amount (format "$%.2f")))}
|
||||
@@ -421,10 +417,10 @@
|
||||
(map :invoice-payment/invoice)
|
||||
(filter identity)
|
||||
(map (fn [invoice]
|
||||
{:link (hu/url (bidi/path-for ssr-routes/only-routes
|
||||
::invoice-route/all-page)
|
||||
{:exact-match-id (:db/id invoice)})
|
||||
:content (str "Inv. " (:invoice/invoice-number invoice))})))
|
||||
{:link (hu/url (bidi/path-for ssr-routes/only-routes
|
||||
::invoice-route/all-page)
|
||||
{:exact-match-id (:db/id invoice)})
|
||||
:content (str "Inv. " (:invoice/invoice-number invoice))})))
|
||||
(some-> p :transaction/_payment ((fn [t]
|
||||
[{:link (hu/url (bidi/path-for client-routes/routes
|
||||
:transactions)
|
||||
@@ -434,8 +430,6 @@
|
||||
|
||||
(def row* (partial helper/row* grid-page))
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
|
||||
(mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
|
||||
@@ -445,7 +439,6 @@
|
||||
(mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
|
||||
(mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))
|
||||
|
||||
|
||||
(mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)))
|
||||
|
||||
(defn delete [{check :entity :as request identity :identity}]
|
||||
@@ -459,7 +452,7 @@
|
||||
#(assert-can-see-client identity (:db/id (:payment/client check))))
|
||||
(notify-if-locked (:db/id (:payment/client check))
|
||||
(:payment/date check))
|
||||
(let [ removing-payments (mapcat (fn [x]
|
||||
(let [removing-payments (mapcat (fn [x]
|
||||
(let [invoice (:invoice-payment/invoice x)
|
||||
new-balance (+ (:invoice/outstanding-balance invoice)
|
||||
(:invoice-payment/amount x))]
|
||||
@@ -475,9 +468,9 @@
|
||||
:payment/status :payment-status/voided}]
|
||||
(audit-transact (cond-> removing-payments
|
||||
true (conj updated-payment)
|
||||
(:transaction/_payment check) (conj [:db/retract (:db/id (first (:transaction/_payment check)))
|
||||
(:transaction/_payment check) (conj [:db/retract (:db/id (first (:transaction/_payment check)))
|
||||
:transaction/payment
|
||||
(:db/id check)]))
|
||||
(:db/id check)]))
|
||||
identity)
|
||||
|
||||
(html-response (row* (:identity request) updated-payment {:delete-after-settle? true :class "live-removed"
|
||||
@@ -578,7 +571,6 @@
|
||||
(assoc-in [:query-params :start] 0)
|
||||
(assoc-in [:query-params :per-page] 250))))
|
||||
|
||||
|
||||
:else
|
||||
selected)
|
||||
updated-count (void-payments-internal ids (:identity request))]
|
||||
@@ -591,7 +583,7 @@
|
||||
|
||||
(defn wrap-status-from-source [handler]
|
||||
(fn [{:keys [matched-current-page-route] :as request}]
|
||||
(let [ request (cond-> request
|
||||
(let [request (cond-> request
|
||||
(= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared)
|
||||
(= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending)
|
||||
(= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided)
|
||||
@@ -605,7 +597,7 @@
|
||||
::route/pending-page (-> (helper/page-route grid-page)
|
||||
(wrap-implied-route-param :status :payment-status/pending))
|
||||
::route/voided-page (-> (helper/page-route grid-page)
|
||||
(wrap-implied-route-param :status :payment-status/voided))
|
||||
(wrap-implied-route-param :status :payment-status/voided))
|
||||
::route/all-page (-> (helper/page-route grid-page)
|
||||
(wrap-implied-route-param :status nil))
|
||||
|
||||
@@ -618,7 +610,6 @@
|
||||
::route/bulk-delete (-> bulk-delete-dialog
|
||||
(wrap-admin))
|
||||
|
||||
|
||||
::route/table (helper/table-route grid-page)}
|
||||
(fn [h]
|
||||
(-> h
|
||||
|
||||
@@ -22,6 +22,10 @@
|
||||
(let [conn (DriverManager/getConnection "jdbc:duckdb:")
|
||||
stmt (.createStatement conn)]
|
||||
(.execute stmt "INSTALL httpfs; LOAD httpfs;")
|
||||
(when-let [key (:aws-access-key-id env)]
|
||||
(.execute stmt (str "SET s3_access_key_id='" key "'"))
|
||||
(.execute stmt (str "SET s3_secret_access_key='" (:aws-secret-access-key env) "'"))
|
||||
(.execute stmt (str "SET s3_region='" (or (:aws-region env) "us-east-1") "'")))
|
||||
(.close stmt)
|
||||
(.addShutdownHook (Runtime/getRuntime)
|
||||
(Thread. #(fn [])))
|
||||
@@ -107,9 +111,9 @@
|
||||
(str entity-type ".jsonl"))]
|
||||
(io/make-parents wal-file)
|
||||
(with-open [w (io/writer wal-file :append true)]
|
||||
(.write w ^String (json/write-str {:seq-no seq-no
|
||||
:record record}))
|
||||
(.write w (int \newline))))
|
||||
(.write w ^String (json/write-str {:seq-no seq-no
|
||||
:record record}))
|
||||
(.write w (int \newline))))
|
||||
(catch Exception e
|
||||
(println "[parquet/wal]" (.getMessage e))))
|
||||
entry))
|
||||
@@ -138,8 +142,8 @@
|
||||
(try
|
||||
(with-open [w (io/writer jsonl-file :append true)]
|
||||
(doseq [r records]
|
||||
(.write w ^String (json/write-str (dissoc r :_seq-no)))
|
||||
(.write w (int \newline))))
|
||||
(.write w ^String (json/write-str (dissoc r :_seq-no)))
|
||||
(.write w (int \newline))))
|
||||
(execute-to-parquet!
|
||||
(format "SELECT * FROM read_json_auto('%s')"
|
||||
(.getAbsolutePath jsonl-file))
|
||||
@@ -150,8 +154,9 @@
|
||||
(.delete ^java.io.File parquet-file)
|
||||
{:key s3-key :status :ok}
|
||||
(catch Exception e
|
||||
(throw (ex-info "Flush failed" {:entity-type entity-type}
|
||||
:error (.getMessage e)))))))))
|
||||
(throw (ex-info "Flush failed"
|
||||
{:entity-type entity-type
|
||||
:error (.getMessage e)}))))))))
|
||||
|
||||
(defn flush-by-date! []
|
||||
"Flush all entity types for today."
|
||||
@@ -218,8 +223,9 @@
|
||||
Returns map with :sql and :count-sql keys."
|
||||
(let [date-strs (date-seq start-date end-date)
|
||||
urls (vec
|
||||
(map #(format "'s3://%%s/sales-details/%%s/%%s.parquet'"
|
||||
*bucket* entity-type %)
|
||||
(map (fn [d]
|
||||
(format "'s3://%s/sales-details/%s/%s.parquet'"
|
||||
*bucket* entity-type d))
|
||||
date-strs))
|
||||
sql (str "SELECT * FROM read_parquet(["
|
||||
(str/join ", " urls)
|
||||
@@ -244,35 +250,34 @@
|
||||
(get-sales-orders start-date end-date {}))
|
||||
([start-date end-date opts]
|
||||
(let [q (parquet-query "sales-order"
|
||||
start-date end-date)
|
||||
base-sql (:sql q)
|
||||
count-sql (:count-sql q)
|
||||
sort (get opts :sort "date")
|
||||
order (get opts :order "DESC")
|
||||
limit (get opts :limit)
|
||||
offset (get opts :offset)
|
||||
start-date end-date)
|
||||
base-sql (:sql q)
|
||||
count-sql (:count-sql q)
|
||||
sort (get opts :sort "date")
|
||||
order (get opts :order "DESC")
|
||||
limit (get opts :limit)
|
||||
offset (get opts :offset)
|
||||
where-str (build-where-clause
|
||||
|
||||
opts
|
||||
[[:client "external_id.client"]
|
||||
[:vendor "external_id.vendor"]
|
||||
[[:client "client-code"]
|
||||
[:vendor "vendor"]
|
||||
[:location "location"]])
|
||||
full-sql (if where-str
|
||||
(str base-sql where-str)
|
||||
base-sql)
|
||||
result (cond-> full-sql
|
||||
sort (str " ORDER BY " sort
|
||||
" " (name order))
|
||||
limit (str " LIMIT " limit)
|
||||
offset (str " OFFSET " offset))
|
||||
full-count (if where-str
|
||||
(str count-sql where-str)
|
||||
count-sql)]
|
||||
{:rows (query-rows result)
|
||||
:count (or
|
||||
(int
|
||||
(query-scalar
|
||||
full-count)) 0)})))
|
||||
full-sql (if where-str
|
||||
(str base-sql where-str)
|
||||
base-sql)
|
||||
result (cond-> full-sql
|
||||
sort (str " ORDER BY " sort
|
||||
" " (name order))
|
||||
limit (str " LIMIT " limit)
|
||||
offset (str " OFFSET " offset))
|
||||
full-count (if where-str
|
||||
(str count-sql where-str)
|
||||
count-sql)]
|
||||
{:rows (query-rows result)
|
||||
:count (or
|
||||
(int
|
||||
(query-scalar
|
||||
full-count)) 0)})))
|
||||
|
||||
(defn query-deduped [entity-type start-date end-date]
|
||||
"Query records deduplicated by external-id (latest _seq_no wins)."
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
tname
|
||||
(+ (get b tname 0.0) total)))))))
|
||||
{}
|
||||
rows)))
|
||||
rows)))
|
||||
(catch Exception e
|
||||
(println "[sales-summaries]" (.getMessage e))
|
||||
{}))))
|
||||
|
||||
Reference in New Issue
Block a user