fix(sales): fix parquet SQL generation and cleanup formatting
- Fix double ORDER BY in sales_orders raw-graphql-ids (was passing full ORDER BY clause from build-sort-clause into get-sales-orders which prepends its own ORDER BY, producing 'ORDER BY ORDER BY ... DESC DESC') - Fix WHERE clause column names in parquet build-where-clause: external_id.client -> client-code, external_id.vendor -> vendor - Fix parquet-query format string (%%s -> %s with proper format call) - Fix ex-info call signature in flush! (was passing :error as third arg instead of inside the data map) - Add S3 credentials to DuckDB connect! so httpfs can read from S3 - Fix parquet buffer indentation and alignment across square/core3, ezcater/core, ezcater_xls, payments, sales_summaries, migrations - Fix broken Datomic query syntax in ezcater/core (upsert-used-subscriptions, upsert-recent find/where clauses mangled by paren-repair) - Uncomment accidentally commented code block in square/core3 - Fix paren/indentation issues in ssr/payments, jobs/sales_summaries
This commit is contained in:
@@ -28,11 +28,9 @@
|
||||
"Authorization" (str "Bearer " (:client/square-auth-token client))
|
||||
"Content-Type" "application/json"}))
|
||||
|
||||
|
||||
(defn ->square-date [d]
|
||||
(f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d))
|
||||
|
||||
|
||||
(def manifold-api-stream
|
||||
(let [stream (s/stream 100)]
|
||||
(->> stream
|
||||
@@ -43,10 +41,10 @@
|
||||
(de/loop [attempt 0]
|
||||
(-> (de/chain (de/future-with (ex/execute-pool)
|
||||
#_(log/info ::request-started
|
||||
:url (:url request)
|
||||
:attempt attempt
|
||||
:source "Square 3"
|
||||
:background-job "Square 3")
|
||||
:url (:url request)
|
||||
:attempt attempt
|
||||
:source "Square 3"
|
||||
:background-job "Square 3")
|
||||
(try
|
||||
(client/request (assoc request
|
||||
:socket-timeout 10000
|
||||
@@ -105,7 +103,6 @@
|
||||
:exception error))
|
||||
[]))))
|
||||
|
||||
|
||||
(def item-cache (atom {}))
|
||||
|
||||
(defn fetch-catalog [client i v]
|
||||
@@ -125,13 +122,11 @@
|
||||
#(do (swap! item-cache assoc i %)
|
||||
%))))
|
||||
|
||||
|
||||
(defn fetch-catalog-cache [client i version]
|
||||
(if (get @item-cache i)
|
||||
(de/success-deferred (get @item-cache i))
|
||||
(fetch-catalog client i version)))
|
||||
|
||||
|
||||
(defn item->category-name-impl [client item version]
|
||||
(capture-context->lc
|
||||
(cond (:item_id (:item_variation_data item))
|
||||
@@ -162,7 +157,6 @@
|
||||
:item item)
|
||||
"Uncategorized"))))
|
||||
|
||||
|
||||
(defn item-id->category-name [client i version]
|
||||
(capture-context->lc
|
||||
(-> [client i]
|
||||
@@ -227,7 +221,6 @@
|
||||
(concat (:orders result) continued-results))))
|
||||
(:orders result)))))))
|
||||
|
||||
|
||||
(defn search
|
||||
([client location start end]
|
||||
(capture-context->lc
|
||||
@@ -251,11 +244,9 @@
|
||||
(concat (:orders result) continued-results))))
|
||||
(:orders result))))))))
|
||||
|
||||
|
||||
(defn amount->money [amt]
|
||||
(* 0.01 (or (:amount amt) 0.0)))
|
||||
|
||||
|
||||
;; to get totals:
|
||||
(comment
|
||||
(reduce
|
||||
@@ -281,7 +272,7 @@
|
||||
:reference-link (str (url/url "https://squareup.com/receipt/preview" (:id t)))
|
||||
:external-id (when (:id t)
|
||||
(str "square/charge/" (:id t)))
|
||||
:processor (cond
|
||||
:processor (cond
|
||||
(#{"OTHER" "THIRD_PARTY_CARD"} (:type t))
|
||||
(condp = (some-> (:note t) str/lower-case)
|
||||
"doordash" :ccp-processor/doordash
|
||||
@@ -354,7 +345,7 @@
|
||||
#:sales-order
|
||||
{:date (if (= "Invoices" (:name (:source order)))
|
||||
(when (:closed_at order)
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
(coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles"))))
|
||||
:client (:db/id client)
|
||||
:location (:square-location/client-location location)
|
||||
@@ -416,7 +407,6 @@
|
||||
:client client
|
||||
:location location)))))))
|
||||
|
||||
|
||||
(defn get-payment [client p]
|
||||
(de/chain (manifold-api-call
|
||||
{:url (str "https://connect.squareup.com/v2/payments/" p)
|
||||
@@ -425,7 +415,6 @@
|
||||
:body
|
||||
:payment))
|
||||
|
||||
|
||||
(defn continue-payout-entry-list [c l poi cursor]
|
||||
(capture-context->lc lc
|
||||
(de/chain
|
||||
@@ -610,8 +599,8 @@
|
||||
so-date (some-> (:sales-order/date order) .toString)
|
||||
client (:sales-order/client order)
|
||||
client-code (when client (if (map? client)
|
||||
(:client/code client)
|
||||
client))]
|
||||
(:client/code client)
|
||||
client))]
|
||||
(parquet/buffer! "sales-order"
|
||||
{:entity-type "sales-order"
|
||||
:external-id so-ext-id
|
||||
@@ -627,32 +616,32 @@
|
||||
(when-let [charges (:sales-order/charges order)]
|
||||
(doseq [chg charges]
|
||||
(parquet/buffer! "charge"
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})
|
||||
{:entity-type "charge"
|
||||
:external-id (:charge/external-id chg)
|
||||
:type-name (:charge/type-name chg)
|
||||
:total (:charge/total chg)
|
||||
:tax (:charge/tax chg)
|
||||
:tip (:charge/tip chg)
|
||||
:date so-date
|
||||
:processor (some-> (:charge/processor chg) name)
|
||||
:sales-order-external-id so-ext-id})
|
||||
(when-let [returns (:charge/returns chg)]
|
||||
(doseq [rt returns]
|
||||
(parquet/buffer! "sales-refund"
|
||||
{:entity-type "sales-refund"
|
||||
:type-name (:type-name rt)
|
||||
:total (:total rt)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
{:entity-type "sales-refund"
|
||||
:type-name (:type-name rt)
|
||||
:total (:total rt)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
(when-let [items (:sales-order/line-items order)]
|
||||
(doseq [li items]
|
||||
(parquet/buffer! "line-item"
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
{:entity-type "line-item"
|
||||
:item-name (:order-line-item/item-name li)
|
||||
:category (:order-line-item/category li)
|
||||
:total (:order-line-item/total li)
|
||||
:tax (:order-line-item/tax li)
|
||||
:discount (:order-line-item/discount li)
|
||||
:sales-order-external-id so-ext-id})))))
|
||||
|
||||
(defn upsert
|
||||
([client]
|
||||
@@ -674,8 +663,7 @@
|
||||
(catch Exception e
|
||||
(log/error ::buffer-failed
|
||||
:exception e
|
||||
:order (:sales-order/external-id order)))))))))))
|
||||
|
||||
:order (:sales-order/external-id order))))))))))))
|
||||
|
||||
(defn upsert-payouts
|
||||
([client]
|
||||
@@ -725,7 +713,6 @@
|
||||
|
||||
(log/info ::done-loading-refunds)))))))
|
||||
|
||||
|
||||
(defn get-cash-shift [client id]
|
||||
(de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id))
|
||||
:method :get
|
||||
@@ -884,8 +871,6 @@
|
||||
d1
|
||||
d2))
|
||||
|
||||
|
||||
|
||||
(defn remove-voided-orders
|
||||
([client]
|
||||
(apply de/zip
|
||||
@@ -912,7 +897,7 @@
|
||||
(:sales-order/external-id o))))))
|
||||
(s/map (fn [[o]]
|
||||
[[:db/retractEntity [:sales-order/external-id (:sales-order/external-id o)]]]))
|
||||
|
||||
|
||||
(s/reduce into [])))
|
||||
|
||||
(fn [results]
|
||||
@@ -920,32 +905,28 @@
|
||||
(doseq [x (partition-all 100 results)]
|
||||
(log/info ::removing-orders
|
||||
:count (count x))
|
||||
@(dc/transact-async conn x)))))
|
||||
@(dc/transact-async conn x)
|
||||
(de/catch (fn [e]
|
||||
(log/warn ::couldnt-remove :error e)
|
||||
nil) ))))))
|
||||
nil)))))))))))
|
||||
|
||||
#_(comment
|
||||
(require 'auto-ap.time-reader)
|
||||
#_(comment
|
||||
(require 'auto-ap.time-reader)
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "DBFS") ]
|
||||
(log/peek :x [ c l])
|
||||
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29")
|
||||
@(let [[c [l]] (get-square-client-and-location "DBFS")]
|
||||
(log/peek :x [c l])
|
||||
(search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29"))
|
||||
|
||||
)
|
||||
@(let [[c [l]] (get-square-client-and-location "NGAK")]
|
||||
(log/peek :x [c l])
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "NGAK") ]
|
||||
(log/peek :x [ c l])
|
||||
|
||||
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
|
||||
(doseq [c (get-square-clients)]
|
||||
(try
|
||||
@(remove-voided-orders c)
|
||||
(catch Exception e
|
||||
nil)))
|
||||
|
||||
|
||||
)
|
||||
(remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15"))
|
||||
(doseq [c (get-square-clients)]
|
||||
(try
|
||||
@(remove-voided-orders c)
|
||||
(catch Exception e
|
||||
nil)))
|
||||
)
|
||||
|
||||
(defn upsert-all [& clients]
|
||||
(capture-context->lc
|
||||
@@ -1014,8 +995,6 @@
|
||||
[:clients clients]
|
||||
@(apply upsert-all clients)))
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(defn refunds-raw-cont
|
||||
([client l cursor so-far]
|
||||
@@ -1045,9 +1024,8 @@
|
||||
(->>
|
||||
@(let [[c [l]] (get-square-client-and-location "NGGG")]
|
||||
|
||||
|
||||
(search c l (time/now) (time/plus (time/now) (time/days -1))))
|
||||
|
||||
|
||||
(filter (fn [r]
|
||||
(str/starts-with? (:created_at r) "2024-03-14"))))
|
||||
|
||||
@@ -1055,7 +1033,6 @@
|
||||
(->>
|
||||
@(let [[c [l]] (get-square-client-and-location "NGGG")]
|
||||
|
||||
|
||||
(refunds-raw-cont c l nil []))
|
||||
(filter (fn [r]
|
||||
(str/starts-with? (:created_at r) "2024-03-14")))))
|
||||
@@ -1089,13 +1066,8 @@
|
||||
[]))]
|
||||
[(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)])
|
||||
:separator \tab)
|
||||
|
||||
|
||||
|
||||
|
||||
;; =>
|
||||
|
||||
|
||||
;; =>
|
||||
|
||||
(require 'auto-ap.time-reader)
|
||||
|
||||
@@ -1104,27 +1076,16 @@
|
||||
(clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")]
|
||||
l
|
||||
|
||||
|
||||
(def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00"
|
||||
#clj-time/date-time "2025-02-28T00:00:00-08:00"))
|
||||
(take 10 (map #(first (deref (order->sales-order c l %))) z)))
|
||||
(take 10 (map #(first (deref (order->sales-order c l %))) z))))
|
||||
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
(->> z
|
||||
(->> z
|
||||
(filter (fn [o]
|
||||
(seq (filter (comp #{"OTHER"} :type) (:tenders o)))))
|
||||
(filter #(not (:name (:source %))))
|
||||
(count)
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
(count))
|
||||
|
||||
(doseq [[code] (seq (dc/q '[:find ?code
|
||||
:in $
|
||||
:where [?o :sales-order/date ?d]
|
||||
@@ -1133,32 +1094,22 @@
|
||||
[?o :sales-order/client ?c]
|
||||
[?c :client/code ?code]]
|
||||
(dc/db conn)))
|
||||
:let [[c [l]] (get-square-client-and-location code)
|
||||
]
|
||||
:let [[c [l]] (get-square-client-and-location code)]
|
||||
order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now))
|
||||
:when (= "Invoices" (:name (:source order) ))
|
||||
:when (= "Invoices" (:name (:source order)))
|
||||
:let [[sales-order] @(order->sales-order c l order)]]
|
||||
|
||||
|
||||
(when (should-import-order? order)
|
||||
(println "DATE IS" (:sales-order/date sales-order))
|
||||
(when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00"))
|
||||
(println "WOULD UPDATE" sales-order)
|
||||
@(dc/transact auto-ap.datomic/conn [sales-order])
|
||||
)
|
||||
#_@(dc/transact )
|
||||
(println "DONE"))
|
||||
|
||||
|
||||
)
|
||||
@(dc/transact auto-ap.datomic/conn [sales-order]))
|
||||
#_@(dc/transact)
|
||||
(println "DONE")))
|
||||
|
||||
#_(filter (comp #{"OTHER"} :type) (mapcat :tenders z))
|
||||
|
||||
|
||||
@(let [[c [l]] (get-square-client-and-location "NGRY")]
|
||||
#_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01"))
|
||||
|
||||
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" )))
|
||||
|
||||
)
|
||||
|
||||
)
|
||||
(order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY")))))
|
||||
|
||||
Reference in New Issue
Block a user