Settlement fixes and perf improvements
This commit is contained in:
@@ -11,14 +11,15 @@
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
[yang.scheduler :as scheduler]
|
||||
[clojure.core.async :as async]))
|
||||
|
||||
(def base-headers {"Square-Version" "2020-08-12"
|
||||
(def base-headers {"Square-Version" "2021-08-18"
|
||||
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
||||
"Content-Type" "application/json"})
|
||||
|
||||
(defn lookup-dates []
|
||||
(->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -15))
|
||||
(->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -10))
|
||||
(time/now)
|
||||
(time/days 5))
|
||||
(map (fn [d]
|
||||
@@ -180,63 +181,66 @@
|
||||
)))
|
||||
0.0))
|
||||
|
||||
(defn order->sales-order [order]
|
||||
(let [[client loc] (location_id->client-location (:location_id order))]
|
||||
(remove-nils
|
||||
#:sales-order
|
||||
{:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))
|
||||
:client [:client/code client]
|
||||
:location loc
|
||||
:external-id (str "square/order/" client "-" loc "-" (:id order))
|
||||
:total (-> order :net_amounts :total_money amount->money)
|
||||
:tax (-> order :net_amounts :tax_money amount->money)
|
||||
:tip (-> order :net_amounts :tip_money amount->money)
|
||||
:discount (-> order :net_amounts :discount_money amount->money)
|
||||
:service-charge (-> order :net_amounts :service_charge_money amount->money)
|
||||
:returns (+ (- (-> order :return_amounts :total_money amount->money)
|
||||
(-> order :return_amounts :tax_money amount->money)
|
||||
(-> order :return_amounts :tip_money amount->money)
|
||||
(-> order :return_amounts :service_charge_money amount->money))
|
||||
(-> order :return_amounts :discount_money amount->money))
|
||||
:charges (->> (:tenders order)
|
||||
(map (fn [t]
|
||||
(remove-nils
|
||||
#:charge
|
||||
{:type-name (:type t)
|
||||
:processor (condp = (some-> (:note t) str/lower-case)
|
||||
"doordash" :ccp-processor/doordash
|
||||
"dd" :ccp-processor/doordash
|
||||
"ubereats" :ccp-processor/uber-eats
|
||||
"ue" :ccp-processor/uber-eats
|
||||
"grubhub" :ccp-processor/grubhub
|
||||
"grub" :ccp-processor/grubhub
|
||||
"gh" :ccp-processor/grubhub
|
||||
(condp = (:name (:source order))
|
||||
"GRUBHUB" :ccp-processor/grubhub
|
||||
"UBEREATS" :ccp-processor/uber-eats
|
||||
"DOORDASH" :ccp-processor/doordash
|
||||
:ccp-processor/na))
|
||||
:total (amount->money (:amount_money t))
|
||||
:tip (amount->money (:tip_money t))}))))
|
||||
:line-items (->> (:line_items order)
|
||||
(map (fn [li]
|
||||
(remove-nils
|
||||
#:order-line-item
|
||||
{:item-name (:name li)
|
||||
:category (if (= "GIFT_CARD" (:item_type li))
|
||||
"Gift Card"
|
||||
(item-id->category-name (:catalog_object_id li)))
|
||||
:total (amount->money (:total_money li))
|
||||
:tax (amount->money (:total_tax_money li))
|
||||
:discount (amount->money (:total_discount_money li))}))))})))
|
||||
|
||||
(defn daily-results [d]
|
||||
(->> (locations)
|
||||
(map :id)
|
||||
(filter location_id->client-location)
|
||||
(mapcat #(search % d))
|
||||
(filter (fn [order]
|
||||
(not= #{"FAILED"}
|
||||
(set (map #(:status (:card_details %)) (:tenders order))))))
|
||||
(map (fn [order]
|
||||
(let [[client loc] (location_id->client-location (:location_id order))]
|
||||
(remove-nils
|
||||
#:sales-order
|
||||
{:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))
|
||||
:client [:client/code client]
|
||||
:location loc
|
||||
:external-id (str "square/order/" client "-" loc "-" (:id order))
|
||||
:total (-> order :net_amounts :total_money amount->money)
|
||||
:tax (-> order :net_amounts :tax_money amount->money)
|
||||
:tip (-> order :net_amounts :tip_money amount->money)
|
||||
:discount (-> order :net_amounts :discount_money amount->money)
|
||||
:service-charge (-> order :net_amounts :service_charge_money amount->money)
|
||||
:returns (+ (- (-> order :return_amounts :total_money amount->money)
|
||||
(-> order :return_amounts :tax_money amount->money)
|
||||
(-> order :return_amounts :tip_money amount->money)
|
||||
(-> order :return_amounts :service_charge_money amount->money))
|
||||
(-> order :return_amounts :discount_money amount->money))
|
||||
:charges (->> (:tenders order)
|
||||
(map (fn [t]
|
||||
(remove-nils
|
||||
#:charge
|
||||
{:type-name (:type t)
|
||||
:processor (condp = (some-> (:note t) str/lower-case)
|
||||
"doordash" :ccp-processor/doordash
|
||||
"dd" :ccp-processor/doordash
|
||||
"ubereats" :ccp-processor/uber-eats
|
||||
"ue" :ccp-processor/uber-eats
|
||||
"grubhub" :ccp-processor/grubhub
|
||||
"grub" :ccp-processor/grubhub
|
||||
"gh" :ccp-processor/grubhub
|
||||
(condp = (:name (:source order))
|
||||
"GRUBHUB" :ccp-processor/grubhub
|
||||
"UBEREATS" :ccp-processor/uber-eats
|
||||
"DOORDASH" :ccp-processor/doordash
|
||||
:ccp-processor/na))
|
||||
:total (amount->money (:amount_money t))
|
||||
:tip (amount->money (:tip_money t))}))))
|
||||
:line-items (->> (:line_items order)
|
||||
(map (fn [li]
|
||||
(remove-nils
|
||||
#:order-line-item
|
||||
{:item-name (:name li)
|
||||
:category (if (= "GIFT_CARD" (:item_type li))
|
||||
"Gift Card"
|
||||
(item-id->category-name (:catalog_object_id li)))
|
||||
:total (amount->money (:total_money li))
|
||||
:tax (amount->money (:total_tax_money li))
|
||||
:discount (amount->money (:total_discount_money li))}))))}))))))
|
||||
(and (= "COMPLETED" (:state order))
|
||||
(not= #{"FAILED"}
|
||||
(set (map #(:status (:card_details %)) (:tenders order)))))))
|
||||
(map order->sales-order)))
|
||||
|
||||
#_(daily-results)
|
||||
|
||||
@@ -260,6 +264,54 @@
|
||||
(log/warn "Retrying after failure " ex)
|
||||
(if (> try-count 4) false true))})))))
|
||||
|
||||
(defn get-settlement-sales-date [settlement]
|
||||
(let [concurrent 10
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(comp (map (fn [p]
|
||||
(lc/with-context {:source "Square settlements loading "}
|
||||
(log/info "looking up payment " p " for settlement " (:id settlement))
|
||||
(->
|
||||
(get-payment p)
|
||||
:created_at
|
||||
coerce/to-date))))
|
||||
(filter identity ))
|
||||
(async/to-chan (->> settlement
|
||||
:entries
|
||||
(map :payment_id)
|
||||
(filter identity)
|
||||
(set)
|
||||
(take 20)
|
||||
)))
|
||||
(->> (async/<!! (async/into [] output-chan))
|
||||
sort
|
||||
(drop 1)
|
||||
first)))
|
||||
|
||||
(defn get-settlement-details [settlements] ;; pairs of [location settlement]
|
||||
(log/info "getting settlement details for " settlements)
|
||||
(let [concurrent 10
|
||||
output-chan (async/chan)]
|
||||
(async/pipeline-blocking concurrent
|
||||
output-chan
|
||||
(map (fn [[location s :as g]]
|
||||
(lc/with-context {:source "Square settlements loading "}
|
||||
(log/info "Looking up settlement " s " for location " location)
|
||||
(let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" location "/settlements/" s)
|
||||
{:headers base-headers
|
||||
:as :json
|
||||
:retry-handler (fn [ex try-count http-context]
|
||||
(log/warn "Retrying after failure " ex)
|
||||
(if (> try-count 4) false true))})))]
|
||||
|
||||
(log/info "sales date for " s " is " (get-settlement-sales-date settlement))
|
||||
(assoc settlement :sales-date (get-settlement-sales-date settlement) ))
|
||||
(log/info "Done loading settlements"))
|
||||
))
|
||||
(async/to-chan settlements))
|
||||
(async/<!! (async/into [] output-chan))))
|
||||
|
||||
(defn settlements
|
||||
([l] (settlements l (lookup-dates)))
|
||||
([l lookup-dates]
|
||||
@@ -278,14 +330,9 @@
|
||||
set
|
||||
seq
|
||||
(map (fn [s]
|
||||
(log/info "Looking up settlement " s " for location " l)
|
||||
(let [n (:body (retry #(client/get (str "https://connect.squareup.com/v1/" l "/settlements/" s)
|
||||
{:headers base-headers
|
||||
:as :json
|
||||
:retry-handler (fn [ex try-count http-context]
|
||||
(log/warn "Retrying after failure " ex)
|
||||
(if (> try-count 4) false true))})))]
|
||||
n))))))
|
||||
[l s]))
|
||||
(get-settlement-details )
|
||||
)))
|
||||
|
||||
(defn daily-settlements
|
||||
([] (daily-settlements (->> (locations)
|
||||
@@ -307,18 +354,9 @@
|
||||
(:entries settlement))))
|
||||
:date (-> (:initiated_at settlement)
|
||||
(coerce/to-date))
|
||||
:sales-date (->> settlement
|
||||
:entries
|
||||
(map :payment_id)
|
||||
(set)
|
||||
(take 10)
|
||||
(map (fn [p]
|
||||
(get-payment p)))
|
||||
(map :created_at)
|
||||
(map coerce/to-date)
|
||||
(sort)
|
||||
(drop 5)
|
||||
first)})))
|
||||
:sales-date (or (:sales-date settlement)
|
||||
(-> (:initiated_at settlement)
|
||||
(coerce/to-date)))})))
|
||||
(filter :expected-deposit/date)))
|
||||
)
|
||||
|
||||
@@ -347,7 +385,7 @@
|
||||
:client [:client/code client]
|
||||
:location location
|
||||
:date (coerce/to-date (:created_at r))
|
||||
}))))))
|
||||
:type (:source_type (get-payment (:payment_id r)))}))))))
|
||||
|
||||
(defn upsert
|
||||
([]
|
||||
@@ -382,10 +420,10 @@
|
||||
(map first)
|
||||
set)
|
||||
_ (log/info (count existing) "settlements already exist")
|
||||
to-create (filter #(not (existing (:expected-deposit/external-id %)))
|
||||
(if location-ids
|
||||
(daily-settlements location-ids)
|
||||
(daily-settlements)))]
|
||||
to-create (filter #(not (existing (:expected-deposit/external-id %)))
|
||||
(if location-ids
|
||||
(daily-settlements location-ids)
|
||||
(daily-settlements)))]
|
||||
(doseq [x (partition-all 20 to-create)]
|
||||
(log/info "Loading expected deposit" (count x))
|
||||
@(d/transact conn x)))
|
||||
@@ -405,8 +443,8 @@
|
||||
(map first)
|
||||
set)
|
||||
_ (log/info (count existing) "refunds already exist")
|
||||
to-create (filter #(not (existing (:sales-refund/external-id %)))
|
||||
(refunds location))]
|
||||
to-create (filter #(not (existing (:sales-refund/external-id %)))
|
||||
(refunds location))]
|
||||
(doseq [x (partition-all 20 to-create)]
|
||||
(log/info "Loading refund" (count x))
|
||||
@(d/transact conn x)))
|
||||
@@ -424,20 +462,26 @@
|
||||
(map (fn [x] [:db/retractEntity x]))))
|
||||
|
||||
(mount/defstate square-loader
|
||||
:start (scheduler/every (* 13 60 1000) upsert)
|
||||
:start (scheduler/every (* 4 59 60 1000) upsert)
|
||||
:stop (scheduler/stop square-loader))
|
||||
|
||||
(mount/defstate square-settlement-loader
|
||||
:start (scheduler/every (* 14 60 1000) upsert-settlements)
|
||||
:start (scheduler/every (* 4 60 60 1000) upsert-settlements)
|
||||
:stop (scheduler/stop square-settlement-loader))
|
||||
|
||||
(mount/defstate square-refund-loader
|
||||
:start (scheduler/every (* 30 60 1000) upsert-refunds)
|
||||
:start (scheduler/every (* 4 58 60 1000) upsert-refunds)
|
||||
:stop (scheduler/stop square-refund-loader))
|
||||
|
||||
(comment
|
||||
(daily-results)
|
||||
(mount/stop (mount/only #{'auto-ap.square.core/square-settlement-loader}))
|
||||
(mount/stop (mount/only #{'auto-ap.square.core/square-refund-loader}))
|
||||
(mount/stop (mount/only #{'auto-ap.square.core/square-loader}))
|
||||
|
||||
(mount/start (mount/only #{'auto-ap.square.core/square-settlement-loader}))
|
||||
(mount/start (mount/only #{'auto-ap.square.core/square-refund-loader}))
|
||||
(mount/start (mount/only #{'auto-ap.square.core/square-loader}))
|
||||
(do (upsert) nil)
|
||||
|
||||
(do @(d/transact conn (reset)) nil))
|
||||
|
||||
Reference in New Issue
Block a user