notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Settlement fixes and perf improvements

Browse Source
This commit is contained in:
Bryce Covert
2021-09-04 10:05:16 -07:00
parent 35d875dd2f
commit b026b2806b
2 changed files with 131 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
sales_queries.txt
View File

@@ -42,7 +42,7 @@
[?s :expected-deposit/client ?c] [?s :expected-deposit/client ?c]
[?s :expected-deposit/total ?t] [?s :expected-deposit/total ?t]
[?s :expected-deposit/fee ?f] [?s :expected-deposit/fee ?f]
[?s :expected-deposit/date ?date] [?s :expected-deposit/sales-date ?date]
[(clj-time.coerce/to-date-time ?date) ?d2] [(clj-time.coerce/to-date-time ?date) ?d2]
[(auto-ap.time/localize ?d2) ?d3] [(auto-ap.time/localize ?d2) ?d3]
[(auto-ap.time/unparse-local ?d3 auto-ap.time/normal-date) ?d4] [(auto-ap.time/unparse-local ?d3 auto-ap.time/normal-date) ?d4]
@@ -69,7 +69,7 @@
] ]
# Refunds # Refunds
[:find ?d4 (sum ?total) (sum ?fee) [:find ?d4 ?t (sum ?total) (sum ?fee)
:with ?r :with ?r
:in $ :in $
:where :where
@@ -77,6 +77,7 @@
[?r :sales-refund/total ?total] [?r :sales-refund/total ?total]
[?r :sales-refund/fee ?fee] [?r :sales-refund/fee ?fee]
[?r :sales-refund/date ?date] [?r :sales-refund/date ?date]
[?r :sales-refund/type ?t]
[(clj-time.coerce/to-date-time ?date) ?d2] [(clj-time.coerce/to-date-time ?date) ?d2]
[(auto-ap.time/localize ?d2) ?d3] [(auto-ap.time/localize ?d2) ?d3]
[(auto-ap.time/unparse-local ?d3 auto-ap.time/normal-date) ?d4] [(auto-ap.time/unparse-local ?d3 auto-ap.time/normal-date) ?d4]

212
src/clj/auto_ap/square/core.clj
View File

@@ -11,14 +11,15 @@
[datomic.api :as d] [datomic.api :as d]
[mount.core :as mount] [mount.core :as mount]
[unilog.context :as lc] [unilog.context :as lc]
[yang.scheduler :as scheduler])) [yang.scheduler :as scheduler]
[clojure.core.async :as async]))
(def base-headers {"Square-Version" "2020-08-12" (def base-headers {"Square-Version" "2021-08-18"
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
"Content-Type" "application/json"}) "Content-Type" "application/json"})
(defn lookup-dates [] (defn lookup-dates []
(->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -15)) (->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -10))
(time/now) (time/now)
(time/days 5)) (time/days 5))
(map (fn [d] (map (fn [d]
@@ -180,63 +181,66 @@
))) )))
0.0)) 0.0))
(defn order->sales-order [order]
(let [[client loc] (location_id->client-location (:location_id order))]
(remove-nils
#:sales-order
{:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))
:client [:client/code client]
:location loc
:external-id (str "square/order/" client "-" loc "-" (:id order))
:total (-> order :net_amounts :total_money amount->money)
:tax (-> order :net_amounts :tax_money amount->money)
:tip (-> order :net_amounts :tip_money amount->money)
:discount (-> order :net_amounts :discount_money amount->money)
:service-charge (-> order :net_amounts :service_charge_money amount->money)
:returns (+ (- (-> order :return_amounts :total_money amount->money)
(-> order :return_amounts :tax_money amount->money)
(-> order :return_amounts :tip_money amount->money)
(-> order :return_amounts :service_charge_money amount->money))
(-> order :return_amounts :discount_money amount->money))
:charges (->> (:tenders order)
(map (fn [t]
(remove-nils
#:charge
{:type-name (:type t)
:processor (condp = (some-> (:note t) str/lower-case)
"doordash" :ccp-processor/doordash
"dd" :ccp-processor/doordash
"ubereats" :ccp-processor/uber-eats
"ue" :ccp-processor/uber-eats
"grubhub" :ccp-processor/grubhub
"grub" :ccp-processor/grubhub
"gh" :ccp-processor/grubhub
(condp = (:name (:source order))
"GRUBHUB" :ccp-processor/grubhub
"UBEREATS" :ccp-processor/uber-eats
"DOORDASH" :ccp-processor/doordash
:ccp-processor/na))
:total (amount->money (:amount_money t))
:tip (amount->money (:tip_money t))}))))
:line-items (->> (:line_items order)
(map (fn [li]
(remove-nils
#:order-line-item
{:item-name (:name li)
:category (if (= "GIFT_CARD" (:item_type li))
"Gift Card"
(item-id->category-name (:catalog_object_id li)))
:total (amount->money (:total_money li))
:tax (amount->money (:total_tax_money li))
:discount (amount->money (:total_discount_money li))}))))})))
(defn daily-results [d] (defn daily-results [d]
(->> (locations) (->> (locations)
(map :id) (map :id)
(filter location_id->client-location) (filter location_id->client-location)
(mapcat #(search % d)) (mapcat #(search % d))
(filter (fn [order] (filter (fn [order]
(not= #{"FAILED"} (and (= "COMPLETED" (:state order))
(set (map #(:status (:card_details %)) (:tenders order)))))) (not= #{"FAILED"}
(map (fn [order] (set (map #(:status (:card_details %)) (:tenders order)))))))
(let [[client loc] (location_id->client-location (:location_id order))] (map order->sales-order)))
(remove-nils
#:sales-order
{:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))
:client [:client/code client]
:location loc
:external-id (str "square/order/" client "-" loc "-" (:id order))
:total (-> order :net_amounts :total_money amount->money)
:tax (-> order :net_amounts :tax_money amount->money)
:tip (-> order :net_amounts :tip_money amount->money)
:discount (-> order :net_amounts :discount_money amount->money)
:service-charge (-> order :net_amounts :service_charge_money amount->money)
:returns (+ (- (-> order :return_amounts :total_money amount->money)
(-> order :return_amounts :tax_money amount->money)
(-> order :return_amounts :tip_money amount->money)
(-> order :return_amounts :service_charge_money amount->money))
(-> order :return_amounts :discount_money amount->money))
:charges (->> (:tenders order)
(map (fn [t]
(remove-nils
#:charge
{:type-name (:type t)
:processor (condp = (some-> (:note t) str/lower-case)
"doordash" :ccp-processor/doordash
"dd" :ccp-processor/doordash
"ubereats" :ccp-processor/uber-eats
"ue" :ccp-processor/uber-eats
"grubhub" :ccp-processor/grubhub
"grub" :ccp-processor/grubhub
"gh" :ccp-processor/grubhub
(condp = (:name (:source order))
"GRUBHUB" :ccp-processor/grubhub
"UBEREATS" :ccp-processor/uber-eats
"DOORDASH" :ccp-processor/doordash
:ccp-processor/na))
:total (amount->money (:amount_money t))
:tip (amount->money (:tip_money t))}))))
:line-items (->> (:line_items order)
(map (fn [li]
(remove-nils
#:order-line-item
{:item-name (:name li)
:category (if (= "GIFT_CARD" (:item_type li))
"Gift Card"
(item-id->category-name (:catalog_object_id li)))
:total (amount->money (:total_money li))
:tax (amount->money (:total_tax_money li))
:discount (amount->money (:total_discount_money li))}))))}))))))
#_(daily-results) #_(daily-results)
@@ -260,6 +264,54 @@
(log/warn "Retrying after failure " ex) (log/warn "Retrying after failure " ex)
(if (> try-count 4) false true))}))))) (if (> try-count 4) false true))})))))
(defn get-settlement-sales-date [settlement]
(let [concurrent 10
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(comp (map (fn [p]
(lc/with-context {:source "Square settlements loading "}
(log/info "looking up payment " p " for settlement " (:id settlement))
(->
(get-payment p)
:created_at
coerce/to-date))))
(filter identity ))
(async/to-chan (->> settlement
:entries
(map :payment_id)
(filter identity)
(set)
(take 20)
)))
(->> (async/<!! (async/into [] output-chan))
sort
(drop 1)
first)))
(defn get-settlement-details [settlements] ;; pairs of [location settlement]
(log/info "getting settlement details for " settlements)
(let [concurrent 10
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [[location s :as g]]
(lc/with-context {:source "Square settlements loading "}
(log/info "Looking up settlement " s " for location " location)
(let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" location "/settlements/" s)
{:headers base-headers
:as :json
:retry-handler (fn [ex try-count http-context]
(log/warn "Retrying after failure " ex)
(if (> try-count 4) false true))})))]
(log/info "sales date for " s " is " (get-settlement-sales-date settlement))
(assoc settlement :sales-date (get-settlement-sales-date settlement) ))
(log/info "Done loading settlements"))
))
(async/to-chan settlements))
(async/<!! (async/into [] output-chan))))
(defn settlements (defn settlements
([l] (settlements l (lookup-dates))) ([l] (settlements l (lookup-dates)))
([l lookup-dates] ([l lookup-dates]
@@ -278,14 +330,9 @@
set set
seq seq
(map (fn [s] (map (fn [s]
(log/info "Looking up settlement " s " for location " l) [l s]))
(let [n (:body (retry #(client/get (str "https://connect.squareup.com/v1/" l "/settlements/" s) (get-settlement-details )
{:headers base-headers )))
:as :json
:retry-handler (fn [ex try-count http-context]
(log/warn "Retrying after failure " ex)
(if (> try-count 4) false true))})))]
n))))))
(defn daily-settlements (defn daily-settlements
([] (daily-settlements (->> (locations) ([] (daily-settlements (->> (locations)
@@ -307,18 +354,9 @@
(:entries settlement)))) (:entries settlement))))
:date (-> (:initiated_at settlement) :date (-> (:initiated_at settlement)
(coerce/to-date)) (coerce/to-date))
:sales-date (->> settlement :sales-date (or (:sales-date settlement)
:entries (-> (:initiated_at settlement)
(map :payment_id) (coerce/to-date)))})))
(set)
(take 10)
(map (fn [p]
(get-payment p)))
(map :created_at)
(map coerce/to-date)
(sort)
(drop 5)
first)})))
(filter :expected-deposit/date))) (filter :expected-deposit/date)))
) )
@@ -347,7 +385,7 @@
:client [:client/code client] :client [:client/code client]
:location location :location location
:date (coerce/to-date (:created_at r)) :date (coerce/to-date (:created_at r))
})))))) :type (:source_type (get-payment (:payment_id r)))}))))))
(defn upsert (defn upsert
([] ([]
@@ -382,10 +420,10 @@
(map first) (map first)
set) set)
_ (log/info (count existing) "settlements already exist") _ (log/info (count existing) "settlements already exist")
to-create (filter #(not (existing (:expected-deposit/external-id %))) to-create (filter #(not (existing (:expected-deposit/external-id %)))
(if location-ids (if location-ids
(daily-settlements location-ids) (daily-settlements location-ids)
(daily-settlements)))] (daily-settlements)))]
(doseq [x (partition-all 20 to-create)] (doseq [x (partition-all 20 to-create)]
(log/info "Loading expected deposit" (count x)) (log/info "Loading expected deposit" (count x))
@(d/transact conn x))) @(d/transact conn x)))
@@ -405,8 +443,8 @@
(map first) (map first)
set) set)
_ (log/info (count existing) "refunds already exist") _ (log/info (count existing) "refunds already exist")
to-create (filter #(not (existing (:sales-refund/external-id %))) to-create (filter #(not (existing (:sales-refund/external-id %)))
(refunds location))] (refunds location))]
(doseq [x (partition-all 20 to-create)] (doseq [x (partition-all 20 to-create)]
(log/info "Loading refund" (count x)) (log/info "Loading refund" (count x))
@(d/transact conn x))) @(d/transact conn x)))
@@ -424,20 +462,26 @@
(map (fn [x] [:db/retractEntity x])))) (map (fn [x] [:db/retractEntity x]))))
(mount/defstate square-loader (mount/defstate square-loader
:start (scheduler/every (* 13 60 1000) upsert) :start (scheduler/every (* 4 59 60 1000) upsert)
:stop (scheduler/stop square-loader)) :stop (scheduler/stop square-loader))
(mount/defstate square-settlement-loader (mount/defstate square-settlement-loader
:start (scheduler/every (* 14 60 1000) upsert-settlements) :start (scheduler/every (* 4 60 60 1000) upsert-settlements)
:stop (scheduler/stop square-settlement-loader)) :stop (scheduler/stop square-settlement-loader))
(mount/defstate square-refund-loader (mount/defstate square-refund-loader
:start (scheduler/every (* 30 60 1000) upsert-refunds) :start (scheduler/every (* 4 58 60 1000) upsert-refunds)
:stop (scheduler/stop square-refund-loader)) :stop (scheduler/stop square-refund-loader))
(comment (comment
(daily-results) (daily-results)
(mount/stop (mount/only #{'auto-ap.square.core/square-settlement-loader}))
(mount/stop (mount/only #{'auto-ap.square.core/square-refund-loader}))
(mount/stop (mount/only #{'auto-ap.square.core/square-loader}))
(mount/start (mount/only #{'auto-ap.square.core/square-settlement-loader}))
(mount/start (mount/only #{'auto-ap.square.core/square-refund-loader}))
(mount/start (mount/only #{'auto-ap.square.core/square-loader}))
(do (upsert) nil) (do (upsert) nil)
(do @(d/transact conn (reset)) nil)) (do @(d/transact conn (reset)) nil))