501 lines
23 KiB
Clojure
501 lines
23 KiB
Clojure
(ns auto-ap.square.core
|
|
(:require [auto-ap.datomic :refer [conn remove-nils]]
|
|
[auto-ap.utils :refer [by]]
|
|
[clj-http.client :as client]
|
|
[clj-time.coerce :as coerce]
|
|
[clj-time.core :as time]
|
|
[clj-time.format :as f]
|
|
[clojure.string :as str]
|
|
[clojure.data.json :as json]
|
|
[clojure.tools.logging :as log]
|
|
[datomic.api :as d]
|
|
[mount.core :as mount]
|
|
[unilog.context :as lc]
|
|
[yang.scheduler :as scheduler]
|
|
[clojure.core.async :as async]))
|
|
|
|
(def base-headers {"Square-Version" "2021-08-18"
|
|
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
|
"Content-Type" "application/json"})
|
|
|
|
(defn lookup-dates []
|
|
(->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -15))
|
|
(time/now)
|
|
(time/days 5))
|
|
(map (fn [d]
|
|
[(auto-ap.time/unparse (time/plus d (time/days 1)) auto-ap.time/iso-date)
|
|
|
|
(auto-ap.time/unparse (time/plus d (time/days 5)) auto-ap.time/iso-date)]))))
|
|
|
|
(defn locations []
|
|
(->> (client/get "https://connect.squareup.com/v2/locations"
|
|
{:headers {"Square-Version" "2020-08-12"
|
|
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
|
"Content-Type" "application/json"}
|
|
:as :json})
|
|
:body
|
|
:locations))
|
|
|
|
(defn fetch-catalog [i]
|
|
(if i
|
|
(try
|
|
(log/info "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i))
|
|
(->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i)
|
|
{:headers base-headers
|
|
:query-params {"include_related_items" "true"}
|
|
:as :json})
|
|
:body
|
|
:object)
|
|
(catch Exception e
|
|
(log/error e)
|
|
nil))
|
|
(log/warn "Trying to look up non existant ")))
|
|
|
|
(def fetch-catalog-fast (memoize fetch-catalog))
|
|
|
|
(defn item-id->category-name [i]
|
|
(let [item (fetch-catalog-fast i)]
|
|
(cond (:item_variation_data item)
|
|
(item-id->category-name (:item_id (:item_variation_data item)))
|
|
|
|
(:category_id (:item_data item))
|
|
(:name (:category_data (fetch-catalog-fast (:category_id (:item_data item)))))
|
|
|
|
(:item_data item)
|
|
"Uncategorized"
|
|
|
|
:else
|
|
(do
|
|
(log/error "couldn't look up" i)
|
|
"Uncategorized"))))
|
|
|
|
(defn categories []
|
|
(by :id (comp :name :category_data) ))
|
|
|
|
(def potential-query
|
|
{"query" {"filter" {"date_time_filter"
|
|
{
|
|
"created_at" {
|
|
"start_at" "2020-08-28T00:00:00-07:00",
|
|
"end_at" "2020-08-28T23:59:59-07:00"
|
|
}
|
|
|
|
|
|
|
|
}
|
|
"state_filter" {"states" ["COMPLETED"]}}
|
|
|
|
"sort" {
|
|
"sort_field" "CREATED_AT"
|
|
"sort_order" "DESC"
|
|
}}})
|
|
|
|
|
|
(defn pc [d] {"query" {"filter" {"date_time_filter"
|
|
{
|
|
"created_at" {
|
|
"start_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles")))
|
|
"end_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (time/plus (coerce/to-date-time d) (time/days 1)) (time/time-zone-for-id "America/Los_Angeles")))
|
|
}
|
|
|
|
|
|
|
|
}
|
|
"state_filter" {"states" ["COMPLETED"]}}
|
|
|
|
"sort" {
|
|
"sort_field" "CREATED_AT"
|
|
"sort_order" "DESC"
|
|
}}})
|
|
|
|
|
|
(defn search
|
|
([l]
|
|
(search l nil))
|
|
([l d]
|
|
(log/info "Searching for" l)
|
|
(let [result (->> (client/post "https://connect.squareup.com/v2/orders/search"
|
|
{:headers {"Square-Version" "2020-08-12"
|
|
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
|
"Content-Type" "application/json"}
|
|
:body (json/write-str (cond-> {"location_ids" [l] "limit" 10000}
|
|
d (merge (pc d))))
|
|
:as :json})
|
|
:body
|
|
:orders)]
|
|
(log/info "found " (count result))
|
|
result)))
|
|
|
|
|
|
(defn order
|
|
|
|
([o]
|
|
(log/info "Searching for" o)
|
|
(let [result (->> (client/get (str "https://connect.squareup.com/v2/orders/" o)
|
|
{:headers {"Square-Version" "2020-08-12"
|
|
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
|
"Content-Type" "application/json"}
|
|
:as :json})
|
|
:body)]
|
|
|
|
(log/info "found " (count result))
|
|
result)))
|
|
|
|
(defn amount->money [amt]
|
|
(* 0.01 (or (:amount amt) 0.0)))
|
|
|
|
(defn location_id->client-location [location]
|
|
({"2RVBYER6QSV7W" ["NGAK" "MH"]
|
|
"8JT71V8XGYAT3" ["NGKG" "NB"]
|
|
"SCX0Y8CTGM1S0" ["NGE1" "UC"]
|
|
"FNH5VRT890WK8" ["NGMJ" "SC"]
|
|
"AMQ0NPA8FGDEF" ["NGPG" "SZ"]
|
|
"ACNTYY8WVZ6DV" ["NGVZ" "NP"]
|
|
"KMVFQ9CRCXJ10" ["NGZO" "VT"]
|
|
"L0J45VZKHWXVR" ["NGRV" "RV"]
|
|
"L3GMNBFARX9GG" ["NGOP" "OP"]
|
|
"LXJCAHYGZVNEJ" ["NGHG" "DC"]
|
|
"LSWNP14T0YKD9" ["NGWL" "WL"]
|
|
"LT322EK0S2TRD" ["NGGH" "FM"]
|
|
"L0Z167T2T7W7F" ["NGEZ" "JS"]
|
|
"LEV4GM1JPJS6R" ["NGEB" "CV"]
|
|
"LQTHXQY69MYB6" ["NGDG" "DB"]
|
|
"L7S9MXZBJ00HY" ["NGGG" "LM"]
|
|
"LRC7WVD77ZM81" ["NGLK" "SM"]
|
|
"FZ3ZYC77T3W1T" ["NGA1" "KA"]
|
|
"LG5X0MHA4NZDM" ["NGSM" "SM"]} location))
|
|
|
|
;; to get totals:
|
|
(comment
|
|
(reduce
|
|
(fn [total i]
|
|
(+ total (+ (- (:sales-order/total i)
|
|
(:sales-order/tax i)
|
|
(:sales-order/tip i)
|
|
(:sales-order/service-charge i))
|
|
(:sales-order/returns i)
|
|
|
|
(:sales-order/discount i)
|
|
)))
|
|
0.0))
|
|
|
|
(defn order->sales-order [order]
|
|
(let [[client loc] (location_id->client-location (:location_id order))]
|
|
(remove-nils
|
|
#:sales-order
|
|
{:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))
|
|
:client [:client/code client]
|
|
:location loc
|
|
:external-id (str "square/order/" client "-" loc "-" (:id order))
|
|
:total (-> order :net_amounts :total_money amount->money)
|
|
:tax (-> order :net_amounts :tax_money amount->money)
|
|
:tip (-> order :net_amounts :tip_money amount->money)
|
|
:discount (-> order :net_amounts :discount_money amount->money)
|
|
:service-charge (-> order :net_amounts :service_charge_money amount->money)
|
|
:returns (+ (- (-> order :return_amounts :total_money amount->money)
|
|
(-> order :return_amounts :tax_money amount->money)
|
|
(-> order :return_amounts :tip_money amount->money)
|
|
(-> order :return_amounts :service_charge_money amount->money))
|
|
(-> order :return_amounts :discount_money amount->money))
|
|
:charges (->> (:tenders order)
|
|
(map (fn [t]
|
|
(remove-nils
|
|
#:charge
|
|
{:type-name (:type t)
|
|
:processor (condp = (some-> (:note t) str/lower-case)
|
|
"doordash" :ccp-processor/doordash
|
|
"dd" :ccp-processor/doordash
|
|
"ubereats" :ccp-processor/uber-eats
|
|
"ue" :ccp-processor/uber-eats
|
|
"grubhub" :ccp-processor/grubhub
|
|
"grub" :ccp-processor/grubhub
|
|
"gh" :ccp-processor/grubhub
|
|
(condp = (:name (:source order))
|
|
"GRUBHUB" :ccp-processor/grubhub
|
|
"UBEREATS" :ccp-processor/uber-eats
|
|
"DOORDASH" :ccp-processor/doordash
|
|
:ccp-processor/na))
|
|
:total (amount->money (:amount_money t))
|
|
:tip (amount->money (:tip_money t))}))))
|
|
:line-items (->> (:line_items order)
|
|
(map (fn [li]
|
|
(remove-nils
|
|
#:order-line-item
|
|
{:item-name (:name li)
|
|
:category (if (= "GIFT_CARD" (:item_type li))
|
|
"Gift Card"
|
|
(item-id->category-name (:catalog_object_id li)))
|
|
:total (amount->money (:total_money li))
|
|
:tax (amount->money (:total_tax_money li))
|
|
:discount (amount->money (:total_discount_money li))}))))})))
|
|
|
|
(defn daily-results [d]
|
|
(->> (locations)
|
|
(map :id)
|
|
(filter location_id->client-location)
|
|
(mapcat #(search % d))
|
|
(filter (fn [order]
|
|
(and (= "COMPLETED" (:state order))
|
|
(not= #{"FAILED"}
|
|
(set (map #(:status (:card_details %)) (:tenders order)))))))
|
|
(map order->sales-order)))
|
|
|
|
#_(daily-results)
|
|
|
|
|
|
(defn retry
|
|
([f] (retry f 0))
|
|
([f i]
|
|
(if (< i 5)
|
|
(try
|
|
(f)
|
|
(catch Exception e
|
|
(log/warn "error pulling http " e)
|
|
(retry f (inc i))))
|
|
(log/error "Too many failures"))))
|
|
|
|
(defn get-payment [p]
|
|
(:payment (:body (retry #(client/get (str "https://connect.squareup.com/v2/payments/" p)
|
|
{:headers base-headers
|
|
:as :json
|
|
:retry-handler (fn [ex try-count http-context]
|
|
(log/warn "Retrying after failure " ex)
|
|
(if (> try-count 4) false true))})))))
|
|
|
|
(defn get-settlement-sales-date [settlement]
|
|
(let [concurrent 10
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(map (fn [p]
|
|
(lc/with-context {:source "Square settlements loading "}
|
|
(log/info "looking up payment " p " for settlement " (:id settlement))
|
|
(println (or
|
|
(-> p
|
|
get-payment
|
|
:created_at
|
|
coerce/to-date)
|
|
(coerce/to-date (time/now))))
|
|
(or
|
|
(-> p
|
|
get-payment
|
|
:created_at
|
|
coerce/to-date)
|
|
(coerce/to-date (time/now))))))
|
|
(async/to-chan (->> settlement
|
|
:entries
|
|
(filter #(= "CHARGE" (:type %)))
|
|
(map :payment_id)
|
|
(filter identity)
|
|
(set)
|
|
(take 20)
|
|
))
|
|
true
|
|
(fn [e]
|
|
(lc/with-context {:source "Square settlements loading "}
|
|
(log/error "Error loading sales date details" e))))
|
|
(->> (async/<!! (async/into [] output-chan))
|
|
sort
|
|
(drop 2)
|
|
first)))
|
|
|
|
(defn get-settlement-details [settlements] ;; pairs of [location settlement]
|
|
(log/info "getting settlement details for " settlements)
|
|
(let [concurrent 10
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(map (fn [[location s :as g]]
|
|
(lc/with-context {:source "Square settlements loading "}
|
|
(log/info "Looking up settlement " s " for location " location)
|
|
(let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" location "/settlements/" s)
|
|
{:headers base-headers
|
|
:as :json
|
|
:retry-handler (fn [ex try-count http-context]
|
|
(log/warn "Retrying after failure " ex)
|
|
(if (> try-count 4) false true))})))]
|
|
|
|
(log/info "sales date for " s " is " (get-settlement-sales-date settlement))
|
|
(assoc settlement :sales-date (get-settlement-sales-date settlement) )))))
|
|
(async/to-chan settlements)
|
|
true
|
|
(fn [e]
|
|
(lc/with-context {:source "Square settlements loading "}
|
|
(log/error "Error loading settlements details" e))))
|
|
(async/<!! (async/into [] output-chan))))
|
|
|
|
(defn settlements
|
|
([l] (settlements l (lookup-dates)))
|
|
([l lookup-dates]
|
|
(log/info "Searching for" l)
|
|
(->> lookup-dates
|
|
(mapcat (fn [[start-date end-date]]
|
|
(log/info "looking up settlements for " l " on dates " start-date " to " end-date)
|
|
(let [settlements (->> (client/get (str "https://connect.squareup.com/v1/" l "/settlements")
|
|
{:headers base-headers
|
|
:query-params {"begin_time" start-date
|
|
"end_time" end-date}
|
|
:as :json})
|
|
:body
|
|
(map :id))]
|
|
settlements)))
|
|
set
|
|
seq
|
|
(map (fn [s]
|
|
[l s]))
|
|
(get-settlement-details )
|
|
)))
|
|
|
|
(defn daily-settlements
|
|
([] (daily-settlements (->> (locations)
|
|
(map :id)
|
|
(filter location_id->client-location))))
|
|
([location-ids]
|
|
(->> location-ids
|
|
(mapcat (fn [l]
|
|
(for [settlement (settlements l)
|
|
:let [[client loc] (location_id->client-location l)]]
|
|
#:expected-deposit {:external-id (str "square/settlement/" (:id settlement))
|
|
:total (amount->money (:total_money settlement))
|
|
:client [:client/code client]
|
|
:location loc
|
|
:fee (- (reduce + 0.0 (map (fn [entry]
|
|
(if (= (:type entry) "REFUND")
|
|
(- (amount->money (:fee_money entry)))
|
|
(amount->money (:fee_money entry))))
|
|
(:entries settlement))))
|
|
:date (-> (:initiated_at settlement)
|
|
(coerce/to-date))
|
|
:sales-date (or (:sales-date settlement)
|
|
(-> (:initiated_at settlement)
|
|
(coerce/to-date)))})))
|
|
(filter :expected-deposit/date)))
|
|
)
|
|
|
|
(defn refunds [l]
|
|
(let [refunds (:refunds (:body (client/get (str "https://connect.squareup.com/v2/refunds?location_id=" l)
|
|
{:headers {"Square-Version" "2021-05-13"
|
|
"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"
|
|
"Content-Type" "application/json"}
|
|
:as :json
|
|
:retry-handler (fn [ex try-count http-context]
|
|
(log/warn "Retrying after failure " ex)
|
|
(if (> try-count 4) false true))})))]
|
|
(->> refunds
|
|
(filter (fn [r] (= "COMPLETED" (:status r))))
|
|
(map (fn [r]
|
|
(let [[client location] (location_id->client-location l)]
|
|
#:sales-refund {:external-id (str "square/refund/" (:id r))
|
|
:total (amount->money (:amount_money r))
|
|
:fee (transduce
|
|
(comp (filter #(= "ADJUSTMENT" (:type %)))
|
|
(map :amount_money)
|
|
(map amount->money))
|
|
+
|
|
0.0
|
|
(:processing_fee r))
|
|
:client [:client/code client]
|
|
:location location
|
|
:date (coerce/to-date (:created_at r))
|
|
:type (:source_type (get-payment (:payment_id r)))}))))))
|
|
|
|
(defn upsert
|
|
([]
|
|
(upsert nil))
|
|
([d]
|
|
(lc/with-context {:source "Square loading"}
|
|
(try
|
|
(let [existing (->> (d/query {:query {:find ['?external-id]
|
|
:in ['$]
|
|
:where ['[_ :sales-order/external-id ?external-id]]}
|
|
:args [(d/db conn)]})
|
|
(map first)
|
|
set)
|
|
_ (log/info (count existing) "Sales orders already exist")
|
|
to-create (filter #(not (existing (:sales-order/external-id %)))
|
|
(daily-results d))]
|
|
(doseq [x (partition-all 20 to-create)]
|
|
(log/info "Loading " (count x))
|
|
@(d/transact conn x)))
|
|
(catch Exception e
|
|
(log/error e))))))
|
|
|
|
(defn upsert-settlements
|
|
([] (upsert-settlements nil))
|
|
([location-ids]
|
|
(lc/with-context {:source "Square settlements loading "}
|
|
(try
|
|
(let [existing (->> (d/query {:query {:find ['?external-id]
|
|
:in ['$]
|
|
:where ['[_ :expected-deposit/external-id ?external-id]]}
|
|
:args [(d/db conn)]})
|
|
(map first)
|
|
set)
|
|
_ (log/info (count existing) "settlements already exist")
|
|
to-create (filter #(not (existing (:expected-deposit/external-id %)))
|
|
(if location-ids
|
|
(daily-settlements location-ids)
|
|
(daily-settlements)))]
|
|
(doseq [x (partition-all 20 to-create)]
|
|
(log/info "Loading expected deposit" (count x))
|
|
@(d/transact conn x)))
|
|
(catch Exception e
|
|
(log/error e)))
|
|
(log/info "Done loading settlements"))))
|
|
|
|
(defn upsert-refunds []
|
|
(doseq [{location :id} (locations)]
|
|
(when (location_id->client-location location)
|
|
(lc/with-context {:source (str "Square refunds loading for " location)}
|
|
(try
|
|
(let [existing (->> (d/query {:query {:find ['?external-id]
|
|
:in ['$]
|
|
:where ['[_ :sales-refund/external-id ?external-id]]}
|
|
:args [(d/db conn)]})
|
|
(map first)
|
|
set)
|
|
_ (log/info (count existing) "refunds already exist")
|
|
to-create (filter #(not (existing (:sales-refund/external-id %)))
|
|
(refunds location))]
|
|
(doseq [x (partition-all 20 to-create)]
|
|
(log/info "Loading refund" (count x))
|
|
@(d/transact conn x)))
|
|
(catch Exception e
|
|
(log/error e)))
|
|
(log/info "Done loading refunds")))))
|
|
|
|
(defn reset []
|
|
(->>
|
|
(d/query {:query {:find ['?e]
|
|
:in ['$]
|
|
:where ['[?e :sales-order/date]]}
|
|
:args [(d/db conn)]})
|
|
(map first)
|
|
(map (fn [x] [:db/retractEntity x]))))
|
|
|
|
(mount/defstate square-loader
|
|
:start (scheduler/every (* 4 59 60 1000) upsert)
|
|
:stop (scheduler/stop square-loader))
|
|
|
|
(mount/defstate square-settlement-loader
|
|
:start (scheduler/every (* 4 60 60 1000) upsert-settlements)
|
|
:stop (scheduler/stop square-settlement-loader))
|
|
|
|
(mount/defstate square-refund-loader
|
|
:start (scheduler/every (* 4 58 60 1000) upsert-refunds)
|
|
:stop (scheduler/stop square-refund-loader))
|
|
|
|
(comment
|
|
(daily-results)
|
|
(mount/stop (mount/only #{'auto-ap.square.core/square-settlement-loader}))
|
|
(mount/stop (mount/only #{'auto-ap.square.core/square-refund-loader}))
|
|
(mount/stop (mount/only #{'auto-ap.square.core/square-loader}))
|
|
|
|
(mount/start (mount/only #{'auto-ap.square.core/square-settlement-loader}))
|
|
(mount/start (mount/only #{'auto-ap.square.core/square-refund-loader}))
|
|
(mount/start (mount/only #{'auto-ap.square.core/square-loader}))
|
|
(do (upsert) nil)
|
|
|
|
(do @(d/transact conn (reset)) nil))
|
|
|