(ns auto-ap.square.core (:require [auto-ap.datomic :refer [conn remove-nils]] [auto-ap.utils :refer [by]] [clj-http.client :as client] [clj-time.coerce :as coerce] [clj-time.core :as time] [clj-time.format :as f] [clojure.string :as str] [clojure.data.json :as json] [clojure.tools.logging :as log] [datomic.api :as d] [mount.core :as mount] [unilog.context :as lc] [yang.scheduler :as scheduler] [clojure.core.async :as async])) (def base-headers {"Square-Version" "2021-08-18" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"}) (defn lookup-dates [] (->> (clj-time.periodic/periodic-seq (time/plus (time/now) (time/days -15)) (time/now) (time/days 5)) (map (fn [d] [(auto-ap.time/unparse (time/plus d (time/days 1)) auto-ap.time/iso-date) (auto-ap.time/unparse (time/plus d (time/days 5)) auto-ap.time/iso-date)])))) (defn locations [] (->> (client/get "https://connect.squareup.com/v2/locations" {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json}) :body :locations)) (defn fetch-catalog [i] (if i (try (log/info "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i)) (->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i) {:headers base-headers :query-params {"include_related_items" "true"} :as :json}) :body :object) (catch Exception e (log/error e) nil)) (log/warn "Trying to look up non existant "))) (def fetch-catalog-fast (memoize fetch-catalog)) (defn item-id->category-name [i] (let [item (fetch-catalog-fast i)] (cond (:item_variation_data item) (item-id->category-name (:item_id (:item_variation_data item))) (:category_id (:item_data item)) (:name (:category_data (fetch-catalog-fast (:category_id (:item_data item))))) (:item_data item) "Uncategorized" :else (do (log/error "couldn't look up" i) "Uncategorized")))) (defn categories [] (by :id (comp :name :category_data) )) (def potential-query {"query" {"filter" {"date_time_filter" { "created_at" { "start_at" "2020-08-28T00:00:00-07:00", "end_at" "2020-08-28T23:59:59-07:00" } } "state_filter" {"states" ["COMPLETED"]}} "sort" { "sort_field" "CREATED_AT" "sort_order" "DESC" }}}) (defn pc [d] {"query" {"filter" {"date_time_filter" { "created_at" { "start_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))) "end_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (time/plus (coerce/to-date-time d) (time/days 1)) (time/time-zone-for-id "America/Los_Angeles"))) } } "state_filter" {"states" ["COMPLETED"]}} "sort" { "sort_field" "CREATED_AT" "sort_order" "DESC" }}}) (defn search ([l] (search l nil)) ([l d] (log/info "Searching for" l) (let [result (->> (client/post "https://connect.squareup.com/v2/orders/search" {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :body (json/write-str (cond-> {"location_ids" [l] "limit" 10000} d (merge (pc d)))) :as :json}) :body :orders)] (log/info "found " (count result)) result))) (defn order ([o] (log/info "Searching for" o) (let [result (->> (client/get (str "https://connect.squareup.com/v2/orders/" o) {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json}) :body)] (log/info "found " (count result)) result))) (defn amount->money [amt] (* 0.01 (or (:amount amt) 0.0))) (defn location_id->client-location [location] ({"2RVBYER6QSV7W" ["NGAK" "MH"] "8JT71V8XGYAT3" ["NGKG" "NB"] "SCX0Y8CTGM1S0" ["NGE1" "UC"] "FNH5VRT890WK8" ["NGMJ" "SC"] "AMQ0NPA8FGDEF" ["NGPG" "SZ"] "ACNTYY8WVZ6DV" ["NGVZ" "NP"] "KMVFQ9CRCXJ10" ["NGZO" "VT"] "L0J45VZKHWXVR" ["NGRV" "RV"] "L3GMNBFARX9GG" ["NGOP" "OP"] "LXJCAHYGZVNEJ" ["NGHG" "DC"] "LSWNP14T0YKD9" ["NGWL" "WL"] "LT322EK0S2TRD" ["NGGH" "FM"] "L0Z167T2T7W7F" ["NGEZ" "JS"] "LEV4GM1JPJS6R" ["NGEB" "CV"] "LQTHXQY69MYB6" ["NGDG" "DB"] "L7S9MXZBJ00HY" ["NGGG" "LM"] "LRC7WVD77ZM81" ["NGLK" "SM"] "FZ3ZYC77T3W1T" ["NGA1" "KA"] "LG5X0MHA4NZDM" ["NGSM" "SM"]} location)) ;; to get totals: (comment (reduce (fn [total i] (+ total (+ (- (:sales-order/total i) (:sales-order/tax i) (:sales-order/tip i) (:sales-order/service-charge i)) (:sales-order/returns i) (:sales-order/discount i) ))) 0.0)) (defn order->sales-order [order] (let [[client loc] (location_id->client-location (:location_id order))] (remove-nils #:sales-order {:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles"))) :client [:client/code client] :location loc :external-id (str "square/order/" client "-" loc "-" (:id order)) :total (-> order :net_amounts :total_money amount->money) :tax (-> order :net_amounts :tax_money amount->money) :tip (-> order :net_amounts :tip_money amount->money) :discount (-> order :net_amounts :discount_money amount->money) :service-charge (-> order :net_amounts :service_charge_money amount->money) :returns (+ (- (-> order :return_amounts :total_money amount->money) (-> order :return_amounts :tax_money amount->money) (-> order :return_amounts :tip_money amount->money) (-> order :return_amounts :service_charge_money amount->money)) (-> order :return_amounts :discount_money amount->money)) :charges (->> (:tenders order) (map (fn [t] (remove-nils #:charge {:type-name (:type t) :processor (condp = (some-> (:note t) str/lower-case) "doordash" :ccp-processor/doordash "dd" :ccp-processor/doordash "ubereats" :ccp-processor/uber-eats "ue" :ccp-processor/uber-eats "grubhub" :ccp-processor/grubhub "grub" :ccp-processor/grubhub "gh" :ccp-processor/grubhub (condp = (:name (:source order)) "GRUBHUB" :ccp-processor/grubhub "UBEREATS" :ccp-processor/uber-eats "DOORDASH" :ccp-processor/doordash :ccp-processor/na)) :total (amount->money (:amount_money t)) :tip (amount->money (:tip_money t))})))) :line-items (->> (:line_items order) (map (fn [li] (remove-nils #:order-line-item {:item-name (:name li) :category (if (= "GIFT_CARD" (:item_type li)) "Gift Card" (item-id->category-name (:catalog_object_id li))) :total (amount->money (:total_money li)) :tax (amount->money (:total_tax_money li)) :discount (amount->money (:total_discount_money li))}))))}))) (defn daily-results [d] (->> (locations) (map :id) (filter location_id->client-location) (mapcat #(search % d)) (filter (fn [order] (and (= "COMPLETED" (:state order)) (not= #{"FAILED"} (set (map #(:status (:card_details %)) (:tenders order))))))) (map order->sales-order))) #_(daily-results) (defn retry ([f] (retry f 0)) ([f i] (if (< i 5) (try (f) (catch Exception e (log/warn "error pulling http " e) (retry f (inc i)))) (log/error "Too many failures")))) (defn get-payment [p] (:payment (:body (retry #(client/get (str "https://connect.squareup.com/v2/payments/" p) {:headers base-headers :as :json :retry-handler (fn [ex try-count http-context] (log/warn "Retrying after failure " ex) (if (> try-count 4) false true))}))))) (defn get-settlement-sales-date [settlement] (let [concurrent 10 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan (map (fn [p] (lc/with-context {:source "Square settlements loading "} (log/info "looking up payment " p " for settlement " (:id settlement)) (println (or (-> p get-payment :created_at coerce/to-date) (coerce/to-date (time/now)))) (or (-> p get-payment :created_at coerce/to-date) (coerce/to-date (time/now)))))) (async/to-chan (->> settlement :entries (filter #(= "CHARGE" (:type %))) (map :payment_id) (filter identity) (set) (take 20) )) true (fn [e] (lc/with-context {:source "Square settlements loading "} (log/error "Error loading sales date details" e)))) (->> (async/ try-count 4) false true))})))] (log/info "sales date for " s " is " (get-settlement-sales-date settlement)) (assoc settlement :sales-date (get-settlement-sales-date settlement) ))))) (async/to-chan settlements) true (fn [e] (lc/with-context {:source "Square settlements loading "} (log/error "Error loading settlements details" e)))) (async/> lookup-dates (mapcat (fn [[start-date end-date]] (log/info "looking up settlements for " l " on dates " start-date " to " end-date) (let [settlements (->> (client/get (str "https://connect.squareup.com/v1/" l "/settlements") {:headers base-headers :query-params {"begin_time" start-date "end_time" end-date} :as :json}) :body (map :id))] settlements))) set seq (map (fn [s] [l s])) (get-settlement-details ) ))) (defn daily-settlements ([] (daily-settlements (->> (locations) (map :id) (filter location_id->client-location)))) ([location-ids] (->> location-ids (mapcat (fn [l] (for [settlement (settlements l) :let [[client loc] (location_id->client-location l)]] #:expected-deposit {:external-id (str "square/settlement/" (:id settlement)) :total (amount->money (:total_money settlement)) :client [:client/code client] :location loc :fee (- (reduce + 0.0 (map (fn [entry] (if (= (:type entry) "REFUND") (- (amount->money (:fee_money entry))) (amount->money (:fee_money entry)))) (:entries settlement)))) :date (-> (:initiated_at settlement) (coerce/to-date)) :sales-date (or (:sales-date settlement) (-> (:initiated_at settlement) (coerce/to-date)))}))) (filter :expected-deposit/date))) ) (defn refunds [l] (let [refunds (:refunds (:body (client/get (str "https://connect.squareup.com/v2/refunds?location_id=" l) {:headers {"Square-Version" "2021-05-13" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json :retry-handler (fn [ex try-count http-context] (log/warn "Retrying after failure " ex) (if (> try-count 4) false true))})))] (->> refunds (filter (fn [r] (= "COMPLETED" (:status r)))) (map (fn [r] (let [[client location] (location_id->client-location l)] #:sales-refund {:external-id (str "square/refund/" (:id r)) :total (amount->money (:amount_money r)) :fee (transduce (comp (filter #(= "ADJUSTMENT" (:type %))) (map :amount_money) (map amount->money)) + 0.0 (:processing_fee r)) :client [:client/code client] :location location :date (coerce/to-date (:created_at r)) :type (:source_type (get-payment (:payment_id r)))})))))) (defn upsert ([] (upsert nil)) ([d] (lc/with-context {:source "Square loading"} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :sales-order/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "Sales orders already exist") to-create (filter #(not (existing (:sales-order/external-id %))) (daily-results d))] (doseq [x (partition-all 20 to-create)] (log/info "Loading " (count x)) @(d/transact conn x))) (catch Exception e (log/error e)))))) (defn upsert-settlements ([] (upsert-settlements nil)) ([location-ids] (lc/with-context {:source "Square settlements loading "} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :expected-deposit/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "settlements already exist") to-create (filter #(not (existing (:expected-deposit/external-id %))) (if location-ids (daily-settlements location-ids) (daily-settlements)))] (doseq [x (partition-all 20 to-create)] (log/info "Loading expected deposit" (count x)) @(d/transact conn x))) (catch Exception e (log/error e))) (log/info "Done loading settlements")))) (defn upsert-refunds [] (doseq [{location :id} (locations)] (when (location_id->client-location location) (lc/with-context {:source (str "Square refunds loading for " location)} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :sales-refund/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "refunds already exist") to-create (filter #(not (existing (:sales-refund/external-id %))) (refunds location))] (doseq [x (partition-all 20 to-create)] (log/info "Loading refund" (count x)) @(d/transact conn x))) (catch Exception e (log/error e))) (log/info "Done loading refunds"))))) (defn reset [] (->> (d/query {:query {:find ['?e] :in ['$] :where ['[?e :sales-order/date]]} :args [(d/db conn)]}) (map first) (map (fn [x] [:db/retractEntity x])))) (mount/defstate square-loader :start (scheduler/every (* 4 59 60 1000) upsert) :stop (scheduler/stop square-loader)) (mount/defstate square-settlement-loader :start (scheduler/every (* 4 60 60 1000) upsert-settlements) :stop (scheduler/stop square-settlement-loader)) (mount/defstate square-refund-loader :start (scheduler/every (* 4 58 60 1000) upsert-refunds) :stop (scheduler/stop square-refund-loader)) (comment (daily-results) (mount/stop (mount/only #{'auto-ap.square.core/square-settlement-loader})) (mount/stop (mount/only #{'auto-ap.square.core/square-refund-loader})) (mount/stop (mount/only #{'auto-ap.square.core/square-loader})) (mount/start (mount/only #{'auto-ap.square.core/square-settlement-loader})) (mount/start (mount/only #{'auto-ap.square.core/square-refund-loader})) (mount/start (mount/only #{'auto-ap.square.core/square-loader})) (do (upsert) nil) (do @(d/transact conn (reset)) nil))