(ns auto-ap.square.core (:require [auto-ap.datomic :refer [conn remove-nils]] [auto-ap.utils :refer [by]] [clj-http.client :as client] [clj-time.coerce :as coerce] [clj-time.core :as time] [clj-time.format :as f] [clojure.data.json :as json] [clojure.tools.logging :as log] [datomic.api :as d] [mount.core :as mount] [unilog.context :as lc] [yang.scheduler :as scheduler])) (defn locations [] (->> (client/get "https://connect.squareup.com/v2/locations" {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json}) :body :locations)) (defn fetch-catalog [i] (if i (try (log/info "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i)) (->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i) {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :query-params {"include_related_items" "true"} :as :json}) :body :object) (catch Exception e (log/error e) nil)) (log/warn "Trying to look up non existant "))) (def fetch-catalog-fast (memoize fetch-catalog)) (defn item-id->category-name [i] (let [item (fetch-catalog-fast i)] (cond (:item_variation_data item) (item-id->category-name (:item_id (:item_variation_data item))) (:category_id (:item_data item)) (:name (:category_data (fetch-catalog-fast (:category_id (:item_data item))))) (:item_data item) "Uncategorized" :else (do (log/error "couldn't look up" i) "Uncategorized")))) (defn categories [] (by :id (comp :name :category_data) )) (def potential-query {"query" {"filter" {"date_time_filter" { "created_at" { "start_at" "2020-08-28T00:00:00-07:00", "end_at" "2020-08-28T23:59:59-07:00" } } "state_filter" {"states" ["COMPLETED"]}} "sort" { "sort_field" "CREATED_AT" "sort_order" "DESC" }}}) (defn pc [d] {"query" {"filter" {"date_time_filter" { "created_at" { "start_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))) "end_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (time/plus (coerce/to-date-time d) (time/days 1)) (time/time-zone-for-id "America/Los_Angeles"))) } } "state_filter" {"states" ["COMPLETED"]}} "sort" { "sort_field" "CREATED_AT" "sort_order" "DESC" }}}) (defn search ([l] (search l nil)) ([l d] (log/info "Searching for" l) (let [result (->> (client/post "https://connect.squareup.com/v2/orders/search" {:headers {"Square-Version" "2020-08-12" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :body (json/write-str (cond-> {"location_ids" [l] "limit" 10000} d (merge (pc d)))) :as :json}) :body :orders)] (println (cond-> {"location_ids" [l] "limit" 4000} d (merge (pc d)))) (log/info "found " (count result)) result))) (defn amount->money [amt] (* 0.01 (or (:amount amt) 0.0))) (defn location_id->client-location [location] ({"2RVBYER6QSV7W" ["NGAK" "MH"] "8JT71V8XGYAT3" ["NGKG" "NB"] "SCX0Y8CTGM1S0" ["NGE1" "UC"] "FNH5VRT890WK8" ["NGMJ" "SC"] "AMQ0NPA8FGDEF" ["NGPG" "SZ"] "ACNTYY8WVZ6DV" ["NGVZ" "NP"] "KMVFQ9CRCXJ10" ["NGZO" "VT"] "L0J45VZKHWXVR" ["NGRV" "RV"]} location)) ;; to get totals: (comment (reduce (fn [total i] (+ total (+ (- (:sales-order/total i) (:sales-order/tax i) (:sales-order/tip i) (:sales-order/service-charge i)) (:sales-order/returns i) (:sales-order/discount i) ))) 0.0)) (defn daily-results [d] (->> (locations) (map :id) (filter location_id->client-location) (mapcat #(search % d)) (filter (fn [order] (not= #{"FAILED"} (set (map #(:status (:card_details %)) (:tenders order)))))) (map (fn [order] (let [[client loc] (location_id->client-location (:location_id order))] (remove-nils #:sales-order {:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles"))) :client [:client/code client] :location loc :external-id (str "square/order/" client "-" loc "-" (:id order)) :total (-> order :net_amounts :total_money amount->money) :tax (-> order :net_amounts :tax_money amount->money) :tip (-> order :net_amounts :tip_money amount->money) :discount (-> order :net_amounts :discount_money amount->money) :service-charge (-> order :net_amounts :service_charge_money amount->money) :returns (+ (- (-> order :return_amounts :total_money amount->money) (-> order :return_amounts :tax_money amount->money) (-> order :return_amounts :tip_money amount->money) (-> order :return_amounts :service_charge_money amount->money) (-> order :return_amounts :discount_money amount->money))) :charges (->> (:tenders order) (map (fn [t] (remove-nils #:charge {:type-name (:type t) :processor (condp = (:note t) "DOORDASH" :ccp-processor/doordash "UBEREATS" :ccp-processor/uber-eats "GRUBHUB" :ccp-processor/grubhub :ccp-processor/na) :total (amount->money (:amount_money t)) :tip (amount->money (:tip_money t))})))) :line-items (->> (:line_items order) (map (fn [li] (remove-nils #:order-line-item {:item-name (:name li) :category (item-id->category-name (:catalog_object_id li)) :total (amount->money (:total_money li)) :tax (amount->money (:total_tax_money li)) :discount (amount->money (:total_discount_money li))}))))})))))) #_(daily-results) (defn retry ([f] (retry f 0)) ([f i] (if (< i 5) (try (f) (catch Exception e (log/warn "error pulling http " e) (retry f (inc i)))) (log/error "Too many failures")))) (defn settlements [l] (log/info "Searching for" l) (let [settlements (->> (client/get (str "https://connect.squareup.com/v1/" l "/settlements") {:headers {"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json}) :body (map :id))] (loop [[s & xs] (take 5 settlements) result []] (log/info "Looking up settlement " s " for location " l) (let [n (:body (retry #(client/get (str "https://connect.squareup.com/v1/" l "/settlements/" s) {:headers {"Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json :retry-handler (fn [ex try-count http-context] (log/warn "Retrying after failure " ex) (if (> try-count 4) false true))})))] (if (seq xs) (recur xs (conj result n)) (conj result n)))))) (defn daily-settlements [] (->> (locations) (map :id) (filter location_id->client-location) (mapcat (fn [l] (for [settlement (settlements l) :let [[client loc] (location_id->client-location l)]] #:expected-deposit {:external-id (str "square/settlement/" (:id settlement)) :total (amount->money (:total_money settlement)) :client [:client/code client] :location loc :fee (- (reduce + 0 (map (fn [entry] (if (= (:type entry) "REFUND") (- (amount->money (:fee_money entry))) (amount->money (:fee_money entry)))) (:entries settlement)))) :date (-> (:initiated_at settlement) (coerce/to-date))}))))) (defn refunds [l] (let [refunds (:refunds (:body (client/get (str "https://connect.squareup.com/v2/refunds?location_id=" l) {:headers {"Square-Version" "2021-05-13" "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" "Content-Type" "application/json"} :as :json :retry-handler (fn [ex try-count http-context] (log/warn "Retrying after failure " ex) (if (> try-count 4) false true))})))] (->> refunds (map (fn [r] (let [[client location] (location_id->client-location l)] #:sales-refund {:external-id (str "square/refund/" (:id r)) :total (amount->money (:amount_money r)) :fee (transduce (comp (filter #(= "ADJUSTMENT" (:type %))) (map :amount_money) (map amount->money)) + 0.0 (:processing_fee r)) :client [:client/code client] :location location :date (coerce/to-date (:created_at r)) })))))) (defn upsert ([] (upsert nil)) ([d] (lc/with-context {:source "Square loading"} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :sales-order/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "Sales orders already exist") to-create (filter #(not (existing (:sales-order/external-id %))) (daily-results d))] (doseq [x (partition-all 20 to-create)] (log/info "Loading " (count x)) @(d/transact conn x))) (catch Exception e (log/error e)))))) (defn upsert-settlements [] (lc/with-context {:source "Square settlements loading "} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :expected-deposit/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "settlements already exist") to-create (filter #(not (existing (:expected-deposit/external-id %))) (daily-settlements))] (doseq [x (partition-all 20 to-create)] (log/info "Loading expected deposit" (count x)) @(d/transact conn x))) (catch Exception e (log/error e))) (log/info "Done loading settlements"))) (defn upsert-refunds [] (doseq [{location :id} (locations)] (when (location_id->client-location location) (lc/with-context {:source (str "Square refunds loading for " location)} (try (let [existing (->> (d/query {:query {:find ['?external-id] :in ['$] :where ['[_ :sales-refund/external-id ?external-id]]} :args [(d/db conn)]}) (map first) set) _ (log/info (count existing) "refunds already exist") to-create (filter #(not (existing (:sales-refund/external-id %))) (refunds location))] (doseq [x (partition-all 20 to-create)] (log/info "Loading refund" (count x)) @(d/transact conn x))) (catch Exception e (log/error e))) (log/info "Done loading refunds"))))) (defn reset [] (->> (d/query {:query {:find ['?e] :in ['$] :where ['[?e :sales-order/date]]} :args [(d/db conn)]}) (map first) (map (fn [x] [:db/retractEntity x])))) (mount/defstate square-loader :start (scheduler/every (* 13 60 1000) upsert) :stop (scheduler/stop square-loader)) (mount/defstate square-settlement-loader :start (scheduler/every (* 14 60 1000) upsert-settlements) :stop (scheduler/stop square-settlement-loader)) (mount/defstate square-refund-loader :start (scheduler/every (* 30 60 1000) upsert-refunds) :stop (scheduler/stop square-refund-loader)) (comment (daily-results) (do (upsert) nil) (do @(d/transact conn (reset)) nil))