diff --git a/scratch-sessions/payouts.repl b/scratch-sessions/payouts.repl index 2967f042..b54086c1 100644 --- a/scratch-sessions/payouts.repl +++ b/scratch-sessions/payouts.repl @@ -10,9 +10,7 @@ (def c (first (get-square-clients "NGOP"))) -(def l {:db/id 17592186051554, - :square-location/name "Almaden", - :square-location/square-id "68MJ8J2RX7KE1"}) +(def l (first (filter :square-location/client-location (:client/square-locations c)))) (def base @(settlements c l)) (def base-result @(daily-settlements c l)) @@ -24,7 +22,8 @@ base-result @(de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/payouts/?" (url/map->query - {:location_id (:square-location/square-id l)}) ) + {:location_id (:square-location/square-id l) + :begin_time "2023-07-22T00:00:00Z"}) ) :method :get :headers (client-base-headers c "2023-04-19") }) @@ -50,7 +49,7 @@ base-result (s/buffer 10) (s/realize-each) (s/reduce conj []))) - #_(fn [settlements] + (fn [settlements] (log/info ::transforming-settlements) (try (->> (for [settlement settlements @@ -71,7 +70,26 @@ base-result first coerce/to-date-time atime/as-local-time - coerce/to-date)]] + coerce/to-date) + + ;; TODO delete this - this is only needed during the short transformation time + equivalent-already-exists? (seq (dc/q '[:find ?s + :in $ ?c ?a + :where + [?s :expected-deposit/date ?ed] + [(>= ?ed #inst "2023-07-10")] + [(<= ?ed #inst "2023-07-26")] + [?s :expected-deposit/client ?c] + [?s :expected-deposit/external-id ?eid] + [(clojure.string/includes? ?eid "settlement")] + [?s :expected-deposit/total ?t] + [(iol-ion.query/dollars= ?t ?a)] + ] + (dc/db conn) + (:db/id c) + (amount->money (:amount_money settlement)) + ))] + :when (not equivalent-already-exists?)] #:expected-deposit {:external-id (str "square/payout/" (:id settlement)) :vendor :vendor/ccp-square :status :expected-deposit-status/pending diff --git a/src/clj/auto_ap/jobs/load_historical_sales.clj b/src/clj/auto_ap/jobs/load_historical_sales.clj index c6134e90..1ff7c445 100644 --- a/src/clj/auto_ap/jobs/load_historical_sales.clj +++ b/src/clj/auto_ap/jobs/load_historical_sales.clj @@ -65,18 +65,7 @@ (println "refunds") @(square3/upsert-refunds client square-location) - - - #_(with-redefs [square3/lookup-dates (fn lookup-dates [] - (->> (per/periodic-seq (time/plus (time/today) (time/days (- days))) - (time/today) - (time/days 2)) - (map (fn [d] - [(atime/unparse (time/plus d (time/days 1)) atime/iso-date) - - (atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))] - - @(square3/upsert-settlements client square-location))))) + @(square3/upsert-payouts client square-location (time/plus (time/now) (time/days (- days))) (time/now))))) (defn load-historical-sales [args] diff --git a/src/clj/auto_ap/square/core3.clj b/src/clj/auto_ap/square/core3.clj index bcac61b4..8b3a79fd 100644 --- a/src/clj/auto_ap/square/core3.clj +++ b/src/clj/auto_ap/square/core3.clj @@ -86,15 +86,6 @@ (when r response-deferred))))) -(defn lookup-dates [] - (->> (periodic/periodic-seq (time/plus (time/now) (time/days -15)) - (time/now) - (time/days 5)) - (map (fn [d] - [(atime/unparse (time/plus d (time/days 1)) atime/iso-date) - - (atime/unparse (time/plus d (time/days 5)) atime/iso-date)])))) - (defn client-locations [client] (capture-context->lc (de/catch @@ -423,110 +414,124 @@ :body :payment)) -(defn settlements - ([client location] (settlements client location (lookup-dates))) - ([client location lookup-dates] +(defn payouts + ([client location] (payouts client location (time/plus (time/now) (time/days -7)) (time/now))) + ([client location start end] (with-context-as {:location (:square-location/client-location location)} lc - (de/chain + (de/chain (manifold-api-call + {:url (str "https://connect.squareup.com/v2/payouts/?" + (url/map->query + {:location_id (:square-location/square-id location) + :begin_time (->square-date start) + :end_time (->square-date end)}) ) + :method :get + :headers (client-base-headers client "2023-04-19") + }) + :body + :payouts + (fn [payouts] + (->> payouts + (s/->source) + (s/map (fn [payout] + (mu/with-context lc + (log/info ::looking-up-payout + :payout-id (:id payout)) + (de/chain + (manifold-api-call + {:url (str "https://connect.squareup.com/v2/payouts/" (:id payout) "/payout-entries") + :method :get + :headers (client-base-headers client "2023-04-19") + :as :json}) + :body + :payout_entries + (fn [b] + (assoc payout :payout_entries b)))))) + (s/buffer 10) + (s/realize-each) + (s/reduce conj []))))))) - (->> lookup-dates - (s/->source) - (s/map (fn [[start-date end-date]] - (mu/with-context lc - (log/info ::searching-settlements - :start-date start-date - :end-date end-date) - (de/chain (manifold-api-call - {:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements") - :method :get - :headers (client-base-headers client) - :query-params {"begin_time" start-date - "end_time" end-date}}) - :body - (fn [settlements] - (map :id settlements)))))) - (s/buffer 3) - (s/realize-each) - (s/map (fn [settlement-set] - settlement-set)) - (s/reduce into #{})) - (fn [settlement-id-set] - (->> settlement-id-set - (s/->source) - (s/map (fn [settlement-id] - (mu/with-context lc - (log/info ::looking-up-settlement - :settlement-id settlement-id) - (de/chain - (manifold-api-call - {:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" settlement-id) - :method :get - :headers (client-base-headers client) - :as :json}) - :body)))) - (s/buffer 10) - (s/realize-each) - (s/reduce conj []))))))) - -(defn daily-settlements +(defn transformed-payouts ([client location] - (daily-settlements client location (settlements client location))) - ([client location settlements] - (de/chain settlements - (fn [settlements] - (log/info ::transforming-settlements) - (try - (->> (for [settlement settlements - :let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s) - :in $ ?settlement-id - :where - [?settlement :expected-deposit/external-id ?settlement-id] - [?settlement :expected-deposit/charges ?c] - [?s :sales-order/charges ?c] - [?s :sales-order/date ?sales-date] - [(clj-time.coerce/to-date-time ?sales-date) ?s2] - [(auto-ap.time/localize ?s2) ?s3] - [(clj-time.coerce/to-local-date ?s3) ?s4]] - (dc/db conn) - (str "square/settlement/" (:id settlement))) - (sort-by last) - last - first - coerce/to-date-time - atime/as-local-time - coerce/to-date)]] - #:expected-deposit {:external-id (str "square/settlement/" (:id settlement)) - :vendor :vendor/ccp-square - :status :expected-deposit-status/pending - :total (amount->money (:total_money settlement)) - :client (:db/id client) - :location (:square-location/client-location location) - :fee (- (reduce + 0.0 (map (fn [entry] - (if (= (:type entry) "REFUND") - (- (amount->money (:fee_money entry))) - (amount->money (:fee_money entry)))) - (:entries settlement)))) - :date (-> (:initiated_at settlement) - (coerce/to-date)) - :sales-date (or best-sales-date - (-> (:initiated_at settlement) - (coerce/to-date))) - :charges (->> (:entries settlement) - (filter :payment_id) - (map (fn [p] {:charge/external-id (str "square/charge/" (:payment_id p))})))}) - (filter :expected-deposit/date) - (into [])) - (catch Throwable e - (log/error ::transform-settlement-failed - :exception e))))))) + (transformed-payouts client location (payouts client location))) + ([client location start end] + (transformed-payouts client location (payouts client location start end))) + ([client location payouts] + (with-context-as {:location (:square-location/client-location location)} lc + (de/chain payouts + (fn [payouts] + (mu/with-context lc + (log/info ::transforming-payouts) + (try + (->> (for [payout payouts + :let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s) + :in $ ?payout-id + :where + [?payout :expected-deposit/external-id ?payout-id] + [?payout :expected-deposit/charges ?c] + [?s :sales-order/charges ?c] + [?s :sales-order/date ?sales-date] + [(clj-time.coerce/to-date-time ?sales-date) ?s2] + [(auto-ap.time/localize ?s2) ?s3] + [(clj-time.coerce/to-local-date ?s3) ?s4]] + (dc/db conn) + (str "square/payout/" (:id payout))) + (sort-by last) + last + first + coerce/to-date-time + atime/as-local-time + coerce/to-date) + + ;; TODO delete this - this is only needed during the short transformation time + equivalent-already-exists? (seq (dc/q '[:find ?s + :in $ ?c ?a + :where + [?s :expected-deposit/date ?ed] + [(>= ?ed #inst "2023-07-10")] + [(<= ?ed #inst "2023-07-26")] + [?s :expected-deposit/client ?c] + [?s :expected-deposit/external-id ?eid] + [(clojure.string/includes? ?eid "settlement")] + [?s :expected-deposit/total ?t] + [(iol-ion.query/dollars= ?t ?a)] + ] + (dc/db conn) + (:db/id client) + (amount->money (:amount_money payout)) + ))] + :when (not equivalent-already-exists?)] + #:expected-deposit {:external-id (str "square/payout/" (:id payout)) + :vendor :vendor/ccp-square + :status :expected-deposit-status/pending + :total (amount->money (:amount_money payout)) + :client (:db/id client) + :location (:square-location/client-location location) + :fee (- (reduce + 0.0 (map (fn [entry] + (if (= (:type entry) "REFUND") + (- (amount->money (:fee_amount_money entry))) + (amount->money (:fee_amount_money entry)))) + (:payout_entries payout)))) + :date (-> (:created_at payout) + (coerce/to-date)) + :sales-date (or best-sales-date + (-> (:created_at payout) + (coerce/to-date))) + :charges (reverse (->> (:payout_entries payout) + (filter (comp :payment_id :type_charge_details)) + (map (fn [p] {:charge/external-id (str "square/charge/" (:payment_id (:type_charge_details p)))}))))}) + (filter :expected-deposit/date) + (into [])) + (catch Throwable e + (log/error ::transform-payout-failed + :exception e))))))))) (defn refunds ([client l] - (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/refunds?location_id=" (:square-location/square-id l)) + (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/refunds?location_id=" (:square-location/square-id l)) :method :get - :headers (client-base-headers client) - :as :json}) + :headers (client-base-headers client) + :as :json}) :body :refunds (fn [refunds] @@ -571,21 +576,23 @@ @(dc/transact-async conn x)))))))) -(defn upsert-settlements +(defn upsert-payouts ([client] (apply de/zip (for [square-location (:client/square-locations client) :when (:square-location/client-location square-location)] - (upsert-settlements client square-location)))) + (upsert-payouts client square-location)))) ([client location] - (with-context-as {:source "Square settlements loading" + (upsert-payouts client location (time/plus (time/now) (time/days -7)) (time/now))) + ([client location start end] + (with-context-as {:source "Square payout loading" :client (:client/code client)} lc - (de/chain (daily-settlements client location) - (fn [settlements] + (de/chain (transformed-payouts client location start end) + (fn [payouts] (mu/with-context lc - (log/info ::started-loading-deposits :settlement-count (count settlements)) - (doseq [x (partition-all 20 settlements)] + (log/info ::started-loading-deposits :payout-count (count payouts)) + (doseq [x (partition-all 20 payouts)] (log/info ::loading-deposits :count (count x)) @(dc/transact-async conn x)) @@ -780,10 +787,10 @@ (mu/with-context lc (log/info ::upsert-orders-started) (upsert client))) - #_(fn [_] ;; settlements are currently disabled (v1) + (fn [_] (mu/with-context lc - (log/info ::upsert-settlements-started) - (upsert-settlements client))) + (log/info ::upsert-payouts-started) + (upsert-payouts client))) (fn [_] (mu/with-context lc (log/info ::upsert-refunds-started)