notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

replaces settlement loading with payout loading.

Browse Source
This commit is contained in:
Bryce
2023-07-25 22:02:36 -07:00
parent 8f5caf9a81
commit 90014fbfcd
3 changed files with 147 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
scratch-sessions/payouts.repl
View File

@@ -10,9 +10,7 @@
(def c (def c
(first (get-square-clients "NGOP"))) (first (get-square-clients "NGOP")))
(def l {:db/id 17592186051554, (def l (first (filter :square-location/client-location (:client/square-locations c))))
:square-location/name "Almaden",
:square-location/square-id "68MJ8J2RX7KE1"})
(def base @(settlements c l)) (def base @(settlements c l))
(def base-result @(daily-settlements c l)) (def base-result @(daily-settlements c l))
@@ -24,7 +22,8 @@ base-result
@(de/chain (manifold-api-call @(de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v2/payouts/?" {:url (str "https://connect.squareup.com/v2/payouts/?"
(url/map->query (url/map->query
{:location_id (:square-location/square-id l)}) ) {:location_id (:square-location/square-id l)
:begin_time "2023-07-22T00:00:00Z"}) )
:method :get :method :get
:headers (client-base-headers c "2023-04-19") :headers (client-base-headers c "2023-04-19")
}) })
@@ -50,7 +49,7 @@ base-result
(s/buffer 10) (s/buffer 10)
(s/realize-each) (s/realize-each)
(s/reduce conj []))) (s/reduce conj [])))
#_(fn [settlements] (fn [settlements]
(log/info ::transforming-settlements) (log/info ::transforming-settlements)
(try (try
(->> (for [settlement settlements (->> (for [settlement settlements
@@ -71,7 +70,26 @@ base-result
first first
coerce/to-date-time coerce/to-date-time
atime/as-local-time atime/as-local-time
coerce/to-date)]] coerce/to-date)
;; TODO delete this - this is only needed during the short transformation time
equivalent-already-exists? (seq (dc/q '[:find ?s
:in $ ?c ?a
:where
[?s :expected-deposit/date ?ed]
[(>= ?ed #inst "2023-07-10")]
[(<= ?ed #inst "2023-07-26")]
[?s :expected-deposit/client ?c]
[?s :expected-deposit/external-id ?eid]
[(clojure.string/includes? ?eid "settlement")]
[?s :expected-deposit/total ?t]
[(iol-ion.query/dollars= ?t ?a)]
]
(dc/db conn)
(:db/id c)
(amount->money (:amount_money settlement))
))]
:when (not equivalent-already-exists?)]
#:expected-deposit {:external-id (str "square/payout/" (:id settlement)) #:expected-deposit {:external-id (str "square/payout/" (:id settlement))
:vendor :vendor/ccp-square :vendor :vendor/ccp-square
:status :expected-deposit-status/pending :status :expected-deposit-status/pending

13
src/clj/auto_ap/jobs/load_historical_sales.clj
View File

@@ -65,18 +65,7 @@
(println "refunds") (println "refunds")
@(square3/upsert-refunds client square-location) @(square3/upsert-refunds client square-location)
@(square3/upsert-payouts client square-location (time/plus (time/now) (time/days (- days))) (time/now)))))
#_(with-redefs [square3/lookup-dates (fn lookup-dates []
(->> (per/periodic-seq (time/plus (time/today) (time/days (- days)))
(time/today)
(time/days 2))
(map (fn [d]
[(atime/unparse (time/plus d (time/days 1)) atime/iso-date)
(atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))]
@(square3/upsert-settlements client square-location)))))
(defn load-historical-sales [args] (defn load-historical-sales [args]

237
src/clj/auto_ap/square/core3.clj
View File

@@ -86,15 +86,6 @@
(when r (when r
response-deferred))))) response-deferred)))))
(defn lookup-dates []
(->> (periodic/periodic-seq (time/plus (time/now) (time/days -15))
(time/now)
(time/days 5))
(map (fn [d]
[(atime/unparse (time/plus d (time/days 1)) atime/iso-date)
(atime/unparse (time/plus d (time/days 5)) atime/iso-date)]))))
(defn client-locations [client] (defn client-locations [client]
(capture-context->lc (capture-context->lc
(de/catch (de/catch
@@ -423,110 +414,124 @@
:body :body
:payment)) :payment))
(defn settlements (defn payouts
([client location] (settlements client location (lookup-dates))) ([client location] (payouts client location (time/plus (time/now) (time/days -7)) (time/now)))
([client location lookup-dates] ([client location start end]
(with-context-as {:location (:square-location/client-location location)} lc (with-context-as {:location (:square-location/client-location location)} lc
(de/chain (de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v2/payouts/?"
(url/map->query
{:location_id (:square-location/square-id location)
:begin_time (->square-date start)
:end_time (->square-date end)}) )
:method :get
:headers (client-base-headers client "2023-04-19")
})
:body
:payouts
(fn [payouts]
(->> payouts
(s/->source)
(s/map (fn [payout]
(mu/with-context lc
(log/info ::looking-up-payout
:payout-id (:id payout))
(de/chain
(manifold-api-call
{:url (str "https://connect.squareup.com/v2/payouts/" (:id payout) "/payout-entries")
:method :get
:headers (client-base-headers client "2023-04-19")
:as :json})
:body
:payout_entries
(fn [b]
(assoc payout :payout_entries b))))))
(s/buffer 10)
(s/realize-each)
(s/reduce conj [])))))))
(->> lookup-dates (defn transformed-payouts
(s/->source)
(s/map (fn [[start-date end-date]]
(mu/with-context lc
(log/info ::searching-settlements
:start-date start-date
:end-date end-date)
(de/chain (manifold-api-call
{:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements")
:method :get
:headers (client-base-headers client)
:query-params {"begin_time" start-date
"end_time" end-date}})
:body
(fn [settlements]
(map :id settlements))))))
(s/buffer 3)
(s/realize-each)
(s/map (fn [settlement-set]
settlement-set))
(s/reduce into #{}))
(fn [settlement-id-set]
(->> settlement-id-set
(s/->source)
(s/map (fn [settlement-id]
(mu/with-context lc
(log/info ::looking-up-settlement
:settlement-id settlement-id)
(de/chain
(manifold-api-call
{:url (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" settlement-id)
:method :get
:headers (client-base-headers client)
:as :json})
:body))))
(s/buffer 10)
(s/realize-each)
(s/reduce conj [])))))))
(defn daily-settlements
([client location] ([client location]
(daily-settlements client location (settlements client location))) (transformed-payouts client location (payouts client location)))
([client location settlements] ([client location start end]
(de/chain settlements (transformed-payouts client location (payouts client location start end)))
(fn [settlements] ([client location payouts]
(log/info ::transforming-settlements) (with-context-as {:location (:square-location/client-location location)} lc
(try (de/chain payouts
(->> (for [settlement settlements (fn [payouts]
:let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s) (mu/with-context lc
:in $ ?settlement-id (log/info ::transforming-payouts)
:where (try
[?settlement :expected-deposit/external-id ?settlement-id] (->> (for [payout payouts
[?settlement :expected-deposit/charges ?c] :let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s)
[?s :sales-order/charges ?c] :in $ ?payout-id
[?s :sales-order/date ?sales-date] :where
[(clj-time.coerce/to-date-time ?sales-date) ?s2] [?payout :expected-deposit/external-id ?payout-id]
[(auto-ap.time/localize ?s2) ?s3] [?payout :expected-deposit/charges ?c]
[(clj-time.coerce/to-local-date ?s3) ?s4]] [?s :sales-order/charges ?c]
(dc/db conn) [?s :sales-order/date ?sales-date]
(str "square/settlement/" (:id settlement))) [(clj-time.coerce/to-date-time ?sales-date) ?s2]
(sort-by last) [(auto-ap.time/localize ?s2) ?s3]
last [(clj-time.coerce/to-local-date ?s3) ?s4]]
first (dc/db conn)
coerce/to-date-time (str "square/payout/" (:id payout)))
atime/as-local-time (sort-by last)
coerce/to-date)]] last
#:expected-deposit {:external-id (str "square/settlement/" (:id settlement)) first
:vendor :vendor/ccp-square coerce/to-date-time
:status :expected-deposit-status/pending atime/as-local-time
:total (amount->money (:total_money settlement)) coerce/to-date)
:client (:db/id client)
:location (:square-location/client-location location) ;; TODO delete this - this is only needed during the short transformation time
:fee (- (reduce + 0.0 (map (fn [entry] equivalent-already-exists? (seq (dc/q '[:find ?s
(if (= (:type entry) "REFUND") :in $ ?c ?a
(- (amount->money (:fee_money entry))) :where
(amount->money (:fee_money entry)))) [?s :expected-deposit/date ?ed]
(:entries settlement)))) [(>= ?ed #inst "2023-07-10")]
:date (-> (:initiated_at settlement) [(<= ?ed #inst "2023-07-26")]
(coerce/to-date)) [?s :expected-deposit/client ?c]
:sales-date (or best-sales-date [?s :expected-deposit/external-id ?eid]
(-> (:initiated_at settlement) [(clojure.string/includes? ?eid "settlement")]
(coerce/to-date))) [?s :expected-deposit/total ?t]
:charges (->> (:entries settlement) [(iol-ion.query/dollars= ?t ?a)]
(filter :payment_id) ]
(map (fn [p] {:charge/external-id (str "square/charge/" (:payment_id p))})))}) (dc/db conn)
(filter :expected-deposit/date) (:db/id client)
(into [])) (amount->money (:amount_money payout))
(catch Throwable e ))]
(log/error ::transform-settlement-failed :when (not equivalent-already-exists?)]
:exception e))))))) #:expected-deposit {:external-id (str "square/payout/" (:id payout))
:vendor :vendor/ccp-square
:status :expected-deposit-status/pending
:total (amount->money (:amount_money payout))
:client (:db/id client)
:location (:square-location/client-location location)
:fee (- (reduce + 0.0 (map (fn [entry]
(if (= (:type entry) "REFUND")
(- (amount->money (:fee_amount_money entry)))
(amount->money (:fee_amount_money entry))))
(:payout_entries payout))))
:date (-> (:created_at payout)
(coerce/to-date))
:sales-date (or best-sales-date
(-> (:created_at payout)
(coerce/to-date)))
:charges (reverse (->> (:payout_entries payout)
(filter (comp :payment_id :type_charge_details))
(map (fn [p] {:charge/external-id (str "square/charge/" (:payment_id (:type_charge_details p)))}))))})
(filter :expected-deposit/date)
(into []))
(catch Throwable e
(log/error ::transform-payout-failed
:exception e)))))))))
(defn refunds (defn refunds
([client l] ([client l]
(de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/refunds?location_id=" (:square-location/square-id l)) (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/refunds?location_id=" (:square-location/square-id l))
:method :get :method :get
:headers (client-base-headers client) :headers (client-base-headers client)
:as :json}) :as :json})
:body :body
:refunds :refunds
(fn [refunds] (fn [refunds]
@@ -571,21 +576,23 @@
@(dc/transact-async conn x)))))))) @(dc/transact-async conn x))))))))
(defn upsert-settlements (defn upsert-payouts
([client] ([client]
(apply de/zip (apply de/zip
(for [square-location (:client/square-locations client) (for [square-location (:client/square-locations client)
:when (:square-location/client-location square-location)] :when (:square-location/client-location square-location)]
(upsert-settlements client square-location)))) (upsert-payouts client square-location))))
([client location] ([client location]
(with-context-as {:source "Square settlements loading" (upsert-payouts client location (time/plus (time/now) (time/days -7)) (time/now)))
([client location start end]
(with-context-as {:source "Square payout loading"
:client (:client/code client)} lc :client (:client/code client)} lc
(de/chain (daily-settlements client location) (de/chain (transformed-payouts client location start end)
(fn [settlements] (fn [payouts]
(mu/with-context lc (mu/with-context lc
(log/info ::started-loading-deposits :settlement-count (count settlements)) (log/info ::started-loading-deposits :payout-count (count payouts))
(doseq [x (partition-all 20 settlements)] (doseq [x (partition-all 20 payouts)]
(log/info ::loading-deposits (log/info ::loading-deposits
:count (count x)) :count (count x))
@(dc/transact-async conn x)) @(dc/transact-async conn x))
@@ -780,10 +787,10 @@
(mu/with-context lc (mu/with-context lc
(log/info ::upsert-orders-started) (log/info ::upsert-orders-started)
(upsert client))) (upsert client)))
#_(fn [_] ;; settlements are currently disabled (v1) (fn [_]
(mu/with-context lc (mu/with-context lc
(log/info ::upsert-settlements-started) (log/info ::upsert-payouts-started)
(upsert-settlements client))) (upsert-payouts client)))
(fn [_] (fn [_]
(mu/with-context lc (mu/with-context lc
(log/info ::upsert-refunds-started) (log/info ::upsert-refunds-started)