From b2e55ee49e89928692d91a39d2d0128e16cec5cb Mon Sep 17 00:00:00 2001
From: Bryce Covert <bryce@brycecovertoperations.com>
Date: Tue, 6 Dec 2022 20:40:38 -0800
Subject: [PATCH] Minor tweaks for performance

---
 .../auto_ap/jobs/register_invoice_import.clj  | 15 +++--
 src/clj/auto_ap/square/core2.clj              | 67 +++++++------------
 2 files changed, 33 insertions(+), 49 deletions(-)

diff --git a/src/clj/auto_ap/jobs/register_invoice_import.clj b/src/clj/auto_ap/jobs/register_invoice_import.clj
index 530e0c24..d45a5d23 100644
--- a/src/clj/auto_ap/jobs/register_invoice_import.clj
+++ b/src/clj/auto_ap/jobs/register_invoice_import.clj
@@ -18,11 +18,16 @@
 (def bucket (:data-bucket env))
 
 (defn s3->csv [url]
-  (->> (-> (s3/get-object {:bucket-name bucket
-                           :key         (str "bulk-import/" url)})
-           :input-stream
-           io/reader
-           csv/read-csv)))
+  (try 
+    (->> (-> (s3/get-object {:bucket-name bucket
+                             :key         (str "bulk-import/" url)})
+             :input-stream
+             io/reader
+             csv/read-csv))
+    (catch Exception e
+      (log/error (str "Could not read the file " url ". Are you sure you uploaded it?"))
+      (log/error e)
+      (throw e))))
 
 (defn register-invoice-import* [data]
   (let [db             (d/db conn)
diff --git a/src/clj/auto_ap/square/core2.clj b/src/clj/auto_ap/square/core2.clj
index 77878b8d..e26827e1 100644
--- a/src/clj/auto_ap/square/core2.clj
+++ b/src/clj/auto_ap/square/core2.clj
@@ -281,38 +281,6 @@
     (throw x)
     x))
 
-(defn get-settlement-sales-date [client settlement]
-  (let [concurrent 4
-        output-chan (async/chan)]
-    (async/pipeline-blocking concurrent
-                             output-chan
-                             (map (fn [p]
-                                    (lc/with-context {:source "Square settlements loading "}
-                                      (log/trace "looking up payment " p " for settlement " (:id settlement))
-                                      (or 
-                                       (-> (get-payment client p)
-                                           :created_at
-                                           coerce/to-date)
-                                       (coerce/to-date (time/now))))))
-                             (async/to-chan! (->> settlement
-                                                 :entries
-                                                 (filter #(= "CHARGE" (:type %)))
-                                                 (map :payment_id)
-                                                 (filter identity)
-                                                 (set)
-                                                 (take 20)
-                                                 ))
-                             true
-                             (fn [e]
-                               (lc/with-context {:source "Square settlements loading "}
-                                 (log/warn "Error loading sales date details" e)
-                                 e)))
-    (->> (async/<!! (async/into [] output-chan))
-         (map halt-if-error)
-         sort
-         (drop 2)
-         first)))
-
 (defn get-settlement-details [client location settlements] ;; pairs of [location settlement]
   (log/info "getting settlement details for " settlements)
   (let [concurrent 4
@@ -322,14 +290,10 @@
                              (map (fn [s]
                                     (lc/with-context {:source "Square settlements loading "}
                                       (log/info "Looking up settlement " s " for location " (:square-location/client-location location))
-                                      (let [settlement (:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" s)
-                                                                                  {:headers (client-base-headers client)
-                                                                                   :as :json
-                                                                                   :retry-handler retry-4})))
-                                            sales-date (get-settlement-sales-date client settlement)]
-                                        
-                                        (log/info "sales date for " s " is " sales-date)
-                                        (assoc settlement :sales-date  sales-date)))))
+                                      (:body (retry #(client/get (str "https://connect.squareup.com/v1/" (:square-location/square-id location) "/settlements/" s)
+                                                                 {:headers       (client-base-headers client)
+                                                                  :as            :json
+                                                                  :retry-handler retry-4}))))))
                              (async/to-chan! settlements)
                              true
                              (fn [e]
@@ -352,8 +316,7 @@
                                                                             "end_time"   end-date}
                                                              :as           :json
                                                              :retry-handler retry-4})) 
-                                         :body
-                                         (map :id))]
+                                         :body)]
                     settlements)))
         set
         seq
@@ -361,7 +324,23 @@
 
 (defn daily-settlements
   ([client location]
-   (->> (for [settlement (settlements client location)]
+   (->> (for [settlement (settlements client location)
+              :let [best-sales-date (->> (d/q '[:find ?s4 (count ?s)
+                                                   :in $ ?settlement-id
+                                                   :where
+                                                   [?settlement :expected-deposit/external-id ?settlement-id]
+                                                   [?settlement :expected-deposit/charges ?c]
+                                                   [?s :sales-order/charges ?c]
+                                                   [?s :sales-order/date ?sales-date]
+                                                   [(clj-time.coerce/to-date-time ?sales-date) ?s2]
+                                                   [(auto-ap.time/localize ?s2) ?s3]
+                                                   [(clj-time.coerce/to-local-date ?s3) ?s4]]
+                                                 (d/db conn)
+                                                 (str "square/settlement/" (:id settlement)))
+                                            (sort-by last)
+                                            last
+                                            first
+                                            coerce/to-date)]]
           #:expected-deposit {:external-id (str "square/settlement/" (:id settlement))
                               :vendor :vendor/ccp-square
                               :status :expected-deposit-status/pending
@@ -375,7 +354,7 @@
                                                          (:entries settlement))))
                               :date (-> (:initiated_at settlement)
                                         (coerce/to-date))
-                              :sales-date (or (:sales-date settlement)
+                              :sales-date (or best-sales-date
                                               (-> (:initiated_at settlement)
                                                   (coerce/to-date)))
                               :charges (->> (:entries settlement)