From 9c1376950b4c9ce5f63e6393a4f44fe5d42ba1d3 Mon Sep 17 00:00:00 2001
From: Bryce Covert <bryce@brycecovertoperations.com>
Date: Wed, 5 Apr 2023 17:19:40 -0700
Subject: [PATCH] (cloud) makes sales export queries super stinkin' fast

---
 iol_ion/src/iol_ion/query.clj        | 27 +++++++++++-
 resources/cloud-migration-schema.edn |  8 ++++
 resources/datomic/ion-config.edn     |  1 +
 src/clj/auto_ap/graphql/clients.clj  | 64 ++++++++++++++--------------
 4 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/iol_ion/src/iol_ion/query.clj b/iol_ion/src/iol_ion/query.clj
index a0950046..fea83611 100644
--- a/iol_ion/src/iol_ion/query.clj
+++ b/iol_ion/src/iol_ion/query.clj
@@ -1,7 +1,8 @@
 (ns iol-ion.query
   (:require [clj-time.core :as time]
             [clj-time.coerce :as coerce]
-            [clj-time.format :as f]))
+            [clj-time.format :as f]
+            [datomic.client.api :as dc]))
 
 #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
 (defn dollars-0? [amt]
@@ -19,7 +20,7 @@
 
 
 (defn recent-date []
-  (coerce/to-date (time/minus (local-now) (time/days 90))))
+  (coerce/to-date (time/minus (local-now) (time/days 10))))
 
 (def excel-formatter (f/with-zone (f/formatter "MM/dd/yyyy") (time/time-zone-for-id "America/Los_Angeles")))
 (defn excel-date [d]
@@ -27,3 +28,25 @@
       (coerce/to-date-time)
       localize
       (f/unparse excel-formatter )))
+
+
+(defn sales-orders-in-range [db client start end]
+  (let [end (or end #inst "2050-01-01")]
+    (into #{}
+          (comp
+            
+            (map (fn [i]
+                   (dc/index-range db
+                                   {:attrid :sales-order/client+date
+                                    :start  [client start]
+                                    :end    [client end]
+                                    :limit  10000
+                                    :offset (* 10000 i)})))
+
+            (take-while seq)
+            (mapcat identity)
+            (map :e))
+          (range))))
+
+
+
diff --git a/resources/cloud-migration-schema.edn b/resources/cloud-migration-schema.edn
index 727e8efd..2883a455 100644
--- a/resources/cloud-migration-schema.edn
+++ b/resources/cloud-migration-schema.edn
@@ -17,4 +17,12 @@
   :db/doc         "Used to find accounts and locations quickly",
   :db/noHistory   true
   }
+
+ {:db/valueType   :db.type/tuple
+  :db/tupleAttrs  [:sales-order/client :sales-order/date]
+  :db/cardinality :db.cardinality/one,
+  :db/ident       :sales-order/client+date
+  :db/doc         "Used to find sales orders quickly",
+  :db/noHistory   true
+  }
  ]
diff --git a/resources/datomic/ion-config.edn b/resources/datomic/ion-config.edn
index fba2a238..64fb6b82 100644
--- a/resources/datomic/ion-config.edn
+++ b/resources/datomic/ion-config.edn
@@ -10,6 +10,7 @@
             iol-ion.query/dollars=
             iol-ion.query/dollars-0?
 
+            iol-ion.query/sales-orders-in-range
             iol-ion.query/local-now
             iol-ion.query/localize
             iol-ion.query/recent-date
diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj
index 8b98e78b..485a0c4e 100644
--- a/src/clj/auto_ap/graphql/clients.clj
+++ b/src/clj/auto_ap/graphql/clients.clj
@@ -218,44 +218,42 @@
                                  (assoc ba :bank-account/yodlee-balance-old nil))
                                bank-accounts))))))))
 
-
-
 (def sales-summary-query
   "[:find ?d4 (sum ?total) (sum ?tax) (sum ?tip)  (sum ?service-charge) (sum ?discount) (sum ?returns)
   :with ?s
-  :in $
-  :where
-  [(ground (iol-ion.query/recent-date)) ?min-d]
-  [?s :sales-order/client [:client/code \"%s\"]]
-[?s :sales-order/date ?d]
-[(>= ?d ?min-d)]
-[?s :sales-order/total ?total]
-[?s :sales-order/tax ?tax]
-[?s :sales-order/tip ?tip]
-[?s :sales-order/service-charge ?service-charge]
-[?s :sales-order/returns ?returns]
-[?s :sales-order/discount ?discount]
-[(iol-ion.query/excel-date ?d) ?d4]
-]")
+ :in $
+ :where
+ [(ground (iol-ion.query/recent-date)) ?min-d]
+  [(ground #inst \"2040-01-01\") ?max-d]
+ [?c :client/code \"%s\"]
+ [(iol-ion.query/sales-orders-in-range $ ?c ?min-d ?max-d) [?s ...]]
+ [?s :sales-order/date ?d]
+ [?s :sales-order/total ?total]
+ [?s :sales-order/tax ?tax]
+ [?s :sales-order/tip ?tip]
+ [?s :sales-order/service-charge ?service-charge]
+ [?s :sales-order/returns ?returns]
+ [?s :sales-order/discount ?discount]
+ [(iol-ion.query/excel-date ?d) ?d4]
+ ]")
 
 (def sales-category-query
   "[:find  ?d4 ?n ?n2 (sum ?total) (sum ?tax) (sum ?discount)
-  :with ?s ?li
-  :in $ 
-  :where 
-  [(ground (iol-ion.query/recent-date)) ?min-d]
-  [?s :sales-order/client [:client/code \"%s\"]]
-  [?s :sales-order/date ?d]
-  [(>= ?d ?min-d)]
-  [?s :sales-order/line-items ?li]
-  [?li :order-line-item/category ?n]
-  [(get-else $ ?li :order-line-item/item-name \"\") ?n2]
-[?li :order-line-item/total ?total]
-[?li :order-line-item/tax ?tax]
-[?li :order-line-item/discount ?discount]
-
-[(iol-ion.query/excel-date ?d) ?d4]
-]")
+   :with ?s ?li
+   :in $ 
+   :where 
+   [(ground (iol-ion.query/recent-date)) ?min-d]
+   [(ground #inst \"2040-01-01\") ?max-d]
+   [?c :client/code \"%s\"]
+   [(iol-ion.query/sales-orders-in-range $ ?c ?min-d ?max-d) [?s ...]]
+   [?s :sales-order/date ?d]
+   [?s :sales-order/line-items ?li]
+   [?li :order-line-item/category ?n]
+   [(get-else $ ?li :order-line-item/item-name \"\") ?n2]
+   [?li :order-line-item/total ?total]
+   [?li :order-line-item/tax ?tax]
+   [?li :order-line-item/discount ?discount]
+   [(iol-ion.query/excel-date ?d) ?d4]]")
 
 (def expected-deposits-query
   "[:find ?d4 ?t ?f
@@ -315,7 +313,7 @@
        [(get-else $ ?ccp :db/ident :na) ?p]
        ))
    [(name ?p) ?p2]
-   [(iol-ion.query/excel-date ?date) ?d4]
+   [(iol-ion.query/excel-date ?date) ?d4]]
 " )
 
 (def refunds-query