(cloud) makes sales export queries super stinkin' fast

iol_ion/src/iol_ion/query.clj

@@ -1,7 +1,8 @@
 (ns iol-ion.query
   (:require [clj-time.core :as time]
             [clj-time.coerce :as coerce]
-            [clj-time.format :as f]))
+            [clj-time.format :as f]
+            [datomic.client.api :as dc]))
 
 #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
 (defn dollars-0? [amt]
@@ -19,7 +20,7 @@
 
 
 (defn recent-date []
-  (coerce/to-date (time/minus (local-now) (time/days 90))))
+  (coerce/to-date (time/minus (local-now) (time/days 10))))
 
 (def excel-formatter (f/with-zone (f/formatter "MM/dd/yyyy") (time/time-zone-for-id "America/Los_Angeles")))
 (defn excel-date [d]
@@ -27,3 +28,25 @@
       (coerce/to-date-time)
       localize
       (f/unparse excel-formatter )))
+
+
+(defn sales-orders-in-range [db client start end]
+  (let [end (or end #inst "2050-01-01")]
+    (into #{}
+          (comp
+            
+            (map (fn [i]
+                   (dc/index-range db
+                                   {:attrid :sales-order/client+date
+                                    :start  [client start]
+                                    :end    [client end]
+                                    :limit  10000
+                                    :offset (* 10000 i)})))
+
+            (take-while seq)
+            (mapcat identity)
+            (map :e))
+          (range))))
+
+
+

resources/cloud-migration-schema.edn

@@ -17,4 +17,12 @@
   :db/doc         "Used to find accounts and locations quickly",
   :db/noHistory   true
   }
+
+ {:db/valueType   :db.type/tuple
+  :db/tupleAttrs  [:sales-order/client :sales-order/date]
+  :db/cardinality :db.cardinality/one,
+  :db/ident       :sales-order/client+date
+  :db/doc         "Used to find sales orders quickly",
+  :db/noHistory   true
+  }
  ]

resources/datomic/ion-config.edn

@@ -10,6 +10,7 @@
             iol-ion.query/dollars=
             iol-ion.query/dollars-0?
 
+            iol-ion.query/sales-orders-in-range
             iol-ion.query/local-now
             iol-ion.query/localize
             iol-ion.query/recent-date

src/clj/auto_ap/graphql/clients.clj

@@ -218,44 +218,42 @@
                                  (assoc ba :bank-account/yodlee-balance-old nil))
                                bank-accounts))))))))
 
-
-
 (def sales-summary-query
   "[:find ?d4 (sum ?total) (sum ?tax) (sum ?tip)  (sum ?service-charge) (sum ?discount) (sum ?returns)
   :with ?s
-  :in $
+ :in $
-  :where
+ :where
-  [(ground (iol-ion.query/recent-date)) ?min-d]
+ [(ground (iol-ion.query/recent-date)) ?min-d]
-  [?s :sales-order/client [:client/code \"%s\"]]
+  [(ground #inst \"2040-01-01\") ?max-d]
-[?s :sales-order/date ?d]
+ [?c :client/code \"%s\"]
-[(>= ?d ?min-d)]
+ [(iol-ion.query/sales-orders-in-range $ ?c ?min-d ?max-d) [?s ...]]
-[?s :sales-order/total ?total]
+ [?s :sales-order/date ?d]
-[?s :sales-order/tax ?tax]
+ [?s :sales-order/total ?total]
-[?s :sales-order/tip ?tip]
+ [?s :sales-order/tax ?tax]
-[?s :sales-order/service-charge ?service-charge]
+ [?s :sales-order/tip ?tip]
-[?s :sales-order/returns ?returns]
+ [?s :sales-order/service-charge ?service-charge]
-[?s :sales-order/discount ?discount]
+ [?s :sales-order/returns ?returns]
-[(iol-ion.query/excel-date ?d) ?d4]
+ [?s :sales-order/discount ?discount]
-]")
+ [(iol-ion.query/excel-date ?d) ?d4]
+ ]")
 
 (def sales-category-query
   "[:find  ?d4 ?n ?n2 (sum ?total) (sum ?tax) (sum ?discount)
-  :with ?s ?li
+   :with ?s ?li
-  :in $ 
+   :in $ 
-  :where 
+   :where 
-  [(ground (iol-ion.query/recent-date)) ?min-d]
+   [(ground (iol-ion.query/recent-date)) ?min-d]
-  [?s :sales-order/client [:client/code \"%s\"]]
+   [(ground #inst \"2040-01-01\") ?max-d]
-  [?s :sales-order/date ?d]
+   [?c :client/code \"%s\"]
-  [(>= ?d ?min-d)]
+   [(iol-ion.query/sales-orders-in-range $ ?c ?min-d ?max-d) [?s ...]]
-  [?s :sales-order/line-items ?li]
+   [?s :sales-order/date ?d]
-  [?li :order-line-item/category ?n]
+   [?s :sales-order/line-items ?li]
-  [(get-else $ ?li :order-line-item/item-name \"\") ?n2]
+   [?li :order-line-item/category ?n]
-[?li :order-line-item/total ?total]
+   [(get-else $ ?li :order-line-item/item-name \"\") ?n2]
-[?li :order-line-item/tax ?tax]
+   [?li :order-line-item/total ?total]
-[?li :order-line-item/discount ?discount]
+   [?li :order-line-item/tax ?tax]
-
+   [?li :order-line-item/discount ?discount]
-[(iol-ion.query/excel-date ?d) ?d4]
+   [(iol-ion.query/excel-date ?d) ?d4]]")
-]")
 
 (def expected-deposits-query
   "[:find ?d4 ?t ?f
@@ -315,7 +313,7 @@
        [(get-else $ ?ccp :db/ident :na) ?p]
        ))
    [(name ?p) ?p2]
-   [(iol-ion.query/excel-date ?date) ?d4]
+   [(iol-ion.query/excel-date ?date) ?d4]]
 " )
 
 (def refunds-query