(cloud) big performance fixes

iol_ion/src/iol_ion/tx.clj

@@ -164,8 +164,7 @@
   (assert (:journal-entry/client ledger-entry) "Must at least provide client when updating ledger")
   (assert (every? :journal-entry-line/account (:journal-entry/line-items ledger-entry)) "must at least provide account when updating ledger")
   (assert (every? :journal-entry-line/location (:journal-entry/line-items ledger-entry)) "Must at least provide location when updating ledger")
-  (let [
+  (let [extant-entry         (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)]
-        extant-entry         (or (when-let [original-entity (:journal-entry/original-entity ledger-entry)]
                                    (dc/pull db extant-read [:journal-entry/original-entity original-entity]))
                                  (when-let [external-id (:journal-entry/external-id ledger-entry)]
                                    (dc/pull db extant-read [:journal-entry/external-id external-id])))

src/clj/auto_ap/datomic.clj

@@ -636,7 +636,7 @@
            true (update :tempids merge (:tempids tx-result)))))
 
      {}
-     (partition-all 50 txes))))
+     (partition-all 200 txes))))
 
 (defn audit-transact [txes id]
   (dc/transact conn {:tx-data (conj txes {:db/id      "datomic.tx"
@@ -664,10 +664,13 @@
   (str (UUID/randomUUID)))
 
 (defn pull-id [db id]
-  (ffirst (dc/q '[:find ?i
+  (if (sequential? id)
-                 :in $ ?i]
+    (ffirst (dc/q '[:find ?i 
-               db
+                    :in $ [?a ?v]
-               id)))
+                    :where [?i ?a ?v]]
+                  db
+                  id))
+    id))
 
 (defn pull-attr [db k id]
   (get (dc/pull db [k] id) k))

src/clj/auto_ap/datomic/vendors.clj

@@ -54,8 +54,9 @@
                 (not (str/blank? (:name-like args)))
                 (merge-query {:query {:in ['?name-like]
                                       :where ['[?e :vendor/name ?n]
-                                              '[(re-find ?name-like ?n)]]}
+                                              '[(re-pattern ?name-like) ?name-like-2]
-                              :args [(re-pattern (str "(?i)" (:name-like args)))]})
+                                              '[(re-find ?name-like-2 ?n)]]}
+                              :args [(str "(?i)"  (:name-like args))]})
 
                 true
                 (merge-query {:query {:find ['?e]

src/clj/auto_ap/ledger.clj

@@ -573,70 +573,74 @@
                  :starting-at starting-at
                  :location    location})))))
 
+(defn find-running-balance-start [{:keys [client account location starting-at]} db ]
+  (let [client  (pull-id db client)
+        account (pull-id db account)]
+    (or 
+      (->> (dc/index-pull db
+                          {:index    :avet
+                           :selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
+                           :start    [:journal-entry-line/client+account+location+date [client account location starting-at]]
+                           :reverse true
+                           :limit   500})
+           (take-while (fn [result]
+                         (= [client
+                             account
+                             location]
+                            (take 3 (:journal-entry-line/client+account+location+date result)))))
+           (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
+                         (>= (compare date starting-at) 0)))
+           first
+           :journal-entry-line/running-balance)
+      0.0)))
 
-(defn find-running-balance-start [account-needing-rebuild db ]
+(defn get-dirty-entries [{:keys [client account location starting-at]} db ]
-  (or 
+  (mu/trace ::get-dirty-entries
-    (->> (dc/index-pull db
+            []
-                        {:index    :avet
+            (let [client  (pull-id db client)
-                         :selector [:db/id :journal-entry-line/running-balance :journal-entry-line/client+account+location+date]
+                  account (pull-id db account)]
-                         :start    [:journal-entry-line/client+account+location+date
+              (into []
-                                    [(:client account-needing-rebuild)
+                    (comp
-                                     (:account account-needing-rebuild)
+                      (mapcat (fn [i]
-                                     (:location account-needing-rebuild)
+                                (dc/index-pull db
-                                     (:starting-at account-needing-rebuild)]]
+                                               {:index    :avet
+                                                :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
+                                                :start    [:journal-entry-line/client+account+location+date
+                                                           [client account location starting-at]]
+                                                :offset   (* i 1000)
+                                                :limit    1000}
+                                               ))
+                              )
                       
-                         :reverse true
+                      (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}]
-                         :limit   500})
+                                    (and 
-         (take-while (fn [result]
+                                      (= client result-client)
-                       (= [(:client account-needing-rebuild)
+                                      (= account result-account)
-                           (:account account-needing-rebuild)
+                                      (= location result-location))))
-                           (:location account-needing-rebuild)]
+                      (map (fn [result]
-                          (take 3 (:journal-entry-line/client+account+location+date result)))))
+                             [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ])))
-         (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
+                    (range)))))
-                       (>= (compare date (:starting-at account-needing-rebuild)) 0)))
-         first
-         :journal-entry-line/running-balance
-         )
-    0.0))
-
-(defn get-dirty-entries [account-needing-rebuild db ]
-  (->> (dc/index-pull db
-                      {:index    :avet
-                       :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
-                       :start    [:journal-entry-line/client+account+location+date
-                                  [(:client account-needing-rebuild)
-                                   (:account account-needing-rebuild)
-                                   (:location account-needing-rebuild)
-                                   (:starting-at account-needing-rebuild)]]
-                       })
-       (take-while (fn [result]
-                     (= [(:client account-needing-rebuild)
-                         (:account account-needing-rebuild)
-                         (:location account-needing-rebuild)]
-                        (take 3 (:journal-entry-line/client+account+location+date result)))))
-       (map (fn [result]
-              [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))))
 
 (defn compute-running-balance [account-needing-refresh]
-  (mu/log ::compute
+  (mu/trace  ::compute
-          :dirty-count (count (:dirty-entries account-needing-refresh)))
+             [:dirty-count (count (:dirty-entries account-needing-refresh))]
-  (second 
+             (second 
-    (reduce
+               (reduce
-      (fn [[running-balance rows] [id debit credit] ]
+                 (fn [[running-balance rows] [id debit credit] ]
-        (let [new-running-balance (+ running-balance
+                   (let [new-running-balance (+ running-balance
-                                     (if (#{:account-type/asset
+                                                (if (#{:account-type/asset
-                                            :account-type/dividend
+                                                       :account-type/dividend
-                                            :account-type/expense} (:account-type account-needing-refresh))
+                                                       :account-type/expense} (:account-type account-needing-refresh))
-                                       (- debit credit)
+                                                  (- debit credit)
-                                       (- credit debit)))]
+                                                  (- credit debit)))]
-          [new-running-balance
+                     [new-running-balance
-           (conj rows 
+                      (conj rows 
-                 {:db/id                              id
+                            {:db/id                              id
-                  :journal-entry-line/running-balance new-running-balance
+                             :journal-entry-line/running-balance new-running-balance
-                  :journal-entry-line/dirty           false})]))
+                             :journal-entry-line/dirty           false})]))
 
-      [(:build-from account-needing-refresh) []]
+                 [(:build-from account-needing-refresh) []]
-      (:dirty-entries account-needing-refresh))))
+                 (:dirty-entries account-needing-refresh)))))
 
 
 (defn refresh-running-balance-cache
@@ -653,7 +657,7 @@
                accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
            (when (seq accounts-needing-rebuild)
              (mu/log ::found-accounts-needing-rebuild
-                     :accounts accounts-needing-rebuild)
+                     :accounts (count accounts-needing-rebuild))
              (audit-transact-batch
                (->>  accounts-needing-rebuild
                      (mapcat (fn [account-needing-rebuild]
@@ -666,6 +670,7 @@
                {:user/name "running-balance-cache"}))))))))
 
 
+;; TODO only enable once IOL is set up in clod
 #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
 #_(mount/defstate running-balance-cache-worker
   :start (scheduler/every (* 15 60 (+ 500 (rand-int 500))) (heartbeat refresh-running-balance-cache "running-balance-cache"))

things-to-search-for.txt

@@ -1,15 +1,9 @@
 or-join syntax changed?
-Make sure no history on ledger
+regex no longer supported as argument for query. Check vendor datomic for how to do it right.
 it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
 ezcater graphql needs search index too
 make sure that temporary ids are set on all new things when using upsert-entity
 Wrap tests around every api call
-Fix tests
-Make a different solution for running balance cache
-* Store the running balance on the line
-* Make ledger changes mark as dirty
-* Recompute the cache periodically, somewhat randomly to avoid races
-* Have a background job recompute all at night.
 
 Fix searching
 * indexing should happen more regularly, and just look for changes since last time it was run
@@ -17,9 +11,27 @@ Fix searching
 Running Balance Cache
 * Add tests for upsert-ledger
 * try again to see if we can get upsert-ledger into the same transaction, making it all or nothing
-* make rest of rebuilding the cache use new index
+* Make a new way to reset the entire cache for a client
-* ensure somehow that the index is always right
-
 
 Address memory
 * JVM settings now and in prod
+
+Release steps:
+Stop prod
+Make database snapshot
+Create new database for prod-cloud (just called prod)
+Restore database
+Transact new schema
+(reset-client+account+location+date)
+(force-rebuild-running-balance-cache)
+Merge branch into master
+Rename prod-cloud to prod everywhere
+Release again
+
+Sanity checks later:
+* Run query 
+
+
+Future improvements:
+Make reports just be based on running-balances
+