Adds the ability to launch a job that restores a database
This commit is contained in:
@@ -14,7 +14,9 @@
|
||||
[com.brunobonacci.mulog :as mu]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[datomic.client.api :as dc]
|
||||
[iol-ion.tx :refer [upsert-ledger]]))
|
||||
[iol-ion.tx :refer [upsert-ledger]]
|
||||
[manifold.deferred :as de]
|
||||
[manifold.stream :as s]))
|
||||
|
||||
(defn datums->impacted-entity [db [e changes]]
|
||||
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
|
||||
@@ -505,50 +507,65 @@
|
||||
(defn reset-client+account+location+date
|
||||
([] (reset-client+account+location+date (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn)))))
|
||||
([clients]
|
||||
(doseq [client clients
|
||||
:let [_ (mu/log ::reseting-index-for :client client)]
|
||||
batch
|
||||
(->> (dc/qseq '[:find (pull ?je [:journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id]}])
|
||||
:in $ ?c
|
||||
:where [?je :journal-entry/client ?c]]
|
||||
(dc/db conn)
|
||||
client
|
||||
|
||||
)
|
||||
(map first)
|
||||
(mapcat (fn [je]
|
||||
(map (fn [jel]
|
||||
{:db/id (:db/id jel)
|
||||
:journal-entry-line/client+account+location+date
|
||||
[(-> je :journal-entry/client :db/id)
|
||||
(-> jel :journal-entry-line/account :db/id)
|
||||
(doseq [[client i] (map vector clients (range))]
|
||||
(mu/trace ::reset-index
|
||||
[:client client]
|
||||
(mu/with-context {:client client
|
||||
:client-index i
|
||||
:client-count (count clients)}
|
||||
(mu/log ::reseting-index)
|
||||
(let [so-far (atom 0)]
|
||||
@(->> (dc/qseq '[:find (pull ?je [:journal-entry/date :journal-entry/client {:journal-entry/line-items [:journal-entry-line/account :journal-entry-line/location :db/id]}])
|
||||
:in $ ?c
|
||||
:where [?je :journal-entry/client ?c]]
|
||||
(dc/db conn)
|
||||
client)
|
||||
(map first)
|
||||
(mapcat (fn [je]
|
||||
(map (fn [jel]
|
||||
{:db/id (:db/id jel)
|
||||
:journal-entry-line/client+account+location+date
|
||||
[(-> je :journal-entry/client :db/id)
|
||||
(-> jel :journal-entry-line/account :db/id)
|
||||
|
||||
(-> jel :journal-entry-line/location)
|
||||
(-> jel :journal-entry-line/location)
|
||||
|
||||
(-> je :journal-entry/date)]})
|
||||
(:journal-entry/line-items je))))
|
||||
(partition-all 500)
|
||||
)]
|
||||
(mu/log ::batch-completed)
|
||||
(dc/transact conn {:tx-data batch}))))
|
||||
(-> je :journal-entry/date)]})
|
||||
(:journal-entry/line-items je))))
|
||||
(partition-all 500)
|
||||
(s/->source)
|
||||
(s/map (fn [batch]
|
||||
(de/future
|
||||
(dc/transact conn {:tx-data batch})
|
||||
(count batch))))
|
||||
(s/buffer 50)
|
||||
(s/realize-each)
|
||||
(s/consume (fn [batch-count]
|
||||
(swap! so-far #(+ % batch-count))
|
||||
(mu/log ::reset :count batch-count
|
||||
:so-far @so-far
|
||||
:client client
|
||||
:client-index i
|
||||
:client-count (count clients))))))
|
||||
(mu/log ::client-completed))))))
|
||||
|
||||
(defn find-mismatch-index []
|
||||
(reduce + 0
|
||||
(for [c (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn)))
|
||||
:let [_ (println "searching for" c)
|
||||
a (->> (dc/index-pull (dc/db conn)
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}]
|
||||
:start [:journal-entry-line/client+account+location+date [c]]})
|
||||
(take-while (fn [result]
|
||||
(= c (first (:journal-entry-line/client+account+location+date result)))
|
||||
))
|
||||
(filter (fn [{index :journal-entry-line/client+account+location+date :as result}]
|
||||
(not= index
|
||||
[(-> result :journal-entry/_line-items :journal-entry/client :db/id)
|
||||
(-> result :journal-entry-line/account :db/id)
|
||||
(-> result :journal-entry-line/location)
|
||||
(-> result :journal-entry/_line-items :journal-entry/date)]))))]]
|
||||
(for [c (map first (dc/q '[:find ?c :where [?c :client/code]] (dc/db conn)))
|
||||
:let [_ (println "searching for" c)
|
||||
a (->> (dc/index-pull (dc/db conn)
|
||||
{:index :avet
|
||||
:selector [:db/id :journal-entry-line/location :journal-entry-line/account :journal-entry-line/client+account+location+date {:journal-entry/_line-items [:journal-entry/date :journal-entry/client]}]
|
||||
:start [:journal-entry-line/client+account+location+date [c]]})
|
||||
(take-while (fn [result]
|
||||
(= c (first (:journal-entry-line/client+account+location+date result)))
|
||||
))
|
||||
(filter (fn [{index :journal-entry-line/client+account+location+date :as result}]
|
||||
(not= index
|
||||
[(-> result :journal-entry/_line-items :journal-entry/client :db/id)
|
||||
(-> result :journal-entry-line/account :db/id)
|
||||
(-> result :journal-entry-line/location)
|
||||
(-> result :journal-entry/_line-items :journal-entry/date)]))))]]
|
||||
(do (println (count a))
|
||||
(count a)))))
|
||||
|
||||
@@ -573,6 +590,24 @@
|
||||
:starting-at starting-at
|
||||
:location location})))))
|
||||
|
||||
(defn all-accounts-needing-rebuild [ db client]
|
||||
(let [client (pull-id db client)]
|
||||
(->> (dc/qseq '[:find ?c ?a ?l (min ?d)
|
||||
:in $ ?c
|
||||
:where
|
||||
[?je :journal-entry/client ?c]
|
||||
[?je :journal-entry/line-items ?jel]
|
||||
[?jel :journal-entry-line/account ?a]
|
||||
[?jel :journal-entry-line/location ?l]
|
||||
[?je :journal-entry/date ?d]]
|
||||
db
|
||||
client)
|
||||
(map (fn [[client account location starting-at ]]
|
||||
{:client client
|
||||
:account account
|
||||
:starting-at starting-at
|
||||
:location location})))))
|
||||
|
||||
(defn find-running-balance-start [{:keys [client account location starting-at]} db ]
|
||||
(let [client (pull-id db client)
|
||||
account (pull-id db account)]
|
||||
@@ -612,10 +647,6 @@
|
||||
(take-while identity)
|
||||
(mapcat identity)
|
||||
(take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}]
|
||||
#_(println
|
||||
[[ client result-client]
|
||||
[ account result-account]
|
||||
[ location result-location]])
|
||||
(and
|
||||
(= client result-client)
|
||||
(= account result-account)
|
||||
@@ -646,31 +677,77 @@
|
||||
(:dirty-entries account-needing-refresh)))))
|
||||
|
||||
|
||||
(defn refresh-running-balance-accounts [accounts-needing-rebuild clients i db]
|
||||
(mu/log ::found-accounts-needing-rebuild
|
||||
:accounts (count accounts-needing-rebuild))
|
||||
(let [so-far (atom 0)]
|
||||
@(->> accounts-needing-rebuild
|
||||
(s/->source)
|
||||
(s/map (fn [account-needing-rebuild]
|
||||
(de/future
|
||||
(mu/with-context {:account account-needing-rebuild}
|
||||
(-> account-needing-rebuild
|
||||
(assoc :build-from (find-running-balance-start account-needing-rebuild db))
|
||||
(assoc :dirty-entries (get-dirty-entries account-needing-rebuild db))
|
||||
(assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild))))
|
||||
(compute-running-balance))))))
|
||||
(s/realize-each)
|
||||
(s/mapcat (fn [x]
|
||||
x))
|
||||
(s/buffer 50)
|
||||
(s/transform (partition-all 500))
|
||||
(s/map (fn [batch]
|
||||
(de/future
|
||||
(dc/transact conn {:tx-data batch})
|
||||
(count batch))))
|
||||
(s/buffer 50)
|
||||
(s/realize-each)
|
||||
(s/consume (fn [batch-count]
|
||||
(swap! so-far #(+ % batch-count))
|
||||
(mu/log ::reset
|
||||
:count batch-count
|
||||
:so-far @so-far
|
||||
:client c
|
||||
:client-index i
|
||||
:client-count (count clients)))))))
|
||||
|
||||
(defn refresh-running-balance-cache
|
||||
([] (refresh-running-balance-cache (shuffle (map first
|
||||
(dc/q '[:find (pull ?c [:client/code :db/id])
|
||||
:where [?c :client/code]]
|
||||
(dc/db conn))))))
|
||||
([clients]
|
||||
(doseq [c clients]
|
||||
(doseq [[c i] (map vector clients (range))]
|
||||
(mu/trace ::building-running-balance
|
||||
[:client c]
|
||||
(mu/with-context {:client c}
|
||||
(mu/with-context {:client c
|
||||
:client-index i
|
||||
:client-count (count clients)}
|
||||
(mu/log ::searching-for-accounts)
|
||||
(let [db (dc/db conn)
|
||||
accounts-needing-rebuild (accounts-needing-rebuild db (:db/id c))]
|
||||
(when (seq accounts-needing-rebuild)
|
||||
(mu/log ::found-accounts-needing-rebuild
|
||||
:accounts (count accounts-needing-rebuild))
|
||||
(audit-transact-batch
|
||||
(->> accounts-needing-rebuild
|
||||
(mapcat (fn [account-needing-rebuild]
|
||||
(mu/with-context {:account account-needing-rebuild}
|
||||
(-> account-needing-rebuild
|
||||
(assoc :build-from (find-running-balance-start account-needing-rebuild db))
|
||||
(assoc :dirty-entries (get-dirty-entries account-needing-rebuild db))
|
||||
(assoc :account-type (:account_type ((build-account-lookup (:client account-needing-rebuild)) (:account account-needing-rebuild))))
|
||||
(compute-running-balance))))))
|
||||
{:user/name "running-balance-cache"}))))))))
|
||||
(refresh-running-balance-accounts accounts-needing-rebuild clients i db)
|
||||
(mu/log ::client-completed))))))))
|
||||
|
||||
(defn rebuild-running-balance-cache
|
||||
([] (rebuild-running-balance-cache (shuffle (map first
|
||||
(dc/q '[:find (pull ?c [:client/code :db/id])
|
||||
:where [?c :client/code]]
|
||||
(dc/db conn))))))
|
||||
([clients]
|
||||
(doseq [[c i] (map vector clients (range))]
|
||||
(mu/trace ::building-running-balance
|
||||
[:client c]
|
||||
(mu/with-context {:client c
|
||||
:client-index i
|
||||
:client-count (count clients)}
|
||||
(mu/log ::searching-for-accounts)
|
||||
(let [db (dc/db conn)
|
||||
accounts-needing-rebuild (all-accounts-needing-rebuild db (:db/id c))]
|
||||
(when (seq accounts-needing-rebuild)
|
||||
(refresh-running-balance-accounts accounts-needing-rebuild clients i db)
|
||||
(mu/log ::client-completed))))))))
|
||||
|
||||
|
||||
;; TODO only enable once IOL is set up in clod
|
||||
|
||||
Reference in New Issue
Block a user