notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Bryce Covert
2022-09-30 06:41:05 -07:00
parent 1f57ed2d1c
commit aebf95a87b
3 changed files with 78 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
scratch-sessions/dump-edn.clj
View File

@@ -253,33 +253,37 @@
"Transacts data from from-ch. Returns a map with:
:result, a return channel getting {:error t} or {:completed n}
:stop, a fn you can use to terminate early."
[conn conc from-ch]
(let [to-ch (a/chan 100)
[conn conc from-ch f]
(let [to-ch (a/chan 400)
done-ch (a/chan)
transact-data (fn [data]
(try
(dc/transact conn {:tx-data data})
; if exception in a transaction
; will close channels and put error
; on done channel.
(catch Throwable t
(.printStackTrace t)
(a/close! from-ch)
(a/close! to-ch)
(a/>!! done-ch {:error t}))))]
transact-data (fn [data result]
(a/go
(try
(a/>! result (a/<! (dca/transact conn {:tx-data (f data)})))
(a/close! result)
; if exception in a transaction
; will close channels and put error
; on done channel.
(catch Throwable t
(.printStackTrace t)
(a/close! from-ch)
(a/close! to-ch)
(a/>! done-ch {:error t})))))]
; go block prints a '.' after every 1000 transactions, puts completed
; report on done channel when no value left to be taken.
(a/go-loop [total 0]
(print ".")
(flush)
(if (= (mod total 2) 0)
(do
(print ".")
(flush)))
(if-let [c (a/<! to-ch)]
(recur (inc total))
(a/>! done-ch {:completed total})))
; pipeline that uses transducer form of map to transact data taken from
; from-ch and puts results on to-ch
(a/pipeline-blocking conc to-ch (map transact-data) from-ch)
(a/pipeline-async conc to-ch transact-data from-ch)
; returns done channel and a function that you can use
; for early termination.
@@ -341,41 +345,50 @@
(cond-> (entity->best-key entity)
(not (vector? (entity->best-key entity))) vector))
tx-chan (a/chan)
pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"}) 30
tx-chan)]]
(println "Inserting " entity ": " (count entities))
(flush)
(doseq [batch (partition-all 2000 entities)]
(let [transaction {:tx-data (->> (d/pull-many remote-db
(->> schema
(filter :db/valueType)
(mapv :db/ident)
(filter #(= entity (namespace %)))
(into [:db/id]))
batch)
(mapv (fn [m ]
(reduce
(fn [m [k v]]
(cond
(= k :db/id)
(-> m
(assoc :entity/migration-key v)
(dissoc :db/id))
(full-dependencies k)
(if (vector? v)
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
(assoc m k [:entity/migration-key (:db/id v)]))
:else
(dissoc m :payment/pdf-data
:payment/memo)))
m
m))))}]
(a/>!! tx-chan (:tx-data transaction))))
(a/close tx-chan)
tx-chan (a/chan 400)
entities->transaction (fn [entities]
(->> (d/pull-many remote-db
(->> schema
(filter :db/valueType)
(mapv :db/ident)
(filter #(= entity (namespace %)))
(into [:db/id]))
entities)
(mapv (fn [m ]
(reduce
(fn [m [k v]]
(cond
(= k :db/id)
(-> m
(assoc :entity/migration-key v)
(dissoc :db/id))
(full-dependencies k)
(if (vector? v)
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
(assoc m k [:entity/migration-key (:db/id v)]))
:else
(dissoc m :payment/pdf-data
:payment/memo)))
m
m)))))
_ (println "Inserting " entity ": " (count entities))
_ (flush)
pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"})
50
tx-chan
entities->transaction)]]
(doseq [batch (partition-all 500 entities)]
(try
(a/>!! tx-chan batch)
(catch Exception e
(println e)
((:stop pipeline)))))
(println "waiting for done from" pipeline)
(flush)
(a/close! tx-chan)
(println (a/<!! (:result pipeline)))
((:stop pipeline))
(println)
(println "Done")))

17
src/clj/auto_ap/graphql/accounts.clj
View File

@@ -161,7 +161,8 @@
(for [result (map first (dc/qseq '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}])
:in $
:where [?aco :account-client-override/client ]
[?aco :account-client-override/search-terms ]]
[?aco :account-client-override/search-terms ]
[_ :account/client-overrides ?aco]]
(dc/db conn)))
:when (:account/numeric-code (:account/_client-overrides result))]
{:id (:db/id (:account/_client-overrides result))
@@ -188,3 +189,17 @@
:location (:account/location result)
:applicability (name (:db/ident (:account/applicability result)))})
"account"))
#_(dc/transact conn
{:tx-data [{:db/ident :account-client-override/k2
:db/valueType :db.type/tuple
:db/tupleAttrs [:account/_client-overrides :account-client-override/client ]
:db/cardinality :db.cardinality/one
:db/unique :db.unique/identity}]})
#_(dc/q '[:find ?o
:where [_ :account-client-override/k ?o]]
(dc/db conn))
#_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k])

3
things-to-search-for.txt
View File

@@ -1,9 +1,8 @@
Look for getting all vendors (too slow) or clients
or-join syntax changed?
helper to lookup rel db-id
New way of automating ledger entries
Make sure no history on ledger
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
automatically rebuild search indexes
ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity
Wrap tests around every api call