notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Bryce Covert
2022-09-30 06:41:05 -07:00
parent 1f57ed2d1c
commit aebf95a87b
3 changed files with 78 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
scratch-sessions/dump-edn.clj
View File

@@ -253,33 +253,37 @@
"Transacts data from from-ch. Returns a map with: "Transacts data from from-ch. Returns a map with:
:result, a return channel getting {:error t} or {:completed n} :result, a return channel getting {:error t} or {:completed n}
:stop, a fn you can use to terminate early." :stop, a fn you can use to terminate early."
[conn conc from-ch] [conn conc from-ch f]
(let [to-ch (a/chan 100) (let [to-ch (a/chan 400)
done-ch (a/chan) done-ch (a/chan)
transact-data (fn [data] transact-data (fn [data result]
(try (a/go
(dc/transact conn {:tx-data data}) (try
; if exception in a transaction (a/>! result (a/<! (dca/transact conn {:tx-data (f data)})))
; will close channels and put error (a/close! result)
; on done channel. ; if exception in a transaction
(catch Throwable t ; will close channels and put error
(.printStackTrace t) ; on done channel.
(a/close! from-ch) (catch Throwable t
(a/close! to-ch) (.printStackTrace t)
(a/>!! done-ch {:error t}))))] (a/close! from-ch)
(a/close! to-ch)
(a/>! done-ch {:error t})))))]
; go block prints a '.' after every 1000 transactions, puts completed ; go block prints a '.' after every 1000 transactions, puts completed
; report on done channel when no value left to be taken. ; report on done channel when no value left to be taken.
(a/go-loop [total 0] (a/go-loop [total 0]
(print ".") (if (= (mod total 2) 0)
(flush) (do
(print ".")
(flush)))
(if-let [c (a/<! to-ch)] (if-let [c (a/<! to-ch)]
(recur (inc total)) (recur (inc total))
(a/>! done-ch {:completed total}))) (a/>! done-ch {:completed total})))
; pipeline that uses transducer form of map to transact data taken from ; pipeline that uses transducer form of map to transact data taken from
; from-ch and puts results on to-ch ; from-ch and puts results on to-ch
(a/pipeline-blocking conc to-ch (map transact-data) from-ch) (a/pipeline-async conc to-ch transact-data from-ch)
; returns done channel and a function that you can use ; returns done channel and a function that you can use
; for early termination. ; for early termination.
@@ -341,41 +345,50 @@
(cond-> (entity->best-key entity) (cond-> (entity->best-key entity)
(not (vector? (entity->best-key entity))) vector)) (not (vector? (entity->best-key entity))) vector))
tx-chan (a/chan) tx-chan (a/chan 400)
pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"}) 30 entities->transaction (fn [entities]
tx-chan)]] (->> (d/pull-many remote-db
(println "Inserting " entity ": " (count entities)) (->> schema
(flush) (filter :db/valueType)
(doseq [batch (partition-all 2000 entities)] (mapv :db/ident)
(let [transaction {:tx-data (->> (d/pull-many remote-db (filter #(= entity (namespace %)))
(->> schema (into [:db/id]))
(filter :db/valueType) entities)
(mapv :db/ident) (mapv (fn [m ]
(filter #(= entity (namespace %))) (reduce
(into [:db/id])) (fn [m [k v]]
batch) (cond
(mapv (fn [m ] (= k :db/id)
(reduce (-> m
(fn [m [k v]] (assoc :entity/migration-key v)
(cond (dissoc :db/id))
(= k :db/id) (full-dependencies k)
(-> m (if (vector? v)
(assoc :entity/migration-key v) (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
(dissoc :db/id)) (assoc m k [:entity/migration-key (:db/id v)]))
(full-dependencies k) :else
(if (vector? v) (dissoc m :payment/pdf-data
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) :payment/memo)))
(assoc m k [:entity/migration-key (:db/id v)])) m
:else m)))))
(dissoc m :payment/pdf-data _ (println "Inserting " entity ": " (count entities))
:payment/memo))) _ (flush)
m pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"})
m))))}] 50
(a/>!! tx-chan (:tx-data transaction)))) tx-chan
(a/close tx-chan) entities->transaction)]]
(doseq [batch (partition-all 500 entities)]
(try
(a/>!! tx-chan batch)
(catch Exception e
(println e)
((:stop pipeline)))))
(println "waiting for done from" pipeline) (println "waiting for done from" pipeline)
(flush) (flush)
(a/close! tx-chan)
(println (a/<!! (:result pipeline))) (println (a/<!! (:result pipeline)))
((:stop pipeline))
(println) (println)
(println "Done"))) (println "Done")))

17
src/clj/auto_ap/graphql/accounts.clj
View File

@@ -161,7 +161,8 @@
(for [result (map first (dc/qseq '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}]) (for [result (map first (dc/qseq '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}])
:in $ :in $
:where [?aco :account-client-override/client ] :where [?aco :account-client-override/client ]
[?aco :account-client-override/search-terms ]] [?aco :account-client-override/search-terms ]
[_ :account/client-overrides ?aco]]
(dc/db conn))) (dc/db conn)))
:when (:account/numeric-code (:account/_client-overrides result))] :when (:account/numeric-code (:account/_client-overrides result))]
{:id (:db/id (:account/_client-overrides result)) {:id (:db/id (:account/_client-overrides result))
@@ -188,3 +189,17 @@
:location (:account/location result) :location (:account/location result)
:applicability (name (:db/ident (:account/applicability result)))}) :applicability (name (:db/ident (:account/applicability result)))})
"account")) "account"))
#_(dc/transact conn
{:tx-data [{:db/ident :account-client-override/k2
:db/valueType :db.type/tuple
:db/tupleAttrs [:account/_client-overrides :account-client-override/client ]
:db/cardinality :db.cardinality/one
:db/unique :db.unique/identity}]})
#_(dc/q '[:find ?o
:where [_ :account-client-override/k ?o]]
(dc/db conn))
#_(dc/pull (dc/db conn) '[*] [:db/ident :account-client-override/k])

3
things-to-search-for.txt
View File

@@ -1,9 +1,8 @@
Look for getting all vendors (too slow) or clients
or-join syntax changed? or-join syntax changed?
helper to lookup rel db-id helper to lookup rel db-id
New way of automating ledger entries
Make sure no history on ledger Make sure no history on ledger
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
automatically rebuild search indexes automatically rebuild search indexes
ezcater graphql needs search index too ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity make sure that temporary ids are set on all new things when using upsert-entity
Wrap tests around every api call