;; This buffer is for Clojure experiments and evaluation.

;; Press C-j to evaluate the last expression.

;; You can also press C-u C-j to evaluate the expression and pretty-print its result.

(ns dump-edn
  (:require #_`[datomic.api :as d]

            [clojure.java.io :as io]
            [amazonica.aws.s3 :as s3]
            [config.core :refer [env]]
            [clojure.core.async :as a]
            [datomic.client.api :as dc]
            [lambdaisland.edn-lines :as ednl]
            [datomic.client.api.async :as dca]
            [datomic.dev-local :as dl]
            [clojure.set :as set]
            [clojure.string :as str]
            [clj-http.client :as client]))


(def remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod")))
(def local-client (dc/client {:server-type :dev-local
                             :system "dev"}))



(def get-schema []
  (let [everything         (->> (d/q '[:find [(pull ?e [:db/ident
                                                        {:db/valueType [:db/ident]}
                                                        {:db/cardinality [:db/ident]}
                                                        :db.attr/preds
                                                        {:db/unique [:db/ident]}
                                                        :db/isComponent
                                                        :db/id
                                                        :db/noHistory
                                                        :db/tupleAttrs
                                                        :db.entity/attrs
                                                        :db.entity/preds
                                                        :db/doc]) ...]
                                       :where [?e :db/ident]]
                                     remote-db))
        schema-attrs       (->> everything
                                (filter :db/ident)
                                (filter (fn [{:db/keys [ident]}]
                                          (if (namespace ident)
                                            (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident))
                                            true
                                            ))))
        meta-schema-schema (filter #(-> % :db/ident not) everything)]
    schema-attrs))

(def best-key-helper
  (->> schema
       (filter :db/valueType)
       (map :db/ident)
       (group-by namespace)
       #_(map (fn [[k v]]
              [k ])))
  )

(def entity->best-key
  {"transaction-rule"
   [:transaction-rule/description, :transaction-rule/note :transaction-rule/vendor]
   "square-location"
   :square-location/square-id,
   "expected-deposit"
   :expected-deposit/date,
   "journal-entry-line"
   [:journal-entry-line/account, :journal-entry-line/debit :journal-entry-line/credit]
   "vendor"
   [:vendor/name,]
   "transaction"
   :transaction/amount,
   "yodlee-provider-account"
   :yodlee-provider-account/id,
   "journal-entry"
   :journal-entry/source,
   "yodlee-merchant"  :yodlee-merchant/yodlee-id,
   "invoice"
   :invoice/invoice-number,
   "vendor-terms-override"
   :vendor-terms-override/client,
   "integration-status"
   :integration-status/state,
   "conformity"       :conformity/conformed-norms-index,
   "user"
   :user/provider-id,
   "sales-refund"
   :sales-refund/total,
   "plaid-account"
   :plaid-account/name,
   "charge"
   [:charge/total, :charge/external-id]
   "location-match"   :location-match/location,
   "vendor-schedule-payment-dom"
   :vendor-schedule-payment-dom/dom,
   "account-client-override"
   :account-client-override/client,
   "plaid-item"
   :plaid-item/client,
   "transaction-account"
   :transaction-account/account,
   "address"
   [:address/street1, :address/city :address/state :address/zip]
   "order-line-item"
   :order-line-item/total,
   "ezcater-location" :ezcater-location/location,
   "account"
   [:account/numeric-code, :account/code :account/name :account/type]
   "intuit-bank-account"
   :intuit-bank-account/name,
   "saved-query"
   :saved-query/guid,
   "ezcater-caterer"
   :ezcater-caterer/uuid,
   "forecasted-transaction"
   :forecasted-transaction/day-of-month,
   "audit"            :audit/user,
   "yodlee-account"
   :yodlee-account/id,
   "transaction-rule-account"
   [:transaction-rule-account/account, :transaction-rule-account/location]
   "ezcater-integration"
   :ezcater-integration/subscriber-uuid,
   "report"
   :report/created,
   "bank-account"
   :bank-account/code,
   "vendor-usage"
   :vendor-usage/key,
   "invoice-expense-account"
   [:invoice-expense-account/expense-account-id, :invoice-expense-account/account :invoice-expense-account/location :invoice-expense-account/amount]
   "sales-order"
   :sales-order/date,
   "client"
   :client/code,
   "email-contact"    :email-contact/email,
   "invoice-payment"
   :invoice-payment/amount,
   "contact"
   [:contact/name, :contact/phone :contact/email]
   "import-batch"
   :import-batch/date,
   "payment"
   [:payment/date, :payment/bank-account]
   "vendor-account-override"
   :vendor-account-override/client})

(def references (filter (comp #{:db.type/ref} :db/ident :db/valueType) schema ))





(def reference->entity  
  (->> (d/q '[:find ?a ?v3
              :in $ $$ [?a ...]
              :where [$$ _ ?a ?e]
              [$ ?e ?v _ _]
              [$ ?v :db/ident ?v2 _ _]
              [(namespace ?v2) ?v3]
              [(namespace ?v2) ?v3]]
            remote-db
            (d/since remote-db #inst "2022-06-01")
            (map :db/ident references)
            )
       (group-by first)
       (map (fn [[k v]]
              [k (disj (set (map second v)) "db")]))
       (into {})))

(def entities-that-need-manual
  (set (map namespace (filter (complement reference->entity) (map :db/ident references)))))

(def manual-dependencies
  {:client/location-matches #{"location-match"}
   :transaction/yodlee-merchant #{"yodlee-merchant"}
   :vendor-account-override/account #{"account"}
   :vendor-account-override/client #{"client"}
   :vendor/secondary-contact #{"contact"}
   :vendor/account-overrides #{"vendor-account-override"}
   :client/bank-accounts #{"bank-account"}
   :transaction-rule/yodlee-merchant #{"yodlee-merchant"}
   :client/forecasted-transactions #{"forecasted-transaction"}
   :transaction/forecast-match #{"forecasted-transaction"}
   :vendor/automatically-paid-when-due #{"client"}
   :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}
   :vendor/terms-overrides #{"vendor-terms-override"}
   :vendor-schedule-payment-dom/client #{"client"}})

(def full-dependencies 
  (update (merge-with into reference->entity manual-dependencies)
          :journal-entry/original-entity
          #(disj % "journal-entry")))

(def entity-dependencies 
  (let [base-dependencies
        (into 
          {}
          (map (fn [i]
                 [i #{}])
               (set (map (comp namespace :db/ident) 
                         (filter :db/valueType
                                 schema)))) 
          ) 
        ]
    (into base-dependencies (reduce
                              (fn [acc [ref deps]]
                                (update acc (namespace ref) (fnil #(into % deps) #{})))
                              {}
                              full-dependencies))))

(def full-dependencies
  {:invoice/client #{"client"},
 :sales-order/client #{"client"},
 :transaction-rule/transaction-approval-status #{},
 :transaction/forecast-match #{"forecasted-transaction"},
 :user/role #{},
 :vendor-schedule-payment-dom/client #{"client"},
 :invoice-payment/payment #{"payment"},
 :transaction-rule/client #{"client"},
 :invoice/status #{},
 :payment/type #{},
 :expected-deposit/client #{"client"},
 :transaction/bank-account #{"bank-account"},
 :transaction-rule-account/account #{"account"},
 :import-batch/status #{},
 :user/clients #{"client"},
 :payment/client #{"client"},
 :expected-deposit/charges #{"charge"},
 :vendor/automatically-paid-when-due #{"client"},
 :payment/invoices #{"invoice"},
 :client/forecasted-transactions #{"forecasted-transaction"},
 :transaction/matched-rule #{"transaction-rule"},
 :invoice/import-status #{},
 :charge/processor #{},
 :expected-deposit/vendor #{"vendor"},
 :client/square-locations #{"square-location"},
 :payment/status #{},
 :client/location-matches #{"location-match"},
 :saved-query/client #{"client"},
 :transaction/payment #{"payment"},
 :transaction-rule/vendor #{"vendor"},
 :plaid-item/client #{"client"},
 :account/applicability #{},
 :journal-entry-line/account #{"account" "bank-account"},
 :client/bank-accounts #{"bank-account"},
 :yodlee-provider-account/client #{"client"},
 :account/vendor-allowance #{},
 :payment/bank-account #{"bank-account"},
 :account/default-allowance #{},
 :transaction-rule/yodlee-merchant #{"yodlee-merchant"},
 :vendor/account-overrides #{"vendor-account-override"},
 :transaction/client #{"client"},
 :invoice/vendor #{"vendor"},
 :sales-order/vendor #{"vendor"},
 :expected-deposit/status #{},
 :journal-entry/original-entity #{"transaction" "invoice"},
 :vendor-usage/client #{"client"},
 :transaction/expected-deposit #{"expected-deposit"},
 :client/ezcater-locations #{"ezcater-location"},
 :journal-entry/client #{"client"},
 :vendor/secondary-contact #{"contact"},
 :journal-entry/line-items #{"journal-entry-line"},
 :vendor/legal-entity-1099-type #{},
 :transaction-rule/bank-account #{"bank-account"},
 :transaction-account/account #{"account"},
 :vendor/terms-overrides #{"vendor-terms-override"},
 :vendor/default-account #{"account"},
 :transaction/yodlee-merchant #{"yodlee-merchant"},
 :sales-refund/client #{"client"},
 :client/emails #{"email-contact"},
 :payment/vendor #{"vendor"},
 :invoice-payment/invoice #{"invoice"},
 :report/client #{"client"},
 :transaction-rule/accounts #{"transaction-rule-account"},
 :charge/client #{"client"},
 :bank-account/type #{},
 :invoice-expense-account/account #{"account"},
 :vendor/legal-entity-tin-type #{},
 :transaction/approval-status #{},
 :import-batch/entry #{"transaction"},
 :bank-account/intuit-bank-account #{"intuit-bank-account"},
 :account/type #{},
 :sales-refund/vendor #{"vendor"},
 :bank-account/yodlee-account #{"yodlee-account"},
 :vendor/address #{"address"},
 :integration-status/state #{},
 :transaction/accounts #{"transaction-account"},
 :sales-order/charges #{"charge"},
 :client/address #{"address"},
 :ezcater-location/caterer #{"ezcater-caterer"},
 :vendor-account-override/client #{"client"},
 :bank-account/integration-status #{"integration-status"},
 :yodlee-provider-account/accounts #{"yodlee-account"},
 :account/invoice-allowance #{},
 :journal-entry/vendor #{"vendor"},
 :plaid-item/accounts #{"plaid-account"},
 :vendor-usage/vendor #{"vendor"},
 :sales-order/line-items #{"order-line-item"},
 :invoice/expense-accounts #{"invoice-expense-account"},
 :account-client-override/client #{"client"},
 :vendor/primary-contact #{"contact"},
 :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"},
 :account/client-overrides #{"account-client-override"},
 :transaction/vendor #{"vendor"},
 :client/square-integration-status #{"integration-status"},
 :ezcater-integration/caterers #{"ezcater-caterer"},
 :vendor-account-override/account #{"account"},
 :import-batch/source #{}})

(def entity-dependencies
  {"transaction-rule"
   #{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account"
     "client"},
   "square-location"             #{},
   "expected-deposit"            #{"vendor" "charge" "client"},
   "journal-entry-line"          #{"account" "bank-account"},
   "vendor"
   #{"vendor-schedule-payment-dom" "address" "account" "client" "contact"
     "vendor-account-override"},
   "transaction"
   #{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant"
     "transaction-account" "forecasted-transaction" "bank-account" "client"
     "payment"},
   "yodlee-provider-account"     #{"yodlee-account" "client"},
   "journal-entry"
   #{"journal-entry-line" "vendor" "transaction" "invoice" "client"},
   "yodlee-merchant"             #{},
   "invoice"                     #{"vendor" "invoice-expense-account" "client"},
   "vendor-terms-override"       #{},
   "integration-status"          #{},
   "conformity"                  #{},
   "user"                        #{"client"},
   "sales-refund"                #{"vendor" "client"},
   "plaid-account"               #{},
   "charge"                      #{"client"},
   "location-match"              #{},
   "vendor-schedule-payment-dom" #{"client"},
   "account-client-override"     #{"client"},
   "plaid-item"                  #{"plaid-account" "client"},
   "transaction-account"         #{"account"},
   "address"                     #{},
   "order-line-item"             #{},
   "ezcater-location"            #{"ezcater-caterer"},
   "account"                     #{"account-client-override"},
   "intuit-bank-account"         #{},
   "saved-query"                 #{"client"},
   "ezcater-caterer"             #{},
   "forecasted-transaction"      #{},
   "audit"                       #{},
   "yodlee-account"              #{},
   "transaction-rule-account"    #{"account"},
   "ezcater-integration"         #{"ezcater-caterer"},
   "report"                      #{"client"},
   "bank-account"                #{"integration-status" "intuit-bank-account" "yodlee-account"},
   "vendor-usage"                #{"vendor" "client"},
   "invoice-expense-account"     #{"account"},
   "sales-order"                 #{"vendor" "charge" "order-line-item" "client"},
   "client"
   #{"square-location" "integration-status" "location-match" "address"
     "ezcater-location" "forecasted-transaction" "bank-account" "email-contact"},
   "email-contact"               #{},
   "invoice-payment"             #{"invoice" "payment"},
   "contact"                     #{},
   "import-batch"                #{"transaction"},
   "payment"                     #{"vendor" "invoice" "bank-account" "client"},
   "vendor-account-override"     #{"account" "client"}})

(defn order-of-insert [entity-dependencies]
  (loop [entity-dependencies entity-dependencies 
         order []]
    (let [next-order (for [[entity deps] entity-dependencies
                           :when         (not (seq deps))]
                       entity)
          next-deps (reduce
                      (fn [entity-dependencies next-entity]
                        (into {}
                              (map
                                (fn [[k v]]
                                  [k (disj v next-entity)])
                                entity-dependencies)))
                      (apply dissoc entity-dependencies next-order)
                      next-order)]
      (println order next-deps)
      (if (seq next-deps)
        (recur next-deps (into order next-order))
        (into order next-order)))))



#_(def best-attributes 
  (->> (map :db/ident (filter :db/valueType schema))
       (d/q
         '[:find ?a2 (count ?e)
           :in $ [?a2 ...]
           :where
           [?a :db/ident ?a2]
           [?e ?a]]
         remote-db)
       (map (fn [[a count]]
              [(namespace a) [a count]]))
       (reduce (fn [acc [namespace [attr count]]]
                 (update acc namespace
                         (fnil 
                           (fn [[curr-attr curr-count]]
                             (if (> count curr-count)
                               [attr count]
                               [curr-attr curr-count]))
                           ["" 0])))
               {})
       (map (fn [[k v]]
              [k (first v)]))))


(defn tx-pipeline
  "Transacts data from from-ch. Returns a map with:
     :result, a return channel getting {:error t} or {:completed n}
     :stop, a fn you can use to terminate early."
  [conn conc from-ch f]
  (let [to-ch (a/chan 400)
        done-ch (a/chan)
        transact-data (fn [data result]
                        (a/go 
                          (try
                            
                            (let [tx-r (a/<! (dca/transact conn {:tx-data (f data)}))]
                              (when (:db/error tx-r)
                                (throw (ex-info "Invalid transaction" tx-r)))
                              (a/>! result tx-r)
                              (a/close! result))
                                        ; if exception in a transaction
                                        ; will close channels and put error
                                        ; on done channel.
                            (catch Throwable t
                              (.printStackTrace t)
                              (a/close! from-ch)
                              (a/close! to-ch)
                              (a/>! done-ch {:error t})))))]

   ; go block prints a '.' after every 1000 transactions, puts completed
   ; report on done channel when no value left to be taken.
   (a/go-loop [total 0]
     (if (= (mod total 2) 0)
       (do  
         (print ".")
         (flush)))
     (if-let [c (a/<! to-ch)]
       (recur (inc total))
       (a/>! done-ch {:completed total})))

   ; pipeline that uses transducer form of map to transact data taken from
   ; from-ch and puts results on to-ch
   (a/pipeline-async conc to-ch transact-data from-ch)

   ; returns done channel and a function that you can use
   ; for early termination.
   {:result done-ch
    :stop (fn [] (a/close! to-ch))}))

(def loaded (atom #{}))

(defn migrate
  ([] (migrate nil))
  ([item-list]
   (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident       :entity/migration-key
                                                                                  :db/unique      :db.unique/identity
                                                                                  :db/cardinality :db.cardinality/one
                                                                                  :db/valueType   :db.type/long}]})
   (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map
                                                                                  (fn [s]
                                                                                    (set/rename-keys s {:db/id :entity/migration-key}))
                                                                                  schema)})

   (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
                                                                                 {:entity/migration-key 17592232621701}
                                                                                 {:entity/migration-key 17592263907739}
                                                                                 {:entity/migration-key 17592271516922}]

                                                                       })

   (dc/transact (dc/connect local-client {:db-name "prod-migration"})
                {:tx-data (->> (d/q '[:find ?e
                                      :in $ $$
                                      :where [$$ ?e :transaction-rule/note _ _ true]
                                      (not [$ ?e :transaction-rule/note] )]
                                    remote-db
                                    (d/history remote-db))
                               (map (fn [[old-rule]]
                                      {:entity/migration-key old-rule
                                       :db/doc               "A transaction rule that was deleted"})))})

   #_(dc/transact (dc/connect local-client {:db-name "prod-migration"})
                  {:tx-data (->> (d/q '[:find (count ?e)
                                        :in $ 
                                        :where
                                        #_[$$ ?e :transaction-account/account _ _ true]
                                        [_ :transaction/accounts ?e]
                                        (not [?e :transaction-account/account _] )
                                        ]
                                      remote-db
                                      #_(d/history remote-db)) 
                                 (map (fn [[old-rule]]
                                        {:entity/migration-key old-rule
                                         :db/doc               "A transaction account that was deleted"})))})


   (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
           :let   [_ (swap! loaded conj entity)
                   _ (println "querying for " entity)
                   entities (d/q '[:find [?e ...]
                                   :in $ [?a ...]
                                   :where [?e ?a]]
                                 remote-db
                                 (cond-> (entity->best-key entity)
                                   (not (vector? (entity->best-key entity))) vector))

                   _ (println "Found some! here's a few: " (take 3 entities))
                   tx-chan (a/chan 50)
                   entities->transaction (fn [entities]
                                           
                                           (->> (d/pull-many remote-db
                                                             (->> schema
                                                                  (filter :db/valueType)
                                                                  (mapv :db/ident)
                                                                  (filter #(= entity (namespace %)))
                                                                  (into [:db/id]))
                                                             entities)
                                                (mapv (fn [m ]
                                                        (reduce
                                                          (fn [m [k v]]
                                                            (cond
                                                              (= k :db/id)
                                                              (-> m
                                                                  (assoc :entity/migration-key v)
                                                                  (dissoc :db/id))
                                                              (full-dependencies k)
                                                              (if (vector? v)
                                                                (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
                                                                (assoc m k [:entity/migration-key (:db/id v)]))
                                                              :else
                                                              (dissoc m :payment/pdf-data
                                                                      :payment/memo)))
                                                          m
                                                          m)))))
                   _ (println "Inserting " entity ": " (count entities))
                   _ (flush)
                   pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"})
                                         10
                                         tx-chan
                                         entities->transaction)]]
     (doseq [batch (partition-all 100 entities)]
       (try 
         (a/>!! tx-chan batch)
         (catch Exception e
           (println e)
           ((:stop pipeline)))))

     (println "waiting for done from" pipeline)
     (flush)
     (a/close! tx-chan)
     (println (a/<!! (:result pipeline)))
     ((:stop pipeline))
     (println)
     (println "Done"))))

(def dumped (atom #{}))


(defn dump-schema [backup]
  (spit (io/file (str backup "/schema.edn"))
        (with-out-str (prn (map
                             (fn [s]
                               (set/rename-keys s {:db/id :entity/migration-key}))
                             schema))))
  (spit (io/file (str backup "/full-dependencies.edn"))
        (with-out-str (prn full-dependencies)))
  (spit (io/file (str backup "/entity-dependencies.edn"))
        (with-out-str (prn entity-dependencies))))

(defn dump-all
  ([] (dump-all nil))
  ([item-list]
   (let [backup-id (str "backups/" (java.util.UUID/randomUUID))]
     (.mkdir (io/file backup-id))
     (dump-schema backup-id)
     (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
             :let   [_ (swap! dumped conj entity)
                     _ (println "querying for " entity)
                     entities (d/q '[:find [?e ...]
                                     :in $ [?a ...]
                                     :where [?e ?a]]
                                   remote-db
                                   (cond-> (entity->best-key entity)
                                     (not (vector? (entity->best-key entity))) vector))

                     entities->transaction (fn [entities]
                                             
                                             (->> (d/pull-many remote-db
                                                               (->> schema
                                                                    (filter :db/valueType)
                                                                    (mapv :db/ident)
                                                                    (filter #(= entity (namespace %)))
                                                                    (into [:db/id]))
                                                               entities)
                                                  (mapv (fn [m ]
                                                          (reduce
                                                            (fn [m [k v]]
                                                              (cond
                                                                (= k :db/id)
                                                                (-> m
                                                                    (assoc :entity/migration-key v)
                                                                    (dissoc :db/id))
                                                                (full-dependencies k)
                                                                (if (vector? v)
                                                                  (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
                                                                  (assoc m k [:entity/migration-key (:db/id v)]))
                                                                :else
                                                                (dissoc m :payment/pdf-data
                                                                        :payment/memo)))
                                                            m
                                                            m)))))
                     _ (println "Inserting " entity ": " (count entities))
                     _ (flush)]]
       (ednl/with-append [append (str backup-id "/" entity ".ednl")]
         (doseq [batch (partition-all 1000 entities)
                 :let [_ (do (print ".") (flush))]
                 item (entities->transaction batch)]
           (try 
             (append item)
             (catch Exception e
               (println e)
               (throw e)))))
       (println "Done")))))

(defn load-from-backup 
  ([backup-id connection] (load-from-backup backup-id connection nil))
  ([backup-id connection item-list]
   (let [schema              (clojure.edn/read-string (slurp (str backup-id "/schema.edn")))
         full-dependencies   (clojure.edn/read-string (slurp (str backup-id "/full-dependencies.edn")))
         entity-dependencies (clojure.edn/read-string (slurp (str backup-id "/entity-dependencies.edn")))]
     (dc/transact connection {:tx-data [{:db/ident       :entity/migration-key
                                         :db/unique      :db.unique/identity
                                         :db/cardinality :db.cardinality/one
                                         :db/valueType   :db.type/long}]})
     (dc/transact connection {:tx-data (map
                                         (fn [s]
                                           (set/rename-keys s {:db/id :entity/migration-key}))
                                         schema)})

     (dc/transact connection {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
                                        {:entity/migration-key 17592232621701}
                                        {:entity/migration-key 17592263907739}
                                        {:entity/migration-key 17592271516922}]

                              })



     (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
             :let   [_ (swap! loaded conj entity)
                     _ (println "querying for " entity)
                     entities (ednl/slurp (str backup-id "/" entity ".ednl"))

                     _ (println "Found some! here's a few: " (take 3 entities))
                     tx-chan (a/chan 50)
                     entities->transaction (fn [entities]
                                             entities)
                     pipeline (tx-pipeline connection
                                           10
                                           tx-chan
                                           entities->transaction)]]
       (doseq [batch (partition-all 100 entities)]
         (try 
           (a/>!! tx-chan batch)
           (catch Exception e
             (println e)
             ((:stop pipeline)))))

       (println "waiting for done from" pipeline)
       (flush)
       (a/close! tx-chan)
       (println (a/<!! (:result pipeline)))
       ((:stop pipeline))
       (println)
       (println "Done")))))

(defn reset-migrate []
  (dc/delete-database local-client {:db-name "prod-migration"})
  (dc/create-database local-client {:db-name "prod-migration"})
  (migrate))



(comment 
  ;; cloud load
  (do
    (let [client (dc/client {:server-type :cloud
                      :region      "us-east-1"
                      :system      "iol-cloud"
                      :endpoint    "https://53syis8n1m.execute-api.us-east-1.amazonaws.com"})
          ;; _ (dc/create-database client {:db-name "prod-mirror"})
          connection (dc/connect client {:db-name "prod-mirror"})]
      (load-from-backup "backups/8e245d3d-be7a-4d90-8e9e-e6a110582658" connection ["transaction"])

      )

    )
  (reset-migrate)

  (migrate)

  (datomic.dev-local/release-db {:db-name "prod-migration"
                                 :system  "dev"})
 
  
  (d/pull remote-db '[*] 17592271182081)
  ;; => {:db/id 17592271182081,
  ;;     :charge/type-name "NO_SALE",
  ;;     :charge/total 0.0,
  ;;     :charge/tip 0.0,
  ;;     :charge/processor #:db{:id 17592237965415},
  ;;     :charge/external-id "square/charge/ndHN8rnam4dsfnDtK78LL1KiuaB"}
  (def local-charges 
    (into #{}
          (map first)
          (dc/q '[:find ?m
                  :where (or [?e :charge/total]
                             [?e :charge/external-id])
                  [?e :entity/migration-key ?m]]
                (dc/db (dc/connect local-client {:db-name "prod-migration"})))))


  (count local-charges)
  

  ;; => [[2815787]]
  (def remote-charges
    (into #{}
          (map first)
          (d/q '[:find ?e
                 :where (or [?e :charge/total]
                            [?e :charge/external-id])]
                remote-db)))
  
  ;; => [[2815955]]

  (d/pull-many remote-db
               (->> schema
                    (filter :db/valueType)
                    (mapv :db/ident)
                    (filter #(= "charge" (namespace %)))
                    (into [:db/id]))
               (vec (clojure.set/difference remote-charges local-charges)))

  (dc/transact 
    (dc/connect local-client {:db-name "prod-migration"})
    {:tx-data (->> (d/pull-many remote-db
                                (->> schema
                                     (filter :db/valueType)
                                     (mapv :db/ident)
                                     (filter #(= "charge" (namespace %)))
                                     (into [:db/id]))
                                (vec (clojure.set/difference remote-charges local-charges))) 
                   (mapv (fn [m ]
                           (reduce
                             (fn [m [k v]]
                               (cond
                                 (= k :db/id)
                                 (-> m
                                     (assoc :entity/migration-key v)
                                     (dissoc :db/id))
                                 (full-dependencies k)
                                 (if (vector? v)
                                   (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
                                   (assoc m k [:entity/migration-key (:db/id v)]))
                                 :else
                                 (dissoc m :payment/pdf-data
                                         :payment/memo)))
                             m
                             m))))})

  (dc/pull (dc/db (dc/connect local-client {:db-name "prod-migration"})) '[*] [:entity/migration-key 17592271182081])
  (d/pull remote-db '[*] 17592232186542)

  )
;; => nil
