From bd658906b5fdf9867385fb6571c2e32d244974db Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 17 Mar 2023 23:29:28 -0700 Subject: [PATCH] Made tweaks to support datomic --- project.clj | 2 +- scratch-sessions/dump-edn.clj | 349 +++++++++++++++++------ src/clj/auto_ap/ssr/company_dropdown.clj | 8 +- 3 files changed, 264 insertions(+), 95 deletions(-) diff --git a/project.clj b/project.clj index c2f84b0f..a006d6ad 100644 --- a/project.clj +++ b/project.clj @@ -151,7 +151,7 @@ [com.bhauman/rebel-readline-cljs "0.1.4" :exclusions [org.clojure/clojurescript]] [javax.servlet/servlet-api "2.5"]] :plugins [[lein-pdo "0.1.1"]] - :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms8G" "-Xmx12G"]} + :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms4G" "-Xmx8G"]} :uberjar diff --git a/scratch-sessions/dump-edn.clj b/scratch-sessions/dump-edn.clj index b8fd7b2f..b61ba2a2 100644 --- a/scratch-sessions/dump-edn.clj +++ b/scratch-sessions/dump-edn.clj @@ -175,12 +175,15 @@ :transaction/yodlee-merchant #{"yodlee-merchant"} :vendor-account-override/account #{"account"} :vendor-account-override/client #{"client"} + :vendor/secondary-contact #{"contact"} :vendor/account-overrides #{"vendor-account-override"} + :client/bank-accounts #{"bank-account"} :transaction-rule/yodlee-merchant #{"yodlee-merchant"} :client/forecasted-transactions #{"forecasted-transaction"} :transaction/forecast-match #{"forecasted-transaction"} :vendor/automatically-paid-when-due #{"client"} :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"} + :vendor/terms-overrides #{"vendor-terms-override"} :vendor-schedule-payment-dom/client #{"client"}}) (def full-dependencies @@ -205,6 +208,162 @@ {} full-dependencies)))) +(def full-dependencies + {:invoice/client #{"client"}, + :sales-order/client #{"client"}, + :transaction-rule/transaction-approval-status #{}, + :transaction/forecast-match #{"forecasted-transaction"}, + :user/role #{}, + :vendor-schedule-payment-dom/client #{"client"}, + :invoice-payment/payment #{"payment"}, + :transaction-rule/client #{"client"}, + :invoice/status #{}, + :payment/type #{}, + :expected-deposit/client #{"client"}, + :transaction/bank-account #{"bank-account"}, + :transaction-rule-account/account #{"account"}, + :import-batch/status #{}, + :user/clients #{"client"}, + :payment/client #{"client"}, + :expected-deposit/charges #{"charge"}, + :vendor/automatically-paid-when-due #{"client"}, + :payment/invoices #{"invoice"}, + :client/forecasted-transactions #{"forecasted-transaction"}, + :transaction/matched-rule #{"transaction-rule"}, + :invoice/import-status #{}, + :charge/processor #{}, + :expected-deposit/vendor #{"vendor"}, + :client/square-locations #{"square-location"}, + :payment/status #{}, + :client/location-matches #{"location-match"}, + :saved-query/client #{"client"}, + :transaction/payment #{"payment"}, + :transaction-rule/vendor #{"vendor"}, + :plaid-item/client #{"client"}, + :account/applicability #{}, + :journal-entry-line/account #{"account" "bank-account"}, + :client/bank-accounts #{"bank-account"}, + :yodlee-provider-account/client #{"client"}, + :account/vendor-allowance #{}, + :payment/bank-account #{"bank-account"}, + :account/default-allowance #{}, + :transaction-rule/yodlee-merchant #{"yodlee-merchant"}, + :vendor/account-overrides #{"vendor-account-override"}, + :transaction/client #{"client"}, + :invoice/vendor #{"vendor"}, + :sales-order/vendor #{"vendor"}, + :expected-deposit/status #{}, + :journal-entry/original-entity #{"transaction" "invoice"}, + :vendor-usage/client #{"client"}, + :transaction/expected-deposit #{"expected-deposit"}, + :client/ezcater-locations #{"ezcater-location"}, + :journal-entry/client #{"client"}, + :vendor/secondary-contact #{"contact"}, + :journal-entry/line-items #{"journal-entry-line"}, + :vendor/legal-entity-1099-type #{}, + :transaction-rule/bank-account #{"bank-account"}, + :transaction-account/account #{"account"}, + :vendor/terms-overrides #{"vendor-terms-override"}, + :vendor/default-account #{"account"}, + :transaction/yodlee-merchant #{"yodlee-merchant"}, + :sales-refund/client #{"client"}, + :client/emails #{"email-contact"}, + :payment/vendor #{"vendor"}, + :invoice-payment/invoice #{"invoice"}, + :report/client #{"client"}, + :transaction-rule/accounts #{"transaction-rule-account"}, + :charge/client #{"client"}, + :bank-account/type #{}, + :invoice-expense-account/account #{"account"}, + :vendor/legal-entity-tin-type #{}, + :transaction/approval-status #{}, + :import-batch/entry #{"transaction"}, + :bank-account/intuit-bank-account #{"intuit-bank-account"}, + :account/type #{}, + :sales-refund/vendor #{"vendor"}, + :bank-account/yodlee-account #{"yodlee-account"}, + :vendor/address #{"address"}, + :integration-status/state #{}, + :transaction/accounts #{"transaction-account"}, + :sales-order/charges #{"charge"}, + :client/address #{"address"}, + :ezcater-location/caterer #{"ezcater-caterer"}, + :vendor-account-override/client #{"client"}, + :bank-account/integration-status #{"integration-status"}, + :yodlee-provider-account/accounts #{"yodlee-account"}, + :account/invoice-allowance #{}, + :journal-entry/vendor #{"vendor"}, + :plaid-item/accounts #{"plaid-account"}, + :vendor-usage/vendor #{"vendor"}, + :sales-order/line-items #{"order-line-item"}, + :invoice/expense-accounts #{"invoice-expense-account"}, + :account-client-override/client #{"client"}, + :vendor/primary-contact #{"contact"}, + :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}, + :account/client-overrides #{"account-client-override"}, + :transaction/vendor #{"vendor"}, + :client/square-integration-status #{"integration-status"}, + :ezcater-integration/caterers #{"ezcater-caterer"}, + :vendor-account-override/account #{"account"}, + :import-batch/source #{}}) + +(def entity-dependencies + {"transaction-rule" + #{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account" + "client"}, + "square-location" #{}, + "expected-deposit" #{"vendor" "charge" "client"}, + "journal-entry-line" #{"account" "bank-account"}, + "vendor" + #{"vendor-schedule-payment-dom" "address" "account" "client" "contact" + "vendor-account-override"}, + "transaction" + #{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant" + "transaction-account" "forecasted-transaction" "bank-account" "client" + "payment"}, + "yodlee-provider-account" #{"yodlee-account" "client"}, + "journal-entry" + #{"journal-entry-line" "vendor" "transaction" "invoice" "client"}, + "yodlee-merchant" #{}, + "invoice" #{"vendor" "invoice-expense-account" "client"}, + "vendor-terms-override" #{}, + "integration-status" #{}, + "conformity" #{}, + "user" #{"client"}, + "sales-refund" #{"vendor" "client"}, + "plaid-account" #{}, + "charge" #{"client"}, + "location-match" #{}, + "vendor-schedule-payment-dom" #{"client"}, + "account-client-override" #{"client"}, + "plaid-item" #{"plaid-account" "client"}, + "transaction-account" #{"account"}, + "address" #{}, + "order-line-item" #{}, + "ezcater-location" #{"ezcater-caterer"}, + "account" #{"account-client-override"}, + "intuit-bank-account" #{}, + "saved-query" #{"client"}, + "ezcater-caterer" #{}, + "forecasted-transaction" #{}, + "audit" #{}, + "yodlee-account" #{}, + "transaction-rule-account" #{"account"}, + "ezcater-integration" #{"ezcater-caterer"}, + "report" #{"client"}, + "bank-account" #{"integration-status" "intuit-bank-account" "yodlee-account"}, + "vendor-usage" #{"vendor" "client"}, + "invoice-expense-account" #{"account"}, + "sales-order" #{"vendor" "charge" "order-line-item" "client"}, + "client" + #{"square-location" "integration-status" "location-match" "address" + "ezcater-location" "forecasted-transaction" "bank-account" "email-contact"}, + "email-contact" #{}, + "invoice-payment" #{"invoice" "payment"}, + "contact" #{}, + "import-batch" #{"transaction"}, + "payment" #{"vendor" "invoice" "bank-account" "client"}, + "vendor-account-override" #{"account" "client"}}) (def order-of-insert (loop [entity-dependencies entity-dependencies order []] @@ -225,6 +384,8 @@ (recur next-deps (into order next-order)) (into order next-order))))) + + #_(def best-attributes (->> (map :db/ident (filter :db/valueType schema)) (d/q @@ -259,8 +420,12 @@ transact-data (fn [data result] (a/go (try - (a/>! result (a/! result tx-r) + (a/close! result)) ; if exception in a transaction ; will close channels and put error ; on done channel. @@ -292,105 +457,109 @@ (def loaded (atom #{})) -(defn migrate [] - (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident :entity/migration-key - :db/unique :db.unique/identity - :db/cardinality :db.cardinality/one - :db/valueType :db.type/long}]}) - (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map - (fn [s] - (set/rename-keys s {:db/id :entity/migration-key})) - schema)}) +(defn migrate + ([] (migrate nil)) + ([item-list] + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident :entity/migration-key + :db/unique :db.unique/identity + :db/cardinality :db.cardinality/one + :db/valueType :db.type/long}]}) + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map + (fn [s] + (set/rename-keys s {:db/id :entity/migration-key})) + schema)}) - (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"} - {:entity/migration-key 17592232621701} - {:entity/migration-key 17592263907739} - {:entity/migration-key 17592271516922}] + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"} + {:entity/migration-key 17592232621701} + {:entity/migration-key 17592263907739} + {:entity/migration-key 17592271516922}] - }) + }) - (dc/transact (dc/connect local-client {:db-name "prod-migration"}) - {:tx-data (->> (d/q '[:find ?e - :in $ $$ - :where [$$ ?e :transaction-rule/note _ _ true] - (not [$ ?e :transaction-rule/note] )] + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) + {:tx-data (->> (d/q '[:find ?e + :in $ $$ + :where [$$ ?e :transaction-rule/note _ _ true] + (not [$ ?e :transaction-rule/note] )] remote-db (d/history remote-db)) - (map (fn [[old-rule]] - {:entity/migration-key old-rule - :db/doc "A transaction rule that was deleted"})))}) + (map (fn [[old-rule]] + {:entity/migration-key old-rule + :db/doc "A transaction rule that was deleted"})))}) - #_(dc/transact (dc/connect local-client {:db-name "prod-migration"}) - {:tx-data (->> (d/q '[:find (count ?e) - :in $ - :where - #_[$$ ?e :transaction-account/account _ _ true] - [_ :transaction/accounts ?e] - (not [?e :transaction-account/account _] ) - ] - remote-db - #_(d/history remote-db)) - (map (fn [[old-rule]] - {:entity/migration-key old-rule - :db/doc "A transaction account that was deleted"})))}) + #_(dc/transact (dc/connect local-client {:db-name "prod-migration"}) + {:tx-data (->> (d/q '[:find (count ?e) + :in $ + :where + #_[$$ ?e :transaction-account/account _ _ true] + [_ :transaction/accounts ?e] + (not [?e :transaction-account/account _] ) + ] + remote-db + #_(d/history remote-db)) + (map (fn [[old-rule]] + {:entity/migration-key old-rule + :db/doc "A transaction account that was deleted"})))}) - (doseq [entity (filter (complement #{"audit"}) order-of-insert) - :let [_ (swap! loaded conj entity) - _ (println "querying for " entity) - entities (d/q '[:find [?e ...] - :in $ [?a ...] - :where [?e ?a]] + (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) order-of-insert)) + :let [_ (swap! loaded conj entity) + _ (println "querying for " entity) + entities (d/q '[:find [?e ...] + :in $ [?a ...] + :where [?e ?a]] remote-db (cond-> (entity->best-key entity) (not (vector? (entity->best-key entity))) vector)) - tx-chan (a/chan 400) - entities->transaction (fn [entities] - (->> (d/pull-many remote-db - (->> schema - (filter :db/valueType) - (mapv :db/ident) - (filter #(= entity (namespace %))) - (into [:db/id])) - entities) - (mapv (fn [m ] - (reduce - (fn [m [k v]] - (cond - (= k :db/id) - (-> m - (assoc :entity/migration-key v) - (dissoc :db/id)) - (full-dependencies k) - (if (vector? v) - (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) - (assoc m k [:entity/migration-key (:db/id v)])) - :else - (dissoc m :payment/pdf-data - :payment/memo))) - m - m))))) - _ (println "Inserting " entity ": " (count entities)) - _ (flush) - pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"}) - 50 - tx-chan - entities->transaction)]] - (doseq [batch (partition-all 500 entities)] - (try - (a/>!! tx-chan batch) - (catch Exception e - (println e) - ((:stop pipeline))))) + _ (println "Found some! here's a few: " (take 3 entities)) + tx-chan (a/chan 50) + entities->transaction (fn [entities] + + (->> (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= entity (namespace %))) + (into [:db/id])) + entities) + (mapv (fn [m ] + (reduce + (fn [m [k v]] + (cond + (= k :db/id) + (-> m + (assoc :entity/migration-key v) + (dissoc :db/id)) + (full-dependencies k) + (if (vector? v) + (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) + (assoc m k [:entity/migration-key (:db/id v)])) + :else + (dissoc m :payment/pdf-data + :payment/memo))) + m + m))))) + _ (println "Inserting " entity ": " (count entities)) + _ (flush) + pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"}) + 10 + tx-chan + entities->transaction)]] + (doseq [batch (partition-all 100 entities)] + (try + (a/>!! tx-chan batch) + (catch Exception e + (println e) + ((:stop pipeline))))) - (println "waiting for done from" pipeline) - (flush) - (a/close! tx-chan) - (println (a/> (d/q '[:find ?c ?n + (let [options (->> (dc/q '[:find ?c ?n :in $ ?user :where [?c :client/name ?n] [(auto-ap.graphql.utils/can-see-client? ?user ?c)]] - (d/db conn) + (dc/db conn) identity) (map (fn [[k v]] {"key" k @@ -105,7 +105,7 @@ document.getElementById(\"company-search-value\").dispatchEvent(new Event('chang (assert-can-see-client identity client-id)) (let [new-session (assoc (:session request) :client (when client-id - (d/pull (d/db conn) [:db/id :client/name :client/code] client-id)))] + (dc/pull (dc/db conn) [:db/id :client/name :client/code] client-id)))] (assoc (html-response (dropdown (assoc request :session new-session)))