notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Made tweaks to support datomic

Browse Source
This commit is contained in:
Bryce Covert
2023-03-17 23:29:28 -07:00
parent f7fe2b2bee
commit bd658906b5
3 changed files with 264 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
project.clj
View File

@@ -151,7 +151,7 @@
[com.bhauman/rebel-readline-cljs "0.1.4" :exclusions [org.clojure/clojurescript]] [com.bhauman/rebel-readline-cljs "0.1.4" :exclusions [org.clojure/clojurescript]]
[javax.servlet/servlet-api "2.5"]] [javax.servlet/servlet-api "2.5"]]
:plugins [[lein-pdo "0.1.1"]] :plugins [[lein-pdo "0.1.1"]]
:jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms8G" "-Xmx12G"]} :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms4G" "-Xmx8G"]}
:uberjar :uberjar

349
scratch-sessions/dump-edn.clj
View File

@@ -175,12 +175,15 @@
:transaction/yodlee-merchant #{"yodlee-merchant"} :transaction/yodlee-merchant #{"yodlee-merchant"}
:vendor-account-override/account #{"account"} :vendor-account-override/account #{"account"}
:vendor-account-override/client #{"client"} :vendor-account-override/client #{"client"}
:vendor/secondary-contact #{"contact"}
:vendor/account-overrides #{"vendor-account-override"} :vendor/account-overrides #{"vendor-account-override"}
:client/bank-accounts #{"bank-account"}
:transaction-rule/yodlee-merchant #{"yodlee-merchant"} :transaction-rule/yodlee-merchant #{"yodlee-merchant"}
:client/forecasted-transactions #{"forecasted-transaction"} :client/forecasted-transactions #{"forecasted-transaction"}
:transaction/forecast-match #{"forecasted-transaction"} :transaction/forecast-match #{"forecasted-transaction"}
:vendor/automatically-paid-when-due #{"client"} :vendor/automatically-paid-when-due #{"client"}
:vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"} :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}
:vendor/terms-overrides #{"vendor-terms-override"}
:vendor-schedule-payment-dom/client #{"client"}}) :vendor-schedule-payment-dom/client #{"client"}})
(def full-dependencies (def full-dependencies
@@ -205,6 +208,162 @@
{} {}
full-dependencies)))) full-dependencies))))
(def full-dependencies
{:invoice/client #{"client"},
:sales-order/client #{"client"},
:transaction-rule/transaction-approval-status #{},
:transaction/forecast-match #{"forecasted-transaction"},
:user/role #{},
:vendor-schedule-payment-dom/client #{"client"},
:invoice-payment/payment #{"payment"},
:transaction-rule/client #{"client"},
:invoice/status #{},
:payment/type #{},
:expected-deposit/client #{"client"},
:transaction/bank-account #{"bank-account"},
:transaction-rule-account/account #{"account"},
:import-batch/status #{},
:user/clients #{"client"},
:payment/client #{"client"},
:expected-deposit/charges #{"charge"},
:vendor/automatically-paid-when-due #{"client"},
:payment/invoices #{"invoice"},
:client/forecasted-transactions #{"forecasted-transaction"},
:transaction/matched-rule #{"transaction-rule"},
:invoice/import-status #{},
:charge/processor #{},
:expected-deposit/vendor #{"vendor"},
:client/square-locations #{"square-location"},
:payment/status #{},
:client/location-matches #{"location-match"},
:saved-query/client #{"client"},
:transaction/payment #{"payment"},
:transaction-rule/vendor #{"vendor"},
:plaid-item/client #{"client"},
:account/applicability #{},
:journal-entry-line/account #{"account" "bank-account"},
:client/bank-accounts #{"bank-account"},
:yodlee-provider-account/client #{"client"},
:account/vendor-allowance #{},
:payment/bank-account #{"bank-account"},
:account/default-allowance #{},
:transaction-rule/yodlee-merchant #{"yodlee-merchant"},
:vendor/account-overrides #{"vendor-account-override"},
:transaction/client #{"client"},
:invoice/vendor #{"vendor"},
:sales-order/vendor #{"vendor"},
:expected-deposit/status #{},
:journal-entry/original-entity #{"transaction" "invoice"},
:vendor-usage/client #{"client"},
:transaction/expected-deposit #{"expected-deposit"},
:client/ezcater-locations #{"ezcater-location"},
:journal-entry/client #{"client"},
:vendor/secondary-contact #{"contact"},
:journal-entry/line-items #{"journal-entry-line"},
:vendor/legal-entity-1099-type #{},
:transaction-rule/bank-account #{"bank-account"},
:transaction-account/account #{"account"},
:vendor/terms-overrides #{"vendor-terms-override"},
:vendor/default-account #{"account"},
:transaction/yodlee-merchant #{"yodlee-merchant"},
:sales-refund/client #{"client"},
:client/emails #{"email-contact"},
:payment/vendor #{"vendor"},
:invoice-payment/invoice #{"invoice"},
:report/client #{"client"},
:transaction-rule/accounts #{"transaction-rule-account"},
:charge/client #{"client"},
:bank-account/type #{},
:invoice-expense-account/account #{"account"},
:vendor/legal-entity-tin-type #{},
:transaction/approval-status #{},
:import-batch/entry #{"transaction"},
:bank-account/intuit-bank-account #{"intuit-bank-account"},
:account/type #{},
:sales-refund/vendor #{"vendor"},
:bank-account/yodlee-account #{"yodlee-account"},
:vendor/address #{"address"},
:integration-status/state #{},
:transaction/accounts #{"transaction-account"},
:sales-order/charges #{"charge"},
:client/address #{"address"},
:ezcater-location/caterer #{"ezcater-caterer"},
:vendor-account-override/client #{"client"},
:bank-account/integration-status #{"integration-status"},
:yodlee-provider-account/accounts #{"yodlee-account"},
:account/invoice-allowance #{},
:journal-entry/vendor #{"vendor"},
:plaid-item/accounts #{"plaid-account"},
:vendor-usage/vendor #{"vendor"},
:sales-order/line-items #{"order-line-item"},
:invoice/expense-accounts #{"invoice-expense-account"},
:account-client-override/client #{"client"},
:vendor/primary-contact #{"contact"},
:vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"},
:account/client-overrides #{"account-client-override"},
:transaction/vendor #{"vendor"},
:client/square-integration-status #{"integration-status"},
:ezcater-integration/caterers #{"ezcater-caterer"},
:vendor-account-override/account #{"account"},
:import-batch/source #{}})
(def entity-dependencies
{"transaction-rule"
#{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account"
"client"},
"square-location" #{},
"expected-deposit" #{"vendor" "charge" "client"},
"journal-entry-line" #{"account" "bank-account"},
"vendor"
#{"vendor-schedule-payment-dom" "address" "account" "client" "contact"
"vendor-account-override"},
"transaction"
#{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant"
"transaction-account" "forecasted-transaction" "bank-account" "client"
"payment"},
"yodlee-provider-account" #{"yodlee-account" "client"},
"journal-entry"
#{"journal-entry-line" "vendor" "transaction" "invoice" "client"},
"yodlee-merchant" #{},
"invoice" #{"vendor" "invoice-expense-account" "client"},
"vendor-terms-override" #{},
"integration-status" #{},
"conformity" #{},
"user" #{"client"},
"sales-refund" #{"vendor" "client"},
"plaid-account" #{},
"charge" #{"client"},
"location-match" #{},
"vendor-schedule-payment-dom" #{"client"},
"account-client-override" #{"client"},
"plaid-item" #{"plaid-account" "client"},
"transaction-account" #{"account"},
"address" #{},
"order-line-item" #{},
"ezcater-location" #{"ezcater-caterer"},
"account" #{"account-client-override"},
"intuit-bank-account" #{},
"saved-query" #{"client"},
"ezcater-caterer" #{},
"forecasted-transaction" #{},
"audit" #{},
"yodlee-account" #{},
"transaction-rule-account" #{"account"},
"ezcater-integration" #{"ezcater-caterer"},
"report" #{"client"},
"bank-account" #{"integration-status" "intuit-bank-account" "yodlee-account"},
"vendor-usage" #{"vendor" "client"},
"invoice-expense-account" #{"account"},
"sales-order" #{"vendor" "charge" "order-line-item" "client"},
"client"
#{"square-location" "integration-status" "location-match" "address"
"ezcater-location" "forecasted-transaction" "bank-account" "email-contact"},
"email-contact" #{},
"invoice-payment" #{"invoice" "payment"},
"contact" #{},
"import-batch" #{"transaction"},
"payment" #{"vendor" "invoice" "bank-account" "client"},
"vendor-account-override" #{"account" "client"}})
(def order-of-insert (def order-of-insert
(loop [entity-dependencies entity-dependencies (loop [entity-dependencies entity-dependencies
order []] order []]
@@ -225,6 +384,8 @@
(recur next-deps (into order next-order)) (recur next-deps (into order next-order))
(into order next-order))))) (into order next-order)))))
#_(def best-attributes #_(def best-attributes
(->> (map :db/ident (filter :db/valueType schema)) (->> (map :db/ident (filter :db/valueType schema))
(d/q (d/q
@@ -259,8 +420,12 @@
transact-data (fn [data result] transact-data (fn [data result]
(a/go (a/go
(try (try
(a/>! result (a/<! (dca/transact conn {:tx-data (f data)})))
(a/close! result) (let [tx-r (a/<! (dca/transact conn {:tx-data (f data)}))]
(when (:db/error tx-r)
(throw (ex-info "Invalid transaction" tx-r)))
(a/>! result tx-r)
(a/close! result))
; if exception in a transaction ; if exception in a transaction
; will close channels and put error ; will close channels and put error
; on done channel. ; on done channel.
@@ -292,105 +457,109 @@
(def loaded (atom #{})) (def loaded (atom #{}))
(defn migrate [] (defn migrate
(dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident :entity/migration-key ([] (migrate nil))
:db/unique :db.unique/identity ([item-list]
:db/cardinality :db.cardinality/one (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident :entity/migration-key
:db/valueType :db.type/long}]}) :db/unique :db.unique/identity
(dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map :db/cardinality :db.cardinality/one
(fn [s] :db/valueType :db.type/long}]})
(set/rename-keys s {:db/id :entity/migration-key})) (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map
schema)}) (fn [s]
(set/rename-keys s {:db/id :entity/migration-key}))
schema)})
(dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"} (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
{:entity/migration-key 17592232621701} {:entity/migration-key 17592232621701}
{:entity/migration-key 17592263907739} {:entity/migration-key 17592263907739}
{:entity/migration-key 17592271516922}] {:entity/migration-key 17592271516922}]
}) })
(dc/transact (dc/connect local-client {:db-name "prod-migration"}) (dc/transact (dc/connect local-client {:db-name "prod-migration"})
{:tx-data (->> (d/q '[:find ?e {:tx-data (->> (d/q '[:find ?e
:in $ $$ :in $ $$
:where [$$ ?e :transaction-rule/note _ _ true] :where [$$ ?e :transaction-rule/note _ _ true]
(not [$ ?e :transaction-rule/note] )] (not [$ ?e :transaction-rule/note] )]
remote-db remote-db
(d/history remote-db)) (d/history remote-db))
(map (fn [[old-rule]] (map (fn [[old-rule]]
{:entity/migration-key old-rule {:entity/migration-key old-rule
:db/doc "A transaction rule that was deleted"})))}) :db/doc "A transaction rule that was deleted"})))})
#_(dc/transact (dc/connect local-client {:db-name "prod-migration"}) #_(dc/transact (dc/connect local-client {:db-name "prod-migration"})
{:tx-data (->> (d/q '[:find (count ?e) {:tx-data (->> (d/q '[:find (count ?e)
:in $ :in $
:where :where
#_[$$ ?e :transaction-account/account _ _ true] #_[$$ ?e :transaction-account/account _ _ true]
[_ :transaction/accounts ?e] [_ :transaction/accounts ?e]
(not [?e :transaction-account/account _] ) (not [?e :transaction-account/account _] )
] ]
remote-db remote-db
#_(d/history remote-db)) #_(d/history remote-db))
(map (fn [[old-rule]] (map (fn [[old-rule]]
{:entity/migration-key old-rule {:entity/migration-key old-rule
:db/doc "A transaction account that was deleted"})))}) :db/doc "A transaction account that was deleted"})))})
(doseq [entity (filter (complement #{"audit"}) order-of-insert) (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) order-of-insert))
:let [_ (swap! loaded conj entity) :let [_ (swap! loaded conj entity)
_ (println "querying for " entity) _ (println "querying for " entity)
entities (d/q '[:find [?e ...] entities (d/q '[:find [?e ...]
:in $ [?a ...] :in $ [?a ...]
:where [?e ?a]] :where [?e ?a]]
remote-db remote-db
(cond-> (entity->best-key entity) (cond-> (entity->best-key entity)
(not (vector? (entity->best-key entity))) vector)) (not (vector? (entity->best-key entity))) vector))
tx-chan (a/chan 400) _ (println "Found some! here's a few: " (take 3 entities))
entities->transaction (fn [entities] tx-chan (a/chan 50)
(->> (d/pull-many remote-db entities->transaction (fn [entities]
(->> schema
(filter :db/valueType) (->> (d/pull-many remote-db
(mapv :db/ident) (->> schema
(filter #(= entity (namespace %))) (filter :db/valueType)
(into [:db/id])) (mapv :db/ident)
entities) (filter #(= entity (namespace %)))
(mapv (fn [m ] (into [:db/id]))
(reduce entities)
(fn [m [k v]] (mapv (fn [m ]
(cond (reduce
(= k :db/id) (fn [m [k v]]
(-> m (cond
(assoc :entity/migration-key v) (= k :db/id)
(dissoc :db/id)) (-> m
(full-dependencies k) (assoc :entity/migration-key v)
(if (vector? v) (dissoc :db/id))
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) (full-dependencies k)
(assoc m k [:entity/migration-key (:db/id v)])) (if (vector? v)
:else (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
(dissoc m :payment/pdf-data (assoc m k [:entity/migration-key (:db/id v)]))
:payment/memo))) :else
m (dissoc m :payment/pdf-data
m))))) :payment/memo)))
_ (println "Inserting " entity ": " (count entities)) m
_ (flush) m)))))
pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"}) _ (println "Inserting " entity ": " (count entities))
50 _ (flush)
tx-chan pipeline (tx-pipeline (dc/connect local-client {:db-name "prod-migration"})
entities->transaction)]] 10
(doseq [batch (partition-all 500 entities)] tx-chan
(try entities->transaction)]]
(a/>!! tx-chan batch) (doseq [batch (partition-all 100 entities)]
(catch Exception e (try
(println e) (a/>!! tx-chan batch)
((:stop pipeline))))) (catch Exception e
(println e)
((:stop pipeline)))))
(println "waiting for done from" pipeline) (println "waiting for done from" pipeline)
(flush) (flush)
(a/close! tx-chan) (a/close! tx-chan)
(println (a/<!! (:result pipeline))) (println (a/<!! (:result pipeline)))
((:stop pipeline)) ((:stop pipeline))
(println) (println)
(println "Done"))) (println "Done"))))
(defn reset-migrate [] (defn reset-migrate []
(dc/delete-database local-client {:db-name "prod-migration"}) (dc/delete-database local-client {:db-name "prod-migration"})
@@ -405,7 +574,7 @@
(migrate) (migrate)
(datomic.dev-local/release-db {:db-name "prod-migration" (datomic.dev-local/release-db {:db-name "prod-migration"
:system "dev"}) :system "dev"})
(d/pull remote-db '[*] 17592271182081) (d/pull remote-db '[*] 17592271182081)
@@ -422,7 +591,7 @@
:where (or [?e :charge/total] :where (or [?e :charge/total]
[?e :charge/external-id]) [?e :charge/external-id])
[?e :entity/migration-key ?m]] [?e :entity/migration-key ?m]]
(dc/db (dc/connect local-client {:db-name "prod-migration"}))))) (dc/db (dc/connect local-client {:db-name "prod-migration"})))))
(count local-charges) (count local-charges)

8
src/clj/auto_ap/ssr/company_dropdown.clj
View File

@@ -6,15 +6,15 @@
[auto-ap.ssr.components.navbar-dropdown :refer [navbar-dropdown]] [auto-ap.ssr.components.navbar-dropdown :refer [navbar-dropdown]]
[auto-ap.ssr.utils :refer [html-response]] [auto-ap.ssr.utils :refer [html-response]]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[datomic.api :as d] [datomic.client.api :as dc]
[hiccup2.core :as hiccup])) [hiccup2.core :as hiccup]))
(defn dropdown-contents [{:keys [identity]}] (defn dropdown-contents [{:keys [identity]}]
(let [options (->> (d/q '[:find ?c ?n (let [options (->> (dc/q '[:find ?c ?n
:in $ ?user :in $ ?user
:where [?c :client/name ?n] :where [?c :client/name ?n]
[(auto-ap.graphql.utils/can-see-client? ?user ?c)]] [(auto-ap.graphql.utils/can-see-client? ?user ?c)]]
(d/db conn) (dc/db conn)
identity) identity)
(map (fn [[k v]] (map (fn [[k v]]
{"key" k {"key" k
@@ -105,7 +105,7 @@ document.getElementById(\"company-search-value\").dispatchEvent(new Event('chang
(assert-can-see-client identity client-id)) (assert-can-see-client identity client-id))
(let [new-session (assoc (:session request) :client (let [new-session (assoc (:session request) :client
(when client-id (when client-id
(d/pull (d/db conn) [:db/id :client/name :client/code] client-id)))] (dc/pull (dc/db conn) [:db/id :client/name :client/code] client-id)))]
(assoc (assoc
(html-response (html-response
(dropdown (assoc request :session new-session))) (dropdown (assoc request :session new-session)))