diff --git a/project.clj b/project.clj index a006d6ad..dd0479e7 100644 --- a/project.clj +++ b/project.clj @@ -17,6 +17,10 @@ org.slf4j/slf4j-nop org.slf4j/slf4j-log4j12]] [com.unbounce/clojure-dogstatsd-client "0.7.0"] + [com.cognitect/hmac-authn "0.1.210"] + [com.datomic/client-cloud "1.0.119"] + [com.cognitect/http-client "1.0.115"] + [lambdaisland/edn-lines "1.0.10"] [bidi "2.1.6"] [ring/ring-defaults "0.3.2" :exclusions [ring ring/ring-core]] [mount "0.1.16"] diff --git a/scratch-sessions/dump-edn.clj b/scratch-sessions/dump-edn.clj index b61ba2a2..c37ed2d6 100644 --- a/scratch-sessions/dump-edn.clj +++ b/scratch-sessions/dump-edn.clj @@ -5,47 +5,51 @@ ;; You can also press C-u C-j to evaluate the expression and pretty-print its result. (ns dump-edn - (:require [datomic.api :as d] + (:require #_`[datomic.api :as d] [clojure.java.io :as io] [amazonica.aws.s3 :as s3] [config.core :refer [env]] [clojure.core.async :as a] [datomic.client.api :as dc] + [lambdaisland.edn-lines :as ednl] [datomic.client.api.async :as dca] [datomic.dev-local :as dl] - [clojure.set :as set])) + [clojure.set :as set] + [clojure.string :as str] + [clj-http.client :as client])) (def remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod"))) (def local-client (dc/client {:server-type :dev-local :system "dev"})) -(dc/list-databases local-client {}) -(def schema (let [everything (->> (d/q '[:find [(pull ?e [:db/ident - {:db/valueType [:db/ident]} - {:db/cardinality [:db/ident]} - :db.attr/preds - {:db/unique [:db/ident]} - :db/isComponent - :db/id - :db/noHistory - :db/tupleAttrs - :db.entity/attrs - :db.entity/preds - :db/doc]) ...] - :where [?e :db/ident]] - remote-db)) - schema-attrs (->> everything - (filter :db/ident) - (filter (fn [{:db/keys [ident]}] - (if (namespace ident) - (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident)) - true - )))) - meta-schema-schema (filter #(-> % :db/ident not) everything)] - schema-attrs)) + +(def get-schema [] + (let [everything (->> (d/q '[:find [(pull ?e [:db/ident + {:db/valueType [:db/ident]} + {:db/cardinality [:db/ident]} + :db.attr/preds + {:db/unique [:db/ident]} + :db/isComponent + :db/id + :db/noHistory + :db/tupleAttrs + :db.entity/attrs + :db.entity/preds + :db/doc]) ...] + :where [?e :db/ident]] + remote-db)) + schema-attrs (->> everything + (filter :db/ident) + (filter (fn [{:db/keys [ident]}] + (if (namespace ident) + (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident)) + true + )))) + meta-schema-schema (filter #(-> % :db/ident not) everything)] + schema-attrs)) (def best-key-helper (->> schema @@ -364,7 +368,8 @@ "import-batch" #{"transaction"}, "payment" #{"vendor" "invoice" "bank-account" "client"}, "vendor-account-override" #{"account" "client"}}) -(def order-of-insert + +(defn order-of-insert [entity-dependencies] (loop [entity-dependencies entity-dependencies order []] (let [next-order (for [[entity deps] entity-dependencies @@ -502,7 +507,7 @@ :db/doc "A transaction account that was deleted"})))}) - (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) order-of-insert)) + (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies))) :let [_ (swap! loaded conj entity) _ (println "querying for " entity) entities (d/q '[:find [?e ...] @@ -561,6 +566,127 @@ (println) (println "Done")))) +(def dumped (atom #{})) + + +(defn dump-schema [backup] + (spit (io/file (str backup "/schema.edn")) + (with-out-str (prn (map + (fn [s] + (set/rename-keys s {:db/id :entity/migration-key})) + schema)))) + (spit (io/file (str backup "/full-dependencies.edn")) + (with-out-str (prn full-dependencies))) + (spit (io/file (str backup "/entity-dependencies.edn")) + (with-out-str (prn entity-dependencies)))) + +(defn dump-all + ([] (dump-all nil)) + ([item-list] + (let [backup-id (str "backups/" (java.util.UUID/randomUUID))] + (.mkdir (io/file backup-id)) + (dump-schema backup-id) + (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies))) + :let [_ (swap! dumped conj entity) + _ (println "querying for " entity) + entities (d/q '[:find [?e ...] + :in $ [?a ...] + :where [?e ?a]] + remote-db + (cond-> (entity->best-key entity) + (not (vector? (entity->best-key entity))) vector)) + + entities->transaction (fn [entities] + + (->> (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= entity (namespace %))) + (into [:db/id])) + entities) + (mapv (fn [m ] + (reduce + (fn [m [k v]] + (cond + (= k :db/id) + (-> m + (assoc :entity/migration-key v) + (dissoc :db/id)) + (full-dependencies k) + (if (vector? v) + (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) + (assoc m k [:entity/migration-key (:db/id v)])) + :else + (dissoc m :payment/pdf-data + :payment/memo))) + m + m))))) + _ (println "Inserting " entity ": " (count entities)) + _ (flush)]] + (ednl/with-append [append (str backup-id "/" entity ".ednl")] + (doseq [batch (partition-all 1000 entities) + :let [_ (do (print ".") (flush))] + item (entities->transaction batch)] + (try + (append item) + (catch Exception e + (println e) + (throw e))))) + (println "Done"))))) + +(defn load-from-backup + ([backup-id connection] (load-from-backup backup-id connection nil)) + ([backup-id connection item-list] + (let [schema (clojure.edn/read-string (slurp (str backup-id "/schema.edn"))) + full-dependencies (clojure.edn/read-string (slurp (str backup-id "/full-dependencies.edn"))) + entity-dependencies (clojure.edn/read-string (slurp (str backup-id "/entity-dependencies.edn")))] + (dc/transact connection {:tx-data [{:db/ident :entity/migration-key + :db/unique :db.unique/identity + :db/cardinality :db.cardinality/one + :db/valueType :db.type/long}]}) + (dc/transact connection {:tx-data (map + (fn [s] + (set/rename-keys s {:db/id :entity/migration-key})) + schema)}) + + (dc/transact connection {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"} + {:entity/migration-key 17592232621701} + {:entity/migration-key 17592263907739} + {:entity/migration-key 17592271516922}] + + }) + + + + (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies))) + :let [_ (swap! loaded conj entity) + _ (println "querying for " entity) + entities (ednl/slurp (str backup-id "/" entity ".ednl")) + + _ (println "Found some! here's a few: " (take 3 entities)) + tx-chan (a/chan 50) + entities->transaction (fn [entities] + entities) + pipeline (tx-pipeline connection + 10 + tx-chan + entities->transaction)]] + (doseq [batch (partition-all 100 entities)] + (try + (a/>!! tx-chan batch) + (catch Exception e + (println e) + ((:stop pipeline))))) + + (println "waiting for done from" pipeline) + (flush) + (a/close! tx-chan) + (println (a/graphql assert-admin can-see-client? is-admin?]] + :refer [->graphql assert-admin can-see-client? is-admin? attach-tracing-resolvers]] [auto-ap.routes.queries :as q] [clj-time.coerce :as coerce] [clojure.java.io :as io] [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] - [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.client.api :as dc] [unilog.context :as lc] - [auto-ap.graphql.utils :refer [attach-tracing-resolvers]] - [com.brunobonacci.mulog :as mu]) + [com.brunobonacci.mulog :as mu] + [datomic.client.api :as dc]) (:import (java.util UUID) (org.apache.commons.codec.binary Base64))) diff --git a/src/clj/auto_ap/graphql/import_batch.clj b/src/clj/auto_ap/graphql/import_batch.clj index e1213387..6029a882 100644 --- a/src/clj/auto_ap/graphql/import_batch.clj +++ b/src/clj/auto_ap/graphql/import_batch.clj @@ -3,12 +3,10 @@ [auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many-by-id]] [auto-ap.graphql.utils :refer - [<-graphql assert-admin ident->enum-f result->page]] + [<-graphql assert-admin ident->enum-f result->page attach-tracing-resolvers]] [clj-time.coerce :as coerce] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d] - [datomic.client.api :as dc] - [auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) + [datomic.client.api :as dc])) (def default-read '[:db/id :import-batch/external-id diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index c7618ace..6dcf715c 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -1,6 +1,7 @@ (ns auto-ap.graphql.invoices (:require - [auto-ap.datomic :refer [conn random-tempid remove-nils upsert-entity]] + [auto-ap.datomic + :refer [conn pull-attr pull-many pull-ref random-tempid upsert-entity]] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.vendors :as d-vendors] @@ -22,10 +23,7 @@ [clj-time.coerce :as coerce] [clj-time.core :as time] [clojure.tools.logging :as log] - [datomic.api :as d] - [manifold.deferred :as de] [com.brunobonacci.mulog :as mu] - [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.client.api :as dc])) (defn ->graphql [invoice user ] @@ -139,7 +137,7 @@ (when (and (= :allowance/denied (:account/invoice-allowance account)) - (not= (:db/id (:vendor/default-account (d/entity (d/db conn) vendor_id))) + (not= (pull-ref (dc/db conn) vendor_id :vendor/default-account) (:db/id account))) (let [err (str "Account isn't allowed for invoice use.")] (throw (ex-info err @@ -262,30 +260,32 @@ (defn all-ids-not-locked [all-ids] (->> all-ids - (d/q '[:find [?i ...] + (dc/q '[:find ?i :in $ [?i ...] :where [?i :invoice/client ?c] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [?i :invoice/date ?d] [(>= ?d ?lu)]] - (d/db conn)))) + (dc/db conn)) + (map first))) (defn void-invoices [context args _] (let [_ (assert-admin (:id context)) args (assoc args :id (:id context)) all-ids (all-ids-not-locked (get-ids-matching-filters args)) - voidable-cash-payments (d/q '[:find [?p ...] - :in $ [?i ...] - :where [?ip :invoice-payment/invoice ?i] - [?ip :invoice-payment/payment ?p] - [?p :payment/type :payment-type/cash] - [?i :invoice/client ?c] - [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] - [?i :invoice/date ?d] - [(>= ?d ?lu)]] - (d/db conn) - all-ids) + voidable-cash-payments (->> (dc/q '[:find ?p + :in $ [?i ...] + :where [?ip :invoice-payment/invoice ?i] + [?ip :invoice-payment/payment ?p] + [?p :payment/type :payment-type/cash] + [?i :invoice/client ?c] + [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] + [?i :invoice/date ?d] + [(>= ?d ?lu)]] + (dc/db conn) + all-ids) + (map first)) ] (log/info "Voiding " (count voidable-cash-payments) "cash payments first") (gq-checks/void-payments-internal voidable-cash-payments (:id context)) @@ -422,11 +422,11 @@ (throw (ex-info "Client is required" {:validation-error "client is required"}))) (let [args (assoc args :id (:id context)) - locations (:client/locations (d/pull (d/db conn) - [:client/locations] - (:client_id args))) + locations (pull-attr (dc/db conn) + :client/locations + (:client_id args)) all-ids (all-ids-not-locked (get-ids-matching-filters args)) - invoices (d/pull-many (d/db conn) '[:db/id :invoice/total] (vec all-ids)) + invoices (pull-many (dc/db conn) '[:db/id :invoice/total] (vec all-ids)) account-total (reduce + 0 (map (fn [x] (:percentage x)) (:accounts args)))] (log/info "client is" locations) (when @@ -434,7 +434,7 @@ (let [error (str "Account total (" account-total ") does not reach 100%")] (throw (ex-info error {:validation-error error})))) (doseq [a (:accounts args) - :let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a))]] + :let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn) [:account/location :account/name] (:account_id a))]] (when (and location (not= location (:location a))) (let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)] (throw (ex-info err {:validation-error err}) ))) diff --git a/src/clj/auto_ap/graphql/plaid.clj b/src/clj/auto_ap/graphql/plaid.clj index 9e887b23..0cc4a970 100644 --- a/src/clj/auto_ap/graphql/plaid.clj +++ b/src/clj/auto_ap/graphql/plaid.clj @@ -20,9 +20,7 @@ [clj-time.coerce :as coerce] [clj-time.core :as time] [clojure.tools.logging :as log] - [datomic.client.api :as dc] - [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn plaid-link-token [context value _] (when-not (:client_id value) diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index a7955966..3cd38a9f 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -19,6 +19,7 @@ :refer [->graphql <-graphql assert-admin + attach-tracing-resolvers assert-can-see-client assert-not-locked assert-power-user @@ -34,10 +35,7 @@ [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] - [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.client.api :as dc] - [datomic.api :as d] - [auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) + [datomic.client.api :as dc])) (def approval-status->graphql (ident->enum-f :transaction/approval-status)) diff --git a/src/clj/auto_ap/jobs/load_historical_sales.clj b/src/clj/auto_ap/jobs/load_historical_sales.clj index 80aba7f6..2051c174 100644 --- a/src/clj/auto_ap/jobs/load_historical_sales.clj +++ b/src/clj/auto_ap/jobs/load_historical_sales.clj @@ -1,27 +1,23 @@ (ns auto-ap.jobs.load-historical-sales (:gen-class) (:require - [unilog.context :as lc] + [auto-ap.datomic :refer [conn]] [auto-ap.jobs.core :refer [execute]] - [amazonica.aws.s3 :as s3] [auto-ap.square.core :as square] [auto-ap.square.core3 :as square3] - [clj-time.periodic :as per] - [clj-time.core :as time] - [datomic.api :as d] - [auto-ap.utils :refer [dollars=]] - [clojure.string :as str] - [clj-time.coerce :as coerce] [auto-ap.time :as atime] - [auto-ap.datomic :refer [conn audit-transact]] - [clojure.data.csv :as csv] - [clojure.java.io :as io] + [clj-time.coerce :as coerce] + [clj-time.core :as time] + [clj-time.periodic :as per] + [clojure.tools.logging :as log] [config.core :refer [env]] - [clojure.tools.logging :as log])) + [datomic.client.api :as dc] + [unilog.context :as lc])) + #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn historical-load-sales [client days] (log/info "loading old approach") - (let [client (d/pull (d/db auto-ap.datomic/conn) + (let [client (dc/pull (dc/db auto-ap.datomic/conn) square/square-read client)] (doseq [square-location (:client/square-locations client) @@ -54,7 +50,7 @@ (defn historical-load-sales2 [client days] (log/info "loading new approach") - (let [client (d/pull (d/db auto-ap.datomic/conn) + (let [client (dc/pull (dc/db auto-ap.datomic/conn) square/square-read client)] (doseq [square-location (:client/square-locations client) @@ -88,7 +84,7 @@ (defn load-historical-sales [args] (let [{:keys [days client]} args client (Long/parseLong client) - feature-flags (-> (d/pull (d/db conn) '[:client/feature-flags] client) + feature-flags (-> (dc/pull (dc/db conn) '[:client/feature-flags] client) :client/feature-flags set)] (if (get feature-flags "new-square") diff --git a/src/clj/auto_ap/jobs/ntg.clj b/src/clj/auto_ap/jobs/ntg.clj index a9e65dab..8a176575 100644 --- a/src/clj/auto_ap/jobs/ntg.clj +++ b/src/clj/auto_ap/jobs/ntg.clj @@ -1,28 +1,18 @@ (ns auto-ap.jobs.ntg (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [conn]] [auto-ap.datomic.clients :as d-clients] - [auto-ap.datomic.invoices :refer [code-invoice]] [auto-ap.jobs.core :refer [execute]] - [auto-ap.ledger :refer [transact-with-ledger]] + [auto-ap.logging :as log] [auto-ap.parse :as parse] - [auto-ap.time :as t] + [auto-ap.time :as atime] [clj-time.coerce :as coerce] [clojure.data.csv :as csv] [clojure.java.io :as io] - [clojure.string :as str] - [auto-ap.logging :as log :refer [capture-context->lc with-context-as]] - [com.unbounce.dogstatsd.core :as statsd] - [config.core :refer [env]] - [datomic.api :as d] - [com.brunobonacci.mulog :as mu] - - [auto-ap.time :as atime]) + [clojure.string :as str]) (:import (java.util UUID))) - (def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env)) (defn read-csv [k] diff --git a/src/clj/auto_ap/jobs/register_invoice_import.clj b/src/clj/auto_ap/jobs/register_invoice_import.clj index 560c2528..fb05d5c3 100644 --- a/src/clj/auto_ap/jobs/register_invoice_import.clj +++ b/src/clj/auto_ap/jobs/register_invoice_import.clj @@ -1,19 +1,20 @@ (ns auto-ap.jobs.register-invoice-import (:gen-class) (:require - [auto-ap.jobs.core :refer [execute]] [amazonica.aws.s3 :as s3] - [datomic.api :as d] - [auto-ap.utils :refer [dollars=]] - [clojure.string :as str] - [clj-time.coerce :as coerce] - [auto-ap.time :as atime] + [auto-ap.datomic :refer [conn pull-attr]] + [auto-ap.jobs.core :refer [execute]] [auto-ap.ledger :refer [transact-with-ledger]] - [auto-ap.datomic :refer [conn]] + [auto-ap.time :as atime] + [auto-ap.utils :refer [dollars=]] + [clj-time.coerce :as coerce] [clojure.data.csv :as csv] [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.logging :as log] [config.core :refer [env]] - [clojure.tools.logging :as log])) + [datomic.client.api :as dc]) + (:import [java.util UUID])) (def bucket (:data-bucket env)) @@ -30,12 +31,11 @@ (throw e)))) (defn register-invoice-import* [data] - (let [db (d/db conn) + (let [db (dc/db conn) i->invoice-id (fn [i] (try (Long/parseLong i) (catch Exception _ - (:db/id (d/pull db '[:db/id] - [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) + (pull-attr db :db/id [:invoice/original-id (Long/parseLong (first (str/split i #"-")))])))) invoice-totals (->> data (drop 1) (group-by first) @@ -57,7 +57,7 @@ location] (drop 1 data) :let [invoice-id (i->invoice-id i) - invoice (d/entity db invoice-id) + invoice (dc/pull db '[*] invoice-id) current-total (:invoice/total invoice) target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) @@ -65,10 +65,10 @@ invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice))) - (d/tempid :db.part/user)) + (str (UUID/randomUUID))) invoice-expense-account (when-not new-account? - (or (d/entity db invoice-expense-account-id) - (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) + (or (dc/pull db '[*] invoice-expense-account-id) + (dc/pull db '[*] [:invoice-expense-account/original-id invoice-expense-account-id]))) current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) target-account-id (Long/parseLong (str/trim target-account)) @@ -84,7 +84,7 @@ target-expense-account-location location - [[_ _ invoice-payment]] (vec (d/q + [[_ _ invoice-payment]] (vec (dc/q '[:find ?p ?a ?ip :in $ ?i :where [?ip :invoice-payment/invoice ?i] diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 976e4aab..f31c19e5 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -418,7 +418,7 @@ (let [db (dc/db conn) tx (audit-transact transaction id) affected-entities (->> (:tx-data tx) - (map (fn [^datomic.db.Datum x] + (map (fn [x] {:e (:e x) :a (:a x) :v (:v x) diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index 18e6c5d0..c4bc3474 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -19,7 +19,6 @@ [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] [config.core :refer [env]] - [datomic.api :as d] [datomic.client.api :as dc] [ring.middleware.json :refer [wrap-json-response]] [venia.core :as venia])) diff --git a/src/clj/auto_ap/routes/queries.clj b/src/clj/auto_ap/routes/queries.clj index 6be5f40f..7793a0d3 100644 --- a/src/clj/auto_ap/routes/queries.clj +++ b/src/clj/auto_ap/routes/queries.clj @@ -9,9 +9,8 @@ [clojure.string :as str] [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] -[config.core :refer [env]] -[datomic.client.api :as dc] - [datomic.api :as d] + [config.core :refer [env]] + [datomic.client.api :as dc] [ring.middleware.json :refer [wrap-json-response]] [ring.util.request :refer [body-string]] diff --git a/src/clj/auto_ap/square/core3.clj b/src/clj/auto_ap/square/core3.clj index ed0b9a73..40a32ac9 100644 --- a/src/clj/auto_ap/square/core3.clj +++ b/src/clj/auto_ap/square/core3.clj @@ -13,7 +13,7 @@ [clojure.set :as set] [clojure.string :as str] [com.brunobonacci.mulog :as mu] - [datomic.api :as d] + [datomic.client.api :as dc] [manifold.deferred :as de] [manifold.executor :as ex] [manifold.stream :as s] @@ -471,7 +471,7 @@ (log/info ::transforming-settlements) (try (->> (for [settlement settlements - :let [best-sales-date (some->> (d/q '[:find ?s4 (count ?s) + :let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s) :in $ ?settlement-id :where [?settlement :expected-deposit/external-id ?settlement-id] @@ -481,7 +481,7 @@ [(clj-time.coerce/to-date-time ?sales-date) ?s2] [(auto-ap.time/localize ?s2) ?s3] [(clj-time.coerce/to-local-date ?s3) ?s4]] - (d/db conn) + (dc/db conn) (str "square/settlement/" (:id settlement))) (sort-by last) last @@ -562,7 +562,7 @@ (doseq [x (partition-all 100 results)] (log/info ::loading-orders :count (count x)) - @(d/transact conn x)))))))) + (dc/transact conn {:tx-data x})))))))) (defn upsert-settlements @@ -582,7 +582,7 @@ (doseq [x (partition-all 20 settlements)] (log/info ::loading-deposits :count (count x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (log/info ::done-loading-deposits))))))) (defn upsert-refunds @@ -603,7 +603,7 @@ (log/info ::loading-refunds :count (count x) :sample (first x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (catch Throwable e (log/error ::upsert-refunds-failed @@ -618,26 +618,26 @@ (defn get-square-clients ([] - (d/q '[:find [(pull ?c [:db/id - :client/square-integration-status - :client/code - :client/square-auth-token - {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] - :in $ - :where [?c :client/square-auth-token] - [?c :client/feature-flags "new-square"]] - (d/db conn))) + (map first (dc/q '[:find (pull ?c [:db/id + :client/square-integration-status + :client/code + :client/square-auth-token + {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) + :in $ + :where [?c :client/square-auth-token] + [?c :client/feature-flags "new-square"]] + (dc/db conn)))) ([ & codes] - (d/q '[:find [(pull ?c [:db/id - :client/code - :client/square-auth-token - {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] - :in $ [?code ...] - :where [?c :client/square-auth-token] - [?c :client/feature-flags "new-square"] - [?c :client/code ?code]] - (d/db conn) - codes))) + (map first (dc/q '[:find (pull ?c [:db/id + :client/code + :client/square-auth-token + {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) + :in $ [?code ...] + :where [?c :client/square-auth-token] + [?c :client/feature-flags "new-square"] + [?c :client/code ?code]] + (dc/db conn) + codes)))) (defn upsert-locations ([] @@ -653,30 +653,30 @@ (:client/square-locations client)))] (de/chain (client-locations client) (fn [client-locations] - @(d/transact conn - (for [square-location client-locations] - {:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user)) - :client/_square-locations (:db/id client) - :square-location/name (:name square-location) - :square-location/square-id (:id square-location)}))))))) + (dc/transact conn + {:tx-data (for [square-location client-locations] + {:db/id (or (square-id->id (:id square-location)) (str (java.util.UUID/randomUUID))) + :client/_square-locations (:db/id client) + :square-location/name (:name square-location) + :square-location/square-id (:id square-location)})})))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn reset [] (->> - (d/query {:query {:find ['?e] + (dc/q {:query {:find ['?e] :in ['$] :where ['(or [?e :sales-order/date] [?e :expected-deposit/date])]} - :args [(d/db conn)]}) + :args [(dc/db conn)]}) (map first) (map (fn [x] [:db/retractEntity x])))) (defn mark-integration-status [client integration-status] - @(d/transact conn - [{:db/id (:db/id client) - :client/square-integration-status (assoc integration-status - :db/id (or (-> client :client/square-integration-status :db/id) - #db/id [:db.part/user]))}])) + (dc/transact conn + {:tx-data [{:db/id (:db/id client) + :client/square-integration-status (assoc integration-status + :db/id (or (-> client :client/square-integration-status :db/id) + (str (java.util.UUID/randomUUID))))}]})) (defn upsert-all [ & clients] (capture-context->lc diff --git a/src/clj/auto_ap/ssr/admin.clj b/src/clj/auto_ap/ssr/admin.clj index 70280683..58cef00d 100644 --- a/src/clj/auto_ap/ssr/admin.clj +++ b/src/clj/auto_ap/ssr/admin.clj @@ -8,7 +8,7 @@ [auto-ap.time :as atime] [clj-time.coerce :as coerce] [clojure.string :as str] - [datomic.api :as d] + [datomic.client.api :as dc] [hiccup2.core :as hiccup])) (defn tx-rows->changes [history] @@ -123,13 +123,13 @@ (get params "entity-id") (get form-params "entity-id"))) history (->> - (d/q '[:find ?a2 ?v (pull ?tx [:db/txInstant :audit/user :db/id]) ?ad + (dc/q '[:find ?a2 ?v (pull ?tx [:db/txInstant :audit/user :db/id]) ?ad :in $ $$ ?i :where [$$ ?i ?a ?v ?tx ?ad] [$ ?a :db/ident ?a2]] - (d/db conn) - (d/history (d/db conn)) + (dc/db conn) + (dc/history (dc/db conn)) entity-id ) tx-rows->changes (sort-by (comp :db/id first)) @@ -166,7 +166,7 @@ :request request) (try (let [entity-id (Long/parseLong entity-id) - data (d/pull (d/db conn) + data (dc/pull (dc/db conn) '[*] entity-id)] diff --git a/src/clj/auto_ap/ssr/company/company_1099.clj b/src/clj/auto_ap/ssr/company/company_1099.clj index 8638fdbd..7391eb37 100644 --- a/src/clj/auto_ap/ssr/company/company_1099.clj +++ b/src/clj/auto_ap/ssr/company/company_1099.clj @@ -8,7 +8,7 @@ [auto-ap.ssr.utils :refer [html-response]] [bidi.bidi :as bidi] [clojure.string :as str] - [datomic.api :as d] + [datomic.client.api :as dc] [hiccup2.core :as hiccup])) (defn cannot-overwrite? [vendor] @@ -17,18 +17,18 @@ (:vendor/legal-entity-tin-type vendor)))) (defn get-1099-companies [user session] - (let [clients (->> (d/q '[:find ?c + (let [clients (->> (dc/q '[:find ?c :in $ ?user :where [?c :client/code] [(auto-ap.graphql.utils/can-see-client? ?user ?c)]] - (d/db conn) user) + (dc/db conn) user) (map first) set) results (cond (and (some-> session :client :db/id) (can-see-client? user (some-> session :client :db/id))) - (d/q '[:find + (dc/q '[:find (pull ?c [:client/code :db/id]) (pull ?v [:db/id :vendor/name @@ -55,11 +55,11 @@ [?p :payment/amount ?a] [?p :payment/vendor ?v]] - (d/db conn) + (dc/db conn) (some-> session :client :db/id)) (is-admin? user) - (d/q '[:find + (dc/q '[:find (pull ?c [:client/code :db/id]) (pull ?v [:db/id :vendor/name @@ -85,10 +85,10 @@ [?p :payment/client ?c] [?p :payment/amount ?a] [?p :payment/vendor ?v]] - (d/db conn)) + (dc/db conn)) :else - (d/q '[:find + (dc/q '[:find (pull ?c [:client/code :db/id]) (pull ?v [:db/id :vendor/name @@ -115,7 +115,7 @@ [?p :payment/amount ?a] [?p :payment/vendor ?v]] - (d/db conn) + (dc/db conn) clients))] (->> results (filter (fn [[_ _ a]] @@ -208,18 +208,18 @@ :else k)) (defn vendor-save [{:keys [form-params identity route-params] :as request}] - (when-not (cannot-overwrite? (d/pull (d/db conn) '[*] (Long/parseLong (:vendor-id route-params)))) - @(d/transact conn [(remove-nils - (-> (form-data->map form-params) - (assoc :db/id (Long/parseLong (:vendor-id route-params))) - (update :vendor/legal-entity-1099-type #(some->> % (keyword "legal-entity-1099-type"))) - (update :vendor/legal-entity-tin-type #(some->> % (keyword "legal-entity-tin-type")))))])) + (when-not (cannot-overwrite? (dc/pull (dc/db conn) '[*] (Long/parseLong (:vendor-id route-params)))) + @(dc/transact conn {:tx-data [(remove-nils + (-> (form-data->map form-params) + (assoc :db/id (Long/parseLong (:vendor-id route-params))) + (update :vendor/legal-entity-1099-type #(some->> % (keyword "legal-entity-1099-type"))) + (update :vendor/legal-entity-tin-type #(some->> % (keyword "legal-entity-tin-type")))))]})) (html-response (table request :flash-id (Long/parseLong (:vendor-id route-params))))) (defn vendor-dialog [request] - (let [vendor (d/pull (d/db conn) '[* {:vendor/legal-entity-1099-type [:db/ident] + (let [vendor (dc/pull (dc/db conn) '[* {:vendor/legal-entity-1099-type [:db/ident] :vendor/legal-entity-tin-type [:db/ident]}] (Long/parseLong (:vendor-id (:params request))))] ;; TODO perms (html-response [:form {:hx-post (bidi/path-for ssr-routes/only-routes