notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Getting closer to datomic cloud

Browse Source
This commit is contained in:
Bryce Covert
2023-03-18 20:07:34 -07:00
parent bd658906b5
commit 78519663ac
17 changed files with 299 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
project.clj
View File

@@ -17,6 +17,10 @@
org.slf4j/slf4j-nop org.slf4j/slf4j-nop
org.slf4j/slf4j-log4j12]] org.slf4j/slf4j-log4j12]]
[com.unbounce/clojure-dogstatsd-client "0.7.0"] [com.unbounce/clojure-dogstatsd-client "0.7.0"]
[com.cognitect/hmac-authn "0.1.210"]
[com.datomic/client-cloud "1.0.119"]
[com.cognitect/http-client "1.0.115"]
[lambdaisland/edn-lines "1.0.10"]
[bidi "2.1.6"] [bidi "2.1.6"]
[ring/ring-defaults "0.3.2" :exclusions [ring ring/ring-core]] [ring/ring-defaults "0.3.2" :exclusions [ring ring/ring-core]]
[mount "0.1.16"] [mount "0.1.16"]

195
scratch-sessions/dump-edn.clj
View File

@@ -5,47 +5,51 @@
;; You can also press C-u C-j to evaluate the expression and pretty-print its result. ;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(ns dump-edn (ns dump-edn
(:require [datomic.api :as d] (:require #_`[datomic.api :as d]
[clojure.java.io :as io] [clojure.java.io :as io]
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[config.core :refer [env]] [config.core :refer [env]]
[clojure.core.async :as a] [clojure.core.async :as a]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[lambdaisland.edn-lines :as ednl]
[datomic.client.api.async :as dca] [datomic.client.api.async :as dca]
[datomic.dev-local :as dl] [datomic.dev-local :as dl]
[clojure.set :as set])) [clojure.set :as set]
[clojure.string :as str]
[clj-http.client :as client]))
(def remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod"))) (def remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod")))
(def local-client (dc/client {:server-type :dev-local (def local-client (dc/client {:server-type :dev-local
:system "dev"})) :system "dev"}))
(dc/list-databases local-client {})
(def schema (let [everything (->> (d/q '[:find [(pull ?e [:db/ident
{:db/valueType [:db/ident]} (def get-schema []
{:db/cardinality [:db/ident]} (let [everything (->> (d/q '[:find [(pull ?e [:db/ident
:db.attr/preds {:db/valueType [:db/ident]}
{:db/unique [:db/ident]} {:db/cardinality [:db/ident]}
:db/isComponent :db.attr/preds
:db/id {:db/unique [:db/ident]}
:db/noHistory :db/isComponent
:db/tupleAttrs :db/id
:db.entity/attrs :db/noHistory
:db.entity/preds :db/tupleAttrs
:db/doc]) ...] :db.entity/attrs
:where [?e :db/ident]] :db.entity/preds
remote-db)) :db/doc]) ...]
schema-attrs (->> everything :where [?e :db/ident]]
(filter :db/ident) remote-db))
(filter (fn [{:db/keys [ident]}] schema-attrs (->> everything
(if (namespace ident) (filter :db/ident)
(re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident)) (filter (fn [{:db/keys [ident]}]
true (if (namespace ident)
)))) (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident))
meta-schema-schema (filter #(-> % :db/ident not) everything)] true
schema-attrs)) ))))
meta-schema-schema (filter #(-> % :db/ident not) everything)]
schema-attrs))
(def best-key-helper (def best-key-helper
(->> schema (->> schema
@@ -364,7 +368,8 @@
"import-batch" #{"transaction"}, "import-batch" #{"transaction"},
"payment" #{"vendor" "invoice" "bank-account" "client"}, "payment" #{"vendor" "invoice" "bank-account" "client"},
"vendor-account-override" #{"account" "client"}}) "vendor-account-override" #{"account" "client"}})
(def order-of-insert
(defn order-of-insert [entity-dependencies]
(loop [entity-dependencies entity-dependencies (loop [entity-dependencies entity-dependencies
order []] order []]
(let [next-order (for [[entity deps] entity-dependencies (let [next-order (for [[entity deps] entity-dependencies
@@ -502,7 +507,7 @@
:db/doc "A transaction account that was deleted"})))}) :db/doc "A transaction account that was deleted"})))})
(doseq [entity (or item-list (filter (complement (conj @loaded "audit")) order-of-insert)) (doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
:let [_ (swap! loaded conj entity) :let [_ (swap! loaded conj entity)
_ (println "querying for " entity) _ (println "querying for " entity)
entities (d/q '[:find [?e ...] entities (d/q '[:find [?e ...]
@@ -561,6 +566,127 @@
(println) (println)
(println "Done")))) (println "Done"))))
(def dumped (atom #{}))
(defn dump-schema [backup]
(spit (io/file (str backup "/schema.edn"))
(with-out-str (prn (map
(fn [s]
(set/rename-keys s {:db/id :entity/migration-key}))
schema))))
(spit (io/file (str backup "/full-dependencies.edn"))
(with-out-str (prn full-dependencies)))
(spit (io/file (str backup "/entity-dependencies.edn"))
(with-out-str (prn entity-dependencies))))
(defn dump-all
([] (dump-all nil))
([item-list]
(let [backup-id (str "backups/" (java.util.UUID/randomUUID))]
(.mkdir (io/file backup-id))
(dump-schema backup-id)
(doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
:let [_ (swap! dumped conj entity)
_ (println "querying for " entity)
entities (d/q '[:find [?e ...]
:in $ [?a ...]
:where [?e ?a]]
remote-db
(cond-> (entity->best-key entity)
(not (vector? (entity->best-key entity))) vector))
entities->transaction (fn [entities]
(->> (d/pull-many remote-db
(->> schema
(filter :db/valueType)
(mapv :db/ident)
(filter #(= entity (namespace %)))
(into [:db/id]))
entities)
(mapv (fn [m ]
(reduce
(fn [m [k v]]
(cond
(= k :db/id)
(-> m
(assoc :entity/migration-key v)
(dissoc :db/id))
(full-dependencies k)
(if (vector? v)
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
(assoc m k [:entity/migration-key (:db/id v)]))
:else
(dissoc m :payment/pdf-data
:payment/memo)))
m
m)))))
_ (println "Inserting " entity ": " (count entities))
_ (flush)]]
(ednl/with-append [append (str backup-id "/" entity ".ednl")]
(doseq [batch (partition-all 1000 entities)
:let [_ (do (print ".") (flush))]
item (entities->transaction batch)]
(try
(append item)
(catch Exception e
(println e)
(throw e)))))
(println "Done")))))
(defn load-from-backup
([backup-id connection] (load-from-backup backup-id connection nil))
([backup-id connection item-list]
(let [schema (clojure.edn/read-string (slurp (str backup-id "/schema.edn")))
full-dependencies (clojure.edn/read-string (slurp (str backup-id "/full-dependencies.edn")))
entity-dependencies (clojure.edn/read-string (slurp (str backup-id "/entity-dependencies.edn")))]
(dc/transact connection {:tx-data [{:db/ident :entity/migration-key
:db/unique :db.unique/identity
:db/cardinality :db.cardinality/one
:db/valueType :db.type/long}]})
(dc/transact connection {:tx-data (map
(fn [s]
(set/rename-keys s {:db/id :entity/migration-key}))
schema)})
(dc/transact connection {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
{:entity/migration-key 17592232621701}
{:entity/migration-key 17592263907739}
{:entity/migration-key 17592271516922}]
})
(doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
:let [_ (swap! loaded conj entity)
_ (println "querying for " entity)
entities (ednl/slurp (str backup-id "/" entity ".ednl"))
_ (println "Found some! here's a few: " (take 3 entities))
tx-chan (a/chan 50)
entities->transaction (fn [entities]
entities)
pipeline (tx-pipeline connection
10
tx-chan
entities->transaction)]]
(doseq [batch (partition-all 100 entities)]
(try
(a/>!! tx-chan batch)
(catch Exception e
(println e)
((:stop pipeline)))))
(println "waiting for done from" pipeline)
(flush)
(a/close! tx-chan)
(println (a/<!! (:result pipeline)))
((:stop pipeline))
(println)
(println "Done")))))
(defn reset-migrate [] (defn reset-migrate []
(dc/delete-database local-client {:db-name "prod-migration"}) (dc/delete-database local-client {:db-name "prod-migration"})
(dc/create-database local-client {:db-name "prod-migration"}) (dc/create-database local-client {:db-name "prod-migration"})
@@ -569,6 +695,19 @@
(comment (comment
;; cloud load
(do
(let [client (dc/client {:server-type :cloud
:region "us-east-1"
:system "iol-cloud"
:endpoint "https://53syis8n1m.execute-api.us-east-1.amazonaws.com"})
;; _ (dc/create-database client {:db-name "prod-mirror"})
connection (dc/connect client {:db-name "prod-mirror"})]
(load-from-backup "backups/8e245d3d-be7a-4d90-8e9e-e6a110582658" connection)
)
)
(reset-migrate) (reset-migrate)
(migrate) (migrate)

8
src/clj/auto_ap/datomic.clj
View File

@@ -11,11 +11,13 @@
(def uri (:datomic-url env)) (def uri (:datomic-url env))
(mount/defstate client (mount/defstate client
:start (dc/client {:server-type :dev-local :start (dc/client {:server-type :cloud
:system "dev"}) :region "us-east-1"
:system "iol-cloud"
:endpoint "https://53syis8n1m.execute-api.us-east-1.amazonaws.com"})
:stop nil) :stop nil)
(mount/defstate conn (mount/defstate conn
:start (dc/connect client {:db-name "prod-migration"}) :start (dc/connect client {:db-name "prod-mirror"})
:stop nil) :stop nil)
#_(def uri "datomic:mem://datomic-transactor:4334/invoice") #_(def uri "datomic:mem://datomic-transactor:4334/invoice")

7
src/clj/auto_ap/graphql/clients.clj
View File

@@ -5,18 +5,17 @@
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.square.core :as square] [auto-ap.square.core :as square]
[auto-ap.graphql.utils [auto-ap.graphql.utils
:refer [->graphql assert-admin can-see-client? is-admin?]] :refer [->graphql assert-admin can-see-client? is-admin? attach-tracing-resolvers]]
[auto-ap.routes.queries :as q] [auto-ap.routes.queries :as q]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[unilog.context :as lc] [unilog.context :as lc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]] [com.brunobonacci.mulog :as mu]
[com.brunobonacci.mulog :as mu]) [datomic.client.api :as dc])
(:import (:import
(java.util UUID) (java.util UUID)
(org.apache.commons.codec.binary Base64))) (org.apache.commons.codec.binary Base64)))

6
src/clj/auto_ap/graphql/import_batch.clj
View File

@@ -3,12 +3,10 @@
[auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many-by-id]] [auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many-by-id]]
[auto-ap.graphql.utils [auto-ap.graphql.utils
:refer :refer
[<-graphql assert-admin ident->enum-f result->page]] [<-graphql assert-admin ident->enum-f result->page attach-tracing-resolvers]]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]] [com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d] [datomic.client.api :as dc]))
[datomic.client.api :as dc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
(def default-read '[:db/id (def default-read '[:db/id
:import-batch/external-id :import-batch/external-id

46
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -1,6 +1,7 @@
(ns auto-ap.graphql.invoices (ns auto-ap.graphql.invoices
(:require (:require
[auto-ap.datomic :refer [conn random-tempid remove-nils upsert-entity]] [auto-ap.datomic
:refer [conn pull-attr pull-many pull-ref random-tempid upsert-entity]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.invoices :as d-invoices]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
@@ -22,10 +23,7 @@
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[datomic.api :as d]
[manifold.deferred :as de]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.client.api :as dc])) [datomic.client.api :as dc]))
(defn ->graphql [invoice user ] (defn ->graphql [invoice user ]
@@ -139,7 +137,7 @@
(when (and (= :allowance/denied (when (and (= :allowance/denied
(:account/invoice-allowance account)) (:account/invoice-allowance account))
(not= (:db/id (:vendor/default-account (d/entity (d/db conn) vendor_id))) (not= (pull-ref (dc/db conn) vendor_id :vendor/default-account)
(:db/id account))) (:db/id account)))
(let [err (str "Account isn't allowed for invoice use.")] (let [err (str "Account isn't allowed for invoice use.")]
(throw (ex-info err (throw (ex-info err
@@ -262,30 +260,32 @@
(defn all-ids-not-locked [all-ids] (defn all-ids-not-locked [all-ids]
(->> all-ids (->> all-ids
(d/q '[:find [?i ...] (dc/q '[:find ?i
:in $ [?i ...] :in $ [?i ...]
:where :where
[?i :invoice/client ?c] [?i :invoice/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?i :invoice/date ?d] [?i :invoice/date ?d]
[(>= ?d ?lu)]] [(>= ?d ?lu)]]
(d/db conn)))) (dc/db conn))
(map first)))
(defn void-invoices [context args _] (defn void-invoices [context args _]
(let [_ (assert-admin (:id context)) (let [_ (assert-admin (:id context))
args (assoc args :id (:id context)) args (assoc args :id (:id context))
all-ids (all-ids-not-locked (get-ids-matching-filters args)) all-ids (all-ids-not-locked (get-ids-matching-filters args))
voidable-cash-payments (d/q '[:find [?p ...] voidable-cash-payments (->> (dc/q '[:find ?p
:in $ [?i ...] :in $ [?i ...]
:where [?ip :invoice-payment/invoice ?i] :where [?ip :invoice-payment/invoice ?i]
[?ip :invoice-payment/payment ?p] [?ip :invoice-payment/payment ?p]
[?p :payment/type :payment-type/cash] [?p :payment/type :payment-type/cash]
[?i :invoice/client ?c] [?i :invoice/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?i :invoice/date ?d] [?i :invoice/date ?d]
[(>= ?d ?lu)]] [(>= ?d ?lu)]]
(d/db conn) (dc/db conn)
all-ids) all-ids)
(map first))
] ]
(log/info "Voiding " (count voidable-cash-payments) "cash payments first") (log/info "Voiding " (count voidable-cash-payments) "cash payments first")
(gq-checks/void-payments-internal voidable-cash-payments (:id context)) (gq-checks/void-payments-internal voidable-cash-payments (:id context))
@@ -422,11 +422,11 @@
(throw (ex-info "Client is required" (throw (ex-info "Client is required"
{:validation-error "client is required"}))) {:validation-error "client is required"})))
(let [args (assoc args :id (:id context)) (let [args (assoc args :id (:id context))
locations (:client/locations (d/pull (d/db conn) locations (pull-attr (dc/db conn)
[:client/locations] :client/locations
(:client_id args))) (:client_id args))
all-ids (all-ids-not-locked (get-ids-matching-filters args)) all-ids (all-ids-not-locked (get-ids-matching-filters args))
invoices (d/pull-many (d/db conn) '[:db/id :invoice/total] (vec all-ids)) invoices (pull-many (dc/db conn) '[:db/id :invoice/total] (vec all-ids))
account-total (reduce + 0 (map (fn [x] (:percentage x)) (:accounts args)))] account-total (reduce + 0 (map (fn [x] (:percentage x)) (:accounts args)))]
(log/info "client is" locations) (log/info "client is" locations)
(when (when
@@ -434,7 +434,7 @@
(let [error (str "Account total (" account-total ") does not reach 100%")] (let [error (str "Account total (" account-total ") does not reach 100%")]
(throw (ex-info error {:validation-error error})))) (throw (ex-info error {:validation-error error}))))
(doseq [a (:accounts args) (doseq [a (:accounts args)
:let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a))]] :let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn) [:account/location :account/name] (:account_id a))]]
(when (and location (not= location (:location a))) (when (and location (not= location (:location a)))
(let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)] (let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)]
(throw (ex-info err {:validation-error err}) ))) (throw (ex-info err {:validation-error err}) )))

4
src/clj/auto_ap/graphql/plaid.clj
View File

@@ -20,9 +20,7 @@
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[datomic.client.api :as dc] [datomic.client.api :as dc]))
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]))
(defn plaid-link-token [context value _] (defn plaid-link-token [context value _]
(when-not (:client_id value) (when-not (:client_id value)

6
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -19,6 +19,7 @@
:refer [->graphql :refer [->graphql
<-graphql <-graphql
assert-admin assert-admin
attach-tracing-resolvers
assert-can-see-client assert-can-see-client
assert-not-locked assert-not-locked
assert-power-user assert-power-user
@@ -34,10 +35,7 @@
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.client.api :as dc]))
[datomic.client.api :as dc]
[datomic.api :as d]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
(def approval-status->graphql (ident->enum-f :transaction/approval-status)) (def approval-status->graphql (ident->enum-f :transaction/approval-status))

26
src/clj/auto_ap/jobs/load_historical_sales.clj
View File

@@ -1,27 +1,23 @@
(ns auto-ap.jobs.load-historical-sales (ns auto-ap.jobs.load-historical-sales
(:gen-class) (:gen-class)
(:require (:require
[unilog.context :as lc] [auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]] [auto-ap.jobs.core :refer [execute]]
[amazonica.aws.s3 :as s3]
[auto-ap.square.core :as square] [auto-ap.square.core :as square]
[auto-ap.square.core3 :as square3] [auto-ap.square.core3 :as square3]
[clj-time.periodic :as per]
[clj-time.core :as time]
[datomic.api :as d]
[auto-ap.utils :refer [dollars=]]
[clojure.string :as str]
[clj-time.coerce :as coerce]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[auto-ap.datomic :refer [conn audit-transact]] [clj-time.coerce :as coerce]
[clojure.data.csv :as csv] [clj-time.core :as time]
[clojure.java.io :as io] [clj-time.periodic :as per]
[clojure.tools.logging :as log]
[config.core :refer [env]] [config.core :refer [env]]
[clojure.tools.logging :as log])) [datomic.client.api :as dc]
[unilog.context :as lc]))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn historical-load-sales [client days] (defn historical-load-sales [client days]
(log/info "loading old approach") (log/info "loading old approach")
(let [client (d/pull (d/db auto-ap.datomic/conn) (let [client (dc/pull (dc/db auto-ap.datomic/conn)
square/square-read square/square-read
client)] client)]
(doseq [square-location (:client/square-locations client) (doseq [square-location (:client/square-locations client)
@@ -54,7 +50,7 @@
(defn historical-load-sales2 [client days] (defn historical-load-sales2 [client days]
(log/info "loading new approach") (log/info "loading new approach")
(let [client (d/pull (d/db auto-ap.datomic/conn) (let [client (dc/pull (dc/db auto-ap.datomic/conn)
square/square-read square/square-read
client)] client)]
(doseq [square-location (:client/square-locations client) (doseq [square-location (:client/square-locations client)
@@ -88,7 +84,7 @@
(defn load-historical-sales [args] (defn load-historical-sales [args]
(let [{:keys [days client]} args (let [{:keys [days client]} args
client (Long/parseLong client) client (Long/parseLong client)
feature-flags (-> (d/pull (d/db conn) '[:client/feature-flags] client) feature-flags (-> (dc/pull (dc/db conn) '[:client/feature-flags] client)
:client/feature-flags :client/feature-flags
set)] set)]
(if (get feature-flags "new-square") (if (get feature-flags "new-square")

16
src/clj/auto_ap/jobs/ntg.clj
View File

@@ -1,28 +1,18 @@
(ns auto-ap.jobs.ntg (ns auto-ap.jobs.ntg
(:require (:require
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice]]
[auto-ap.jobs.core :refer [execute]] [auto-ap.jobs.core :refer [execute]]
[auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.logging :as log]
[auto-ap.parse :as parse] [auto-ap.parse :as parse]
[auto-ap.time :as t] [auto-ap.time :as atime]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.string :as str] [clojure.string :as str])
[auto-ap.logging :as log :refer [capture-context->lc with-context-as]]
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[datomic.api :as d]
[com.brunobonacci.mulog :as mu]
[auto-ap.time :as atime])
(:import (:import
(java.util UUID))) (java.util UUID)))
(def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env)) (def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env))
(defn read-csv [k] (defn read-csv [k]

32
src/clj/auto_ap/jobs/register_invoice_import.clj
View File

@@ -1,19 +1,20 @@
(ns auto-ap.jobs.register-invoice-import (ns auto-ap.jobs.register-invoice-import
(:gen-class) (:gen-class)
(:require (:require
[auto-ap.jobs.core :refer [execute]]
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[datomic.api :as d] [auto-ap.datomic :refer [conn pull-attr]]
[auto-ap.utils :refer [dollars=]] [auto-ap.jobs.core :refer [execute]]
[clojure.string :as str]
[clj-time.coerce :as coerce]
[auto-ap.time :as atime]
[auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.datomic :refer [conn]] [auto-ap.time :as atime]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.logging :as log]
[config.core :refer [env]] [config.core :refer [env]]
[clojure.tools.logging :as log])) [datomic.client.api :as dc])
(:import [java.util UUID]))
(def bucket (:data-bucket env)) (def bucket (:data-bucket env))
@@ -30,12 +31,11 @@
(throw e)))) (throw e))))
(defn register-invoice-import* [data] (defn register-invoice-import* [data]
(let [db (d/db conn) (let [db (dc/db conn)
i->invoice-id (fn [i] i->invoice-id (fn [i]
(try (Long/parseLong i) (try (Long/parseLong i)
(catch Exception _ (catch Exception _
(:db/id (d/pull db '[:db/id] (pull-attr db :db/id [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))
[:invoice/original-id (Long/parseLong (first (str/split i #"-")))])))))
invoice-totals (->> data invoice-totals (->> data
(drop 1) (drop 1)
(group-by first) (group-by first)
@@ -57,7 +57,7 @@
location] (drop 1 data) location] (drop 1 data)
:let [invoice-id (i->invoice-id i) :let [invoice-id (i->invoice-id i)
invoice (d/entity db invoice-id) invoice (dc/pull db '[*] invoice-id)
current-total (:invoice/total invoice) current-total (:invoice/total invoice)
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
@@ -65,10 +65,10 @@
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
(:db/id (first (:invoice/expense-accounts invoice))) (:db/id (first (:invoice/expense-accounts invoice)))
(d/tempid :db.part/user)) (str (UUID/randomUUID)))
invoice-expense-account (when-not new-account? invoice-expense-account (when-not new-account?
(or (d/entity db invoice-expense-account-id) (or (dc/pull db '[*] invoice-expense-account-id)
(d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) (dc/pull db '[*] [:invoice-expense-account/original-id invoice-expense-account-id])))
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
target-account-id (Long/parseLong (str/trim target-account)) target-account-id (Long/parseLong (str/trim target-account))
@@ -84,7 +84,7 @@
target-expense-account-location location target-expense-account-location location
[[_ _ invoice-payment]] (vec (d/q [[_ _ invoice-payment]] (vec (dc/q
'[:find ?p ?a ?ip '[:find ?p ?a ?ip
:in $ ?i :in $ ?i
:where [?ip :invoice-payment/invoice ?i] :where [?ip :invoice-payment/invoice ?i]

2
src/clj/auto_ap/ledger.clj
View File

@@ -418,7 +418,7 @@
(let [db (dc/db conn) (let [db (dc/db conn)
tx (audit-transact transaction id) tx (audit-transact transaction id)
affected-entities (->> (:tx-data tx) affected-entities (->> (:tx-data tx)
(map (fn [^datomic.db.Datum x] (map (fn [x]
{:e (:e x) {:e (:e x)
:a (:a x) :a (:a x)
:v (:v x) :v (:v x)

1
src/clj/auto_ap/routes/exports.clj
View File

@@ -19,7 +19,6 @@
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd] [com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]] [config.core :refer [env]]
[datomic.api :as d]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia])) [venia.core :as venia]))

5
src/clj/auto_ap/routes/queries.clj
View File

@@ -9,9 +9,8 @@
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd] [com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]] [config.core :refer [env]]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[datomic.api :as d]
[ring.middleware.json :refer [wrap-json-response]] [ring.middleware.json :refer [wrap-json-response]]
[ring.util.request :refer [body-string]] [ring.util.request :refer [body-string]]

76
src/clj/auto_ap/square/core3.clj
View File

@@ -13,7 +13,7 @@
[clojure.set :as set] [clojure.set :as set]
[clojure.string :as str] [clojure.string :as str]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
[datomic.api :as d] [datomic.client.api :as dc]
[manifold.deferred :as de] [manifold.deferred :as de]
[manifold.executor :as ex] [manifold.executor :as ex]
[manifold.stream :as s] [manifold.stream :as s]
@@ -471,7 +471,7 @@
(log/info ::transforming-settlements) (log/info ::transforming-settlements)
(try (try
(->> (for [settlement settlements (->> (for [settlement settlements
:let [best-sales-date (some->> (d/q '[:find ?s4 (count ?s) :let [best-sales-date (some->> (dc/q '[:find ?s4 (count ?s)
:in $ ?settlement-id :in $ ?settlement-id
:where :where
[?settlement :expected-deposit/external-id ?settlement-id] [?settlement :expected-deposit/external-id ?settlement-id]
@@ -481,7 +481,7 @@
[(clj-time.coerce/to-date-time ?sales-date) ?s2] [(clj-time.coerce/to-date-time ?sales-date) ?s2]
[(auto-ap.time/localize ?s2) ?s3] [(auto-ap.time/localize ?s2) ?s3]
[(clj-time.coerce/to-local-date ?s3) ?s4]] [(clj-time.coerce/to-local-date ?s3) ?s4]]
(d/db conn) (dc/db conn)
(str "square/settlement/" (:id settlement))) (str "square/settlement/" (:id settlement)))
(sort-by last) (sort-by last)
last last
@@ -562,7 +562,7 @@
(doseq [x (partition-all 100 results)] (doseq [x (partition-all 100 results)]
(log/info ::loading-orders (log/info ::loading-orders
:count (count x)) :count (count x))
@(d/transact conn x)))))))) (dc/transact conn {:tx-data x}))))))))
(defn upsert-settlements (defn upsert-settlements
@@ -582,7 +582,7 @@
(doseq [x (partition-all 20 settlements)] (doseq [x (partition-all 20 settlements)]
(log/info ::loading-deposits (log/info ::loading-deposits
:count (count x)) :count (count x))
@(d/transact conn x)) (dc/transact conn {:tx-data x}))
(log/info ::done-loading-deposits))))))) (log/info ::done-loading-deposits)))))))
(defn upsert-refunds (defn upsert-refunds
@@ -603,7 +603,7 @@
(log/info ::loading-refunds (log/info ::loading-refunds
:count (count x) :count (count x)
:sample (first x)) :sample (first x))
@(d/transact conn x)) (dc/transact conn {:tx-data x}))
(catch Throwable e (catch Throwable e
(log/error ::upsert-refunds-failed (log/error ::upsert-refunds-failed
@@ -618,26 +618,26 @@
(defn get-square-clients (defn get-square-clients
([] ([]
(d/q '[:find [(pull ?c [:db/id (map first (dc/q '[:find (pull ?c [:db/id
:client/square-integration-status :client/square-integration-status
:client/code :client/code
:client/square-auth-token :client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}])
:in $ :in $
:where [?c :client/square-auth-token] :where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"]] [?c :client/feature-flags "new-square"]]
(d/db conn))) (dc/db conn))))
([ & codes] ([ & codes]
(d/q '[:find [(pull ?c [:db/id (map first (dc/q '[:find (pull ?c [:db/id
:client/code :client/code
:client/square-auth-token :client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}])
:in $ [?code ...] :in $ [?code ...]
:where [?c :client/square-auth-token] :where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"] [?c :client/feature-flags "new-square"]
[?c :client/code ?code]] [?c :client/code ?code]]
(d/db conn) (dc/db conn)
codes))) codes))))
(defn upsert-locations (defn upsert-locations
([] ([]
@@ -653,30 +653,30 @@
(:client/square-locations client)))] (:client/square-locations client)))]
(de/chain (client-locations client) (de/chain (client-locations client)
(fn [client-locations] (fn [client-locations]
@(d/transact conn (dc/transact conn
(for [square-location client-locations] {:tx-data (for [square-location client-locations]
{:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user)) {:db/id (or (square-id->id (:id square-location)) (str (java.util.UUID/randomUUID)))
:client/_square-locations (:db/id client) :client/_square-locations (:db/id client)
:square-location/name (:name square-location) :square-location/name (:name square-location)
:square-location/square-id (:id square-location)}))))))) :square-location/square-id (:id square-location)})}))))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn reset [] (defn reset []
(->> (->>
(d/query {:query {:find ['?e] (dc/q {:query {:find ['?e]
:in ['$] :in ['$]
:where ['(or [?e :sales-order/date] :where ['(or [?e :sales-order/date]
[?e :expected-deposit/date])]} [?e :expected-deposit/date])]}
:args [(d/db conn)]}) :args [(dc/db conn)]})
(map first) (map first)
(map (fn [x] [:db/retractEntity x])))) (map (fn [x] [:db/retractEntity x]))))
(defn mark-integration-status [client integration-status] (defn mark-integration-status [client integration-status]
@(d/transact conn (dc/transact conn
[{:db/id (:db/id client) {:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status :client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id) :db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}])) (str (java.util.UUID/randomUUID))))}]}))
(defn upsert-all [ & clients] (defn upsert-all [ & clients]
(capture-context->lc (capture-context->lc

10
src/clj/auto_ap/ssr/admin.clj
View File

@@ -8,7 +8,7 @@
[auto-ap.time :as atime] [auto-ap.time :as atime]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[clojure.string :as str] [clojure.string :as str]
[datomic.api :as d] [datomic.client.api :as dc]
[hiccup2.core :as hiccup])) [hiccup2.core :as hiccup]))
(defn tx-rows->changes [history] (defn tx-rows->changes [history]
@@ -123,13 +123,13 @@
(get params "entity-id") (get params "entity-id")
(get form-params "entity-id"))) (get form-params "entity-id")))
history (->> history (->>
(d/q '[:find ?a2 ?v (pull ?tx [:db/txInstant :audit/user :db/id]) ?ad (dc/q '[:find ?a2 ?v (pull ?tx [:db/txInstant :audit/user :db/id]) ?ad
:in $ $$ ?i :in $ $$ ?i
:where :where
[$$ ?i ?a ?v ?tx ?ad] [$$ ?i ?a ?v ?tx ?ad]
[$ ?a :db/ident ?a2]] [$ ?a :db/ident ?a2]]
(d/db conn) (dc/db conn)
(d/history (d/db conn)) (dc/history (dc/db conn))
entity-id ) entity-id )
tx-rows->changes tx-rows->changes
(sort-by (comp :db/id first)) (sort-by (comp :db/id first))
@@ -166,7 +166,7 @@
:request request) :request request)
(try (try
(let [entity-id (Long/parseLong entity-id) (let [entity-id (Long/parseLong entity-id)
data (d/pull (d/db conn) data (dc/pull (dc/db conn)
'[*] '[*]
entity-id)] entity-id)]

32
src/clj/auto_ap/ssr/company/company_1099.clj
View File

@@ -8,7 +8,7 @@
[auto-ap.ssr.utils :refer [html-response]] [auto-ap.ssr.utils :refer [html-response]]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[clojure.string :as str] [clojure.string :as str]
[datomic.api :as d] [datomic.client.api :as dc]
[hiccup2.core :as hiccup])) [hiccup2.core :as hiccup]))
(defn cannot-overwrite? [vendor] (defn cannot-overwrite? [vendor]
@@ -17,18 +17,18 @@
(:vendor/legal-entity-tin-type vendor)))) (:vendor/legal-entity-tin-type vendor))))
(defn get-1099-companies [user session] (defn get-1099-companies [user session]
(let [clients (->> (d/q '[:find ?c (let [clients (->> (dc/q '[:find ?c
:in $ ?user :in $ ?user
:where [?c :client/code] :where [?c :client/code]
[(auto-ap.graphql.utils/can-see-client? ?user ?c)]] [(auto-ap.graphql.utils/can-see-client? ?user ?c)]]
(d/db conn) user) (dc/db conn) user)
(map first) (map first)
set) set)
results (cond results (cond
(and (some-> session :client :db/id) (and (some-> session :client :db/id)
(can-see-client? user (can-see-client? user
(some-> session :client :db/id))) (some-> session :client :db/id)))
(d/q '[:find (dc/q '[:find
(pull ?c [:client/code :db/id]) (pull ?c [:client/code :db/id])
(pull ?v [:db/id (pull ?v [:db/id
:vendor/name :vendor/name
@@ -55,11 +55,11 @@
[?p :payment/amount ?a] [?p :payment/amount ?a]
[?p :payment/vendor ?v]] [?p :payment/vendor ?v]]
(d/db conn) (dc/db conn)
(some-> session :client :db/id)) (some-> session :client :db/id))
(is-admin? user) (is-admin? user)
(d/q '[:find (dc/q '[:find
(pull ?c [:client/code :db/id]) (pull ?c [:client/code :db/id])
(pull ?v [:db/id (pull ?v [:db/id
:vendor/name :vendor/name
@@ -85,10 +85,10 @@
[?p :payment/client ?c] [?p :payment/client ?c]
[?p :payment/amount ?a] [?p :payment/amount ?a]
[?p :payment/vendor ?v]] [?p :payment/vendor ?v]]
(d/db conn)) (dc/db conn))
:else :else
(d/q '[:find (dc/q '[:find
(pull ?c [:client/code :db/id]) (pull ?c [:client/code :db/id])
(pull ?v [:db/id (pull ?v [:db/id
:vendor/name :vendor/name
@@ -115,7 +115,7 @@
[?p :payment/amount ?a] [?p :payment/amount ?a]
[?p :payment/vendor ?v]] [?p :payment/vendor ?v]]
(d/db conn) (dc/db conn)
clients))] clients))]
(->> results (->> results
(filter (fn [[_ _ a]] (filter (fn [[_ _ a]]
@@ -208,18 +208,18 @@
:else k)) :else k))
(defn vendor-save [{:keys [form-params identity route-params] :as request}] (defn vendor-save [{:keys [form-params identity route-params] :as request}]
(when-not (cannot-overwrite? (d/pull (d/db conn) '[*] (Long/parseLong (:vendor-id route-params)))) (when-not (cannot-overwrite? (dc/pull (dc/db conn) '[*] (Long/parseLong (:vendor-id route-params))))
@(d/transact conn [(remove-nils @(dc/transact conn {:tx-data [(remove-nils
(-> (form-data->map form-params) (-> (form-data->map form-params)
(assoc :db/id (Long/parseLong (:vendor-id route-params))) (assoc :db/id (Long/parseLong (:vendor-id route-params)))
(update :vendor/legal-entity-1099-type #(some->> % (keyword "legal-entity-1099-type"))) (update :vendor/legal-entity-1099-type #(some->> % (keyword "legal-entity-1099-type")))
(update :vendor/legal-entity-tin-type #(some->> % (keyword "legal-entity-tin-type")))))])) (update :vendor/legal-entity-tin-type #(some->> % (keyword "legal-entity-tin-type")))))]}))
(html-response (html-response
(table request :flash-id (Long/parseLong (:vendor-id route-params))))) (table request :flash-id (Long/parseLong (:vendor-id route-params)))))
(defn vendor-dialog [request] (defn vendor-dialog [request]
(let [vendor (d/pull (d/db conn) '[* {:vendor/legal-entity-1099-type [:db/ident] (let [vendor (dc/pull (dc/db conn) '[* {:vendor/legal-entity-1099-type [:db/ident]
:vendor/legal-entity-tin-type [:db/ident]}] (Long/parseLong (:vendor-id (:params request))))] ;; TODO perms :vendor/legal-entity-tin-type [:db/ident]}] (Long/parseLong (:vendor-id (:params request))))] ;; TODO perms
(html-response (html-response
[:form {:hx-post (bidi/path-for ssr-routes/only-routes [:form {:hx-post (bidi/path-for ssr-routes/only-routes