From 65ecee70148b9ae44a0ccd05c1c666a44eaa68e0 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 28 Sep 2022 09:03:28 -0700 Subject: [PATCH] making cloud runnable again. --- project.clj | 9 ++--- src/clj/auto_ap/datomic.clj | 3 +- src/clj/auto_ap/ezcater/core.clj | 50 ++++++++++++++-------------- src/clj/auto_ap/graphql/ezcater.clj | 6 ++-- src/clj/auto_ap/graphql/invoices.clj | 13 ++++---- src/clj/auto_ap/import/yodlee2.clj | 5 ++- src/clj/auto_ap/ledger.clj | 4 +-- src/clj/auto_ap/square/core.clj | 2 +- src/clj/auto_ap/square/core2.clj | 44 ++++++++++++------------ src/clj/user.clj | 1 + things-to-search-for.txt | 2 ++ 11 files changed, 70 insertions(+), 69 deletions(-) diff --git a/project.clj b/project.clj index 6a71ed37..c2f84b0f 100644 --- a/project.clj +++ b/project.clj @@ -6,7 +6,8 @@ :username "datomic@brycecovertoperations.com" :password "9a382afc-d119-44db-83c2-98d8057d7666"}} - :dependencies [[org.clojure/clojure "1.10.1"] + :dependencies [[com.google.guava/guava "31.1-jre"] + [org.clojure/clojure "1.10.1"] [com.datomic/dev-local "1.0.243"] [com.datomic/datomic-pro "0.9.5783" :exclusions [com.google.guava/guava @@ -43,7 +44,7 @@ [digest "1.4.10"] [clj-http "3.11.0" :exclusions [org.apache.httpcomponents/httpclient org.apache.httpcomponents/httpcore]] [clj-pdf "2.2.33"] - [org.clojure/core.async "1.3.610"] + [org.clojure/core.async "1.5.648"] [fogus/ring-edn "0.3.0"] [buddy/buddy-auth "2.2.0" :exclusions [com.fasterxml.jackson.dataformat/jackson-dataformat-cbor @@ -128,7 +129,7 @@ :profiles { :dev {:resource-paths ["resources" "target"] - :dependencies [[binaryage/devtools "1.0.2"] + :dependencies [#_[binaryage/devteols "1.0.2"] [postgresql/postgresql "9.3-1102.jdbc41"] [org.clojure/java.jdbc "0.7.11"] [etaoin "0.4.1"] @@ -171,7 +172,7 @@ org.eclipse.jetty.websocket/websocket-server org.eclipse.jetty.websocket/websocket-servlet args4j]]]} - :provided {:dependencies [[org.clojure/clojurescript "1.10.773" + :provided {:dependencies [[org.clojure/clojurescript "1.11.4" :exclusions [com.google.code.findbugs/jsr305 com.fasterxml.jackson.core/jackson-core]] [reagent "1.0.0" :exclusions [cljsjs/react cljsjs/react-dom cljsjs/react-dom-server] ] diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index 31131b6d..0865f185 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -685,7 +685,7 @@ (into (map (fn [i] (if is-component? [:db/retractEntity i] [:db/retract e a i ])) retract-ids)) - (into (map (fn [i] [:db/add e a (:db/id i)]) new-rels)) + (into (map (fn [i] [:db/add e a i]) new-rels)) (into (mapcat (fn [i] (upsert-entity db i)) vs))))) (defn reset-scalars [db e a vs] @@ -703,6 +703,7 @@ (into (map (fn [i] [:db/add e a i]) new))))) +;; TODO unit test this (defn upsert-entity [db entity] (assert (:db/id entity) "Cannot upsert without :db/id") (let [e (:db/id entity) diff --git a/src/clj/auto_ap/ezcater/core.clj b/src/clj/auto_ap/ezcater/core.clj index 0b35d7e2..c8b4cbba 100644 --- a/src/clj/auto_ap/ezcater/core.clj +++ b/src/clj/auto_ap/ezcater/core.clj @@ -1,7 +1,7 @@ (ns auto-ap.ezcater.core (:require - [auto-ap.datomic :refer [conn]] - [datomic.api :as d] + [auto-ap.datomic :refer [conn random-tempid]] + [datomic.client.api :as dc] [clj-http.client :as client] [venia.core :as v] [clojure.string :as str] @@ -35,40 +35,40 @@ :subscriptions)) (defn get-integrations [] - (map first (d/q '[:find (pull ?i [:ezcater-integration/api-key + (map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key :ezcater-integration/subscriber-uuid :db/id :ezcater-integration/integration-status [:db/id]]) :in $ :where [?i :ezcater-integration/api-key]] - (d/db conn)))) + (dc/db conn)))) (defn mark-integration-status [integration integration-status] - @(d/transact conn - [{:db/id (:db/id integration) - :ezcater-integration/integration-status (assoc integration-status - :db/id (or (-> integration :ezcater-integration/integration-status :db/id) - #db/id [:db.part/user]))}])) + (dc/transact conn + {:tx-data [{:db/id (:db/id integration) + :ezcater-integration/integration-status (assoc integration-status + :db/id (or (-> integration :ezcater-integration/integration-status :db/id) + (random-tempid)))}]})) (defn upsert-caterers ([integration] - @(d/transact + (dc/transact conn - (for [caterer (get-caterers integration)] - {:db/id (:db/id integration) - :ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")") - :ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer))) - :ezcater-caterer/uuid (:uuid caterer)}]})))) + {:tx-data (for [caterer (get-caterers integration)] + {:db/id (:db/id integration) + :ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")") + :ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer))) + :ezcater-caterer/uuid (:uuid caterer)}]})}))) (defn upsert-used-subscriptions ([integration] (let [extant (get-subscriptions integration) - to-ensure (set (map first (d/q '[:find ?cu + to-ensure (set (map first (dc/q '[:find ?cu :in $ :where [_ :client/ezcater-locations ?el] [?el :ezcater-location/caterer ?c] [?c :ezcater-caterer/uuid ?cu]] - (d/db conn)))) + (dc/db conn)))) to-create (set/difference to-ensure (set (map :parentId extant)))] @@ -114,7 +114,7 @@ :integration-status/message (.getMessage e)})))))) (defn get-caterer [caterer-uuid] - (d/pull (d/db conn) + (dc/pull (dc/db conn) '[:ezcater-caterer/name {:ezcater-integration/_caterers [:ezcater-integration/api-key]} {:ezcater-location/_caterer [:ezcater-location/location @@ -270,12 +270,12 @@ (alog/info ::try-import-order :json json) - @(d/transact conn (filter identity - [(some-> json - (lookup-order) - (order->sales-order) - (update :sales-order/date coerce/to-date) - (update-in [:sales-order/charges 0 :charge/date] coerce/to-date))]))) + (dc/transact conn {:tx-data (filter identity + [(some-> json + (lookup-order) + (order->sales-order) + (update :sales-order/date coerce/to-date) + (update-in [:sales-order/charges 0 :charge/date] coerce/to-date))])})) (defn upsert-recent [] (upsert-ezcater) @@ -346,6 +346,6 @@ updated-order))] (alog/info :found-orders-to-update :orders orders-to-update) - @(d/transact conn orders-to-update))) + @(dc/transact conn {:tx-data orders-to-update}))) diff --git a/src/clj/auto_ap/graphql/ezcater.clj b/src/clj/auto_ap/graphql/ezcater.clj index 90304b3d..b9186202 100644 --- a/src/clj/auto_ap/graphql/ezcater.clj +++ b/src/clj/auto_ap/graphql/ezcater.clj @@ -4,16 +4,16 @@ [auto-ap.graphql.utils :refer [assert-admin cleanse-query]] [auto-ap.graphql.vendors :refer [partial-match-first]] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d] + [datomic.client.api :as dc] [auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) (defn search [context args _] (assert-admin (:id context)) (let [search-query (cleanse-query (:query args)) - data (d/q '[:find ?n ?i ?s + data (dc/q '[:find ?n ?i ?s :in $ ?q :where [(fulltext $ :ezcater-caterer/search-terms ?q) [[?i ?n _ ?s]]]] - (d/db conn) + (dc/db conn) search-query)] (->> data (sort-by (comp - last)) diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index 978424fb..c7618ace 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -1,8 +1,6 @@ (ns auto-ap.graphql.invoices (:require - [auto-ap.datomic - :refer [conn remove-nils upsert-entity]] - [auto-ap.ledger :refer [transact-with-ledger]] + [auto-ap.datomic :refer [conn random-tempid remove-nils upsert-entity]] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.vendors :as d-vendors] @@ -92,10 +90,10 @@ (throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number :validation-error (str "Invoice '" invoice_number "' already exists.")})))) (defn expense-account->entity [{:keys [id account_id amount location]}] - (remove-nils #:invoice-expense-account {:amount amount - :db/id id - :account account_id - :location location})) + #:invoice-expense-account {:amount amount + :db/id (or id (random-tempid)) + :account account_id + :location location}) (defn add-invoice-transaction [{:keys [total invoice_number scheduled_payment client_id vendor_id date due expense_accounts]}] (let [vendor (d-vendors/get-by-id vendor_id) @@ -109,6 +107,7 @@ due)) _ (when-not (:db/id account) (throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))] + `(upsert-entity ~{:db/id "invoice" :invoice/invoice-number invoice_number :invoice/client client_id diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj index 103e6b23..978fefe9 100644 --- a/src/clj/auto_ap/import/yodlee2.clj +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -10,9 +10,8 @@ [clojure.string :as str] [com.unbounce.dogstatsd.core :as statsd] [datomic.client.api :as dc] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) + [digest :as di] + [unilog.context :as lc])) #_{:clj-kondo/ignore [:unresolved-var]} (defn yodlee->transaction [transaction use-date-instead-of-post-date?] diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 0fd6d327..097ed885 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -227,8 +227,8 @@ :end end}))))) (defn entities-since-last-ledger-entry [] - (count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5))) - :end (coerce/to-date (time/now))}))) + (count (dc/tx-range conn {:start (c/to-date (t/plus (t/now) (t/days -5))) + :end (c/to-date (t/now))}))) (defn mismatched-transactions ([] diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index 0c891b7e..61d0edff 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -496,7 +496,7 @@ {:tx-data [{:db/id (:db/id client) :client/square-integration-status (assoc integration-status :db/id (or (-> client :client/square-integration-status :db/id) - #db/id [:db.part/user]))}]})) + (random-tempid)))}]})) (defn upsert-all [ & clients] (doseq [client (apply get-square-clients clients) diff --git a/src/clj/auto_ap/square/core2.clj b/src/clj/auto_ap/square/core2.clj index 9eae401e..9adc48cd 100644 --- a/src/clj/auto_ap/square/core2.clj +++ b/src/clj/auto_ap/square/core2.clj @@ -1,6 +1,6 @@ (ns auto-ap.square.core2 (:require - [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.datomic :refer [conn remove-nils random-tempid]] [auto-ap.time :as atime] [clj-http.client :as client] [clj-time.coerce :as coerce] @@ -13,7 +13,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [cemerick.url :as url] - [datomic.api :as d] + [datomic.client.api :as dc] [slingshot.slingshot :refer [try+]] [unilog.context :as lc])) @@ -399,7 +399,7 @@ (lc/with-context {:source "Square loading"} (doseq [x (partition-all 20 (daily-results client location start end))] (log/info "Loading " (count x)) - @(d/transact conn x))))) + (dc/transact conn {:tx-data x}))))) (defn upsert-settlements ([client] @@ -411,7 +411,7 @@ :client (:client/code client)} (doseq [x (partition-all 20 (daily-settlements client location))] (log/info "Loading expected deposit" (count x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (log/info "Done loading settlements")))) (defn upsert-refunds @@ -425,7 +425,7 @@ :location (:square-location/client-location client)} (doseq [x (partition-all 20 (refunds client location))] (log/info "Loading refund" (count x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (log/info "Done loading refunds")))) (def square-read [:db/id @@ -435,7 +435,7 @@ (defn get-square-clients ([] - (d/q '[:find [(pull ?c [:db/id + (dc/q '[:find [(pull ?c [:db/id :client/square-integration-status :client/code :client/square-auth-token @@ -443,9 +443,9 @@ :in $ :where [?c :client/square-auth-token] [?c :client/feature-flags "new-square"]] - (d/db conn))) + (dc/db conn))) ([ & codes] - (d/q '[:find [(pull ?c [:db/id + (dc/q '[:find [(pull ?c [:db/id :client/code :client/square-auth-token {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] @@ -453,7 +453,7 @@ :where [?c :client/square-auth-token] [?c :client/feature-flags "new-square"] [?c :client/code ?code]] - (d/db conn) + (dc/db conn) codes))) (defn upsert-locations @@ -466,31 +466,29 @@ [(:square-location/square-id sl) (:db/id sl)]) (:client/square-locations client)))] - (->> (for [square-location (client-locations client)] - {:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user)) - :client/_square-locations (:db/id client) - :square-location/name (:name square-location) - :square-location/square-id (:id square-location)}) - (d/transact conn) - deref)))) + (dc/transact conn {:tx-data (for [square-location (client-locations client)] + {:db/id (or (square-id->id (:id square-location)) (random-tempid)) + :client/_square-locations (:db/id client) + :square-location/name (:name square-location) + :square-location/square-id (:id square-location)})})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn reset [] (->> - (d/query {:query {:find ['?e] + (dc/q {:query {:find ['?e] :in ['$] :where ['(or [?e :sales-order/date] [?e :expected-deposit/date])]} - :args [(d/db conn)]}) + :args [(dc/db conn)]}) (map first) (map (fn [x] [:db/retractEntity x])))) (defn mark-integration-status [client integration-status] - @(d/transact conn - [{:db/id (:db/id client) - :client/square-integration-status (assoc integration-status - :db/id (or (-> client :client/square-integration-status :db/id) - #db/id [:db.part/user]))}])) + (dc/transact conn + {:tx-data [{:db/id (:db/id client) + :client/square-integration-status (assoc integration-status + :db/id (or (-> client :client/square-integration-status :db/id) + (random-tempid)))}]})) (defn upsert-all [ & clients] (doseq [client (apply get-square-clients clients) diff --git a/src/clj/user.clj b/src/clj/user.clj index db760499..466435b6 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -444,6 +444,7 @@ (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))) (defn restart-db [] + (require 'datomic.dev-local) (datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) (mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})) (start-db)) diff --git a/things-to-search-for.txt b/things-to-search-for.txt index a045c14a..d62dcb6c 100644 --- a/things-to-search-for.txt +++ b/things-to-search-for.txt @@ -5,3 +5,5 @@ New way of automating ledger entries Make sure no history on ledger it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes automatically rebuild search indexes +ezcater graphql needs search index too +make sure that temporary ids are set on all new things when using upsert-entity