notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

making cloud runnable again.

Browse Source
This commit is contained in:
Bryce Covert
2022-09-28 09:03:28 -07:00
parent d02fba2b44
commit 65ecee7014
11 changed files with 70 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
project.clj
View File

@@ -6,7 +6,8 @@
:username "datomic@brycecovertoperations.com" :username "datomic@brycecovertoperations.com"
:password "9a382afc-d119-44db-83c2-98d8057d7666"}} :password "9a382afc-d119-44db-83c2-98d8057d7666"}}
:dependencies [[org.clojure/clojure "1.10.1"] :dependencies [[com.google.guava/guava "31.1-jre"]
[org.clojure/clojure "1.10.1"]
[com.datomic/dev-local "1.0.243"] [com.datomic/dev-local "1.0.243"]
[com.datomic/datomic-pro "0.9.5783" [com.datomic/datomic-pro "0.9.5783"
:exclusions [com.google.guava/guava :exclusions [com.google.guava/guava
@@ -43,7 +44,7 @@
[digest "1.4.10"] [digest "1.4.10"]
[clj-http "3.11.0" :exclusions [org.apache.httpcomponents/httpclient org.apache.httpcomponents/httpcore]] [clj-http "3.11.0" :exclusions [org.apache.httpcomponents/httpclient org.apache.httpcomponents/httpcore]]
[clj-pdf "2.2.33"] [clj-pdf "2.2.33"]
[org.clojure/core.async "1.3.610"] [org.clojure/core.async "1.5.648"]
[fogus/ring-edn "0.3.0"] [fogus/ring-edn "0.3.0"]
[buddy/buddy-auth "2.2.0" [buddy/buddy-auth "2.2.0"
:exclusions [com.fasterxml.jackson.dataformat/jackson-dataformat-cbor :exclusions [com.fasterxml.jackson.dataformat/jackson-dataformat-cbor
@@ -128,7 +129,7 @@
:profiles { :profiles {
:dev :dev
{:resource-paths ["resources" "target"] {:resource-paths ["resources" "target"]
:dependencies [[binaryage/devtools "1.0.2"] :dependencies [#_[binaryage/devteols "1.0.2"]
[postgresql/postgresql "9.3-1102.jdbc41"] [postgresql/postgresql "9.3-1102.jdbc41"]
[org.clojure/java.jdbc "0.7.11"] [org.clojure/java.jdbc "0.7.11"]
[etaoin "0.4.1"] [etaoin "0.4.1"]
@@ -171,7 +172,7 @@
org.eclipse.jetty.websocket/websocket-server org.eclipse.jetty.websocket/websocket-server
org.eclipse.jetty.websocket/websocket-servlet org.eclipse.jetty.websocket/websocket-servlet
args4j]]]} args4j]]]}
:provided {:dependencies [[org.clojure/clojurescript "1.10.773" :provided {:dependencies [[org.clojure/clojurescript "1.11.4"
:exclusions [com.google.code.findbugs/jsr305 :exclusions [com.google.code.findbugs/jsr305
com.fasterxml.jackson.core/jackson-core]] com.fasterxml.jackson.core/jackson-core]]
[reagent "1.0.0" :exclusions [cljsjs/react cljsjs/react-dom cljsjs/react-dom-server] ] [reagent "1.0.0" :exclusions [cljsjs/react cljsjs/react-dom cljsjs/react-dom-server] ]

3
src/clj/auto_ap/datomic.clj
View File

@@ -685,7 +685,7 @@
(into (map (fn [i] (if is-component? (into (map (fn [i] (if is-component?
[:db/retractEntity i] [:db/retractEntity i]
[:db/retract e a i ])) retract-ids)) [:db/retract e a i ])) retract-ids))
(into (map (fn [i] [:db/add e a (:db/id i)]) new-rels)) (into (map (fn [i] [:db/add e a i]) new-rels))
(into (mapcat (fn [i] (upsert-entity db i)) vs))))) (into (mapcat (fn [i] (upsert-entity db i)) vs)))))
(defn reset-scalars [db e a vs] (defn reset-scalars [db e a vs]
@@ -703,6 +703,7 @@
(into (map (fn [i] [:db/add e a i]) new))))) (into (map (fn [i] [:db/add e a i]) new)))))
;; TODO unit test this
(defn upsert-entity [db entity] (defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id") (assert (:db/id entity) "Cannot upsert without :db/id")
(let [e (:db/id entity) (let [e (:db/id entity)

50
src/clj/auto_ap/ezcater/core.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.ezcater.core (ns auto-ap.ezcater.core
(:require (:require
[auto-ap.datomic :refer [conn]] [auto-ap.datomic :refer [conn random-tempid]]
[datomic.api :as d] [datomic.client.api :as dc]
[clj-http.client :as client] [clj-http.client :as client]
[venia.core :as v] [venia.core :as v]
[clojure.string :as str] [clojure.string :as str]
@@ -35,40 +35,40 @@
:subscriptions)) :subscriptions))
(defn get-integrations [] (defn get-integrations []
(map first (d/q '[:find (pull ?i [:ezcater-integration/api-key (map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid :ezcater-integration/subscriber-uuid
:db/id :db/id
:ezcater-integration/integration-status [:db/id]]) :ezcater-integration/integration-status [:db/id]])
:in $ :in $
:where [?i :ezcater-integration/api-key]] :where [?i :ezcater-integration/api-key]]
(d/db conn)))) (dc/db conn))))
(defn mark-integration-status [integration integration-status] (defn mark-integration-status [integration integration-status]
@(d/transact conn (dc/transact conn
[{:db/id (:db/id integration) {:tx-data [{:db/id (:db/id integration)
:ezcater-integration/integration-status (assoc integration-status :ezcater-integration/integration-status (assoc integration-status
:db/id (or (-> integration :ezcater-integration/integration-status :db/id) :db/id (or (-> integration :ezcater-integration/integration-status :db/id)
#db/id [:db.part/user]))}])) (random-tempid)))}]}))
(defn upsert-caterers (defn upsert-caterers
([integration] ([integration]
@(d/transact (dc/transact
conn conn
(for [caterer (get-caterers integration)] {:tx-data (for [caterer (get-caterers integration)]
{:db/id (:db/id integration) {:db/id (:db/id integration)
:ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")") :ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")")
:ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer))) :ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer)))
:ezcater-caterer/uuid (:uuid caterer)}]})))) :ezcater-caterer/uuid (:uuid caterer)}]})})))
(defn upsert-used-subscriptions (defn upsert-used-subscriptions
([integration] ([integration]
(let [extant (get-subscriptions integration) (let [extant (get-subscriptions integration)
to-ensure (set (map first (d/q '[:find ?cu to-ensure (set (map first (dc/q '[:find ?cu
:in $ :in $
:where [_ :client/ezcater-locations ?el] :where [_ :client/ezcater-locations ?el]
[?el :ezcater-location/caterer ?c] [?el :ezcater-location/caterer ?c]
[?c :ezcater-caterer/uuid ?cu]] [?c :ezcater-caterer/uuid ?cu]]
(d/db conn)))) (dc/db conn))))
to-create (set/difference to-create (set/difference
to-ensure to-ensure
(set (map :parentId extant)))] (set (map :parentId extant)))]
@@ -114,7 +114,7 @@
:integration-status/message (.getMessage e)})))))) :integration-status/message (.getMessage e)}))))))
(defn get-caterer [caterer-uuid] (defn get-caterer [caterer-uuid]
(d/pull (d/db conn) (dc/pull (dc/db conn)
'[:ezcater-caterer/name '[:ezcater-caterer/name
{:ezcater-integration/_caterers [:ezcater-integration/api-key]} {:ezcater-integration/_caterers [:ezcater-integration/api-key]}
{:ezcater-location/_caterer [:ezcater-location/location {:ezcater-location/_caterer [:ezcater-location/location
@@ -270,12 +270,12 @@
(alog/info (alog/info
::try-import-order ::try-import-order
:json json) :json json)
@(d/transact conn (filter identity (dc/transact conn {:tx-data (filter identity
[(some-> json [(some-> json
(lookup-order) (lookup-order)
(order->sales-order) (order->sales-order)
(update :sales-order/date coerce/to-date) (update :sales-order/date coerce/to-date)
(update-in [:sales-order/charges 0 :charge/date] coerce/to-date))]))) (update-in [:sales-order/charges 0 :charge/date] coerce/to-date))])}))
(defn upsert-recent [] (defn upsert-recent []
(upsert-ezcater) (upsert-ezcater)
@@ -346,6 +346,6 @@
updated-order))] updated-order))]
(alog/info :found-orders-to-update (alog/info :found-orders-to-update
:orders orders-to-update) :orders orders-to-update)
@(d/transact conn orders-to-update))) @(dc/transact conn {:tx-data orders-to-update})))

6
src/clj/auto_ap/graphql/ezcater.clj
View File

@@ -4,16 +4,16 @@
[auto-ap.graphql.utils :refer [assert-admin cleanse-query]] [auto-ap.graphql.utils :refer [assert-admin cleanse-query]]
[auto-ap.graphql.vendors :refer [partial-match-first]] [auto-ap.graphql.vendors :refer [partial-match-first]]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]] [com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d] [datomic.client.api :as dc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) [auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
(defn search [context args _] (defn search [context args _]
(assert-admin (:id context)) (assert-admin (:id context))
(let [search-query (cleanse-query (:query args)) (let [search-query (cleanse-query (:query args))
data (d/q '[:find ?n ?i ?s data (dc/q '[:find ?n ?i ?s
:in $ ?q :in $ ?q
:where [(fulltext $ :ezcater-caterer/search-terms ?q) [[?i ?n _ ?s]]]] :where [(fulltext $ :ezcater-caterer/search-terms ?q) [[?i ?n _ ?s]]]]
(d/db conn) (dc/db conn)
search-query)] search-query)]
(->> data (->> data
(sort-by (comp - last)) (sort-by (comp - last))

13
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -1,8 +1,6 @@
(ns auto-ap.graphql.invoices (ns auto-ap.graphql.invoices
(:require (:require
[auto-ap.datomic [auto-ap.datomic :refer [conn random-tempid remove-nils upsert-entity]]
:refer [conn remove-nils upsert-entity]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.invoices :as d-invoices]
[auto-ap.datomic.vendors :as d-vendors] [auto-ap.datomic.vendors :as d-vendors]
@@ -92,10 +90,10 @@
(throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number :validation-error (str "Invoice '" invoice_number "' already exists.")})))) (throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number :validation-error (str "Invoice '" invoice_number "' already exists.")}))))
(defn expense-account->entity [{:keys [id account_id amount location]}] (defn expense-account->entity [{:keys [id account_id amount location]}]
(remove-nils #:invoice-expense-account {:amount amount #:invoice-expense-account {:amount amount
:db/id id :db/id (or id (random-tempid))
:account account_id :account account_id
:location location})) :location location})
(defn add-invoice-transaction [{:keys [total invoice_number scheduled_payment client_id vendor_id date due expense_accounts]}] (defn add-invoice-transaction [{:keys [total invoice_number scheduled_payment client_id vendor_id date due expense_accounts]}]
(let [vendor (d-vendors/get-by-id vendor_id) (let [vendor (d-vendors/get-by-id vendor_id)
@@ -109,6 +107,7 @@
due)) due))
_ (when-not (:db/id account) _ (when-not (:db/id account)
(throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))] (throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))]
`(upsert-entity ~{:db/id "invoice" `(upsert-entity ~{:db/id "invoice"
:invoice/invoice-number invoice_number :invoice/invoice-number invoice_number
:invoice/client client_id :invoice/client client_id

5
src/clj/auto_ap/import/yodlee2.clj
View File

@@ -10,9 +10,8 @@
[clojure.string :as str] [clojure.string :as str]
[com.unbounce.dogstatsd.core :as statsd] [com.unbounce.dogstatsd.core :as statsd]
[datomic.client.api :as dc] [datomic.client.api :as dc]
[mount.core :as mount] [digest :as di]
[unilog.context :as lc] [unilog.context :as lc]))
[yang.scheduler :as scheduler]))
#_{:clj-kondo/ignore [:unresolved-var]} #_{:clj-kondo/ignore [:unresolved-var]}
(defn yodlee->transaction [transaction use-date-instead-of-post-date?] (defn yodlee->transaction [transaction use-date-instead-of-post-date?]

4
src/clj/auto_ap/ledger.clj
View File

@@ -227,8 +227,8 @@
:end end}))))) :end end})))))
(defn entities-since-last-ledger-entry [] (defn entities-since-last-ledger-entry []
(count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5))) (count (dc/tx-range conn {:start (c/to-date (t/plus (t/now) (t/days -5)))
:end (coerce/to-date (time/now))}))) :end (c/to-date (t/now))})))
(defn mismatched-transactions (defn mismatched-transactions
([] ([]

2
src/clj/auto_ap/square/core.clj
View File

@@ -496,7 +496,7 @@
{:tx-data [{:db/id (:db/id client) {:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status :client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id) :db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}]})) (random-tempid)))}]}))
(defn upsert-all [ & clients] (defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients) (doseq [client (apply get-square-clients clients)

44
src/clj/auto_ap/square/core2.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.square.core2 (ns auto-ap.square.core2
(:require (:require
[auto-ap.datomic :refer [conn remove-nils]] [auto-ap.datomic :refer [conn remove-nils random-tempid]]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[clj-http.client :as client] [clj-http.client :as client]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
@@ -13,7 +13,7 @@
[clojure.string :as str] [clojure.string :as str]
[clojure.tools.logging :as log] [clojure.tools.logging :as log]
[cemerick.url :as url] [cemerick.url :as url]
[datomic.api :as d] [datomic.client.api :as dc]
[slingshot.slingshot :refer [try+]] [slingshot.slingshot :refer [try+]]
[unilog.context :as lc])) [unilog.context :as lc]))
@@ -399,7 +399,7 @@
(lc/with-context {:source "Square loading"} (lc/with-context {:source "Square loading"}
(doseq [x (partition-all 20 (daily-results client location start end))] (doseq [x (partition-all 20 (daily-results client location start end))]
(log/info "Loading " (count x)) (log/info "Loading " (count x))
@(d/transact conn x))))) (dc/transact conn {:tx-data x})))))
(defn upsert-settlements (defn upsert-settlements
([client] ([client]
@@ -411,7 +411,7 @@
:client (:client/code client)} :client (:client/code client)}
(doseq [x (partition-all 20 (daily-settlements client location))] (doseq [x (partition-all 20 (daily-settlements client location))]
(log/info "Loading expected deposit" (count x)) (log/info "Loading expected deposit" (count x))
@(d/transact conn x)) (dc/transact conn {:tx-data x}))
(log/info "Done loading settlements")))) (log/info "Done loading settlements"))))
(defn upsert-refunds (defn upsert-refunds
@@ -425,7 +425,7 @@
:location (:square-location/client-location client)} :location (:square-location/client-location client)}
(doseq [x (partition-all 20 (refunds client location))] (doseq [x (partition-all 20 (refunds client location))]
(log/info "Loading refund" (count x)) (log/info "Loading refund" (count x))
@(d/transact conn x)) (dc/transact conn {:tx-data x}))
(log/info "Done loading refunds")))) (log/info "Done loading refunds"))))
(def square-read [:db/id (def square-read [:db/id
@@ -435,7 +435,7 @@
(defn get-square-clients (defn get-square-clients
([] ([]
(d/q '[:find [(pull ?c [:db/id (dc/q '[:find [(pull ?c [:db/id
:client/square-integration-status :client/square-integration-status
:client/code :client/code
:client/square-auth-token :client/square-auth-token
@@ -443,9 +443,9 @@
:in $ :in $
:where [?c :client/square-auth-token] :where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"]] [?c :client/feature-flags "new-square"]]
(d/db conn))) (dc/db conn)))
([ & codes] ([ & codes]
(d/q '[:find [(pull ?c [:db/id (dc/q '[:find [(pull ?c [:db/id
:client/code :client/code
:client/square-auth-token :client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...]
@@ -453,7 +453,7 @@
:where [?c :client/square-auth-token] :where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"] [?c :client/feature-flags "new-square"]
[?c :client/code ?code]] [?c :client/code ?code]]
(d/db conn) (dc/db conn)
codes))) codes)))
(defn upsert-locations (defn upsert-locations
@@ -466,31 +466,29 @@
[(:square-location/square-id sl) [(:square-location/square-id sl)
(:db/id sl)]) (:db/id sl)])
(:client/square-locations client)))] (:client/square-locations client)))]
(->> (for [square-location (client-locations client)] (dc/transact conn {:tx-data (for [square-location (client-locations client)]
{:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user)) {:db/id (or (square-id->id (:id square-location)) (random-tempid))
:client/_square-locations (:db/id client) :client/_square-locations (:db/id client)
:square-location/name (:name square-location) :square-location/name (:name square-location)
:square-location/square-id (:id square-location)}) :square-location/square-id (:id square-location)})}))))
(d/transact conn)
deref))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn reset [] (defn reset []
(->> (->>
(d/query {:query {:find ['?e] (dc/q {:query {:find ['?e]
:in ['$] :in ['$]
:where ['(or [?e :sales-order/date] :where ['(or [?e :sales-order/date]
[?e :expected-deposit/date])]} [?e :expected-deposit/date])]}
:args [(d/db conn)]}) :args [(dc/db conn)]})
(map first) (map first)
(map (fn [x] [:db/retractEntity x])))) (map (fn [x] [:db/retractEntity x]))))
(defn mark-integration-status [client integration-status] (defn mark-integration-status [client integration-status]
@(d/transact conn (dc/transact conn
[{:db/id (:db/id client) {:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status :client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id) :db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}])) (random-tempid)))}]}))
(defn upsert-all [ & clients] (defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients) (doseq [client (apply get-square-clients clients)

1
src/clj/user.clj
View File

@@ -444,6 +444,7 @@
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
(defn restart-db [] (defn restart-db []
(require 'datomic.dev-local)
(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) (datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})
(mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})) (mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))
(start-db)) (start-db))

2
things-to-search-for.txt
View File

@@ -5,3 +5,5 @@ New way of automating ledger entries
Make sure no history on ledger Make sure no history on ledger
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
automatically rebuild search indexes automatically rebuild search indexes
ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity