notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

making cloud runnable again.

Browse Source
This commit is contained in:
Bryce Covert
2022-09-28 09:03:28 -07:00
parent d02fba2b44
commit 65ecee7014
11 changed files with 70 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
project.clj
View File

@@ -6,7 +6,8 @@
:username "datomic@brycecovertoperations.com"
:password "9a382afc-d119-44db-83c2-98d8057d7666"}}
:dependencies [[org.clojure/clojure "1.10.1"]
:dependencies [[com.google.guava/guava "31.1-jre"]
[org.clojure/clojure "1.10.1"]
[com.datomic/dev-local "1.0.243"]
[com.datomic/datomic-pro "0.9.5783"
:exclusions [com.google.guava/guava
@@ -43,7 +44,7 @@
[digest "1.4.10"]
[clj-http "3.11.0" :exclusions [org.apache.httpcomponents/httpclient org.apache.httpcomponents/httpcore]]
[clj-pdf "2.2.33"]
[org.clojure/core.async "1.3.610"]
[org.clojure/core.async "1.5.648"]
[fogus/ring-edn "0.3.0"]
[buddy/buddy-auth "2.2.0"
:exclusions [com.fasterxml.jackson.dataformat/jackson-dataformat-cbor
@@ -128,7 +129,7 @@
:profiles {
:dev
{:resource-paths ["resources" "target"]
:dependencies [[binaryage/devtools "1.0.2"]
:dependencies [#_[binaryage/devteols "1.0.2"]
[postgresql/postgresql "9.3-1102.jdbc41"]
[org.clojure/java.jdbc "0.7.11"]
[etaoin "0.4.1"]
@@ -171,7 +172,7 @@
org.eclipse.jetty.websocket/websocket-server
org.eclipse.jetty.websocket/websocket-servlet
args4j]]]}
:provided {:dependencies [[org.clojure/clojurescript "1.10.773"
:provided {:dependencies [[org.clojure/clojurescript "1.11.4"
:exclusions [com.google.code.findbugs/jsr305
com.fasterxml.jackson.core/jackson-core]]
[reagent "1.0.0" :exclusions [cljsjs/react cljsjs/react-dom cljsjs/react-dom-server] ]

3
src/clj/auto_ap/datomic.clj
View File

@@ -685,7 +685,7 @@
(into (map (fn [i] (if is-component?
[:db/retractEntity i]
[:db/retract e a i ])) retract-ids))
(into (map (fn [i] [:db/add e a (:db/id i)]) new-rels))
(into (map (fn [i] [:db/add e a i]) new-rels))
(into (mapcat (fn [i] (upsert-entity db i)) vs)))))
(defn reset-scalars [db e a vs]
@@ -703,6 +703,7 @@
(into (map (fn [i] [:db/add e a i]) new)))))
;; TODO unit test this
(defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id")
(let [e (:db/id entity)

50
src/clj/auto_ap/ezcater/core.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.ezcater.core
(:require
[auto-ap.datomic :refer [conn]]
[datomic.api :as d]
[auto-ap.datomic :refer [conn random-tempid]]
[datomic.client.api :as dc]
[clj-http.client :as client]
[venia.core :as v]
[clojure.string :as str]
@@ -35,40 +35,40 @@
:subscriptions))
(defn get-integrations []
(map first (d/q '[:find (pull ?i [:ezcater-integration/api-key
(map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid
:db/id
:ezcater-integration/integration-status [:db/id]])
:in $
:where [?i :ezcater-integration/api-key]]
(d/db conn))))
(dc/db conn))))
(defn mark-integration-status [integration integration-status]
@(d/transact conn
[{:db/id (:db/id integration)
:ezcater-integration/integration-status (assoc integration-status
:db/id (or (-> integration :ezcater-integration/integration-status :db/id)
#db/id [:db.part/user]))}]))
(dc/transact conn
{:tx-data [{:db/id (:db/id integration)
:ezcater-integration/integration-status (assoc integration-status
:db/id (or (-> integration :ezcater-integration/integration-status :db/id)
(random-tempid)))}]}))
(defn upsert-caterers
([integration]
@(d/transact
(dc/transact
conn
(for [caterer (get-caterers integration)]
{:db/id (:db/id integration)
:ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")")
:ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer)))
:ezcater-caterer/uuid (:uuid caterer)}]}))))
{:tx-data (for [caterer (get-caterers integration)]
{:db/id (:db/id integration)
:ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")")
:ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer)))
:ezcater-caterer/uuid (:uuid caterer)}]})})))
(defn upsert-used-subscriptions
([integration]
(let [extant (get-subscriptions integration)
to-ensure (set (map first (d/q '[:find ?cu
to-ensure (set (map first (dc/q '[:find ?cu
:in $
:where [_ :client/ezcater-locations ?el]
[?el :ezcater-location/caterer ?c]
[?c :ezcater-caterer/uuid ?cu]]
(d/db conn))))
(dc/db conn))))
to-create (set/difference
to-ensure
(set (map :parentId extant)))]
@@ -114,7 +114,7 @@
:integration-status/message (.getMessage e)}))))))
(defn get-caterer [caterer-uuid]
(d/pull (d/db conn)
(dc/pull (dc/db conn)
'[:ezcater-caterer/name
{:ezcater-integration/_caterers [:ezcater-integration/api-key]}
{:ezcater-location/_caterer [:ezcater-location/location
@@ -270,12 +270,12 @@
(alog/info
::try-import-order
:json json)
@(d/transact conn (filter identity
[(some-> json
(lookup-order)
(order->sales-order)
(update :sales-order/date coerce/to-date)
(update-in [:sales-order/charges 0 :charge/date] coerce/to-date))])))
(dc/transact conn {:tx-data (filter identity
[(some-> json
(lookup-order)
(order->sales-order)
(update :sales-order/date coerce/to-date)
(update-in [:sales-order/charges 0 :charge/date] coerce/to-date))])}))
(defn upsert-recent []
(upsert-ezcater)
@@ -346,6 +346,6 @@
updated-order))]
(alog/info :found-orders-to-update
:orders orders-to-update)
@(d/transact conn orders-to-update)))
@(dc/transact conn {:tx-data orders-to-update})))

6
src/clj/auto_ap/graphql/ezcater.clj
View File

@@ -4,16 +4,16 @@
[auto-ap.graphql.utils :refer [assert-admin cleanse-query]]
[auto-ap.graphql.vendors :refer [partial-match-first]]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]
[datomic.client.api :as dc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
(defn search [context args _]
(assert-admin (:id context))
(let [search-query (cleanse-query (:query args))
data (d/q '[:find ?n ?i ?s
data (dc/q '[:find ?n ?i ?s
:in $ ?q
:where [(fulltext $ :ezcater-caterer/search-terms ?q) [[?i ?n _ ?s]]]]
(d/db conn)
(dc/db conn)
search-query)]
(->> data
(sort-by (comp - last))

13
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -1,8 +1,6 @@
(ns auto-ap.graphql.invoices
(:require
[auto-ap.datomic
:refer [conn remove-nils upsert-entity]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.datomic :refer [conn random-tempid remove-nils upsert-entity]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices]
[auto-ap.datomic.vendors :as d-vendors]
@@ -92,10 +90,10 @@
(throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number :validation-error (str "Invoice '" invoice_number "' already exists.")}))))
(defn expense-account->entity [{:keys [id account_id amount location]}]
(remove-nils #:invoice-expense-account {:amount amount
:db/id id
:account account_id
:location location}))
#:invoice-expense-account {:amount amount
:db/id (or id (random-tempid))
:account account_id
:location location})
(defn add-invoice-transaction [{:keys [total invoice_number scheduled_payment client_id vendor_id date due expense_accounts]}]
(let [vendor (d-vendors/get-by-id vendor_id)
@@ -109,6 +107,7 @@
due))
_ (when-not (:db/id account)
(throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))]
`(upsert-entity ~{:db/id "invoice"
:invoice/invoice-number invoice_number
:invoice/client client_id

5
src/clj/auto_ap/import/yodlee2.clj
View File

@@ -10,9 +10,8 @@
[clojure.string :as str]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.client.api :as dc]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
[digest :as di]
[unilog.context :as lc]))
#_{:clj-kondo/ignore [:unresolved-var]}
(defn yodlee->transaction [transaction use-date-instead-of-post-date?]

4
src/clj/auto_ap/ledger.clj
View File

@@ -227,8 +227,8 @@
:end end})))))
(defn entities-since-last-ledger-entry []
(count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5)))
:end (coerce/to-date (time/now))})))
(count (dc/tx-range conn {:start (c/to-date (t/plus (t/now) (t/days -5)))
:end (c/to-date (t/now))})))
(defn mismatched-transactions
([]

2
src/clj/auto_ap/square/core.clj
View File

@@ -496,7 +496,7 @@
{:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}]}))
(random-tempid)))}]}))
(defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients)

44
src/clj/auto_ap/square/core2.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.square.core2
(:require
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.datomic :refer [conn remove-nils random-tempid]]
[auto-ap.time :as atime]
[clj-http.client :as client]
[clj-time.coerce :as coerce]
@@ -13,7 +13,7 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[cemerick.url :as url]
[datomic.api :as d]
[datomic.client.api :as dc]
[slingshot.slingshot :refer [try+]]
[unilog.context :as lc]))
@@ -399,7 +399,7 @@
(lc/with-context {:source "Square loading"}
(doseq [x (partition-all 20 (daily-results client location start end))]
(log/info "Loading " (count x))
@(d/transact conn x)))))
(dc/transact conn {:tx-data x})))))
(defn upsert-settlements
([client]
@@ -411,7 +411,7 @@
:client (:client/code client)}
(doseq [x (partition-all 20 (daily-settlements client location))]
(log/info "Loading expected deposit" (count x))
@(d/transact conn x))
(dc/transact conn {:tx-data x}))
(log/info "Done loading settlements"))))
(defn upsert-refunds
@@ -425,7 +425,7 @@
:location (:square-location/client-location client)}
(doseq [x (partition-all 20 (refunds client location))]
(log/info "Loading refund" (count x))
@(d/transact conn x))
(dc/transact conn {:tx-data x}))
(log/info "Done loading refunds"))))
(def square-read [:db/id
@@ -435,7 +435,7 @@
(defn get-square-clients
([]
(d/q '[:find [(pull ?c [:db/id
(dc/q '[:find [(pull ?c [:db/id
:client/square-integration-status
:client/code
:client/square-auth-token
@@ -443,9 +443,9 @@
:in $
:where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"]]
(d/db conn)))
(dc/db conn)))
([ & codes]
(d/q '[:find [(pull ?c [:db/id
(dc/q '[:find [(pull ?c [:db/id
:client/code
:client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...]
@@ -453,7 +453,7 @@
:where [?c :client/square-auth-token]
[?c :client/feature-flags "new-square"]
[?c :client/code ?code]]
(d/db conn)
(dc/db conn)
codes)))
(defn upsert-locations
@@ -466,31 +466,29 @@
[(:square-location/square-id sl)
(:db/id sl)])
(:client/square-locations client)))]
(->> (for [square-location (client-locations client)]
{:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user))
:client/_square-locations (:db/id client)
:square-location/name (:name square-location)
:square-location/square-id (:id square-location)})
(d/transact conn)
deref))))
(dc/transact conn {:tx-data (for [square-location (client-locations client)]
{:db/id (or (square-id->id (:id square-location)) (random-tempid))
:client/_square-locations (:db/id client)
:square-location/name (:name square-location)
:square-location/square-id (:id square-location)})}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn reset []
(->>
(d/query {:query {:find ['?e]
(dc/q {:query {:find ['?e]
:in ['$]
:where ['(or [?e :sales-order/date]
[?e :expected-deposit/date])]}
:args [(d/db conn)]})
:args [(dc/db conn)]})
(map first)
(map (fn [x] [:db/retractEntity x]))))
(defn mark-integration-status [client integration-status]
@(d/transact conn
[{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}]))
(dc/transact conn
{:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id)
(random-tempid)))}]}))
(defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients)

1
src/clj/user.clj
View File

@@ -444,6 +444,7 @@
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
(defn restart-db []
(require 'datomic.dev-local)
(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})
(mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))
(start-db))

2
things-to-search-for.txt
View File

@@ -5,3 +5,5 @@ New way of automating ledger entries
Make sure no history on ledger
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
automatically rebuild search indexes
ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity