notid/integreat
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Makes integreat run on datomic cloud

Browse Source
This commit is contained in:
Bryce Covert
2022-08-23 12:13:12 -07:00
parent 58b9dcf272
commit d02fba2b44
58 changed files with 2163 additions and 1257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
src/clj/auto_ap/datomic.clj
View File

@@ -1,25 +1,31 @@
(ns auto-ap.datomic
(:require
[auto-ap.utils :refer [default-pagination-size]]
[auto-ap.utils :refer [default-pagination-size by]]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[datomic.api :as d]
[mount.core :as mount]))
[datomic.client.api :as dc]
[mount.core :as mount])
(:import
(java.util UUID)))
(def uri (:datomic-url env))
(mount/defstate client
:start (dc/client {:server-type :dev-local
:system "dev"})
:stop nil)
(mount/defstate conn
:start (d/connect uri)
:stop (d/release conn))
:start (dc/connect client {:db-name "prod-migration"})
:stop nil)
#_(def uri "datomic:mem://datomic-transactor:4334/invoice")
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn create-database []
#_(defn create-database []
(d/create-database uri))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn drop-database []
#_(defn drop-database []
(d/delete-database uri))
(defn remove-nils [m]
@@ -35,26 +41,6 @@
result
nil)))
(defn replace-nils-with-retract [updated original]
(let [result (reduce-kv
(fn [[m & retractions] k v]
(cond (and (nil? v)
(not (nil? (get original k))))
(into [m] (conj retractions [:db/retract (:db/id original) k (or (:db/id (get original k))
(get original k))]))
(nil? v)
(into [m] retractions)
:else
(into [(assoc m k v)] retractions)))
[{}]
updated)]
(if (seq result)
result
nil)))
(def vendor-schema
[{:db/ident :vendor/original-id
:db/valueType :db.type/long
@@ -101,8 +87,7 @@
:db/isComponent true
:db/cardinality :db.cardinality/one
:db/doc "The vendor's secondary contact"}
{:db/id #db/id[:db.part/db]
:db/ident :vendor/address
{:db/ident :vendor/address
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one
:db/isComponent true
@@ -588,6 +573,20 @@
q
(:sort args)))
(defn add-sorter-fields-2 [q sort-map args]
(log/info "sort-map" (pr-str sort-map))
(reduce
(fn [q {:keys [sort-key]}]
(merge-query q
{:query {:find [(last (last (sort-map
sort-key
(println "Warning, trying to sort by unsupported field" sort-key))))]
:where (sort-map
sort-key
(println "Warning, trying to sort by unsupported field" sort-key))}}))
q
(:sort args)))
(defn apply-sort-3 [args results]
(let [sort-bys (conj (:sort args)
{:sort-key "default" :asc (if (contains? args :default-asc?)
@@ -625,8 +624,7 @@
:audit/user (str (:user/role id) "-" (:user/name id))
:audit/batch batch-id})
_ (log/info "transacting batch " batch-id " " (count batch))
tx-result @(d/transact conn batch)
_ (Thread/sleep 100)]
tx-result (dc/transact conn {:tx-data batch})]
(cond-> full-tx
(:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result)))
@@ -639,5 +637,255 @@
(partition-all 50 txes))))
(defn audit-transact [txes id]
@(d/transact conn (conj txes {:db/id "datomic.tx"
:audit/user (str (:user/role id) "-" (:user/name id))})))
(dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx"
:audit/user (str (:user/role id) "-" (:user/name id))})}))
(defn pull-many [db read ids ]
(->> (dc/q '[:find (pull ?e r)
:in $ [?e ...] r]
db
ids
read)
(map first)))
(defn pull-many-by-id [db read ids ]
(into {}
(map (fn [[e]]
[(:db/id e) e]))
(dc/q '[:find (pull ?e r)
:in $ [?e ...] r]
db
ids
read)))
(defn random-tempid []
(str (UUID/randomUUID)))
(defn pull-attr [db k id]
(get (dc/pull db [k] id) k))
(defn pull-ref [db k id]
(:db/id (pull-attr db k id)))
(declare upsert-entity)
(defn reset-rels [db e a vs]
(assert (every? :db/id vs) (format "In order to reset attribute %s, every value must have :db/id" a))
(let [ids (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
new-id-set (set (map :db/id vs))
retract-ids (filter (complement new-id-set) ids)
{is-component? :db/isComponent} (dc/pull db [:db/isComponent] a)
new-rels (filter (complement (set ids)) (map :db/id vs))]
(-> []
(into (map (fn [i] (if is-component?
[:db/retractEntity i]
[:db/retract e a i ])) retract-ids))
(into (map (fn [i] [:db/add e a (:db/id i)]) new-rels))
(into (mapcat (fn [i] (upsert-entity db i)) vs)))))
(defn reset-scalars [db e a vs]
(let [extant (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
retracts (filter (complement (set vs)) extant)
new (filter (complement (set extant)) vs)]
(-> []
(into (map (fn [i] [:db/retract e a i ]) retracts))
(into (map (fn [i] [:db/add e a i]) new)))))
(defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id")
(let [e (:db/id entity)
is-new? (string? e)
extant-entity (when-not is-new?
(dc/pull db (keys entity) (:db/id entity)))
ident->value-type (by :db/ident (comp :db/ident
:db/valueType)
(pull-many
db
[:db/valueType :db/ident]
(keys entity)))
ops (->> entity
(reduce
(fn [ops [a v]]
(cond
(= :db/id a)
ops
(or (= v (a extant-entity))
(= v (:db/ident (a extant-entity) :nope))
(= v (:db/id (a extant-entity)) :nope))
ops
(and (nil? v)
(not (nil? (a extant-entity))))
(conj ops [:db/retract e a (cond-> (a extant-entity)
(:db/id (a extant-entity)) :db/id)])
(nil? v)
ops
;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq)
(and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v))
(into ops (reset-rels db e a v))
(and (sequential? v) (not= :db.type/ref (ident->value-type a)))
(into ops (reset-scalars db e a v))
(and (map? v)
(= :db.type/ref (ident->value-type a)))
(let [id (or (:db/id v) (random-tempid))]
(-> ops
(conj [:db/add e a id])
(into (upsert-entity db (assoc v :db/id id)))))
:else
(conj ops [:db/add e a v])
))
[]))]
ops))
(defn plus [db e a amount]
[[:db/add e a (-> (dc/pull db [a] e) a (+ amount))]])
(comment
(dc/pull (dc/db conn) '[*] 175921860633685)
(upsert-entity (dc/db conn) {:db/id 175921860633685 :invoice/invoice-number nil :invoice/date #inst "2021-01-01" :invoice/expense-accounts [:reset-rels [{:db/id "new" :invoice-expense-account/amount 1}]]})
(upsert-entity (dc/db conn) {:invoice/client #:db{:id 79164837221949},
:invoice/status #:db{:id 101155069755470, :ident :invoice-status/paid},
:invoice/due #inst "2020-12-23T08:00:00.000-00:00",
:invoice/invoice-number "12648",
:invoice/import-status
:import-status/imported,
:invoice/vendor nil,
:invoice/date #inst "2020-12-16T08:00:00.000-00:00",
:entity/migration-key 17592234924273,
:db/id 175921860633685,
:invoice/outstanding-balance 0.0,
:invoice/expense-accounts
[{:entity/migration-key 17592234924274,
:invoice-expense-account/location nil
:invoice-expense-account/amount 360.0,
:invoice-expense-account/account #:db{:id 92358976759248}}],})
#_(dc/pull (dc/db conn) auto-ap.datomic.clients 79164837221904)
(upsert-entity (dc/db conn) {:client/name "20Twenty - WG Development LLC",
:client/square-locations
[{:db/id 83562883711605,
:entity/migration-key 17592258901782,
:square-location/square-id "L2579ATQ0X1ET",
:square-location/name "20Twenty",
:square-location/client-location "WG"}],
:client/square-auth-token
"EAAAEEr749Ea6AdPTdngsmUPwIM3ETbPwcx3QQl_NS0KWuIL-JNzAg4f3W9DGQhb",
:client/bank-accounts
[{:bank-account/sort-order 2,
:bank-account/include-in-reports true,
:bank-account/number "3467",
:bank-account/code "20TY-WFCC3467",
:bank-account/locations ["WG"],
:entity/migration-key 17592245102834,
:bank-account/current-balance 11160.289999999979,
:bank-account/name "Wells Fargo CC - 3467",
:db/id 83562883732805,
:bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00",
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755504, :ident :bank-account-type/credit},
:bank-account/intuit-bank-account #:db{:id 105553116286744},
:bank-account/integration-status
{:db/id 74766790691480,
:entity/migration-key 17592267080690,
:integration-status/last-updated #inst "2022-08-23T03:47:44.892-00:00",
:integration-status/last-attempt #inst "2022-08-23T03:47:44.892-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}},
:bank-account/bank-name "Wells Fargo"}
{:bank-account/sort-order 0,
:bank-account/include-in-reports true,
:bank-account/numeric-code 11301,
:bank-account/check-number 301,
:bank-account/number "1734742859",
:bank-account/code "20TY-WF2882",
:bank-account/locations ["WG"],
:bank-account/bank-code "11-4288/1210 4285",
:entity/migration-key 17592241193004,
:bank-account/current-balance -47342.54000000085,
:bank-account/name "Wells Fargo Main - 2859",
:db/id 83562883732846,
:bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00",
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755468, :ident :bank-account-type/check},
:bank-account/intuit-bank-account #:db{:id 105553116286745},
:bank-account/routing "121042882",
:bank-account/integration-status
{:db/id 74766790691458,
:entity/migration-key 17592267080255,
:integration-status/last-updated #inst "2022-08-23T03:46:45.879-00:00",
:integration-status/last-attempt #inst "2022-08-23T03:46:45.879-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}},
:bank-account/bank-name "Wells Fargo"}
{:bank-account/sort-order 1,
:bank-account/include-in-reports true,
:bank-account/numeric-code 20101,
:bank-account/yodlee-account-id 27345526,
:bank-account/number "41006",
:bank-account/code "20TY-Amex41006",
:bank-account/locations ["WG"],
:entity/migration-key 17592241193006,
:bank-account/current-balance 9674.069999999963,
:bank-account/name "Amex - 41006",
:db/id 83562883732847,
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755504, :ident :bank-account-type/credit},
:bank-account/bank-name "American Express"}
{:bank-account/sort-order 3,
:bank-account/include-in-reports true,
:bank-account/numeric-code 11101,
:bank-account/code "20TY-0",
:bank-account/locations ["WG"],
:entity/migration-key 17592241193005,
:bank-account/current-balance 0.0,
:bank-account/name "CASH",
:db/id 83562883732848,
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755469, :ident :bank-account-type/cash}}],
:entity/migration-key 17592241193003,
:db/id 79164837221904,
:client/address
{:db/id 105553116285906,
:entity/migration-key 17592250661126,
:address/street1 "1389 Lincoln Ave",
:address/city "San Jose",
:address/state "CA",
:address/zip "95125"},
:client/code "NY",
:client/locations ["WE" "NG"],
:client/square-integration-status
{:db/id 74766790691447,
:entity/migration-key 17592267072653,
:integration-status/last-updated #inst "2022-08-23T13:09:16.082-00:00",
:integration-status/last-attempt #inst "2022-08-23T13:08:47.018-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}}})
)