notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes issues with a neverending search. Also fixes client saving to match normal test.

Browse Source
This commit is contained in:
Bryce Covert
2023-03-24 13:01:33 -07:00
parent 7fd3528d0f
commit 4f94602716
8 changed files with 112 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
iol_ion/src/iol_ion/tx.clj
View File

@@ -73,6 +73,13 @@
db db
[:db/valueType :db/ident] [:db/valueType :db/ident]
(keys entity))) (keys entity)))
ident->cardinality (by :db/ident (comp :db/ident
:db/cardinality)
(pull-many
db
[:db/cardinality :db/ident]
(keys entity)))
ops (->> entity ops (->> entity
(reduce (reduce
(fn [ops [a v]] (fn [ops [a v]]
@@ -90,8 +97,14 @@
(and (nil? v) (and (nil? v)
(not (nil? (a extant-entity)))) (not (nil? (a extant-entity))))
(conj ops [:db/retract e a (cond-> (a extant-entity) (if (= :db.cardinality/many (ident->cardinality a))
(:db/id (a extant-entity)) :db/id)]) (into ops (map (fn [v]
[:db/retract e a (cond-> v
(:db/id v) :db/id)])
(a extant-entity)))
(conj ops [:db/retract e a (cond-> (a extant-entity)
(:db/id (a extant-entity)) :db/id)]))
(nil? v) (nil? v)
ops ops
@@ -104,6 +117,10 @@
(and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v))
(into ops (reset-rels db e a v)) (into ops (reset-rels db e a v))
(= :db.cardinality/many (ident->cardinality a))
(into ops (reset-scalars db e a v))
(and (sequential? v) (not= :db.type/ref (ident->value-type a))) (and (sequential? v) (not= :db.type/ref (ident->value-type a)))
(into ops (reset-scalars db e a v)) (into ops (reset-scalars db e a v))

4
src/clj/auto_ap/datomic.clj
View File

@@ -617,6 +617,7 @@
:count (count results)}) :count (count results)})
(defn audit-transact-batch [txes id] (defn audit-transact-batch [txes id]
(println "transacting batch")
(let [batch-id (.toString (java.util.UUID/randomUUID))] (let [batch-id (.toString (java.util.UUID/randomUUID))]
(reduce (reduce
(fn [full-tx batch] (fn [full-tx batch]
@@ -636,7 +637,8 @@
true (update :tempids merge (:tempids tx-result))))) true (update :tempids merge (:tempids tx-result)))))
{} {}
(partition-all 200 txes)))) (partition-all 200 txes)))
(println "done"))
(defn audit-transact [txes id] (defn audit-transact [txes id]
(dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx" (dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx"

22
src/clj/auto_ap/graphql/clients.clj
View File

@@ -3,6 +3,7 @@
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [audit-transact conn]] [auto-ap.datomic :refer [audit-transact conn]]
[auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.clients :as d-clients]
[iol-ion.tx :refer [upsert-entity]]
[auto-ap.graphql.utils [auto-ap.graphql.utils
:refer [->graphql :refer [->graphql
assert-admin assert-admin
@@ -21,7 +22,8 @@
[unilog.context :as lc] [unilog.context :as lc]
[mount.core :as mount] [mount.core :as mount]
[yang.scheduler :as scheduler] [yang.scheduler :as scheduler]
[iol-ion.tx :refer [random-tempid]]) [iol-ion.tx :refer [random-tempid]]
[com.brunobonacci.mulog :as mu])
(:import (:import
(java.util UUID) (java.util UUID)
(org.apache.commons.codec.binary Base64))) (org.apache.commons.codec.binary Base64)))
@@ -99,7 +101,7 @@
:email-contact/description (:description e)}) :email-contact/description (:description e)})
(:emails edit_client)) (:emails edit_client))
:client/feature-flags (:feature_flags edit_client) :client/feature-flags (:feature_flags edit_client)
:client/ezcater-locations (map :client/ezcater-locations (map
(fn [el] (fn [el]
{:db/id (or (:id el) (random-tempid)) {:db/id (or (:id el) (random-tempid))
@@ -112,14 +114,15 @@
(map (fn [lm] {:db/id (or (:id lm ) (random-tempid)) (map (fn [lm] {:db/id (or (:id lm ) (random-tempid))
:location-match/location (:location lm) :location-match/location (:location lm)
:location-match/matches [(:match lm)]}))) :location-match/matches [(:match lm)]})))
:client/address {:db/id (or (:id (:address edit_client)) (random-tempid)) :client/address (when (seq (filter identity (vals (:address edit_client))))
:address/street1 (:street1 (:address edit_client)) {:db/id (or (:id (:address edit_client)) (random-tempid))
:address/street2 (:street2 (:address edit_client)) :address/street1 (:street1 (:address edit_client))
:address/city (:city (:address edit_client)) :address/street2 (:street2 (:address edit_client))
:address/state (:state (:address edit_client)) :address/city (:city (:address edit_client))
:address/zip (:zip (:address edit_client))} :address/state (:state (:address edit_client))
:address/zip (:zip (:address edit_client))})
:client/bank-accounts (map (fn [ba] :client/bank-accounts (map (fn [ba]
{:db/id (:id ba) {:db/id (or (:id ba) (random-tempid))
:bank-account/code (:code ba) :bank-account/code (:code ba)
:bank-account/bank-name (:bank_name ba) :bank-account/bank-name (:bank_name ba)
:bank-account/bank-code (:bank_code ba) :bank-account/bank-code (:bank_code ba)
@@ -145,6 +148,7 @@
(:bank_accounts edit_client)) (:bank_accounts edit_client))
} }
_ (mu/log ::upserting :up updated-entity)
_ (assert-no-shared-transaction-sources client-code [`(upsert-entity ~updated-entity)]) _ (assert-no-shared-transaction-sources client-code [`(upsert-entity ~updated-entity)])
_ (log/info "upserting client" updated-entity) _ (log/info "upserting client" updated-entity)

53
src/clj/auto_ap/ledger.clj
View File

@@ -588,6 +588,7 @@
account account
location] location]
(take 3 (:journal-entry-line/client+account+location+date result))))) (take 3 (:journal-entry-line/client+account+location+date result)))))
(take 5)
(drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}] (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}]
(>= (compare date starting-at) 0))) (>= (compare date starting-at) 0)))
first first
@@ -595,31 +596,33 @@
0.0))) 0.0)))
(defn get-dirty-entries [{:keys [client account location starting-at]} db ] (defn get-dirty-entries [{:keys [client account location starting-at]} db ]
(mu/trace ::get-dirty-entries (let [client (pull-id db client)
[] account (pull-id db account)]
(let [client (pull-id db client) (into []
account (pull-id db account)] (comp
(into [] (map (fn [i]
(comp (dc/index-pull db
(mapcat (fn [i] {:index :avet
(dc/index-pull db :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date]
{:index :avet :start [:journal-entry-line/client+account+location+date
:selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date] [client account location starting-at]]
:start [:journal-entry-line/client+account+location+date :offset (* i 1000)
[client account location starting-at]] :limit 1000}
:offset (* i 1000) )))
:limit 1000} (take-while identity)
)) (mapcat identity)
) (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}]
#_(println
(take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}] [[ client result-client]
(and [ account result-account]
(= client result-client) [ location result-location]])
(= account result-account) (and
(= location result-location)))) (= client result-client)
(map (fn [result] (= account result-account)
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))) (= location result-location))))
(range))))) (map (fn [result]
[(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ])))
(range 100))))
(defn compute-running-balance [account-needing-refresh] (defn compute-running-balance [account-needing-refresh]
(mu/trace ::compute (mu/trace ::compute

1
test/clj/auto_ap/integration/graphql/clients.clj
View File

@@ -106,7 +106,6 @@
} }
create-result (sut/edit-client {:id (admin-token)} {:edit_client create-client-request} nil)] create-result (sut/edit-client {:id (admin-token)} {:edit_client create-client-request} nil)]
(mu/log ::result :result create-result)
(is (some? (-> create-result :id))) (is (some? (-> create-result :id)))
(is (some? (-> create-result :bank_accounts first :id))) (is (some? (-> create-result :bank_accounts first :id)))
(is (= (set (keys create-client-request)) (disj (set (keys create-result)) (is (= (set (keys create-client-request)) (disj (set (keys create-result))

92
test/clj/auto_ap/integration/graphql/ledger/running_balance.clj
View File

@@ -1,12 +1,11 @@
(ns auto-ap.integration.graphql.ledger.running-balance (ns auto-ap.integration.graphql.ledger.running-balance
(:require (:require
[auto-ap.datomic :refer [conn pull-attr]] [auto-ap.datomic :refer [conn pull-attr]]
[auto-ap.graphql.ledger :as sut] [auto-ap.ledger :as sut]
[iol-ion.tx :refer [upsert-entity upsert-ledger]] [iol-ion.tx :refer [upsert-ledger]]
[auto-ap.integration.util :refer [wrap-setup]] [auto-ap.integration.util :refer [wrap-setup]]
[clojure.test :as t :refer [deftest is testing use-fixtures]] [clojure.test :as t :refer [deftest is testing use-fixtures]]
[datomic.client.api :as d] [datomic.client.api :as d]))
[datomic.client.api :as dc]))
(use-fixtures :each wrap-setup) (use-fixtures :each wrap-setup)
@@ -23,49 +22,48 @@
line-2-1 line-2-1
line-2-2 line-2-2
line-3-1 line-3-1
line-3-2]} (:tempids (doto (d/transact conn {:tx-data [{:db/id "test-account-1" line-3-2]} (:tempids (d/transact conn {:tx-data [{:db/id "test-account-1"
:account/type :account-type/asset} :account/type :account-type/asset}
{:db/id "test-account-2" {:db/id "test-account-2"
:account/type :account-type/equity} :account/type :account-type/equity}
{:db/id "test-client" {:db/id "test-client"
:client/code "TEST"} :client/code "TEST"}
`(upsert-ledger {:db/id "journal-entry-1" `(upsert-ledger {:db/id "journal-entry-1"
:journal-entry/external-id "1" :journal-entry/external-id "1"
:journal-entry/date #inst "2022-01-01" :journal-entry/date #inst "2022-01-01"
:journal-entry/client "test-client" :journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-1-1" :journal-entry/line-items [{:db/id "line-1-1"
:journal-entry-line/account "test-account-1" :journal-entry-line/account "test-account-1"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/debit 10.0} :journal-entry-line/debit 10.0}
{:db/id "line-1-2" {:db/id "line-1-2"
:journal-entry-line/account "test-account-2" :journal-entry-line/account "test-account-2"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/credit 10.0}]}) :journal-entry-line/credit 10.0}]})
`(upsert-ledger {:db/id "journal-entry-2" `(upsert-ledger {:db/id "journal-entry-2"
:journal-entry/date #inst "2022-01-02" :journal-entry/date #inst "2022-01-02"
:journal-entry/external-id "2" :journal-entry/external-id "2"
:journal-entry/client "test-client" :journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-2-1" :journal-entry/line-items [{:db/id "line-2-1"
:journal-entry-line/account "test-account-1" :journal-entry-line/account "test-account-1"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/debit 50.0} :journal-entry-line/debit 50.0}
{:db/id "line-2-2" {:db/id "line-2-2"
:journal-entry-line/account "test-account-2" :journal-entry-line/account "test-account-2"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/credit 50.0}]}) :journal-entry-line/credit 50.0}]})
`(upsert-ledger {:db/id "journal-entry-3" `(upsert-ledger {:db/id "journal-entry-3"
:journal-entry/date #inst "2022-01-03" :journal-entry/date #inst "2022-01-03"
:journal-entry/external-id "3" :journal-entry/external-id "3"
:journal-entry/client "test-client" :journal-entry/client "test-client"
:journal-entry/line-items [{:db/id "line-3-1" :journal-entry/line-items [{:db/id "line-3-1"
:journal-entry-line/account "test-account-1" :journal-entry-line/account "test-account-1"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/debit 150.0} :journal-entry-line/debit 150.0}
{:db/id "line-3-2" {:db/id "line-3-2"
:journal-entry-line/account "test-account-2" :journal-entry-line/account "test-account-2"
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/credit 150.0}]})]}) :journal-entry-line/credit 150.0}]})]}))]
clojure.pprint/pprint))]
(testing "should set running-balance on ledger entries missing them" (testing "should set running-balance on ledger entries missing them"

3
test/clj/auto_ap/integration/util.clj
View File

@@ -5,8 +5,9 @@
(defn wrap-setup (defn wrap-setup
[f] [f]
(require 'datomic.dev-local)
(with-redefs [auto-ap.datomic/client (dc/client {:server-type :dev-local (with-redefs [auto-ap.datomic/client (dc/client {:server-type :dev-local
:system "dev"})] :system "test"})]
(dc/create-database client {:db-name "test"}) (dc/create-database client {:db-name "test"})
(with-redefs [auto-ap.datomic/conn (dc/connect client {:db-name "test"})] (with-redefs [auto-ap.datomic/conn (dc/connect client {:db-name "test"})]
(transact-schema conn) (transact-schema conn)

3
things-to-search-for.txt
View File

@@ -1,9 +1,10 @@
or-join syntax changed? or-join syntax changed?
regex no longer supported as argument for query. Check vendor datomic for how to do it right. (probably fixed) regex no longer supported as argument for query. Check vendor datomic for how to do it right.
it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes
ezcater graphql needs search index too ezcater graphql needs search index too
make sure that temporary ids are set on all new things when using upsert-entity make sure that temporary ids are set on all new things when using upsert-entity
Wrap tests around every api call Wrap tests around every api call
upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null
Fix searching Fix searching
* indexing should happen more regularly, and just look for changes since last time it was run * indexing should happen more regularly, and just look for changes since last time it was run