diff --git a/iol_ion/src/iol_ion/tx.clj b/iol_ion/src/iol_ion/tx.clj index 2f08346a..7d848327 100644 --- a/iol_ion/src/iol_ion/tx.clj +++ b/iol_ion/src/iol_ion/tx.clj @@ -73,6 +73,13 @@ db [:db/valueType :db/ident] (keys entity))) + + ident->cardinality (by :db/ident (comp :db/ident + :db/cardinality) + (pull-many + db + [:db/cardinality :db/ident] + (keys entity))) ops (->> entity (reduce (fn [ops [a v]] @@ -90,8 +97,14 @@ (and (nil? v) (not (nil? (a extant-entity)))) - (conj ops [:db/retract e a (cond-> (a extant-entity) - (:db/id (a extant-entity)) :db/id)]) + (if (= :db.cardinality/many (ident->cardinality a)) + (into ops (map (fn [v] + [:db/retract e a (cond-> v + (:db/id v) :db/id)]) + (a extant-entity))) + + (conj ops [:db/retract e a (cond-> (a extant-entity) + (:db/id (a extant-entity)) :db/id)])) (nil? v) ops @@ -104,6 +117,10 @@ (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) (into ops (reset-rels db e a v)) + + (= :db.cardinality/many (ident->cardinality a)) + (into ops (reset-scalars db e a v)) + (and (sequential? v) (not= :db.type/ref (ident->value-type a))) (into ops (reset-scalars db e a v)) diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index da7d70cf..c48c288a 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -617,6 +617,7 @@ :count (count results)}) (defn audit-transact-batch [txes id] + (println "transacting batch") (let [batch-id (.toString (java.util.UUID/randomUUID))] (reduce (fn [full-tx batch] @@ -636,7 +637,8 @@ true (update :tempids merge (:tempids tx-result))))) {} - (partition-all 200 txes)))) + (partition-all 200 txes))) + (println "done")) (defn audit-transact [txes id] (dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx" diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index fc483bc2..a8cef96e 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -3,6 +3,7 @@ [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [audit-transact conn]] [auto-ap.datomic.clients :as d-clients] + [iol-ion.tx :refer [upsert-entity]] [auto-ap.graphql.utils :refer [->graphql assert-admin @@ -21,7 +22,8 @@ [unilog.context :as lc] [mount.core :as mount] [yang.scheduler :as scheduler] - [iol-ion.tx :refer [random-tempid]]) + [iol-ion.tx :refer [random-tempid]] + [com.brunobonacci.mulog :as mu]) (:import (java.util UUID) (org.apache.commons.codec.binary Base64))) @@ -99,7 +101,7 @@ :email-contact/description (:description e)}) (:emails edit_client)) - :client/feature-flags (:feature_flags edit_client) + :client/feature-flags (:feature_flags edit_client) :client/ezcater-locations (map (fn [el] {:db/id (or (:id el) (random-tempid)) @@ -112,14 +114,15 @@ (map (fn [lm] {:db/id (or (:id lm ) (random-tempid)) :location-match/location (:location lm) :location-match/matches [(:match lm)]}))) - :client/address {:db/id (or (:id (:address edit_client)) (random-tempid)) - :address/street1 (:street1 (:address edit_client)) - :address/street2 (:street2 (:address edit_client)) - :address/city (:city (:address edit_client)) - :address/state (:state (:address edit_client)) - :address/zip (:zip (:address edit_client))} + :client/address (when (seq (filter identity (vals (:address edit_client)))) + {:db/id (or (:id (:address edit_client)) (random-tempid)) + :address/street1 (:street1 (:address edit_client)) + :address/street2 (:street2 (:address edit_client)) + :address/city (:city (:address edit_client)) + :address/state (:state (:address edit_client)) + :address/zip (:zip (:address edit_client))}) :client/bank-accounts (map (fn [ba] - {:db/id (:id ba) + {:db/id (or (:id ba) (random-tempid)) :bank-account/code (:code ba) :bank-account/bank-name (:bank_name ba) :bank-account/bank-code (:bank_code ba) @@ -145,6 +148,7 @@ (:bank_accounts edit_client)) } + _ (mu/log ::upserting :up updated-entity) _ (assert-no-shared-transaction-sources client-code [`(upsert-entity ~updated-entity)]) _ (log/info "upserting client" updated-entity) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index ce638696..20e1c186 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -588,6 +588,7 @@ account location] (take 3 (:journal-entry-line/client+account+location+date result))))) + (take 5) (drop-while (fn [{[_ _ _ date] :journal-entry-line/client+account+location+date}] (>= (compare date starting-at) 0))) first @@ -595,31 +596,33 @@ 0.0))) (defn get-dirty-entries [{:keys [client account location starting-at]} db ] - (mu/trace ::get-dirty-entries - [] - (let [client (pull-id db client) - account (pull-id db account)] - (into [] - (comp - (mapcat (fn [i] - (dc/index-pull db - {:index :avet - :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date] - :start [:journal-entry-line/client+account+location+date - [client account location starting-at]] - :offset (* i 1000) - :limit 1000} - )) - ) - - (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}] - (and - (= client result-client) - (= account result-account) - (= location result-location)))) - (map (fn [result] - [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))) - (range))))) + (let [client (pull-id db client) + account (pull-id db account)] + (into [] + (comp + (map (fn [i] + (dc/index-pull db + {:index :avet + :selector [:db/id :journal-entry-line/debit :journal-entry-line/credit :journal-entry-line/client+account+location+date] + :start [:journal-entry-line/client+account+location+date + [client account location starting-at]] + :offset (* i 1000) + :limit 1000} + ))) + (take-while identity) + (mapcat identity) + (take-while (fn [{[result-client result-account result-location] :journal-entry-line/client+account+location+date}] + #_(println + [[ client result-client] + [ account result-account] + [ location result-location]]) + (and + (= client result-client) + (= account result-account) + (= location result-location)))) + (map (fn [result] + [(:db/id result) (:journal-entry-line/debit result 0.0) (:journal-entry-line/credit result 0.0) ]))) + (range 100)))) (defn compute-running-balance [account-needing-refresh] (mu/trace ::compute diff --git a/test/clj/auto_ap/integration/graphql/clients.clj b/test/clj/auto_ap/integration/graphql/clients.clj index 644dc42c..9b3d9db4 100644 --- a/test/clj/auto_ap/integration/graphql/clients.clj +++ b/test/clj/auto_ap/integration/graphql/clients.clj @@ -106,7 +106,6 @@ } create-result (sut/edit-client {:id (admin-token)} {:edit_client create-client-request} nil)] - (mu/log ::result :result create-result) (is (some? (-> create-result :id))) (is (some? (-> create-result :bank_accounts first :id))) (is (= (set (keys create-client-request)) (disj (set (keys create-result)) diff --git a/test/clj/auto_ap/integration/graphql/ledger/running_balance.clj b/test/clj/auto_ap/integration/graphql/ledger/running_balance.clj index c5dfaa6d..aa563a8d 100644 --- a/test/clj/auto_ap/integration/graphql/ledger/running_balance.clj +++ b/test/clj/auto_ap/integration/graphql/ledger/running_balance.clj @@ -1,12 +1,11 @@ (ns auto-ap.integration.graphql.ledger.running-balance (:require [auto-ap.datomic :refer [conn pull-attr]] - [auto-ap.graphql.ledger :as sut] - [iol-ion.tx :refer [upsert-entity upsert-ledger]] + [auto-ap.ledger :as sut] + [iol-ion.tx :refer [upsert-ledger]] [auto-ap.integration.util :refer [wrap-setup]] [clojure.test :as t :refer [deftest is testing use-fixtures]] - [datomic.client.api :as d] - [datomic.client.api :as dc])) + [datomic.client.api :as d])) (use-fixtures :each wrap-setup) @@ -23,49 +22,48 @@ line-2-1 line-2-2 line-3-1 - line-3-2]} (:tempids (doto (d/transact conn {:tx-data [{:db/id "test-account-1" - :account/type :account-type/asset} - {:db/id "test-account-2" - :account/type :account-type/equity} - {:db/id "test-client" - :client/code "TEST"} - `(upsert-ledger {:db/id "journal-entry-1" - :journal-entry/external-id "1" - :journal-entry/date #inst "2022-01-01" - :journal-entry/client "test-client" - :journal-entry/line-items [{:db/id "line-1-1" - :journal-entry-line/account "test-account-1" - :journal-entry-line/location "A" - :journal-entry-line/debit 10.0} - {:db/id "line-1-2" - :journal-entry-line/account "test-account-2" - :journal-entry-line/location "A" - :journal-entry-line/credit 10.0}]}) - `(upsert-ledger {:db/id "journal-entry-2" - :journal-entry/date #inst "2022-01-02" - :journal-entry/external-id "2" - :journal-entry/client "test-client" - :journal-entry/line-items [{:db/id "line-2-1" - :journal-entry-line/account "test-account-1" - :journal-entry-line/location "A" - :journal-entry-line/debit 50.0} - {:db/id "line-2-2" - :journal-entry-line/account "test-account-2" - :journal-entry-line/location "A" - :journal-entry-line/credit 50.0}]}) - `(upsert-ledger {:db/id "journal-entry-3" - :journal-entry/date #inst "2022-01-03" - :journal-entry/external-id "3" - :journal-entry/client "test-client" - :journal-entry/line-items [{:db/id "line-3-1" - :journal-entry-line/account "test-account-1" - :journal-entry-line/location "A" - :journal-entry-line/debit 150.0} - {:db/id "line-3-2" - :journal-entry-line/account "test-account-2" - :journal-entry-line/location "A" - :journal-entry-line/credit 150.0}]})]}) - clojure.pprint/pprint))] + line-3-2]} (:tempids (d/transact conn {:tx-data [{:db/id "test-account-1" + :account/type :account-type/asset} + {:db/id "test-account-2" + :account/type :account-type/equity} + {:db/id "test-client" + :client/code "TEST"} + `(upsert-ledger {:db/id "journal-entry-1" + :journal-entry/external-id "1" + :journal-entry/date #inst "2022-01-01" + :journal-entry/client "test-client" + :journal-entry/line-items [{:db/id "line-1-1" + :journal-entry-line/account "test-account-1" + :journal-entry-line/location "A" + :journal-entry-line/debit 10.0} + {:db/id "line-1-2" + :journal-entry-line/account "test-account-2" + :journal-entry-line/location "A" + :journal-entry-line/credit 10.0}]}) + `(upsert-ledger {:db/id "journal-entry-2" + :journal-entry/date #inst "2022-01-02" + :journal-entry/external-id "2" + :journal-entry/client "test-client" + :journal-entry/line-items [{:db/id "line-2-1" + :journal-entry-line/account "test-account-1" + :journal-entry-line/location "A" + :journal-entry-line/debit 50.0} + {:db/id "line-2-2" + :journal-entry-line/account "test-account-2" + :journal-entry-line/location "A" + :journal-entry-line/credit 50.0}]}) + `(upsert-ledger {:db/id "journal-entry-3" + :journal-entry/date #inst "2022-01-03" + :journal-entry/external-id "3" + :journal-entry/client "test-client" + :journal-entry/line-items [{:db/id "line-3-1" + :journal-entry-line/account "test-account-1" + :journal-entry-line/location "A" + :journal-entry-line/debit 150.0} + {:db/id "line-3-2" + :journal-entry-line/account "test-account-2" + :journal-entry-line/location "A" + :journal-entry-line/credit 150.0}]})]}))] (testing "should set running-balance on ledger entries missing them" diff --git a/test/clj/auto_ap/integration/util.clj b/test/clj/auto_ap/integration/util.clj index dca88736..b35f81c7 100644 --- a/test/clj/auto_ap/integration/util.clj +++ b/test/clj/auto_ap/integration/util.clj @@ -5,8 +5,9 @@ (defn wrap-setup [f] + (require 'datomic.dev-local) (with-redefs [auto-ap.datomic/client (dc/client {:server-type :dev-local - :system "dev"})] + :system "test"})] (dc/create-database client {:db-name "test"}) (with-redefs [auto-ap.datomic/conn (dc/connect client {:db-name "test"})] (transact-schema conn) diff --git a/things-to-search-for.txt b/things-to-search-for.txt index b5f80b64..bbba9a05 100644 --- a/things-to-search-for.txt +++ b/things-to-search-for.txt @@ -1,9 +1,10 @@ or-join syntax changed? -regex no longer supported as argument for query. Check vendor datomic for how to do it right. +(probably fixed) regex no longer supported as argument for query. Check vendor datomic for how to do it right. it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes ezcater graphql needs search index too make sure that temporary ids are set on all new things when using upsert-entity Wrap tests around every api call +upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null Fix searching * indexing should happen more regularly, and just look for changes since last time it was run