diff --git a/iol_ion/src/iol_ion/query.clj b/iol_ion/src/iol_ion/query.clj index b2c2742b..89162ef1 100644 --- a/iol_ion/src/iol_ion/query.clj +++ b/iol_ion/src/iol_ion/query.clj @@ -57,3 +57,14 @@ (or (= "admin" (:user/role identity)) ((set (map :db/id (:user/clients identity))) (:db/id client)) ((set (map :db/id (:user/clients identity))) client))) + + +(defn ->pattern [x] + (. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE))) + + +(defn dom [^java.util.Date x] + (-> x + (.toInstant) + (.atZone (java.time.ZoneId/of "US/Pacific")) + (.get java.time.temporal.ChronoField/DAY_OF_MONTH))) diff --git a/iol_ion/src/iol_ion/tx.clj b/iol_ion/src/iol_ion/tx.clj index 0f4e3727..b610b9b9 100644 --- a/iol_ion/src/iol_ion/tx.clj +++ b/iol_ion/src/iol_ion/tx.clj @@ -376,7 +376,7 @@ (defn upsert-transaction [db transaction] ;; because some transactions will reference temp ids, you have to dissoc them, like :transaction/payment - (let [upserted-entity (upsert-entity db (dissoc transaction :transaction/payment)) + (let [upserted-entity (upsert-entity db (dissoc transaction :transaction/payment :import-batch/_entry)) with-transaction (try (dc/with db {:tx-data upserted-entity}) (catch ClassCastException e (println "Dev local does not support with in tx functions. :(") diff --git a/resources/datomic/ion-config.edn b/resources/datomic/ion-config.edn index b60df7b4..bff09256 100644 --- a/resources/datomic/ion-config.edn +++ b/resources/datomic/ion-config.edn @@ -17,5 +17,9 @@ iol-ion.query/recent-date iol-ion.query/excel-date iol-ion.query/can-see-client? + + iol-ion.query/dom + iol-ion.query/->pattern + clojure.string/includes? ] :app-name "iol-cloud"} diff --git a/src/clj/auto_ap/datomic/clients.clj b/src/clj/auto_ap/datomic/clients.clj index a12ccffe..b43d29c5 100644 --- a/src/clj/auto_ap/datomic/clients.clj +++ b/src/clj/auto_ap/datomic/clients.clj @@ -65,8 +65,8 @@ (update :bank-account/sort-order (fn [so] (or so i))))) (range) bas))))) (defn get-all [] - (->> (dc/q '[:find (pull ?e ?r) - :in $ ?r + (->> (dc/q '[:find (pull ?e r) + :in $ r :where [?e :client/name]] (dc/db conn) full-read) diff --git a/src/clj/auto_ap/graphql/ledger.clj b/src/clj/auto_ap/graphql/ledger.clj index 3df91768..d367b649 100644 --- a/src/clj/auto_ap/graphql/ledger.clj +++ b/src/clj/auto_ap/graphql/ledger.clj @@ -15,16 +15,20 @@ [auto-ap.ledger :refer [build-account-lookup]] [auto-ap.ledger.reports :as l-reports] [auto-ap.parse.util :as parse] + [auto-ap.pdf.ledger + :refer [print-balance-sheet + print-cash-flows + print-journal-detail-report + print-pnl]] [auto-ap.time :as atime] [auto-ap.utils :refer [by dollars=]] - [auto-ap.pdf.ledger :refer [print-balance-sheet print-pnl print-journal-detail-report print-cash-flows]] [clj-time.coerce :as coerce] [clj-time.core :as t] [clojure.data.csv :as csv] [clojure.tools.logging :as log] [com.brunobonacci.mulog :as mu] [datomic.client.api :as dc] - [iol-ion.tx :refer [upsert-ledger]]) + [iol-ion.tx :refer [random-tempid upsert-ledger]]) (:import (org.apache.commons.codec.binary Base64))) @@ -343,29 +347,37 @@ (map first) (pull-many (dc/db conn) [:db/id :vendor/name]) (by :vendor/name))) - all-clients (mu/trace ::get-all-clients [] - (by :client/code (d-clients/get-all ))) + client-locked-lookup (mu/trace ::get-all-clients [] + (->> (dc/q '[:find ?code ?locked-until + :in $ + :where [?c :client/code ?code] + [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?locked-until]] + (dc/db conn)) + (into {}))) all-client-bank-accounts (mu/trace ::get-all-client-bank-accounts - [] - (reduce - (fn [acc client] - (assoc acc (:client/code client) - (set (->> (:client/bank-accounts client) - (map :bank-account/code) - )))) - {} - (d-clients/get-all))) + [] + (->> (dc/q '[:find ?code ?ba-code + :in $ + :where [?c :client/code ?code] + [?c :client/bank-accounts ?ba] + [?ba :bank-account/code ?ba-code]] + (dc/db conn)) + (reduce + (fn [acc [code ba-code]] + (update acc code (fnil conj #{}) ba-code)) + {}))) all-client-locations (mu/trace ::get-all-client-locations [] - (reduce - (fn [acc client] - (assoc acc (:client/code client) - (-> (set (:client/locations client)) - (conj "HQ") - (conj "A")))) - {} - (d-clients/get-all))) + (->> (dc/q '[:find ?code ?location + :in $ + :where [?c :client/code ?code] + [?c :client/locations ?location]] + (dc/db conn)) + (reduce + (fn [acc [code ba-code]] + (update acc code (fnil conj #{"HQ" "A"}) ba-code)) + {}))) new-hidden-vendors (reduce (fn [new-vendors {:keys [vendor_name]}] @@ -396,13 +408,13 @@ (doall (map (assoc-error (fn [entry] (let [vendor (all-vendors (:vendor_name entry))] - (when-not (all-clients (:client_code entry)) + (when-not (client-locked-lookup (:client_code entry)) (throw (ex-info (str "Client '" (:client_code entry )"' not found.") {:status :error}) )) (when-not vendor (throw (ex-info (str "Vendor '" (:vendor_name entry) "' not found.") {:status :error}))) (when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry)) (throw (ex-info (str "Date must be MM/dd/yyyy") {:status :error}))) - (when-let [locked-until (:client/locked-until (all-clients (:client_code entry)))] + (when-let [locked-until (client-locked-lookup (:client_code entry))] (when (and (not (t/after? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))) (coerce/to-date-time locked-until))) (not (t/equal? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))) @@ -478,7 +490,8 @@ (:location ea) "'") {:status :error}))) - (remove-nils (cond-> {:journal-entry-line/location (:location ea) + (remove-nils (cond-> {:db/id (random-tempid) + :journal-entry-line/location (:location ea) :journal-entry-line/debit (when (> debit 0) debit) :journal-entry-line/credit (when (> credit 0) diff --git a/src/clj/auto_ap/graphql/transaction_rules.clj b/src/clj/auto_ap/graphql/transaction_rules.clj index 8661ab5d..d75c8446 100644 --- a/src/clj/auto_ap/graphql/transaction_rules.clj +++ b/src/clj/auto_ap/graphql/transaction_rules.clj @@ -117,6 +117,10 @@ :in ['$ ] :where []} :args [(dc/db conn)]} + description + (merge-query {:query {:in ['?descr] + :where ['[(iol-ion.query/->pattern ?descr) ?description-regex]]} + :args [description]}) (limited-clients id) (merge-query {:query {:in ['[?xx ...]] @@ -129,10 +133,8 @@ :args [(:db/id bank-account)]}) description - (merge-query {:query {:in ['?description-regex] - :where ['[?e :transaction/description-original ?do] - '[(re-find ?description-regex ?do)]]} - :args [(rm/->pattern description)]}) + (merge-query {:query {:where ['[?e :transaction/description-original ?do] + '[(re-find ?description-regex ?do)]]}}) yodlee-merchant (merge-query {:query {:in ['?yodlee-merchant-id] @@ -154,18 +156,14 @@ dom-lte (merge-query {:query {:in ['?dom-lte] :where ['[?e :transaction/date ?transaction-date] - '[(.toInstant ^java.util.Date ?transaction-date ) ?transaction-instant] - '[(.atZone ^java.time.Instant ?transaction-instant (java.time.ZoneId/of "US/Pacific")) ?transaction-local] - '[(.get ?transaction-local java.time.temporal.ChronoField/DAY_OF_MONTH) ?dom] + '[(iol-ion.query/dom ?transaction-date) ?dom] '[(<= ?dom ?dom-lte)]]} :args [dom-lte]}) dom-gte (merge-query {:query {:in ['?dom-gte] :where ['[?e :transaction/date ?transaction-date] - '[(.toInstant ^java.util.Date ?transaction-date ) ?transaction-instant] - '[(.atZone ^java.time.Instant ?transaction-instant (java.time.ZoneId/of "US/Pacific")) ?transaction-local] - '[(.get ?transaction-local java.time.temporal.ChronoField/DAY_OF_MONTH) ?dom] + '[(iol-ion.query/dom ?transaction-date) ?dom] '[(>= ?dom ?dom-gte)]]} :args [dom-gte]}) diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index 885c756b..70204c54 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -498,7 +498,7 @@ conj [] transaction_ids) - transaction-rule (update (tr/get-by-id transaction_rule_id) :transaction-rule/description #(some-> % rm/->pattern))] + transaction-rule (update (tr/get-by-id transaction_rule_id) :transaction-rule/description #(some-> % iol-ion.query/->pattern))] (doseq [transaction transactions] (when (not (rm/rule-applies? transaction transaction-rule)) (throw (ex-info "Transaction rule does not apply" {:validation-error "Transaction rule does not apply" diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 112c34d6..fad70033 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -1,9 +1,7 @@ (ns auto-ap.import.transactions - (:require [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic.accounts :as a] - [iol-ion.tx :refer [upsert-transaction upsert-invoice]] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] @@ -15,7 +13,8 @@ [clojure.core.cache :as cache] [clojure.tools.logging :as log] [datomic.client.api :as dc] - [digest :as di])) + [digest :as di] + [iol-ion.tx :refer [upsert-invoice upsert-transaction]])) (defn rough-match [client-id bank-account-id amount] (if (and client-id bank-account-id amount) @@ -309,8 +308,7 @@ (:transaction/bank-account transaction)) extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing) (:transaction/bank-account transaction)) - action (categorize-transaction transaction bank-account extant) - transaction (assoc transaction :import-batch/_entry import-id)] + action (categorize-transaction transaction bank-account extant)] (swap! stats #(update % (condp = action :import :import-batch/imported @@ -319,7 +317,9 @@ :error :import-batch/error :not-ready :import-batch/not-ready) inc)) (when (= :import action) - (audit-transact [`(upsert-transaction ~(transaction->txs transaction bank-account rule-applying-function))] + (audit-transact [`(upsert-transaction ~(transaction->txs transaction bank-account rule-applying-function)) + {:db/id import-id + :import-batch/entry (:db/id transaction)}] {:user/name user :user/role ":admin"})))) @@ -356,7 +356,8 @@ (assoc transaction :transaction/id #_{:clj-kondo/ignore [:unresolved-var]} (di/sha-256 raw-id) - :transaction/raw-id raw-id))) + :transaction/raw-id raw-id + :db/id (random-tempid)))) (range) group))))) diff --git a/src/clj/auto_ap/rule_matching.clj b/src/clj/auto_ap/rule_matching.clj index 00863499..c399eaf8 100644 --- a/src/clj/auto_ap/rule_matching.clj +++ b/src/clj/auto_ap/rule_matching.clj @@ -1,8 +1,6 @@ (ns auto-ap.rule-matching - (:require [iol-ion.tx :refer [random-tempid]])) - -(defn ->pattern [x] - (. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE))) + (:require [iol-ion.tx :refer [random-tempid]] + [iol-ion.query :refer [->pattern]])) (defn rule-applies? [transaction {:keys [:transaction-rule/description :transaction-rule/dom-gte :transaction-rule/dom-lte diff --git a/test/clj/auto_ap/integration/graphql/clients.clj b/test/clj/auto_ap/integration/graphql/clients.clj index 9604e9a8..ac08e5ec 100644 --- a/test/clj/auto_ap/integration/graphql/clients.clj +++ b/test/clj/auto_ap/integration/graphql/clients.clj @@ -55,7 +55,7 @@ :square_integration_status :yodlee_provider_accounts :plaid_items :id))) (testing "Should be able to retrieve created client" - (let [[created-client] (sut/get-client {:id (admin-token)} nil nil)] + (let [created-client (sut/get-admin-client {:id (admin-token)} {:id (:id create-result)} nil)] (is (some? (-> created-client :id))) (is (some? (-> created-client :bank_accounts first :id))) (is (= (set (keys create-client-request)) (disj (set (keys created-client)) diff --git a/test/clj/iol_ion/integration/tx.clj b/test/clj/iol_ion/integration/tx.clj index 5db4cb79..c9cbc3e9 100644 --- a/test/clj/iol_ion/integration/tx.clj +++ b/test/clj/iol_ion/integration/tx.clj @@ -2,8 +2,8 @@ (:require [auto-ap.datomic :refer [conn]] [auto-ap.integration.util - :refer [setup-test-data test-invoice wrap-setup apply-tx]] - [clojure.test :as t :refer [deftest is use-fixtures testing]] + :refer [apply-tx setup-test-data test-invoice test-transaction wrap-setup]] + [clojure.test :as t :refer [deftest is testing use-fixtures]] [datomic.client.api :as dc] [iol-ion.tx :as sut])) @@ -82,3 +82,55 @@ (is (= nil (dc/pull db-after journal-pull [:journal-entry/original-entity invoice-id]))))))))) + +(deftest upsert-transaction [] + (testing "should upsert transaction" + (let [{:strs [test-client-id + test-bank-account-id + test-account-id + test-vendor-id + test-transaction-id + test-import-batch-id + ]} (setup-test-data + [(test-transaction :db/id "test-transaction-id" + ) + {:db/id "test-import-batch-id" + :import-batch/date #inst "2022-01-01"}]) + update (sut/upsert-transaction (dc/db conn) {:db/id test-transaction-id + :transaction/id "hello" + :transaction/bank-account test-bank-account-id + :transaction/amount 500.00 + :transaction/client test-client-id + :transaction/date #inst "2022-01-01" + :transaction/vendor test-vendor-id + :transaction/approval-status :transaction-approval-status/approved + :transaction/accounts [ + {:db/id "account" + :transaction-account/account test-account-id + :transaction-account/location "A" + :transaction-account/amount 500.00}]})] + + (is (nil? (:db/id (dc/pull (dc/db conn) journal-pull + [:journal-entry/original-entity test-transaction-id])))) + (let [db-after (apply-tx update)] + (testing "should create journal entry" + (is (= #:journal-entry{:date #inst "2022-01-01T00:00:00.000-00:00", + :original-entity #:db{:id test-transaction-id}, + :client #:db{:id test-client-id}, + :source "transaction", + :cleared true, + :amount 500.0, + :vendor #:db{:id test-vendor-id}, + :line-items + [#:journal-entry-line{:location "A", + :dirty true, + :debit 500.0} + #:journal-entry-line{:account + #:account{:name "Account"}, + :location "A", + :credit 500.0, + :dirty true}]} + (dc/pull db-after journal-pull + [:journal-entry/original-entity test-transaction-id]))))) + + )))