diff --git a/iol_ion/src/iol_ion/tx.clj b/iol_ion/src/iol_ion/tx.clj index a0b26120..9012f73c 100644 --- a/iol_ion/src/iol_ion/tx.clj +++ b/iol_ion/src/iol_ion/tx.clj @@ -259,7 +259,7 @@ :journal-entry/vendor (:db/id (:invoice/vendor entity)) :journal-entry/amount (Math/abs (:invoice/total entity)) - :journal-entry/line-items (into [(cond-> {:db/id (str (:db/id entity) "-" 0) + :journal-entry/line-items (into [(cond-> {:db/id (str raw-invoice-id "-" 0) :journal-entry-line/account :account/accounts-payable :journal-entry-line/location "A" } @@ -267,7 +267,7 @@ (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))] (map-indexed (fn [i ea] (cond-> - {:db/id (str (:db/id entity) "-" (inc i)) + {:db/id (str raw-invoice-id "-" (inc i)) :journal-entry-line/account (:db/id (:invoice-expense-account/account ea)) :journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ") } @@ -291,6 +291,7 @@ journal-entry (invoice->journal-entry (:db-after with-invoice) invoice-id (:db/id invoice))] + (into upserted-entity (if journal-entry (upsert-ledger db journal-entry) @@ -311,3 +312,82 @@ (if existing? [] (upsert-invoice db invoice)))) + + +(defn transaction->journal-entry + ([db transaction-id] + (transaction->journal-entry db transaction-id transaction-id)) + ;; the 3-arity version allows you to pass a potential tempid in instead of the invoice-id, + ;; which would be a temporary value after the transaction + ([db transaction-id raw-transaction-id] + (let [entity (dc/pull db [:transaction/client + :transaction/date + :transaction/description-original + :db/id + :transaction/vendor + :transaction/amount + :transaction/cleared-against + {:transaction/accounts [:transaction-account/account + :transaction-account/location + :transaction-account/amount] + :transaction/approval-status [:db/ident] + :transaction/bank-account [:db/id {:bank-account/type [:db/ident]}]}] + transaction-id) + decreasing? (< (or (:transaction/amount entity) 0.0) 0.0) + credit-from-bank? decreasing? + debit-from-bank? (not decreasing?)] + (when (and (not (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))) + (not (= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))) + (:transaction/amount entity) + (not (< -0.001 (:transaction/amount entity) 0.001))) + (remove-nils + {:journal-entry/source "transaction" + :journal-entry/client (:db/id (:transaction/client entity)) + :journal-entry/date (:transaction/date entity) + :journal-entry/original-entity raw-transaction-id + :journal-entry/alternate-description (:transaction/description-original entity) + :journal-entry/vendor (:db/id (:transaction/vendor entity)) + :journal-entry/amount (Math/abs (:transaction/amount entity)) + :journal-entry/cleared-against (:transaction/cleared-against entity) + + :journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity)) + :db/id (str raw-transaction-id "-" 0) + :journal-entry-line/location "A" + :journal-entry-line/credit (when credit-from-bank? + (Math/abs (:transaction/amount entity))) + :journal-entry-line/debit (when debit-from-bank? + (Math/abs (:transaction/amount entity)))}) + ] + (map-indexed + (fn [i a] + (remove-nils{ + :db/id (str raw-transaction-id "-" (inc i)) + :journal-entry-line/account (:db/id (:transaction-account/account a)) + :journal-entry-line/location (:transaction-account/location a) + :journal-entry-line/debit (when credit-from-bank? + (Math/abs (:transaction-account/amount a))) + :journal-entry-line/credit (when debit-from-bank? + (Math/abs (:transaction-account/amount a)))})) + (if (seq (:transaction/accounts entity)) + (:transaction/accounts entity) + [{:transaction-account/amount (:transaction/amount entity)}]))) + + :journal-entry/cleared true}))))) + +(defn upsert-transaction [db transaction] + ;; because some transactions will reference temp ids, you have to dissoc them, like :transaction/payment + (let [upserted-entity (upsert-entity db (dissoc transaction :transaction/payment)) + with-transaction (try (dc/with db {:tx-data upserted-entity}) + (catch ClassCastException e + (println "Dev local does not support with in tx functions. :(") + (dc/with (dc/with-db @(resolve 'auto-ap.datomic/conn)) {:tx-data upserted-entity}) + )) + transaction-id (or (-> with-transaction :tempids (get (:db/id transaction))) + (:db/id transaction)) + journal-entry (transaction->journal-entry (:db-after with-transaction) + transaction-id + (:db/id transaction))] + (into (upsert-entity db transaction) + (if journal-entry + (upsert-ledger db journal-entry) + [[:db/retractEntity [:journal-entry/original-entity (:db/id transaction)]]])))) diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj index ded584c9..a1678b4f 100644 --- a/src/clj/auto_ap/datomic/invoices.clj +++ b/src/clj/auto_ap/datomic/invoices.clj @@ -6,8 +6,7 @@ apply-sort-3 conn merge-query - pull-many - remove-nils]] + pull-many]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [limited-clients]] @@ -15,7 +14,8 @@ [clj-time.coerce :as coerce] [clj-time.core :as time] [clojure.set :refer [rename-keys]] - [datomic.client.api :as dc])) + [datomic.client.api :as dc] + [iol-ion.tx :refer [random-tempid]])) (def default-read '[* {:invoice/client [:client/name :db/id :client/locations :client/code]} @@ -323,7 +323,8 @@ (next-dom schedule-payment-dom) coerce/to-date) :else nil) - default-expense-account #:invoice-expense-account {:account (d-vendors/account-for-client-id vendor client-id) + default-expense-account #:invoice-expense-account {:db/id (random-tempid) + :account (d-vendors/account-for-client-id vendor client-id) :location (:invoice/location invoice) :amount (:invoice/total invoice)}] (cond-> invoice diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index 52c456f5..b4f37b45 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -17,8 +17,6 @@ assert-power-user attach-tracing-resolvers enum->keyword]] - [auto-ap.ledger - :refer [transact-batch-with-ledger]] [auto-ap.rule-matching :as rm] [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index bf6a2fe8..885c756b 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -1,6 +1,6 @@ (ns auto-ap.graphql.transactions (:require - [auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils]] + [auto-ap.datomic :refer [conn pull-attr pull-many pull-ref remove-nils audit-transact audit-transact-batch]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.invoices :as d-invoices] @@ -19,8 +19,6 @@ ident->enum-f snake->kebab]] [auto-ap.import.transactions :as i-transactions] - [auto-ap.ledger - :refer [transact-batch-with-ledger transact-with-ledger]] [auto-ap.rule-matching :as rm] [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] @@ -28,7 +26,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [datomic.client.api :as dc] - [iol-ion.tx :refer [random-tempid upsert-entity]] + [iol-ion.tx :refer [random-tempid upsert-transaction]] [com.brunobonacci.mulog :as mu])) (def approval-status->graphql (ident->enum-f :transaction/approval-status)) @@ -91,11 +89,11 @@ all-ids-not-locked)] (log/info "Unapproving " (count all-ids) args) - (transact-batch-with-ledger + (audit-transact-batch (->> all-ids (mapv (fn [t] - {:db/id t - :transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")}))) + `(upsert-transaction ~{:db/id t + :transaction/approval-status (enum->keyword (:status args) "transaction-approval-status")})))) (:id context)) {:message (str "Succesfully changed " (count all-ids) " transactions to be " (name (:status args) ) ".")})) @@ -172,12 +170,12 @@ (throw (ex-info err {:validation-error err}) )))) (log/info "Bulk coding " (count all-ids) args) - (transact-batch-with-ledger + (audit-transact-batch (map (fn [i] - `(upsert-entity ~(cond-> i - (:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status")) - (:vendor args) (assoc :transaction/vendor (:vendor args)) - (seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations))))) + `(upsert-transaction ~(cond-> i + (:approval_status args) (assoc :transaction/approval-status (enum->keyword (:approval_status args) "transaction-approval-status")) + (:vendor args) (assoc :transaction/vendor (:vendor args)) + (seq (:accounts args)) (assoc :transaction/accounts (maybe-code-accounts i (:accounts args) locations))))) transactions) (:id context)) {:message (str "Successfully coded " (count all-ids) " transactions.")})) @@ -190,7 +188,7 @@ db (dc/db conn)] (log/info "Deleting " (count all-ids) args) - (transact-batch-with-ledger + (audit-transact-batch (mapcat (fn [i] (let [transaction (dc/pull db [:transaction/payment :transaction/expected-deposit @@ -272,46 +270,36 @@ (when (not= :payment-status/cleared (-> payment :payment/status :db/ident)) (throw (ex-info "Payment can't be undone because it isn't cleared." {:validation-error "Payment can't be undone because it isn't cleared."}))) (if is-autopay-payment? - (transact-with-ledger - (cond-> [{:db/id (:db/id payment) - :payment/status :payment-status/pending} - {:db/id transaction-id - :transaction/approval-status :transaction-approval-status/unapproved} + (audit-transact + (-> [{:db/id (:db/id payment) + :payment/status :payment-status/pending} + `(upsert-transaction + ~{:db/id transaction-id + :transaction/approval-status :transaction-approval-status/unapproved + :transaction/payment nil + :transaction/vendor nil + :transaction/location nil + :transaction/accounts nil}) - [:db/retractEntity (:db/id payment) ] - [:db/retract transaction-id :transaction/payment (:db/id payment)] - [:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))]] + [:db/retractEntity (:db/id payment) ]] - (:transaction/location transaction) - (conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)]) - - (seq (:transaction/accounts transaction)) - (into (map (fn [a] - [:db/retract transaction-id :transaction/accounts (:db/id a)]) - (:transaction/accounts transaction))) - - true - (into (map (fn [[invoice-payment]] - [:db/retractEntity invoice-payment]) - (dc/q {:query {:find ['?ip] - :in ['$ '?p] - :where ['[?ip :invoice-payment/payment ?p]]} - :args [(dc/db conn) (:db/id payment)]} )))) + (into (map (fn [[invoice-payment]] + [:db/retractEntity invoice-payment]) + (dc/q {:query {:find ['?ip] + :in ['$ '?p] + :where ['[?ip :invoice-payment/payment ?p]]} + :args [(dc/db conn) (:db/id payment)]} )))) (:id context)) - (transact-with-ledger - (into (cond-> [{:db/id (:db/id payment) - :payment/status :payment-status/pending} - {:db/id transaction-id - :transaction/approval-status :transaction-approval-status/unapproved} - [:db/retract transaction-id :transaction/payment (:db/id payment)] - [:db/retract transaction-id :transaction/vendor (:db/id (:transaction/vendor transaction))] - ] - (:transaction/location transaction) - (conj [:db/retract transaction-id :transaction/location (:transaction/location transaction)])) - - (map (fn [a] - [:db/retract transaction-id :transaction/accounts (:db/id a)]) - (:transaction/accounts transaction))) + (audit-transact + [{:db/id (:db/id payment) + :payment/status :payment-status/pending} + `(upsert-transaction + ~{:db/id transaction-id + :transaction/approval-status :transaction-approval-status/unapproved + :transaction/payment nil + :transaction/vendor nil + :transaction/location nil + :transaction/accounts nil})] (:id context))) (-> (d-transactions/get-by-id transaction-id) approval-status->graphql @@ -372,14 +360,14 @@ (when missing-locations (throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) ) - (transact-with-ledger [`(upsert-entity ~{:db/id id - :transaction/vendor vendor_id - :transaction/approval-status (some->> approval_status - name - snake->kebab - (keyword "transaction-approval-status")) - :transaction/accounts (map transaction-account->entity accounts) - :transaction/forecast-match forecast_match})] + (audit-transact [`(upsert-transaction ~{:db/id id + :transaction/vendor vendor_id + :transaction/approval-status (some->> approval_status + name + snake->kebab + (keyword "transaction-approval-status")) + :transaction/accounts (map transaction-account->entity accounts) + :transaction/forecast-match forecast_match})] (:id context)) (-> (d-transactions/get-by-id id) approval-status->graphql @@ -398,22 +386,22 @@ (when-not (dollars= (- (:transaction/amount transaction)) (:payment/amount payment)) (throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"}))) - (transact-with-ledger (into + (audit-transact (into [{:db/id (:db/id payment) :payment/status :payment-status/cleared :payment/date (coerce/to-date (first (sort [(:payment/date payment) (:transaction/date transaction)])))} - {:db/id (:db/id transaction) - :transaction/payment (:db/id payment) - :transaction/vendor (:db/id (:payment/vendor payment)) - :transaction/location "A" - :transaction/approval-status :transaction-approval-status/approved - :transaction/accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :transaction-account/location "A" - :transaction-account/amount (Math/abs (:transaction/amount transaction))}]}] - (map (fn [x] [:db/retractEntity (:db/id x)] ) - (:transaction/accounts transaction))) + `(upsert-transaction + ~{:db/id (:db/id transaction) + :transaction/payment (:db/id payment) + :transaction/vendor (:db/id (:payment/vendor payment)) + :transaction/location "A" + :transaction/approval-status :transaction-approval-status/approved + :transaction/accounts [{:db/id (random-tempid) + :transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :transaction-account/location "A" + :transaction-account/amount (Math/abs (:transaction/amount transaction))}]})]) (:id context))) (-> (d-transactions/get-by-id transaction_id) approval-status->graphql @@ -447,7 +435,7 @@ (:db/id (:transaction/bank-account transaction)) (:db/id (:transaction/client transaction)))] (log/info "Adding a new payment" payment-tx) - (transact-with-ledger payment-tx (:id context))) + (audit-transact payment-tx (:id context))) (-> (d-transactions/get-by-id transaction_id) approval-status->graphql @@ -484,8 +472,7 @@ unpaid_invoice_ids) (:db/id (:transaction/bank-account transaction)) (:db/id (:transaction/client transaction)))] - (log/info "Adding a new payment" payment-tx) - (transact-with-ledger payment-tx (:id context))) + (audit-transact payment-tx (:id context))) (-> (d-transactions/get-by-id transaction_id) approval-status->graphql @@ -521,19 +508,15 @@ (when (:transaction/payment transaction) (throw (ex-info "Transaction already associated with a payment" {:validation-error "Transaction already associated with a payment"})))) - (transact-with-ledger (transduce - (map #(into - [(remove-nils (rm/apply-rule {:db/id (:db/id %) - :transaction/amount (:transaction/amount %)} - transaction-rule + (audit-transact (mapv (fn [t] + `(upsert-transaction + ~(remove-nils (rm/apply-rule {:db/id (:db/id t) + :transaction/amount (:transaction/amount t)} + transaction-rule - (or (-> % :transaction/bank-account :bank-account/locations) - (-> % :transaction/client :client/locations))))] - (map (fn [x] [:db/retractEntity (:db/id x)] ) - (:transaction/accounts %)))) - into - [] - transactions) + (or (-> t :transaction/bank-account :bank-account/locations) + (-> t :transaction/client :client/locations)))))) + transactions) (:id context)) ) (transduce diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 51245ed9..112c34d6 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -1,12 +1,12 @@ (ns auto-ap.import.transactions + (:require [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic.accounts :as a] - [iol-ion.tx :refer [upsert-entity]] + [iol-ion.tx :refer [upsert-transaction upsert-invoice]] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] - [auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.rule-matching :as rm] [auto-ap.time :as atime] [auto-ap.utils :refer [dollars=]] @@ -113,17 +113,17 @@ (defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id] (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) (let [payment-id (random-tempid)] - (-> [`(upsert-entity ~(assoc transaction - :transaction/payment payment-id - :transaction/approval-status :transaction-approval-status/approved - :transaction/vendor vendor - :transaction/location "A" - :transaction/accounts - [#:transaction-account - {:db/id (random-tempid) - :account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :location "A" - :amount (Math/abs (:transaction/amount transaction))}]))] + (-> [`(upsert-transaction ~(assoc transaction + :transaction/payment payment-id + :transaction/approval-status :transaction-approval-status/approved + :transaction/vendor vendor + :transaction/location "A" + :transaction/accounts + [#:transaction-account + {:db/id (random-tempid) + :account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :location "A" + :amount (Math/abs (:transaction/amount transaction))}]))] (conj {:payment/bank-account bank-account-id :payment/client client-id @@ -137,9 +137,9 @@ [{:invoice-payment/invoice invoice-id :invoice-payment/payment payment-id :invoice-payment/amount invoice-amount} - {:db/id invoice-id - :invoice/outstanding-balance 0.0 - :invoice/status :invoice-status/paid}]) + `(upsert-invoice ~{:db/id invoice-id + :invoice/outstanding-balance 0.0 + :invoice/status :invoice-status/paid})]) invoice-payments))))) (defn extract-check-number [{:transaction/keys [description-original]}] @@ -255,25 +255,25 @@ #_maybe-autopay-invoices #(maybe-code % apply-rules valid-locations) identity)] - [(-> transaction - (assoc :transaction/client client-id) - (assoc :transaction/bank-account bank-account-id) - (assoc :transaction/approval-status :transaction-approval-status/unapproved) - maybe-assoc-check-number - code-fn - remove-nils)])) + (-> transaction + (assoc :transaction/client client-id) + (assoc :transaction/bank-account bank-account-id) + (assoc :transaction/approval-status :transaction-approval-status/unapproved) + maybe-assoc-check-number + code-fn + remove-nils))) (defn get-existing [bank-account] (log/info "looking up bank account data for" bank-account) (into {} (dc/q {:query {:find ['?tid '?as2] - :in ['$ '?ba] - :where ['[?e :transaction/bank-account ?ba] - '[?e :transaction/id ?tid] - '[?e :transaction/approval-status ?as] - '[?as :db/ident ?as2]]} - :args [(dc/db conn) bank-account]}))) + :in ['$ '?ba] + :where ['[?e :transaction/bank-account ?ba] + '[?e :transaction/id ?tid] + '[?e :transaction/approval-status ?as] + '[?as :db/ident ?as2]]} + :args [(dc/db conn) bank-account]}))) (defprotocol ImportBatch (import-transaction! [this transaction]) @@ -319,7 +319,7 @@ :error :import-batch/error :not-ready :import-batch/not-ready) inc)) (when (= :import action) - (transact-with-ledger (transaction->txs transaction bank-account rule-applying-function) + (audit-transact [`(upsert-transaction ~(transaction->txs transaction bank-account rule-applying-function))] {:user/name user :user/role ":admin"})))) diff --git a/src/clj/auto_ap/jobs/sysco.clj b/src/clj/auto_ap/jobs/sysco.clj index 1e110975..ace57821 100644 --- a/src/clj/auto_ap/jobs/sysco.clj +++ b/src/clj/auto_ap/jobs/sysco.clj @@ -3,10 +3,10 @@ [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [conn]] [auto-ap.jobs.core :refer [execute]] + [auto-ap.datomic :refer [audit-transact]] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.invoices :refer [code-invoice]] [iol-ion.tx :refer [propose-invoice]] - [auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.parse :as parse] [auto-ap.time :as t] [clj-time.coerce :as coerce] @@ -145,11 +145,10 @@ (s3/copy-object {:source-bucket-name (:data-bucket env) :destination-bucket-name (:data-bucket env) :source-key k - :destination-key (doto (str "sysco/error/" - (.getName (io/file k))) - println)})) + :destination-key (str "sysco/error/" + (.getName (io/file k)))})) []))))) -result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]) +result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})]) (doseq [k keys] (mark-key k)))) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 18cddd0a..bef28bb6 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -459,45 +459,6 @@ :priority :low} nil)) -(defn transact-with-ledger [transaction id] - (let [db (dc/db conn) - tx (audit-transact transaction id) - affected-entities (->> (:tx-data tx) - (map (fn [x] - {:e (:e x) - :a (:a x) - :v (:v x) - :added (:added x)})) - (group-by :e) - (mapcat #(datums->impacted-entity db %)) - (set)) - ledger-txs (->> affected-entities - (map #(entity-change->ledger (:db-after tx) %)) - (filter seq) - (map (fn [l] - `(upsert-ledger ~l))))] - (when (seq ledger-txs) - (audit-transact ledger-txs id)) - tx)) - -(defn transact-batch-with-ledger [txes id] - (let [batch-id (.toString (java.util.UUID/randomUUID))] - (reduce - (fn [full-tx batch] - (let [batch (conj (vec batch) {:db/id "datomic.tx" - :audit/batch batch-id}) - _ (log/info "transacting batch " batch-id " " (count batch)) - tx-result (transact-with-ledger batch id)] - - (cond-> full-tx - (:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result))) - (not (:tx-data full-tx)) (assoc :tx-data (vec (:tx-data tx-result))) - (not (:db-before full-tx)) (assoc :db-before (:db-before tx-result)) - true (assoc :db-after (:db-after tx-result)) - true (update :tempids merge (:tempids tx-result))))) - - {} - (partition-all 50 txes)))) (defn build-account-lookup [client-id] (let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 6cc2892c..bbea9b46 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -20,7 +20,7 @@ [config.core :refer [env]] [datomic.client.api :as dc] [digest] - [iol-ion.tx :refer [propose-invoice]] + [iol-ion.tx :refer [propose-invoice random-tempid]] [ring.middleware.json :refer [wrap-json-response]] [unilog.context :as lc]) (:import @@ -150,7 +150,8 @@ matching-client) text full-text))] - #:invoice {:invoice/client matching-client + #:invoice {:db/id (random-tempid) + :invoice/client matching-client :invoice/client-identifier (or account-number customer-identifier) :invoice/vendor (:db/id matching-vendor) :invoice/source-url source-url @@ -175,29 +176,6 @@ invoice) -(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}] - (try - (->> (dc/q - (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] - :in ['$ '?invoice-number '?vendor '?client] - :where '[[?e :invoice/invoice-number ?invoice-number] - [?e :invoice/vendor ?vendor] - [?e :invoice/client ?client] - [?e :invoice/outstanding-balance ?outstanding-balance] - [?e :invoice/status ?status] - [?e :invoice/import-status ?import-status] - [?import-status :db/ident ?import-status2]]} - :args [(dc/db conn) invoice-number vendor client]})) - first - boolean) - (catch Exception e - (throw (ex-info (str "Failed to find potential matching invoice with" - " invoice " invoice-number - " vendor " vendor - " client " client - ". " - (.toString e)) - {:invoice-number invoice-number}))))) (defn invoice-rows->transaction [rows user] (->> rows diff --git a/src/clj/auto_ap/rule_matching.clj b/src/clj/auto_ap/rule_matching.clj index a548ad1f..00863499 100644 --- a/src/clj/auto_ap/rule_matching.clj +++ b/src/clj/auto_ap/rule_matching.clj @@ -1,4 +1,5 @@ -(ns auto-ap.rule-matching) +(ns auto-ap.rule-matching + (:require [iol-ion.tx :refer [random-tempid]])) (defn ->pattern [x] (. java.util.regex.Pattern (compile x java.util.regex.Pattern/CASE_INSENSITIVE))) @@ -110,11 +111,13 @@ (->> valid-locations (map (fn [cents location] - {:transaction-account/account (:db/id (:transaction-rule-account/account tra)) + {:db/id (random-tempid) + :transaction-account/account (:db/id (:transaction-rule-account/account tra)) :transaction-account/amount (* 0.01 cents) :transaction-account/location location}) (spread-cents cents-to-distribute (count valid-locations)))) - [(cond-> {:transaction-account/account (:db/id (:transaction-rule-account/account tra)) + [(cond-> {:db/id (random-tempid) + :transaction-account/account (:db/id (:transaction-rule-account/account tra)) :transaction-account/amount (* 0.01 cents-to-distribute)} (:transaction-rule-account/location tra) (assoc :transaction-account/location (:transaction-rule-account/location tra)))]))) (filter (comp seq :transaction-rule-account/account) (:transaction-rule/accounts rule)))) diff --git a/src/clj/user.clj b/src/clj/user.clj index 72182446..e81868bf 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -2,7 +2,7 @@ (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [conn pull-attr random-tempid]] - [auto-ap.ledger :as l :refer [transact-with-ledger]] + [auto-ap.ledger :as l ] [auto-ap.server] [auto-ap.square.core :as square] [auto-ap.square.core2 :as square2] @@ -68,48 +68,6 @@ -#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(defn mark-until-date [client end] - (doseq [p (->> - (dc/q {:query {:find '[?e] - :in '[$ ?client ?end ] - :where [ - '[?e :invoice/client ?c] - '[?c :client/code ?client] - '[?e :invoice/date ?d ] - '[(<= ?d ?end) ]]} - :args [(dc/db conn) - client - (c/to-date end)]}) - (mapv first) - (mapv (fn [i] - {:db/id i - :invoice/exclude-from-ledger true})) - (partition-all 100))] - - (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) - (println "process 100")) - - (doseq [p (->> - (dc/q {:query {:find '[?e] - :in '[$ ?client ?end ] - :where [ - '[?e :transaction/client ?c] - '[?c :client/code ?client] - '[?e :transaction/date ?d ] - '[(<= ?d ?end) ]]} - :args [(dc/db conn) - client - (c/to-date end)]}) - (mapv first) - (mapv (fn [i] - {:db/id i - :transaction/approval-status :transaction-approval-status/excluded})) - (partition-all 100))] - - (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) - (println "process 100"))) - #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-accounts [conn] (let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) @@ -443,6 +401,9 @@ (mu/start-publisher! {:type :dev}) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))) +(defn start-search [] + (mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer}))) + (defn restart-db [] #_(require 'datomic.dev-local) #_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) diff --git a/test/clj/auto_ap/import/transactions_test.clj b/test/clj/auto_ap/import/transactions_test.clj index 392cbd06..e41504a1 100644 --- a/test/clj/auto_ap/import/transactions_test.clj +++ b/test/clj/auto_ap/import/transactions_test.clj @@ -3,6 +3,7 @@ [auto-ap.datomic :refer [conn]] [auto-ap.import.transactions :as sut] [auto-ap.integration.util :refer [wrap-setup]] + [iol-ion.tx :refer [upsert-transaction]] [datomic.client.api :as dc] [clj-time.coerce :as coerce] [clojure.test :as t] @@ -83,14 +84,13 @@ :client/code "TEST" :client/locations ["Z" "E"] :client/bank-accounts ["bank-account-id"]}]})) - _ (println bank-account-id client-id) result (sut/transaction->txs base-transaction (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) noop-rule)] - (t/is (= [(assoc base-transaction + (t/is (= (assoc base-transaction :transaction/approval-status :transaction-approval-status/unapproved :transaction/bank-account bank-account-id - :transaction/client client-id)] + :transaction/client client-id) result)))) (t/testing "Should match an uncleared check" @@ -109,7 +109,7 @@ (dc/transact conn) :tempids)] - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/description-original "CHECK 10001" :transaction/amount -30.0) (dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id) @@ -121,7 +121,7 @@ (t/testing "Should match a check that matches on amount if check number does not match" - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/description-original "CHECK 12301" :transaction/amount -30.0) (dc/pull (dc/db conn ) sut/bank-account-pull bank-account-id) @@ -133,7 +133,7 @@ (t/testing "Should not match an already matched check" (dc/transact conn {:tx-data [{:db/id payment-id :payment/status :payment-status/cleared}]}) - (let [[result] (sut/transaction->txs (assoc base-transaction + (let [result (sut/transaction->txs (assoc base-transaction :transaction/description-original "CHECK 10001" :transaction/amount -30.0) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) @@ -162,7 +162,7 @@ (t/testing "Should match within 10 days" - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/date #inst "2021-07-03T00:00:00-08:00" :transaction/amount 100.0) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) @@ -175,7 +175,7 @@ (:transaction/expected-deposit transaction-result))))) (t/testing "Should copy vendor from expected-depoisit" - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/vendor :vendor/ccp-square) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) noop-rule)] @@ -183,7 +183,7 @@ (:transaction/vendor transaction-result))))) (t/testing "Should credit CCP" - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/date #inst "2021-07-03T00:00:00-08:00" :transaction/amount 100.0) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) @@ -194,7 +194,7 @@ (:transaction/accounts transaction-result))))) (t/testing "Should not match old expected deposits" - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/date #inst "2021-07-13" :transaction/amount 100.0) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) @@ -202,7 +202,7 @@ (t/is (not (:transaction/expected-deposit transaction-result))))) (t/testing "Should only match exact." - (let [[transaction-result] (sut/transaction->txs (assoc base-transaction + (let [transaction-result (sut/transaction->txs (assoc base-transaction :transaction/date "2021-07-03" :transaction/amount 100.01) (dc/pull (dc/db conn) sut/bank-account-pull bank-account-id) diff --git a/things-to-search-for.txt b/things-to-search-for.txt index 59c77777..ec2d21c3 100644 --- a/things-to-search-for.txt +++ b/things-to-search-for.txt @@ -1,6 +1,5 @@ * try again to see if we can get upsert-ledger into the same transaction, making it all or nothing * look for all usages of transact-with-ledger -* propose-invoice should use upsert invoice as well it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes upsertledger - matching transaction rule might not assign an account. Other things might not assign accounts. This is an assertion that is commented out. Determine consequence of disabling @@ -33,6 +32,7 @@ Just use a periodic request or event instead of a job for running balance cache, get rid of account-groups move to solr upsertentity Look at how address works on client save. There's agood chance that we should make saving a rel with only a temp id just resolve it to null +new way to do backups