adds detailed tracing for ledger loads.
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[com.brunobonacci.mulog :as mu]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]
|
||||
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]])
|
||||
@@ -291,175 +292,186 @@
|
||||
|
||||
(defn import-ledger [context args _]
|
||||
(assert-admin (:id context))
|
||||
(lc/with-context {:area "import ledger"}
|
||||
(let [all-vendors (->> (d/q '[:find [?e ...]
|
||||
:in $
|
||||
:where [?e :vendor/name]]
|
||||
(d/db conn))
|
||||
(d/pull-many (d/db conn) d-vendors/default-read)
|
||||
(by :vendor/name))
|
||||
all-clients (by :client/code (d-clients/get-all ))
|
||||
all-client-bank-accounts (reduce
|
||||
(fn [acc client]
|
||||
(assoc acc (:client/code client)
|
||||
(set (->> (:client/bank-accounts client)
|
||||
(map :bank-account/code)
|
||||
))))
|
||||
{}
|
||||
(d-clients/get-all))
|
||||
all-client-locations (reduce
|
||||
(fn [acc client]
|
||||
(assoc acc (:client/code client)
|
||||
(-> (set (:client/locations client))
|
||||
(conj "HQ")
|
||||
(conj "A"))))
|
||||
{}
|
||||
(d-clients/get-all))
|
||||
new-hidden-vendors (reduce
|
||||
(fn [new-vendors {:keys [vendor_name]}]
|
||||
(if (or (all-vendors vendor_name)
|
||||
(new-vendors vendor_name))
|
||||
new-vendors
|
||||
(assoc new-vendors vendor_name
|
||||
{:vendor/name vendor_name
|
||||
:vendor/hidden true
|
||||
:db/id vendor_name})))
|
||||
{}
|
||||
(:entries args))
|
||||
_ (audit-transact-batch (vec (vals new-hidden-vendors)) (:id context))
|
||||
all-vendors (->> (d/q '[:find [?e ...]
|
||||
:in $
|
||||
:where [?e :vendor/name]]
|
||||
(d/db conn))
|
||||
(d/pull-many (d/db conn) d-vendors/default-read)
|
||||
(by :vendor/name))
|
||||
all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
|
||||
transaction (doall (map
|
||||
(assoc-error (fn [entry]
|
||||
(let [vendor (all-vendors (:vendor_name entry))]
|
||||
(when-not (all-clients (:client_code entry))
|
||||
(throw (ex-info (str "Client '" (:client_code entry )"' not found.") {:status :error}) ))
|
||||
(when-not vendor
|
||||
(throw (ex-info (str "Vendor '" (:vendor_name entry) "' not found.") {:status :error})))
|
||||
(when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
|
||||
(throw (ex-info (str "Date must be MM/dd/yyyy") {:status :error})))
|
||||
(when-let [locked-until (:client/locked-until (all-clients (:client_code entry)))]
|
||||
(when (and (not (t/after? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry))))
|
||||
(coerce/to-date-time locked-until)))
|
||||
(not (t/equal? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry))))
|
||||
(coerce/to-date-time locked-until))))
|
||||
(throw (ex-info (str "Client's data is locked until " locked-until) {:status :error}))))
|
||||
|
||||
(when-not (dollars= (reduce (fnil + 0.0 0.0) 0.0 (map :debit (:line_items entry)))
|
||||
(reduce (fnil + 0.0 0.0) 0.0 (map :credit (:line_items entry))))
|
||||
(throw (ex-info (str "Debits '"
|
||||
(reduce (fnil + 0.0 0.0) 0 (map :debit (:line_items entry)))
|
||||
"' and credits '"
|
||||
(reduce (fnil + 0.0 0.0) 0 (map :credit (:line_items entry)))
|
||||
"' do not add up.")
|
||||
{:status :error})))
|
||||
(when (dollars= (reduce (fnil + 0.0 0.0) 0.0 (map :debit (:line_items entry)))
|
||||
0.0)
|
||||
(throw (ex-info (str "Cannot have ledger entries that total $0.00")
|
||||
{:status :ignored})))
|
||||
(assoc entry
|
||||
:status :success
|
||||
:tx
|
||||
(remove-nils
|
||||
{:journal-entry/source (:source entry)
|
||||
:journal-entry/client [:client/code (:client_code entry)]
|
||||
:journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
|
||||
:journal-entry/external-id (:external_id entry)
|
||||
:journal-entry/vendor (:db/id (all-vendors (:vendor_name entry)))
|
||||
:journal-entry/amount (:amount entry)
|
||||
:journal-entry/note (:note entry)
|
||||
:journal-entry/cleared-against (:cleared_against entry)
|
||||
(let [all-vendors (mu/trace ::get-all-vendors
|
||||
[]
|
||||
(->> (d/q '[:find [?e ...]
|
||||
:in $
|
||||
:where [?e :vendor/name]]
|
||||
(d/db conn))
|
||||
(d/pull-many (d/db conn) d-vendors/default-read)
|
||||
(by :vendor/name)))
|
||||
all-clients (by :client/code (d-clients/get-all ))
|
||||
all-client-bank-accounts (reduce
|
||||
(fn [acc client]
|
||||
(assoc acc (:client/code client)
|
||||
(set (->> (:client/bank-accounts client)
|
||||
(map :bank-account/code)
|
||||
))))
|
||||
{}
|
||||
(d-clients/get-all))
|
||||
all-client-locations (reduce
|
||||
(fn [acc client]
|
||||
(assoc acc (:client/code client)
|
||||
(-> (set (:client/locations client))
|
||||
(conj "HQ")
|
||||
(conj "A"))))
|
||||
{}
|
||||
(d-clients/get-all))
|
||||
new-hidden-vendors (reduce
|
||||
(fn [new-vendors {:keys [vendor_name]}]
|
||||
(if (or (all-vendors vendor_name)
|
||||
(new-vendors vendor_name))
|
||||
new-vendors
|
||||
(assoc new-vendors vendor_name
|
||||
{:vendor/name vendor_name
|
||||
:vendor/hidden true
|
||||
:db/id vendor_name})))
|
||||
{}
|
||||
(:entries args))
|
||||
_ (mu/trace ::upsert-new-vendors
|
||||
[]
|
||||
(audit-transact-batch (vec (vals new-hidden-vendors)) (:id context)))
|
||||
all-vendors (->> (d/q '[:find [?e ...]
|
||||
:in $
|
||||
:where [?e :vendor/name]]
|
||||
(d/db conn))
|
||||
(d/pull-many (d/db conn) d-vendors/default-read)
|
||||
(by :vendor/name))
|
||||
all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
|
||||
transaction (mu/trace ::build-transaction
|
||||
[:count (count (:entries args))]
|
||||
(doall (map
|
||||
(assoc-error (fn [entry]
|
||||
(let [vendor (all-vendors (:vendor_name entry))]
|
||||
(when-not (all-clients (:client_code entry))
|
||||
(throw (ex-info (str "Client '" (:client_code entry )"' not found.") {:status :error}) ))
|
||||
(when-not vendor
|
||||
(throw (ex-info (str "Vendor '" (:vendor_name entry) "' not found.") {:status :error})))
|
||||
(when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
|
||||
(throw (ex-info (str "Date must be MM/dd/yyyy") {:status :error})))
|
||||
(when-let [locked-until (:client/locked-until (all-clients (:client_code entry)))]
|
||||
(when (and (not (t/after? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry))))
|
||||
(coerce/to-date-time locked-until)))
|
||||
(not (t/equal? (coerce/to-date-time (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry))))
|
||||
(coerce/to-date-time locked-until))))
|
||||
(throw (ex-info (str "Client's data is locked until " locked-until) {:status :error}))))
|
||||
|
||||
(when-not (dollars= (reduce (fnil + 0.0 0.0) 0.0 (map :debit (:line_items entry)))
|
||||
(reduce (fnil + 0.0 0.0) 0.0 (map :credit (:line_items entry))))
|
||||
(throw (ex-info (str "Debits '"
|
||||
(reduce (fnil + 0.0 0.0) 0 (map :debit (:line_items entry)))
|
||||
"' and credits '"
|
||||
(reduce (fnil + 0.0 0.0) 0 (map :credit (:line_items entry)))
|
||||
"' do not add up.")
|
||||
{:status :error})))
|
||||
(when (dollars= (reduce (fnil + 0.0 0.0) 0.0 (map :debit (:line_items entry)))
|
||||
0.0)
|
||||
(throw (ex-info (str "Cannot have ledger entries that total $0.00")
|
||||
{:status :ignored})))
|
||||
(assoc entry
|
||||
:status :success
|
||||
:tx
|
||||
(remove-nils
|
||||
{:journal-entry/source (:source entry)
|
||||
:journal-entry/client [:client/code (:client_code entry)]
|
||||
:journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
|
||||
:journal-entry/external-id (:external_id entry)
|
||||
:journal-entry/vendor (:db/id (all-vendors (:vendor_name entry)))
|
||||
:journal-entry/amount (:amount entry)
|
||||
:journal-entry/note (:note entry)
|
||||
:journal-entry/cleared-against (:cleared_against entry)
|
||||
|
||||
:journal-entry/line-items
|
||||
(mapv (fn [ea]
|
||||
(let [debit (or (:debit ea) 0.0)
|
||||
credit (or (:credit ea) 0.0)]
|
||||
(when (and (not (get
|
||||
(get all-client-locations (:client_code entry))
|
||||
(:location ea)))
|
||||
(not= "A" (:location ea)))
|
||||
(throw (ex-info (str "Location '" (:location ea) "' not found.")
|
||||
{:status :error})))
|
||||
(when (and (<= debit 0.0)
|
||||
(<= credit 0.0))
|
||||
(throw (ex-info (str "Line item amount " (or debit credit) " must be greater than 0.")
|
||||
{:status :error})))
|
||||
(when (and (not (all-accounts (:account_identifier ea)))
|
||||
(not (get
|
||||
(get all-client-bank-accounts (:client_code entry))
|
||||
(:account_identifier ea))))
|
||||
(throw (ex-info (str "Account '" (:account_identifier ea) "' not found.")
|
||||
{:status :error})))
|
||||
(let [matching-account (when (re-matches #"^[0-9]+$" (:account_identifier ea))
|
||||
(a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))]
|
||||
(when (and matching-account
|
||||
(:account/location matching-account)
|
||||
(not= (:account/location matching-account)
|
||||
(:location ea)))
|
||||
(throw (ex-info (str "Account '"
|
||||
(:account/numeric-code matching-account)
|
||||
"' requires location '"
|
||||
(:account/location matching-account)
|
||||
"' but got '"
|
||||
(:location ea)
|
||||
"'")
|
||||
{:status :error})))
|
||||
:journal-entry/line-items
|
||||
(mapv (fn [ea]
|
||||
(let [debit (or (:debit ea) 0.0)
|
||||
credit (or (:credit ea) 0.0)]
|
||||
(when (and (not (get
|
||||
(get all-client-locations (:client_code entry))
|
||||
(:location ea)))
|
||||
(not= "A" (:location ea)))
|
||||
(throw (ex-info (str "Location '" (:location ea) "' not found.")
|
||||
{:status :error})))
|
||||
(when (and (<= debit 0.0)
|
||||
(<= credit 0.0))
|
||||
(throw (ex-info (str "Line item amount " (or debit credit) " must be greater than 0.")
|
||||
{:status :error})))
|
||||
(when (and (not (all-accounts (:account_identifier ea)))
|
||||
(not (get
|
||||
(get all-client-bank-accounts (:client_code entry))
|
||||
(:account_identifier ea))))
|
||||
(throw (ex-info (str "Account '" (:account_identifier ea) "' not found.")
|
||||
{:status :error})))
|
||||
(let [matching-account (when (re-matches #"^[0-9]+$" (:account_identifier ea))
|
||||
(a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))]
|
||||
(when (and matching-account
|
||||
(:account/location matching-account)
|
||||
(not= (:account/location matching-account)
|
||||
(:location ea)))
|
||||
(throw (ex-info (str "Account '"
|
||||
(:account/numeric-code matching-account)
|
||||
"' requires location '"
|
||||
(:account/location matching-account)
|
||||
"' but got '"
|
||||
(:location ea)
|
||||
"'")
|
||||
{:status :error})))
|
||||
|
||||
(when (and matching-account
|
||||
(not (:account/location matching-account))
|
||||
(= "A" (:location ea)))
|
||||
(throw (ex-info (str "Account '"
|
||||
(:account/numeric-code matching-account)
|
||||
"' cannot use location '"
|
||||
(:location ea)
|
||||
"'")
|
||||
{:status :error})))
|
||||
(remove-nils (cond-> {:journal-entry-line/location (:location ea)
|
||||
:journal-entry-line/debit (when (> debit 0)
|
||||
debit)
|
||||
:journal-entry-line/credit (when (> credit 0)
|
||||
credit)}
|
||||
matching-account (assoc :journal-entry-line/account (:db/id matching-account))
|
||||
(not matching-account) (assoc :journal-entry-line/account [:bank-account/code (:account_identifier ea)]))))))
|
||||
(:line_items entry))
|
||||
|
||||
:journal-entry/cleared true})))))
|
||||
(:entries args)))
|
||||
errors (filter #(= (:status %) :error) transaction)
|
||||
ignored (filter #(= (:status %) :ignored) transaction)
|
||||
success (filter #(= (:status %) :success) transaction)
|
||||
retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:external_id x)]])
|
||||
success)
|
||||
ignore-retraction (->> ignored
|
||||
(map :external_id )
|
||||
(d/q '[:find [?je ...]
|
||||
:in $ [?ei ...]
|
||||
:where [?je :journal-entry/external-id ?ei]]
|
||||
(d/db conn)
|
||||
)
|
||||
(map (fn [je] [:db/retractEntity je])))]
|
||||
(log/info "manual ledger import has " (count success) " new rows")
|
||||
(log/info errors)
|
||||
|
||||
(when (and matching-account
|
||||
(not (:account/location matching-account))
|
||||
(= "A" (:location ea)))
|
||||
(throw (ex-info (str "Account '"
|
||||
(:account/numeric-code matching-account)
|
||||
"' cannot use location '"
|
||||
(:location ea)
|
||||
"'")
|
||||
{:status :error})))
|
||||
(remove-nils (cond-> {:journal-entry-line/location (:location ea)
|
||||
:journal-entry-line/debit (when (> debit 0)
|
||||
debit)
|
||||
:journal-entry-line/credit (when (> credit 0)
|
||||
credit)}
|
||||
matching-account (assoc :journal-entry-line/account (:db/id matching-account))
|
||||
(not matching-account) (assoc :journal-entry-line/account [:bank-account/code (:account_identifier ea)]))))))
|
||||
(:line_items entry))
|
||||
|
||||
:journal-entry/cleared true})))))
|
||||
(:entries args))))
|
||||
errors (filter #(= (:status %) :error) transaction)
|
||||
ignored (filter #(= (:status %) :ignored) transaction)
|
||||
success (filter #(= (:status %) :success) transaction)
|
||||
retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:external_id x)]])
|
||||
success)
|
||||
ignore-retraction (->> ignored
|
||||
(map :external_id )
|
||||
(d/q '[:find [?je ...]
|
||||
:in $ [?ei ...]
|
||||
:where [?je :journal-entry/external-id ?ei]]
|
||||
(d/db conn)
|
||||
)
|
||||
(map (fn [je] [:db/retractEntity je])))]
|
||||
(log/info "manual ledger import has " (count success) " new rows")
|
||||
(log/info errors)
|
||||
|
||||
|
||||
(audit-transact-batch retraction (:id context))
|
||||
(mu/trace ::retraction-tx
|
||||
[:count (count retraction)]
|
||||
(audit-transact-batch retraction (:id context)))
|
||||
(mu/trace ::ignore-retraction-tx
|
||||
[:count (count ignore-retraction)]
|
||||
(when (seq ignore-retraction)
|
||||
(audit-transact-batch ignore-retraction (:id context)))
|
||||
#_(log/info (map :tx success))
|
||||
(audit-transact-batch (map :tx success) (:id context))
|
||||
|
||||
{:successful (map (fn [x] {:external_id (:external_id x)}) success)
|
||||
:ignored (map (fn [x]
|
||||
{:external_id (:external_id x)})
|
||||
ignored)
|
||||
:existing []
|
||||
:errors (map (fn [x] {:external_id (:external_id x)
|
||||
:error (:error x)}) errors)})))
|
||||
(audit-transact-batch ignore-retraction (:id context))))
|
||||
#_(log/info (map :tx success))
|
||||
(mu/trace ::success-tx
|
||||
[:count (count success)]
|
||||
(audit-transact-batch (map :tx success) (:id context)))
|
||||
|
||||
{:successful (map (fn [x] {:external_id (:external_id x)}) success)
|
||||
:ignored (map (fn [x]
|
||||
{:external_id (:external_id x)})
|
||||
ignored)
|
||||
:existing []
|
||||
:errors (map (fn [x] {:external_id (:external_id x)
|
||||
:error (:error x)}) errors)}))
|
||||
|
||||
(defn build-running-balance
|
||||
([lookup-account all-ledger-entries]
|
||||
|
||||
Reference in New Issue
Block a user