data cleanup, new metrics for data integrity, preventing bad clients that swap bank accounts.
This commit is contained in:
@@ -39,6 +39,21 @@
|
||||
:canned-acl "public-read")
|
||||
(str "https://integreat-signature-images.s3.amazonaws.com/" signature-id ".jpg")))))
|
||||
|
||||
(defn assert-no-shared-transaction-sources [client-code txes]
|
||||
(let [new-db (:db-after (d/with (d/db conn)
|
||||
txes))]
|
||||
(when (seq (->> (d/q '[:find ?src (count ?ba)
|
||||
:in $ ?c
|
||||
:where [?c :client/bank-accounts ?ba]
|
||||
(or
|
||||
[?ba :bank-account/intuit-bank-account ?src]
|
||||
[?ba :bank-account/plaid-account ?src]
|
||||
[?ba :bank-account/yodlee-account-id ?src])]
|
||||
new-db [:client/code client-code])
|
||||
(filter (fn [[src cnt]]
|
||||
(> cnt 1)))))
|
||||
(throw (ex-info "Cannot reuse yodlee/plaid/intuit account" {:validation-error (str "Cannot reuse yodlee/plaid/intuit account")})))))
|
||||
|
||||
(defn edit-client [context {:keys [edit_client new_bank_accounts] :as args} value]
|
||||
(assert-admin (:id context))
|
||||
(when-not (:id edit_client)
|
||||
@@ -71,10 +86,11 @@
|
||||
(map (fn [ba] [:db/retract (:id ba) :bank-account/plaid-account (:db/id (:bank-account/plaid-account (d/entity (d/db conn) (:id ba))))])))))
|
||||
|
||||
|
||||
client-code (if (str/blank? (:client/code client))
|
||||
(:code edit_client)
|
||||
(:client/code client))
|
||||
transactions (into [(remove-nils {:db/id id
|
||||
:client/code (if (str/blank? (:client/code client))
|
||||
(:code edit_client)
|
||||
(:client/code client))
|
||||
:client/code client-code
|
||||
:client/name (:name edit_client)
|
||||
:client/matches (:matches edit_client)
|
||||
:client/signature-file signature-file
|
||||
@@ -127,6 +143,7 @@
|
||||
)
|
||||
(:forecasted_transactions edit_client))]]
|
||||
reverts)
|
||||
_ (assert-no-shared-transaction-sources client-code transactions)
|
||||
_ (log/info "upserting client" transactions)
|
||||
result (audit-transact transactions (:id context))]
|
||||
(-> result :tempids (get id) (or id) d-clients/get-by-id
|
||||
|
||||
@@ -42,87 +42,89 @@
|
||||
|
||||
(defmethod entity-change->ledger :invoice
|
||||
[db [type id]]
|
||||
(let [entity (d/pull db ['* {:invoice/vendor '[*]
|
||||
:invoice/payment '[*]
|
||||
:invoice/status '[:db/ident]
|
||||
:invoice/import-status '[:db/ident]}] id)
|
||||
credit-invoice? (< (:invoice/total entity) 0.0)]
|
||||
(when-not (or
|
||||
(not (:invoice/total entity))
|
||||
(= true (:invoice/exclude-from-ledger entity))
|
||||
(= :import-status/pending (:db/ident (:invoice/import-status entity)))
|
||||
(= :invoice-status/voided (:db/ident (:invoice/status entity)))
|
||||
(dollars-0? (:invoice/total entity)))
|
||||
(remove-nils
|
||||
{:journal-entry/source "invoice"
|
||||
:journal-entry/client (:db/id (:invoice/client entity))
|
||||
:journal-entry/date (:invoice/date entity)
|
||||
:journal-entry/original-entity (:db/id entity)
|
||||
:journal-entry/vendor (:db/id (:invoice/vendor entity))
|
||||
:journal-entry/amount (Math/abs (:invoice/total entity))
|
||||
(when id
|
||||
(let [entity (d/pull db ['* {:invoice/vendor '[*]
|
||||
:invoice/payment '[*]
|
||||
:invoice/status '[:db/ident]
|
||||
:invoice/import-status '[:db/ident]}] id)
|
||||
credit-invoice? (< (:invoice/total entity) 0.0)]
|
||||
(when-not (or
|
||||
(not (:invoice/total entity))
|
||||
(= true (:invoice/exclude-from-ledger entity))
|
||||
(= :import-status/pending (:db/ident (:invoice/import-status entity)))
|
||||
(= :invoice-status/voided (:db/ident (:invoice/status entity)))
|
||||
(dollars-0? (:invoice/total entity)))
|
||||
(remove-nils
|
||||
{:journal-entry/source "invoice"
|
||||
:journal-entry/client (:db/id (:invoice/client entity))
|
||||
:journal-entry/date (:invoice/date entity)
|
||||
:journal-entry/original-entity (:db/id entity)
|
||||
:journal-entry/vendor (:db/id (:invoice/vendor entity))
|
||||
:journal-entry/amount (Math/abs (:invoice/total entity))
|
||||
|
||||
:journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable
|
||||
:journal-entry-line/location "A"
|
||||
}
|
||||
credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity)))
|
||||
(not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
|
||||
(map (fn [ea]
|
||||
(cond->
|
||||
{:journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
|
||||
:journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
|
||||
:journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable
|
||||
:journal-entry-line/location "A"
|
||||
}
|
||||
credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea)))
|
||||
(not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea)))))
|
||||
(:invoice/expense-accounts entity)))
|
||||
:journal-entry/cleared (and (< (:invoice/outstanding-balance entity) 0.01)
|
||||
(every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity))
|
||||
)}))))
|
||||
credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity)))
|
||||
(not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
|
||||
(map (fn [ea]
|
||||
(cond->
|
||||
{:journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
|
||||
:journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
|
||||
}
|
||||
credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea)))
|
||||
(not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea)))))
|
||||
(:invoice/expense-accounts entity)))
|
||||
:journal-entry/cleared (and (< (:invoice/outstanding-balance entity) 0.01)
|
||||
(every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity))
|
||||
)})))))
|
||||
|
||||
(defmethod entity-change->ledger :transaction
|
||||
[db [type id]]
|
||||
(let [entity (d/pull db ['* {:transaction/vendor '[*]
|
||||
:transaction/client '[*]
|
||||
:transaction/approval-status '[*]
|
||||
:transaction/bank-account '[* {:bank-account/type [:db/ident]}]
|
||||
:transaction/accounts '[*
|
||||
{:transaction-account/account [*]}] }] id)
|
||||
bank-account-type (-> entity :transaction/bank-account :bank-account/type :db/ident)
|
||||
decreasing? (< (:transaction/amount entity) 0.0)
|
||||
credit-from-bank? decreasing?
|
||||
debit-from-bank? (not decreasing?)]
|
||||
(when-not (or (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))
|
||||
(= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))
|
||||
(dollars-0? (:transaction/amount entity)))
|
||||
(remove-nils
|
||||
{:journal-entry/source "transaction"
|
||||
:journal-entry/client (:db/id (:transaction/client entity))
|
||||
:journal-entry/date (:transaction/date entity)
|
||||
:journal-entry/original-entity (:db/id entity)
|
||||
:journal-entry/alternate-description (:transaction/description-original entity)
|
||||
:journal-entry/vendor (:db/id (:transaction/vendor entity))
|
||||
:journal-entry/amount (Math/abs (:transaction/amount entity))
|
||||
:journal-entry/cleared-against (:transaction/cleared-against entity)
|
||||
(when id
|
||||
(let [entity (d/pull db ['* {:transaction/vendor '[*]
|
||||
:transaction/client '[*]
|
||||
:transaction/approval-status '[*]
|
||||
:transaction/bank-account '[* {:bank-account/type [:db/ident]}]
|
||||
:transaction/accounts '[*
|
||||
{:transaction-account/account [*]}] }] id)
|
||||
bank-account-type (-> entity :transaction/bank-account :bank-account/type :db/ident)
|
||||
decreasing? (< (:transaction/amount entity) 0.0)
|
||||
credit-from-bank? decreasing?
|
||||
debit-from-bank? (not decreasing?)]
|
||||
(when-not (or (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))
|
||||
(= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))
|
||||
(dollars-0? (:transaction/amount entity)))
|
||||
(remove-nils
|
||||
{:journal-entry/source "transaction"
|
||||
:journal-entry/client (:db/id (:transaction/client entity))
|
||||
:journal-entry/date (:transaction/date entity)
|
||||
:journal-entry/original-entity (:db/id entity)
|
||||
:journal-entry/alternate-description (:transaction/description-original entity)
|
||||
:journal-entry/vendor (:db/id (:transaction/vendor entity))
|
||||
:journal-entry/amount (Math/abs (:transaction/amount entity))
|
||||
:journal-entry/cleared-against (:transaction/cleared-against entity)
|
||||
|
||||
:journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity))
|
||||
:journal-entry-line/location "A"
|
||||
:journal-entry-line/credit (when credit-from-bank?
|
||||
(Math/abs (:transaction/amount entity)))
|
||||
:journal-entry-line/debit (when debit-from-bank?
|
||||
(Math/abs (:transaction/amount entity)))})
|
||||
]
|
||||
(map
|
||||
(fn [a]
|
||||
(remove-nils{:journal-entry-line/account (:db/id (:transaction-account/account a))
|
||||
:journal-entry-line/location (:transaction-account/location a)
|
||||
:journal-entry-line/debit (when credit-from-bank?
|
||||
(Math/abs (:transaction-account/amount a)))
|
||||
:journal-entry-line/credit (when debit-from-bank?
|
||||
(Math/abs (:transaction-account/amount a)))}))
|
||||
(if (seq (:transaction/accounts entity))
|
||||
(:transaction/accounts entity)
|
||||
[{:transaction-account/amount (:transaction/amount entity)}])))
|
||||
|
||||
:journal-entry/cleared true}))))
|
||||
:journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity))
|
||||
:journal-entry-line/location "A"
|
||||
:journal-entry-line/credit (when credit-from-bank?
|
||||
(Math/abs (:transaction/amount entity)))
|
||||
:journal-entry-line/debit (when debit-from-bank?
|
||||
(Math/abs (:transaction/amount entity)))})
|
||||
]
|
||||
(map
|
||||
(fn [a]
|
||||
(remove-nils{:journal-entry-line/account (:db/id (:transaction-account/account a))
|
||||
:journal-entry-line/location (:transaction-account/location a)
|
||||
:journal-entry-line/debit (when credit-from-bank?
|
||||
(Math/abs (:transaction-account/amount a)))
|
||||
:journal-entry-line/credit (when debit-from-bank?
|
||||
(Math/abs (:transaction-account/amount a)))}))
|
||||
(if (seq (:transaction/accounts entity))
|
||||
(:transaction/accounts entity)
|
||||
[{:transaction-account/amount (:transaction/amount entity)}])))
|
||||
|
||||
:journal-entry/cleared true})))))
|
||||
|
||||
(defmethod entity-change->ledger :expected-deposit
|
||||
[db [type id]]
|
||||
@@ -382,30 +384,38 @@
|
||||
(do
|
||||
(log/warn (count mismatched-ts) " transactions exist but don't match ledger ")
|
||||
(doseq [[m] mismatched-ts]
|
||||
(touch-transaction m)))))
|
||||
(touch-transaction m))
|
||||
(statsd/gauge "data.mismatched_transactions" (count (mismatched-transactions))))
|
||||
(statsd/gauge "data.mismatched_transactions" 0)))
|
||||
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
|
||||
(let [unbalanced-ts (unbalanced-transactions)]
|
||||
(if (seq unbalanced-ts)
|
||||
(do
|
||||
(log/warn (count unbalanced-ts) " transactions exist but don't have matching debits/credits ")
|
||||
(doseq [m unbalanced-ts]
|
||||
(touch-transaction m)))))
|
||||
(touch-transaction m))
|
||||
(statsd/gauge "data.unbalanced_transactions" (count (unbalanced-transactions))))
|
||||
(statsd/gauge "data.unbalanced_transactions" 0)))
|
||||
(log/info "Finished fixing transactions that are in the ledger but are wrong")
|
||||
(let [mismatched-is (mismatched-invoices)]
|
||||
(if (seq mismatched-is)
|
||||
(do
|
||||
(log/warn (count mismatched-is) " invoice exist but don't match ledger ")
|
||||
(doseq [[m] mismatched-is]
|
||||
(touch-invoice m)))))
|
||||
(touch-invoice m))
|
||||
(statsd/gauge "data.mismatched_invoices" (count (mismatched-invoices))))
|
||||
(statsd/gauge "data.mismatched_invoices" 0)))
|
||||
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
|
||||
(let [unbalanced-invoices (unbalanced-invoices)]
|
||||
(if (seq unbalanced-invoices)
|
||||
(do
|
||||
(log/warn (count unbalanced-invoices) " invoices exist but don't have matching debits/credits ")
|
||||
(doseq [m unbalanced-invoices]
|
||||
(touch-invoice m)))))
|
||||
(touch-invoice m))
|
||||
(statsd/gauge "data.unbalanced_invoices" (count (unbalanced-invoices))))
|
||||
(statsd/gauge "data.unbalanced_invoices" 0)))
|
||||
|
||||
(log/info "Finished fixing invoices that are in the ledger but are wrong")
|
||||
(log/info "Finish fixing invoices that are in the ledger but are wrong")
|
||||
(statsd/event {:title "Finished Reconciling Ledger"
|
||||
:text "This process looks for unbalance ledger entries, or missing ledger entries"
|
||||
:priority :low}
|
||||
|
||||
@@ -744,3 +744,6 @@
|
||||
(sort)
|
||||
vec))
|
||||
|
||||
(defn init-repl []
|
||||
(set! nrepl.middleware.print/*print-fn* clojure.pprint/pprint))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user