notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

much more background process tracking

Browse Source
This commit is contained in:
Bryce Covert
2022-06-22 10:43:37 -07:00
parent 480066b202
commit 1db8d7a52c
15 changed files with 379 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
src/clj/auto_ap/ledger.clj
View File

@@ -196,47 +196,41 @@
:stop (-> process-txes-worker :running? (reset! false)))
(defn reconcile-ledger []
(lc/with-context {:source "reconcile-ledger"}
(try
(log/info "Attempting to reconcile the ledger")
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :transaction/date]
'[?t :transaction/amount ?amt]
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :transaction/date]
'[?t :transaction/amount ?amt]
'[(not= 0.0 ?amt)]
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
'(not [?t :transaction/approval-status :transaction-approval-status/suppressed])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :invoice/date]
'[?t :invoice/total ?amt]
'[(not= 0.0 ?amt)]
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
'(not [?t :transaction/approval-status :transaction-approval-status/suppressed])
'(not [?t :invoice/status :invoice-status/voided])
'(not [?t :invoice/import-status :import-status/pending])
'(not [?t :invoice/exclude-from-ledger true])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :invoice/date]
'[?t :invoice/total ?amt]
'[(not= 0.0 ?amt)]
'(not [?t :invoice/status :invoice-status/voided])
'(not [?t :invoice/import-status :import-status/pending])
'(not [?t :invoice/exclude-from-ledger true])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
(when (seq repairs)
(log/info (take 3 repairs))
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs))
(log/info "Finished reconciling ledger"))
(catch Exception e
(log/error e)))))
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
(when (seq repairs)
(log/info (take 3 repairs))
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs))))
(mount/defstate reconciliation-frequency :start (* 1000 60 60))
(mount/defstate ledger-reconciliation-worker
:start (scheduler/every reconciliation-frequency reconcile-ledger)
:start (scheduler/every reconciliation-frequency (heartbeat reconcile-ledger "reconcile-ledger"))
:stop (scheduler/stop ledger-reconciliation-worker))
@@ -371,56 +365,52 @@
invoice-accounts)))
(defn touch-broken-ledger []
(lc/with-context {:source "touch-broken-ledger"}
(statsd/event {:title "Reconciling Ledger"
:text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low}
nil)
(try
(log/info "Attempting to fix transactions that are in the ledger but are wrong")
(let [mismatched-ts (mismatched-transactions)]
(if (seq mismatched-ts)
(do
(log/warn (count mismatched-ts) " transactions exist but don't match ledger " (pr-str (take 10 mismatched-ts) ))
(doseq [[m] mismatched-ts]
(touch-transaction m))
(statsd/gauge "data.mismatched_transactions" (count (mismatched-transactions))))
(statsd/gauge "data.mismatched_transactions" 0.0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-ts (unbalanced-transactions)]
(if (seq unbalanced-ts)
(do
(log/warn (count unbalanced-ts) " transactions exist but don't have matching debits/credits (" (pr-str (take 10 unbalanced-ts) ) ")")
(doseq [m unbalanced-ts]
(touch-transaction m))
(statsd/gauge "data.unbalanced_transactions" (count (unbalanced-transactions))))
(statsd/gauge "data.unbalanced_transactions" 0.0)))
(log/info "Finished fixing transactions that are in the ledger but are wrong")
(let [mismatched-is (mismatched-invoices)]
(if (seq mismatched-is)
(do
(log/warn (count mismatched-is) " invoice exist but don't match ledger ")
(doseq [[m] mismatched-is]
(touch-invoice m))
(statsd/gauge "data.mismatched_invoices" (count (mismatched-invoices))))
(statsd/gauge "data.mismatched_invoices" 0.0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-invoices (unbalanced-invoices)]
(if (seq unbalanced-invoices)
(do
(log/warn (count unbalanced-invoices) " invoices exist but don't have matching debits/credits ")
(doseq [m unbalanced-invoices]
(touch-invoice m))
(statsd/gauge "data.unbalanced_invoices" (count (unbalanced-invoices))))
(statsd/gauge "data.unbalanced_invoices" 0.0)))
(statsd/event {:title "Reconciling Ledger"
:text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low}
nil)
(log/info "Attempting to fix transactions that are in the ledger but are wrong")
(let [mismatched-ts (mismatched-transactions)]
(if (seq mismatched-ts)
(do
(log/warn (count mismatched-ts) " transactions exist but don't match ledger " (pr-str (take 10 mismatched-ts) ))
(doseq [[m] mismatched-ts]
(touch-transaction m))
(statsd/gauge "data.mismatched_transactions" (count (mismatched-transactions))))
(statsd/gauge "data.mismatched_transactions" 0.0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-ts (unbalanced-transactions)]
(if (seq unbalanced-ts)
(do
(log/warn (count unbalanced-ts) " transactions exist but don't have matching debits/credits (" (pr-str (take 10 unbalanced-ts) ) ")")
(doseq [m unbalanced-ts]
(touch-transaction m))
(statsd/gauge "data.unbalanced_transactions" (count (unbalanced-transactions))))
(statsd/gauge "data.unbalanced_transactions" 0.0)))
(log/info "Finished fixing transactions that are in the ledger but are wrong")
(let [mismatched-is (mismatched-invoices)]
(if (seq mismatched-is)
(do
(log/warn (count mismatched-is) " invoice exist but don't match ledger ")
(doseq [[m] mismatched-is]
(touch-invoice m))
(statsd/gauge "data.mismatched_invoices" (count (mismatched-invoices))))
(statsd/gauge "data.mismatched_invoices" 0.0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-invoices (unbalanced-invoices)]
(if (seq unbalanced-invoices)
(do
(log/warn (count unbalanced-invoices) " invoices exist but don't have matching debits/credits ")
(doseq [m unbalanced-invoices]
(touch-invoice m))
(statsd/gauge "data.unbalanced_invoices" (count (unbalanced-invoices))))
(statsd/gauge "data.unbalanced_invoices" 0.0)))
(log/info "Finish fixing invoices that are in the ledger but are wrong")
(statsd/event {:title "Finished Reconciling Ledger"
:text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low}
nil)
(catch Exception e
(log/error e)))))
(log/info "Finish fixing invoices that are in the ledger but are wrong")
(statsd/event {:title "Finished Reconciling Ledger"
:text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low}
nil))
(mount/defstate touch-broken-ledger-worker
:start (scheduler/every reconciliation-frequency (heartbeat touch-broken-ledger "touch-broken-ledger"))