From 616798cebfb44118f0226571f5cacea56df7903b Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 24 Jun 2022 09:28:43 -0700 Subject: [PATCH] making ledger reconciliation be more user friendly. --- src/clj/auto_ap/ledger.clj | 283 ++++++++++++++++++++++--------------- 1 file changed, 171 insertions(+), 112 deletions(-) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index e557e605..2f859465 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -255,122 +255,181 @@ :db/doc "touching invoice to update ledger"} (entity-change->ledger (d/db conn) [:invoice e])])) -(defn mismatched-transactions [] - (let [jel-accounts (reduce - (fn [acc [e lia]] - (update acc e (fnil conj #{} ) lia)) - {} - (d/query {:query {:find ['?e '?lia] - :in ['$] - :where ['[?je :journal-entry/line-items ?li] - '[?je :journal-entry/original-entity ?e] - '[?li :journal-entry-line/account ?lia] - '[?lia :account/name]]} - :args [(d/db auto-ap.datomic/conn)]})) - transaction-accounts (reduce - (fn [acc [e lia]] - (update acc e (fnil conj #{} ) lia)) - {} - (d/query {:query {:find ['?e '?lia] - :in ['$] - :where ['[?e :transaction/accounts ?li] - '(not [?e :transaction/approval-status :transaction-approval-status/excluded]) - '(not [?e :transaction/approval-status :transaction-approval-status/suppressed]) - '[?li :transaction-account/account ?lia] +(defn mismatched-transactions + ([] + (mismatched-transactions (c/to-date (t/minus (t/now) (t/days 7))) + (c/to-date (t/minus (t/now) (t/hours 1)))) ) + ([changed-between-start changed-between-end] + (let [entities-to-consider (d/q '[:find [?t ...] + :in $ ?log ?start ?end + :where + [(tx-ids ?log ?start ?end) [?tx ...]] + [(tx-data ?log ?tx) [[?t]]] + [?t :transaction/date]] + (d/db auto-ap.datomic/conn) + (d/log auto-ap.datomic/conn) + changed-between-start + changed-between-end) + _ (log/info "checking" (count entities-to-consider) "transactions looking for mismatches between" changed-between-start changed-between-end) + jel-accounts (reduce + (fn [acc [e lia]] + (update acc e (fnil conj #{} ) lia)) + {} + (d/q '[:find ?e ?lia + :in $ [?e ...] + :where + [?je :journal-entry/original-entity ?e] + [?je :journal-entry/line-items ?li] + [?li :journal-entry-line/account ?lia] + [?lia :account/name]] + (d/db auto-ap.datomic/conn) + entities-to-consider)) + transaction-accounts (reduce + (fn [acc [e lia]] + (update acc e (fnil conj #{} ) lia)) + {} + (d/q '[:find ?e ?lia + :in $ [?e ...] + :where + [?e :transaction/date ?d] + [?e :transaction/accounts ?li] + (not [?e :transaction/approval-status :transaction-approval-status/excluded]) + (not [?e :transaction/approval-status :transaction-approval-status/suppressed]) + [?li :transaction-account/account ?lia] + [?lia :account/name] + [?e :transaction/amount ?amt] + [(not= ?amt 0.0)]] + (d/db auto-ap.datomic/conn) + entities-to-consider))] + (->> transaction-accounts + (filter + (fn [[e accounts]] (not= accounts (get jel-accounts e)))))))) - '[?lia :account/name] - '[?e :transaction/amount ?amt] - '[(not= ?amt 0.0)]]} - :args [(d/db auto-ap.datomic/conn)]}))] - (->> transaction-accounts - (filter - (fn [[e accounts]] (not= accounts (get jel-accounts e)))) - (filter (fn [[e]] - (let [[tx-date] (d/q '[:find [(max ?d) ...] - :in $ ?t - :where - [?t _ _ ?tx] - [?tx :db/txInstant ?d]] - (d/history (d/db auto-ap.datomic/conn)) - e)] - (t/before? (c/to-date-time tx-date) - (t/minus (t/now) (t/hours 1))))))))) +(defn unbalanced-transactions + ([] (unbalanced-transactions (c/to-date (t/minus (t/now) (t/days 7))) + (c/to-date (t/minus (t/now) (t/hours 1))))) + ([changed-between-start changed-between-end] + (let [entities-to-consider (d/q '[:find [?je ...] + :in $ ?log ?start ?end + :where + [(tx-ids ?log ?start ?end) [?tx ...]] + [(tx-data ?log ?tx) [[?je]]] + [?je :journal-entry/amount] + [?je :journal-entry/original-entity ?i] + [?i :transaction/date]] + (d/db auto-ap.datomic/conn) + (d/log auto-ap.datomic/conn) + changed-between-start + changed-between-end)] + (log/info "checking" (count entities-to-consider) "transaction journal entries looking for mismatches between" changed-between-start changed-between-end) + (->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit) + :with ?jel + :in $ [?je ...] + :where [?je :journal-entry/amount ?a] + [?je :journal-entry/line-items ?jel] + [(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] + [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] + ] + (d/db auto-ap.datomic/conn) + entities-to-consider) + (filter (fn [[_ a d c]] + (or (not (dollars= a d)) + (not (dollars= a c))))) + (map first) + (map (fn [je] + (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) + je)))) + (map :db/id))))) -(defn unbalanced-transactions [] - (->> (d/query {:query {:find ['?je '?a '(sum ?debit) '(sum ?credit)] - :with ['?jel] - :in '[$] - :where ['[?je :journal-entry/amount ?a] - '[?je :journal-entry/line-items ?jel] - '[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] - '[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]] - } - :args [(d/db auto-ap.datomic/conn)]}) - (filter (fn [[_ a d c]] - (or (not (dollars= a d)) - (not (dollars= a c))))) - (map first) - (map (fn [je] - (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) - je)))) - (filter (fn [transaction?] - (:transaction/amount transaction?))) - (map :db/id))) +(defn unbalanced-invoices + ([] (unbalanced-invoices (c/to-date (t/minus (t/now) (t/days 7))) + (c/to-date (t/minus (t/now) (t/hours 1))))) + ([changed-between-start changed-between-end] + (let [entities-to-consider (d/q '[:find [?je ...] + :in $ ?log ?start ?end + :where + [(tx-ids ?log ?start ?end) [?tx ...]] + [(tx-data ?log ?tx) [[?je]]] + [?je :journal-entry/amount] + [?je :journal-entry/original-entity ?i] + [?i :invoice/date]] + (d/db auto-ap.datomic/conn) + (d/log auto-ap.datomic/conn) + changed-between-start + changed-between-end)] + (log/info "checking" (count entities-to-consider) "invoice journal entries looking for mismatches between" changed-between-start changed-between-end) + (->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit) + :with ?jel + :in $ [?je ...] + :where [?je :journal-entry/amount ?a] + [?je :journal-entry/line-items ?jel] + [(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] + [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] + ] + (d/db auto-ap.datomic/conn) + entities-to-consider) + (filter (fn [[_ a d c]] + (or (not (dollars= a d)) + (not (dollars= a c))))) + (map first) + (map (fn [je] + (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) + je)))) + (map :db/id))))) -(defn unbalanced-invoices [] - (->> (d/query {:query {:find ['?je '?a '(sum ?debit) '(sum ?credit)] - :with ['?jel] - :in '[$] - :where ['[?je :journal-entry/amount ?a] - '[?je :journal-entry/line-items ?jel] - '[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] - '[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]] - } - :args [(d/db auto-ap.datomic/conn)]}) - (filter (fn [[_ a d c]] - (or (not (dollars= a d)) - (not (dollars= a c))))) - (map first) - (map (fn [je] - (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) - je)))) - (filter (fn [invoice?] - (:invoice/total invoice?))) - (map :db/id))) +(defn mismatched-invoices + ([] + (mismatched-invoices (c/to-date (t/minus (t/now) (t/days 7))) + (c/to-date (t/minus (t/now) (t/hours 1)))) ) -(defn mismatched-invoices [] - (let [jel-accounts (reduce - (fn [acc [e lia]] - (update acc e (fnil conj #{} ) lia)) - {} - (d/query {:query {:find ['?e '?lia] - :in ['$] - :where ['[?je :journal-entry/line-items ?li] - '[?je :journal-entry/original-entity ?e] - '[?li :journal-entry-line/account ?lia] - '(not [?lia :account/numeric-code 21000]) - '[?lia :account/name]]} - :args [(d/db auto-ap.datomic/conn)]})) - invoice-accounts (reduce - (fn [acc [e lia]] - (update acc e (fnil conj #{} ) lia)) - {} - (d/query {:query {:find ['?e '?lia] - :in ['$] - :where ['[?e :invoice/expense-accounts ?li] - '(not [?e :invoice/total 0.0]) - '[?li :invoice-expense-account/account ?lia] - '[?lia :account/name] - '(not [?lia :account/numeric-code 21000]) - '(not [?e :invoice/status :invoice-status/voided]) - '(not [?e :invoice/exclude-from-ledger true]) - '[?e :invoice/import-status :import-status/imported]]} - :args [(d/db auto-ap.datomic/conn)]})) - ] - (filter - (fn [[e accounts]] - (not= accounts (get jel-accounts e))) - invoice-accounts))) + ([changed-between-start changed-between-end] + (let [entities-to-consider (d/q '[:find [?i ...] + :in $ ?log ?start ?end + :where + [(tx-ids ?log ?start ?end) [?tx ...]] + [(tx-data ?log ?tx) [[?i]]] + [?i :invoice/date]] + (d/db auto-ap.datomic/conn) + (d/log auto-ap.datomic/conn) + changed-between-start + changed-between-end) + _ (log/info (count entities-to-consider) "invoices have changed between" changed-between-start "and" changed-between-end) + jel-accounts (reduce + (fn [acc [e lia]] + (update acc e (fnil conj #{} ) lia)) + {} + (d/q '[:find ?e ?lia + :in $ [?e ...] + :where + [?je :journal-entry/original-entity ?e] + [?je :journal-entry/line-items ?li] + [?li :journal-entry-line/account ?lia] + (not [?lia :account/numeric-code 21000]) + [?lia :account/name]] + (d/db auto-ap.datomic/conn) + entities-to-consider)) + invoice-accounts (reduce + (fn [acc [e lia]] + (update acc e (fnil conj #{} ) lia)) + {} + (d/q '[:find ?e ?lia + :in $ [?e ...] + :where + [?e :invoice/expense-accounts ?li] + (not [?e :invoice/total 0.0]) + [?li :invoice-expense-account/account ?lia] + [?lia :account/name] + (not [?lia :account/numeric-code 21000]) + (not [?e :invoice/status :invoice-status/voided]) + (not [?e :invoice/exclude-from-ledger true]) + [?e :invoice/import-status :import-status/imported]] + (d/db auto-ap.datomic/conn) + entities-to-consider)) + ] + (filter + (fn [[e accounts]] + (not= accounts (get jel-accounts e))) + invoice-accounts)))) (defn touch-broken-ledger [] (statsd/event {:title "Reconciling Ledger"