notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

data cleanup, new metrics for data integrity, preventing bad clients that swap bank accounts.

Browse Source
This commit is contained in:
Bryce Covert
2022-01-08 07:28:14 -08:00
parent 379d8003f1
commit deeadb16b6
8 changed files with 10009 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
curl-format.txt Normal file
View File

@@ -0,0 +1,8 @@
time_namelookup: %{time_namelookup}s\n
time_connect: %{time_connect}s\n
time_appconnect: %{time_appconnect}s\n
time_pretransfer: %{time_pretransfer}s\n
time_redirect: %{time_redirect}s\n
time_starttransfer: %{time_starttransfer}s\n
----------\n
time_total: %{time_total}s\n

93
scratch-sessions/data-cleanup.clj
View File

@@ -24,6 +24,9 @@
(f updated))) (f updated)))
(println "success!")) (println "success!"))
(let [invoices-without-clients (check-fn '[:find ?i (let [invoices-without-clients (check-fn '[:find ?i
:in $ :in $
:where :where
@@ -78,58 +81,62 @@
[?t :invoice-payment/amount] [?t :invoice-payment/amount]
(not [?t :invoice-payment/invoice ])] (not [?t :invoice-payment/invoice ])]
#(= (count %) 0)) #(= (count %) 0))
banks-without-types (check-fn '[:find ?ba
:in $
:where [?ba :bank-account/name]
(not [?ba :bank-account/type])]
#(= (count %) 0))
transaction transaction fix-transaction
(->> [[:db/retractEntity [:client/code "CBC"]]] ]
(into (map (fn [[i]]
[:db/retractEntity i])
(d/q '[:find ?i
:in $
:where (or [?i :invoice/client [:client/code "CBC"]]
[?i :vendor-usage/client [:client/code "CBC"]]
[?i :transaction/client [:client/code "CBC"]]
[?i :payment/client [:client/code "CBC"]]
[?i :transaction-rule/client [:client/code "CBC"]]
[?i :journal-entry/client [:client/code "CBC"]]
[?i :user/clients [:client/code "CBC"]])]
(d/db c))))
(into (map (fn [[i]]
[:db/retractEntity i])
(d/q '[:find ?ip
:in $
:where [?i :invoice/client [:client/code "CBC"]]
[?ip :invoice-payment/invoice ?i]]
(d/db c)))))]
(check-transaction [invoices-without-clients transactions-without-clients (check-transaction [invoices-without-clients transactions-without-clients
account-overide-without-clients vendor-schedule-payment-dom account-overide-without-clients vendor-schedule-payment-dom
payments-without-clients vendor-usage-without-client payments-without-clients vendor-usage-without-client
journal-entries-without-client userless-users journal-entries-without-client userless-users
invoice-payments-without-invoices] invoice-payments-without-invoices banks-without-types]
transaction) transaction)
(auto-ap.datomic/audit-transact-batch transaction {:user/role "admin" )
:user/name "Removing CBC"}))
(clojure.pprint/pprint (get-idents)) (def fix-transaction
(get-schema "invoice-payment") (d/transact c (into [{:db/id "datomic.tx"
:db/doc "Fixing issue with a transaction that was adjusting bank accounts"}]
(->>
(d/pull (d/db c) '[*] 17592250556002)
:transaction/accounts
(group-by #(dissoc % :db/id))
(vals)
(mapcat rest)
(map :db/id)
(map (fn [to-delete]
[:db/retractEntity to-delete]))
))))
(d/q '[:find ?n
:in $
:where [_ ?a 17592186046483]
[?a :db/ident ?n]]
(d/db c))
(->> (d/q '[:find ?ip
:in $
:where [?ip :invoice-payment/amount]
(not [?ip :invoice-payment/invoice]) (comment
#_(not [?ip :invoice-payment/payment])] (defn delete-client []
(d/db c)) (->> [[:db/retractEntity [:client/code "NGGL"]]]
(map (fn [[ip]] (into (map (fn [[i]]
[:db/retractEntity ip])) [:db/retractEntity i])
(into [{:db/id "datomic.tx" :db/doc "Removing payments that were disconnected from invoices"}]) (d/q '[:find ?i
(d/transact c) :in $
deref :where (or [?i :invoice/client [:client/code "NGGL"]]
) [?i :vendor-usage/client [:client/code "NGGL"]]
[?i :transaction/client [:client/code "NGGL"]]
[?i :payment/client [:client/code "NGGL"]]
[?i :transaction-rule/client [:client/code "NGGL"]]
[?i :journal-entry/client [:client/code "NGGL"]]
[?i :user/clients [:client/code "NGGL"]])]
(d/db c))))
(into (map (fn [[i]]
[:db/retractEntity i])
(d/q '[:find ?ip
:in $
:where [?i :invoice/client [:client/code "NGGL"]]
[?ip :invoice-payment/invoice ?i]]
(d/db c)))))))

58
scratch-sessions/finding-invoices-with-partial-cents.clj Normal file
View File

@@ -0,0 +1,58 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(defn bad? [a]
(> (Math/abs (- (Math/round (* 100 a))
(* 100 a)))
0.0001))
(defn improved [t]
(* (Math/round (* 100 (first t))) 0.01))
(def bad-db (d/db auto-ap.datomic/conn))
(def bad-invoices (d/q '[:find ?t (user/improved ?total)
:in $
:where
#_[?t :invoice/client [:client/code "NGAK"]]
[?t :invoice/total ?total]
[(user/bad? ?total)]
]
bad-db ))
(d/transact auto-ap.datomic/conn (->> (d/q '[:find ?ea (user/improved ?total)
:in $ [?t ... ]
:where
#_[?t :invoice/client [:client/code "NGAK"]]
[?t :invoice/total ?total]
[?t :invoice/expense-accounts ?ea]
[?ea :invoice-expense-account/amount ?eam]
]
(d/db auto-ap.datomic/conn) (mapv first bad-invoices))
(mapv (fn [[ea total]]
{:db/id ea
:invoice-expense-account/amount total}))))
(d/transact auto-ap.datomic/conn (->> (d/q '[:find ?t (user/improved ?total)
:in $
:where
#_[?t :invoice/client [:client/code "NGAK"]]
[?t :invoice/total ?total]
[(user/bad? ?total)]
]
(d/db auto-ap.datomic/conn) )
(mapv (fn [[invoice total]]
{:db/id invoice
:invoice/total total}))))
(d/q '[:find ?t ?total
:in $
:where
#_[?t :invoice/client [:client/code "NGAK"]]
[?t :invoice-expense-account/amount ?total]
[(user/bad? ?total)]
]
(d/db auto-ap.datomic/conn) )

248
scratch-sessions/fix-other-checks-different-descriptions Normal file
View File

@@ -0,0 +1,248 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(def potential-dups
(->> (d/q '[:find ?tx ?amount ?date ?ba
:in $
:where
[?ba :bank-account/intuit-bank-account]
[?tx :transaction/bank-account ?ba]
[?tx :transaction/amount ?amount]
[?tx :transaction/date ?date]
(not [?tx :transaction/approval-status :transaction-approval-status/suppressed])]
(d/db auto-ap.datomic/conn))
(group-by (fn [[tx amount date ba]]
[amount date ba]))
(filter (fn [[g txes]]
(> (count txes) 1)))
(vals)))
(def ids-to-suppress
(let [db (d/db auto-ap.datomic/conn)]
(->> potential-dups
(map (fn [tx-ids]
(d/q '[:find (pull ?t [:db/id :transaction/description-original :transaction/raw-id]) ?d
:in $ $$ [?t ...]
:where
[$$ ?t :transaction/id _ ?tx true]
[?tx :db/txInstant ?d]]
db (d/history db)
(map first tx-ids))
))
(filter (fn [tx-sets]
(some (fn [[_ added-date]]
(t/after? (c/to-date-time added-date) (c/to-date-time #inst "2021-12-01")))
tx-sets)))
(mapcat (fn [tx-sets]
(->> tx-sets
(map (fn [[tx-data]]
(let [alternate-description (str/replace (:transaction/description-original tx-data) #" \d+$" "" )
alternate-id (first (d/q '[:find [?alternate ...]
:in $ [?alternate ...] ?t-num ?ad
:where [?alternate :transaction/description-original ?ad]
(not [(= ?alternate ?t-num)])]
db
(map (comp :db/id first) tx-sets)
(:db/id tx-data)
alternate-description))]
[(:db/id tx-data)
alternate-id])))
(filter last)
set)))
(map (fn [pairs]
(d/pull-many
db
[:db/id :transaction/description-original :transaction/bank-account :transaction/amount :transaction/date {:transaction/payment [:db/id :payment/date :payment/amount :payment/check-number]}]
pairs)))
(map (fn [[l r]]
(cond (and (:transaction/payment l)
(:transaction/payment r))
(:db/id r)
(:transaction/payment l)
(:db/id r)
(:transaction/payment r)
(:db/id l)
:else
(:db/id r)))))))
(count ids-to-suppress)
(clojure.pprint/pprint
(mapcat (fn [i]
(let [transaction (d/entity (d/db auto-ap.datomic/conn) i)
payment-id (-> transaction :transaction/payment :db/id)
expected-deposit-id (-> transaction :transaction/expected-deposit :db/id)
transaction-tx {:db/id i
:transaction/approval-status :transaction-approval-status/suppressed}]
(cond->> [{:db/id "datomic.tx"
:db/doc "Fixing duplicates caused by missing number"}
transaction-tx
[:db/retractEntity [:journal-entry/original-entity i]]]
payment-id (into [{:db/id payment-id
:payment/status :payment-status/pending}
[:db/retract (:db/id transaction) :transaction/payment payment-id]])
expected-deposit-id (into [{:db/id expected-deposit-id
:expected-deposit/status :expected-deposit-status/pending}
[:db/retract (:db/id transaction) :transaction/expected-deposit expected-deposit-id]]))))
ids-to-suppress))
(auto-ap.datomic/audit-transact-batch
(mapcat (fn [i]
(let [transaction (d/entity (d/db auto-ap.datomic/conn) i)
payment-id (-> transaction :transaction/payment :db/id)
expected-deposit-id (-> transaction :transaction/expected-deposit :db/id)
transaction-tx {:db/id i
:transaction/approval-status :transaction-approval-status/suppressed}]
(cond->> [{:db/id "datomic.tx"
:db/doc "Fixing duplicates caused by missing number"}
transaction-tx
[:db/retractEntity [:journal-entry/original-entity i]]]
payment-id (into [{:db/id payment-id
:payment/status :payment-status/pending}
[:db/retract (:db/id transaction) :transaction/payment payment-id]])
expected-deposit-id (into [{:db/id expected-deposit-id
:expected-deposit/status :expected-deposit-status/pending}
[:db/retract (:db/id transaction) :transaction/expected-deposit expected-deposit-id]]))))
ids-to-suppress)
{:user/role ":admin"
:user/name "Bryce"})
(take 10 potential-dups)
[[#:transaction{:description-original "Check Paid - 233 - Check Paid 233"} #inst "2021-12-17T18:46:15.419-00:00"]
[#:transaction{:description-original "Check Paid - 233 - Check Paid 233"} #inst "2021-12-17T06:10:34.587-00:00"]
[#:transaction{:description-original "Check Paid - 233 - Check Paid"} #inst "2021-12-18T06:02:24.053-00:00"]
]
[[{:db/id 17592251275669,
:transaction/description-original "CHECK 4136"} #inst "2021-12-18T06:04:25.450-00:00"]
[{:db/id 17592251219175, :transaction/description-original "CHECK 4136 4136"} #inst "2021-12-17T19:01:35.161-00:00"]]
(entity-history 17592251219175);; => [[13194204708070 :transaction/location "A"] [13194204708070 :transaction/bank-account 17592220107992] [13194204708070 :transaction/check-number 4136] [13194204708070 :transaction/status "POSTED"] [13194204708070 :transaction/raw-id "2021-12-15T00:00:00.000-08:00-17592220107992-CHECK 4136 4136--252.0-0-17592219470393"] [13194204708070 :transaction/payment 17592250880737] [13194204708070 :transaction/vendor 17592236287037] [13194204708070 :transaction/accounts 17592251219176] [13194204708070 :transaction/id "d74f6f645e9e31e1f22d65a1eb41fcfa0c5f0a439c903f562f4b146e3945410f"] [13194204708070 :transaction/description-original "CHECK 4136 4136"] [13194204708070 :transaction/amount -252.0] [13194204708070 :transaction/date #inst "2021-12-15T08:00:00.000-00:00"] [13194204708070 :transaction/client 17592219470393] [13194204708070 :transaction/approval-status 17592231963877]]
(entity-history 13194204708070)
(clojure.pprint/pprint (auto-ap.intuit.core/get-transactions "2021-12-01" "2021-12-30" "NGSM - City National - Main - 5955 ---NGSM-CN5955"))
@(d/transact auto-ap.datomic/conn (->> (d/q '[:find ?ba ?d
:in $ $$
:where [?ba :bank-account/name]
(not [?ba :bank-account/type])
[$$ ?ba :bank-account/type ?t _ true]
[?t :db/ident ?d]
]
(d/db auto-ap.datomic/conn)
(d/history (d/db auto-ap.datomic/conn)))
(map (fn [[ba t]]
{:db/id ba
:bank-account/type t}))))
[[{:db/id 17592251275311,
:transaction/raw-id
"2021-12-06T00:00:00.000-08:00-17592232886804-Credit (Any Type) Deposit-256.05-0-17592232886786"}
#inst "2021-12-18T06:03:45.924-00:00"]
[{:db/id 17592251196869,
:transaction/raw-id
"2021-12-06T00:00:00.000-08:00-17592232886804-Credit (Any Type) Deposit 108981169-256.05-0-17592232886786"}
#inst "2021-12-17T18:46:15.419-00:00"]
[{:db/id 17592251196869,
:transaction/raw-id
"2021-12-06T00:00:00.000-08:00-17592232886804-Credit (Any Type) Deposit 108981169-256.05-0-17592232886786"}
#inst "2021-12-17T06:12:30.538-00:00"]]
(clojure.pprint/pprint (entity-history 17592251189853))
(clojure.pprint/pprint (entity-history 17592250137454))
(clojure.pprint/pprint (entity-history 13194204678748))
(clojure.pprint/pprint (entity-history 13194203626349))
(clojure.pprint/pprint (tx-detail 13194204678748))
(def potential-dups
(let [problem-date? (fn [[_ date-added]]
(t/within?
(t/interval
(c/to-date-time #inst "2021-12-16T00:00:00-08:00")
(c/to-date-time #inst "2021-12-18T00:00:00-08:00"))
(c/to-date-time date-added)))]
(->> (d/q '[:find ?tx ?date-added ?check-number ?amount ?date ?ba
:in $ $$
:where
[?ba :bank-account/intuit-bank-account]
[?tx :transaction/bank-account ?ba]
[?tx :transaction/amount ?check-number]
[?tx :transaction/amount ?amount]
[$$ ?tx :transaction/date ?date ?dtx true]
[$$ ?dtx :db/txInstant ?date-added]
(not [?tx :transaction/approval-status :transaction-approval-status/suppressed])]
(d/db auto-ap.datomic/conn)
(d/history (d/db auto-ap.datomic/conn)))
(group-by (fn [[tx date-added check-number amount date ba]]
[check-number amount date ba]))
(filter (fn [[g txes]]
(and
(> (count txes) 1)
(not (every? problem-date? txes))
(some problem-date? txes))))
(vals))))
(def ids-to-suppress (->> potential-dups
(map (fn [dup-set]
(last
(->> dup-set
(sort-by (fn [[_ date-added]]
date-added))
(map (fn [[tx]]
tx))
))))))
(auto-ap.datomic/audit-transact-batch
(mapcat (fn [i]
(let [transaction (d/entity (d/db auto-ap.datomic/conn) i)
payment-id (-> transaction :transaction/payment :db/id)
expected-deposit-id (-> transaction :transaction/expected-deposit :db/id)
transaction-tx {:db/id i
:transaction/approval-status :transaction-approval-status/suppressed}]
(cond->> [{:db/id "datomic.tx"
:db/doc "Fixing duplicates caused by change in check text"}
transaction-tx
[:db/retractEntity [:journal-entry/original-entity i]]]
payment-id (into [{:db/id payment-id
:payment/status :payment-status/pending}
[:db/retract (:db/id transaction) :transaction/payment payment-id]])
expected-deposit-id (into [{:db/id expected-deposit-id
:expected-deposit/status :expected-deposit-status/pending}
[:db/retract (:db/id transaction) :transaction/expected-deposit expected-deposit-id]]))))
ids-to-suppress)
{:user/role ":admin"
:user/name "Bryce"})
(count potential-dups)
(clojure.pprint/pprint potential-dups)
17592250137454
(entity-history 17592248570305)
(entity-history 13194202059200)
(entity-history 17592248569907)
(entity-history 13194202058802)
(doc every?)

9532
scratch-sessions/fixing-duplicated-transactions.clj Normal file
View File

File diff suppressed because it is too large Load Diff

23
src/clj/auto_ap/graphql/clients.clj
View File

@@ -39,6 +39,21 @@
:canned-acl "public-read") :canned-acl "public-read")
(str "https://integreat-signature-images.s3.amazonaws.com/" signature-id ".jpg"))))) (str "https://integreat-signature-images.s3.amazonaws.com/" signature-id ".jpg")))))
(defn assert-no-shared-transaction-sources [client-code txes]
(let [new-db (:db-after (d/with (d/db conn)
txes))]
(when (seq (->> (d/q '[:find ?src (count ?ba)
:in $ ?c
:where [?c :client/bank-accounts ?ba]
(or
[?ba :bank-account/intuit-bank-account ?src]
[?ba :bank-account/plaid-account ?src]
[?ba :bank-account/yodlee-account-id ?src])]
new-db [:client/code client-code])
(filter (fn [[src cnt]]
(> cnt 1)))))
(throw (ex-info "Cannot reuse yodlee/plaid/intuit account" {:validation-error (str "Cannot reuse yodlee/plaid/intuit account")})))))
(defn edit-client [context {:keys [edit_client new_bank_accounts] :as args} value] (defn edit-client [context {:keys [edit_client new_bank_accounts] :as args} value]
(assert-admin (:id context)) (assert-admin (:id context))
(when-not (:id edit_client) (when-not (:id edit_client)
@@ -71,10 +86,11 @@
(map (fn [ba] [:db/retract (:id ba) :bank-account/plaid-account (:db/id (:bank-account/plaid-account (d/entity (d/db conn) (:id ba))))]))))) (map (fn [ba] [:db/retract (:id ba) :bank-account/plaid-account (:db/id (:bank-account/plaid-account (d/entity (d/db conn) (:id ba))))])))))
client-code (if (str/blank? (:client/code client))
(:code edit_client)
(:client/code client))
transactions (into [(remove-nils {:db/id id transactions (into [(remove-nils {:db/id id
:client/code (if (str/blank? (:client/code client)) :client/code client-code
(:code edit_client)
(:client/code client))
:client/name (:name edit_client) :client/name (:name edit_client)
:client/matches (:matches edit_client) :client/matches (:matches edit_client)
:client/signature-file signature-file :client/signature-file signature-file
@@ -127,6 +143,7 @@
) )
(:forecasted_transactions edit_client))]] (:forecasted_transactions edit_client))]]
reverts) reverts)
_ (assert-no-shared-transaction-sources client-code transactions)
_ (log/info "upserting client" transactions) _ (log/info "upserting client" transactions)
result (audit-transact transactions (:id context))] result (audit-transact transactions (:id context))]
(-> result :tempids (get id) (or id) d-clients/get-by-id (-> result :tempids (get id) (or id) d-clients/get-by-id

168
src/clj/auto_ap/ledger.clj
View File

@@ -42,87 +42,89 @@
(defmethod entity-change->ledger :invoice (defmethod entity-change->ledger :invoice
[db [type id]] [db [type id]]
(let [entity (d/pull db ['* {:invoice/vendor '[*] (when id
:invoice/payment '[*] (let [entity (d/pull db ['* {:invoice/vendor '[*]
:invoice/status '[:db/ident] :invoice/payment '[*]
:invoice/import-status '[:db/ident]}] id) :invoice/status '[:db/ident]
credit-invoice? (< (:invoice/total entity) 0.0)] :invoice/import-status '[:db/ident]}] id)
(when-not (or credit-invoice? (< (:invoice/total entity) 0.0)]
(not (:invoice/total entity)) (when-not (or
(= true (:invoice/exclude-from-ledger entity)) (not (:invoice/total entity))
(= :import-status/pending (:db/ident (:invoice/import-status entity))) (= true (:invoice/exclude-from-ledger entity))
(= :invoice-status/voided (:db/ident (:invoice/status entity))) (= :import-status/pending (:db/ident (:invoice/import-status entity)))
(dollars-0? (:invoice/total entity))) (= :invoice-status/voided (:db/ident (:invoice/status entity)))
(remove-nils (dollars-0? (:invoice/total entity)))
{:journal-entry/source "invoice" (remove-nils
:journal-entry/client (:db/id (:invoice/client entity)) {:journal-entry/source "invoice"
:journal-entry/date (:invoice/date entity) :journal-entry/client (:db/id (:invoice/client entity))
:journal-entry/original-entity (:db/id entity) :journal-entry/date (:invoice/date entity)
:journal-entry/vendor (:db/id (:invoice/vendor entity)) :journal-entry/original-entity (:db/id entity)
:journal-entry/amount (Math/abs (:invoice/total entity)) :journal-entry/vendor (:db/id (:invoice/vendor entity))
:journal-entry/amount (Math/abs (:invoice/total entity))
:journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable :journal-entry/line-items (into [(cond-> {:journal-entry-line/account :account/accounts-payable
:journal-entry-line/location "A" :journal-entry-line/location "A"
}
credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity)))
(not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
(map (fn [ea]
(cond->
{:journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
:journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
} }
credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea))) credit-invoice? (assoc :journal-entry-line/debit (Math/abs (:invoice/total entity)))
(not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea))))) (not credit-invoice?) (assoc :journal-entry-line/credit (Math/abs (:invoice/total entity))))]
(:invoice/expense-accounts entity))) (map (fn [ea]
:journal-entry/cleared (and (< (:invoice/outstanding-balance entity) 0.01) (cond->
(every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity)) {:journal-entry-line/account (:db/id (:invoice-expense-account/account ea))
)})))) :journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ")
}
credit-invoice? (assoc :journal-entry-line/credit (Math/abs (:invoice-expense-account/amount ea)))
(not credit-invoice?) (assoc :journal-entry-line/debit (Math/abs (:invoice-expense-account/amount ea)))))
(:invoice/expense-accounts entity)))
:journal-entry/cleared (and (< (:invoice/outstanding-balance entity) 0.01)
(every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity))
)})))))
(defmethod entity-change->ledger :transaction (defmethod entity-change->ledger :transaction
[db [type id]] [db [type id]]
(let [entity (d/pull db ['* {:transaction/vendor '[*] (when id
:transaction/client '[*] (let [entity (d/pull db ['* {:transaction/vendor '[*]
:transaction/approval-status '[*] :transaction/client '[*]
:transaction/bank-account '[* {:bank-account/type [:db/ident]}] :transaction/approval-status '[*]
:transaction/accounts '[* :transaction/bank-account '[* {:bank-account/type [:db/ident]}]
{:transaction-account/account [*]}] }] id) :transaction/accounts '[*
bank-account-type (-> entity :transaction/bank-account :bank-account/type :db/ident) {:transaction-account/account [*]}] }] id)
decreasing? (< (:transaction/amount entity) 0.0) bank-account-type (-> entity :transaction/bank-account :bank-account/type :db/ident)
credit-from-bank? decreasing? decreasing? (< (:transaction/amount entity) 0.0)
debit-from-bank? (not decreasing?)] credit-from-bank? decreasing?
(when-not (or (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity))) debit-from-bank? (not decreasing?)]
(= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity))) (when-not (or (= :transaction-approval-status/excluded (:db/ident (:transaction/approval-status entity)))
(dollars-0? (:transaction/amount entity))) (= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status entity)))
(remove-nils (dollars-0? (:transaction/amount entity)))
{:journal-entry/source "transaction" (remove-nils
:journal-entry/client (:db/id (:transaction/client entity)) {:journal-entry/source "transaction"
:journal-entry/date (:transaction/date entity) :journal-entry/client (:db/id (:transaction/client entity))
:journal-entry/original-entity (:db/id entity) :journal-entry/date (:transaction/date entity)
:journal-entry/alternate-description (:transaction/description-original entity) :journal-entry/original-entity (:db/id entity)
:journal-entry/vendor (:db/id (:transaction/vendor entity)) :journal-entry/alternate-description (:transaction/description-original entity)
:journal-entry/amount (Math/abs (:transaction/amount entity)) :journal-entry/vendor (:db/id (:transaction/vendor entity))
:journal-entry/cleared-against (:transaction/cleared-against entity) :journal-entry/amount (Math/abs (:transaction/amount entity))
:journal-entry/cleared-against (:transaction/cleared-against entity)
:journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity)) :journal-entry/line-items (into [(remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity))
:journal-entry-line/location "A" :journal-entry-line/location "A"
:journal-entry-line/credit (when credit-from-bank? :journal-entry-line/credit (when credit-from-bank?
(Math/abs (:transaction/amount entity))) (Math/abs (:transaction/amount entity)))
:journal-entry-line/debit (when debit-from-bank? :journal-entry-line/debit (when debit-from-bank?
(Math/abs (:transaction/amount entity)))}) (Math/abs (:transaction/amount entity)))})
] ]
(map (map
(fn [a] (fn [a]
(remove-nils{:journal-entry-line/account (:db/id (:transaction-account/account a)) (remove-nils{:journal-entry-line/account (:db/id (:transaction-account/account a))
:journal-entry-line/location (:transaction-account/location a) :journal-entry-line/location (:transaction-account/location a)
:journal-entry-line/debit (when credit-from-bank? :journal-entry-line/debit (when credit-from-bank?
(Math/abs (:transaction-account/amount a))) (Math/abs (:transaction-account/amount a)))
:journal-entry-line/credit (when debit-from-bank? :journal-entry-line/credit (when debit-from-bank?
(Math/abs (:transaction-account/amount a)))})) (Math/abs (:transaction-account/amount a)))}))
(if (seq (:transaction/accounts entity)) (if (seq (:transaction/accounts entity))
(:transaction/accounts entity) (:transaction/accounts entity)
[{:transaction-account/amount (:transaction/amount entity)}]))) [{:transaction-account/amount (:transaction/amount entity)}])))
:journal-entry/cleared true})))) :journal-entry/cleared true})))))
(defmethod entity-change->ledger :expected-deposit (defmethod entity-change->ledger :expected-deposit
[db [type id]] [db [type id]]
@@ -382,30 +384,38 @@
(do (do
(log/warn (count mismatched-ts) " transactions exist but don't match ledger ") (log/warn (count mismatched-ts) " transactions exist but don't match ledger ")
(doseq [[m] mismatched-ts] (doseq [[m] mismatched-ts]
(touch-transaction m))))) (touch-transaction m))
(statsd/gauge "data.mismatched_transactions" (count (mismatched-transactions))))
(statsd/gauge "data.mismatched_transactions" 0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up") (log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-ts (unbalanced-transactions)] (let [unbalanced-ts (unbalanced-transactions)]
(if (seq unbalanced-ts) (if (seq unbalanced-ts)
(do (do
(log/warn (count unbalanced-ts) " transactions exist but don't have matching debits/credits ") (log/warn (count unbalanced-ts) " transactions exist but don't have matching debits/credits ")
(doseq [m unbalanced-ts] (doseq [m unbalanced-ts]
(touch-transaction m))))) (touch-transaction m))
(statsd/gauge "data.unbalanced_transactions" (count (unbalanced-transactions))))
(statsd/gauge "data.unbalanced_transactions" 0)))
(log/info "Finished fixing transactions that are in the ledger but are wrong") (log/info "Finished fixing transactions that are in the ledger but are wrong")
(let [mismatched-is (mismatched-invoices)] (let [mismatched-is (mismatched-invoices)]
(if (seq mismatched-is) (if (seq mismatched-is)
(do (do
(log/warn (count mismatched-is) " invoice exist but don't match ledger ") (log/warn (count mismatched-is) " invoice exist but don't match ledger ")
(doseq [[m] mismatched-is] (doseq [[m] mismatched-is]
(touch-invoice m))))) (touch-invoice m))
(statsd/gauge "data.mismatched_invoices" (count (mismatched-invoices))))
(statsd/gauge "data.mismatched_invoices" 0)))
(log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up") (log/info "Attempting to fix transactions that are in the ledger but debits/credits don't add up")
(let [unbalanced-invoices (unbalanced-invoices)] (let [unbalanced-invoices (unbalanced-invoices)]
(if (seq unbalanced-invoices) (if (seq unbalanced-invoices)
(do (do
(log/warn (count unbalanced-invoices) " invoices exist but don't have matching debits/credits ") (log/warn (count unbalanced-invoices) " invoices exist but don't have matching debits/credits ")
(doseq [m unbalanced-invoices] (doseq [m unbalanced-invoices]
(touch-invoice m))))) (touch-invoice m))
(statsd/gauge "data.unbalanced_invoices" (count (unbalanced-invoices))))
(statsd/gauge "data.unbalanced_invoices" 0)))
(log/info "Finished fixing invoices that are in the ledger but are wrong") (log/info "Finish fixing invoices that are in the ledger but are wrong")
(statsd/event {:title "Finished Reconciling Ledger" (statsd/event {:title "Finished Reconciling Ledger"
:text "This process looks for unbalance ledger entries, or missing ledger entries" :text "This process looks for unbalance ledger entries, or missing ledger entries"
:priority :low} :priority :low}

3
src/clj/user.clj
View File

@@ -744,3 +744,6 @@
(sort) (sort)
vec)) vec))
(defn init-repl []
(set! nrepl.middleware.print/*print-fn* clojure.pprint/pprint))