Makes logging unified
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
(ns auto-ap.import.common
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn pull-ref random-tempid]]
|
||||
[clojure.tools.logging :as log]
|
||||
[auto-ap.logging :as alog]
|
||||
[datomic.api :as dc]))
|
||||
|
||||
(defn bank-account->integration-id [bank-account]
|
||||
@@ -25,5 +25,5 @@
|
||||
:integration-status/state :integration-state/failed
|
||||
:integration-status/last-attempt (java.util.Date.)
|
||||
:integration-status/message (.getMessage e)}}])
|
||||
(log/warn e)
|
||||
(alog/warn ::integration-failed :error e)
|
||||
nil)))
|
||||
|
||||
@@ -4,15 +4,13 @@
|
||||
[auto-ap.import.common :refer [wrap-integration]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.intuit.core :as i]
|
||||
[auto-ap.logging :as alog]
|
||||
[auto-ap.time :as atime]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[datomic.api :as dc]
|
||||
[mount.core :as mount]
|
||||
[yang.scheduler :as scheduler]))
|
||||
[datomic.api :as dc]))
|
||||
|
||||
(defn get-intuit-bank-accounts [db]
|
||||
(dc/q '[:find ?external-id ?ba ?c
|
||||
@@ -28,7 +26,7 @@
|
||||
(try
|
||||
(Integer/parseInt (:Num transaction))
|
||||
(catch NumberFormatException e
|
||||
(log/warn "Got an invalid check number " e)
|
||||
(alog/warn ::invalid-check-number :check-number (:Num transaction) :error e)
|
||||
nil)))]
|
||||
(cond-> {:transaction/description-original (:Memo/Description transaction)
|
||||
:transaction/amount (Double/parseDouble (:Amount transaction))
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.data.csv :as csv]
|
||||
[datomic.api :as dc]
|
||||
[unilog.context :as lc]))
|
||||
[datomic.api :as dc]))
|
||||
|
||||
|
||||
|
||||
@@ -33,33 +32,32 @@
|
||||
(c/assoc-or-error :transaction/amount #(c/parse-amount transaction))))
|
||||
|
||||
(defn import-batch [transactions user]
|
||||
(lc/with-context {:source "Manual import transactions"}
|
||||
(let [bank-account-code->client (into {}
|
||||
(dc/q '[:find ?bac ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/code ?bac]]
|
||||
(dc/db conn)))
|
||||
bank-account-code->bank-account (into {}
|
||||
(dc/q '[:find ?bac ?ba
|
||||
:in $
|
||||
:where [?ba :bank-account/code ?bac]]
|
||||
(dc/db conn)))
|
||||
import-batch (t/start-import-batch :import-source/manual user)
|
||||
transactions (->> transactions
|
||||
(map (fn [t]
|
||||
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
|
||||
(t/apply-synthetic-ids ))]
|
||||
(try
|
||||
(doseq [transaction transactions]
|
||||
(when-not (seq (:errors transaction))
|
||||
(t/import-transaction! import-batch transaction)))
|
||||
(let [bank-account-code->client (into {}
|
||||
(dc/q '[:find ?bac ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/code ?bac]]
|
||||
(dc/db conn)))
|
||||
bank-account-code->bank-account (into {}
|
||||
(dc/q '[:find ?bac ?ba
|
||||
:in $
|
||||
:where [?ba :bank-account/code ?bac]]
|
||||
(dc/db conn)))
|
||||
import-batch (t/start-import-batch :import-source/manual user)
|
||||
transactions (->> transactions
|
||||
(map (fn [t]
|
||||
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
|
||||
(t/apply-synthetic-ids ))]
|
||||
(try
|
||||
(doseq [transaction transactions]
|
||||
(when-not (seq (:errors transaction))
|
||||
(t/import-transaction! import-batch transaction)))
|
||||
|
||||
(t/finish! import-batch)
|
||||
(assoc (t/get-stats import-batch)
|
||||
:failed-validation (count (filter :errors transactions))
|
||||
:sample-error (first (first (map :errors (filter :errors transactions)))))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(t/get-stats import-batch))))))
|
||||
(t/finish! import-batch)
|
||||
(assoc (t/get-stats import-batch)
|
||||
:failed-validation (count (filter :errors transactions))
|
||||
:sample-error (first (first (map :errors (filter :errors transactions)))))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(t/get-stats import-batch)))))
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
[digest :as di]
|
||||
[manifold.deferred :as de]
|
||||
[manifold.executor :as ex]
|
||||
[unilog.context :as lc]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn get-plaid-accounts [db]
|
||||
@@ -73,32 +72,31 @@
|
||||
"name" (:plaid-merchant/name result)}))))
|
||||
|
||||
(defn import-plaid-int []
|
||||
(lc/with-context {:source "Import plaid transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user")
|
||||
end (atime/local-now)
|
||||
start (time/plus end (time/days -30))
|
||||
plaid-merchant->vendor-id (build-plaid-merchant->vendor-id)]
|
||||
(try
|
||||
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (dc/db conn))
|
||||
:let [transaction-result (wrap-integration #(p/get-transactions access-token external-id start end)
|
||||
bank-account-id)
|
||||
accounts-by-id (by :account_id (:accounts transaction-result))]
|
||||
transaction (:transactions transaction-result)]
|
||||
(when (not (:pending transaction))
|
||||
(t/import-transaction! import-batch (assoc (plaid->transaction (assoc transaction
|
||||
:account
|
||||
(accounts-by-id (:account_id transaction)))
|
||||
plaid-merchant->vendor-id)
|
||||
(let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user")
|
||||
end (atime/local-now)
|
||||
start (time/plus end (time/days -30))
|
||||
plaid-merchant->vendor-id (build-plaid-merchant->vendor-id)]
|
||||
(try
|
||||
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (dc/db conn))
|
||||
:let [transaction-result (wrap-integration #(p/get-transactions access-token external-id start end)
|
||||
bank-account-id)
|
||||
accounts-by-id (by :account_id (:accounts transaction-result))]
|
||||
transaction (:transactions transaction-result)]
|
||||
(when (not (:pending transaction))
|
||||
(t/import-transaction! import-batch (assoc (plaid->transaction (assoc transaction
|
||||
:account
|
||||
(accounts-by-id (:account_id transaction)))
|
||||
plaid-merchant->vendor-id)
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))))
|
||||
(try
|
||||
(rebuild-search-index)
|
||||
(catch Exception e
|
||||
(alog/error ::cant-index-plaid
|
||||
:error e)))
|
||||
(t/finish! import-batch)
|
||||
(try
|
||||
(rebuild-search-index)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
(alog/error ::cant-index-plaid
|
||||
:error e)))
|
||||
(t/finish! import-batch)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)))))
|
||||
|
||||
(def import-plaid (allow-once import-plaid-int))
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as t]
|
||||
[clojure.core.cache :as cache]
|
||||
[clojure.tools.logging :as log]
|
||||
[auto-ap.logging :as alog]
|
||||
[datomic.api :as dc]
|
||||
[digest :as di]))
|
||||
|
||||
@@ -30,11 +30,11 @@
|
||||
|
||||
|
||||
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
||||
(log/info "Searching for a matching check for "
|
||||
{:client-id client-id
|
||||
(alog/info ::searching
|
||||
:client-id client-id
|
||||
:check-number check-number
|
||||
:bank-account-id bank-account-id
|
||||
:amount amount})
|
||||
:amount amount)
|
||||
(cond (not (and client-id bank-account-id))
|
||||
nil
|
||||
|
||||
@@ -57,7 +57,6 @@
|
||||
(rough-match client-id bank-account-id amount)))
|
||||
|
||||
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
||||
(log/info "trying to find uncleared autopay invoices")
|
||||
(let [candidate-invoices-vendor-groups (->> (dc/q {:find ['?vendor-id '?e '?total '?sd]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
@@ -77,12 +76,15 @@
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
||||
considerations
|
||||
))
|
||||
(alog/info ::unfulfilled-autoapayment-considerations
|
||||
:count (count considerations)
|
||||
:amount amount)
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
||||
(log/info "trying to find unpaid invoices for " client-id amount)
|
||||
(alog/info ::searching-unpaid-invoice
|
||||
:client-id client-id
|
||||
:amount amount)
|
||||
(let [candidate-invoices-vendor-groups (->> (dc/q {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
@@ -102,7 +104,10 @@
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
||||
(alog/info ::unpaid-invoice-considerations-found
|
||||
:client-id client-id
|
||||
:amount amount
|
||||
:count (count considerations))
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
||||
@@ -112,7 +117,9 @@
|
||||
[])))
|
||||
|
||||
(defn add-new-payment [transaction [[vendor] :as invoice-payments] bank-account-id client-id]
|
||||
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
||||
(alog/info ::adding-payment
|
||||
:transaction-id (:transaction/id transaction)
|
||||
:invoices (count invoice-payments))
|
||||
(let [payment-id (random-tempid)]
|
||||
(-> [[:upsert-transaction
|
||||
(assoc transaction
|
||||
@@ -279,7 +286,6 @@
|
||||
|
||||
|
||||
(defn get-existing [bank-account]
|
||||
(log/info "looking up bank account data for" bank-account)
|
||||
(into {}
|
||||
(dc/q '[:find ?tid ?as2
|
||||
:in $ ?ba
|
||||
@@ -314,7 +320,8 @@
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch")
|
||||
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
|
||||
(log/info "Importing transactions from " source)
|
||||
(alog/info ::starting-transaction-import
|
||||
:source source)
|
||||
|
||||
(reify ImportBatch
|
||||
(import-transaction! [_ transaction]
|
||||
@@ -344,15 +351,17 @@
|
||||
@stats)
|
||||
|
||||
(fail! [_ error]
|
||||
(log/errorf "Couldn't complete import %d with error." import-id)
|
||||
(log/error error)
|
||||
(alog/error ::cant-complete-import
|
||||
:import-id import-id
|
||||
:error error)
|
||||
|
||||
@(dc/transact-async conn [(merge {:db/id import-id
|
||||
:import-batch/status :import-status/completed
|
||||
:import-batch/error-message (str error)}
|
||||
@stats)]))
|
||||
|
||||
(finish! [_]
|
||||
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
|
||||
(alog/info ::finished :import-id import-id :source source :stats (pr-str @stats))
|
||||
@(dc/transact conn [(merge {:db/id import-id
|
||||
|
||||
:import-batch/status :import-status/completed}
|
||||
|
||||
@@ -10,8 +10,7 @@
|
||||
[clojure.string :as str]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[datomic.api :as dc]
|
||||
[digest :as di]
|
||||
[unilog.context :as lc]))
|
||||
[digest :as di]))
|
||||
|
||||
#_{:clj-kondo/ignore [:unresolved-var]}
|
||||
(defn yodlee->transaction [transaction use-date-instead-of-post-date?]
|
||||
@@ -45,43 +44,42 @@
|
||||
:status status}))
|
||||
|
||||
(defn import-yodlee2-int []
|
||||
(lc/with-context {:source "Import yodlee2 transactions"}
|
||||
(statsd/event {:title "Yodlee2 import started"
|
||||
:text "Starting"
|
||||
:priority :low}
|
||||
nil)
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
|
||||
(try
|
||||
(let [account-lookup (dc/q '[:find ?ya ?ba ?cd ?ud
|
||||
:in $
|
||||
:where
|
||||
[?ba :bank-account/yodlee-account ?y]
|
||||
[(get-else $ ?ba :bank-account/use-date-instead-of-post-date? false) ?ud]
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?c :client/code ?cd]
|
||||
[?y :yodlee-account/id ?ya]
|
||||
]
|
||||
(dc/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-code use-date-instead-of-post-date?] account-lookup
|
||||
transaction (wrap-integration #(client2/get-specific-transactions client-code yodlee-account)
|
||||
bank-account)]
|
||||
(t/import-transaction! import-batch (assoc (yodlee->transaction transaction use-date-instead-of-post-date?)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client [:client/code client-code])))
|
||||
(statsd/event {:title "Yodlee2 import started"
|
||||
:text "Starting"
|
||||
:priority :low}
|
||||
nil)
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
|
||||
(try
|
||||
(let [account-lookup (dc/q '[:find ?ya ?ba ?cd ?ud
|
||||
:in $
|
||||
:where
|
||||
[?ba :bank-account/yodlee-account ?y]
|
||||
[(get-else $ ?ba :bank-account/use-date-instead-of-post-date? false) ?ud]
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?c :client/code ?cd]
|
||||
[?y :yodlee-account/id ?ya]
|
||||
]
|
||||
(dc/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-code use-date-instead-of-post-date?] account-lookup
|
||||
transaction (wrap-integration #(client2/get-specific-transactions client-code yodlee-account)
|
||||
bank-account)]
|
||||
(t/import-transaction! import-batch (assoc (yodlee->transaction transaction use-date-instead-of-post-date?)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client [:client/code client-code])))
|
||||
|
||||
(t/finish! import-batch))
|
||||
(statsd/event {:title "Yodlee2 import Finished"
|
||||
:text (pr-str (t/get-stats import-batch))
|
||||
:priority :low}
|
||||
nil)
|
||||
(t/finish! import-batch))
|
||||
(statsd/event {:title "Yodlee2 import Finished"
|
||||
:text (pr-str (t/get-stats import-batch))
|
||||
:priority :low}
|
||||
nil)
|
||||
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(statsd/event {:title "Yodlee2 import failed"
|
||||
:text (str e)
|
||||
:alert-type :warning
|
||||
:priority :normal}
|
||||
nil))))))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(statsd/event {:title "Yodlee2 import failed"
|
||||
:text (str e)
|
||||
:alert-type :warning
|
||||
:priority :normal}
|
||||
nil)))))
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user