fixed warnings.

This commit is contained in:
2022-07-26 05:56:41 -07:00
parent f6f73a6110
commit 96c80853ef
133 changed files with 670 additions and 1640 deletions

View File

@@ -1,15 +1,14 @@
(ns auto-ap.import.manual
(:require [auto-ap.datomic :refer [conn]]
[auto-ap.import.manual.common :as c]
[auto-ap.import.transactions :as t]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv]
[clojure.tools.logging :as log]
[datomic.api :as d]
[unilog.context :as lc]))
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.import.manual.common :as c]
[auto-ap.import.transactions :as t]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv]
[datomic.api :as d]
[unilog.context :as lc]))
(defn manual-import-batch [transactions user]
)
(def columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code])
@@ -19,9 +18,9 @@
(drop 1)
(map (fn [row]
(into {} (->> (map vector columns row)
(filter (fn [[k v]] k))))))))
(filter (fn [[k _]] k))))))))
(defn manual->transaction [{:keys [description-original amount bank-account-code date] :as transaction} bank-account-lookup client-lookup]
(defn manual->transaction [{:keys [description-original bank-account-code] :as transaction} bank-account-lookup client-lookup]
(-> {:transaction/description-original description-original
:transaction/status "POSTED"}
(c/assoc-or-error :transaction/client #(if-let [client-id (client-lookup bank-account-code)]
@@ -46,21 +45,21 @@
(d/q '[:find ?bac ?ba
:in $
:where [?ba :bank-account/code ?bac]]
(d/db conn)))]
(let [import-batch (t/start-import-batch :import-source/manual user)
transactions (->> transactions
(map (fn [t]
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
(t/apply-synthetic-ids ))]
(try
(doseq [transaction transactions]
(when-not (seq (:errors transaction))
(t/import-transaction! import-batch transaction)))
(d/db conn)))
import-batch (t/start-import-batch :import-source/manual user)
transactions (->> transactions
(map (fn [t]
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
(t/apply-synthetic-ids ))]
(try
(doseq [transaction transactions]
(when-not (seq (:errors transaction))
(t/import-transaction! import-batch transaction)))
(t/finish! import-batch)
(assoc (t/get-stats import-batch)
:failed-validation (count (filter :errors transactions))
:sample-error (first (first (map :errors (filter :errors transactions)))))
(catch Exception e
(t/fail! import-batch e)
(t/get-stats import-batch)))))))
(t/finish! import-batch)
(assoc (t/get-stats import-batch)
:failed-validation (count (filter :errors transactions))
:sample-error (first (first (map :errors (filter :errors transactions)))))
(catch Exception e
(t/fail! import-batch e)
(t/get-stats import-batch))))))

View File

@@ -9,6 +9,7 @@
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[datomic.api :as d]
[digest :as di]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
@@ -28,7 +29,8 @@
(defn plaid->transaction [t]
#:transaction {:description-original (:name t)
:raw-id (:transaction_id t)
:id (digest/sha-256 (:transaction_id t))
:id #_{:clj-kondo/ignore [:unresolved-var]}
(di/sha-256 (:transaction_id t))
:amount (if (= "credit" (:type (:account t)))
(- (double (:amount t)))
(double (:amount t)))
@@ -36,7 +38,7 @@
:status "POSTED"})
(defn import-plaid []
(defn import-plaid-int []
(lc/with-context {:source "Import plaid transactions"}
(let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user")
end (atime/local-now)
@@ -57,7 +59,7 @@
(catch Exception e
(t/fail! import-batch e))))))
(def import-plaid (allow-once import-plaid))
(def import-plaid (allow-once import-plaid-int))
(mount/defstate import-worker
:start (scheduler/every (* 1000 60 60 3) (heartbeat import-plaid "import-plaid"))

View File

@@ -1,18 +1,19 @@
(ns auto-ap.import.transactions
(:require [auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.datomic.transactions :as d-transactions]
[auto-ap.rule-matching :as rm]
[auto-ap.utils :refer [dollars=]]
[auto-ap.yodlee.core :as client]
[clj-time.coerce :as coerce]
[clj-time.core :as t]
[clojure.core.cache :as cache]
[clojure.tools.logging :as log]
[datomic.api :as d]
[digest :refer [sha-256]]))
(:require
[auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.datomic.transactions :as d-transactions]
[auto-ap.rule-matching :as rm]
[auto-ap.time :as atime]
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
[clj-time.core :as t]
[clojure.core.cache :as cache]
[clojure.tools.logging :as log]
[datomic.api :as d]
[digest :as di]))
(defn rough-match [client-id bank-account-id amount]
(if (and client-id bank-account-id amount)
@@ -35,7 +36,8 @@
(cond (not (and client-id bank-account-id))
nil
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
(:transaction/payment (d-transactions/get-by-id [:transaction/id #_{:clj-kondo/ignore [:unresolved-var]}
(di/sha-256 (str id))]))
nil
check-number
@@ -119,7 +121,7 @@
:payment/type :payment-type/debit
:payment/status :payment-status/cleared
:db/id payment-id})
(into (mapcat (fn [[vendor invoice-id invoice-amount]]
(into (mapcat (fn [[_ invoice-id invoice-amount]]
[{:invoice-payment/invoice invoice-id
:invoice-payment/payment payment-id
:invoice-payment/amount invoice-amount}
@@ -143,7 +145,7 @@
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
(try
(Integer/parseInt check-number)
(catch NumberFormatException e
(catch NumberFormatException _
nil))
nil))
@@ -238,8 +240,7 @@
))))
(defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations]
(if-let [unpaid-invoices (seq (match-transaction-to-unpaid-invoices amount client))]
nil
(when (seq (match-transaction-to-unpaid-invoices amount client))
(apply-rules transaction valid-locations)))
(defn transaction->txs [transaction bank-account apply-rules]
@@ -294,7 +295,7 @@
(log/info "Importing transactions from " source)
(reify ImportBatch
(import-transaction! [this transaction]
(import-transaction! [_ transaction]
(let [bank-account (d/pull (d/db conn)
[:bank-account/code
:db/id
@@ -318,10 +319,10 @@
{:user/name user
:user/role ":admin"}))))
(get-stats [this]
(get-stats [_]
@stats)
(fail! [this error]
(fail! [_ error]
(log/errorf "Couldn't complete import %d with error." import-id)
(log/error error)
@(d/transact conn [(merge {:db/id import-id
@@ -329,7 +330,7 @@
:import-batch/error-message (str error)}
@stats)]))
(finish! [this]
(finish! [_]
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
@(d/transact conn [(merge {:db/id import-id
@@ -339,7 +340,7 @@
(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index]
(str (str (some-> date coerce/to-date-time auto-ap.time/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
(str (str (some-> date coerce/to-date-time atime/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
(defn apply-synthetic-ids [transactions]
(->> transactions
@@ -349,7 +350,8 @@
(map (fn [index transaction]
(let [raw-id (synthetic-key transaction index)]
(assoc transaction
:transaction/id (digest/sha-256 raw-id)
:transaction/id #_{:clj-kondo/ignore [:unresolved-var]}
(di/sha-256 raw-id)
:transaction/raw-id raw-id)))
(range)
group)))))

View File

@@ -3,16 +3,14 @@
[auto-ap.datomic :refer [conn]]
[auto-ap.import.transactions :as t]
[auto-ap.time :as atime]
[auto-ap.utils :refer [allow-once heartbeat]]
[auto-ap.utils :refer [allow-once]]
[auto-ap.yodlee.core :as client]
[clj-time.coerce :as coerce]
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]
[digest :refer [sha-256]]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
[digest :as di]))
(defn yodlee->transaction [transaction use-date-instead-of-post-date?]
(let [{post-date :postDate
@@ -31,7 +29,8 @@
date (atime/parse date "YYYY-MM-dd")]
#:transaction
{:post-date (coerce/to-date (atime/parse post-date "YYYY-MM-dd"))
:id (sha-256 (str id))
:id #_{:clj-kondo/ignore [:unresolved-var]}
(di/sha-256 (str id))
:raw-id (str id)
:account-id account-id
:date (if use-date-instead-of-post-date?

View File

@@ -12,7 +12,7 @@
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
(defn import-yodlee2 []
(defn import-yodlee2-int []
(lc/with-context {:source "Import yodlee2 transactions"}
(statsd/event {:title "Yodlee2 import started"
:text "Starting"
@@ -53,7 +53,7 @@
(def import-yodlee2 (allow-once import-yodlee2))
(def import-yodlee2 (allow-once import-yodlee2-int))
(mount/defstate import-worker