enormous refactor but simplified much!
This commit is contained in:
Bryce Covert
74
src/clj/auto_ap/import/intuit.clj
Normal file
74
src/clj/auto_ap/import/intuit.clj
Normal file
@@ -0,0 +1,74 @@
|
||||
(ns auto-ap.import.intuit
|
||||
(:require [amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic :refer [conn remove-nils]]
|
||||
[auto-ap.intuit.core :as i]
|
||||
[auto-ap.utils :refer [by allow-once]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clj-time.format :as f]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(defn get-intuit-bank-accounts [db]
|
||||
(d/q '[:find ?external-id ?ba ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/intuit-bank-account ?iab]
|
||||
[?iab :intuit-bank-account/external-id ?external-id]]
|
||||
db))
|
||||
|
||||
(defn intuit->transaction [transaction]
|
||||
{:transaction/description-original (:Memo/Description transaction)
|
||||
:transaction/amount (Double/parseDouble (:Amount transaction))
|
||||
:transaction/date (coerce/to-date (auto-ap.time/parse (:Date transaction) auto-ap.time/iso-date))
|
||||
:transaction/status "POSTED"})
|
||||
|
||||
(defn intuits->transactions [transactions bank-account-id client-id]
|
||||
(->> transactions
|
||||
(map intuit->transaction)
|
||||
(map #(assoc %
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))
|
||||
(t/apply-synthetic-ids)))
|
||||
|
||||
(defn import-intuit []
|
||||
(lc/with-context {:source "Import intuit transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
|
||||
db (d/db conn)
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
(try
|
||||
(doseq [[external-id bank-account-id client-id] (get-intuit-bank-accounts db)
|
||||
transaction (-> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
|
||||
(auto-ap.time/unparse end auto-ap.time/iso-date)
|
||||
external-id)
|
||||
(intuits->transactions bank-account-id client-id))]
|
||||
(t/import-transaction! import-batch transaction))
|
||||
(t/finish! import-batch)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def upsert-transactions (allow-once upsert-transactions))
|
||||
|
||||
(defn upsert-accounts []
|
||||
(let [token (i/get-fresh-access-token)
|
||||
bank-accounts (i/get-bank-accounts token)]
|
||||
@(d/transact conn (mapv
|
||||
(fn [ba]
|
||||
{:intuit-bank-account/external-id (:name ba)
|
||||
:intuit-bank-account/name (:name ba)})
|
||||
bank-accounts))))
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) import-intuit)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
(mount/defstate account-worker
|
||||
:start (scheduler/every (* 1000 60 60 24) upsert-accounts)
|
||||
:stop (scheduler/stop account-worker))
|
||||
20
src/clj/auto_ap/import/intuit_test.clj
Normal file
20
src/clj/auto_ap/import/intuit_test.clj
Normal file
@@ -0,0 +1,20 @@
|
||||
(ns auto-ap.import.intuit-test
|
||||
(:require [auto-ap.import.intuit :as sut]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(def base-transaction {:Memo/Description "this is a description"
|
||||
:Amount "45.23"
|
||||
:Date "2021-10-11"})
|
||||
|
||||
(t/deftest intuit->transaction
|
||||
(t/testing "Should parse dates"
|
||||
(t/is (= #inst "2021-01-01T00:00:00-08:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-01-01")))))
|
||||
(t/is (= #inst "2021-06-01T00:00:00-07:00" (:transaction/date (sut/intuit->transaction (assoc base-transaction :Date "2021-06-01")))))))
|
||||
|
||||
|
||||
(t/deftest intuits->transactions
|
||||
(t/testing "should give unique ids to duplicates"
|
||||
(t/is (= ["2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-0-345"
|
||||
"2021-10-11T00:00:00.000-07:00-123-this is a description-45.23-1-345"] (map :transaction/raw-id (sut/intuits->transactions [base-transaction base-transaction]
|
||||
123
|
||||
345))))))
|
||||
66
src/clj/auto_ap/import/manual.clj
Normal file
66
src/clj/auto_ap/import/manual.clj
Normal file
@@ -0,0 +1,66 @@
|
||||
(ns auto-ap.import.manual
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.manual.common :as c]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[unilog.context :as lc]))
|
||||
|
||||
(defn manual-import-batch [transactions user]
|
||||
)
|
||||
|
||||
(def columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code])
|
||||
|
||||
(defn tabulate-data [data]
|
||||
(->>
|
||||
(csv/read-csv data :separator \tab)
|
||||
(drop 1)
|
||||
(map (fn [row]
|
||||
(into {} (->> (map vector columns row)
|
||||
(filter (fn [[k v]] k))))))))
|
||||
|
||||
(defn manual->transaction [{:keys [description-original amount bank-account-code date] :as transaction} bank-account-lookup client-lookup]
|
||||
(-> {:transaction/description-original description-original
|
||||
:transaction/status "POSTED"}
|
||||
(c/assoc-or-error :transaction/client #(if-let [client-id (client-lookup bank-account-code)]
|
||||
client-id
|
||||
(throw (Exception. (str "Cannot find client for bank account code " bank-account-code)))))
|
||||
(c/assoc-or-error :transaction/bank-account #(if-let [bank-account-id (bank-account-lookup bank-account-code)]
|
||||
bank-account-id
|
||||
(throw (Exception. (str "Cannot find bank account by code " bank-account-code)))))
|
||||
(c/assoc-or-error :transaction/date #(coerce/to-date (c/parse-date transaction)))
|
||||
(c/assoc-or-error :transaction/amount #(c/parse-amount transaction))))
|
||||
|
||||
(defn import-batch [transactions user]
|
||||
(lc/with-context {:source "Manual import transactions"}
|
||||
(let [bank-account-code->client (into {}
|
||||
(d/q '[:find ?bac ?c
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/code ?bac]]
|
||||
(d/db conn)))
|
||||
bank-account-code->bank-account (into {}
|
||||
(d/q '[:find ?bac ?ba
|
||||
:in $
|
||||
:where [?ba :bank-account/code ?bac]]
|
||||
(d/db conn)))]
|
||||
(let [import-batch (t/start-import-batch :import-source/manual user)
|
||||
transactions (->> transactions
|
||||
(map (fn [t]
|
||||
(manual->transaction t bank-account-code->bank-account bank-account-code->client)))
|
||||
(t/apply-synthetic-ids ))]
|
||||
(try
|
||||
(doseq [transaction transactions]
|
||||
(when-not (seq (:errors transaction))
|
||||
(t/import-transaction! import-batch transaction)))
|
||||
|
||||
(t/finish! import-batch)
|
||||
(assoc (t/get-stats import-batch)
|
||||
:failed-validation (count (filter :errors transactions))
|
||||
:sample-error (first (first (map :errors (filter :errors transactions)))))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e)
|
||||
(t/get-stats import-batch)))))))
|
||||
53
src/clj/auto_ap/import/manual/common.clj
Normal file
53
src/clj/auto_ap/import/manual/common.clj
Normal file
@@ -0,0 +1,53 @@
|
||||
(ns auto-ap.import.manual.common
|
||||
(:require [auto-ap.datomic.accounts :as d-accounts]
|
||||
[auto-ap.parse.util :as parse-u]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn parse-amount [i]
|
||||
(try
|
||||
(Double/parseDouble (str/replace (or (second
|
||||
(re-matches #"[^0-9\.,\\-]*([0-9\.,\\-]+)[^0-9\.,]*" (:amount i)))
|
||||
"0")
|
||||
#"," ""))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse total from value '" (:amount i) "'") e)))))
|
||||
|
||||
(defn parse-account-numeric-code [i]
|
||||
(try
|
||||
(when-let [account-numeric-code (:account-numeric-code i)]
|
||||
(:db/id (d-accounts/get-account-by-numeric-code-and-sets (Integer/parseInt account-numeric-code)
|
||||
["default"])))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse expense account from value '" (:account-numeric-code i) "'") e)))))
|
||||
|
||||
(defn parse-account-id [i]
|
||||
(try
|
||||
(Long/parseLong (second
|
||||
(re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i))))
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e)))))
|
||||
|
||||
(defn parse-date [{:keys [raw-date]}]
|
||||
(when-not
|
||||
(re-find #"\d{1,2}/\d{1,2}/\d{4}" raw-date)
|
||||
(throw (Exception. (str "Date " raw-date " must match MM/dd/yyyy"))))
|
||||
(try
|
||||
|
||||
(parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date)
|
||||
(catch Exception e
|
||||
(throw (Exception. (str "Could not parse date from '" raw-date "'") e)))))
|
||||
|
||||
(defn parse-or-error [key f]
|
||||
(fn [x]
|
||||
(try
|
||||
(assoc x key (f x))
|
||||
(catch Exception e
|
||||
(update x :errors conj {:info (.getMessage e)
|
||||
:details (str e)})))))
|
||||
|
||||
(defn assoc-or-error [m k f]
|
||||
(try
|
||||
(assoc m k (f))
|
||||
(catch Exception e
|
||||
(update m :errors conj {:info (.getMessage e)
|
||||
:details (str e)}))))
|
||||
60
src/clj/auto_ap/import/plaid.clj
Normal file
60
src/clj/auto_ap/import/plaid.clj
Normal file
@@ -0,0 +1,60 @@
|
||||
(ns auto-ap.import.plaid
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.plaid.core :as p]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[clj-time.core :as time]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]
|
||||
[clj-time.coerce :as coerce]))
|
||||
|
||||
|
||||
(defn get-plaid-accounts [db]
|
||||
(-> (d/q '[:find ?ba ?c ?external-id ?t
|
||||
:in $
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/plaid-account ?pa]
|
||||
[?pa :plaid-account/external-id ?external-id]
|
||||
[?pi :plaid-item/accounts ?pa]
|
||||
[?pi :plaid-item/access-token ?t]]
|
||||
db )))
|
||||
|
||||
|
||||
(defn plaid->transaction [t]
|
||||
#:transaction {:description-original (:name t)
|
||||
:raw-id (:transaction_id t)
|
||||
:id (digest/sha-256 (:transaction_id t))
|
||||
:amount (double (:amount t))
|
||||
:date (coerce/to-date (auto-ap.time/parse (:date t) auto-ap.time/iso-date))
|
||||
:status "POSTED"})
|
||||
|
||||
|
||||
(defn import-plaid []
|
||||
(lc/with-context {:source "Import plaid transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/plaid "Automated plaid user")
|
||||
end (auto-ap.time/local-now)
|
||||
start (time/plus end (time/days -30))]
|
||||
(try
|
||||
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn))
|
||||
transaction (:transactions (p/get-transactions access-token external-id start end))]
|
||||
(when (not (:pending transaction))
|
||||
(t/import-transaction! import-batch (assoc (plaid->transaction transaction)
|
||||
:transaction/bank-account bank-account-id
|
||||
:transaction/client client-id))))
|
||||
(t/finish! import-batch)
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def import-plaid (allow-once import-plaid))
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 3) import-plaid)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
|
||||
|
||||
336
src/clj/auto_ap/import/transactions.clj
Normal file
336
src/clj/auto_ap/import/transactions.clj
Normal file
@@ -0,0 +1,336 @@
|
||||
(ns auto-ap.import.transactions
|
||||
(:require [auto-ap.datomic :refer [audit-transact conn remove-nils uri]]
|
||||
[auto-ap.datomic.accounts :as a]
|
||||
[auto-ap.datomic.checks :as d-checks]
|
||||
[auto-ap.datomic.transaction-rules :as tr]
|
||||
[auto-ap.datomic.transactions :as d-transactions]
|
||||
[auto-ap.rule-matching :as rm]
|
||||
[auto-ap.utils :refer [dollars=]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as t]
|
||||
[clojure.core.cache :as cache]
|
||||
[clojure.tools.logging :as log]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]))
|
||||
|
||||
(defn rough-match [client-id bank-account-id amount]
|
||||
(if (and client-id bank-account-id amount)
|
||||
(let [[matching-checks] (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})]
|
||||
(if (= 1 (count matching-checks))
|
||||
(first matching-checks)
|
||||
nil))
|
||||
nil))
|
||||
|
||||
|
||||
(defn transaction->existing-payment [_ check-number client-id bank-account-id amount id]
|
||||
(log/info "Searching for a matching check for "
|
||||
{:client-id client-id
|
||||
:check-number check-number
|
||||
:bank-account-id bank-account-id
|
||||
:amount amount})
|
||||
(cond (not (and client-id bank-account-id))
|
||||
nil
|
||||
|
||||
(:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))]))
|
||||
nil
|
||||
|
||||
check-number
|
||||
(or (-> (d-checks/get-graphql {:client-id client-id
|
||||
:bank-account-id bank-account-id
|
||||
:check-number check-number
|
||||
:amount (- amount)
|
||||
:status :payment-status/pending})
|
||||
first
|
||||
first)
|
||||
(rough-match client-id bank-account-id amount))
|
||||
|
||||
:else
|
||||
(rough-match client-id bank-account-id amount)))
|
||||
|
||||
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
|
||||
(log/info "trying to find uncleared autopay invoices")
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/scheduled-payment ?sd]
|
||||
'[?e :invoice/status :invoice-status/paid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/total ?total]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "considerations for transaction of" amount)
|
||||
considerations
|
||||
))
|
||||
|
||||
(defn match-transaction-to-unpaid-invoices [amount client-id]
|
||||
(log/info "trying to find unpaid invoices for " client-id amount)
|
||||
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
|
||||
:in ['$ '?client-id]
|
||||
:where ['[?e :invoice/client ?client-id]
|
||||
'[?e :invoice/status :invoice-status/unpaid]
|
||||
'(not [_ :invoice-payment/invoice ?e])
|
||||
'[?e :invoice/vendor ?vendor-id]
|
||||
'[?e :invoice/outstanding-balance ?outstanding-balance]
|
||||
'[?e :invoice/date ?d]]}
|
||||
:args [(d/db conn) client-id]})
|
||||
(sort-by last) ;; sort by scheduled payment date
|
||||
(group-by first) ;; group by vendors
|
||||
vals)
|
||||
considerations (for [candidate-invoices candidate-invoices-vendor-groups
|
||||
invoice-count (range 1 32)
|
||||
consideration (partition invoice-count 1 candidate-invoices)
|
||||
:when (dollars= (reduce (fn [acc [_ _ amount]]
|
||||
(+ acc amount)) 0.0 consideration)
|
||||
(- amount))]
|
||||
consideration)]
|
||||
(log/info "Found " (count considerations) "unpaid invoice considerations for transaction of" amount)
|
||||
considerations))
|
||||
|
||||
(defn match-transaction-to-single-unfulfilled-autopayments [amount client-id]
|
||||
(let [considerations (match-transaction-to-unfulfilled-autopayments amount client-id)]
|
||||
(if (= 1 (count considerations))
|
||||
(first considerations)
|
||||
[])))
|
||||
|
||||
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
|
||||
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
|
||||
(let [payment-id (d/tempid :db.part/user)]
|
||||
(-> tx
|
||||
|
||||
(conj {:payment/bank-account bank-account-id
|
||||
:payment/client client-id
|
||||
:payment/amount (- (:transaction/amount transaction))
|
||||
:payment/vendor vendor
|
||||
:payment/date (:transaction/date transaction)
|
||||
:payment/type :payment-type/debit
|
||||
:payment/status :payment-status/cleared
|
||||
:db/id payment-id})
|
||||
(into (mapcat (fn [[vendor invoice-id invoice-amount]]
|
||||
[{:invoice-payment/invoice invoice-id
|
||||
:invoice-payment/payment payment-id
|
||||
:invoice-payment/amount invoice-amount}
|
||||
{:db/id invoice-id
|
||||
:invoice/outstanding-balance 0.0
|
||||
:invoice/status :invoice-status/paid}])
|
||||
invoice-payments))
|
||||
(update 0 assoc
|
||||
:transaction/payment payment-id
|
||||
:transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/vendor vendor
|
||||
:transaction/location "A")
|
||||
(conj [:reset (:db/id transaction) :transaction/accounts
|
||||
[#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (:transaction/amount transaction))}]]))))
|
||||
|
||||
(defn extract-check-number [{:transaction/keys [description-original]}]
|
||||
|
||||
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
|
||||
(try
|
||||
(Integer/parseInt check-number)
|
||||
(catch NumberFormatException e
|
||||
nil))
|
||||
nil))
|
||||
|
||||
(defn find-expected-deposit [client-id amount date]
|
||||
(when date
|
||||
(-> (d/q
|
||||
'[:find ?ed
|
||||
:in $ ?c ?a ?d-start
|
||||
:where
|
||||
[?ed :expected-deposit/client ?c]
|
||||
(not [?ed :expected-deposit/status :expected-deposit-status/cleared])
|
||||
[?ed :expected-deposit/date ?d]
|
||||
[(>= ?d ?d-start)]
|
||||
[?ed :expected-deposit/total ?a2]
|
||||
[(auto-ap.utils/dollars= ?a2 ?a)]
|
||||
]
|
||||
(d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
|
||||
first
|
||||
first)))
|
||||
|
||||
|
||||
(defn categorize-transaction [transaction bank-account existing]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(cond (= :transaction-approval-status/suppressed (existing (:transaction/id transaction)))
|
||||
:suppressed
|
||||
|
||||
(existing (:transaction/id transaction))
|
||||
:extant
|
||||
|
||||
(not (:transaction/client transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/bank-account transaction))
|
||||
:error
|
||||
|
||||
(not (:transaction/id transaction))
|
||||
:error
|
||||
|
||||
(not= "POSTED" (:transaction/status transaction))
|
||||
:not-posted
|
||||
|
||||
(and (:bank-account/start-date bank-account)
|
||||
(not (t/after? (coerce/to-date-time (:transaction/date transaction))
|
||||
(-> bank-account :bank-account/start-date coerce/to-date-time))))
|
||||
:not-ready
|
||||
|
||||
:else
|
||||
:import)))
|
||||
|
||||
(defn transaction->txs [transaction bank-account apply-rules]
|
||||
(let [bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))]
|
||||
(into []
|
||||
(let [{:transaction/keys [amount id date]} transaction
|
||||
check-number (extract-check-number transaction)
|
||||
existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit (:db/id client) amount (coerce/to-date-time date)))]
|
||||
(cond->
|
||||
[(assoc transaction :transaction/approval-status :transaction-approval-status/unapproved)]
|
||||
check-number (update 0 #(assoc % :transaction/check-number check-number))
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils))))))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
(defn get-existing [bank-account]
|
||||
(log/info "looking up bank account data for" bank-account)
|
||||
(into {}
|
||||
(d/query {:query {:find ['?tid '?as2]
|
||||
:in ['$ '?ba]
|
||||
:where ['[?e :transaction/bank-account ?ba]
|
||||
'[?e :transaction/id ?tid]
|
||||
'[?e :transaction/approval-status ?as]
|
||||
'[?as :db/ident ?as2]]}
|
||||
:args [(d/db (d/connect uri)) bank-account]})))
|
||||
|
||||
(defprotocol ImportBatch
|
||||
(import-transaction! [this transaction])
|
||||
(get-stats [this ])
|
||||
(finish! [this])
|
||||
(fail! [this error]))
|
||||
|
||||
(defn start-import-batch [source user]
|
||||
(let [stats (atom {:import-batch/imported 0
|
||||
:import-batch/suppressed 0
|
||||
:import-batch/error 0
|
||||
:import-batch/not-ready 0
|
||||
:import-batch/extant 0})
|
||||
extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 ))
|
||||
import-id (get (:tempids @(d/transact conn [{:db/id "import-batch"
|
||||
:import-batch/date (coerce/to-date (t/now))
|
||||
:import-batch/source source
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch")
|
||||
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
|
||||
(log/info "Importing transactions from " source)
|
||||
|
||||
(reify ImportBatch
|
||||
(import-transaction! [this transaction]
|
||||
(let [bank-account (d/pull (d/db conn)
|
||||
[:bank-account/code
|
||||
:db/id
|
||||
:bank-account/locations
|
||||
:bank-account/start-date
|
||||
{:client/_bank-accounts [:client/code :client/locations :db/id]} ]
|
||||
(:transaction/bank-account transaction))
|
||||
extant (get (swap! extant-cache cache/through-cache (:transaction/bank-account transaction) get-existing)
|
||||
(:transaction/bank-account transaction))
|
||||
action (categorize-transaction transaction bank-account extant)
|
||||
transaction (assoc transaction :import-batch/_entry import-id)]
|
||||
(swap! stats
|
||||
#(update % (condp = action
|
||||
:import :import-batch/imported
|
||||
:extant :import-batch/extant
|
||||
:suppressed :import-batch/suppressed
|
||||
:error :import-batch/error
|
||||
:not-read :import-batch/not-ready) inc))
|
||||
(when (= :import action)
|
||||
(audit-transact (transaction->txs transaction bank-account rule-applying-function)
|
||||
{:user/name user
|
||||
:user/role ":admin"}))))
|
||||
|
||||
(get-stats [this]
|
||||
@stats)
|
||||
|
||||
(fail! [this error]
|
||||
(log/errorf "Couldn't complete import %d with error." import-id)
|
||||
(log/error error)
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
:import-batch/status :import-status/completed
|
||||
:import-batch/error-message (str error)}
|
||||
@stats)]))
|
||||
|
||||
(finish! [this]
|
||||
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
|
||||
@(d/transact conn [(merge {:db/id import-id
|
||||
|
||||
:import-batch/status :import-status/completed}
|
||||
@stats)])))))
|
||||
|
||||
|
||||
|
||||
(defn synthetic-key [{:transaction/keys [date bank-account description-original amount client] } index]
|
||||
(str (str (some-> date coerce/to-date-time auto-ap.time/localize)) "-" bank-account "-" description-original "-" amount "-" index "-" client))
|
||||
|
||||
(defn apply-synthetic-ids [transactions]
|
||||
(->> transactions
|
||||
(group-by #(select-keys % [:transaction/date :transaction/bank-account :transaction/description-original :transaction/amount :transaction/client]))
|
||||
(vals)
|
||||
(mapcat (fn [group]
|
||||
(map (fn [index transaction]
|
||||
(let [raw-id (synthetic-key transaction index)]
|
||||
(assoc transaction
|
||||
:transaction/id (digest/sha-256 raw-id)
|
||||
:transaction/raw-id raw-id)))
|
||||
(range)
|
||||
group)))))
|
||||
|
||||
70
src/clj/auto_ap/import/yodlee.clj
Normal file
70
src/clj/auto_ap/import/yodlee.clj
Normal file
@@ -0,0 +1,70 @@
|
||||
(ns auto-ap.import.yodlee
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.time :as atime]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.yodlee.core :as client]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clojure.string :as str]
|
||||
[datomic.api :as d]
|
||||
[digest :refer [sha-256]]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]
|
||||
[clojure.tools.logging :as log]))
|
||||
|
||||
(defn yodlee->transaction [transaction]
|
||||
(let [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
{amount :amount} :amount
|
||||
{description-original :original
|
||||
description-simple :simple} :description
|
||||
{merchant-id :id
|
||||
merchant-name :name} :merchant
|
||||
base-type :baseType
|
||||
type :type
|
||||
status :status} transaction
|
||||
amount (if (= "DEBIT" base-type)
|
||||
(- amount)
|
||||
amount)
|
||||
date (atime/parse date "YYYY-MM-dd")]
|
||||
#:transaction
|
||||
{:post-date (coerce/to-date (atime/parse post-date "YYYY-MM-dd"))
|
||||
:id (sha-256 (str id))
|
||||
:raw-id (str id)
|
||||
:account-id account-id
|
||||
:date (coerce/to-date date)
|
||||
:amount (double amount)
|
||||
:description-original (some-> description-original (str/replace #"\s+" " "))
|
||||
:description-simple (some-> description-simple (str/replace #"\s+" " "))
|
||||
:type type
|
||||
:status status}))
|
||||
|
||||
(defn import-yodlee []
|
||||
(lc/with-context {:source "Import yodlee transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")]
|
||||
(try
|
||||
(let [account-lookup (d/q '[:find ?ya ?ba ?c
|
||||
:in $
|
||||
:where [?ba :bank-account/yodlee-account-id ?ya]
|
||||
[?c :client/bank-accounts ?ba]]
|
||||
(d/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-id] account-lookup
|
||||
transaction (client/get-specific-transactions yodlee-account)]
|
||||
(log/info "importing")
|
||||
(t/import-transaction! import-batch (assoc (yodlee->transaction transaction)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client client-id)))
|
||||
|
||||
(t/finish! import-batch))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
(def import-yodlee (allow-once import-yodlee))
|
||||
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 4) import-yodlee)
|
||||
:stop (scheduler/stop import-worker))
|
||||
45
src/clj/auto_ap/import/yodlee2.clj
Normal file
45
src/clj/auto_ap/import/yodlee2.clj
Normal file
@@ -0,0 +1,45 @@
|
||||
(ns auto-ap.import.yodlee2
|
||||
(:require [auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.import.yodlee :as y]
|
||||
[auto-ap.utils :refer [allow-once]]
|
||||
[auto-ap.yodlee.core2 :as client2]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
|
||||
(defn import-yodlee2 []
|
||||
(lc/with-context {:source "Import yodlee2 transactions"}
|
||||
(let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
|
||||
(try
|
||||
(let [account-lookup (d/q '[:find ?ya ?ba ?cd
|
||||
:in $
|
||||
:where
|
||||
[?ba :bank-account/yodlee-account ?y]
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?c :client/code ?cd]
|
||||
[?y :yodlee-account/id ?ya]]
|
||||
(d/db conn))]
|
||||
(doseq [[yodlee-account bank-account client-code] account-lookup
|
||||
transaction (client2/get-specific-transactions client-code yodlee-account)]
|
||||
(t/import-transaction! import-batch (assoc (y/yodlee->transaction transaction)
|
||||
:transaction/bank-account bank-account
|
||||
:transaction/client [:client/code client-code])))
|
||||
|
||||
(t/finish! import-batch))
|
||||
(catch Exception e
|
||||
(t/fail! import-batch e))))))
|
||||
|
||||
|
||||
|
||||
(def import-yodlee2 (allow-once import-yodlee2))
|
||||
|
||||
|
||||
(mount/defstate import-worker
|
||||
:start (scheduler/every (* 1000 60 60 4) import-yodlee2)
|
||||
:stop (scheduler/stop import-worker))
|
||||
|
||||
(mount/defstate account-worker
|
||||
:start (scheduler/every (* 5 60 1000) client2/upsert-accounts)
|
||||
:stop (scheduler/stop account-worker))
|
||||
Reference in New Issue
Block a user