New import approach to capture the batch.
This commit is contained in:
@@ -172,85 +172,112 @@
|
||||
first
|
||||
first)))
|
||||
|
||||
(defn transactions->txs [transactions transaction->bank-account apply-rules existing]
|
||||
(into []
|
||||
(defn transactions->txs [transactions import-id transaction->bank-account apply-rules existing]
|
||||
(let [grouped-transactions (group-by
|
||||
(fn [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
status :status :as transaction}]
|
||||
(let [bank-account (transaction->bank-account transaction)
|
||||
bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
||||
|
||||
date (time/parse date "YYYY-MM-dd")]
|
||||
|
||||
(for [transaction transactions
|
||||
:let [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
{amount :amount} :amount
|
||||
{description-original :original
|
||||
description-simple :simple} :description
|
||||
{merchant-id :id
|
||||
merchant-name :name} :merchant
|
||||
base-type :baseType
|
||||
type :type
|
||||
status :status}
|
||||
transaction
|
||||
amount (if (= "DEBIT" base-type)
|
||||
(- amount)
|
||||
amount)
|
||||
check-number (extract-check-number transaction)
|
||||
bank-account (transaction->bank-account transaction)
|
||||
bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
||||
|
||||
date (time/parse date "YYYY-MM-dd")]
|
||||
:when (and client-id
|
||||
(not (existing (sha-256 (str id))))
|
||||
(= "POSTED" status)
|
||||
(cond (existing (sha-256 (str id)))
|
||||
:extant
|
||||
|
||||
(or (not (:start-date bank-account))
|
||||
(t/after? date (:start-date bank-account))))]
|
||||
(let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id ))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit client-id amount date))]
|
||||
(cond->
|
||||
[#:transaction
|
||||
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
|
||||
:id (sha-256 (str id))
|
||||
:raw-id (str id)
|
||||
:account-id account-id
|
||||
:date (coerce/to-date date)
|
||||
:amount (double amount)
|
||||
:description-original (some-> description-original (str/replace #"\s+" " "))
|
||||
:description-simple (some-> description-simple (str/replace #"\s+" " "))
|
||||
:approval-status :transaction-approval-status/unapproved
|
||||
:type type
|
||||
:status status
|
||||
:client client-id
|
||||
:check-number check-number
|
||||
:bank-account bank-account-id}]
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
(not client-id)
|
||||
:error
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
(not= "POSTED" status)
|
||||
:not-posted
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils))))))
|
||||
(and (:start-date bank-account)
|
||||
(not (t/after? date (:start-date bank-account))))
|
||||
:not-ready
|
||||
|
||||
:else
|
||||
:import)))
|
||||
transactions)]
|
||||
|
||||
(update grouped-transactions :import
|
||||
(fn [transactions]
|
||||
(into []
|
||||
|
||||
(for [transaction transactions
|
||||
:let [{post-date :postDate
|
||||
account-id :accountId
|
||||
date :date
|
||||
id :id
|
||||
{amount :amount} :amount
|
||||
{description-original :original
|
||||
description-simple :simple} :description
|
||||
{merchant-id :id
|
||||
merchant-name :name} :merchant
|
||||
base-type :baseType
|
||||
type :type
|
||||
status :status}
|
||||
transaction
|
||||
amount (if (= "DEBIT" base-type)
|
||||
(- amount)
|
||||
amount)
|
||||
check-number (extract-check-number transaction)
|
||||
bank-account (transaction->bank-account transaction)
|
||||
bank-account-id (:db/id bank-account)
|
||||
client (:client/_bank-accounts bank-account)
|
||||
client-id (:db/id client)
|
||||
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
|
||||
date (time/parse date "YYYY-MM-dd")]]
|
||||
(let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
|
||||
autopay-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unfulfilled-autopayments amount client-id ))
|
||||
unpaid-invoices-matches (when-not existing-check
|
||||
(match-transaction-to-unpaid-invoices amount client-id ))
|
||||
expected-deposit (when (and (> amount 0.0)
|
||||
(not existing-check))
|
||||
(find-expected-deposit client-id amount date))]
|
||||
(cond->
|
||||
[#:transaction
|
||||
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
|
||||
:id (sha-256 (str id))
|
||||
:raw-id (str id)
|
||||
:account-id account-id
|
||||
:date (coerce/to-date date)
|
||||
:amount (double amount)
|
||||
:import-batch/_entry import-id
|
||||
:description-original (some-> description-original (str/replace #"\s+" " "))
|
||||
:description-simple (some-> description-simple (str/replace #"\s+" " "))
|
||||
:approval-status :transaction-approval-status/unapproved
|
||||
:type type
|
||||
:status status
|
||||
:client client-id
|
||||
:check-number check-number
|
||||
:bank-account bank-account-id}]
|
||||
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
|
||||
:transaction/payment {:db/id (:db/id existing-check)
|
||||
:payment/status :payment-status/cleared}
|
||||
:transaction/vendor (:db/id (:payment/vendor existing-check))
|
||||
:transaction/location "A"
|
||||
:transaction/accounts [#:transaction-account
|
||||
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
||||
:location "A"
|
||||
:amount (Math/abs (double amount))}]))
|
||||
|
||||
;; temporarily removed to automatically match autopaid invoices
|
||||
#_(and (not existing-check)
|
||||
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
|
||||
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
|
||||
:expected-deposit/status :expected-deposit-status/cleared}))
|
||||
|
||||
|
||||
(and (not (seq autopay-invoices-matches))
|
||||
(not (seq unpaid-invoices-matches))
|
||||
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
|
||||
true (update 0 remove-nils)))))))))
|
||||
|
||||
|
||||
(defn batch-transact [transactions]
|
||||
@@ -282,40 +309,22 @@
|
||||
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
|
||||
(assoc bank-account
|
||||
:client/_bank-accounts client))))))))
|
||||
(defn start-import [source user]
|
||||
(get (:tempids @(d/transact conn [{:db/id "import-batch"
|
||||
:import-batch/date (coerce/to-date (t/now))
|
||||
:import-batch/source source
|
||||
:import-batch/status :import-status/started
|
||||
:import-batch/user-name user}])) "import-batch"))
|
||||
|
||||
(defn manual-import [manual-transactions]
|
||||
(lc/with-context {:source "manual import"}
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
|
||||
:bank-account-id bank-account-id
|
||||
:date (time/unparse date "YYYY-MM-dd")
|
||||
:amount {:amount amount}
|
||||
:description {:original description-original
|
||||
:simple high-level-category}
|
||||
:status "POSTED"})
|
||||
(range)
|
||||
transaction-group))))
|
||||
all-rules (tr/get-all)
|
||||
all-bank-accounts (by :db/id (get-all-bank-accounts))
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
|
||||
(log/info "Importing " (count transformed-transactions) " manual transactions")
|
||||
|
||||
(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(audit-transact tx {:user/name "Yodlee import"
|
||||
:user/role ":admin"}))
|
||||
(log/info "Imported manual transactions"))))
|
||||
(defn finish-import [i]
|
||||
@(d/transact conn [(assoc i :import-batch/status :import-status/completed)]))
|
||||
|
||||
(defn grouped-import [manual-transactions]
|
||||
|
||||
(defn grouped-import [manual-transactions import-id]
|
||||
(lc/with-context {:source "grouped import"}
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount]))
|
||||
(group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
@@ -331,32 +340,15 @@
|
||||
transaction-group))))
|
||||
all-rules (tr/get-all)
|
||||
all-bank-accounts (by :db/id (get-all-bank-accounts))
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
|
||||
(log/info "Importing " (count transformed-transactions) " grouped transactions")
|
||||
transaction->bank-account (comp all-bank-accounts :bank-account-id)
|
||||
transactions (transactions->txs transformed-transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(log/info "Importing " (count (:import transformed-transactions)) " grouped transactions")
|
||||
|
||||
#_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(doseq [tx (:import transactions)]
|
||||
(audit-transact tx {:user/name "Yodlee import"
|
||||
:user/role ":admin"}))
|
||||
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))
|
||||
(log/info "Imported grouped transactions"))))
|
||||
|
||||
(defn grouped-new [manual-transactions]
|
||||
(let [transformed-transactions (->> manual-transactions
|
||||
(filter #(= "posted" (:status %)))
|
||||
(group-by #(select-keys % [:date :description-original :amount]))
|
||||
(vals)
|
||||
(mapcat (fn [transaction-group]
|
||||
(map
|
||||
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
|
||||
(assoc transaction :id
|
||||
(str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)))
|
||||
(range)
|
||||
transaction-group))))
|
||||
existing (get-existing)]
|
||||
(map #(if (existing (sha-256 (str (:id %))))
|
||||
(assoc % :new? false)
|
||||
(assoc % :new? true))
|
||||
transformed-transactions)))
|
||||
(log/info "Imported grouped transactions")
|
||||
transactions)))
|
||||
|
||||
(defn do-import
|
||||
([]
|
||||
@@ -365,12 +357,18 @@
|
||||
(lc/with-context {:source "Import yodlee transactions"}
|
||||
(do
|
||||
(log/info "importing from yodlee")
|
||||
(let [all-bank-accounts (get-all-bank-accounts)
|
||||
transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
|
||||
all-rules (tr/get-all)]
|
||||
(doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(audit-transact tx {:user/name "Yodlee import"
|
||||
:user/role ":admin"})))))))
|
||||
(let [import-id (start-import :import-source/yodlee "Automated Yodlee User")
|
||||
all-bank-accounts (get-all-bank-accounts)
|
||||
transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
|
||||
all-rules (tr/get-all)
|
||||
transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(doseq [tx (:import transactions)]
|
||||
(audit-transact tx {:user/name "Yodlee import"
|
||||
:user/role ":admin"}))
|
||||
(finish-import {:db/id import-id
|
||||
:import-batch/imported (count (:import transactions))
|
||||
:import-batch/extant (count (:extant transactions))
|
||||
:import-batch/suppressed (count (:suppresed transactions))}))))))
|
||||
|
||||
(defn do-import2
|
||||
([]
|
||||
@@ -380,15 +378,20 @@
|
||||
|
||||
(do
|
||||
(log/info "importing from yodlee2")
|
||||
(let [all-bank-accounts (get-all-bank-accounts)
|
||||
(let [import-id (start-import :import-source/yodlee "Automated Yodlee User")
|
||||
all-bank-accounts (get-all-bank-accounts)
|
||||
transaction->bank-account (comp (by (comp :yodlee-account/id :bank-account/yodlee-account) all-bank-accounts) :accountId)
|
||||
all-rules (tr/get-all)]
|
||||
all-rules (tr/get-all)
|
||||
transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
|
||||
(log/info "COUNT" (count (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))))
|
||||
(doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
|
||||
(doseq [tx (:import transactions)]
|
||||
|
||||
(audit-transact tx {:user/name "Yodlee import2"
|
||||
:user/role ":admin"})))))))
|
||||
:user/role ":admin"}))
|
||||
(finish-import {:db/id import-id
|
||||
:import-batch/imported (count (:import transactions))
|
||||
:import-batch/extant (count (:extant transactions))
|
||||
:import-batch/suppressed (count (:suppresed transactions))}))))))
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user