notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

New import approach to capture the batch.

Browse Source
This commit is contained in:
Bryce Covert
2021-12-15 20:35:37 -08:00
parent b7324af538
commit 26a949e63f
12 changed files with 335 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/clj/auto_ap/datomic/migrate.clj
View File

@@ -378,7 +378,51 @@
:db/doc "An unhashed version of the id" :db/doc "An unhashed version of the id"
:db/valueType :db.type/string :db/valueType :db.type/string
:db/cardinality :db.cardinality/one}]] :db/cardinality :db.cardinality/one}]]
:depends-on [:auto-ap/base-schema]}} :depends-on [:auto-ap/base-schema]}
:auto-ap/add-transaction-import2 {:txes [[{:db/ident :import-batch/external-id
:db/doc "An identifier for the import batch"
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/unique :db.unique/identity}
{:db/ident :import-batch/entry
:db/doc "Links to everything that was imported"
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
{:db/ident :import-batch/imported
:db/doc "How many entries were imported"
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one}
{:db/ident :import-batch/extant
:db/doc "How many entries were already present"
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one}
{:db/ident :import-batch/suppressed
:db/doc "How many entries were explicitly ignored"
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one}
{:db/ident :import-batch/date
:db/doc "The date the import occurred"
:db/valueType :db.type/instant
:db/cardinality :db.cardinality/one}
{:db/ident :import-batch/source
:db/doc "What system triggered the import"
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one}
{:db/ident :import-batch/status
:db/doc "What system triggered the import"
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one}
{:db/ident :import-source/intuit}
{:db/ident :import-source/manual}
{:db/ident :import-source/yodlee}
{:db/ident :import-source/yodlee2}
{:db/ident :import-status/started}
{:db/ident :import-status/completed}
{:db/ident :import-batch/user-name
:db/doc "Who triggred this import"
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one}]]
:depends-on [:auto-ap/base-schema]}}

5
src/clj/auto_ap/datomic/transactions.clj
View File

@@ -43,6 +43,11 @@
:where ['[?e :transaction/bank-account ?bank-account-id]]} :where ['[?e :transaction/bank-account ?bank-account-id]]}
:args [(:bank-account-id args)]}) :args [(:bank-account-id args)]})
(:import-batch-id args)
(merge-query {:query {:in ['?import-batch-id]
:where ['[?import-batch-id :import-batch/entry ?e]]}
:args [(:import-batch-id args)]})
(:account-id args) (:account-id args)
(merge-query {:query {:in ['?account-id] (merge-query {:query {:in ['?account-id]
:where ['[?e :transaction/accounts ?accounts] :where ['[?e :transaction/accounts ?accounts]

101
src/clj/auto_ap/graphql.clj
View File

@@ -1,37 +1,40 @@
(ns auto-ap.graphql (ns auto-ap.graphql
(:require [auto-ap.datomic :refer [merge-query uri]] (:require
[auto-ap.datomic.checks :as d-checks] [auto-ap.datomic :refer [merge-query uri]]
[auto-ap.datomic.sales-orders :as d-sales-orders] [auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.users :as d-users] [auto-ap.datomic.sales-orders :as d-sales-orders]
[auto-ap.graphql.accounts :as gq-accounts] [auto-ap.datomic.users :as d-users]
[auto-ap.graphql.checks :as gq-checks] [auto-ap.graphql.accounts :as gq-accounts]
[auto-ap.graphql.clients :as gq-clients] [auto-ap.graphql.checks :as gq-checks]
[auto-ap.graphql.expected-deposit :as gq-expected-deposit] [auto-ap.graphql.clients :as gq-clients]
[auto-ap.graphql.invoices :as gq-invoices] [auto-ap.graphql.expected-deposit :as gq-expected-deposit]
[auto-ap.graphql.ledger :as gq-ledger] [auto-ap.graphql.import-batch :as gq-import-batches]
[auto-ap.graphql.sales-orders :as gq-sales-orders] [auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts]
[auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts] [auto-ap.graphql.invoices :as gq-invoices]
[auto-ap.graphql.transaction-rules :as gq-transaction-rules] [auto-ap.graphql.ledger :as gq-ledger]
[auto-ap.graphql.transactions :as gq-transactions] [auto-ap.graphql.sales-orders :as gq-sales-orders]
[auto-ap.graphql.users :as gq-users] [auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] [auto-ap.graphql.transactions :as gq-transactions]
[auto-ap.graphql.vendors :as gq-vendors] [auto-ap.graphql.users :as gq-users]
[auto-ap.graphql.yodlee-merchants :as ym] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
[auto-ap.graphql.yodlee2 :as gq-yodlee2] [auto-ap.graphql.vendors :as gq-vendors]
[auto-ap.logging :refer [error-event info-event warn-event]] [auto-ap.graphql.yodlee-merchants :as ym]
[auto-ap.time :as time] [auto-ap.graphql.yodlee2 :as gq-yodlee2]
[clj-time.coerce :as coerce] [auto-ap.logging :refer [error-event info-event warn-event]]
[clj-time.core :as t] [auto-ap.time :as time]
[clojure.string :as str] [clj-time.coerce :as coerce]
[clojure.tools.logging :as log] [clj-time.core :as t]
[clojure.walk :as walk] [clojure.string :as str]
[com.walmartlabs.lacinia :refer [execute]] [clojure.tools.logging :as log]
[com.walmartlabs.lacinia.schema :as schema] [clojure.walk :as walk]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]] [com.walmartlabs.lacinia :refer [execute]]
[datomic.api :as d] [com.walmartlabs.lacinia.schema :as schema]
[unilog.context :as lc] [com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[yang.time :refer [time-it]]) [datomic.api :as d]
(:import clojure.lang.IPersistentMap)) [unilog.context :as lc]
[yang.time :refer [time-it]])
(:import
(clojure.lang IPersistentMap)))
(def integreat-schema (def integreat-schema
{ {
@@ -452,6 +455,21 @@
:start {:type 'Int} :start {:type 'Int}
:end {:type 'Int}}} :end {:type 'Int}}}
:import_batch {:fields {:user_name {:type 'String}
:id {:type :id}
:date {:type :iso_date}
:imported {:type 'Int}
:extant {:type 'Int}
:suppressed {:type 'Int}
:status {:type :import_batch_status}
:source {:type :import_batch_source}}}
:import_batch_page {:fields {:data {:type '(list :import_batch)}
:count {:type 'Int}
:total {:type 'Int}
:start {:type 'Int}
:end {:type 'Int}}}
:transaction_rule_page {:fields {:transaction_rules {:type '(list :transaction_rule)} :transaction_rule_page {:fields {:transaction_rules {:type '(list :transaction_rule)}
:count {:type 'Int} :count {:type 'Int}
:total {:type 'Int} :total {:type 'Int}
@@ -633,6 +651,11 @@
:resolve :get-transaction-page} :resolve :get-transaction-page}
:import_batch_page {:type :import_batch_page
:args {:filters {:type :import_batch_filters}}
:resolve :get-import-batch-page}
:transaction_rule_page {:type :transaction_rule_page :transaction_rule_page {:type :transaction_rule_page
:args {:client_id {:type :id} :args {:client_id {:type :id}
:vendor_id {:type :id} :vendor_id {:type :id}
@@ -706,6 +729,7 @@
:transaction_filters {:fields {:client_id {:type :id} :transaction_filters {:fields {:client_id {:type :id}
:exact_match_id {:type :id} :exact_match_id {:type :id}
:import_batch_id {:type :id}
:vendor_id {:type :id} :vendor_id {:type :id}
:bank_account_id {:type :id} :bank_account_id {:type :id}
@@ -720,6 +744,9 @@
:sort {:type '(list :sort_item)} :sort {:type '(list :sort_item)}
:approval_status {:type :transaction_approval_status} :approval_status {:type :transaction_approval_status}
:unresolved {:type 'Boolean}}} :unresolved {:type 'Boolean}}}
:import_batch_filters {:fields {:start {:type 'Int}
:per_page {:type 'Int}
:sort {:type '(list :sort_item)}}}
:ledger_filters {:fields {:client_id {:type :id} :ledger_filters {:fields {:client_id {:type :id}
:vendor_id {:type :id} :vendor_id {:type :id}
@@ -935,6 +962,13 @@
{:enum-value :cash} {:enum-value :cash}
{:enum-value :debit}]} {:enum-value :debit}]}
:import_batch_source {:values [{:enum-value :intuit}
{:enum-value :yodlee}
{:enum-value :yodlee2}
{:enum-value :manual}]}
:import_batch_status {:values [{:enum-value :started}
{:enum-value :completed}]}
:processor {:values [{:enum-value :na} :processor {:values [{:enum-value :na}
{:enum-value :doordash} {:enum-value :doordash}
{:enum-value :uber_eats} {:enum-value :uber_eats}
@@ -1328,6 +1362,7 @@
:get-potential-unpaid-invoices-matches gq-transactions/get-potential-unpaid-invoices-matches :get-potential-unpaid-invoices-matches gq-transactions/get-potential-unpaid-invoices-matches
:get-accounts gq-accounts/get-accounts :get-accounts gq-accounts/get-accounts
:get-transaction-page gq-transactions/get-transaction-page :get-transaction-page gq-transactions/get-transaction-page
:get-import-batch-page gq-import-batches/get-import-batch-page
:get-ledger-page gq-ledger/get-ledger-page :get-ledger-page gq-ledger/get-ledger-page
:get-sales-order-page gq-sales-orders/get-sales-orders-page :get-sales-order-page gq-sales-orders/get-sales-orders-page
:get-balance-sheet gq-ledger/get-balance-sheet :get-balance-sheet gq-ledger/get-balance-sheet

23
src/clj/auto_ap/intuit/import.clj
View File

@@ -22,12 +22,16 @@
db))) db)))
(defn upsert-transactions [] (defn upsert-transactions []
(let [db (d/db conn)] (let [db (d/db conn)
import-id (y/start-import :import-source/intuit "Automated Intuit User")
stats (atom {:import-batch/imported 0
:import-batch/extant 0
:import-batch/suppressed 0})]
(doseq [[bank-account external-id] (get-intuit-bank-accounts db) (doseq [[bank-account external-id] (get-intuit-bank-accounts db)
:let [bank-account (d/entity db bank-account) :let [bank-account (d/entity db bank-account)
end (auto-ap.time/local-now) end (auto-ap.time/local-now)
start (time/plus end (time/days -30))] start (time/plus end (time/days -30))]]
]
(log/infof "importing from %s to %s for %s" start end external-id) (log/infof "importing from %s to %s for %s" start end external-id)
(let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) (let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
(auto-ap.time/unparse end auto-ap.time/iso-date) (auto-ap.time/unparse end auto-ap.time/iso-date)
@@ -40,8 +44,13 @@
:date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
:status "posted"})))] :status "posted"})))]
(log/infof "%d transactions found" (count transactions)) (log/infof "%d transactions found" (count transactions))
#_(y/grouped-import transactions) (let [result (y/grouped-import transactions import-id)]
transactions)))) (swap! stats
(fn [s]
(merge-with + s {:import-batch/imported (count (:import result))
:import-batch/extant (count (:extant result))
:import-batch/suppressed (count (:suppressed result))}))))))
(y/finish-import (assoc @stats :db/id import-id))))
(defn dry-run-upsert-transactions [] (defn dry-run-upsert-transactions []
@@ -55,9 +64,7 @@
start (time/plus end (time/days -30))] start (time/plus end (time/days -30))]
r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
(auto-ap.time/unparse end auto-ap.time/iso-date) (auto-ap.time/unparse end auto-ap.time/iso-date)
external-id) external-id)]
]
{:client-id (:db/id (:client/_bank-accounts bank-account)) {:client-id (:db/id (:client/_bank-accounts bank-account))
:client-code (:client/code (:client/_bank-accounts bank-account)) :client-code (:client/code (:client/_bank-accounts bank-account))
:bank-account-id (:db/id bank-account) :bank-account-id (:db/id bank-account)

63
src/clj/auto_ap/routes/invoices.clj
View File

@@ -11,7 +11,7 @@
[auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.routes.utils :refer [wrap-secure]]
[auto-ap.time-utils :refer [next-dom]] [auto-ap.time-utils :refer [next-dom]]
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
[auto-ap.yodlee.import :refer [manual-import]] [auto-ap.yodlee.import :refer [grouped-import] :as y]
[clj-time.coerce :as coerce :refer [to-date]] [clj-time.coerce :as coerce :refer [to-date]]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
@@ -439,36 +439,41 @@
(try (try
#_(throw (Exception. "Unexpected error")) #_(throw (Exception. "Unexpected error"))
(let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code] (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code]
all-clients (d-clients/get-all) all-clients (d-clients/get-all)
all-bank-accounts (mapcat :client/bank-accounts all-clients) all-bank-accounts (mapcat :client/bank-accounts all-clients)
all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) all-clients (merge (by :client/code all-clients) (by :client/name all-clients))
all-bank-accounts (merge (by :bank-account/code all-bank-accounts)) all-bank-accounts (merge (by :bank-account/code all-bank-accounts))
rows (->> (str/split data #"\n" ) rows (->> (str/split data #"\n" )
(drop 1) (drop 1)
(map #(str/split % #"\t")) (map #(str/split % #"\t"))
(map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns))))
(map (parse-or-error :amount parse-amount)) (map (parse-or-error :amount parse-amount))
(map (parse-or-error :date parse-date)) (map (parse-or-error :date parse-date))
(map (fn [{:keys [bank-account-code] :as row}] (map (fn [{:keys [bank-account-code] :as row}]
(if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))] (if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))]
(assoc row :bank-account-id bank-account-id) (assoc row :bank-account-id bank-account-id)
(update row :errors conj {:info (str "Cannot find bank by code " bank-account-code) (update row :errors conj {:info (str "Cannot find bank by code " bank-account-code)
:details (str "Cannot find bank by code " bank-account-code)})))) :details (str "Cannot find bank by code " bank-account-code)}))))
(map (fn [{:keys [client-code] :as row}] (map (fn [{:keys [client-code] :as row}]
(if-let [client-id (:db/id (all-clients client-code))] (if-let [client-id (:db/id (all-clients client-code))]
(assoc row :client-id client-id) (assoc row :client-id client-id)
(update row :errors conj {:info (str "Cannot find client by code " client-code) (update row :errors conj {:info (str "Cannot find client by code " client-code)
:details (str "Cannot find client by code " client-code)}))))) :details (str "Cannot find client by code " client-code)})))))
error-rows (filter :errors rows) error-rows (filter :errors rows)
_ (log/info "Importing " (count rows) "raw transactions") _ (log/info "Importing " (count rows) "raw transactions")
raw-transactions (vec (->> rows raw-transactions (vec (->> rows
(filter #(not (seq (:errors %))) ) (filter #(not (seq (:errors %))) )
(map (fn [row] (map (fn [row]
(select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id])))))] (select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id])))))
import-id (y/start-import :import-source/manual (:user/name user))
(manual-import raw-transactions) result (grouped-import raw-transactions import-id)]
(y/finish-import {:db/id import-id
:import-batch/imported (count (:import result))
:import-batch/suppressed (count (:suppressed result))
:import-batch/extant (count (:extant result))})
{:status 200 {:status 200
:body (pr-str {:imported (count raw-transactions) :body (pr-str {:imported (count (:import result))
:errors (map #(dissoc % :date) error-rows)}) :errors (map #(dissoc % :date) error-rows)})
:headers {"Content-Type" "application/edn"}}) :headers {"Content-Type" "application/edn"}})
(catch Exception e (catch Exception e

279
src/clj/auto_ap/yodlee/import.clj
View File

@@ -172,85 +172,112 @@
first first
first))) first)))
(defn transactions->txs [transactions transaction->bank-account apply-rules existing] (defn transactions->txs [transactions import-id transaction->bank-account apply-rules existing]
(into [] (let [grouped-transactions (group-by
(fn [{post-date :postDate
account-id :accountId
date :date
id :id
status :status :as transaction}]
(let [bank-account (transaction->bank-account transaction)
bank-account-id (:db/id bank-account)
client (:client/_bank-accounts bank-account)
client-id (:db/id client)
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
date (time/parse date "YYYY-MM-dd")]
(for [transaction transactions (cond (existing (sha-256 (str id)))
:let [{post-date :postDate :extant
account-id :accountId
date :date
id :id
{amount :amount} :amount
{description-original :original
description-simple :simple} :description
{merchant-id :id
merchant-name :name} :merchant
base-type :baseType
type :type
status :status}
transaction
amount (if (= "DEBIT" base-type)
(- amount)
amount)
check-number (extract-check-number transaction)
bank-account (transaction->bank-account transaction)
bank-account-id (:db/id bank-account)
client (:client/_bank-accounts bank-account)
client-id (:db/id client)
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
date (time/parse date "YYYY-MM-dd")]
:when (and client-id
(not (existing (sha-256 (str id))))
(= "POSTED" status)
(or (not (:start-date bank-account)) (not client-id)
(t/after? date (:start-date bank-account))))] :error
(let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
autopay-invoices-matches (when-not existing-check
(match-transaction-to-unfulfilled-autopayments amount client-id ))
unpaid-invoices-matches (when-not existing-check
(match-transaction-to-unpaid-invoices amount client-id ))
expected-deposit (when (and (> amount 0.0)
(not existing-check))
(find-expected-deposit client-id amount date))]
(cond->
[#:transaction
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
:id (sha-256 (str id))
:raw-id (str id)
:account-id account-id
:date (coerce/to-date date)
:amount (double amount)
:description-original (some-> description-original (str/replace #"\s+" " "))
:description-simple (some-> description-simple (str/replace #"\s+" " "))
:approval-status :transaction-approval-status/unapproved
:type type
:status status
:client client-id
:check-number check-number
:bank-account bank-account-id}]
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
:transaction/payment {:db/id (:db/id existing-check)
:payment/status :payment-status/cleared}
:transaction/vendor (:db/id (:payment/vendor existing-check))
:transaction/location "A"
:transaction/accounts [#:transaction-account
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:location "A"
:amount (Math/abs (double amount))}]))
;; temporarily removed to automatically match autopaid invoices (not= "POSTED" status)
#_(and (not existing-check) :not-posted
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
:expected-deposit/status :expected-deposit-status/cleared}))
(and (not (seq autopay-invoices-matches)) (and (:start-date bank-account)
(not (seq unpaid-invoices-matches)) (not (t/after? date (:start-date bank-account))))
(not expected-deposit)) (update 0 #(apply-rules % valid-locations)) :not-ready
true (update 0 remove-nils))))))
:else
:import)))
transactions)]
(update grouped-transactions :import
(fn [transactions]
(into []
(for [transaction transactions
:let [{post-date :postDate
account-id :accountId
date :date
id :id
{amount :amount} :amount
{description-original :original
description-simple :simple} :description
{merchant-id :id
merchant-name :name} :merchant
base-type :baseType
type :type
status :status}
transaction
amount (if (= "DEBIT" base-type)
(- amount)
amount)
check-number (extract-check-number transaction)
bank-account (transaction->bank-account transaction)
bank-account-id (:db/id bank-account)
client (:client/_bank-accounts bank-account)
client-id (:db/id client)
valid-locations (or (:bank-account/locations bank-account) (:client/locations client))
date (time/parse date "YYYY-MM-dd")]]
(let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id)
autopay-invoices-matches (when-not existing-check
(match-transaction-to-unfulfilled-autopayments amount client-id ))
unpaid-invoices-matches (when-not existing-check
(match-transaction-to-unpaid-invoices amount client-id ))
expected-deposit (when (and (> amount 0.0)
(not existing-check))
(find-expected-deposit client-id amount date))]
(cond->
[#:transaction
{:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd"))
:id (sha-256 (str id))
:raw-id (str id)
:account-id account-id
:date (coerce/to-date date)
:amount (double amount)
:import-batch/_entry import-id
:description-original (some-> description-original (str/replace #"\s+" " "))
:description-simple (some-> description-simple (str/replace #"\s+" " "))
:approval-status :transaction-approval-status/unapproved
:type type
:status status
:client client-id
:check-number check-number
:bank-account bank-account-id}]
existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved
:transaction/payment {:db/id (:db/id existing-check)
:payment/status :payment-status/cleared}
:transaction/vendor (:db/id (:payment/vendor existing-check))
:transaction/location "A"
:transaction/accounts [#:transaction-account
{:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
:location "A"
:amount (Math/abs (double amount))}]))
;; temporarily removed to automatically match autopaid invoices
#_(and (not existing-check)
(seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id)
expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit
:expected-deposit/status :expected-deposit-status/cleared}))
(and (not (seq autopay-invoices-matches))
(not (seq unpaid-invoices-matches))
(not expected-deposit)) (update 0 #(apply-rules % valid-locations))
true (update 0 remove-nils)))))))))
(defn batch-transact [transactions] (defn batch-transact [transactions]
@@ -282,40 +309,22 @@
(map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}] (map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}]
(assoc bank-account (assoc bank-account
:client/_bank-accounts client)))))))) :client/_bank-accounts client))))))))
(defn start-import [source user]
(get (:tempids @(d/transact conn [{:db/id "import-batch"
:import-batch/date (coerce/to-date (t/now))
:import-batch/source source
:import-batch/status :import-status/started
:import-batch/user-name user}])) "import-batch"))
(defn manual-import [manual-transactions] (defn finish-import [i]
(lc/with-context {:source "manual import"} @(d/transact conn [(assoc i :import-batch/status :import-status/completed)]))
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount]))
(vals)
(mapcat (fn [transaction-group]
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount}
:description {:original description-original
:simple high-level-category}
:status "POSTED"})
(range)
transaction-group))))
all-rules (tr/get-all)
all-bank-accounts (by :db/id (get-all-bank-accounts))
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
(log/info "Importing " (count transformed-transactions) " manual transactions")
(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
(audit-transact tx {:user/name "Yodlee import"
:user/role ":admin"}))
(log/info "Imported manual transactions"))))
(defn grouped-import [manual-transactions]
(defn grouped-import [manual-transactions import-id]
(lc/with-context {:source "grouped import"} (lc/with-context {:source "grouped import"}
(let [transformed-transactions (->> manual-transactions (let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %))) (filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount])) (group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id]))
(vals) (vals)
(mapcat (fn [transaction-group] (mapcat (fn [transaction-group]
(map (map
@@ -331,32 +340,15 @@
transaction-group)))) transaction-group))))
all-rules (tr/get-all) all-rules (tr/get-all)
all-bank-accounts (by :db/id (get-all-bank-accounts)) all-bank-accounts (by :db/id (get-all-bank-accounts))
transaction->bank-account (comp all-bank-accounts :bank-account-id)] transaction->bank-account (comp all-bank-accounts :bank-account-id)
(log/info "Importing " (count transformed-transactions) " grouped transactions") transactions (transactions->txs transformed-transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
(log/info "Importing " (count (:import transformed-transactions)) " grouped transactions")
#_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] (doseq [tx (:import transactions)]
(audit-transact tx {:user/name "Yodlee import" (audit-transact tx {:user/name "Yodlee import"
:user/role ":admin"})) :user/role ":admin"}))
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)) (log/info "Imported grouped transactions")
(log/info "Imported grouped transactions")))) transactions)))
(defn grouped-new [manual-transactions]
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount]))
(vals)
(mapcat (fn [transaction-group]
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
(assoc transaction :id
(str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)))
(range)
transaction-group))))
existing (get-existing)]
(map #(if (existing (sha-256 (str (:id %))))
(assoc % :new? false)
(assoc % :new? true))
transformed-transactions)))
(defn do-import (defn do-import
([] ([]
@@ -365,12 +357,18 @@
(lc/with-context {:source "Import yodlee transactions"} (lc/with-context {:source "Import yodlee transactions"}
(do (do
(log/info "importing from yodlee") (log/info "importing from yodlee")
(let [all-bank-accounts (get-all-bank-accounts) (let [import-id (start-import :import-source/yodlee "Automated Yodlee User")
transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId) all-bank-accounts (get-all-bank-accounts)
all-rules (tr/get-all)] transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId)
(doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] all-rules (tr/get-all)
(audit-transact tx {:user/name "Yodlee import" transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
:user/role ":admin"}))))))) (doseq [tx (:import transactions)]
(audit-transact tx {:user/name "Yodlee import"
:user/role ":admin"}))
(finish-import {:db/id import-id
:import-batch/imported (count (:import transactions))
:import-batch/extant (count (:extant transactions))
:import-batch/suppressed (count (:suppresed transactions))}))))))
(defn do-import2 (defn do-import2
([] ([]
@@ -380,15 +378,20 @@
(do (do
(log/info "importing from yodlee2") (log/info "importing from yodlee2")
(let [all-bank-accounts (get-all-bank-accounts) (let [import-id (start-import :import-source/yodlee "Automated Yodlee User")
all-bank-accounts (get-all-bank-accounts)
transaction->bank-account (comp (by (comp :yodlee-account/id :bank-account/yodlee-account) all-bank-accounts) :accountId) transaction->bank-account (comp (by (comp :yodlee-account/id :bank-account/yodlee-account) all-bank-accounts) :accountId)
all-rules (tr/get-all)] all-rules (tr/get-all)
transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
(log/info "COUNT" (count (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))) (doseq [tx (:import transactions)]
(doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
(audit-transact tx {:user/name "Yodlee import2" (audit-transact tx {:user/name "Yodlee import2"
:user/role ":admin"}))))))) :user/role ":admin"}))
(finish-import {:db/id import-id
:import-batch/imported (count (:import transactions))
:import-batch/extant (count (:extant transactions))
:import-batch/suppressed (count (:suppresed transactions))}))))))

12
src/clj/user.clj
View File

@@ -566,7 +566,7 @@
(str (UUID/randomUUID))) (str (UUID/randomUUID)))
expected-deposit-id (or (first (find-queries [client-code "expected-deposit"])) expected-deposit-id (or (first (find-queries [client-code "expected-deposit"]))
(str (UUID/randomUUID))) (str (UUID/randomUUID)))
tender-id (or (first (find-queries [client-code "charge"])) tender-id (or (first (find-queries [client-code ":charge"]))
(str (UUID/randomUUID))) (str (UUID/randomUUID)))
refund-id (or (first (find-queries [client-code "sales-refund"])) refund-id (or (first (find-queries [client-code "sales-refund"]))
(str (UUID/randomUUID)))] (str (UUID/randomUUID)))]
@@ -597,7 +597,7 @@
(let [sales-summary-id (first (find-queries [client-code "service-charge"])) (let [sales-summary-id (first (find-queries [client-code "service-charge"]))
sales-category-id (first (find-queries [client-code "item-name"])) sales-category-id (first (find-queries [client-code "item-name"]))
expected-deposit-id (first (find-queries [client-code "expected-deposit"])) expected-deposit-id (first (find-queries [client-code "expected-deposit"]))
tender-id (first (find-queries [client-code "charge"])) tender-id (first (find-queries [client-code ":charge"]))
refund-id (first (find-queries [client-code "sales-refund"]))] refund-id (first (find-queries [client-code "sales-refund"]))]
(println "For" client-code ":") (println "For" client-code ":")
@@ -605,9 +605,8 @@
(println "Sales Category: " (str "https://app.integreatconsult.com/api/queries/" sales-category-id "/results/json")) (println "Sales Category: " (str "https://app.integreatconsult.com/api/queries/" sales-category-id "/results/json"))
(println "Expected Deposits: " (str "https://app.integreatconsult.com/api/queries/" expected-deposit-id "/results/json")) (println "Expected Deposits: " (str "https://app.integreatconsult.com/api/queries/" expected-deposit-id "/results/json"))
(println "Tenders: " (str "https://app.integreatconsult.com/api/queries/" tender-id "/results/json")) (println "Tenders: " (str "https://app.integreatconsult.com/api/queries/" tender-id "/results/json"))
(println "Sales Category: " (str "https://app.integreatconsult.com/api/queries/" refund-id "/results/json")) (println "Refund: " (str "https://app.integreatconsult.com/api/queries/" refund-id "/results/json"))
(println "") (println "")))
))
(defn historical-load-sales [client-code days] (defn historical-load-sales [client-code days]
@@ -726,8 +725,7 @@
:where [_ :db/ident ?i] :where [_ :db/ident ?i]
[(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix) [(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix)
(mapcat identity) (mapcat identity)
vec vec))
))
(defn manually-add-transaction [] (defn manually-add-transaction []
(auto-ap.yodlee.import/transactions->txs [{:postDate "2014-01-04" (auto-ap.yodlee.import/transactions->txs [{:postDate "2014-01-04"

1
src/cljc/auto_ap/client_routes.cljc
View File

@@ -11,6 +11,7 @@
"users" :admin-users "users" :admin-users
"rules" :admin-rules "rules" :admin-rules
"accounts" :admin-accounts "accounts" :admin-accounts
"import-batches" :admin-import-batches
"reminders" :admin-reminders "reminders" :admin-reminders
"vendors" :admin-vendors "vendors" :admin-vendors
"excel-import" :admin-excel-import "excel-import" :admin-excel-import

9
src/cljs/auto_ap/views/components/admin/side_bar.cljs
View File

@@ -77,5 +77,12 @@
[:span {:class "icon"} [:span {:class "icon"}
[:i {:class "fa fa-download"}]] [:i {:class "fa fa-download"}]]
[:span {:class "name"} "Excel Invoices"]]]] [:span {:class "name"} "Excel Invoices"]]]
[:li.menu-item
[:a {:href (bidi/path-for routes/routes :admin-import-batches) , :class (str "item" (active-when ap = :admin-import-batches))}
[:span {:class "icon"}
[:i {:class "fa fa-download"}]]
[:span {:class "name"} "Import Batches"]]]]
(into [:div ] (r/children (r/current-component)))])) (into [:div ] (r/children (r/current-component)))]))

4
src/cljs/auto_ap/views/main.cljs
View File

@@ -29,6 +29,7 @@
[auto-ap.views.pages.admin.vendors :refer [admin-vendors-page]] [auto-ap.views.pages.admin.vendors :refer [admin-vendors-page]]
[auto-ap.views.pages.admin.excel-import :refer [admin-excel-import-page]] [auto-ap.views.pages.admin.excel-import :refer [admin-excel-import-page]]
[auto-ap.views.pages.admin.users :refer [admin-users-page]] [auto-ap.views.pages.admin.users :refer [admin-users-page]]
[auto-ap.views.pages.admin.import-batches :refer [import-batches-page]]
[auto-ap.views.pages.admin.yodlee :refer [admin-yodlee-page]] [auto-ap.views.pages.admin.yodlee :refer [admin-yodlee-page]]
[auto-ap.views.pages.admin.yodlee2 :as yodlee2] [auto-ap.views.pages.admin.yodlee2 :as yodlee2]
[auto-ap.entities.clients :as clients])) [auto-ap.entities.clients :as clients]))
@@ -104,6 +105,9 @@
(defmethod page :admin-vendors [_] (defmethod page :admin-vendors [_]
(admin-vendors-page)) (admin-vendors-page))
(defmethod page :admin-import-batches [_]
[import-batches-page])
(defmethod page :admin-yodlee [_] (defmethod page :admin-yodlee [_]
(admin-yodlee-page)) (admin-yodlee-page))

1
src/cljs/auto_ap/views/pages/transactions.cljs
View File

@@ -34,6 +34,7 @@
:exact-match-id (some-> (:exact-match-id params) str) :exact-match-id (some-> (:exact-match-id params) str)
:unresolved (:unresolved params) :unresolved (:unresolved params)
:location (:location params) :location (:location params)
:import-batch-id (some-> (:import-batch-id params) str)
:amount-lte (:amount-lte (:amount-range params)) :amount-lte (:amount-lte (:amount-range params))
:description (:description params) :description (:description params)
:approval-status (condp = @(re-frame/subscribe [::subs/active-page]) :approval-status (condp = @(re-frame/subscribe [::subs/active-page])

9
src/cljs/auto_ap/views/pages/transactions/side_bar.cljs
View File

@@ -111,13 +111,20 @@
:on-change (dispatch-value-change [::data-page/filter-changed data-page :description])} ]]]] :on-change (dispatch-value-change [::data-page/filter-changed data-page :description])} ]]]]
(when-let [exact-match-id @(re-frame/subscribe [::data-page/filter data-page :exact-match-id])] (when-let [exact-match-id @(re-frame/subscribe [::data-page/filter data-page :exact-match-id])]
[:div [:div
[:p.menu-label "Specific Payment"] [:p.menu-label "Specific Transaction"]
[:span.tag.is-medium exact-match-id " " [:span.tag.is-medium exact-match-id " "
[:button.delete.is-small {:on-click [:button.delete.is-small {:on-click
(dispatch-event [::data-page/filter-changed data-page :exact-match-id nil])}]]]) (dispatch-event [::data-page/filter-changed data-page :exact-match-id nil])}]]])
(when (= "admin" (:user/role user)) (when (= "admin" (:user/role user))
[:<> [:<>
(when-let [import-batch-id @(re-frame/subscribe [::data-page/filter data-page :import-batch-id])]
[:div
[:p.menu-label "Import Batch"]
[:span.tag.is-medium import-batch-id " "
[:button.delete.is-small {:on-click
(dispatch-event [::data-page/filter-changed data-page :import-batch-id nil])}]]])
[:p.menu-label "Admin only"] [:p.menu-label "Admin only"]
[:div [:div
[switch-field {:id "unresolved-only" [switch-field {:id "unresolved-only"