From 26a949e63f1de3cc366948f143f820f9153eea9a Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Wed, 15 Dec 2021 20:35:37 -0800 Subject: [PATCH] New import approach to capture the batch. --- src/clj/auto_ap/datomic/migrate.clj | 46 ++- src/clj/auto_ap/datomic/transactions.clj | 5 + src/clj/auto_ap/graphql.clj | 101 ++++--- src/clj/auto_ap/intuit/import.clj | 23 +- src/clj/auto_ap/routes/invoices.clj | 63 ++-- src/clj/auto_ap/yodlee/import.clj | 279 +++++++++--------- src/clj/user.clj | 12 +- src/cljc/auto_ap/client_routes.cljc | 1 + .../views/components/admin/side_bar.cljs | 9 +- src/cljs/auto_ap/views/main.cljs | 4 + .../auto_ap/views/pages/transactions.cljs | 1 + .../views/pages/transactions/side_bar.cljs | 9 +- 12 files changed, 335 insertions(+), 218 deletions(-) diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index c136ffdf..a73d839a 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -378,7 +378,51 @@ :db/doc "An unhashed version of the id" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]] - :depends-on [:auto-ap/base-schema]}} + :depends-on [:auto-ap/base-schema]} + :auto-ap/add-transaction-import2 {:txes [[{:db/ident :import-batch/external-id + :db/doc "An identifier for the import batch" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity} + {:db/ident :import-batch/entry + :db/doc "Links to everything that was imported" + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + {:db/ident :import-batch/imported + :db/doc "How many entries were imported" + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/extant + :db/doc "How many entries were already present" + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/suppressed + :db/doc "How many entries were explicitly ignored" + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/date + :db/doc "The date the import occurred" + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/source + :db/doc "What system triggered the import" + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + {:db/ident :import-batch/status + :db/doc "What system triggered the import" + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + {:db/ident :import-source/intuit} + {:db/ident :import-source/manual} + {:db/ident :import-source/yodlee} + {:db/ident :import-source/yodlee2} + {:db/ident :import-status/started} + {:db/ident :import-status/completed} + {:db/ident :import-batch/user-name + :db/doc "Who triggred this import" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one}]] + :depends-on [:auto-ap/base-schema]}} diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj index d892f8b2..b69c9cc6 100644 --- a/src/clj/auto_ap/datomic/transactions.clj +++ b/src/clj/auto_ap/datomic/transactions.clj @@ -43,6 +43,11 @@ :where ['[?e :transaction/bank-account ?bank-account-id]]} :args [(:bank-account-id args)]}) + (:import-batch-id args) + (merge-query {:query {:in ['?import-batch-id] + :where ['[?import-batch-id :import-batch/entry ?e]]} + :args [(:import-batch-id args)]}) + (:account-id args) (merge-query {:query {:in ['?account-id] :where ['[?e :transaction/accounts ?accounts] diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index f5492f53..1521667a 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -1,37 +1,40 @@ (ns auto-ap.graphql - (:require [auto-ap.datomic :refer [merge-query uri]] - [auto-ap.datomic.checks :as d-checks] - [auto-ap.datomic.sales-orders :as d-sales-orders] - [auto-ap.datomic.users :as d-users] - [auto-ap.graphql.accounts :as gq-accounts] - [auto-ap.graphql.checks :as gq-checks] - [auto-ap.graphql.clients :as gq-clients] - [auto-ap.graphql.expected-deposit :as gq-expected-deposit] - [auto-ap.graphql.invoices :as gq-invoices] - [auto-ap.graphql.ledger :as gq-ledger] - [auto-ap.graphql.sales-orders :as gq-sales-orders] - [auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts] - [auto-ap.graphql.transaction-rules :as gq-transaction-rules] - [auto-ap.graphql.transactions :as gq-transactions] - [auto-ap.graphql.users :as gq-users] - [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] - [auto-ap.graphql.vendors :as gq-vendors] - [auto-ap.graphql.yodlee-merchants :as ym] - [auto-ap.graphql.yodlee2 :as gq-yodlee2] - [auto-ap.logging :refer [error-event info-event warn-event]] - [auto-ap.time :as time] - [clj-time.coerce :as coerce] - [clj-time.core :as t] - [clojure.string :as str] - [clojure.tools.logging :as log] - [clojure.walk :as walk] - [com.walmartlabs.lacinia :refer [execute]] - [com.walmartlabs.lacinia.schema :as schema] - [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d] - [unilog.context :as lc] - [yang.time :refer [time-it]]) - (:import clojure.lang.IPersistentMap)) + (:require + [auto-ap.datomic :refer [merge-query uri]] + [auto-ap.datomic.checks :as d-checks] + [auto-ap.datomic.sales-orders :as d-sales-orders] + [auto-ap.datomic.users :as d-users] + [auto-ap.graphql.accounts :as gq-accounts] + [auto-ap.graphql.checks :as gq-checks] + [auto-ap.graphql.clients :as gq-clients] + [auto-ap.graphql.expected-deposit :as gq-expected-deposit] + [auto-ap.graphql.import-batch :as gq-import-batches] + [auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts] + [auto-ap.graphql.invoices :as gq-invoices] + [auto-ap.graphql.ledger :as gq-ledger] + [auto-ap.graphql.sales-orders :as gq-sales-orders] + [auto-ap.graphql.transaction-rules :as gq-transaction-rules] + [auto-ap.graphql.transactions :as gq-transactions] + [auto-ap.graphql.users :as gq-users] + [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] + [auto-ap.graphql.vendors :as gq-vendors] + [auto-ap.graphql.yodlee-merchants :as ym] + [auto-ap.graphql.yodlee2 :as gq-yodlee2] + [auto-ap.logging :refer [error-event info-event warn-event]] + [auto-ap.time :as time] + [clj-time.coerce :as coerce] + [clj-time.core :as t] + [clojure.string :as str] + [clojure.tools.logging :as log] + [clojure.walk :as walk] + [com.walmartlabs.lacinia :refer [execute]] + [com.walmartlabs.lacinia.schema :as schema] + [com.walmartlabs.lacinia.util :refer [attach-resolvers]] + [datomic.api :as d] + [unilog.context :as lc] + [yang.time :refer [time-it]]) + (:import + (clojure.lang IPersistentMap))) (def integreat-schema { @@ -452,6 +455,21 @@ :start {:type 'Int} :end {:type 'Int}}} + :import_batch {:fields {:user_name {:type 'String} + :id {:type :id} + :date {:type :iso_date} + :imported {:type 'Int} + :extant {:type 'Int} + :suppressed {:type 'Int} + :status {:type :import_batch_status} + :source {:type :import_batch_source}}} + + :import_batch_page {:fields {:data {:type '(list :import_batch)} + :count {:type 'Int} + :total {:type 'Int} + :start {:type 'Int} + :end {:type 'Int}}} + :transaction_rule_page {:fields {:transaction_rules {:type '(list :transaction_rule)} :count {:type 'Int} :total {:type 'Int} @@ -633,6 +651,11 @@ :resolve :get-transaction-page} + :import_batch_page {:type :import_batch_page + :args {:filters {:type :import_batch_filters}} + + :resolve :get-import-batch-page} + :transaction_rule_page {:type :transaction_rule_page :args {:client_id {:type :id} :vendor_id {:type :id} @@ -706,6 +729,7 @@ :transaction_filters {:fields {:client_id {:type :id} :exact_match_id {:type :id} + :import_batch_id {:type :id} :vendor_id {:type :id} :bank_account_id {:type :id} @@ -720,6 +744,9 @@ :sort {:type '(list :sort_item)} :approval_status {:type :transaction_approval_status} :unresolved {:type 'Boolean}}} + :import_batch_filters {:fields {:start {:type 'Int} + :per_page {:type 'Int} + :sort {:type '(list :sort_item)}}} :ledger_filters {:fields {:client_id {:type :id} :vendor_id {:type :id} @@ -935,6 +962,13 @@ {:enum-value :cash} {:enum-value :debit}]} + :import_batch_source {:values [{:enum-value :intuit} + {:enum-value :yodlee} + {:enum-value :yodlee2} + {:enum-value :manual}]} + :import_batch_status {:values [{:enum-value :started} + {:enum-value :completed}]} + :processor {:values [{:enum-value :na} {:enum-value :doordash} {:enum-value :uber_eats} @@ -1328,6 +1362,7 @@ :get-potential-unpaid-invoices-matches gq-transactions/get-potential-unpaid-invoices-matches :get-accounts gq-accounts/get-accounts :get-transaction-page gq-transactions/get-transaction-page + :get-import-batch-page gq-import-batches/get-import-batch-page :get-ledger-page gq-ledger/get-ledger-page :get-sales-order-page gq-sales-orders/get-sales-orders-page :get-balance-sheet gq-ledger/get-balance-sheet diff --git a/src/clj/auto_ap/intuit/import.clj b/src/clj/auto_ap/intuit/import.clj index 6aec5663..d341357e 100644 --- a/src/clj/auto_ap/intuit/import.clj +++ b/src/clj/auto_ap/intuit/import.clj @@ -22,12 +22,16 @@ db))) (defn upsert-transactions [] - (let [db (d/db conn)] + (let [db (d/db conn) + import-id (y/start-import :import-source/intuit "Automated Intuit User") + stats (atom {:import-batch/imported 0 + :import-batch/extant 0 + :import-batch/suppressed 0})] + (doseq [[bank-account external-id] (get-intuit-bank-accounts db) :let [bank-account (d/entity db bank-account) end (auto-ap.time/local-now) - start (time/plus end (time/days -30))] - ] + start (time/plus end (time/days -30))]] (log/infof "importing from %s to %s for %s" start end external-id) (let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) (auto-ap.time/unparse end auto-ap.time/iso-date) @@ -40,8 +44,13 @@ :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) :status "posted"})))] (log/infof "%d transactions found" (count transactions)) - #_(y/grouped-import transactions) - transactions)))) + (let [result (y/grouped-import transactions import-id)] + (swap! stats + (fn [s] + (merge-with + s {:import-batch/imported (count (:import result)) + :import-batch/extant (count (:extant result)) + :import-batch/suppressed (count (:suppressed result))})))))) + (y/finish-import (assoc @stats :db/id import-id)))) (defn dry-run-upsert-transactions [] @@ -55,9 +64,7 @@ start (time/plus end (time/days -30))] r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) (auto-ap.time/unparse end auto-ap.time/iso-date) - external-id) - - ] + external-id)] {:client-id (:db/id (:client/_bank-accounts bank-account)) :client-code (:client/code (:client/_bank-accounts bank-account)) :bank-account-id (:db/id bank-account) diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 44ebf5a1..9ce3ba29 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -11,7 +11,7 @@ [auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.time-utils :refer [next-dom]] [auto-ap.utils :refer [by]] - [auto-ap.yodlee.import :refer [manual-import]] + [auto-ap.yodlee.import :refer [grouped-import] :as y] [clj-time.coerce :as coerce :refer [to-date]] [clj-time.core :as time] [clojure.data.csv :as csv] @@ -439,36 +439,41 @@ (try #_(throw (Exception. "Unexpected error")) (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-code :client-code] - all-clients (d-clients/get-all) - all-bank-accounts (mapcat :client/bank-accounts all-clients) - all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) - all-bank-accounts (merge (by :bank-account/code all-bank-accounts)) - rows (->> (str/split data #"\n" ) - (drop 1) - (map #(str/split % #"\t")) - (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) - (map (parse-or-error :amount parse-amount)) - (map (parse-or-error :date parse-date)) - (map (fn [{:keys [bank-account-code] :as row}] - (if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))] - (assoc row :bank-account-id bank-account-id) - (update row :errors conj {:info (str "Cannot find bank by code " bank-account-code) - :details (str "Cannot find bank by code " bank-account-code)})))) - (map (fn [{:keys [client-code] :as row}] - (if-let [client-id (:db/id (all-clients client-code))] - (assoc row :client-id client-id) - (update row :errors conj {:info (str "Cannot find client by code " client-code) - :details (str "Cannot find client by code " client-code)}))))) - error-rows (filter :errors rows) + all-clients (d-clients/get-all) + all-bank-accounts (mapcat :client/bank-accounts all-clients) + all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) + all-bank-accounts (merge (by :bank-account/code all-bank-accounts)) + rows (->> (str/split data #"\n" ) + (drop 1) + (map #(str/split % #"\t")) + (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) + (map (parse-or-error :amount parse-amount)) + (map (parse-or-error :date parse-date)) + (map (fn [{:keys [bank-account-code] :as row}] + (if-let [bank-account-id (:db/id (all-bank-accounts bank-account-code))] + (assoc row :bank-account-id bank-account-id) + (update row :errors conj {:info (str "Cannot find bank by code " bank-account-code) + :details (str "Cannot find bank by code " bank-account-code)})))) + (map (fn [{:keys [client-code] :as row}] + (if-let [client-id (:db/id (all-clients client-code))] + (assoc row :client-id client-id) + (update row :errors conj {:info (str "Cannot find client by code " client-code) + :details (str "Cannot find client by code " client-code)}))))) + error-rows (filter :errors rows) _ (log/info "Importing " (count rows) "raw transactions") - raw-transactions (vec (->> rows - (filter #(not (seq (:errors %))) ) - (map (fn [row] - (select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id])))))] - - (manual-import raw-transactions) + raw-transactions (vec (->> rows + (filter #(not (seq (:errors %))) ) + (map (fn [row] + (select-keys row [:description-original :client-code :status :high-level-category :amount :bank-account-id :date :client-id]))))) + import-id (y/start-import :import-source/manual (:user/name user)) + result (grouped-import raw-transactions import-id)] + (y/finish-import {:db/id import-id + :import-batch/imported (count (:import result)) + :import-batch/suppressed (count (:suppressed result)) + :import-batch/extant (count (:extant result))}) + {:status 200 - :body (pr-str {:imported (count raw-transactions) + :body (pr-str {:imported (count (:import result)) :errors (map #(dissoc % :date) error-rows)}) :headers {"Content-Type" "application/edn"}}) (catch Exception e diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 6558048e..f9de2aee 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -172,85 +172,112 @@ first first))) -(defn transactions->txs [transactions transaction->bank-account apply-rules existing] - (into [] +(defn transactions->txs [transactions import-id transaction->bank-account apply-rules existing] + (let [grouped-transactions (group-by + (fn [{post-date :postDate + account-id :accountId + date :date + id :id + status :status :as transaction}] + (let [bank-account (transaction->bank-account transaction) + bank-account-id (:db/id bank-account) + client (:client/_bank-accounts bank-account) + client-id (:db/id client) + valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) + + date (time/parse date "YYYY-MM-dd")] - (for [transaction transactions - :let [{post-date :postDate - account-id :accountId - date :date - id :id - {amount :amount} :amount - {description-original :original - description-simple :simple} :description - {merchant-id :id - merchant-name :name} :merchant - base-type :baseType - type :type - status :status} - transaction - amount (if (= "DEBIT" base-type) - (- amount) - amount) - check-number (extract-check-number transaction) - bank-account (transaction->bank-account transaction) - bank-account-id (:db/id bank-account) - client (:client/_bank-accounts bank-account) - client-id (:db/id client) - valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) - - date (time/parse date "YYYY-MM-dd")] - :when (and client-id - (not (existing (sha-256 (str id)))) - (= "POSTED" status) + (cond (existing (sha-256 (str id))) + :extant - (or (not (:start-date bank-account)) - (t/after? date (:start-date bank-account))))] - (let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id) - autopay-invoices-matches (when-not existing-check - (match-transaction-to-unfulfilled-autopayments amount client-id )) - unpaid-invoices-matches (when-not existing-check - (match-transaction-to-unpaid-invoices amount client-id )) - expected-deposit (when (and (> amount 0.0) - (not existing-check)) - (find-expected-deposit client-id amount date))] - (cond-> - [#:transaction - {:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd")) - :id (sha-256 (str id)) - :raw-id (str id) - :account-id account-id - :date (coerce/to-date date) - :amount (double amount) - :description-original (some-> description-original (str/replace #"\s+" " ")) - :description-simple (some-> description-simple (str/replace #"\s+" " ")) - :approval-status :transaction-approval-status/unapproved - :type type - :status status - :client client-id - :check-number check-number - :bank-account bank-account-id}] - existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved - :transaction/payment {:db/id (:db/id existing-check) - :payment/status :payment-status/cleared} - :transaction/vendor (:db/id (:payment/vendor existing-check)) - :transaction/location "A" - :transaction/accounts [#:transaction-account - {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) - :location "A" - :amount (Math/abs (double amount))}])) + (not client-id) + :error - ;; temporarily removed to automatically match autopaid invoices - #_(and (not existing-check) - (seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id) - expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit - :expected-deposit/status :expected-deposit-status/cleared})) - + (not= "POSTED" status) + :not-posted - (and (not (seq autopay-invoices-matches)) - (not (seq unpaid-invoices-matches)) - (not expected-deposit)) (update 0 #(apply-rules % valid-locations)) - true (update 0 remove-nils)))))) + (and (:start-date bank-account) + (not (t/after? date (:start-date bank-account)))) + :not-ready + + :else + :import))) + transactions)] + + (update grouped-transactions :import + (fn [transactions] + (into [] + + (for [transaction transactions + :let [{post-date :postDate + account-id :accountId + date :date + id :id + {amount :amount} :amount + {description-original :original + description-simple :simple} :description + {merchant-id :id + merchant-name :name} :merchant + base-type :baseType + type :type + status :status} + transaction + amount (if (= "DEBIT" base-type) + (- amount) + amount) + check-number (extract-check-number transaction) + bank-account (transaction->bank-account transaction) + bank-account-id (:db/id bank-account) + client (:client/_bank-accounts bank-account) + client-id (:db/id client) + valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) + date (time/parse date "YYYY-MM-dd")]] + (let [existing-check (transaction->existing-payment transaction check-number client-id bank-account-id amount id) + autopay-invoices-matches (when-not existing-check + (match-transaction-to-unfulfilled-autopayments amount client-id )) + unpaid-invoices-matches (when-not existing-check + (match-transaction-to-unpaid-invoices amount client-id )) + expected-deposit (when (and (> amount 0.0) + (not existing-check)) + (find-expected-deposit client-id amount date))] + (cond-> + [#:transaction + {:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd")) + :id (sha-256 (str id)) + :raw-id (str id) + :account-id account-id + :date (coerce/to-date date) + :amount (double amount) + :import-batch/_entry import-id + :description-original (some-> description-original (str/replace #"\s+" " ")) + :description-simple (some-> description-simple (str/replace #"\s+" " ")) + :approval-status :transaction-approval-status/unapproved + :type type + :status status + :client client-id + :check-number check-number + :bank-account bank-account-id}] + existing-check (update 0 #(assoc % :transaction/approval-status :transaction-approval-status/approved + :transaction/payment {:db/id (:db/id existing-check) + :payment/status :payment-status/cleared} + :transaction/vendor (:db/id (:payment/vendor existing-check)) + :transaction/location "A" + :transaction/accounts [#:transaction-account + {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) + :location "A" + :amount (Math/abs (double amount))}])) + + ;; temporarily removed to automatically match autopaid invoices + #_(and (not existing-check) + (seq autopay-invoices-matches)) #_(add-new-payment autopay-invoices-matches bank-account-id client-id) + expected-deposit (update 0 #(assoc % :transaction/expected-deposit {:db/id expected-deposit + :expected-deposit/status :expected-deposit-status/cleared})) + + + (and (not (seq autopay-invoices-matches)) + (not (seq unpaid-invoices-matches)) + (not expected-deposit)) (update 0 #(apply-rules % valid-locations)) + true (update 0 remove-nils))))))))) (defn batch-transact [transactions] @@ -282,40 +309,22 @@ (map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}] (assoc bank-account :client/_bank-accounts client)))))))) +(defn start-import [source user] + (get (:tempids @(d/transact conn [{:db/id "import-batch" + :import-batch/date (coerce/to-date (t/now)) + :import-batch/source source + :import-batch/status :import-status/started + :import-batch/user-name user}])) "import-batch")) -(defn manual-import [manual-transactions] - (lc/with-context {:source "manual import"} - (let [transformed-transactions (->> manual-transactions - (filter #(= "posted" (:status %))) - (group-by #(select-keys % [:date :description-original :amount])) - (vals) - (mapcat (fn [transaction-group] - (map - (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] - {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) - :bank-account-id bank-account-id - :date (time/unparse date "YYYY-MM-dd") - :amount {:amount amount} - :description {:original description-original - :simple high-level-category} - :status "POSTED"}) - (range) - transaction-group)))) - all-rules (tr/get-all) - all-bank-accounts (by :db/id (get-all-bank-accounts)) - transaction->bank-account (comp all-bank-accounts :bank-account-id)] - (log/info "Importing " (count transformed-transactions) " manual transactions") - - (doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] - (audit-transact tx {:user/name "Yodlee import" - :user/role ":admin"})) - (log/info "Imported manual transactions")))) +(defn finish-import [i] + @(d/transact conn [(assoc i :import-batch/status :import-status/completed)])) -(defn grouped-import [manual-transactions] + +(defn grouped-import [manual-transactions import-id] (lc/with-context {:source "grouped import"} (let [transformed-transactions (->> manual-transactions (filter #(= "posted" (:status %))) - (group-by #(select-keys % [:date :description-original :amount])) + (group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id])) (vals) (mapcat (fn [transaction-group] (map @@ -331,32 +340,15 @@ transaction-group)))) all-rules (tr/get-all) all-bank-accounts (by :db/id (get-all-bank-accounts)) - transaction->bank-account (comp all-bank-accounts :bank-account-id)] - (log/info "Importing " (count transformed-transactions) " grouped transactions") + transaction->bank-account (comp all-bank-accounts :bank-account-id) + transactions (transactions->txs transformed-transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + (log/info "Importing " (count (:import transformed-transactions)) " grouped transactions") - #_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + (doseq [tx (:import transactions)] (audit-transact tx {:user/name "Yodlee import" :user/role ":admin"})) - (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)) - (log/info "Imported grouped transactions")))) - -(defn grouped-new [manual-transactions] - (let [transformed-transactions (->> manual-transactions - (filter #(= "posted" (:status %))) - (group-by #(select-keys % [:date :description-original :amount])) - (vals) - (mapcat (fn [transaction-group] - (map - (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] - (assoc transaction :id - (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id))) - (range) - transaction-group)))) - existing (get-existing)] - (map #(if (existing (sha-256 (str (:id %)))) - (assoc % :new? false) - (assoc % :new? true)) - transformed-transactions))) + (log/info "Imported grouped transactions") + transactions))) (defn do-import ([] @@ -365,12 +357,18 @@ (lc/with-context {:source "Import yodlee transactions"} (do (log/info "importing from yodlee") - (let [all-bank-accounts (get-all-bank-accounts) - transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId) - all-rules (tr/get-all)] - (doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] - (audit-transact tx {:user/name "Yodlee import" - :user/role ":admin"}))))))) + (let [import-id (start-import :import-source/yodlee "Automated Yodlee User") + all-bank-accounts (get-all-bank-accounts) + transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId) + all-rules (tr/get-all) + transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + (doseq [tx (:import transactions)] + (audit-transact tx {:user/name "Yodlee import" + :user/role ":admin"})) + (finish-import {:db/id import-id + :import-batch/imported (count (:import transactions)) + :import-batch/extant (count (:extant transactions)) + :import-batch/suppressed (count (:suppresed transactions))})))))) (defn do-import2 ([] @@ -380,15 +378,20 @@ (do (log/info "importing from yodlee2") - (let [all-bank-accounts (get-all-bank-accounts) + (let [import-id (start-import :import-source/yodlee "Automated Yodlee User") + all-bank-accounts (get-all-bank-accounts) transaction->bank-account (comp (by (comp :yodlee-account/id :bank-account/yodlee-account) all-bank-accounts) :accountId) - all-rules (tr/get-all)] + all-rules (tr/get-all) + transactions (transactions->txs transactions import-id transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] - (log/info "COUNT" (count (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))) - (doseq [tx (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + (doseq [tx (:import transactions)] (audit-transact tx {:user/name "Yodlee import2" - :user/role ":admin"}))))))) + :user/role ":admin"})) + (finish-import {:db/id import-id + :import-batch/imported (count (:import transactions)) + :import-batch/extant (count (:extant transactions)) + :import-batch/suppressed (count (:suppresed transactions))})))))) diff --git a/src/clj/user.clj b/src/clj/user.clj index 744761bb..d7b9ec86 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -566,7 +566,7 @@ (str (UUID/randomUUID))) expected-deposit-id (or (first (find-queries [client-code "expected-deposit"])) (str (UUID/randomUUID))) - tender-id (or (first (find-queries [client-code "charge"])) + tender-id (or (first (find-queries [client-code ":charge"])) (str (UUID/randomUUID))) refund-id (or (first (find-queries [client-code "sales-refund"])) (str (UUID/randomUUID)))] @@ -597,7 +597,7 @@ (let [sales-summary-id (first (find-queries [client-code "service-charge"])) sales-category-id (first (find-queries [client-code "item-name"])) expected-deposit-id (first (find-queries [client-code "expected-deposit"])) - tender-id (first (find-queries [client-code "charge"])) + tender-id (first (find-queries [client-code ":charge"])) refund-id (first (find-queries [client-code "sales-refund"]))] (println "For" client-code ":") @@ -605,9 +605,8 @@ (println "Sales Category: " (str "https://app.integreatconsult.com/api/queries/" sales-category-id "/results/json")) (println "Expected Deposits: " (str "https://app.integreatconsult.com/api/queries/" expected-deposit-id "/results/json")) (println "Tenders: " (str "https://app.integreatconsult.com/api/queries/" tender-id "/results/json")) - (println "Sales Category: " (str "https://app.integreatconsult.com/api/queries/" refund-id "/results/json")) - (println "") - )) + (println "Refund: " (str "https://app.integreatconsult.com/api/queries/" refund-id "/results/json")) + (println ""))) (defn historical-load-sales [client-code days] @@ -726,8 +725,7 @@ :where [_ :db/ident ?i] [(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix) (mapcat identity) - vec - )) + vec)) (defn manually-add-transaction [] (auto-ap.yodlee.import/transactions->txs [{:postDate "2014-01-04" diff --git a/src/cljc/auto_ap/client_routes.cljc b/src/cljc/auto_ap/client_routes.cljc index 0ff3b14f..cd105b0e 100644 --- a/src/cljc/auto_ap/client_routes.cljc +++ b/src/cljc/auto_ap/client_routes.cljc @@ -11,6 +11,7 @@ "users" :admin-users "rules" :admin-rules "accounts" :admin-accounts + "import-batches" :admin-import-batches "reminders" :admin-reminders "vendors" :admin-vendors "excel-import" :admin-excel-import diff --git a/src/cljs/auto_ap/views/components/admin/side_bar.cljs b/src/cljs/auto_ap/views/components/admin/side_bar.cljs index 099eae4a..2e70be0e 100644 --- a/src/cljs/auto_ap/views/components/admin/side_bar.cljs +++ b/src/cljs/auto_ap/views/components/admin/side_bar.cljs @@ -77,5 +77,12 @@ [:span {:class "icon"} [:i {:class "fa fa-download"}]] - [:span {:class "name"} "Excel Invoices"]]]] + [:span {:class "name"} "Excel Invoices"]]] + [:li.menu-item + [:a {:href (bidi/path-for routes/routes :admin-import-batches) , :class (str "item" (active-when ap = :admin-import-batches))} + [:span {:class "icon"} + [:i {:class "fa fa-download"}]] + + [:span {:class "name"} "Import Batches"]]]] + (into [:div ] (r/children (r/current-component)))])) diff --git a/src/cljs/auto_ap/views/main.cljs b/src/cljs/auto_ap/views/main.cljs index 0a4b3855..bb316472 100644 --- a/src/cljs/auto_ap/views/main.cljs +++ b/src/cljs/auto_ap/views/main.cljs @@ -29,6 +29,7 @@ [auto-ap.views.pages.admin.vendors :refer [admin-vendors-page]] [auto-ap.views.pages.admin.excel-import :refer [admin-excel-import-page]] [auto-ap.views.pages.admin.users :refer [admin-users-page]] + [auto-ap.views.pages.admin.import-batches :refer [import-batches-page]] [auto-ap.views.pages.admin.yodlee :refer [admin-yodlee-page]] [auto-ap.views.pages.admin.yodlee2 :as yodlee2] [auto-ap.entities.clients :as clients])) @@ -104,6 +105,9 @@ (defmethod page :admin-vendors [_] (admin-vendors-page)) +(defmethod page :admin-import-batches [_] + [import-batches-page]) + (defmethod page :admin-yodlee [_] (admin-yodlee-page)) diff --git a/src/cljs/auto_ap/views/pages/transactions.cljs b/src/cljs/auto_ap/views/pages/transactions.cljs index a98d89e2..62269501 100644 --- a/src/cljs/auto_ap/views/pages/transactions.cljs +++ b/src/cljs/auto_ap/views/pages/transactions.cljs @@ -34,6 +34,7 @@ :exact-match-id (some-> (:exact-match-id params) str) :unresolved (:unresolved params) :location (:location params) + :import-batch-id (some-> (:import-batch-id params) str) :amount-lte (:amount-lte (:amount-range params)) :description (:description params) :approval-status (condp = @(re-frame/subscribe [::subs/active-page]) diff --git a/src/cljs/auto_ap/views/pages/transactions/side_bar.cljs b/src/cljs/auto_ap/views/pages/transactions/side_bar.cljs index 4544f468..242fbc75 100644 --- a/src/cljs/auto_ap/views/pages/transactions/side_bar.cljs +++ b/src/cljs/auto_ap/views/pages/transactions/side_bar.cljs @@ -111,13 +111,20 @@ :on-change (dispatch-value-change [::data-page/filter-changed data-page :description])} ]]]] (when-let [exact-match-id @(re-frame/subscribe [::data-page/filter data-page :exact-match-id])] [:div - [:p.menu-label "Specific Payment"] + [:p.menu-label "Specific Transaction"] [:span.tag.is-medium exact-match-id " " [:button.delete.is-small {:on-click (dispatch-event [::data-page/filter-changed data-page :exact-match-id nil])}]]]) + (when (= "admin" (:user/role user)) [:<> + (when-let [import-batch-id @(re-frame/subscribe [::data-page/filter data-page :import-batch-id])] + [:div + [:p.menu-label "Import Batch"] + [:span.tag.is-medium import-batch-id " " + [:button.delete.is-small {:on-click + (dispatch-event [::data-page/filter-changed data-page :import-batch-id nil])}]]]) [:p.menu-label "Admin only"] [:div [switch-field {:id "unresolved-only"