From 77d7b99ebb06db9097fd1503d0bcdf3fbf68d7f3 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 9 Nov 2018 23:28:56 -0800 Subject: [PATCH] bulk upload a little better. --- src/clj/auto_ap/routes/invoices.clj | 127 +++++++++++++++------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index a5600f49..76c13a98 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -1,17 +1,15 @@ (ns auto-ap.routes.invoices - (:require [auto-ap.db.companies :as companies] - [auto-ap.db.vendors :as vendors] - [auto-ap.db.invoices :as invoices] - [auto-ap.db.invoices-checks :as invoices-checks] - [auto-ap.db.checks :as checks] - [auto-ap.db.utils :refer [query]] + (:require [auto-ap.datomic.clients :as d-clients] + [auto-ap.datomic.vendors :as d-vendors] + [auto-ap.datomic.invoices :as d-invoices] [auto-ap.yodlee.import :refer [manual-import]] [auto-ap.utils :refer [by]] + [auto-ap.datomic :refer [remove-nils uri]] + [datomic.api :as d] [auto-ap.parse :as parse] [auto-ap.graphql.utils :refer [assert-admin]] [auto-ap.routes.utils :refer [wrap-secure]] [clj-time.coerce :refer [to-date]] - [auto-ap.db.invoices-expense-accounts :as expense-accounts] [ring.middleware.json :refer [wrap-json-response]] [compojure.core :refer [GET POST context defroutes wrap-routes]] @@ -23,24 +21,24 @@ nil n)))) -(defn assoc-company-code [i] +(defn assoc-client-code [i] (-> i - (assoc :company-code (first (str/split (:location i) #"-" ))) + (assoc :client-code (first (str/split (:location i) #"-" ))) (assoc :default-location (second (str/split (:location i) #"-" ))))) -(defn parse-company [{:keys [company-code company]} companies] - (if-let [id (:id (or (companies company-code) - (companies company)))] +(defn parse-client [{:keys [client-code client]} clients] + (if-let [id (:db/id (or (clients client-code) + (clients client)))] id - (throw (Exception. (str "Company code '" company-code "' and company named '" company "' not found."))))) + (throw (Exception. (str "Company code '" client-code "' and company named '" client "' not found."))))) (defn parse-invoice-number [{:keys [invoice-number]}] (or invoice-number "")) (defn parse-vendor [{:keys [vendor-name check]} vendors] - (let [id (:id (vendors vendor-name))] - (cond id - id + (let [v (vendors vendor-name)] + (cond v + v (= "Cash" check) nil @@ -48,6 +46,9 @@ :else (throw (Exception. (str "Vendor '" vendor-name "' not found.")))))) +(defn parse-vendor-id [{:keys [vendor]}] + (:db/id vendor)) + (defn parse-amount [i] (try (Double/parseDouble (str/replace (or (second @@ -144,16 +145,17 @@ {{:keys [excel-rows]} :edn-params user :identity} (assert-admin user) (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :company :bill-entered :bill-rejected :added-on :exported-on :default-expense-account] - all-vendors (by :name (vendors/get-all)) - all-companies (companies/get-all) - all-companies (merge (by :code all-companies) (by :name all-companies)) + all-vendors (by :vendor/name (d-vendors/get-graphql {})) + all-clients (d-clients/get-all) + all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) rows (->> (str/split excel-rows #"\n" ) (map #(str/split % #"\t")) (map #(into {} (map (fn [c k] [k c] ) % columns))) (map reset-id) - (map assoc-company-code) - (map (parse-or-error :company-id #(parse-company % all-companies))) - (map (parse-or-error :vendor-id #(parse-vendor % all-vendors))) + (map assoc-client-code) + (map (parse-or-error :client-id #(parse-client % all-clients))) + (map (parse-or-error :vendor #(parse-vendor % all-vendors))) + (map (parse-or-error :vendor-id #(parse-vendor-id %))) (map (parse-or-error :default-expense-account parse-default-expense-account)) (map (parse-or-error :invoice-number parse-invoice-number)) (map (parse-or-error :total parse-amount)) @@ -164,43 +166,54 @@ (not= "Cash" (:check %)))) (map :vendor-name) set) - insert-rows (vec (->> (filter #(not (seq (:errors %))) rows) - (map (fn [{:keys [vendor-id total company-id amount date invoice-number default-location default-expense-account check]}] - {:vendor-id vendor-id - :company-id company-id - :default-location default-location - :default-expense-account default-expense-account - :total total - :outstanding-balance (if (= "Cash" check) - 0 - total) - :imported true - :status (if (= "Cash" check) - "paid" - "unpaid") - :invoice-number invoice-number - :date date})))) - - inserted-rows (invoices/upsert-multi! insert-rows) - already-imported-count (- (count insert-rows) (count inserted-rows))] - (doseq [inserted-row inserted-rows - :when (= "paid" (:status inserted-row))] - (let [inserted-check (checks/insert! {:vendor-id (:vendor-id inserted-row) - :company-id (:company-id inserted-row) - :bank-account-id 0 - :type "cash" - :amount (:total inserted-row) - :status "cleared" - :date (:date inserted-row)})] - (invoices-checks/insert-multi! [{:amount (:total inserted-row) - :invoice-id (:id inserted-row) - :check-id (:id inserted-check)}]))) - (expense-accounts/assign-defaults!) - - + get-dupe-keys (fn [i] + (select-keys i [:invoice/vendor :invoice/client :invoice/invoice-number])) + existing-rows (->> (d-invoices/raw-graphql {}) + (filter (fn [i] (not= :invoice-status/voided (:invoice/status i)))) + (map (fn [i] (-> i + (update :invoice/client :db/id) + (update :invoice/vendor :db/id)))) + (map get-dupe-keys) + set) + total-rows (->> (filter #(not (seq (:errors %))) rows) + (map (fn [{:keys [vendor-id total client-id amount date invoice-number default-location default-expense-account check vendor]}] + #:invoice {:db/id (.toString (java.util.UUID/randomUUID)) + :vendor vendor-id + :client client-id + :default-location default-location + :default-expense-account default-expense-account + :total total + :outstanding-balance (if (= "Cash" check) + 0.0 + total) + :status (if (= "Cash" check) + :invoice-status/paid + :invoice-status/unpaid) + :invoice-number invoice-number + :date (to-date date) + :expense-accounts [#:invoice-expense-account {:expense-account-id (or default-expense-account (:vendor/default-expense-account vendor)) + :location default-location + :amount total}]}))) + insert-rows (vec (->> total-rows + (filter #(not (existing-rows (get-dupe-keys %) ))) + (mapcat (fn [i] + (if (= :invoice-status/paid (:invoice/status i)) + [i #:invoice-payment {:invoice (:db/id i) + :amount (:invoice/total i) + :payment (remove-nils #:payment {:db/id (.toString (java.util.UUID/randomUUID)) + :vendor (:invoice/vendor i) + :client (:invoice/client i) + :type :payment-type/cash + :amount (:invoice/total i) + :status :payment-status/cleared + :date (:invoice/date i)})}] + [i]))) + (map remove-nils))) + inserted-rows @(d/transact (d/connect uri) insert-rows) + already-imported-count (- (count total-rows) (count insert-rows))] {:status 200 - :body (pr-str {:imported (count inserted-rows) + :body (pr-str {:imported (count total-rows) :already-imported already-imported-count :vendors-not-found vendors-not-found :errors (map #(dissoc % :date) error-rows)})