(ns auto-ap.ssr.admin.excel-invoice (:require [auto-ap.datomic :refer [audit-transact conn remove-nils]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.import.manual.common :as c] [auto-ap.logging :as alog] [auto-ap.routes.admin.excel-invoices :as route] [auto-ap.routes.utils :refer [wrap-admin wrap-client-redirect-unauthenticated]] [auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr.components :as com] [auto-ap.ssr.components.inputs :as inputs] [auto-ap.ssr.form-cursor :as fc] [auto-ap.ssr.hiccup-helper :as hh] [auto-ap.ssr.hx :as hx] [auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]] [auto-ap.ssr.ui :refer [base-page]] [auto-ap.ssr.utils :refer [apply-middleware-to-all-handlers html-response wrap-form-4xx-2 wrap-schema-enforce]] [auto-ap.utils :refer [by]] [bidi.bidi :as bidi] [clj-time.coerce :as coerce] [clojure.string :as str] [datomic.api :as dc] [digest] [hiccup2.core :as hiccup :refer [raw]])) (defn validate-invoice [invoice] (when-not (:invoice/client invoice) (throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.") {:invoice-number (:invoice/invoice-number invoice) :customer-identifier (:invoice/client-identifier invoice)}))) (doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]] (when (not (get invoice k)) (throw (ex-info (str (name k) "not found on invoice " invoice) invoice)))) invoice) (defn reset-id [i] (update i :invoice-number (fn [n] (if (re-matches #"#+" n) nil n)))) (defn assoc-client-code [i] (let [[client-code default-location] (str/split (:location i) #"-")] (cond-> i client-code (assoc :client-code client-code) default-location (assoc :default-location default-location) (not client-code) (update :errors conj {:info "No client code found"}) (not default-location) (update :errors conj {:info "No default location found"})))) (defn parse-client [{:keys [client-code client default-location]} clients] (if-let [id (:db/id (or (clients client-code) (clients client)))] (do (when (not ((set (:client/locations (or (clients client-code) (clients client)))) default-location)) (throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'.")))) id) (throw (Exception. (str "Client code '" client-code "' and client named '" client "' not found."))))) (defn parse-invoice-number [{:keys [invoice-number]}] (or invoice-number "")) (defn parse-vendor [{:keys [vendor-name check]} vendor-name->vendor] (let [v (vendor-name->vendor vendor-name)] (cond v v (= "Cash" check) nil :else (throw (Exception. (str "Vendor '" vendor-name "' not found.")))))) (defn parse-automatically-paid-when-due [{:keys [vendor client-id]}] (boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id))) (defn parse-schedule-payment-dom [{:keys [vendor client-id]}] (get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor)) client-id)) (defn invoice-rows->transaction [rows user] (->> rows (mapcat (fn [{:keys [vendor-id total client-id date invoice-number default-location check automatically-paid-when-due account-id]}] (let [payment-id (.toString (java.util.UUID/randomUUID)) transaction-id (.toString (java.util.UUID/randomUUID)) invoice #:invoice {:db/id (.toString (java.util.UUID/randomUUID)) :vendor vendor-id :client client-id :default-location default-location :location default-location :import-status :import-status/imported :automatically-paid-when-due automatically-paid-when-due :total total :outstanding-balance (if (= "Cash" check) 0.0 total) :status (if (= "Cash" check) :invoice-status/paid :invoice-status/unpaid) :invoice-number invoice-number :date (coerce/to-date date)} payment (when (= :invoice-status/paid (:invoice/status invoice)) #:invoice-payment {:invoice (:db/id invoice) :amount (:invoice/total invoice) :payment (remove-nils #:payment {:db/id payment-id :vendor (:invoice/vendor invoice) :client (:invoice/client invoice) :type :payment-type/cash :amount (:invoice/total invoice) :status :payment-status/cleared :date (:invoice/date invoice)})}) transaction (when (= :invoice-status/paid (:invoice/status invoice)) (let [[[bank-account]] (seq (dc/q '[:find ?ba :in $ ?c :where [?c :client/bank-accounts ?ba] [?ba :bank-account/type :bank-account-type/cash]] (dc/db conn) client-id))] [:upsert-transaction #:transaction {:amount (- (:invoice/total invoice)) :payment payment-id :client (:invoice/client invoice) :status "POSTED" :bank-account bank-account :db/id #_{:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id) :id #_{:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id) :raw-id transaction-id :vendor (:invoice/vendor invoice) :description-original "Cash payment" :date (coerce/to-date date) :approval-status :transaction-approval-status/approved :accounts [{:db/id (str #_{:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id) "-account") :transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :transaction-account/location "A" :transaction-account/amount (Math/abs (:invoice/total invoice))}]}]))] [[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice)) account-id)] (some-> payment remove-nils) transaction]))) (filter identity))) (defn parse-invoice-rows [excel-rows] (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code] tabulated (->> (str/split excel-rows #"\n") (map #(str/split % #"\t")) (map #(into {} (map (fn [c k] [k c]) % columns)))) vendor-name->vendor (->> (set (map :vendor-name tabulated)) (dc/q '[:find ?n ?v :in $ [?n ...] :where [?v :vendor/name ?n]] (dc/db conn)) (into {})) all-clients (merge (into {} (dc/q '[:find ?n (pull ?v [:db/id :client/locations]) :in $ :where [?v :client/name ?n]] (dc/db conn))) (into {} (dc/q '[:find ?n (pull ?v [:db/id :client/locations]) :in $ :where [?v :client/code ?n]] (dc/db conn)))) rows (->> tabulated (map reset-id) (map assoc-client-code) (map (c/parse-or-error :client-id #(parse-client % all-clients))) (map (c/parse-or-error :vendor-id #(parse-vendor % vendor-name->vendor))) (map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %))) (map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %))) (map (c/parse-or-error :account-id c/parse-account-numeric-code)) (map (c/parse-or-error :invoice-number parse-invoice-number)) (map (c/parse-or-error :total c/parse-amount)) (map (c/parse-or-error :date c/parse-date)))] (alog/info ::parsed :rows (take 10 rows)) rows)) (defn bulk-upload-invoices [excel-rows user] (let [parsed-invoice-rows (parse-invoice-rows excel-rows) existing-rows (set (d-invoices/get-existing-set)) grouped-rows (group-by (fn [i] (cond (seq (:errors i)) :error (existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)]) :exists :else :new)) parsed-invoice-rows) vendors-not-found (->> parsed-invoice-rows (filter #(and (nil? (:vendor-id %)) (not= "Cash" (:check %)))) (map :vendor-name) set)] (audit-transact (invoice-rows->transaction (:new grouped-rows) user) user) {:imported (count (:new grouped-rows)) :already-imported (count (:exists grouped-rows)) :vendors-not-found vendors-not-found :errors (map #(dissoc % :date) (:error grouped-rows))})) (def sample "6/16/17 Acme Bread NMKT-CB 3/26/56 12:00 AM $54.00 Naschmarkt X 7/31/17 8:26 AM 8/1/17 3:57 PM 31000 6/20/17 Acme Bread NMKT-CB 3/19/58 12:00 AM $54.00 Naschmarkt X 7/31/17 8:26 AM 8/1/17 3:57 PM 6/21/17 Acme Bread NMKT-CB 11/13/58 12:00 AM $54.00 Naschmarkt X 7/31/17 8:26 AM 8/1/17 3:57 PM") (defn form* [{:keys [form-params form-errors]} & children] (fc/start-form form-params form-errors [:form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/import) :hx-swap "outerHTML"} [:div {:class "flex flex-col px-4 py-3 space-y-3 w-full"} [:h1.text-2xl.mb-3.font-bold "Import invoices from excel"] (fc/with-field :tsv (com/validated-field {:label "Tab-separated invoices" :errors (fc/field-errors)} [:textarea {:class (hh/add-class "w-full h-96" inputs/default-input-classes) :placeholder (hiccup/raw sample) :name (fc/field-name)} (fc/field-value)])) (com/form-errors {:errors (:errors fc/*form-errors*)}) (com/validated-save-button {:color :primary :class "place-self-end w-32" :errors (seq form-errors)} "Import") children]])) (defn page [{:keys [form-params form-errors] :as request}] (base-page request (com/page {:nav com/admin-aside-nav :client-selection (:client-selection request) :clients (:clients request) :client (:client request) :identity (:identity request) :request request} (com/breadcrumbs {} [:a {:href (bidi/path-for ssr-routes/only-routes ::route/page)} "Admin"]) [:div.flex.space-x-4 (com/content-card {:class "w-3/4"} (form* {:form-params {} :form-errors []}))]) "Admin")) (defn form [{:keys [form-params form-errors] :as request}] (html-response (form* {:form-params (or form-params {}) :form-errors (or form-errors [])}))) (defn import [{:keys [form-params form-errors] :as request}] (html-response (let [result (bulk-upload-invoices (:tsv form-params) (:identity request))] (form* {:form-params form-params :form-errors form-errors} [:div.flex.space-x-4 (com/pill {:color :primary} (format "%d imported" (:imported result))) (com/pill {:color :secondary} (format "%d extant" (:already-imported result))) (when (seq (:vendors-not-found result)) (list (com/pill {:color :yellow "@mouseover" "show=true" "@mouseout" "show=false" "x-tooltip" "{content: ()=>$refs.tooltip.innerHTML , allowHTML: true}"} (format "%d vendors not found" (count (:vendors-not-found result)))) [:template {:x-ref "tooltip"} [:ul (for [n (take 5 (:vendors-not-found result))] [:li n])]]))] (when (seq (:errors result)) (com/field {:label "Errors"} (com/data-grid {:headers [(com/data-grid-header {} "Date") (com/data-grid-header {} "Invoice #") (com/data-grid-header {} "Client") (com/data-grid-header {} "Vendor") (com/data-grid-header {} "Amount") (com/data-grid-header {} "Errors")]} (for [r (:errors result)] (com/data-grid-row {} (com/data-grid-cell {} (:raw-date r)) (com/data-grid-cell {} (:invoice-number r)) (com/data-grid-cell {} (:client-name r)) (com/data-grid-cell {} (:vendor-name r)) (com/data-grid-cell {} (:amount r)) (com/data-grid-cell {} (str/join ", " (map :info (:errors r))))))))))))) (def key->handler (apply-middleware-to-all-handlers (->> {::route/page page ::route/import (-> import (wrap-schema-enforce :form-schema [:map [:tsv :string]]) (wrap-nested-form-params) (wrap-form-4xx-2 form))}) (fn [h] (-> h (wrap-admin) (wrap-client-redirect-unauthenticated)))))