521 lines
29 KiB
Clojure
521 lines
29 KiB
Clojure
(ns auto-ap.routes.invoices
|
|
(:require
|
|
[amazonica.aws.s3 :as s3]
|
|
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
|
|
[auto-ap.datomic.accounts :as a]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[auto-ap.datomic.invoices :as d-invoices]
|
|
[auto-ap.datomic.vendors :as d-vendors]
|
|
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
|
|
[auto-ap.import.manual :as manual]
|
|
[auto-ap.import.manual.common :as c]
|
|
[auto-ap.parse :as parse]
|
|
[auto-ap.routes.utils :refer [wrap-secure]]
|
|
[auto-ap.utils :refer [by]]
|
|
[clj-time.coerce :as coerce :refer [to-date]]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.java.io :as io]
|
|
[clojure.string :as str]
|
|
[clojure.tools.logging :as log]
|
|
[config.core :refer [env]]
|
|
[datomic.client.api :as dc]
|
|
[digest]
|
|
[iol-ion.tx :refer [propose-invoice random-tempid]]
|
|
[ring.middleware.json :refer [wrap-json-response]]
|
|
[unilog.context :as lc])
|
|
(:import
|
|
(java.util UUID)))
|
|
|
|
(defn reset-id [i]
|
|
(update i :invoice-number
|
|
(fn [n] (if (re-matches #"#+" n)
|
|
nil
|
|
n))))
|
|
|
|
(defn assoc-client-code [i]
|
|
(let [[client-code default-location] (str/split (:location i) #"-")]
|
|
(cond-> i
|
|
client-code (assoc :client-code client-code)
|
|
default-location (assoc :default-location default-location)
|
|
(not client-code) (update :errors conj {:info "No client code found"})
|
|
(not default-location) (update :errors conj {:info "No default location found"}))))
|
|
|
|
(defn parse-client [{:keys [client-code client default-location]} clients]
|
|
(if-let [id (:db/id (or (clients client-code)
|
|
(clients client)))]
|
|
(do
|
|
(when (not ((set (:client/locations (or (clients client-code)
|
|
(clients client))))
|
|
default-location))
|
|
(throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'."))))
|
|
id)
|
|
(throw (Exception. (str "Client code '" client-code "' and client named '" client "' not found.")))))
|
|
|
|
(defn parse-invoice-number [{:keys [invoice-number]}]
|
|
(or invoice-number ""))
|
|
|
|
(defn parse-vendor [{:keys [vendor-name check]} vendor-name->vendor]
|
|
(let [v (vendor-name->vendor vendor-name)]
|
|
(cond v
|
|
v
|
|
|
|
(= "Cash" check)
|
|
nil
|
|
|
|
:else
|
|
(throw (Exception. (str "Vendor '" vendor-name "' not found."))))))
|
|
|
|
(defn parse-automatically-paid-when-due [{:keys [vendor client-id]}]
|
|
(boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id)))
|
|
|
|
(defn parse-schedule-payment-dom [{:keys [vendor client-id]}]
|
|
(get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor))
|
|
client-id))
|
|
|
|
(defn parse-invoice-rows [excel-rows]
|
|
(let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code]
|
|
tabulated (->> (str/split excel-rows #"\n")
|
|
(map #(str/split % #"\t"))
|
|
(map #(into {} (map (fn [c k] [k c]) % columns))))
|
|
vendor-name->vendor (->>
|
|
(set (map :vendor-name tabulated))
|
|
(dc/q '[:find ?n ?v
|
|
:in $ [?n ...]
|
|
:where [?v :vendor/name ?n]]
|
|
(dc/db conn)
|
|
)
|
|
(into {}))
|
|
all-clients (merge (into {}(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
|
|
:in $
|
|
:where [?v :client/name ?n]]
|
|
(dc/db conn)))
|
|
(into {}
|
|
(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
|
|
:in $
|
|
:where [?v :client/code ?n]]
|
|
(dc/db conn))))
|
|
rows (->> tabulated
|
|
(map reset-id)
|
|
(map assoc-client-code)
|
|
(map (c/parse-or-error :client-id #(parse-client % all-clients)))
|
|
(map (c/parse-or-error :vendor-id #(parse-vendor % vendor-name->vendor)))
|
|
(map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
|
|
(map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
|
|
(map (c/parse-or-error :account-id c/parse-account-numeric-code))
|
|
(map (c/parse-or-error :invoice-number parse-invoice-number))
|
|
(map (c/parse-or-error :total c/parse-amount))
|
|
(map (c/parse-or-error :date c/parse-date)))]
|
|
|
|
rows))
|
|
|
|
(defn match-vendor [vendor-code forced-vendor]
|
|
(when (and (not forced-vendor) (str/blank? vendor-code))
|
|
(throw (ex-info (str "No vendor found. Please supply an forced vendor.")
|
|
{:vendor-code vendor-code})))
|
|
(let [vendor-id (or forced-vendor
|
|
(->> (dc/q
|
|
{:query {:find ['?vendor]
|
|
:in ['$ '?vendor-name]
|
|
:where ['[?vendor :vendor/name ?vendor-name]]}
|
|
:args [(dc/db conn) vendor-code]})
|
|
first
|
|
first))]
|
|
(when-not vendor-id
|
|
(throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.")
|
|
{:vendor-code vendor-code})))
|
|
|
|
(if-let [matching-vendor (->> (dc/q
|
|
{:query {:find [(list 'pull '?vendor-id d-vendors/default-read)]
|
|
:in ['$ '?vendor-id]}
|
|
:args [(dc/db conn) vendor-id]})
|
|
first
|
|
first)]
|
|
matching-vendor
|
|
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
|
|
{:vendor-code vendor-code})))))
|
|
|
|
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}]
|
|
(let [matching-client (cond
|
|
account-number (d-clients/exact-match account-number)
|
|
customer-identifier (d-clients/best-match customer-identifier)
|
|
client-override (Long/parseLong client-override))
|
|
|
|
matching-vendor (match-vendor vendor-code vendor-override)
|
|
matching-location (or (when-not (str/blank? location-override)
|
|
location-override)
|
|
(parse/best-location-match (dc/pull (dc/db conn)
|
|
[{:client/location-matches [:location-match/location :location-match/matches]}
|
|
:client/default-location
|
|
:client/locations]
|
|
matching-client)
|
|
text
|
|
full-text))]
|
|
#:invoice {:db/id (random-tempid)
|
|
:invoice/client matching-client
|
|
:invoice/client-identifier (or account-number customer-identifier)
|
|
:invoice/vendor (:db/id matching-vendor)
|
|
:invoice/source-url source-url
|
|
:invoice/invoice-number invoice-number
|
|
:invoice/total (Double/parseDouble total)
|
|
:invoice/date (to-date date)
|
|
:invoice/location matching-location
|
|
:invoice/import-status (or import-status :import-status/pending)
|
|
:invoice/outstanding-balance (Double/parseDouble total)
|
|
:invoice/status :invoice-status/unpaid}))
|
|
|
|
(defn validate-invoice [invoice user]
|
|
(when-not (:invoice/client invoice)
|
|
(throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.")
|
|
{:invoice-number (:invoice/invoice-number invoice)
|
|
:customer-identifier (:invoice/client-identifier invoice)})))
|
|
(assert-can-see-client user (:invoice/client invoice))
|
|
(doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]]
|
|
(when (not (get invoice k))
|
|
(throw (ex-info (str (name k) "not found on invoice " invoice)
|
|
invoice))))
|
|
|
|
invoice)
|
|
|
|
|
|
(defn invoice-rows->transaction [rows user]
|
|
(->> rows
|
|
(mapcat (fn [{:keys [vendor-id total client-id date invoice-number default-location check automatically-paid-when-due]}]
|
|
(let [payment-id (.toString (java.util.UUID/randomUUID))
|
|
transaction-id (.toString (java.util.UUID/randomUUID))
|
|
invoice #:invoice {:db/id (.toString (java.util.UUID/randomUUID))
|
|
:vendor vendor-id
|
|
:client client-id
|
|
:default-location default-location
|
|
:location default-location
|
|
:import-status :import-status/imported
|
|
:automatically-paid-when-due automatically-paid-when-due
|
|
:total total
|
|
:outstanding-balance (if (= "Cash" check)
|
|
0.0
|
|
total)
|
|
:status (if (= "Cash" check)
|
|
:invoice-status/paid
|
|
:invoice-status/unpaid)
|
|
:invoice-number invoice-number
|
|
:date (to-date date)}
|
|
payment (when (= :invoice-status/paid (:invoice/status invoice))
|
|
#:invoice-payment {:invoice (:db/id invoice)
|
|
:amount (:invoice/total invoice)
|
|
:payment (remove-nils #:payment {:db/id payment-id
|
|
:vendor (:invoice/vendor invoice)
|
|
:client (:invoice/client invoice)
|
|
:type :payment-type/cash
|
|
:amount (:invoice/total invoice)
|
|
:status :payment-status/cleared
|
|
:date (:invoice/date invoice)})})
|
|
transaction (when (= :invoice-status/paid (:invoice/status invoice))
|
|
(let [[[bank-account]] (dc/q '[:find ?ba
|
|
:in $ ?c
|
|
:where [?c :client/bank-accounts ?ba]
|
|
[?ba :bank-account/type :bank-account-type/cash]
|
|
]
|
|
(dc/db conn)
|
|
client-id)]
|
|
#:transaction {:amount (- (:invoice/total invoice))
|
|
:payment payment-id
|
|
:client (:invoice/client invoice)
|
|
:status "POSTED"
|
|
:bank-account bank-account
|
|
:id #_ {:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id)
|
|
:raw-id transaction-id
|
|
:vendor (:invoice/vendor invoice)
|
|
:description-original "Cash payment"
|
|
:date (to-date date)
|
|
:approval-status :transaction-approval-status/approved
|
|
:accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
|
:transaction-account/location "A"
|
|
:transaction-account/amount (Math/abs (:invoice/total invoice))}]}))
|
|
]
|
|
[`(propose-invoice ~(d-invoices/code-invoice (validate-invoice (remove-nils invoice)
|
|
user)))
|
|
(some-> payment remove-nils)
|
|
transaction])))
|
|
(filter identity)))
|
|
|
|
(defn admin-only-if-multiple-clients [is]
|
|
(let [client-count (->> is
|
|
(map :invoice/client)
|
|
set
|
|
count)]
|
|
(map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is)))
|
|
|
|
(defn import-uploaded-invoice [user imports]
|
|
(lc/with-context {:area "upload-invoice"}
|
|
(log/info "Number of invoices to import is" (count imports))
|
|
(let [potential-invoices (->> imports
|
|
(map import->invoice)
|
|
(map #(validate-invoice % user))
|
|
admin-only-if-multiple-clients
|
|
(mapv d-invoices/code-invoice)
|
|
(mapv (fn [i] `(propose-invoice ~i))))]
|
|
|
|
(log/info "creating invoice" potential-invoices)
|
|
(let [tx (audit-transact potential-invoices user)]
|
|
(when-not (seq (dc/q '[:find ?i
|
|
:in $ [?i ...]
|
|
:where [?i :invoice/invoice-number]]
|
|
(:db-after tx)
|
|
(map :e (:tx-data tx))))
|
|
(throw (ex-info "No new invoices found."
|
|
{})))
|
|
tx))))
|
|
|
|
(defn validate-account-rows [rows code->existing-account]
|
|
(when-let [bad-types (seq (->> rows
|
|
(filter (fn [[account _ _ type]]
|
|
(and (not (code->existing-account (Integer/parseInt account)))
|
|
(not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))]
|
|
(throw (ex-info (str "You are adding accounts without a valid type")
|
|
{:rows bad-types})))
|
|
(when-let [duplicate-rows (seq (->> rows
|
|
(filter (fn [[account]]
|
|
(not-empty account)))
|
|
(group-by (fn [[account]]
|
|
account))
|
|
vals
|
|
(filter #(> (count %) 1))
|
|
(filter (fn [duplicates]
|
|
(apply not= duplicates)))
|
|
#_(map (fn [[[_ account]]]
|
|
account))))]
|
|
(throw (ex-info (str "You have duplicated rows with different values.")
|
|
{:rows duplicate-rows}))))
|
|
|
|
(defn import-account-overrides [customer filename]
|
|
(let [[_ & rows] (-> filename (io/reader) csv/read-csv)
|
|
[client-id] (first (dc/q (-> {:query {:find ['?e]
|
|
:in ['$ '?z]
|
|
:where [['?e :client/code '?z]]}
|
|
:args [(dc/db conn) customer]})))
|
|
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
|
|
{:account/applicability [:db/ident]}
|
|
:db/id])]
|
|
:in ['$]
|
|
:where ['[?e :account/name]]}
|
|
:args [(dc/db conn)]})))
|
|
|
|
existing-account-overrides (dc/q (-> {:query {:find ['?e]
|
|
:in ['$ '?client-id]
|
|
:where [['?e :account-client-override/client '?client-id]]}
|
|
:args [(dc/db conn) client-id]}))
|
|
rows (transduce (comp
|
|
(map (fn [[_ account account-name override-name _ type]]
|
|
[account account-name override-name type]))
|
|
(filter (fn [[account]]
|
|
(not-empty account))))
|
|
conj
|
|
[]
|
|
rows)
|
|
|
|
_ (validate-account-rows rows code->existing-account)
|
|
rows (vec (set rows))
|
|
|
|
txes (transduce
|
|
(comp
|
|
(mapcat (fn parse-map [[account account-name override-name type]]
|
|
(let [code (some-> account
|
|
not-empty
|
|
Integer/parseInt)
|
|
existing (code->existing-account code)]
|
|
(cond (not code)
|
|
[]
|
|
|
|
(and existing (or (#{:account-applicability/optional :account-applicability/customized}
|
|
(:db/ident (:account/applicability existing)))
|
|
(and (not-empty override-name)
|
|
(not-empty account-name)
|
|
(not= override-name account-name))))
|
|
[{:db/id (:db/id existing)
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
|
|
(not existing)
|
|
[{:account/applicability :account-applicability/customized
|
|
:account/name account-name
|
|
:account/account-set "default"
|
|
:account/numeric-code code
|
|
:account/code (str code)
|
|
:account/type (if (str/blank? type)
|
|
:account-type/expense
|
|
(keyword "account-type" (str/lower-case type)))
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
:else
|
|
[])))))
|
|
|
|
conj
|
|
(mapv
|
|
(fn [[x]]
|
|
[:db/retractEntity x])
|
|
existing-account-overrides)
|
|
rows)]
|
|
|
|
(dc/transact conn {:tx-data txes})
|
|
txes))
|
|
|
|
(defn import-transactions-cleared-against [file]
|
|
(let [[_ & rows] (-> file (io/reader) csv/read-csv)
|
|
txes (transduce
|
|
(comp
|
|
(filter (fn [[transaction-id _]]
|
|
(dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id))))
|
|
(map (fn [[transaction-id cleared-against]]
|
|
{:db/id (Long/parseLong transaction-id)
|
|
:transaction/cleared-against cleared-against})))
|
|
conj
|
|
[]
|
|
rows)]
|
|
(audit-transact txes nil)))
|
|
|
|
(defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}]
|
|
(assert-admin user)
|
|
(try
|
|
(let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))]
|
|
{:status 200
|
|
:body (pr-str stats)
|
|
:headers {"Content-Type" "application/edn"}})
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}})))
|
|
|
|
(defn upload-invoices [{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"
|
|
location :location
|
|
location-2 "location"
|
|
vendor :vendor
|
|
vendor-2 "vendor"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
location (or location location-2)
|
|
vendor (some-> (or vendor vendor-2)
|
|
(Long/parseLong))
|
|
{:keys [filename tempfile]} files]
|
|
(lc/with-context {:parsing-file filename}
|
|
(log/info tempfile)
|
|
(try
|
|
(let [extension (last (str/split (.getName (io/file filename)) #"\."))
|
|
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)
|
|
_ (s3/put-object (:data-bucket env)
|
|
s3-location
|
|
(io/input-stream tempfile)
|
|
{:content-type (if (= "csv" extension)
|
|
"text/csv"
|
|
"application/pdf")
|
|
:content-length (.length tempfile)})
|
|
imports (->> (parse/parse-file (.getPath tempfile) filename)
|
|
(map #(assoc %
|
|
:client-override client
|
|
:location-override location
|
|
:vendor-override vendor
|
|
:source-url (str "http://" (:data-bucket env)
|
|
".s3-website-us-east-1.amazonaws.com/"
|
|
s3-location))))]
|
|
(import-uploaded-invoice user imports))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(log/warn e)
|
|
{:status 400
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}})))))
|
|
|
|
(defn bulk-upload-invoices [{{:keys [excel-rows]} :edn-params user :identity}]
|
|
(assert-admin user)
|
|
(let [parsed-invoice-rows (parse-invoice-rows excel-rows)
|
|
existing-rows (set (d-invoices/get-existing-set))
|
|
grouped-rows (group-by
|
|
(fn [i]
|
|
(cond (seq (:errors i))
|
|
:error
|
|
|
|
(existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)])
|
|
:exists
|
|
|
|
:else
|
|
:new))
|
|
parsed-invoice-rows)
|
|
vendors-not-found (->> parsed-invoice-rows
|
|
(filter #(and (nil? (:vendor-id %))
|
|
(not= "Cash" (:check %))))
|
|
(map :vendor-name)
|
|
set)
|
|
_ (audit-transact (invoice-rows->transaction (:new grouped-rows)
|
|
user)
|
|
user)]
|
|
{:status 200
|
|
:body (pr-str {:imported (count (:new grouped-rows))
|
|
:already-imported (count (:exists grouped-rows))
|
|
:vendors-not-found vendors-not-found
|
|
:errors (map #(dissoc % :date) (:error grouped-rows))})
|
|
:headers {"Content-Type" "application/edn"}}))
|
|
|
|
(defn cleared-against [{{files :file
|
|
files-2 "file"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
(import-transactions-cleared-against (.getPath tempfile))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}}))))
|
|
|
|
|
|
|
|
(defn bulk-account-overrides [{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
{:status 200
|
|
:body (import-account-overrides client (.getPath tempfile))
|
|
:headers {"Content-Type" "application/json"}}
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body {:message (.getMessage e)
|
|
:data (ex-data e)}
|
|
:headers {"Content-Type" "application/json"}}))))
|
|
|
|
(def routes {"api/" {"transactions/" {:post {#"batch-upload/?" :batch-upload-transactions
|
|
#"cleared-against/?" :cleared-against}}
|
|
"invoices/" {:post {#"upload/?" :upload-invoices
|
|
#"upload-integreat/?" :bulk-upload-invoices}}
|
|
:post {#"account-overrides/?" :bulk-account-overrides}}})
|
|
|
|
(def match->handler {:batch-upload-transactions (wrap-secure batch-upload-transactions)
|
|
:upload-invoices (wrap-secure upload-invoices)
|
|
:bulk-upload-invoices (wrap-secure bulk-upload-invoices)
|
|
:cleared-against (wrap-secure cleared-against)
|
|
:bulk-account-overrides (wrap-secure (wrap-json-response bulk-account-overrides))})
|