346 lines
18 KiB
Clojure
346 lines
18 KiB
Clojure
(ns auto-ap.routes.invoices
|
|
(:require
|
|
[amazonica.aws.s3 :as s3]
|
|
[auto-ap.datomic :refer [audit-transact conn]]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[com.brunobonacci.mulog :as mu]
|
|
[auto-ap.datomic.invoices :as d-invoices]
|
|
[auto-ap.datomic.vendors :as d-vendors]
|
|
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
|
|
[auto-ap.import.manual :as manual]
|
|
[auto-ap.logging :as alog]
|
|
[auto-ap.parse :as parse]
|
|
[auto-ap.routes.utils :refer [wrap-secure]]
|
|
[auto-ap.utils :refer [by]]
|
|
[clj-time.coerce :as coerce :refer [to-date]]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.java.io :as io]
|
|
[clojure.string :as str]
|
|
[config.core :refer [env]]
|
|
[datomic.api :as dc]
|
|
[iol-ion.tx :refer [random-tempid]]
|
|
[ring.middleware.json :refer [wrap-json-response]])
|
|
(:import
|
|
(java.util UUID)))
|
|
|
|
(defn match-vendor [vendor-code forced-vendor]
|
|
(when (and (not forced-vendor) (str/blank? vendor-code))
|
|
(throw (ex-info (str "No vendor found. Please supply an forced vendor.")
|
|
{:vendor-code vendor-code})))
|
|
(let [vendor-id (or forced-vendor
|
|
(->> (dc/q
|
|
{:find ['?vendor]
|
|
:in ['$ '?vendor-name]
|
|
:where ['[?vendor :vendor/name ?vendor-name]]}
|
|
(dc/db conn) vendor-code)
|
|
first
|
|
first))]
|
|
(when-not vendor-id
|
|
(throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.")
|
|
{:vendor-code vendor-code})))
|
|
|
|
(if-let [matching-vendor (->> (dc/q
|
|
{:find [(list 'pull '?vendor-id d-vendors/default-read)]
|
|
:in ['$ '?vendor-id]}
|
|
(dc/db conn) vendor-id)
|
|
first
|
|
first)]
|
|
matching-vendor
|
|
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
|
|
{:vendor-code vendor-code})))))
|
|
|
|
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}]
|
|
(let [matching-client (cond
|
|
account-number (:db/id (d-clients/exact-match account-number))
|
|
customer-identifier (:db/id (d-clients/best-match customer-identifier))
|
|
client-override (Long/parseLong client-override))
|
|
|
|
matching-vendor (match-vendor vendor-code vendor-override)
|
|
matching-location (or (when-not (str/blank? location-override)
|
|
location-override)
|
|
(parse/best-location-match (dc/pull (dc/db conn)
|
|
[{:client/location-matches [:location-match/location :location-match/matches]}
|
|
:client/default-location
|
|
:client/locations]
|
|
matching-client)
|
|
text
|
|
full-text))]
|
|
#:invoice {:db/id (random-tempid)
|
|
:invoice/client matching-client
|
|
:invoice/client-identifier (or account-number customer-identifier)
|
|
:invoice/vendor (:db/id matching-vendor)
|
|
:invoice/source-url source-url
|
|
:invoice/invoice-number invoice-number
|
|
:invoice/total (Double/parseDouble total)
|
|
:invoice/date (to-date date)
|
|
:invoice/location matching-location
|
|
:invoice/import-status (or import-status :import-status/pending)
|
|
:invoice/outstanding-balance (Double/parseDouble total)
|
|
:invoice/status :invoice-status/unpaid}))
|
|
|
|
(defn validate-invoice [invoice user]
|
|
(when-not (:invoice/client invoice)
|
|
(throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.")
|
|
{:invoice-number (:invoice/invoice-number invoice)
|
|
:customer-identifier (:invoice/client-identifier invoice)})))
|
|
(assert-can-see-client user (:invoice/client invoice))
|
|
(doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]]
|
|
(when (not (get invoice k))
|
|
(throw (ex-info (str (name k) "not found on invoice " invoice)
|
|
invoice))))
|
|
|
|
invoice)
|
|
|
|
|
|
(defn admin-only-if-multiple-clients [is]
|
|
(let [client-count (->> is
|
|
(map :invoice/client)
|
|
set
|
|
count)]
|
|
(map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is)))
|
|
|
|
(defn import-uploaded-invoice [user imports]
|
|
(alog/info ::importing-uploaded :count (count imports))
|
|
(let [potential-invoices (->> imports
|
|
(map import->invoice)
|
|
(map #(validate-invoice % user))
|
|
admin-only-if-multiple-clients
|
|
(mapv d-invoices/code-invoice)
|
|
(mapv (fn [i] [:propose-invoice i])))]
|
|
|
|
(alog/info ::creating-invoice :invoices potential-invoices)
|
|
(let [tx (audit-transact potential-invoices user)]
|
|
(when-not (seq (dc/q '[:find ?i
|
|
:in $ [?i ...]
|
|
:where [?i :invoice/invoice-number]]
|
|
(:db-after tx)
|
|
(map :e (:tx-data tx))))
|
|
(throw (ex-info "No new invoices found."
|
|
{})))
|
|
tx)))
|
|
|
|
(defn validate-account-rows [rows code->existing-account]
|
|
(when-let [bad-types (seq (->> rows
|
|
(filter (fn [[account _ _ type]]
|
|
(and (not (code->existing-account (Integer/parseInt account)))
|
|
(not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))]
|
|
(throw (ex-info (str "You are adding accounts without a valid type")
|
|
{:rows bad-types})))
|
|
(when-let [duplicate-rows (seq (->> rows
|
|
(filter (fn [[account]]
|
|
(not-empty account)))
|
|
(group-by (fn [[account]]
|
|
account))
|
|
vals
|
|
(filter #(> (count %) 1))
|
|
(filter (fn [duplicates]
|
|
(apply not= duplicates)))
|
|
#_(map (fn [[[_ account]]]
|
|
account))))]
|
|
(throw (ex-info (str "You have duplicated rows with different values.")
|
|
{:rows duplicate-rows}))))
|
|
|
|
(defn import-account-overrides [customer filename]
|
|
(let [[_ & rows] (-> filename (io/reader) csv/read-csv)
|
|
[client-id] (first (dc/q (-> {:find ['?e]
|
|
:in ['$ '?z]
|
|
:where [['?e :client/code '?z]]}
|
|
(dc/db conn)
|
|
customer)))
|
|
code->existing-account (by :account/numeric-code (map first (dc/q {:find ['(pull ?e [:account/numeric-code
|
|
{:account/applicability [:db/ident]}
|
|
:db/id])]
|
|
:in ['$]
|
|
:where ['[?e :account/name]]}
|
|
(dc/db conn))))
|
|
|
|
existing-account-overrides (dc/q (-> {:find ['?e]
|
|
:in ['$ '?client-id]
|
|
:where [['?e :account-client-override/client '?client-id]]}
|
|
(dc/db conn)
|
|
client-id))
|
|
rows (transduce (comp
|
|
(map (fn [[_ account account-name override-name _ type]]
|
|
[account account-name override-name type]))
|
|
(filter (fn [[account]]
|
|
(not-empty account))))
|
|
conj
|
|
[]
|
|
rows)
|
|
|
|
_ (validate-account-rows rows code->existing-account)
|
|
rows (vec (set rows))
|
|
|
|
txes (transduce
|
|
(comp
|
|
(mapcat (fn parse-map [[account account-name override-name type]]
|
|
(let [code (some-> account
|
|
not-empty
|
|
Integer/parseInt)
|
|
existing (code->existing-account code)]
|
|
(cond (not code)
|
|
[]
|
|
|
|
(and existing (or (#{:account-applicability/optional :account-applicability/customized}
|
|
(:db/ident (:account/applicability existing)))
|
|
(and (not-empty override-name)
|
|
(not-empty account-name)
|
|
(not= override-name account-name))))
|
|
[{:db/id (:db/id existing)
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
|
|
(not existing)
|
|
[{:account/applicability :account-applicability/customized
|
|
:account/name account-name
|
|
:account/account-set "default"
|
|
:account/numeric-code code
|
|
:account/code (str code)
|
|
:account/type (if (str/blank? type)
|
|
:account-type/expense
|
|
(keyword "account-type" (str/lower-case type)))
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
:else
|
|
[])))))
|
|
|
|
conj
|
|
(mapv
|
|
(fn [[x]]
|
|
[:db/retractEntity x])
|
|
existing-account-overrides)
|
|
rows)]
|
|
|
|
@(dc/transact conn txes)
|
|
txes))
|
|
|
|
(defn import-transactions-cleared-against [file]
|
|
(let [[_ & rows] (-> file (io/reader) csv/read-csv)
|
|
txes (transduce
|
|
(comp
|
|
(filter (fn [[transaction-id _]]
|
|
(dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id))))
|
|
(map (fn [[transaction-id cleared-against]]
|
|
{:db/id (Long/parseLong transaction-id)
|
|
:transaction/cleared-against cleared-against})))
|
|
conj
|
|
[]
|
|
rows)]
|
|
(audit-transact txes nil)))
|
|
|
|
(defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}]
|
|
(assert-admin user)
|
|
(try
|
|
(let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))]
|
|
{:status 200
|
|
:body (pr-str stats)
|
|
:headers {"Content-Type" "application/edn"}})
|
|
(catch Exception e
|
|
(alog/error ::couldnt-batch-upload
|
|
:error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}})))
|
|
|
|
(defn upload-invoices [{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"
|
|
location :location
|
|
location-2 "location"
|
|
vendor :vendor
|
|
vendor-2 "vendor"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
location (or location location-2)
|
|
vendor (some-> (or vendor vendor-2)
|
|
(Long/parseLong))
|
|
{:keys [filename tempfile]} files]
|
|
(mu/with-context {:parsing-file filename}
|
|
(try
|
|
(let [extension (last (str/split (.getName (io/file filename)) #"\."))
|
|
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)
|
|
_ (s3/put-object (:data-bucket env)
|
|
s3-location
|
|
(io/input-stream tempfile)
|
|
{:content-type (if (= "csv" extension)
|
|
"text/csv"
|
|
"application/pdf")
|
|
:content-length (.length tempfile)})
|
|
imports (->> (parse/parse-file (.getPath tempfile) filename)
|
|
(map #(assoc %
|
|
:client-override client
|
|
:location-override location
|
|
:vendor-override vendor
|
|
:source-url (str "https://" (:data-bucket env)
|
|
"/"
|
|
s3-location))))]
|
|
(import-uploaded-invoice user imports))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(alog/warn ::couldnt-import-upload
|
|
:error e)
|
|
{:status 400
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}})))))
|
|
|
|
(defn cleared-against [{{files :file
|
|
files-2 "file"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
(import-transactions-cleared-against (.getPath tempfile))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(alog/error ::error :error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}}))))
|
|
|
|
|
|
|
|
(defn bulk-account-overrides [{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"} :params
|
|
user :identity}]
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
{:status 200
|
|
:body (import-account-overrides client (.getPath tempfile))
|
|
:headers {"Content-Type" "application/json"}}
|
|
(catch Exception e
|
|
(alog/error ::error :error e)
|
|
{:status 500
|
|
:body {:message (.getMessage e)
|
|
:data (ex-data e)}
|
|
:headers {"Content-Type" "application/json"}}))))
|
|
|
|
(def routes {"api/" {"transactions/" {:post {#"batch-upload/?" :batch-upload-transactions
|
|
#"cleared-against/?" :cleared-against}}
|
|
"invoices/" {:post {#"upload/?" :upload-invoices}}
|
|
:post {#"account-overrides/?" :bulk-account-overrides}}})
|
|
|
|
(def match->handler {:batch-upload-transactions (wrap-secure batch-upload-transactions)
|
|
:upload-invoices (wrap-secure upload-invoices)
|
|
:cleared-against (wrap-secure cleared-against)
|
|
:bulk-account-overrides (wrap-secure (wrap-json-response bulk-account-overrides))})
|