514 lines
29 KiB
Clojure
514 lines
29 KiB
Clojure
(ns auto-ap.routes.invoices
|
|
(:require [auto-ap.datomic :refer [remove-nils uri conn]]
|
|
[auto-ap.datomic.clients :as d-clients]
|
|
[auto-ap.datomic.invoices :as d-invoices]
|
|
[auto-ap.datomic.accounts :as a]
|
|
[auto-ap.datomic.vendors :as d-vendors]
|
|
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
|
|
[auto-ap.import.manual :as manual]
|
|
[auto-ap.import.manual.common :as c]
|
|
[auto-ap.parse :as parse]
|
|
[auto-ap.routes.utils :refer [wrap-secure]]
|
|
[auto-ap.utils :refer [by]]
|
|
[clj-time.coerce :as coerce :refer [to-date]]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.java.io :as io]
|
|
[clojure.string :as str]
|
|
[clojure.tools.logging :as log]
|
|
[compojure.core :refer [context defroutes POST wrap-routes]]
|
|
[datomic.api :as d]
|
|
[ring.middleware.json :refer [wrap-json-response]]
|
|
[unilog.context :as lc]
|
|
[amazonica.aws.s3 :as s3]
|
|
[config.core :refer [env]]
|
|
[digest])
|
|
(:import java.util.UUID))
|
|
|
|
(defn reset-id [i]
|
|
(update i :invoice-number
|
|
(fn [n] (if (re-matches #"#+" n)
|
|
nil
|
|
n))))
|
|
|
|
(defn assoc-client-code [i]
|
|
(let [[client-code default-location] (str/split (:location i) #"-")]
|
|
(cond-> i
|
|
client-code (assoc :client-code client-code)
|
|
default-location (assoc :default-location default-location)
|
|
(not client-code) (update :errors conj {:info "No client code found"})
|
|
(not default-location) (update :errors conj {:info "No default location found"}))))
|
|
|
|
(defn parse-client [{:keys [client-code client default-location]} clients]
|
|
(if-let [id (:db/id (or (clients client-code)
|
|
(clients client)))]
|
|
(do
|
|
(when (not ((set (:client/locations (or (clients client-code)
|
|
(clients client))))
|
|
default-location))
|
|
(throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'."))))
|
|
id)
|
|
(throw (Exception. (str "Client code '" client-code "' and client named '" client "' not found.")))))
|
|
|
|
(defn parse-invoice-number [{:keys [invoice-number]}]
|
|
(or invoice-number ""))
|
|
|
|
(defn parse-vendor [{:keys [vendor-name check]} vendors]
|
|
(let [v (vendors vendor-name)]
|
|
(cond v
|
|
v
|
|
|
|
(= "Cash" check)
|
|
nil
|
|
|
|
:else
|
|
(throw (Exception. (str "Vendor '" vendor-name "' not found."))))))
|
|
|
|
(defn parse-vendor-id [{:keys [vendor]}]
|
|
(:db/id vendor))
|
|
|
|
(defn parse-automatically-paid-when-due [{:keys [vendor client-id]}]
|
|
(boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id)))
|
|
|
|
(defn parse-schedule-payment-dom [{:keys [vendor client-id]}]
|
|
(get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor))
|
|
client-id))
|
|
|
|
(defn parse-invoice-rows [excel-rows]
|
|
(let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code]
|
|
all-vendors (by :vendor/name (d-vendors/get-graphql {}))
|
|
all-clients (d-clients/get-all)
|
|
all-clients (merge (by :client/code all-clients) (by :client/name all-clients))
|
|
rows (->> (str/split excel-rows #"\n")
|
|
(map #(str/split % #"\t"))
|
|
(map #(into {} (map (fn [c k] [k c]) % columns)))
|
|
(map reset-id)
|
|
(map assoc-client-code)
|
|
(map (c/parse-or-error :client-id #(parse-client % all-clients)))
|
|
(map (c/parse-or-error :vendor #(parse-vendor % all-vendors)))
|
|
(map (c/parse-or-error :vendor-id #(parse-vendor-id %)))
|
|
(map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
|
|
(map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
|
|
(map (c/parse-or-error :account-id c/parse-account-numeric-code))
|
|
(map (c/parse-or-error :invoice-number parse-invoice-number))
|
|
(map (c/parse-or-error :total c/parse-amount))
|
|
(map (c/parse-or-error :date c/parse-date)))]
|
|
|
|
rows))
|
|
|
|
(defn match-vendor [vendor-code forced-vendor]
|
|
(when (and (not forced-vendor) (str/blank? vendor-code))
|
|
(throw (ex-info (str "No vendor found. Please supply an forced vendor.")
|
|
{:vendor-code vendor-code})))
|
|
(let [vendor-id (or forced-vendor
|
|
(->> (d/query
|
|
{:query {:find ['?vendor]
|
|
:in ['$ '?vendor-name]
|
|
:where ['[?vendor :vendor/name ?vendor-name]]}
|
|
:args [(d/db (d/connect uri)) vendor-code]})
|
|
first
|
|
first))]
|
|
(when-not vendor-id
|
|
(throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.")
|
|
{:vendor-code vendor-code})))
|
|
|
|
(if-let [matching-vendor (->> (d/query
|
|
{:query {:find [(list 'pull '?vendor-id d-vendors/default-read)]
|
|
:in ['$ '?vendor-id]}
|
|
:args [(d/db (d/connect uri)) vendor-id]})
|
|
first
|
|
first)]
|
|
matching-vendor
|
|
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
|
|
{:vendor-code vendor-code})))))
|
|
|
|
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override]} clients]
|
|
(let [[matching-client similarity] (cond
|
|
account-number (parse/best-match clients account-number 0.0)
|
|
customer-identifier (parse/best-match clients customer-identifier)
|
|
client-override [(first (filter (fn [c]
|
|
(= (:db/id c) (Long/parseLong client-override)))
|
|
clients))
|
|
1.0])
|
|
matching-vendor (match-vendor vendor-code vendor-override)
|
|
matching-location (or (when-not (str/blank? location-override)
|
|
location-override)
|
|
(parse/best-location-match matching-client text full-text))]
|
|
(remove-nils #:invoice {:invoice/client (:db/id matching-client)
|
|
:invoice/client-identifier (or account-number customer-identifier)
|
|
:invoice/vendor (:db/id matching-vendor)
|
|
:invoice/similarity (some-> similarity double (#(- 1.0 %)))
|
|
:invoice/source-url source-url
|
|
:invoice/invoice-number invoice-number
|
|
:invoice/total (Double/parseDouble total)
|
|
:invoice/date (to-date date)
|
|
:invoice/location matching-location
|
|
:invoice/import-status :import-status/pending
|
|
:invoice/outstanding-balance (Double/parseDouble total)
|
|
:invoice/status :invoice-status/unpaid})))
|
|
|
|
(defn validate-invoice [invoice user]
|
|
(when-not (:invoice/client invoice)
|
|
(throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.")
|
|
{:invoice-number (:invoice/invoice-number invoice)
|
|
:customer-identifier (:invoice/client-identifier invoice)})))
|
|
(assert-can-see-client user (:invoice/client invoice))
|
|
(doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]]
|
|
(when (not (get invoice k))
|
|
(throw (ex-info (str (name k) "not found on invoice " invoice)
|
|
invoice))))
|
|
|
|
invoice)
|
|
|
|
(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}]
|
|
(try
|
|
(->> (d/query
|
|
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
|
|
:in ['$ '?invoice-number '?vendor '?client]
|
|
:where '[[?e :invoice/invoice-number ?invoice-number]
|
|
[?e :invoice/vendor ?vendor]
|
|
[?e :invoice/client ?client]
|
|
[?e :invoice/outstanding-balance ?outstanding-balance]
|
|
[?e :invoice/status ?status]
|
|
[?e :invoice/import-status ?import-status]
|
|
[?import-status :db/ident ?import-status2]]}
|
|
:args [(d/db (d/connect uri)) invoice-number vendor client]}))
|
|
first
|
|
boolean)
|
|
(catch Exception e
|
|
(throw (ex-info (str "Failed to find potential matching invoice with"
|
|
" invoice " invoice-number
|
|
" vendor " vendor
|
|
" client " client
|
|
". "
|
|
(.toString e))
|
|
{:invoice-number invoice-number})))))
|
|
|
|
(defn invoice-rows->transaction [rows user]
|
|
(->> rows
|
|
(mapcat (fn [{:keys [vendor-id total client-id date invoice-number default-location check automatically-paid-when-due]}]
|
|
(let [payment-id (.toString (java.util.UUID/randomUUID))
|
|
transaction-id (.toString (java.util.UUID/randomUUID))
|
|
invoice #:invoice {:db/id (.toString (java.util.UUID/randomUUID))
|
|
:vendor vendor-id
|
|
:client client-id
|
|
:default-location default-location
|
|
:location default-location
|
|
:import-status :import-status/imported
|
|
:automatically-paid-when-due automatically-paid-when-due
|
|
:total total
|
|
:outstanding-balance (if (= "Cash" check)
|
|
0.0
|
|
total)
|
|
:status (if (= "Cash" check)
|
|
:invoice-status/paid
|
|
:invoice-status/unpaid)
|
|
:invoice-number invoice-number
|
|
:date (to-date date)}
|
|
payment (when (= :invoice-status/paid (:invoice/status invoice))
|
|
#:invoice-payment {:invoice (:db/id invoice)
|
|
:amount (:invoice/total invoice)
|
|
:payment (remove-nils #:payment {:db/id payment-id
|
|
:vendor (:invoice/vendor invoice)
|
|
:client (:invoice/client invoice)
|
|
:type :payment-type/cash
|
|
:amount (:invoice/total invoice)
|
|
:status :payment-status/cleared
|
|
:date (:invoice/date invoice)})})
|
|
transaction (when (= :invoice-status/paid (:invoice/status invoice))
|
|
(let [[bank-account] (d/q '[:find [?ba ...]
|
|
:in $ ?c
|
|
:where [?c :client/bank-accounts ?ba]
|
|
[?ba :bank-account/type :bank-account-type/cash]
|
|
]
|
|
(d/db conn)
|
|
client-id)]
|
|
#:transaction {:amount (- (:invoice/total invoice))
|
|
:payment payment-id
|
|
:client (:invoice/client invoice)
|
|
:status "POSTED"
|
|
:bank-account bank-account
|
|
:id #_ {:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id)
|
|
:raw-id transaction-id
|
|
:vendor (:invoice/vendor invoice)
|
|
:description-original "Cash payment"
|
|
:date (to-date date)
|
|
:approval-status :transaction-approval-status/approved
|
|
:accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"]))
|
|
:transaction-account/location "A"
|
|
:transaction-account/amount (Math/abs (:invoice/total invoice))}]}))
|
|
]
|
|
[[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice)
|
|
user))]
|
|
(some-> payment remove-nils)
|
|
transaction])))
|
|
(filter identity)))
|
|
|
|
(defn admin-only-if-multiple-clients [is]
|
|
(let [client-count (->> is
|
|
(map :invoice/client)
|
|
set
|
|
count)]
|
|
(map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is)))
|
|
|
|
(defn import-uploaded-invoice [user imports]
|
|
(lc/with-context {:area "upload-invoice"}
|
|
(log/info "Number of invoices to import is" (count imports))
|
|
(let [clients (d-clients/get-all)
|
|
potential-invoices (->> imports
|
|
(mapv #(import->invoice % clients))
|
|
(mapv #(validate-invoice % user))
|
|
admin-only-if-multiple-clients
|
|
(filter #(not (extant-invoice? %)))
|
|
(mapv d-invoices/code-invoice)
|
|
(mapv (fn [i] [:propose-invoice i])))]
|
|
(when-not (seq potential-invoices)
|
|
(throw (ex-info "No new invoices found."
|
|
{})))
|
|
(log/info "creating invoice" potential-invoices)
|
|
@(d/transact (d/connect uri) potential-invoices))))
|
|
|
|
(defn validate-account-rows [rows code->existing-account]
|
|
(when-let [bad-types (seq (->> rows
|
|
(filter (fn [[account _ _ type]]
|
|
(and (not (code->existing-account (Integer/parseInt account)))
|
|
(not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))]
|
|
(throw (ex-info (str "You are adding accounts without a valid type")
|
|
{:rows bad-types})))
|
|
(when-let [duplicate-rows (seq (->> rows
|
|
(filter (fn [[account]]
|
|
(not-empty account)))
|
|
(group-by (fn [[account]]
|
|
account))
|
|
vals
|
|
(filter #(> (count %) 1))
|
|
(filter (fn [duplicates]
|
|
(apply not= duplicates)))
|
|
#_(map (fn [[[_ account]]]
|
|
account))))]
|
|
(throw (ex-info (str "You have duplicated rows with different values.")
|
|
{:rows duplicate-rows}))))
|
|
|
|
(defn import-account-overrides [customer filename]
|
|
(let [conn (d/connect uri)
|
|
[_ & rows] (-> filename (io/reader) csv/read-csv)
|
|
[client-id] (first (d/query (-> {:query {:find ['?e]
|
|
:in ['$ '?z]
|
|
:where [['?e :client/code '?z]]}
|
|
:args [(d/db (d/connect uri)) customer]})))
|
|
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
|
|
{:account/applicability [:db/ident]}
|
|
:db/id])]
|
|
:in ['$]
|
|
:where ['[?e :account/name]]}
|
|
:args [(d/db conn)]})))
|
|
|
|
existing-account-overrides (d/query (-> {:query {:find ['?e]
|
|
:in ['$ '?client-id]
|
|
:where [['?e :account-client-override/client '?client-id]]}
|
|
:args [(d/db (d/connect uri)) client-id]}))
|
|
rows (transduce (comp
|
|
(map (fn [[_ account account-name override-name _ type]]
|
|
[account account-name override-name type]))
|
|
(filter (fn [[account]]
|
|
(not-empty account))))
|
|
conj
|
|
[]
|
|
rows)
|
|
|
|
_ (validate-account-rows rows code->existing-account)
|
|
rows (vec (set rows))
|
|
|
|
txes (transduce
|
|
(comp
|
|
(mapcat (fn parse-map [[account account-name override-name type]]
|
|
(let [code (some-> account
|
|
not-empty
|
|
Integer/parseInt)
|
|
existing (code->existing-account code)]
|
|
(cond (not code)
|
|
[]
|
|
|
|
(and existing (or (#{:account-applicability/optional :account-applicability/customized}
|
|
(:db/ident (:account/applicability existing)))
|
|
(and (not-empty override-name)
|
|
(not-empty account-name)
|
|
(not= override-name account-name))))
|
|
[{:db/id (:db/id existing)
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
|
|
(not existing)
|
|
[{:account/applicability :account-applicability/customized
|
|
:account/name account-name
|
|
:account/account-set "default"
|
|
:account/numeric-code code
|
|
:account/code (str code)
|
|
:account/type (if (str/blank? type)
|
|
:account-type/expense
|
|
(keyword "account-type" (str/lower-case type)))
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
:else
|
|
[])))))
|
|
|
|
conj
|
|
(mapv
|
|
(fn [[x]]
|
|
[:db/retractEntity x])
|
|
existing-account-overrides)
|
|
rows)]
|
|
|
|
@(d/transact conn txes)
|
|
txes))
|
|
|
|
(defn import-transactions-cleared-against [file]
|
|
(let [[_ & rows] (-> file (io/reader) csv/read-csv)
|
|
txes (transduce
|
|
(comp
|
|
(filter (fn [[transaction-id _]]
|
|
(d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id))))
|
|
(map (fn [[transaction-id cleared-against]]
|
|
{:db/id (Long/parseLong transaction-id)
|
|
:transaction/cleared-against cleared-against})))
|
|
conj
|
|
[]
|
|
rows)]
|
|
@(d/transact (d/connect uri) txes)))
|
|
|
|
(defroutes routes
|
|
(wrap-routes
|
|
(context "/" []
|
|
(context "/transactions" []
|
|
(POST "/batch-upload"
|
|
{{:keys [data]} :edn-params user :identity}
|
|
(assert-admin user)
|
|
(try
|
|
(let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))]
|
|
{:status 200
|
|
:body (pr-str stats)
|
|
:headers {"Content-Type" "application/edn"}})
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}}))))
|
|
|
|
(context "/invoices" []
|
|
(POST "/upload"
|
|
{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"
|
|
location :location
|
|
location-2 "location"
|
|
vendor :vendor
|
|
vendor-2 "vendor"} :params
|
|
user :identity}
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
location (or location location-2)
|
|
vendor (some-> (or vendor vendor-2)
|
|
(Long/parseLong))
|
|
{:keys [filename tempfile]} files]
|
|
(lc/with-context {:parsing-file filename}
|
|
(try
|
|
(let [extension (last (str/split (.getName (io/file filename)) #"\."))
|
|
s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension)
|
|
_ (s3/put-object :bucket-name (:data-bucket env)
|
|
:key s3-location
|
|
:input-stream (io/input-stream tempfile)
|
|
:metadata {:content-type "application/pdf"})
|
|
imports (->> (parse/parse-file (.getPath tempfile) filename)
|
|
(map #(assoc %
|
|
:client-override client
|
|
:location-override location
|
|
:vendor-override vendor
|
|
:source-url (str "http://" (:data-bucket env)
|
|
".s3-website-us-east-1.amazonaws.com/"
|
|
s3-location))))]
|
|
(import-uploaded-invoice user imports))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(log/warn e)
|
|
{:status 400
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}})))))
|
|
|
|
(POST "/upload-integreat"
|
|
{{:keys [excel-rows]} :edn-params user :identity}
|
|
(assert-admin user)
|
|
(let [parsed-invoice-rows (parse-invoice-rows excel-rows)
|
|
existing-rows (set (d-invoices/get-existing-set))
|
|
grouped-rows (group-by
|
|
(fn [i]
|
|
(cond (seq (:errors i))
|
|
:error
|
|
|
|
(existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)])
|
|
:exists
|
|
|
|
:else
|
|
:new))
|
|
parsed-invoice-rows)
|
|
vendors-not-found (->> parsed-invoice-rows
|
|
(filter #(and (nil? (:vendor-id %))
|
|
(not= "Cash" (:check %))))
|
|
(map :vendor-name)
|
|
set)
|
|
_ @(d/transact (d/connect uri) (invoice-rows->transaction (:new grouped-rows)
|
|
user))]
|
|
{:status 200
|
|
:body (pr-str {:imported (count (:new grouped-rows))
|
|
:already-imported (count (:exists grouped-rows))
|
|
:vendors-not-found vendors-not-found
|
|
:errors (map #(dissoc % :date) (:error grouped-rows))})
|
|
:headers {"Content-Type" "application/edn"}})))
|
|
(POST "/transactions/cleared-against"
|
|
{{files :file
|
|
files-2 "file"} :params
|
|
user :identity}
|
|
(let [files (or files files-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
(import-transactions-cleared-against (.getPath tempfile))
|
|
{:status 200
|
|
:body (pr-str {})
|
|
:headers {"Content-Type" "application/edn"}}
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body (pr-str {:message (.getMessage e)
|
|
:error (.toString e)
|
|
:data (ex-data e)})
|
|
:headers {"Content-Type" "application/edn"}}))))
|
|
(wrap-json-response (POST "/account-overrides"
|
|
{{files :file
|
|
files-2 "file"
|
|
client :client
|
|
client-2 "client"} :params
|
|
user :identity}
|
|
(let [files (or files files-2)
|
|
client (or client client-2)
|
|
{:keys [tempfile]} files]
|
|
(assert-admin user)
|
|
(try
|
|
{:status 200
|
|
:body (import-account-overrides client (.getPath tempfile))
|
|
:headers {"Content-Type" "application/json"}}
|
|
(catch Exception e
|
|
(log/error e)
|
|
{:status 500
|
|
:body {:message (.getMessage e)
|
|
:data (ex-data e)}
|
|
:headers {"Content-Type" "application/json"}}))))))
|
|
wrap-secure))
|