(ns auto-ap.routes.invoices (:require [auto-ap.datomic :refer [remove-nils uri conn]] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] [auto-ap.import.manual :as manual] [auto-ap.import.manual.common :as c] [auto-ap.parse :as parse] [auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.utils :refer [by]] [clj-time.coerce :as coerce :refer [to-date]] [clojure.data.csv :as csv] [clojure.java.io :as io] [clojure.string :as str] [clojure.tools.logging :as log] [compojure.core :refer [context defroutes POST wrap-routes]] [datomic.api :as d] [ring.middleware.json :refer [wrap-json-response]] [unilog.context :as lc] [amazonica.aws.s3 :as s3] [config.core :refer [env]] [digest]) (:import java.util.UUID)) (defn reset-id [i] (update i :invoice-number (fn [n] (if (re-matches #"#+" n) nil n)))) (defn assoc-client-code [i] (let [[client-code default-location] (str/split (:location i) #"-")] (cond-> i client-code (assoc :client-code client-code) default-location (assoc :default-location default-location) (not client-code) (update :errors conj {:info "No client code found"}) (not default-location) (update :errors conj {:info "No default location found"})))) (defn parse-client [{:keys [client-code client default-location]} clients] (if-let [id (:db/id (or (clients client-code) (clients client)))] (do (when (not ((set (:client/locations (or (clients client-code) (clients client)))) default-location)) (throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'.")))) id) (throw (Exception. (str "Client code '" client-code "' and client named '" client "' not found."))))) (defn parse-invoice-number [{:keys [invoice-number]}] (or invoice-number "")) (defn parse-vendor [{:keys [vendor-name check]} vendors] (let [v (vendors vendor-name)] (cond v v (= "Cash" check) nil :else (throw (Exception. (str "Vendor '" vendor-name "' not found.")))))) (defn parse-vendor-id [{:keys [vendor]}] (:db/id vendor)) (defn parse-automatically-paid-when-due [{:keys [vendor client-id]}] (boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id))) (defn parse-schedule-payment-dom [{:keys [vendor client-id]}] (get (by (comp :db/id :vendor-schedule-payment-dom/client) :vendor-schedule-payment-dom/dom (:vendor/schedule-payment-dom vendor)) client-id)) (defn parse-invoice-rows [excel-rows] (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code] all-vendors (by :vendor/name (d-vendors/get-graphql {})) all-clients (d-clients/get-all) all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) rows (->> (str/split excel-rows #"\n") (map #(str/split % #"\t")) (map #(into {} (map (fn [c k] [k c]) % columns))) (map reset-id) (map assoc-client-code) (map (c/parse-or-error :client-id #(parse-client % all-clients))) (map (c/parse-or-error :vendor #(parse-vendor % all-vendors))) (map (c/parse-or-error :vendor-id #(parse-vendor-id %))) (map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %))) (map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %))) (map (c/parse-or-error :account-id c/parse-account-numeric-code)) (map (c/parse-or-error :invoice-number parse-invoice-number)) (map (c/parse-or-error :total c/parse-amount)) (map (c/parse-or-error :date c/parse-date)))] rows)) (defn match-vendor [vendor-code forced-vendor] (when (and (not forced-vendor) (str/blank? vendor-code)) (throw (ex-info (str "No vendor found. Please supply an forced vendor.") {:vendor-code vendor-code}))) (let [vendor-id (or forced-vendor (->> (d/query {:query {:find ['?vendor] :in ['$ '?vendor-name] :where ['[?vendor :vendor/name ?vendor-name]]} :args [(d/db (d/connect uri)) vendor-code]}) first first))] (when-not vendor-id (throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.") {:vendor-code vendor-code}))) (if-let [matching-vendor (->> (d/query {:query {:find [(list 'pull '?vendor-id d-vendors/default-read)] :in ['$ '?vendor-id]} :args [(d/db (d/connect uri)) vendor-id]}) first first)] matching-vendor (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) (defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override]} clients] (let [[matching-client similarity] (cond account-number (parse/best-match clients account-number 0.0) customer-identifier (parse/best-match clients customer-identifier) client-override [(first (filter (fn [c] (= (:db/id c) (Long/parseLong client-override))) clients)) 1.0]) matching-vendor (match-vendor vendor-code vendor-override) matching-location (or (when-not (str/blank? location-override) location-override) (parse/best-location-match matching-client text full-text))] (remove-nils #:invoice {:invoice/client (:db/id matching-client) :invoice/client-identifier (or account-number customer-identifier) :invoice/vendor (:db/id matching-vendor) :invoice/similarity (some-> similarity double (#(- 1.0 %))) :invoice/source-url source-url :invoice/invoice-number invoice-number :invoice/total (Double/parseDouble total) :invoice/date (to-date date) :invoice/location matching-location :invoice/import-status :import-status/pending :invoice/outstanding-balance (Double/parseDouble total) :invoice/status :invoice-status/unpaid}))) (defn validate-invoice [invoice user] (when-not (:invoice/client invoice) (throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.") {:invoice-number (:invoice/invoice-number invoice) :customer-identifier (:invoice/client-identifier invoice)}))) (assert-can-see-client user (:invoice/client invoice)) (doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]] (when (not (get invoice k)) (throw (ex-info (str (name k) "not found on invoice " invoice) invoice)))) invoice) (defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}] (try (->> (d/query (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] :in ['$ '?invoice-number '?vendor '?client] :where '[[?e :invoice/invoice-number ?invoice-number] [?e :invoice/vendor ?vendor] [?e :invoice/client ?client] [?e :invoice/outstanding-balance ?outstanding-balance] [?e :invoice/status ?status] [?e :invoice/import-status ?import-status] [?import-status :db/ident ?import-status2]]} :args [(d/db (d/connect uri)) invoice-number vendor client]})) first boolean) (catch Exception e (throw (ex-info (str "Failed to find potential matching invoice with" " invoice " invoice-number " vendor " vendor " client " client ". " (.toString e)) {:invoice-number invoice-number}))))) (defn invoice-rows->transaction [rows user] (->> rows (mapcat (fn [{:keys [vendor-id total client-id date invoice-number default-location check automatically-paid-when-due]}] (let [payment-id (.toString (java.util.UUID/randomUUID)) transaction-id (.toString (java.util.UUID/randomUUID)) invoice #:invoice {:db/id (.toString (java.util.UUID/randomUUID)) :vendor vendor-id :client client-id :default-location default-location :location default-location :import-status :import-status/imported :automatically-paid-when-due automatically-paid-when-due :total total :outstanding-balance (if (= "Cash" check) 0.0 total) :status (if (= "Cash" check) :invoice-status/paid :invoice-status/unpaid) :invoice-number invoice-number :date (to-date date)} payment (when (= :invoice-status/paid (:invoice/status invoice)) #:invoice-payment {:invoice (:db/id invoice) :amount (:invoice/total invoice) :payment (remove-nils #:payment {:db/id payment-id :vendor (:invoice/vendor invoice) :client (:invoice/client invoice) :type :payment-type/cash :amount (:invoice/total invoice) :status :payment-status/cleared :date (:invoice/date invoice)})}) transaction (when (= :invoice-status/paid (:invoice/status invoice)) (let [[bank-account] (d/q '[:find [?ba ...] :in $ ?c :where [?c :client/bank-accounts ?ba] [?ba :bank-account/type :bank-account-type/cash] ] (d/db conn) client-id)] #:transaction {:amount (- (:invoice/total invoice)) :payment payment-id :client (:invoice/client invoice) :status "POSTED" :bank-account bank-account :id #_ {:clj-kondo/ignore [:unresolved-var]} (digest/sha-256 transaction-id) :raw-id transaction-id :vendor (:invoice/vendor invoice) :description-original "Cash payment" :date (to-date date) :approval-status :transaction-approval-status/approved :accounts [{:transaction-account/account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :transaction-account/location "A" :transaction-account/amount (Math/abs (:invoice/total invoice))}]})) ] [[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice) user))] (some-> payment remove-nils) transaction]))) (filter identity))) (defn admin-only-if-multiple-clients [is] (let [client-count (->> is (map :invoice/client) set count)] (map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is))) (defn import-uploaded-invoice [user imports] (lc/with-context {:area "upload-invoice"} (log/info "Number of invoices to import is" (count imports)) (let [clients (d-clients/get-all) potential-invoices (->> imports (mapv #(import->invoice % clients)) (mapv #(validate-invoice % user)) admin-only-if-multiple-clients (filter #(not (extant-invoice? %))) (mapv d-invoices/code-invoice) (mapv (fn [i] [:propose-invoice i])))] (when-not (seq potential-invoices) (throw (ex-info "No new invoices found." {}))) (log/info "creating invoice" potential-invoices) @(d/transact (d/connect uri) potential-invoices)))) (defn validate-account-rows [rows code->existing-account] (when-let [bad-types (seq (->> rows (filter (fn [[account _ _ type]] (and (not (code->existing-account (Integer/parseInt account))) (not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))] (throw (ex-info (str "You are adding accounts without a valid type") {:rows bad-types}))) (when-let [duplicate-rows (seq (->> rows (filter (fn [[account]] (not-empty account))) (group-by (fn [[account]] account)) vals (filter #(> (count %) 1)) (filter (fn [duplicates] (apply not= duplicates))) #_(map (fn [[[_ account]]] account))))] (throw (ex-info (str "You have duplicated rows with different values.") {:rows duplicate-rows})))) (defn import-account-overrides [customer filename] (let [conn (d/connect uri) [_ & rows] (-> filename (io/reader) csv/read-csv) [client-id] (first (d/query (-> {:query {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} :args [(d/db (d/connect uri)) customer]}))) code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} :args [(d/db conn)]}))) existing-account-overrides (d/query (-> {:query {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} :args [(d/db (d/connect uri)) client-id]})) rows (transduce (comp (map (fn [[_ account account-name override-name _ type]] [account account-name override-name type])) (filter (fn [[account]] (not-empty account)))) conj [] rows) _ (validate-account-rows rows code->existing-account) rows (vec (set rows)) txes (transduce (comp (mapcat (fn parse-map [[account account-name override-name type]] (let [code (some-> account not-empty Integer/parseInt) existing (code->existing-account code)] (cond (not code) [] (and existing (or (#{:account-applicability/optional :account-applicability/customized} (:db/ident (:account/applicability existing))) (and (not-empty override-name) (not-empty account-name) (not= override-name account-name)))) [{:db/id (:db/id existing) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] (not existing) [{:account/applicability :account-applicability/customized :account/name account-name :account/account-set "default" :account/numeric-code code :account/code (str code) :account/type (if (str/blank? type) :account-type/expense (keyword "account-type" (str/lower-case type))) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] :else []))))) conj (mapv (fn [[x]] [:db/retractEntity x]) existing-account-overrides) rows)] @(d/transact conn txes) txes)) (defn import-transactions-cleared-against [file] (let [[_ & rows] (-> file (io/reader) csv/read-csv) txes (transduce (comp (filter (fn [[transaction-id _]] (d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id)))) (map (fn [[transaction-id cleared-against]] {:db/id (Long/parseLong transaction-id) :transaction/cleared-against cleared-against}))) conj [] rows)] @(d/transact (d/connect uri) txes))) (defroutes routes (wrap-routes (context "/" [] (context "/transactions" [] (POST "/batch-upload" {{:keys [data]} :edn-params user :identity} (assert-admin user) (try (let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))] {:status 200 :body (pr-str stats) :headers {"Content-Type" "application/edn"}}) (catch Exception e (log/error e) {:status 500 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}})))) (context "/invoices" [] (POST "/upload" {{files :file files-2 "file" client :client client-2 "client" location :location location-2 "location" vendor :vendor vendor-2 "vendor"} :params user :identity} (let [files (or files files-2) client (or client client-2) location (or location location-2) vendor (some-> (or vendor vendor-2) (Long/parseLong)) {:keys [filename tempfile]} files] (lc/with-context {:parsing-file filename} (try (let [extension (last (str/split (.getName (io/file filename)) #"\.")) s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension) _ (s3/put-object :bucket-name (:data-bucket env) :key s3-location :input-stream (io/input-stream tempfile) :metadata {:content-type "application/pdf"}) imports (->> (parse/parse-file (.getPath tempfile) filename) (map #(assoc % :client-override client :location-override location :vendor-override vendor :source-url (str "http://" (:data-bucket env) ".s3-website-us-east-1.amazonaws.com/" s3-location))))] (import-uploaded-invoice user imports)) {:status 200 :body (pr-str {}) :headers {"Content-Type" "application/edn"}} (catch Exception e (log/warn e) {:status 400 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}}))))) (POST "/upload-integreat" {{:keys [excel-rows]} :edn-params user :identity} (assert-admin user) (let [parsed-invoice-rows (parse-invoice-rows excel-rows) existing-rows (set (d-invoices/get-existing-set)) grouped-rows (group-by (fn [i] (cond (seq (:errors i)) :error (existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)]) :exists :else :new)) parsed-invoice-rows) vendors-not-found (->> parsed-invoice-rows (filter #(and (nil? (:vendor-id %)) (not= "Cash" (:check %)))) (map :vendor-name) set) _ @(d/transact (d/connect uri) (invoice-rows->transaction (:new grouped-rows) user))] {:status 200 :body (pr-str {:imported (count (:new grouped-rows)) :already-imported (count (:exists grouped-rows)) :vendors-not-found vendors-not-found :errors (map #(dissoc % :date) (:error grouped-rows))}) :headers {"Content-Type" "application/edn"}}))) (POST "/transactions/cleared-against" {{files :file files-2 "file"} :params user :identity} (let [files (or files files-2) {:keys [tempfile]} files] (assert-admin user) (try (import-transactions-cleared-against (.getPath tempfile)) {:status 200 :body (pr-str {}) :headers {"Content-Type" "application/edn"}} (catch Exception e (log/error e) {:status 500 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}})))) (wrap-json-response (POST "/account-overrides" {{files :file files-2 "file" client :client client-2 "client"} :params user :identity} (let [files (or files files-2) client (or client client-2) {:keys [tempfile]} files] (assert-admin user) (try {:status 200 :body (import-account-overrides client (.getPath tempfile)) :headers {"Content-Type" "application/json"}} (catch Exception e (log/error e) {:status 500 :body {:message (.getMessage e) :data (ex-data e)} :headers {"Content-Type" "application/json"}})))))) wrap-secure))