(ns auto-ap.routes.invoices (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [audit-transact conn]] [auto-ap.datomic.clients :as d-clients] [com.brunobonacci.mulog :as mu] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] [auto-ap.import.manual :as manual] [auto-ap.logging :as alog] [auto-ap.parse :as parse] [auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.utils :refer [by]] [clj-time.coerce :as coerce :refer [to-date]] [clojure.data.csv :as csv] [clojure.java.io :as io] [clojure.string :as str] [config.core :refer [env]] [datomic.api :as dc] [iol-ion.tx :refer [random-tempid]] [ring.middleware.json :refer [wrap-json-response]]) (:import (java.util UUID))) (defn match-vendor [vendor-code forced-vendor] (when (and (not forced-vendor) (str/blank? vendor-code)) (throw (ex-info (str "No vendor found. Please supply an forced vendor.") {:vendor-code vendor-code}))) (let [vendor-id (or forced-vendor (->> (dc/q {:find ['?vendor] :in ['$ '?vendor-name] :where ['[?vendor :vendor/name ?vendor-name]]} (dc/db conn) vendor-code) first first))] (when-not vendor-id (throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.") {:vendor-code vendor-code}))) (if-let [matching-vendor (->> (dc/q {:find [(list 'pull '?vendor-id d-vendors/default-read)] :in ['$ '?vendor-id]} (dc/db conn) vendor-id) first first)] matching-vendor (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) (defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}] (let [matching-client (cond account-number (:db/id (d-clients/exact-match account-number)) customer-identifier (:db/id (d-clients/best-match customer-identifier)) client-override (Long/parseLong client-override)) _ (alog/info ::client-matched :account-number account-number :customer-identifier customer-identifier :client-override client-override :matching (when matching-client (dc/pull (dc/db conn) [:client/name :client/code] matching-client))) matching-vendor (match-vendor vendor-code vendor-override) matching-location (or (when-not (str/blank? location-override) location-override) (parse/best-location-match (dc/pull (dc/db conn) [{:client/location-matches [:location-match/location :location-match/matches]} :client/default-location :client/locations] matching-client) text full-text))] #:invoice {:db/id (random-tempid) :invoice/client matching-client :invoice/client-identifier (or account-number customer-identifier) :invoice/vendor (:db/id matching-vendor) :invoice/source-url source-url :invoice/invoice-number invoice-number :invoice/total (Double/parseDouble total) :invoice/date (to-date date) :invoice/location matching-location :invoice/import-status (or import-status :import-status/pending) :invoice/outstanding-balance (Double/parseDouble total) :invoice/status :invoice-status/unpaid})) (defn validate-invoice [invoice user] (when-not (:invoice/client invoice) (throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.") {:invoice-number (:invoice/invoice-number invoice) :customer-identifier (:invoice/client-identifier invoice)}))) (assert-can-see-client user (:invoice/client invoice)) (doseq [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]] (when (not (get invoice k)) (throw (ex-info (str (name k) "not found on invoice " invoice) invoice)))) invoice) (defn admin-only-if-multiple-clients [is] (let [client-count (->> is (map :invoice/client) set count)] (map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is))) (defn import-uploaded-invoice [user imports] (alog/info ::importing-uploaded :count (count imports)) (let [potential-invoices (->> imports (map import->invoice) (map #(validate-invoice % user)) admin-only-if-multiple-clients (mapv d-invoices/code-invoice) (mapv (fn [i] [:propose-invoice i])))] (alog/info ::creating-invoice :invoices potential-invoices) (let [tx (audit-transact potential-invoices user)] (when-not (seq (dc/q '[:find ?i :in $ [?i ...] :where [?i :invoice/invoice-number]] (:db-after tx) (map :e (:tx-data tx)))) (throw (ex-info "No new invoices found." {}))) tx))) (defn validate-account-rows [rows code->existing-account] (when-let [bad-types (seq (->> rows (filter (fn [[account _ _ type]] (and (not (code->existing-account (Integer/parseInt account))) (not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))] (throw (ex-info (str "You are adding accounts without a valid type") {:rows bad-types}))) (when-let [duplicate-rows (seq (->> rows (filter (fn [[account]] (not-empty account))) (group-by (fn [[account]] account)) vals (filter #(> (count %) 1)) (filter (fn [duplicates] (apply not= duplicates))) #_(map (fn [[[_ account]]] account))))] (throw (ex-info (str "You have duplicated rows with different values.") {:rows duplicate-rows})))) (defn import-account-overrides [customer filename] (let [[_ & rows] (-> filename (io/reader) csv/read-csv) [client-id] (first (dc/q (-> {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} (dc/db conn) customer))) code->existing-account (by :account/numeric-code (map first (dc/q {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} (dc/db conn)))) existing-account-overrides (dc/q (-> {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} (dc/db conn) client-id)) rows (transduce (comp (map (fn [[_ account account-name override-name _ type]] [account account-name override-name type])) (filter (fn [[account]] (not-empty account)))) conj [] rows) _ (validate-account-rows rows code->existing-account) rows (vec (set rows)) txes (transduce (comp (mapcat (fn parse-map [[account account-name override-name type]] (let [code (some-> account not-empty Integer/parseInt) existing (code->existing-account code)] (cond (not code) [] (and existing (or (#{:account-applicability/optional :account-applicability/customized} (:db/ident (:account/applicability existing))) (and (not-empty override-name) (not-empty account-name) (not= override-name account-name)))) [{:db/id (:db/id existing) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] (not existing) [{:account/applicability :account-applicability/customized :account/name account-name :account/account-set "default" :account/numeric-code code :account/code (str code) :account/type (if (str/blank? type) :account-type/expense (keyword "account-type" (str/lower-case type))) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] :else []))))) conj (mapv (fn [[x]] [:db/retractEntity x]) existing-account-overrides) rows)] @(dc/transact conn txes) txes)) (defn import-transactions-cleared-against [file] (let [[_ & rows] (-> file (io/reader) csv/read-csv) txes (transduce (comp (filter (fn [[transaction-id _]] (dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id)))) (map (fn [[transaction-id cleared-against]] {:db/id (Long/parseLong transaction-id) :transaction/cleared-against cleared-against}))) conj [] rows)] (audit-transact txes nil))) (defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}] (assert-admin user) (try (let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))] {:status 200 :body (pr-str stats) :headers {"Content-Type" "application/edn"}}) (catch Exception e (alog/error ::couldnt-batch-upload :error e) {:status 500 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}}))) (defn upload-invoices [{{files :file files-2 "file" client :client client-2 "client" location :location location-2 "location" vendor :vendor vendor-2 "vendor"} :params user :identity}] (let [files (or files files-2) client (or client client-2) location (or location location-2) vendor (some-> (or vendor vendor-2) (Long/parseLong)) {:keys [filename tempfile]} files] (mu/with-context {:parsing-file filename} (try (let [extension (last (str/split (.getName (io/file filename)) #"\.")) s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension) _ (s3/put-object (:data-bucket env) s3-location (io/input-stream tempfile) {:content-type (if (= "csv" extension) "text/csv" "application/pdf") :content-length (.length tempfile)}) imports (->> (parse/parse-file (.getPath tempfile) filename :allow-glimpse? true) (map #(assoc % :client-override client :location-override location :vendor-override vendor :source-url (str "https://" (:data-bucket env) "/" s3-location))))] (import-uploaded-invoice user imports)) {:status 200 :body (pr-str {}) :headers {"Content-Type" "application/edn"}} (catch Exception e (alog/warn ::couldnt-import-upload :error e) {:status 400 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}}))))) (defn cleared-against [{{files :file files-2 "file"} :params user :identity}] (let [files (or files files-2) {:keys [tempfile]} files] (assert-admin user) (try (import-transactions-cleared-against (.getPath tempfile)) {:status 200 :body (pr-str {}) :headers {"Content-Type" "application/edn"}} (catch Exception e (alog/error ::error :error e) {:status 500 :body (pr-str {:message (.getMessage e) :error (.toString e) :data (ex-data e)}) :headers {"Content-Type" "application/edn"}})))) (defn bulk-account-overrides [{{files :file files-2 "file" client :client client-2 "client"} :params user :identity}] (let [files (or files files-2) client (or client client-2) {:keys [tempfile]} files] (assert-admin user) (try {:status 200 :body (import-account-overrides client (.getPath tempfile)) :headers {"Content-Type" "application/json"}} (catch Exception e (alog/error ::error :error e) {:status 500 :body {:message (.getMessage e) :data (ex-data e)} :headers {"Content-Type" "application/json"}})))) (def routes {"api/" {"transactions/" {:post {#"batch-upload/?" :batch-upload-transactions #"cleared-against/?" :cleared-against}} "invoices/" {:post {#"upload/?" :upload-invoices}} :post {#"account-overrides/?" :bulk-account-overrides}}}) (def match->handler {:batch-upload-transactions (wrap-secure batch-upload-transactions) :upload-invoices (wrap-secure upload-invoices) :cleared-against (wrap-secure cleared-against) :bulk-account-overrides (wrap-secure (wrap-json-response bulk-account-overrides))})