diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 0f87a248..35eec030 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -483,7 +483,11 @@ :auto-ap/add-invoice-similarity {:txes [[{:db/ident :invoice/similarity :db/doc "How close an invoice matches its import" :db/valueType :db.type/double - :db/cardinality :db.cardinality/one}]]}} + :db/cardinality :db.cardinality/one}]]} + :auto-ap/add-source-url-admin-only {:txes [[{:db/ident :invoice/source-url-admin-only + :db/doc "Can only admins see this invoice?" + :db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one}]]}} diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index b81b81e5..432a897c 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -6,12 +6,12 @@ [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.checks :as gq-checks] [auto-ap.graphql.utils - :refer [->graphql - <-graphql + :refer [<-graphql assert-admin assert-can-see-client assert-power-user - enum->keyword]] + enum->keyword] + :as u] [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] [clj-time.core :as time] @@ -20,6 +20,13 @@ [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.api :as d])) +(defn ->graphql [invoice user ] + (if (= "admin" (:user/role user)) + (u/->graphql invoice) + (u/->graphql (if (:invoice/source-url-admin-only invoice) + (dissoc invoice :invoice/source-url) + invoice)))) + (defn get-invoice-page [context args _] (let [args (assoc args :id (:id context)) @@ -28,7 +35,7 @@ (<-graphql) (update :status enum->keyword "invoice-status") (d-invoices/get-graphql))] - [{:invoices (map ->graphql invoices) + [{:invoices (mapv #(->graphql % (:id context)) invoices) :outstanding outstanding :total invoice-count :count (count invoices) @@ -38,7 +45,7 @@ (defn get-all-invoices [context args _] (assert-admin (:id context)) (map - ->graphql + u/->graphql (first (d-invoices/get-graphql (assoc (<-graphql args) :count Integer/MAX_VALUE))))) @@ -147,7 +154,7 @@ (let [transaction-result (audit-transact [(add-invoice-transaction in)] (:id context))] (-> (d-invoices/get-by-id (get-in transaction-result [:tempids "invoice"])) - (->graphql)))) + (->graphql (:id context))))) (defn assert-bank-account-belongs [client-id bank-account-id] (when-not ((set (map :db/id (:client/bank-accounts (d-clients/get-by-id client-id)))) bank-account-id) @@ -166,7 +173,7 @@ bank-account-id type (:id context)) - ->graphql))) + u/->graphql))) (defn edit-invoice [context {{:keys [id due invoice_number total date expense_accounts scheduled_payment] :as in} :invoice} _] (let [invoice (d-invoices/get-by-id id) @@ -196,7 +203,7 @@ (map (fn [d] [:db/retract id :invoice/expense-accounts d]) deleted)) (:id context)) (-> (d-invoices/get-by-id id) - (->graphql)))) + (->graphql (:id context))))) (defn void-invoice [context {id :invoice_id} _] (let [invoice (d-invoices/get-by-id id) @@ -210,7 +217,7 @@ (:invoice/expense-accounts invoice))}] (:id context)) - (-> (d-invoices/get-by-id id) (->graphql)))) + (-> (d-invoices/get-by-id id) (->graphql (:id context))))) (defn unvoid-invoice [context {id :invoice_id} _] (let [invoice (d-invoices/get-by-id id) @@ -239,7 +246,7 @@ (:id context)) (-> (d-invoices/get-by-id id) - (->graphql)))) + (->graphql (:id context))))) (defn unautopay-invoice [context {id :invoice_id} _] (let [invoice (d/entity (d/db conn) id) @@ -251,7 +258,7 @@ (:id context)) (-> (d-invoices/get-by-id id) - (->graphql)))) + (->graphql (:id context))))) (defn edit-expense-accounts [context args _] (assert-can-see-client (:id context) (:db/id (:invoice/client (d-invoices/get-by-id (:invoice_id args))))) @@ -268,27 +275,28 @@ (map (fn [d] [:db/retract invoice-id :invoice/expense-accounts d]) deleted)) (:id context)) (->graphql - (d-invoices/get-by-id (:invoice_id args))))) + (d-invoices/get-by-id (:invoice_id args)) + (:id context)))) (def objects {:invoice - {:fields {:id {:type :id} - :original_id {:type 'Int} - :client_identifier {:type 'String} - :total {:type 'String} - :source_url {:type 'String} - :outstanding_balance {:type 'String} - :invoice_number {:type 'String} - :status {:type 'String} - :expense_accounts {:type '(list :invoices_expense_accounts)} - :similarity {:type 'Float} - :date {:type :iso_date} - :due {:type :iso_date} - :client_id {:type 'Int} - :payments {:type '(list :invoice_payment)} - :vendor {:type :vendor} - :client {:type :client} - :scheduled_payment {:type :iso_date}}} + {:fields {:id {:type :id} + :original_id {:type 'Int} + :client_identifier {:type 'String} + :total {:type 'String} + :source_url {:type 'String} + :outstanding_balance {:type 'String} + :invoice_number {:type 'String} + :status {:type 'String} + :expense_accounts {:type '(list :invoices_expense_accounts)} + :similarity {:type 'Float} + :date {:type :iso_date} + :due {:type :iso_date} + :client_id {:type 'Int} + :payments {:type '(list :invoice_payment)} + :vendor {:type :vendor} + :client {:type :client} + :scheduled_payment {:type :iso_date}}} :invoices_expense_accounts {:fields {:id {:type :id} diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 45990005..8f542624 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -25,11 +25,11 @@ (defn reset-id [i] (update i :invoice-number (fn [n] (if (re-matches #"#+" n) - nil - n)))) + nil + n)))) (defn assoc-client-code [i] - (let [[client-code default-location] (str/split (:location i) #"-" )] + (let [[client-code default-location] (str/split (:location i) #"-")] (cond-> i client-code (assoc :client-code client-code) default-location (assoc :default-location default-location) @@ -38,13 +38,12 @@ (defn parse-client [{:keys [client-code client default-location]} clients] (if-let [id (:db/id (or (clients client-code) - (clients client)))] + (clients client)))] (do (when (not ((set (:client/locations (or (clients client-code) (clients client)))) default-location)) - (throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'."))) - ) + (throw (Exception. (str "Location '" default-location "' not found for client '" client-code "'.")))) id) (throw (Exception. (str "Client code '" client-code "' and client named '" client "' not found."))))) @@ -74,12 +73,12 @@ (defn parse-invoice-rows [excel-rows] (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code] - all-vendors (by :vendor/name (d-vendors/get-graphql {})) + all-vendors (by :vendor/name (d-vendors/get-graphql {})) all-clients (d-clients/get-all) all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) - rows (->> (str/split excel-rows #"\n" ) + rows (->> (str/split excel-rows #"\n") (map #(str/split % #"\t")) - (map #(into {} (map (fn [c k] [k c] ) % columns))) + (map #(into {} (map (fn [c k] [k c]) % columns))) (map reset-id) (map assoc-client-code) (map (c/parse-or-error :client-id #(parse-client % all-clients))) @@ -92,12 +91,8 @@ (map (c/parse-or-error :total c/parse-amount)) (map (c/parse-or-error :date c/parse-date)))] - rows)) - - - (defn match-vendor [vendor-code forced-vendor] (when (and (not forced-vendor) (str/blank? vendor-code)) (throw (ex-info (str "No vendor found. Please supply an forced vendor.") @@ -124,7 +119,6 @@ (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) - (defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override]} clients] (let [[matching-client similarity] (cond account-number (parse/best-match clients account-number 0.0) @@ -150,11 +144,6 @@ :invoice/outstanding-balance (Double/parseDouble total) :invoice/status :invoice-status/unpaid}))) - - - - - (defn validate-invoice [invoice user] (when-not (:invoice/client invoice) (throw (ex-info (str "Searched clients for '" (:invoice/client-identifier invoice) "'. No client found in file. Select a client first.") @@ -171,16 +160,16 @@ (defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}] (try (->> (d/query - (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] - :in ['$ '?invoice-number '?vendor '?client] - :where '[[?e :invoice/invoice-number ?invoice-number] - [?e :invoice/vendor ?vendor] - [?e :invoice/client ?client] - [?e :invoice/outstanding-balance ?outstanding-balance] - [?e :invoice/status ?status] - [?e :invoice/import-status ?import-status] - [?import-status :db/ident ?import-status2]]} - :args [(d/db (d/connect uri)) invoice-number vendor client]})) + (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] + :in ['$ '?invoice-number '?vendor '?client] + :where '[[?e :invoice/invoice-number ?invoice-number] + [?e :invoice/vendor ?vendor] + [?e :invoice/client ?client] + [?e :invoice/outstanding-balance ?outstanding-balance] + [?e :invoice/status ?status] + [?e :invoice/import-status ?import-status] + [?import-status :db/ident ?import-status2]]} + :args [(d/db (d/connect uri)) invoice-number vendor client]})) first boolean) (catch Exception e @@ -194,7 +183,7 @@ (defn invoice-rows->transaction [rows user] (->> rows - (mapcat (fn [{:keys [vendor-id total client-id amount date invoice-number default-location account-id check vendor automatically-paid-when-due schedule-payment-dom]}] + (mapcat (fn [{:keys [vendor-id total client-id date invoice-number default-location check automatically-paid-when-due]}] (let [invoice #:invoice {:db/id (.toString (java.util.UUID/randomUUID)) :vendor vendor-id :client client-id @@ -202,16 +191,16 @@ :location default-location :import-status :import-status/imported :automatically-paid-when-due automatically-paid-when-due - :total total + :total total :outstanding-balance (if (= "Cash" check) - 0.0 - total) + 0.0 + total) :status (if (= "Cash" check) - :invoice-status/paid - :invoice-status/unpaid) + :invoice-status/paid + :invoice-status/unpaid) :invoice-number invoice-number :date (to-date date)} - payment (if (= :invoice-status/paid (:invoice/status invoice)) + payment (when (= :invoice-status/paid (:invoice/status invoice)) #:invoice-payment {:invoice (:db/id invoice) :amount (:invoice/total invoice) :payment (remove-nils #:payment {:db/id (.toString (java.util.UUID/randomUUID)) @@ -219,21 +208,28 @@ :client (:invoice/client invoice) :type :payment-type/cash :amount (:invoice/total invoice) - :status :payment-status/cleared - :date (:invoice/date invoice)})} - )] + :status :payment-status/cleared + :date (:invoice/date invoice)})})] [[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice) user))] (some-> payment remove-nils)]))) (filter identity))) +(defn admin-only-if-multiple-clients [is] + (let [client-count (->> is + (map :invoice/client) + set + count)] + (map #(assoc % :invoice/source-url-admin-only (boolean (> client-count 1))) is))) + (defn import-uploaded-invoice [user imports] (lc/with-context {:area "upload-invoice"} - (log/info "Number of invoices to import is" (count imports) ) + (log/info "Number of invoices to import is" (count imports)) (let [clients (d-clients/get-all) potential-invoices (->> imports (mapv #(import->invoice % clients)) (mapv #(validate-invoice % user)) + admin-only-if-multiple-clients (filter #(not (extant-invoice? %))) (mapv d-invoices/code-invoice) (mapv (fn [i] [:propose-invoice i])))] @@ -243,16 +239,14 @@ (log/info "creating invoice" potential-invoices) @(d/transact (d/connect uri) potential-invoices)))) - (defn validate-account-rows [rows code->existing-account] (when-let [bad-types (seq (->> rows - (filter (fn [[account _ _ type :as row]] + (filter (fn [[account _ _ type]] (and (not (code->existing-account (Integer/parseInt account))) - (not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type))) - ))))] - (throw (ex-info (str "You are adding accounts without a valid type" ) + (not (#{"Asset" "Liability" "Revenue" "Expense" "Equity" "Dividend"} type)))))))] + (throw (ex-info (str "You are adding accounts without a valid type") {:rows bad-types}))) -(when-let [duplicate-rows (seq (->> rows + (when-let [duplicate-rows (seq (->> rows (filter (fn [[account]] (not-empty account))) (group-by (fn [[account]] @@ -262,32 +256,28 @@ (filter (fn [duplicates] (apply not= duplicates))) #_(map (fn [[[_ account]]] - account)) - ))] - (throw (ex-info (str "You have duplicated rows with different values." ) + account))))] + (throw (ex-info (str "You have duplicated rows with different values.") {:rows duplicate-rows})))) (defn import-account-overrides [customer filename] (let [conn (d/connect uri) - [header & rows] (-> filename (io/reader) csv/read-csv) + [_ & rows] (-> filename (io/reader) csv/read-csv) [client-id] (first (d/query (-> {:query {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} :args [(d/db (d/connect uri)) customer]}))) - headers (map read-string header) code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} - :args [(d/db conn)]}))) + :args [(d/db conn)]}))) existing-account-overrides (d/query (-> {:query {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} :args [(d/db (d/connect uri)) client-id]})) - - rows (transduce (comp (map (fn [[_ account account-name override-name _ type]] [account account-name override-name type])) @@ -296,7 +286,7 @@ conj [] rows) - + _ (validate-account-rows rows code->existing-account) rows (vec (set rows)) @@ -314,8 +304,7 @@ (:db/ident (:account/applicability existing))) (and (not-empty override-name) (not-empty account-name) - (not= override-name account-name) - ))) + (not= override-name account-name)))) [{:db/id (:db/id existing) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) @@ -342,17 +331,15 @@ [:db/retractEntity x]) existing-account-overrides) rows)] - + @(d/transact conn txes) txes)) - - (defn import-transactions-cleared-against [file] - (let [[header & rows] (-> file (io/reader) csv/read-csv) + (let [[_ & rows] (-> file (io/reader) csv/read-csv) txes (transduce - (comp - (filter (fn [[transaction-id cleared-against]] + (comp + (filter (fn [[transaction-id _]] (d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id)))) (map (fn [[transaction-id cleared-against]] {:db/id (Long/parseLong transaction-id) @@ -362,140 +349,137 @@ rows)] @(d/transact (d/connect uri) txes))) - - - (defroutes routes (wrap-routes (context "/" [] - (context "/transactions" [] - (POST "/batch-upload" - {{:keys [data]} :edn-params user :identity} - (assert-admin user) - (try - (let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))] - {:status 200 - :body (pr-str stats) - :headers {"Content-Type" "application/edn"}}) - (catch Exception e - (log/error e) - {:status 500 - :body (pr-str {:message (.getMessage e) - :error (.toString e) - :data (ex-data e)}) - :headers {"Content-Type" "application/edn"}})))) + (context "/transactions" [] + (POST "/batch-upload" + {{:keys [data]} :edn-params user :identity} + (assert-admin user) + (try + (let [stats (manual/import-batch (manual/tabulate-data data) (:user/name user))] + {:status 200 + :body (pr-str stats) + :headers {"Content-Type" "application/edn"}}) + (catch Exception e + (log/error e) + {:status 500 + :body (pr-str {:message (.getMessage e) + :error (.toString e) + :data (ex-data e)}) + :headers {"Content-Type" "application/edn"}})))) - (context "/invoices" [] - (POST "/upload" + (context "/invoices" [] + (POST "/upload" + {{files :file + files-2 "file" + client :client + client-2 "client" + location :location + location-2 "location" + vendor :vendor + vendor-2 "vendor"} :params + user :identity} + (let [files (or files files-2) + client (or client client-2) + location (or location location-2) + vendor (some-> (or vendor vendor-2) + (Long/parseLong)) + {:keys [filename tempfile]} files] + (lc/with-context {:parsing-file filename} + (try + (let [extension (last (str/split (.getName (io/file filename)) #"\.")) + s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension) + _ (s3/put-object :bucket-name (:data-bucket env) + :key s3-location + :input-stream (io/input-stream tempfile) + :metadata {:content-type "application/pdf"}) + imports (->> (parse/parse-file (.getPath tempfile) filename) + (map #(assoc % + :client-override client + :location-override location + :vendor-override vendor + :source-url (str "http://" (:data-bucket env) + ".s3-website-us-east-1.amazonaws.com/" + s3-location))))] + (import-uploaded-invoice user imports)) + {:status 200 + :body (pr-str {}) + :headers {"Content-Type" "application/edn"}} + (catch Exception e + (log/warn e) + {:status 400 + :body (pr-str {:message (.getMessage e) + :error (.toString e) + :data (ex-data e)}) + :headers {"Content-Type" "application/edn"}}))))) + + (POST "/upload-integreat" + {{:keys [excel-rows]} :edn-params user :identity} + (assert-admin user) + (let [parsed-invoice-rows (parse-invoice-rows excel-rows) + existing-rows (set (d-invoices/get-existing-set)) + grouped-rows (group-by + (fn [i] + (cond (seq (:errors i)) + :error + + (existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)]) + :exists + + :else + :new)) + parsed-invoice-rows) + vendors-not-found (->> parsed-invoice-rows + (filter #(and (nil? (:vendor-id %)) + (not= "Cash" (:check %)))) + (map :vendor-name) + set) + _ @(d/transact (d/connect uri) (invoice-rows->transaction (:new grouped-rows) + user))] + {:status 200 + :body (pr-str {:imported (count (:new grouped-rows)) + :already-imported (count (:exists grouped-rows)) + :vendors-not-found vendors-not-found + :errors (map #(dissoc % :date) (:error grouped-rows))}) + :headers {"Content-Type" "application/edn"}}))) + (POST "/transactions/cleared-against" + {{files :file + files-2 "file"} :params + user :identity} + (let [files (or files files-2) + {:keys [tempfile]} files] + (assert-admin user) + (try + (import-transactions-cleared-against (.getPath tempfile)) + {:status 200 + :body (pr-str {}) + :headers {"Content-Type" "application/edn"}} + (catch Exception e + (log/error e) + {:status 500 + :body (pr-str {:message (.getMessage e) + :error (.toString e) + :data (ex-data e)}) + :headers {"Content-Type" "application/edn"}})))) + (wrap-json-response (POST "/account-overrides" {{files :file files-2 "file" client :client - client-2 "client" - location :location - location-2 "location" - vendor :vendor - vendor-2 "vendor"} :params :as params + client-2 "client"} :params user :identity} (let [files (or files files-2) client (or client client-2) - location (or location location-2) - vendor (some-> (or vendor vendor-2) - (Long/parseLong)) - {:keys [filename tempfile]} files] - (lc/with-context {:parsing-file filename} - (try - (let [extension (last (str/split (.getName (io/file filename)) #"\." )) - s3-location (str "invoice-files/" (str (UUID/randomUUID)) "." extension) - _ (s3/put-object :bucket-name (:data-bucket env) - :key s3-location - :input-stream (io/input-stream tempfile) - :metadata {:content-type "application/pdf"}) - imports (->> (parse/parse-file (.getPath tempfile) filename) - (map #(assoc % - :client-override client - :location-override location - :vendor-override vendor - :source-url (str "http://" (:data-bucket env) - ".s3-website-us-east-1.amazonaws.com/" - s3-location))))] - (import-uploaded-invoice user imports)) - {:status 200 - :body (pr-str {}) - :headers {"Content-Type" "application/edn"}} - (catch Exception e - (log/warn e) - {:status 400 - :body (pr-str {:message (.getMessage e) - :error (.toString e) - :data (ex-data e)}) - :headers {"Content-Type" "application/edn"}}))))) - - (POST "/upload-integreat" - {{:keys [excel-rows]} :edn-params user :identity} - (assert-admin user) - (let [parsed-invoice-rows (parse-invoice-rows excel-rows) - existing-rows (set (d-invoices/get-existing-set )) - grouped-rows (group-by - (fn [i] - (cond (seq (:errors i)) - :error - - (existing-rows [(:vendor-id i) (:client-id i) (:invoice-number i)]) - :exists - - :else - :new)) - parsed-invoice-rows) - vendors-not-found (->> parsed-invoice-rows - (filter #(and (nil? (:vendor-id %)) - (not= "Cash" (:check %)))) - (map :vendor-name) - set) - inserted-rows @(d/transact (d/connect uri) (invoice-rows->transaction (:new grouped-rows) - user))] - {:status 200 - :body (pr-str {:imported (count (:new grouped-rows)) - :already-imported (count (:exists grouped-rows)) - :vendors-not-found vendors-not-found - :errors (map #(dissoc % :date) (:error grouped-rows))}) - :headers {"Content-Type" "application/edn"}}))) - (POST "/transactions/cleared-against" - {{files :file - files-2 "file"} :params :as params - user :identity} - (let [files (or files files-2) - {:keys [filename tempfile]} files] + {:keys [tempfile]} files] (assert-admin user) (try - (import-transactions-cleared-against (.getPath tempfile)) {:status 200 - :body (pr-str {}) - :headers {"Content-Type" "application/edn"}} + :body (import-account-overrides client (.getPath tempfile)) + :headers {"Content-Type" "application/json"}} (catch Exception e (log/error e) {:status 500 - :body (pr-str {:message (.getMessage e) - :error (.toString e) - :data (ex-data e)}) - :headers {"Content-Type" "application/edn"}})))) - (wrap-json-response (POST "/account-overrides" - {{files :file - files-2 "file" - client :client - client-2 "client"} :params :as params - user :identity} - (let [files (or files files-2) - client (or client client-2) - {:keys [filename tempfile]} files] - (assert-admin user) - (try - {:status 200 - :body (import-account-overrides client (.getPath tempfile)) - :headers {"Content-Type" "application/json"}} - (catch Exception e - (log/error e) - {:status 500 - :body {:message (.getMessage e) - :data (ex-data e)} - :headers {"Content-Type" "application/json"}})))))) + :body {:message (.getMessage e) + :data (ex-data e)} + :headers {"Content-Type" "application/json"}})))))) wrap-secure))