From 7c01a04ee8a4614025bc19d5fcc55a7f4ba2c4b2 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Mon, 11 Feb 2019 20:48:43 -0800 Subject: [PATCH] Added the ability to import invoices again --- src/clj/auto_ap/datomic/invoices.clj | 7 ++- src/clj/auto_ap/datomic/migrate.clj | 8 ++- .../datomic/migrate/invoice_converter.clj | 58 +++++++++++++++++++ .../auto_ap/datomic/migrate/rename_codes.clj | 18 +++--- src/clj/auto_ap/graphql.clj | 2 +- src/clj/auto_ap/graphql/invoices.clj | 1 + src/clj/auto_ap/parse.clj | 28 ++++++--- src/clj/auto_ap/parse/templates.clj | 10 +++- src/clj/auto_ap/routes/invoices.clj | 58 +++++++++++++++++++ .../views/components/invoices/side_bar.cljs | 9 ++- src/cljs/auto_ap/views/main.cljs | 5 ++ .../auto_ap/views/pages/import_invoices.cljs | 12 +++- .../auto_ap/views/pages/unpaid_invoices.cljs | 2 +- 13 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 src/clj/auto_ap/datomic/migrate/invoice_converter.clj diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj index df840b33..c4de6557 100644 --- a/src/clj/auto_ap/datomic/invoices.clj +++ b/src/clj/auto_ap/datomic/invoices.clj @@ -1,7 +1,8 @@ (ns auto-ap.datomic.invoices (:require [datomic.api :as d] - [auto-ap.datomic :refer [uri]] + [auto-ap.datomic :refer [uri remove-nils]] [auto-ap.graphql.utils :refer [limited-clients]] + [auto-ap.parse :as parse] [clj-time.coerce :as c] [clojure.set :refer [rename-keys]] [clojure.string :as str])) @@ -49,6 +50,8 @@ (:original-id args) (add-arg '?original-id (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong ) '[?e :invoice/client ?c] '[?c :client/original-id ?original-id]) + (:import-status args) (add-arg '?import-status (keyword "import-status" (:import-status args)) + '[?e :invoice/import-status ?import-status]) (:status args) (add-arg '?status (keyword "invoice-status" (:status args)) '[?e :invoice/status ?status]) (:vendor-id args) (add-arg '?vendor-id (:vendor-id args) @@ -127,3 +130,5 @@ (map first) (<-datomic))) + + diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index e069b548..b0e94980 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -35,7 +35,7 @@ (defn -main [& args] (println "Creating database...") - #_(d/delete-database uri) + (d/delete-database uri) (d/create-database uri) (let [ @@ -55,6 +55,12 @@ :auto-ap/add-bank-account-codes {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes :requires [:auto-ap/add-bank-account-codes-schema]} :auto-ap/add-nick-the-greek {:txes [[{:client/name "Nick the Greek" :client/code "NGAK" :client/locations ["MH"] :client/bank-accounts [{:bank-account/code "NGAK-0" :bank-account/type :bank-account-type/cash :bank-account/name "Cash"}]}]] :requires [:auto-ap/add-bank-account-codes]} :auto-ap/rename-codes-1 {:txes-fn 'auto-ap.datomic.migrate.rename-codes/rename-codes-1 :requires [:auto-ap/add-nick-the-greek]} + :auto-ap/invoice-converter {:txes auto-ap.datomic.migrate.invoice-converter/add-matches :requires [:auto-ap/rename-codes-1]} + :auto-ap/starter {:txes auto-ap.datomic.migrate.invoice-converter/add-starter :requires [:auto-ap/invoice-converter]} + :auto-ap/add-default-location {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location :requires [:auto-ap/invoice-converter]} + :auto-ap/add-default-location-2 {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location-2 :requires [:auto-ap/add-default-location]} + :auto-ap/add-import-status {:txes auto-ap.datomic.migrate.invoice-converter/add-import-status :requires [:auto-ap/add-default-location-2]} + :auto-ap/add-import-status-existing-invoices {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-import-status-existing-invoices :requires [:auto-ap/add-import-status]} }] (println "Conforming database...") (println (c/ensure-conforms conn norms-map)) diff --git a/src/clj/auto_ap/datomic/migrate/invoice_converter.clj b/src/clj/auto_ap/datomic/migrate/invoice_converter.clj new file mode 100644 index 00000000..96d5dc63 --- /dev/null +++ b/src/clj/auto_ap/datomic/migrate/invoice_converter.clj @@ -0,0 +1,58 @@ +(ns auto-ap.datomic.migrate.invoice-converter + (:require [datomic.api :as d])) + +(def add-matches + [[{:db/ident :client/matches + :db/valueType :db.type/string + :db/cardinality :db.cardinality/many + :db/doc "The strings that match the client"} + {:db/ident :client/location-matches + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/many + :db/isComponent true + :db/doc "The mapping from string to location"} + + {:db/ident :location-match/matches + :db/valueType :db.type/string + :db/cardinality :db.cardinality/many + :db/doc "The strings that match the location"} + + {:db/ident :location-match/location + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The location of the location match"} + ]]) + +(def add-starter + [[{:db/id [:client/code "CBC"] + :client/matches ["campbell brewing company"] + :client/location-matches [{:location-match/location "CB" + :location-match/matches ["campbell brewing company"]}]}]]) + +(defn add-default-location [conn] + [[{:db/ident :client/default-location + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The default location if one can't be found"}]]) + +(defn add-default-location-2 [conn] + [[{:db/id [:client/code "CBC"] + :client/default-location "CB"}]]) + +(def add-import-status + [[{:db/ident :invoice/import-status + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The status of importing the transaction"} + + {:db/ident :import-status/pending} + {:db/ident :import-status/imported}]]) + +(defn add-import-status-existing-invoices [conn] + (let [existing-invoices (->> (d/query {:query {:find ['?e] + :in ['$] + :where ['[?e :invoice/invoice-number]]} + :args [(d/db conn)]}))] + [(map (fn [i] {:db/id (first i) + :invoice/import-status :import-status/imported}) + existing-invoices)])) diff --git a/src/clj/auto_ap/datomic/migrate/rename_codes.clj b/src/clj/auto_ap/datomic/migrate/rename_codes.clj index 326ffea4..7226f5e7 100644 --- a/src/clj/auto_ap/datomic/migrate/rename_codes.clj +++ b/src/clj/auto_ap/datomic/migrate/rename_codes.clj @@ -27,11 +27,15 @@ ) (defn rename-codes-1 [conn] - (apply concat [(rename "WE" "WME" (d/connect uri)) - (rename "HM" "HIM" (d/connect uri)) - (rename "BES" "SBE" (d/connect uri)) - (rename "BES" "SBE" (d/connect uri)) - (rename "ORA" "OMG" (d/connect uri)) - (rename "INT" "IGC" (d/connect uri)) - (rename "MV" "MVSC" (d/connect uri))])) + (let [result (apply concat [(rename "WE" "WME" (d/connect uri)) + (rename "HM" "HIM" (d/connect uri)) + (rename "BES" "SBE" (d/connect uri)) + (rename "BES" "SBE" (d/connect uri)) + (rename "ORA" "OMG" (d/connect uri)) + (rename "INT" "IGC" (d/connect uri)) + (rename "MV" "MVSC" (d/connect uri))])] + (if (seq (seq result)) + result + [[]])) + ) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 5aa9cbde..31991d0a 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -205,7 +205,7 @@ :queries {:invoice_page {:type '(list :invoice_page) - :args {:imported {:type 'Boolean} + :args {:import_status {:type 'String} :status {:type 'String} :client_id {:type :id} :vendor_id {:type :id} diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index 1d106544..4c6d4601 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -44,6 +44,7 @@ :invoice/invoice-number invoice_number :invoice/client client_id :invoice/vendor vendor_id + :invoice/import-status :import-status/imported :invoice/total total :invoice/outstanding-balance total :invoice/status :invoice-status/unpaid diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj index c8e80383..b8322722 100644 --- a/src/clj/auto_ap/parse.clj +++ b/src/clj/auto_ap/parse.clj @@ -38,9 +38,11 @@ str/trim ) [value-parser parser-params] (-> template :parser k)] (assoc result k (parse-value value-parser parser-params value)))) - {:vendor-code (:vendor template)}))])) + {:vendor-code (:vendor template) + :text text}))])) (defn parse [text] + (println text) (->> t/pdf-templates (filter (partial template-applies? text)) first @@ -67,13 +69,25 @@ [file filename] (excel/parse-file file filename)) -(defn best-match [clients client-identifier] +(defn best-match [clients invoice-client-name] (->> clients - (map (fn [client] - (if-let [matches (:client/matches client)] - [client (apply min (map #(m/jaccard (.toLowerCase client-identifier) %) matches))] - [client 1]))) + + (mapcat (fn [{:keys [:db/id :client/matches] :as client :or {matches []}}] + (println matches) + (map (fn [m] + [client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))]) + matches))) (filter #(< (second %) 0.25)) (sort-by second) - ffirst)) + +(defn best-location-match [client text] + (or (->> client + :client/location-matches + (mapcat (fn [{:keys [:location-match/location :location-match/matches]}] + (map (fn [match] [location match]) matches))) + (filter (fn [[location match]] (re-find (re-pattern (str "(?i)" match)) text)) ) + first + first) + (:client/default-location client) + (first (:client/locations client)))) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index eb204e11..8ab123a0 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -25,7 +25,15 @@ :date #"INVOICE DATE\s*\n.*\s+([0-9]+/[0-9]+/[0-9]+)" :total #"INVOICE TOTAL\s+([0-9.]+)"} :parser {:date [:clj-time "MM/dd/yy"]} - :multi #"\f\f"}]) + :multi #"\f\f"} + + {:vendor "Carbonic Service Inc" + :keywords [#"CARBONIC SERVICE INC"] + :extract {:invoice-number #"Invoice #\s*\n\s*[\w\.]+\s+[\w\./]+(.*)\s*\n" + :customer-identifier #"Bill To[^\n]+\n[^\n]*\n([\w ]+)\s{2,}" + :date #"Invoice #\s*\n\s*[\w\.]+\s+([\w\./]+)" + :total #"Total\s+\$([0-9.]+)"} + :parser {:date [:clj-time "MM/dd/yy"]}}]) (def excel-templates [{:vendor "Isp Productions" diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 5d4ad175..2cc7d1e6 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -114,6 +114,7 @@ :vendor vendor-id :client client-id :default-location default-location + :import-status :import-status/imported :default-expense-account default-expense-account :total total :outstanding-balance (if (= "Cash" check) @@ -143,6 +144,56 @@ (map remove-nils) )) + +(defn import-uploaded-invoice [imports] + (let [clients (d-clients/get-all) + _ (println imports) + + transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text] :as info}] + (let [[matching-vendor default-expense-account] (->> (d/query + (cond-> {:query {:find ['?vendor '?default-expense-account] + :in ['$ '?vendor-name] + :where ['[?vendor :vendor/name ?vendor-name] + '[?vendor :vendor/default-expense-account ?default-expense-account]]} + :args [(d/db (d/connect uri)) vendor-code]})) + first) + matching-client (parse/best-match clients customer-identifier) + matching-location (parse/best-location-match matching-client text ) + _ (println "LOCATION" matching-location) + + [existing-id existing-outstanding-balance existing-status import-status] (->> (d/query + (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status] + :in ['$ '?invoice-number '?vendor '?client] + :where '[[?e :invoice/invoice-number ?invoice-number] + [?e :invoice/vendor ?vendor] + [?e :invoice/client ?client] + [?e :invoice/outstanding-balance ?outstanding-balance] + [?e :invoice/status ?status] + [?e :invoice/import-status ?import-status]]} + :args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]})) + first)] + (if (= :import-status/imported import-status) + result + (conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client) + :invoice/vendor matching-vendor + :invoice/invoice-number invoice-number + :invoice/total (Double/parseDouble total) + :invoice/date (to-date date) + :invoice/import-status :import-status/pending + :invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total)) + :invoice/status (or existing-status :invoice-status/unpaid) + :invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:expense-account-id default-expense-account + :location matching-location + :amount (Double/parseDouble total)}]) + :db/id existing-id + }))) + )) + [] + imports)] + (println transactions) + @(d/transact (d/connect uri) transactions) + )) + (defroutes routes (wrap-routes (context "/" [] @@ -184,6 +235,13 @@ :headers {"Content-Type" "application/edn"}}))) (context "/invoices" [] + (POST "/upload" + {{ files "file"} :params :as params} + (let [{:keys [filename tempfile]} files] + (import-uploaded-invoice (parse/parse-file (.getPath tempfile) filename)) + {:status 200 + :body (pr-str {}) + :headers {"Content-Type" "application/edn"}})) (POST "/upload-integreat" {{:keys [excel-rows]} :edn-params user :identity} (assert-admin user) diff --git a/src/cljs/auto_ap/views/components/invoices/side_bar.cljs b/src/cljs/auto_ap/views/components/invoices/side_bar.cljs index bda9e7fd..f4923341 100644 --- a/src/cljs/auto_ap/views/components/invoices/side_bar.cljs +++ b/src/cljs/auto_ap/views/components/invoices/side_bar.cljs @@ -27,7 +27,14 @@ [:span {:class "icon icon-accounting-invoice-mail" :style {:font-size "25px"}}] - [:span {:class "name"} "Paid Invoices"]]]]] + [:span {:class "name"} "Paid Invoices"]]] + [:li.menu-item + [:a.item {:href (bidi/path-for routes/routes :import-invoices) + :class [(active-when ap = :import-invoices)]} + + [:span {:class "icon icon-accounting-invoice-mail" :style {:font-size "25px"}}] + + [:span {:class "name"} "Import Invoices"]]]]] [:div rest] [:div {:class "compose has-text-centered"} diff --git a/src/cljs/auto_ap/views/main.cljs b/src/cljs/auto_ap/views/main.cljs index 24240c1f..f4de920e 100644 --- a/src/cljs/auto_ap/views/main.cljs +++ b/src/cljs/auto_ap/views/main.cljs @@ -8,6 +8,7 @@ [auto-ap.views.utils :refer [active-when active-when= login-url dispatch-event]] [auto-ap.views.components.layouts :refer [side-bar-layout]] [auto-ap.views.pages.unpaid-invoices :refer [unpaid-invoices-page]] + [auto-ap.views.pages.import-invoices :refer [import-invoices-page]] [auto-ap.views.pages.paid-invoices :refer [paid-invoices-page]] [auto-ap.views.pages.transactions :refer [transactions-page]] [auto-ap.views.pages.login :refer [login-page]] @@ -25,6 +26,10 @@ (defmethod page :unpaid-invoices [_] (unpaid-invoices-page {:status "unpaid"})) +(defmethod page :import-invoices [_] + (import-invoices-page )) + + (defmethod page :paid-invoices [_] (unpaid-invoices-page {:status "paid"})) diff --git a/src/cljs/auto_ap/views/pages/import_invoices.cljs b/src/cljs/auto_ap/views/pages/import_invoices.cljs index 7d41cbb0..fcb89fb2 100644 --- a/src/cljs/auto_ap/views/pages/import_invoices.cljs +++ b/src/cljs/auto_ap/views/pages/import_invoices.cljs @@ -4,6 +4,8 @@ [auto-ap.events :as events] [auto-ap.subs :as subs] [auto-ap.entities.clients :as client] + [auto-ap.views.components.layouts :refer [side-bar-layout]] + [auto-ap.views.components.invoices.side-bar :refer [invoices-side-bar]] [auto-ap.entities.vendors :as vendor] [auto-ap.views.components.invoice-table :refer [invoice-table] :as invoice-table] [cljsjs.dropzone :as dropzone] @@ -56,7 +58,7 @@ (assoc-in [:status :loading] true) (assoc-in [::params] params)) :graphql {:token (-> cofx :db :user) - :query-obj (invoice-table/query (assoc params :imported false)) + :query-obj (invoice-table/query (assoc params :import-status "pending")) :on-success [::received]}})) (re-frame/reg-event-db @@ -88,7 +90,7 @@ :on-success on-success }})) -(def import-invoices-page +(def import-invoices-content (with-meta (fn [] (let [invoice-page (re-frame/subscribe [::invoice-page]) @@ -129,3 +131,9 @@ "Reject all"]])]])) {:component-will-mount (fn [] (re-frame/dispatch-sync [::invalidated]))})) + +(defn import-invoices-page [] + [side-bar-layout {:side-bar [invoices-side-bar {}] + :main [import-invoices-content ]}]) + + diff --git a/src/cljs/auto_ap/views/pages/unpaid_invoices.cljs b/src/cljs/auto_ap/views/pages/unpaid_invoices.cljs index 7719bf59..cb73742d 100644 --- a/src/cljs/auto_ap/views/pages/unpaid_invoices.cljs +++ b/src/cljs/auto_ap/views/pages/unpaid_invoices.cljs @@ -80,7 +80,7 @@ (assoc-in [:status :loading] true) (assoc-in [::params] params)) :graphql {:token (-> cofx :db :user) - :query-obj (invoice-table/query (-> params (assoc :imported true) (dissoc :invoice-number-like-current)) ) + :query-obj (invoice-table/query (-> params (assoc :import-status "imported") (dissoc :invoice-number-like-current)) ) :on-success [::received]}})) (re-frame/reg-event-db ::unmount-invoices