Added the ability to import invoices again

src/clj/auto_ap/datomic/invoices.clj

@@ -1,7 +1,8 @@
 (ns auto-ap.datomic.invoices
   (:require [datomic.api :as d]
-            [auto-ap.datomic :refer [uri]]
+            [auto-ap.datomic :refer [uri remove-nils]]
             [auto-ap.graphql.utils :refer [limited-clients]]
+            [auto-ap.parse :as parse]
             [clj-time.coerce :as c]
             [clojure.set :refer [rename-keys]]
             [clojure.string :as str]))
@@ -49,6 +50,8 @@
           (:original-id args) (add-arg '?original-id (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong )
                                        '[?e :invoice/client ?c]
                                        '[?c :client/original-id ?original-id]) 
+          (:import-status args) (add-arg '?import-status (keyword "import-status" (:import-status args))
+                                  '[?e :invoice/import-status ?import-status])
           (:status args) (add-arg '?status (keyword "invoice-status" (:status args))
                                   '[?e :invoice/status ?status])
           (:vendor-id args) (add-arg '?vendor-id (:vendor-id args)
@@ -127,3 +130,5 @@
        (map first)
        
        (<-datomic)))
+
+

src/clj/auto_ap/datomic/migrate.clj

@@ -35,7 +35,7 @@
 
 (defn -main [& args]
     (println "Creating database...")
-  #_(d/delete-database uri)
+  (d/delete-database uri)
   (d/create-database uri)
   
   (let [
@@ -55,6 +55,12 @@
                      :auto-ap/add-bank-account-codes {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes :requires [:auto-ap/add-bank-account-codes-schema]}
                      :auto-ap/add-nick-the-greek {:txes [[{:client/name "Nick the Greek" :client/code "NGAK" :client/locations ["MH"] :client/bank-accounts [{:bank-account/code "NGAK-0" :bank-account/type :bank-account-type/cash :bank-account/name "Cash"}]}]] :requires [:auto-ap/add-bank-account-codes]}
                      :auto-ap/rename-codes-1 {:txes-fn 'auto-ap.datomic.migrate.rename-codes/rename-codes-1 :requires [:auto-ap/add-nick-the-greek]}
+                     :auto-ap/invoice-converter {:txes auto-ap.datomic.migrate.invoice-converter/add-matches :requires [:auto-ap/rename-codes-1]}
+                     :auto-ap/starter {:txes auto-ap.datomic.migrate.invoice-converter/add-starter :requires [:auto-ap/invoice-converter]}
+                     :auto-ap/add-default-location {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location :requires [:auto-ap/invoice-converter]}
+                     :auto-ap/add-default-location-2 {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location-2 :requires [:auto-ap/add-default-location]}
+                     :auto-ap/add-import-status {:txes auto-ap.datomic.migrate.invoice-converter/add-import-status :requires [:auto-ap/add-default-location-2]}
+                     :auto-ap/add-import-status-existing-invoices {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-import-status-existing-invoices :requires [:auto-ap/add-import-status]}
                      }]
       (println "Conforming database...")
       (println (c/ensure-conforms  conn norms-map))

src/clj/auto_ap/datomic/migrate/invoice_converter.clj

@@ -0,0 +1,58 @@
+(ns auto-ap.datomic.migrate.invoice-converter
+  (:require [datomic.api :as d]))
+
+(def add-matches
+  [[{:db/ident :client/matches
+     :db/valueType :db.type/string
+     :db/cardinality :db.cardinality/many
+     :db/doc "The strings that match the client"}
+    {:db/ident :client/location-matches
+     :db/valueType :db.type/ref
+     :db/cardinality :db.cardinality/many
+     :db/isComponent true
+     :db/doc "The mapping from string to location"}
+
+    {:db/ident :location-match/matches
+     :db/valueType :db.type/string
+     :db/cardinality :db.cardinality/many
+     :db/doc "The strings that match the location"}
+
+    {:db/ident :location-match/location
+     :db/valueType :db.type/string
+     :db/cardinality :db.cardinality/one
+     :db/doc "The location of the location match"}
+    ]])
+
+(def add-starter
+  [[{:db/id [:client/code "CBC"]
+     :client/matches ["campbell brewing company"]
+     :client/location-matches [{:location-match/location "CB"
+                                :location-match/matches ["campbell brewing company"]}]}]])
+
+(defn add-default-location [conn]
+  [[{:db/ident :client/default-location
+     :db/valueType :db.type/string
+     :db/cardinality :db.cardinality/one
+     :db/doc "The default location if one can't be found"}]])
+
+(defn add-default-location-2 [conn]
+  [[{:db/id [:client/code "CBC"]
+     :client/default-location "CB"}]])
+
+(def add-import-status
+  [[{:db/ident :invoice/import-status
+     :db/valueType :db.type/ref
+     :db/cardinality :db.cardinality/one
+     :db/doc "The status of importing the transaction"}
+
+    {:db/ident :import-status/pending}
+    {:db/ident :import-status/imported}]])
+
+(defn add-import-status-existing-invoices [conn]
+  (let [existing-invoices (->> (d/query {:query {:find ['?e]
+                                                 :in ['$]
+                                                 :where ['[?e :invoice/invoice-number]]}
+                                         :args [(d/db conn)]}))]
+    [(map (fn [i] {:db/id (first i)
+                   :invoice/import-status :import-status/imported})
+          existing-invoices)]))

src/clj/auto_ap/datomic/migrate/rename_codes.clj

@@ -27,11 +27,15 @@
   )
 
 (defn rename-codes-1 [conn]
-  (apply concat [(rename  "WE" "WME" (d/connect uri))
+  (let [result (apply concat [(rename  "WE" "WME" (d/connect uri))
-                 (rename  "HM" "HIM" (d/connect uri))
+                              (rename  "HM" "HIM" (d/connect uri))
-                 (rename  "BES" "SBE" (d/connect uri))
+                              (rename  "BES" "SBE" (d/connect uri))
-                 (rename  "BES" "SBE" (d/connect uri))
+                              (rename  "BES" "SBE" (d/connect uri))
-                 (rename  "ORA" "OMG" (d/connect uri))
+                              (rename  "ORA" "OMG" (d/connect uri))
-                 (rename  "INT" "IGC" (d/connect uri))
+                              (rename  "INT" "IGC" (d/connect uri))
-                 (rename  "MV" "MVSC" (d/connect uri))]))
+                              (rename  "MV" "MVSC" (d/connect uri))])]
+    (if (seq (seq result))
+      result
+      [[]]))
+  )
 

src/clj/auto_ap/graphql.clj

@@ -205,7 +205,7 @@
 
    :queries
    {:invoice_page {:type '(list :invoice_page)
-                   :args {:imported {:type 'Boolean}
+                   :args {:import_status {:type 'String}
                           :status {:type 'String}
                           :client_id {:type :id}
                           :vendor_id {:type :id}

src/clj/auto_ap/graphql/invoices.clj

@@ -44,6 +44,7 @@
                       :invoice/invoice-number invoice_number
                       :invoice/client client_id
                       :invoice/vendor vendor_id
+                      :invoice/import-status :import-status/imported
                       :invoice/total total
                       :invoice/outstanding-balance total
                       :invoice/status :invoice-status/unpaid

src/clj/auto_ap/parse.clj

@@ -38,9 +38,11 @@
                                   str/trim )
                     [value-parser parser-params] (-> template :parser k)]
                 (assoc result k (parse-value value-parser parser-params value))))
-            {:vendor-code (:vendor template)}))]))
+            {:vendor-code (:vendor template)
+             :text text}))]))
 
 (defn parse [text]
+  (println text)
   (->> t/pdf-templates
        (filter (partial template-applies? text))
        first
@@ -67,13 +69,25 @@
   [file filename]
   (excel/parse-file file filename))
 
-(defn best-match [clients client-identifier]
+(defn best-match [clients invoice-client-name]
   (->> clients
-       (map (fn [client]
+       
-              (if-let [matches (:client/matches client)]
+       (mapcat (fn [{:keys [:db/id :client/matches] :as client :or {matches []}}]
-                [client (apply min (map #(m/jaccard (.toLowerCase client-identifier) %) matches))]
+                 (println matches)
-                [client 1])))
+                 (map (fn [m]
+                        [client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))])
+                      matches)))
        (filter #(< (second %) 0.25))
        (sort-by second)
-       
        ffirst))
+
+(defn best-location-match [client text]
+  (or (->> client
+           :client/location-matches
+           (mapcat (fn [{:keys [:location-match/location :location-match/matches]}]
+                     (map (fn [match] [location match]) matches)))
+           (filter (fn [[location match]] (re-find (re-pattern (str "(?i)" match)) text)) )
+           first
+           first)
+      (:client/default-location client)
+      (first (:client/locations client))))

src/clj/auto_ap/parse/templates.clj

@@ -25,7 +25,15 @@
               :date #"INVOICE DATE\s*\n.*\s+([0-9]+/[0-9]+/[0-9]+)"
               :total #"INVOICE TOTAL\s+([0-9.]+)"}
     :parser {:date [:clj-time "MM/dd/yy"]}
-    :multi #"\f\f"}])
+    :multi #"\f\f"}
+
+   {:vendor "Carbonic Service Inc"
+    :keywords [#"CARBONIC SERVICE INC"]
+    :extract {:invoice-number #"Invoice #\s*\n\s*[\w\.]+\s+[\w\./]+(.*)\s*\n"
+              :customer-identifier #"Bill To[^\n]+\n[^\n]*\n([\w ]+)\s{2,}"
+              :date #"Invoice #\s*\n\s*[\w\.]+\s+([\w\./]+)"
+              :total #"Total\s+\$([0-9.]+)"}
+    :parser {:date [:clj-time "MM/dd/yy"]}}])
 
 (def excel-templates
   [{:vendor "Isp Productions"

src/clj/auto_ap/routes/invoices.clj

@@ -114,6 +114,7 @@
                                           :vendor vendor-id
                                           :client client-id
                                           :default-location default-location
+                                          :import-status :import-status/imported
                                           :default-expense-account default-expense-account
                                           :total total 
                                           :outstanding-balance (if (= "Cash" check)
@@ -143,6 +144,56 @@
        (map remove-nils)
        ))
 
+
+(defn import-uploaded-invoice [imports]
+  (let [clients (d-clients/get-all)
+        _ (println imports)
+
+        transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text] :as info}]
+                               (let [[matching-vendor default-expense-account] (->> (d/query
+                                                                                     (cond-> {:query {:find ['?vendor '?default-expense-account]
+                                                                                                      :in ['$ '?vendor-name]
+                                                                                                      :where ['[?vendor :vendor/name ?vendor-name]
+                                                                                                              '[?vendor :vendor/default-expense-account ?default-expense-account]]}
+                                                                                              :args [(d/db (d/connect uri)) vendor-code]}))
+                                                                                    first)
+                                     matching-client (parse/best-match clients customer-identifier)
+                                     matching-location (parse/best-location-match matching-client text )
+                                     _ (println "LOCATION" matching-location)
+
+                                     [existing-id existing-outstanding-balance existing-status import-status] (->> (d/query
+                                                                                                      (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status]
+                                                                                                                       :in ['$ '?invoice-number '?vendor '?client]
+                                                                                                                       :where '[[?e :invoice/invoice-number ?invoice-number]
+                                                                                                                                [?e :invoice/vendor ?vendor]
+                                                                                                                                [?e :invoice/client ?client]
+                                                                                                                                [?e :invoice/outstanding-balance ?outstanding-balance]
+                                                                                                                                [?e :invoice/status ?status]
+                                                                                                                                [?e :invoice/import-status ?import-status]]}
+                                                                                                               :args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]}))
+                                                                                                     first)]
+                                 (if (= :import-status/imported import-status)
+                                   result
+                                   (conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client)
+                                                                        :invoice/vendor matching-vendor
+                                                                        :invoice/invoice-number invoice-number
+                                                                        :invoice/total (Double/parseDouble total)
+                                                                        :invoice/date (to-date date)
+                                                                        :invoice/import-status :import-status/pending
+                                                                        :invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total))
+                                                                        :invoice/status (or existing-status :invoice-status/unpaid)
+                                                                        :invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:expense-account-id default-expense-account
+                                                                                                                                                    :location matching-location
+                                                                                                                                                    :amount (Double/parseDouble total)}])
+                                                                        :db/id existing-id
+                                                                        })))
+                                 ))
+                             []
+                             imports)]
+    (println transactions)
+    @(d/transact (d/connect uri) transactions)
+    ))
+
 (defroutes routes
   (wrap-routes
    (context "/" []
@@ -184,6 +235,13 @@
                               :headers {"Content-Type" "application/edn"}})))
 
             (context "/invoices" []
+                     (POST "/upload"
+                           {{  files "file"} :params :as params}
+                           (let [{:keys [filename tempfile]} files]
+                             (import-uploaded-invoice (parse/parse-file (.getPath tempfile) filename))
+                             {:status 200
+                              :body (pr-str {})
+                              :headers {"Content-Type" "application/edn"}}))
                      (POST "/upload-integreat"
                            {{:keys [excel-rows]}  :edn-params user :identity}
                            (assert-admin user)

src/cljs/auto_ap/views/components/invoices/side_bar.cljs

@@ -27,7 +27,14 @@
 
         [:span {:class "icon icon-accounting-invoice-mail" :style {:font-size "25px"}}]
         
-        [:span {:class "name"} "Paid Invoices"]]]]]
+        [:span {:class "name"} "Paid Invoices"]]]
+      [:li.menu-item
+       [:a.item {:href (bidi/path-for routes/routes :import-invoices)
+                 :class [(active-when ap = :import-invoices)]}
+
+        [:span {:class "icon icon-accounting-invoice-mail" :style {:font-size "25px"}}]
+        
+        [:span {:class "name"} "Import Invoices"]]]]]
     [:div
      rest]
     [:div {:class "compose has-text-centered"}

src/cljs/auto_ap/views/main.cljs

@@ -8,6 +8,7 @@
    [auto-ap.views.utils :refer [active-when active-when= login-url dispatch-event]]
    [auto-ap.views.components.layouts :refer [side-bar-layout]]
    [auto-ap.views.pages.unpaid-invoices :refer [unpaid-invoices-page]]
+   [auto-ap.views.pages.import-invoices :refer [import-invoices-page]]
    [auto-ap.views.pages.paid-invoices :refer [paid-invoices-page]]
    [auto-ap.views.pages.transactions :refer [transactions-page]]
    [auto-ap.views.pages.login :refer [login-page]]
@@ -25,6 +26,10 @@
 (defmethod page :unpaid-invoices [_]
   (unpaid-invoices-page {:status "unpaid"}))
 
+(defmethod page :import-invoices [_]
+  (import-invoices-page ))
+
+
 (defmethod page :paid-invoices [_]
   (unpaid-invoices-page {:status "paid"}))
 

src/cljs/auto_ap/views/pages/import_invoices.cljs

@@ -4,6 +4,8 @@
             [auto-ap.events :as events]
             [auto-ap.subs :as subs]
             [auto-ap.entities.clients :as client]
+            [auto-ap.views.components.layouts :refer [side-bar-layout]]
+            [auto-ap.views.components.invoices.side-bar :refer [invoices-side-bar]]
             [auto-ap.entities.vendors :as vendor]
             [auto-ap.views.components.invoice-table :refer [invoice-table] :as invoice-table]
             [cljsjs.dropzone :as dropzone]
@@ -56,7 +58,7 @@
             (assoc-in  [:status :loading] true)
             (assoc-in [::params] params))
     :graphql {:token (-> cofx :db :user)
-              :query-obj (invoice-table/query (assoc params :imported false)) 
+              :query-obj (invoice-table/query (assoc params :import-status "pending")) 
               :on-success [::received]}}))
 
 (re-frame/reg-event-db
@@ -88,7 +90,7 @@
            :on-success on-success
            }}))
 
-(def import-invoices-page
+(def import-invoices-content
   (with-meta
     (fn []
       (let [invoice-page (re-frame/subscribe [::invoice-page])
@@ -129,3 +131,9 @@
               "Reject all"]])]]))
     {:component-will-mount (fn []
                              (re-frame/dispatch-sync [::invalidated]))}))
+
+(defn import-invoices-page []
+  [side-bar-layout {:side-bar [invoices-side-bar {}]
+                    :main [import-invoices-content ]}])
+
+

src/cljs/auto_ap/views/pages/unpaid_invoices.cljs

@@ -80,7 +80,7 @@
             (assoc-in [:status :loading] true)
             (assoc-in [::params] params))
     :graphql {:token (-> cofx :db :user)
-              :query-obj (invoice-table/query (-> params (assoc :imported true) (dissoc :invoice-number-like-current)) ) 
+              :query-obj (invoice-table/query (-> params (assoc :import-status "imported") (dissoc :invoice-number-like-current)) ) 
               :on-success [::received]}}))
 (re-frame/reg-event-db
  ::unmount-invoices