notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

merged.

Browse Source
This commit is contained in:
Bryce Covert
2019-10-26 22:48:51 -07:00
parent b4cce91f57 332eae2f58
commit bf89befbc9
7 changed files with 271 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
src/clj/auto_ap/parse.clj
View File

@@ -1,54 +1,46 @@
(ns auto-ap.parse (ns auto-ap.parse
(:require [auto-ap.parse.excel :as excel] (:require [auto-ap.parse.excel :as excel]
[auto-ap.parse.templates :as t] [auto-ap.parse.templates :as t]
[auto-ap.parse.util :as u]
[auto-ap.parse.csv :as csv]
[clj-fuzzy.metrics :as m] [clj-fuzzy.metrics :as m]
[clojure.java.shell :as sh] [clojure.java.shell :as sh]
[clojure.string :as str] [clojure.string :as str]
[clj-time.format :as f] [clj-time.format :as f]
[clj-time.core :as time])) [clj-time.core :as time]
[clojure.set :as set]))
(defmulti parse-value (fn [method _ _]
method))
(defmethod parse-value :trim-commas
[_ _ value]
(str/replace value #"," "")
)
(defmethod parse-value :clj-time
[_ format value]
(time/from-time-zone (f/parse (f/formatter format) value)
(time/time-zone-for-id "America/Los_Angeles")))
(defmethod parse-value nil
[_ _ value]
value)
(def last-text (atom nil)) (def last-text (atom nil))
(defn template-applies? [text {:keys [keywords]}] (defn template-applies? [text {:keys [keywords]}]
(every? #(re-find % text) keywords)) (every? #(re-find % text) keywords))
(defn extract-template [text template] (defn extract-template
(if (:multi template) ([text template]
(mapcat (if (:multi template)
#(extract-template % (dissoc template :multi)) (mapcat
(str/split text (:multi template))) #(extract-template % text (dissoc template :multi))
(str/split text (:multi template)))
(when template (extract-template text text template)))
[(->> template ([text full-text template]
:extract (when (and template
(reduce-kv (or (not (:multi-match? template))
(fn [result k v] (re-find (:multi-match? template) text )))
(let [value (some-> (first (map second (re-seq v text))) [(->> template
str/trim ) :extract
[value-parser parser-params] (-> template :parser k)] (reduce-kv
(assoc result k (parse-value value-parser parser-params value)))) (fn [result k v]
{:vendor-code (:vendor template) (let [value (some-> (or (first (map second (re-seq v text)))
:text text}))]))) (first (map second (re-seq v full-text))))
str/trim )
[value-parser parser-params] (-> template :parser k)]
(assoc result k (u/parse-value value-parser parser-params value))))
{:vendor-code (:vendor template)
:text text
:full-text full-text}))])))
(defn parse [text] (defn parse [text]
(reset! last-text text) (reset! last-text text)
@@ -67,6 +59,11 @@
:out :out
parse)) parse))
(defmethod parse-file
"csv"
[file filename]
(csv/parse-file file filename))
(defmethod parse-file (defmethod parse-file
"xls" "xls"
[file filename] [file filename]
@@ -79,22 +76,50 @@
(excel/parse-file file filename)) (excel/parse-file file filename))
(defn best-match [clients invoice-client-name] (defn best-match [clients invoice-client-name]
(->> clients (let [fuzzy-match (->> clients
(mapcat (fn [{:keys [:db/id :client/matches :client/name] :as client :or {matches []}}]
(map (fn [m]
(println m invoice-client-name)
[client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))])
(conj matches name))))
(filter #(< (second %) 0.25))
(sort-by second)
ffirst)
(mapcat (fn [{:keys [:db/id :client/matches :client/name] :as client :or {matches []}}] word-set (set (str/split (.toLowerCase invoice-client-name) #"\s" ))
(map (fn [m] client-word-match (->> clients
[client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))]) (map
(conj matches name)))) (fn [{:keys [:db/id :client/matches :client/name] :as client :or {matches []}}]
(filter #(< (second %) 0.25)) (let [client-words (-> #{}
(sort-by second) (into
ffirst)) (mapcat
(fn [match] (str/split (.toLowerCase match) #"\s" ))
matches))
(into
(str/split (.toLowerCase name) #"\s" )))]
[client (count (set/intersection client-words word-set))])))
(filter (fn [[_ c]] (> c 0)))
(sort-by (fn [[_ c]] c))
reverse
ffirst)]
(or fuzzy-match client-word-match)))
(defn best-location-match [client text] (defn best-location-match [client text full-text]
(or (->> client (or (->> client
:client/location-matches :client/location-matches
(mapcat (fn [{:keys [:location-match/location :location-match/matches]}] (mapcat (fn [{:keys [:location-match/location :location-match/matches]}]
(map (fn [match] [location match]) matches))) (map (fn [match] [location match]) matches)))
(filter (fn [[location match]] (re-find (re-pattern (str "(?i)" match)) text)) ) (filter (fn [[location match]]
(println "loc " location match text)
(re-find (re-pattern (str "(?i)" match)) text)) )
first
first)
(->> client
:client/location-matches
(mapcat (fn [{:keys [:location-match/location :location-match/matches]}]
(map (fn [match] [location match]) matches)))
(filter (fn [[location match]] (re-find (re-pattern (str "(?i)" match)) full-text)) )
first first
first) first)
(:client/default-location client) (:client/default-location client)

46
src/clj/auto_ap/parse/csv.clj Normal file
View File

@@ -0,0 +1,46 @@
(ns auto-ap.parse.csv
(:require [auto-ap.parse.util :as u]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]))
(defn determine
[[header :as z]]
(prn header)
(cond (str/includes? (second header) "Customer's PO No.")
:mama-lus
:else
nil))
(defmulti parse-csv
determine
:default (fn default [rows]
nil))
(defmethod parse-csv :mama-lus
[rows]
(println "MAMA LU")
(transduce
(comp (drop 1)
(map (fn [[_ po-number despatch-number invoice-number invoice-date customer value :as row]]
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number )
:date (u/parse-value :clj-time "MM/dd/yy HH:ss" invoice-date)
:total value
:text (str/join " " row)})))
conj
[]
rows))
(defmethod parse-csv nil
[rows]
nil)
(defn parse-file [file filename]
(println "HEREERE")
(with-open [reader (io/reader file)]
(let [rows (csv/read-csv reader :separator \,)]
(parse-csv rows))))

64
src/clj/auto_ap/parse/excel.clj
View File

@@ -10,38 +10,40 @@
(every? #(re-find % text) keywords)) (every? #(re-find % text) keywords))
(defn extract [wb {:keys [extract vendor]}] (defn extract [wb {:keys [extract vendor]}]
(reduce-kv (if (fn? extract)
(fn [invoice k [regex offset-row offset-column extract-regex]] (extract wb vendor)
(assoc invoice k [(reduce-kv
(->> wb (fn [invoice k [regex offset-row offset-column extract-regex]]
(d/sheet-seq) (assoc invoice k
first (->> wb
(d/cell-seq) (d/sheet-seq)
(filter (fn [cell] first
(re-find regex (str (d/read-cell cell))))) (d/cell-seq)
(map (fn [cell] (filter (fn [cell]
(let [address (.getAddress cell) (re-find regex (str (d/read-cell cell)))))
cell-value (str (d/read-cell (d/select-cell (.toString (CellAddress. (+ offset-row (.getRow address)) (+ offset-column (.getColumn address)) )) (map (fn [cell]
(first (d/sheet-seq wb)))))] (let [address (.getAddress cell)
(if extract-regex cell-value (str (d/read-cell (d/select-cell (.toString (CellAddress. (+ offset-row (.getRow address)) (+ offset-column (.getColumn address)) ))
(second (re-find extract-regex cell-value)) (first (d/sheet-seq wb)))))]
(if extract-regex
(second (re-find extract-regex cell-value))
cell-value)))) cell-value))))
first))) first)))
{:vendor-code vendor} {:vendor-code vendor}
extract)) extract)]))
(defn parse-file (defn parse-file
[file filename] [file filename]
[(let [wb (d/load-workbook file) (let [wb (d/load-workbook file)
text (->> wb text (->> wb
(d/sheet-seq) (d/sheet-seq)
first first
(d/cell-seq) (d/cell-seq)
(map d/read-cell) (map d/read-cell)
(str/join " "))] (str/join " "))]
(->> t/excel-templates (->> t/excel-templates
(filter (partial template-applies? text)) (filter (partial template-applies? text))
first first
(extract wb) (extract wb)
))]) )))

65
src/clj/auto_ap/parse/templates.clj
View File

@@ -1,4 +1,7 @@
(ns auto-ap.parse.templates) (ns auto-ap.parse.templates
(:require [dk.ative.docjure.spreadsheet :as d]
[clojure.string :as str])
(:import (org.apache.poi.ss.util CellAddress)))
(def pdf-templates (def pdf-templates
@@ -48,7 +51,40 @@
:customer-identifier #"Bill To:[^\n]+\n\s*([\w ]+)" :customer-identifier #"Bill To:[^\n]+\n\s*([\w ]+)"
:invoice-number #"Invoice\s([\w\./]+)*" :invoice-number #"Invoice\s([\w\./]+)*"
:total #"Total Invoice\s+([0-9.]+)"} :total #"Total Invoice\s+([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]}}]) :parser {:date [:clj-time "MM/dd/yy"]}}
{:vendor "Southbay Fresh Produce"
:keywords [#"SOUTH BAY FRESH PRODUCE"]
:extract {:date #"^([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"FAX:[^\n]+\n\s+([A-Za-z ]+)\s{2}"
:invoice-number #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"
:total #"\$([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}
:multi #"\n"
:multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"}
{:vendor "Performance Food Group"
:keywords [#"performancefoodservice"]
:extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"DELIVER TO[^\n]+\n\s*[\S ]+?(?=\s{2,}([\S ]+?)\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"DELIVER TO[^\n]+\n.+?(?=\d+)(\d+)\s*\n"
:total #"([0-9.]+)\s+Status Code"}
:parser {:date [:clj-time "MM/dd/yy"]}}
{:vendor "US Foods"
:keywords [#"US Foods"]
:extract {:date #"INVOICE NUMBER[^\n]+\n\n\d+\s+\d+\s+([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"BILL TO[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE NUMBER[^\n]+\n\n\d+\s+(\d+)"
:total #"DELIVERED AMOUNT\s+\$([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}
{:vendor "Sysco"
:keywords [#"SYSCO"]
:extract {:date #"INVOICE NUMBER[^\n]+\n([^\n]+)\n"
:customer-identifier #"INVOICE NUMBER[^\n]+\n[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})"
:total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}])
(defn offset [c x y]
(.toString (CellAddress. (+ y (.getRow (.getAddress c))) (+ x (.getColumn (.getAddress c))) )))
(def excel-templates (def excel-templates
[{:vendor "Isp Productions" [{:vendor "Isp Productions"
@@ -62,4 +98,27 @@
:extract {:customer-identifier [#"Customer #" 1 0] :extract {:customer-identifier [#"Customer #" 1 0]
:total [#"Total Invoice" 0 5] :total [#"Total Invoice" 0 5]
:date [#"Date" 0 0 #"Date: (.*)"] :date [#"Date" 0 0 #"Date: (.*)"]
:invoice-number [#"Invoice #" 0 0 #"Invoice #: (.*)"]}}]) :invoice-number [#"Invoice #" 0 0 #"Invoice #: (.*)"]}}
{:vendor "DVW Commercial"
:keywords [#"Thank you!!!"]
:extract (fn [wb vendor]
(let [[sheet] (d/sheet-seq wb)]
(transduce (comp (filter (fn [c]
(re-find #"Invoice" (str (d/read-cell c)))))
(map (fn [c]
(let [customer-identifier (d/read-cell (->> (d/select-cell (offset c -3 0) sheet)
(iterate (fn [c]
(d/select-cell (offset c 0 -1) sheet)))
(filter (fn [c]
(not (str/blank? (d/read-cell c)))))
first))]
{:customer-identifier customer-identifier
:text customer-identifier
:full-text customer-identifier
:date (d/read-cell (d/select-cell (offset c 2 0) sheet))
:invoice-number (d/read-cell (d/select-cell (offset c 4 0) sheet))
:total (str (d/read-cell (d/select-cell (offset c 8 0) sheet)))
:vendor-code vendor}))))
conj
[]
(d/cell-seq sheet))))}])

25
src/clj/auto_ap/parse/util.clj Normal file
View File

@@ -0,0 +1,25 @@
(ns auto-ap.parse.util
(:require [clj-fuzzy.metrics :as m]
[clojure.java.shell :as sh]
[clojure.string :as str]
[clj-time.format :as f]
[clj-time.core :as time]))
(defmulti parse-value (fn [method _ _]
method))
(defmethod parse-value :trim-commas
[_ _ value]
(str/replace value #"," "")
)
(defmethod parse-value :clj-time
[_ format value]
(time/from-time-zone (f/parse (f/formatter format) value)
(time/time-zone-for-id "America/Los_Angeles")))
(defmethod parse-value nil
[_ _ value]
value)

50
src/clj/auto_ap/routes/invoices.clj
View File

@@ -8,6 +8,7 @@
[auto-ap.datomic :refer [remove-nils uri]] [auto-ap.datomic :refer [remove-nils uri]]
[datomic.api :as d] [datomic.api :as d]
[auto-ap.parse :as parse] [auto-ap.parse :as parse]
[auto-ap.parse.util :as parse-u]
[auto-ap.graphql.utils :refer [assert-admin]] [auto-ap.graphql.utils :refer [assert-admin]]
[auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.routes.utils :refer [wrap-secure]]
[clj-time.coerce :refer [to-date]] [clj-time.coerce :refer [to-date]]
@@ -89,7 +90,7 @@
(defn parse-date [{:keys [raw-date]}] (defn parse-date [{:keys [raw-date]}]
(try (try
(parse/parse-value :clj-time "MM/dd/yyyy" raw-date) (parse-u/parse-value :clj-time "MM/dd/yyyy" raw-date)
(catch Exception e (catch Exception e
(throw (Exception. (str "Could not parse date from '" raw-date "'") e))))) (throw (Exception. (str "Could not parse date from '" raw-date "'") e)))))
@@ -167,32 +168,41 @@
(let [clients (d-clients/get-all) (let [clients (d-clients/get-all)
_ (clojure.pprint/pprint imports) _ (clojure.pprint/pprint imports)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text] :as info}] transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text full-text] :as info}]
(println "searching for" vendor-code)
(let [[matching-vendor default-account] (->> (d/query (let [[matching-vendor default-account] (->> (d/query
(cond-> {:query {:find ['?vendor '?default-account] {:query {:find ['?vendor '?default-account]
:in ['$ '?vendor-name] :in ['$ '?vendor-name]
:where ['[?vendor :vendor/name ?vendor-name] :where ['[?vendor :vendor/name ?vendor-name]
'[?vendor :vendor/default-account ?default-account]]} '[?vendor :vendor/default-account ?default-account]]}
:args [(d/db (d/connect uri)) vendor-code]})) :args [(d/db (d/connect uri)) vendor-code]})
first) first)
_ (println "matching" customer-identifier "-" matching-vendor)
matching-client (parse/best-match clients customer-identifier) matching-client (parse/best-match clients customer-identifier)
_ (println "New invoice matches client '" matching-client "', vendor '" matching-vendor "', account '" default-account "'") _ (println "New invoice matches client" matching-client)
matching-location (parse/best-location-match matching-client text ) matching-location (parse/best-location-match matching-client text full-text)
[existing-id existing-outstanding-balance existing-status import-status] (->> (d/query [existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] (->> (d/query
:in ['$ '?invoice-number '?vendor '?client] (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:where '[[?e :invoice/invoice-number ?invoice-number] :in ['$ '?invoice-number '?vendor '?client]
[?e :invoice/vendor ?vendor] :where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/client ?client] [?e :invoice/vendor ?vendor]
[?e :invoice/outstanding-balance ?outstanding-balance] [?e :invoice/client ?client]
[?e :invoice/status ?status] [?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/import-status ?import-status] [?e :invoice/status ?status]
[?import-status :db/ident ?import-status2]]} [?e :invoice/import-status ?import-status]
:args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]})) [?import-status :db/ident ?import-status2]]}
first)] :args [(d/db (d/connect uri)) invoice-number matching-vendor (:db/id matching-client)]}))
first))]
(if (= :import-status/imported import-status) (cond
(not (and matching-location matching-client))
result result
(= :import-status/imported import-status)
result
:else
(conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client) (conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client)
:invoice/vendor matching-vendor :invoice/vendor matching-vendor
:invoice/invoice-number invoice-number :invoice/invoice-number invoice-number

3
src/cljs/auto_ap/views/pages/import_invoices.cljs
View File

@@ -7,6 +7,7 @@
[auto-ap.views.components.layouts :refer [side-bar-layout]] [auto-ap.views.components.layouts :refer [side-bar-layout]]
[auto-ap.views.components.invoices.side-bar :refer [invoices-side-bar]] [auto-ap.views.components.invoices.side-bar :refer [invoices-side-bar]]
[auto-ap.views.utils :refer [dispatch-event]] [auto-ap.views.utils :refer [dispatch-event]]
[auto-ap.utils :refer [by]]
[auto-ap.entities.vendors :as vendor] [auto-ap.entities.vendors :as vendor]
[auto-ap.views.components.invoice-table :refer [invoice-table] :as invoice-table] [auto-ap.views.components.invoice-table :refer [invoice-table] :as invoice-table]
[cljsjs.dropzone :as dropzone] [cljsjs.dropzone :as dropzone]
@@ -67,6 +68,8 @@
(fn [db [_ data]] (fn [db [_ data]]
(-> db (-> db
(assoc ::invoice-page (first (:invoice-page data))) (assoc ::invoice-page (first (:invoice-page data)))
(update-in [::invoice-page] (fn [ip]
(assoc ip :checked (by :id (:invoices ip)))))
(assoc-in [:status :loading] false)))) (assoc-in [:status :loading] false))))
(re-frame/reg-event-fx (re-frame/reg-event-fx