This commit is contained in:
Bryce Covert
2019-12-21 14:57:48 -08:00
14 changed files with 332 additions and 50 deletions

View File

@@ -47,6 +47,12 @@
:invoice-status/paid
:invoice-status/unpaid)]])})}]] )
(def add-client-identifier
[[{:db/ident :invoice/client-identifier
:db/doc "An identifier found in an uploaded invoice"
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one}]])
(defn -main [& args]
(println "Creating database ..." uri)
(doto (d/create-database uri) println)
@@ -125,18 +131,13 @@
:requires [:auto-ap/add-hidden-to-vendor]}
:auto-ap/convert-invoices {:txes-fn `add-general-ledger/convert-invoices
:requires [:auto-ap/convert-vendors]}
:auto-ap/add-yodlee-merchant2 {:txes add-general-ledger/add-yodlee-merchant :requires [:auto-ap/convert-vendors]}
:auto-ap/add-external-id-to-ledger {:txes add-general-ledger/add-external-id-to-ledger :requires [:auto-ap/add-yodlee-merchant2]}
:auto-ap/add-exclude-to-transaction {:txes add-general-ledger/add-exclude-to-transaction :requires [:auto-ap/add-external-id-to-ledger]}
:auto-ap/add-client-identifier2 {:txes add-client-identifier :requires [:auto-ap/make-every-account-visible]}
:auto-ap/add-transaction-rules {:txes add-general-ledger/add-transaction-rules :requires [:auto-ap/add-exclude-to-transaction]}
:auto-ap/add-bank-account-locations {:txes add-general-ledger/add-bank-account-locations :requires [:auto-ap/add-transaction-rules]}
:auto-ap/convert-transactions {:txes-fn `add-general-ledger/convert-transactions :requires [:auto-ap/add-bank-account-locations]}
}]
:auto-ap/convert-transactions {:txes-fn `add-general-ledger/convert-transactions :requires [:auto-ap/add-bank-account-locations]}}]
(println "Conforming database...")
(c/ensure-conforms conn norms-map)
(when (not (seq args))

View File

@@ -70,6 +70,7 @@
:address {:type :address}
:location_matches {:type '(list :location_match)}
:locations {:type '(list String)}
:matches {:type '(list String)}
:bank_accounts {:type '(list :bank_account)}}}
:contact
{:fields {:id {:type :id}
@@ -253,6 +254,7 @@
:invoice
{:fields {:id {:type :id}
:original_id {:type 'Int}
:client_identifier {:type 'String}
:total {:type 'String}
:outstanding_balance {:type 'String}
:invoice_number {:type 'String}
@@ -473,6 +475,7 @@
:email {:type 'String}
:address {:type :add_address}
:locations {:type '(list String)}
:matches {:type '(list String)}
:location_matches {:type '(list :edit_location_match)}
:bank_accounts {:type '(list :edit_bank_account)}}}
:edit_bank_account

View File

@@ -25,11 +25,17 @@
id (or (:db/id client) "new-client")
_ (println id)
_ (println edit_client)
_ (when client
@(d/transact (d/connect uri)
(into
(mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client))
(mapv (fn [m] [:db/retract (:db/id client) :client/matches m]) (:client/matches client)))))
transactions [(remove-nils {:db/id id
:client/code (if (str/blank? (:client/code client))
(:code edit_client)
(:client/code client))
:client/name (:name edit_client)
:client/matches (:matches edit_client)
:client/email (:email edit_client)
:client/locations (filter identity (:locations edit_client))
:client/location-matches (->> (:location_matches edit_client)
@@ -65,7 +71,16 @@
})]
result @(d/transact (d/connect uri) transactions)]
(println result "ID" id)
(-> result :tempids (get id) (or id) d-clients/get-by-id ->graphql)))
(-> result :tempids (get id) (or id) d-clients/get-by-id
(update :client/location-matches
(fn [lms]
(mapcat (fn [lm]
(map (fn [m]
{:location-match/match m
:location-match/location (:location-match/location lm)})
(:location-match/matches lm)))
lms)))
->graphql)))
(defn get-client [context args value]

View File

@@ -19,6 +19,7 @@
(defn extract-template
([text template]
(println "template" template)
(if (:multi template)
(mapcat
#(extract-template % text (dissoc template :multi))
@@ -37,12 +38,15 @@
(first (map second (re-seq v full-text))))
str/trim )
[value-parser parser-params] (-> template :parser k)]
(assoc result k (u/parse-value value-parser parser-params value))))
(assoc result k (try (u/parse-value value-parser parser-params value)
(catch Exception e
(println e))))))
{:vendor-code (:vendor template)
:text text
:full-text full-text}))])))
(defn parse [text]
(println "Parsing PDF " text)
(reset! last-text text)
(->> t/pdf-templates
(filter (partial template-applies? text))
@@ -79,7 +83,6 @@
(let [fuzzy-match (->> clients
(mapcat (fn [{:keys [:db/id :client/matches :client/name] :as client :or {matches []}}]
(map (fn [m]
(println m invoice-client-name)
[client (m/jaccard (.toLowerCase invoice-client-name) (.toLowerCase m))])
(conj matches name))))
(filter #(< (second %) 0.25))
@@ -111,7 +114,6 @@
(map (fn [match] [location match]) matches)))
(filter (fn [[location match]]
(println "loc " location match text)
(re-find (re-pattern (str "(?i)" match)) text)) )
first
first)
@@ -124,3 +126,10 @@
first)
(:client/default-location client)
(first (:client/locations client))))
(defn dbg-parse [v]
(doto
(map
(fn [x] (dissoc x :full-text :text))
(parse v))
clojure.pprint/pprint ))

View File

@@ -20,16 +20,19 @@
(defmethod parse-csv :mama-lus
[rows]
(println "MAMA LU")
(println "MAMA LU4")
(transduce
(comp (drop 1)
(map (fn [[_ po-number despatch-number invoice-number invoice-date customer value :as row]]
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number )
:date (u/parse-value :clj-time "MM/dd/yy HH:ss" invoice-date)
:total value
:text (str/join " " row)})))
:date (try (u/parse-value :clj-time "M/d/yyyy HH:ss" invoice-date)
(catch Exception _
(u/parse-value :clj-time "M/d/yyyy" invoice-date)))
:total (str/replace value #"," "")
:text (str/join " " row)
:full-text (str/join " " row)})))
conj
[]
rows))

View File

@@ -5,7 +5,9 @@
(def pdf-templates
[{:vendor "CHFW"
[
;; CHEF's WAREHOUSE
{:vendor "CHFW"
:keywords [#"CHEF'S WAREHOUSE"]
:extract {:total #"2 WKS C\.C\.\s+([\d.,]+)"
:customer-identifier #"\n([A-Z][A-Z ]+)\s{2,}"
@@ -13,6 +15,7 @@
:invoice-number #"\s+[0-9]+/[0-9]+/[0-9]+\s+([0-9]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}
;; GGM
{:vendor "Golden Gate Meat Company, Inc"
:keywords [#"Golden Gate Meat"]
:extract {:total #"Invoice Total\:\s+\$([\d.,]+)"
@@ -22,6 +25,7 @@
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}}
;; CINTAS
{:vendor "CINTAS"
:keywords [#"CINTAS CORPORATION"]
:extract {:invoice-number #"INVOICE\s#\s+([\d.,]+)"
@@ -31,13 +35,17 @@
:parser {:date [:clj-time "MM/dd/yy"]}
:multi #"\f\f"}
;; CARBONIC
{:vendor "Carbonic Service Inc"
:keywords [#"CARBONIC SERVICE INC"]
:extract {:invoice-number #"Invoice #\s*\n\s*[\w\.]+\s+[\w\./]+(.*)\s*\n"
:customer-identifier #"Bill To[^\n]+\n[^\n]*\n([\w ]+)\s{2,}"
:date #"Invoice #\s*\n\s*[\w\.]+\s+([\w\./]+)"
:total #"Total\s+\$([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]}}
:total #"Total\s+\$([0-9.,]+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]}}
;; DVW
{:vendor "DVW Commercial"
:keywords [#"DVW Commercial"]
:extract {:date #"\s*([0-9]+/[0-9]+/[0-9]+)"
@@ -45,6 +53,8 @@
:invoice-number #"Invoice\s*\n\s*([\w\./]+)*"
:total #"Total:\s+\$ ([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]}}
;; DAYLIGHT FOOD
{:vendor "Daylight Foods"
:keywords [#"DAYLIGHT FOODS"]
:extract {:date #"\n\s*Date[^\n]+\n\s*([0-9]+/[0-9]+/[0-9]+)"
@@ -52,36 +62,193 @@
:invoice-number #"Invoice\s([\w\./]+)*"
:total #"Total Invoice\s+([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]}}
;; SOUTHBAY FRESH
{:vendor "Southbay Fresh Produce"
:keywords [#"SOUTH BAY FRESH PRODUCE"]
:extract {:date #"^([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"FAX:[^\n]+\n\s+([A-Za-z ]+)\s{2}"
:invoice-number #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"
:total #"\$([0-9.]+)"}
:customer-identifier #"To:[^\n]*\n\s+([A-Za-z' ]+)\s{2}"
:invoice-number #"INV #\/(\d+)"
:total #"\$([0-9.]+)\."}
:parser {:date [:clj-time "MM/dd/yyyy"]}
:multi #"\n"
:multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+(\d+)"}
{:vendor "Performance Food Group"
:multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+INV "}
;; PFG - LEDYARD
{:vendor "Performance Food Group - LEDYARD"
:keywords [#"performancefoodservice"]
:extract {:date #"DELIVER TO[^\n]+\n.+?(?=[0-9]+/[0-9]+/[0-9]+)([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"DELIVER TO[^\n]+\n\s*[\S ]+?(?=\s{2,}([\S ]+?)\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"DELIVER TO[^\n]+\n.+?(?=\d+)(\d+)\s*\n"
:total #"([0-9.]+)\s+Status Code"}
:parser {:date [:clj-time "MM/dd/yy"]}}
:total #"([0-9.\-]+)\s+Status Code"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
;; SOUTHERN GLAZER'S
{:vendor "Southern Glazers"
:keywords [#"Southern Glazer's"]
:extract {:date #"INVOICE DATE(?s:.*)(?= (?:[0-9]+/[0-9]+/[0-9]+)\s+([0-9]+/[0-9]+/[0-9]+)) "
:customer-identifier #"SOLD TO:(?:.*)(?=\n)\n(.*)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE\n(?:.*?)(?=\d{4,})(\d+)"
:total #"PAY THIS AMOUNT(?s:.*)(?= ([0-9,]+\.[0-9]{2}))"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]}}
;; GOLDEN BRANDS
{:vendor "Golden Brands San Jose"
:keywords [#"GOLDEN BRANDS"]
:extract {:date #"0430\n(.*)"
:customer-identifier #"Account:(?:.*\n)(.*(?=\s{2,}))"
:invoice-number #"Invoice#: (\d+)"
:total #"Invoice Total\s+([0-9]+\.[0-9]{2})"}
:parser {:date [:clj-time "EEE MMM dd, yyyy HH:mm aa"]
:total [:trim-commas nil]}}
;; WINE WAREHOUSE
{:vendor "Wine Warehouse"
:keywords [#"WINE WAREHOUSE"]
:extract {:date #"INVOICE DATE\s+([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"SHIP-TO-PARTY.*\n(.*?)(?=\s{2,})"
:invoice-number #"INV #\s+(\d+)"
:total #"PLEASE PAY THIS AMOUNT\s+([0-9]+\.[0-9]{2})"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}}
;; REGAL
{:vendor "Regal Wine Co"
:keywords [#"REGAL WINE"]
:extract {:date #"INVOICE DATE.*\n\n(?:.*?)([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"INVOICE\n(.*?)\s{2,}"
:invoice-number #"INVOICE NUMBER.*\n\n(?:.*?)(\d+)"
:total #"Total Amount Due(?:.*?)([0-9,]+\.[0-9]{2})"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]}}
;; ALSCO
{:vendor "Alsco"
:keywords [#"Alsco"]
:extract {:date #"Invoice Date:\s+(.*)"
:customer-identifier #"Invoice F o r(?:.*?)\n\s+(.*?)\s{2,}"
:invoice-number #" (\S+)\n\s+Invoice Date"
:total #"Invoice Total\s+\$([0-9,]+\.[0-9]{2})"}
:parser {:date [:clj-time "MMM dd yyyy"]
:total [:trim-commas nil]}}
;; SUNCREST
{:vendor "Suncrest USA Inc"
:keywords [#"Suncrest.*Invoice"]
:extract {:date #"Date.*\n\s*\n(?:.*?)([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To(?:.*?)\n\n(.*?)\s{2,}"
:invoice-number #"Invoice #.*\n\s*\n(?:.*?)\s{2,}(\d{5,})"
:total #"Balance Due\s+\$([0-9,]+\.[0-9]{2})"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}}
;; SUNCREST STATEMENT
{:vendor "Suncrest USA Inc"
:keywords [#"Suncrest.*\n.*Statement"]
:extract {:date #"^([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"To:(?:.*?)\n\s*(.*?)\s{2,}"
:invoice-number #"INV #(\d+)"
:total #"Orig\. Amount \$([0-9,]+\.[0-9]{2})"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}
:multi #"\n"
:multi-match? #"^[0-9]+/[0-9]+/[0-9]+\s+INV "}
;; US FOODS
{:vendor "US Foods"
:keywords [#"US Foods"]
:extract {:date #"INVOICE NUMBER[^\n]+\n\n\d+\s+\d+\s+([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"BILL TO[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE NUMBER[^\n]+\n\n\d+\s+(\d+)"
:total #"DELIVERED AMOUNT\s+\$([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}
:total #"(?:DELIVERED AMOUNT|PLEASE REMIT).*(?=\$)\$([0-9.,]+)\s*\n"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas nil]}}
;; SYSCO
{:vendor "Sysco"
:keywords [#"SYSCO"]
:extract {:date #"INVOICE NUMBER[^\n]+\n([^\n]+)\n"
:customer-identifier #"INVOICE NUMBER[^\n]+\n[^\n]+\n([\S ]+?)(?=\s{2,})" ;; ([\S ]+)\s{2,}
:invoice-number #"INVOICE NUMBER[^\n]+\n[^\n]+\n.*?(?=[\d]{9})(\d{9})"
:total #"\s{2,}INVOICE\s{2,}.*?(?=TOTAL)TOTAL\s+([0-9.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}])
:parser {:date [:clj-time "MM/dd/yyyy"]}}
;; LE BOULANGER
{:vendor "Le Boulanger"
:keywords [#"Le Boulanger"]
:extract {:date #"Invoice Date: ([^\n]+)\n"
:customer-identifier #"Ship to\n+\s+([\S ]+?)(?=\s{2,})"
:invoice-number #"Invoice No: ([^\n]+)\n"
:total #" Total:\s+([\d\.]+)"}
:parser {:date [:clj-time "MMM dd, yyyy"]}}
;; A&B
{:vendor "A&B Produce"
:keywords [#"ABProduce"]
:extract {:date #"^\s+([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"BILL TO:[^\n]+\n[^\n]+\n[^\n]+\n(.*)\s{2,}"
:invoice-number #"(\d+)\s+(?:INV|C/M)"
:total #" (?:INV|C/M)\s+([\d\.\-]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}
:multi #"\n"
:multi-match? #"^\s+[0-9]+/[0-9]+/[0-9]+\s+\d+\s+(INV|C/M)\s+"}
;; CHEF's CHOICE
{:vendor "Chef's Choice Produce Co"
:keywords [#"(2170 MARTIN AVENUE|213-3886)"]
:extract {:date #"([0-9/]{10,10})"
:customer-identifier #"\n B\s+([\S ]+?)(?=\s{2,}I) "
:invoice-number #"^0*([0-9]+)"
:total #"INVOICE\s+([\d\.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]} ;; may want to try two approaches [:clj-time ["MM/dd/yyyy" "MM1dd1yyyy"]]
:multi #"\n"
:multi-match? #"\s+INVOICE\s+"}
;; FRESH AND BELT
{:vendor "Fresh and Best Produce"
:keywords [#"freshbestproduce"]
:extract {:date #"\n\s+([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To[^\n]+\n([A-Za-z ']+)"
:invoice-number #"\n\s+[0-9/]+\s+(\d+)"
:total #"Balance Due\s+\$([0-9\.]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]}}
;; PFG - ROMA
{:vendor "Performance Food Group - ROMA"
:keywords [#"Performance Food Group, Inc\n\f"]
:extract {:date #"Date: ([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})"
:invoice-number #"INVOICE NO.\s+ ([\d]+)"
:total #"([\d\.,]+)\s+INVOICE TOTAL"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas nil]}}
;; PFG - ROMA LOOK 1
{:vendor "Performance Food Group - ROMA"
:keywords [#"inquiries call 1-800-233-6211"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"BILL TO:\s+([\S ]+?)(?=\s{2,})"
:invoice-number #"^\s+([\dA-Z]+)"
:total #"([\d\.,\-]+\.[\d\-]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}
:multi #"\n"
:multi-match? #"^\s+[\d]{6,8}\s+\d+"}
;; JFC
{:vendor "JFC International"
:keywords [#"48490 MILMONT DRIVE"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"SOLD\s+([\S ]+?)(?=(\s{2,}|\n))"
:invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)"
:total #"(?:INVOICE|TOTAL)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}}])
(defn offset [c x y]
(.toString (CellAddress. (+ y (.getRow (.getAddress c))) (+ x (.getColumn (.getAddress c))) )))

View File

@@ -14,6 +14,17 @@
(str/replace value #"," "")
)
(defmethod parse-value :trim-commas-and-negate
[_ _ value]
(let [[_ raw-value] (re-find #"([\d\.]+)"
(-> value
(str/replace #"," "")
(str/replace #"-" "")))]
(if (or (str/includes? value "-")
(str/includes? value "CR"))
(str (- (Double/parseDouble raw-value)))
(str raw-value))))
(defmethod parse-value :clj-time
[_ format value]
(time/from-time-zone (f/parse (f/formatter format) value)

View File

@@ -19,6 +19,7 @@
(defroutes routes
(GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers}
(println "Authenticating with" r "..." code)
(try
(let [auth (-> "https://accounts.google.com/o/oauth2/token"
(http/post
@@ -44,17 +45,19 @@
;; TODO - these namespaces are not being transmitted/deserialized properly
(if (and token user)
{:status 301
:headers {"Location" (str "/?jwt=" (jwt/sign (doto {:user (:name profile)
:exp (time/plus (time/now) (time/days 30))
:user/clients (map (fn [c]
(dissoc c :client/bank-accounts ))
(:user/clients user))
:user/role (name (:user/role user))
:user/name (:name profile)}
println)
(:jwt-secret env)
{:alg :hs512}))}}
(let [jwt (jwt/sign (doto {:user (:name profile)
:exp (time/plus (time/now) (time/days 30))
:user/clients (map (fn [c]
(dissoc c :client/bank-accounts :client/location-matches))
(:user/clients user))
:user/role (name (:user/role user))
:user/name (:name profile)}
println)
(:jwt-secret env)
{:alg :hs512})]
(println "authenticated. using jwt" jwt)
{:status 301
:headers {"Location" (str "/?jwt=" jwt)}})
{:status 401
:body "Couldn't authenticate"}))
(catch Exception e

View File

@@ -166,7 +166,6 @@
(defn import-uploaded-invoice [imports]
(let [clients (d-clients/get-all)
_ (clojure.pprint/pprint imports)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier total date vendor-code text full-text] :as info}]
(println "searching for" vendor-code)
@@ -204,6 +203,7 @@
:else
(conj result (remove-nils #:invoice {:invoice/client (:db/id matching-client)
:invoice/client-identifier customer-identifier
:invoice/vendor matching-vendor
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
@@ -219,8 +219,7 @@
))
[]
imports)]
@(d/transact (d/connect uri) transactions)))
@(d/transact (d/connect uri) (vec (set transactions)))))
(defroutes routes
(wrap-routes

View File

@@ -163,6 +163,21 @@
(recur (concat transactions transaction-batch) (+ batch-size skip))
transactions)))))
(defn count-specific-transactions [account]
(let [cob-session (login-cobrand)
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
(doto println)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json})
:body
:transaction
)))
(defn get-access-token []
(let [cob-session (login-cobrand)
user-session (login-user cob-session)