Allows upload of CSV of sysco with line item parsing

This commit is contained in:
2026-05-26 21:53:04 -07:00
parent 200056098f
commit a4d7ac5982
5 changed files with 187 additions and 77 deletions

View File

@@ -1759,4 +1759,38 @@ Id,Sysco Category,Sysco Description,Integreat Account,Integreat Account Code,Nic
1758,MEATS,PORK BELLY SKIN ON P12 COV,Beef/Pork Costs,51110,
1759,MEATS,PORK SHANK BONE KUROBUTA PR12,Beef/Pork Costs,51110,
1760,CANNED AND DRY,SEASONING ITALIAN WHL,Food Costs,50000,
1761,PRODUCE,MUSHROOM PORTABELLA CAP 4-5,Produce Costs,51200,
1761,PRODUCE,MUSHROOM PORTABELLA CAP 4-5,Produce Costs,51200,
1762,PAPER & DISP,BAG PAPER 250 CT,Paper Costs,55000,
1763,MEATS,BEEF SHLDR TERES MAJOR SEL,Beef/Pork Costs,51110,
1764,PAPER & DISP,BOWL PLASTIC COATING 42 OZ,Paper Costs,55000,
1765,PAPER & DISP,BOX CATERING 21X13X4.25 LOGO,Paper Costs,55000,
1766,CANNED AND DRY,CANDY MILK CHOC SHELLS,Food Costs,50000,
1767,CANNED AND DRY,CHOCOLATE DUBAI PISTCHO KUNFEH,Food Costs,50000,
1768,PAPER & DISP,CONTAINER PAPER 1/30 OZ NTG,Paper Costs,55000,
1769,PAPER & DISP,CONTAINER PAPER 4/110OZ NTG,Paper Costs,55000,
1770,PAPER & DISP,CUP PAPER COLD 22 OZ LOGO NTG,Paper Costs,55000,
1771,PAPER & DISP,CUP PORTION PLAS CLR 1.50 OZ,Paper Costs,55000,
1772,CANNED AND DRY,DESSERT CUP,Food Costs,50000,
1773,FROZEN,DESSERT MINI PLAIN BEIGNET,Food Costs,50000,
1774,CANNED AND DRY,DIP GARLIC TOUM,Food Costs,50000,
1775,CANNED AND DRY,DRINK ENERGY ORANGE SPRKLNG,Soft Beverage Costs,52000,
1776,CANNED AND DRY,DRINK ENERGY PEACH VIBE SPRKLG,Soft Beverage Costs,52000,
1777,CANNED AND DRY,DRINK ENERGY TROPICAL VIBE,Soft Beverage Costs,52000,
1778,PAPER & DISP,FILM PVC 18X2000 ROLL,Paper Costs,55000,
1779,CANNED AND DRY,JUICE CONC MANDARIN CARDAMOM,Food Costs,50000,
1780,CANNED AND DRY,JUICE CONC STRAWB DRAGON,Food Costs,50000,
1781,PAPER & DISP,LID CLEAR PET 42 OZ,Paper Costs,55000,
1782,PAPER & DISP,LID DOME DESSERT CUP,Paper Costs,55000,
1783,PAPER & DISP,NAPKIN 2PLY INTR FOLD 6.3X8.26,Paper Costs,55000,
1784,CANNED AND DRY,PASTE HERB HARISSA MOROCCAN,Food Costs,50000,
1785,CANNED AND DRY,PASTE TAHINI DRESSING,Food Costs,50000,
1786,FROZEN,PASTRY BEIGNET MN FLD CHOCCRML,Food Costs,50000,
1787,CANNED AND DRY,PEPPER BANANA MILD RING,Food Costs,50000,
1788,CANNED AND DRY,RICE MIX NICKS,Food Costs,50000,
1789,CANNED AND DRY,SODA CHERRY VISSINADA GREEK,Soft Beverage Costs,52000,
1790,CANNED AND DRY,SODA COLA PEPSI ZERO SUGAR,Soft Beverage Costs,52000,
1791,CANNED AND DRY,SODA PEPSI COLA,Soft Beverage Costs,52000,
1792,FROZEN,SPANAKOPITA SPINACH COOKED,Food Costs,50000,
1793,PAPER & DISP,SPOON PLAS TEA PP X-HVY BLK,Paper Costs,55000,
1794,PAPER & DISP,WRAP PAPER 14X14 LOGO VER2,Paper Costs,55000,
1795,DAIRY PRODUCTS,YOGURT FRZN NF NICK THE GREEK,Dairy Costs,51300,
1 Id Sysco Category Sysco Description Integreat Account Integreat Account Code Nick's changes
1759 1758 MEATS PORK BELLY SKIN ON P12 COV Beef/Pork Costs 51110
1760 1759 MEATS PORK SHANK BONE KUROBUTA PR12 Beef/Pork Costs 51110
1761 1760 CANNED AND DRY SEASONING ITALIAN WHL Food Costs 50000
1762 1761 PRODUCE MUSHROOM PORTABELLA CAP 4-5 Produce Costs 51200
1763 1762 PAPER & DISP BAG PAPER 250 CT Paper Costs 55000
1764 1763 MEATS BEEF SHLDR TERES MAJOR SEL Beef/Pork Costs 51110
1765 1764 PAPER & DISP BOWL PLASTIC COATING 42 OZ Paper Costs 55000
1766 1765 PAPER & DISP BOX CATERING 21X13X4.25 LOGO Paper Costs 55000
1767 1766 CANNED AND DRY CANDY MILK CHOC SHELLS Food Costs 50000
1768 1767 CANNED AND DRY CHOCOLATE DUBAI PISTCHO KUNFEH Food Costs 50000
1769 1768 PAPER & DISP CONTAINER PAPER 1/30 OZ NTG Paper Costs 55000
1770 1769 PAPER & DISP CONTAINER PAPER 4/110OZ NTG Paper Costs 55000
1771 1770 PAPER & DISP CUP PAPER COLD 22 OZ LOGO NTG Paper Costs 55000
1772 1771 PAPER & DISP CUP PORTION PLAS CLR 1.50 OZ Paper Costs 55000
1773 1772 CANNED AND DRY DESSERT CUP Food Costs 50000
1774 1773 FROZEN DESSERT MINI PLAIN BEIGNET Food Costs 50000
1775 1774 CANNED AND DRY DIP GARLIC TOUM Food Costs 50000
1776 1775 CANNED AND DRY DRINK ENERGY ORANGE SPRKLNG Soft Beverage Costs 52000
1777 1776 CANNED AND DRY DRINK ENERGY PEACH VIBE SPRKLG Soft Beverage Costs 52000
1778 1777 CANNED AND DRY DRINK ENERGY TROPICAL VIBE Soft Beverage Costs 52000
1779 1778 PAPER & DISP FILM PVC 18X2000 ROLL Paper Costs 55000
1780 1779 CANNED AND DRY JUICE CONC MANDARIN CARDAMOM Food Costs 50000
1781 1780 CANNED AND DRY JUICE CONC STRAWB DRAGON Food Costs 50000
1782 1781 PAPER & DISP LID CLEAR PET 42 OZ Paper Costs 55000
1783 1782 PAPER & DISP LID DOME DESSERT CUP Paper Costs 55000
1784 1783 PAPER & DISP NAPKIN 2PLY INTR FOLD 6.3X8.26 Paper Costs 55000
1785 1784 CANNED AND DRY PASTE HERB HARISSA MOROCCAN Food Costs 50000
1786 1785 CANNED AND DRY PASTE TAHINI DRESSING Food Costs 50000
1787 1786 FROZEN PASTRY BEIGNET MN FLD CHOCCRML Food Costs 50000
1788 1787 CANNED AND DRY PEPPER BANANA MILD RING Food Costs 50000
1789 1788 CANNED AND DRY RICE MIX NICKS Food Costs 50000
1790 1789 CANNED AND DRY SODA CHERRY VISSINADA GREEK Soft Beverage Costs 52000
1791 1790 CANNED AND DRY SODA COLA PEPSI ZERO SUGAR Soft Beverage Costs 52000
1792 1791 CANNED AND DRY SODA PEPSI COLA Soft Beverage Costs 52000
1793 1792 FROZEN SPANAKOPITA SPINACH COOKED Food Costs 50000
1794 1793 PAPER & DISP SPOON PLAS TEA PP X-HVY BLK Paper Costs 55000
1795 1794 PAPER & DISP WRAP PAPER 14X14 LOGO VER2 Paper Costs 55000
1796 1795 DAIRY PRODUCTS YOGURT FRZN NF NICK THE GREEK Dairy Costs 51300

View File

@@ -18,7 +18,7 @@
[datomic.api :as dc])
(:import (java.util UUID)))
(def sysco-name->line (atom nil))
(def sysco-name->line (atom nil))
(defn get-sysco->line []
(when (nil? @sysco-name->line)
(reset! sysco-name->line
@@ -90,16 +90,60 @@
tax)
updated-invoice (assoc invoice :invoice/expense-accounts
(for [[account amount] items-with-tax]
#:invoice-expense-account {:db/id (random-tempid)
:account account
#:invoice-expense-account {:db/id (random-tempid)
:account account
:location (:invoice/location invoice)
:amount amount}))]
:amount amount}))]
(if (check-okay-amount? updated-invoice)
updated-invoice
(do (alog/warn ::itemized-expenses-not-adding-up
:invoice updated-invoice)
invoice))))
(defn code-invoices-list-items [invoice]
(with-precision 2
(let [line-items (:line-items invoice)
invoice-total (if (string? (:invoice/total invoice))
(Double/parseDouble (:invoice/total invoice))
(:invoice/total invoice))
abs-total (Math/abs invoice-total)
expense-accounts (reduce
(fn [acc {:keys [description amount]}]
(let [account (get-line-account description)]
(update acc account (fnil + 0.0) amount)))
{}
line-items)
total-line-amount (reduce + 0.0 (vals expense-accounts))
accounts (if (zero? total-line-amount)
[]
(vec (for [[account amount] expense-accounts]
(let [ratio (/ amount total-line-amount)
cents (int (Math/round (* ratio abs-total 100)))]
#:invoice-expense-account {:db/id (random-tempid)
:account account
:location (:invoice/location invoice)
:amount (* 0.01 cents)}))))
accounts (mapv
(fn [a]
(update a :invoice-expense-account/amount
#(with-precision 2
(double (.setScale (bigdec %) 2 java.math.RoundingMode/HALF_UP)))))
accounts)
leftover (with-precision 2 (.round (bigdec (- abs-total
(reduce + 0.0 (map :invoice-expense-account/amount accounts))))
*math-context*))
accounts (if (seq accounts)
(update-in accounts [(dec (count accounts)) :invoice-expense-account/amount] #(+ % (double leftover)))
[])]
(dissoc (assoc invoice :invoice/expense-accounts accounts) :line-items))))
(defn maybe-code-line-items [invoice]
(if (and (seq (:line-items invoice))
(= "Sysco" (-> (dc/pull (dc/db conn) [:vendor/name] (:invoice/vendor invoice))
:vendor/name)))
(code-invoices-list-items invoice)
(dissoc invoice :line-items)))
(defn extract-invoice-details [csv-rows sysco-vendor]
(let [[header-row & csv-rows] csv-rows
header-row (into {} (map vector header-keys header-row))
@@ -139,31 +183,31 @@
:invoice-number (header-row "InvoiceNumber")
:customer-name (header-row "CustomerName"))
(cond-> #:invoice {:invoice-number (header-row "InvoiceNumber")
(cond-> #:invoice {:invoice-number (header-row "InvoiceNumber")
:db/id (random-tempid)
:total (+ total tax)
:total (+ total tax)
:outstanding-balance (+ total tax)
:location (parse/best-location-match (dc/pull (dc/db conn)
[{:client/location-matches [:location-match/location :location-match/matches]}
:client/default-location
:client/locations]
(:db/id matching-client))
location-hint
location-hint)
:date (coerce/to-date date)
:vendor (:db/id sysco-vendor)
:client (:db/id matching-client)
:import-status :import-status/imported
:status :invoice-status/unpaid
:client-identifier customer-identifier}
:location (parse/best-location-match (dc/pull (dc/db conn)
[{:client/location-matches [:location-match/location :location-match/matches]}
:client/default-location
:client/locations]
(:db/id matching-client))
location-hint
location-hint)
:date (coerce/to-date date)
:vendor (:db/id sysco-vendor)
:client (:db/id matching-client)
:import-status :import-status/imported
:status :invoice-status/unpaid
:client-identifier customer-identifier}
true (code-invoice)
code-items (code-individual-items csv-rows tax))))
(defn mark-key [k]
(s3/copy-object {:source-bucket-name bucket-name
(s3/copy-object {:source-bucket-name bucket-name
:destination-bucket-name bucket-name
:destination-key (str/replace-first k "pending" "imported")
:source-key k})
:destination-key (str/replace-first k "pending" "imported")
:source-key k})
(s3/delete-object {:bucket-name bucket-name
:key k}))
@@ -180,7 +224,7 @@
([] (get-test-invoice-file 999))
([i]
(nth (->> (s3/list-objects-v2 {:bucket-name "data.prod.app.integreatconsult.com"
:prefix "sysco/imported"})
:prefix "sysco/imported"})
:object-summaries
(map :key))
i)))
@@ -205,10 +249,10 @@
(defn import-sysco []
(let [sysco-vendor (get-sysco-vendor)
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "sysco/pending"})
:object-summaries
(map :key))]
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "sysco/pending"})
:object-summaries
(map :key))]
(alog/info ::importing-sysco
:count (count keys)
@@ -219,10 +263,10 @@
(try
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
invoice-url (str "https://" (:data-bucket env) "/" invoice-key)]
(s3/copy-object {:source-bucket-name (:data-bucket env)
(s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key invoice-key})
:source-key k
:destination-key invoice-key})
[[:propose-invoice
(-> k
read-sysco-csv
@@ -232,13 +276,13 @@
(alog/error ::cant-load-file
:file k
:error e e)
(s3/copy-object {:source-bucket-name (:data-bucket env)
(s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key (str "sysco/error/"
(.getName (io/file k)))})
:source-key k
:destination-key (str "sysco/error/"
(.getName (io/file k)))})
[])))))
result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})])
(doseq [k keys]
(mark-key k))))

View File

@@ -26,6 +26,9 @@
(str/includes? (str header) "DISCOUNT_MESSAGE")
:wismettac
(str/includes? (str header) "Item Description")
:sysco-invoices-list
(str/includes? (str header) "Status")
:ledyard
@@ -34,7 +37,6 @@
(str/includes? (str header) "PARENT CUSTOMER NAME")
:worldwide
:else
nil)]
(alog/info ::csv-type-determined :type csv-type)
@@ -108,7 +110,7 @@
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number)
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:total (str/replace value #"," "")
:text (str/join " " row)
:full-text (str/join " " row)})))
@@ -124,7 +126,7 @@
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number)
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:total (str/replace value #"," "")
:text (str/join " " row)
:full-text (str/join " " row)})))
@@ -171,13 +173,13 @@
(transduce
(comp
(map (fn [[invoice-number date due amount standard :as row]]
{:vendor-code "Performance Food Group - LEDYARD"
{:vendor-code "Performance Food Group - LEDYARD"
:invoice-number invoice-number
:date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"]))
:date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"]))
:due (some-> due not-empty (parse-date-fallover ["MM/dd/yy"]))
:total (str/replace amount #"[\$,]" "")
:text (str/join " " row)
:full-text (str/join " " row)})))
:total (str/replace amount #"[\$,]" "")
:text (str/join " " row)
:full-text (str/join " " row)})))
conj
[]
(drop 1 rows)))
@@ -187,17 +189,41 @@
(transduce
(comp
(map (fn [[_ customer-name _ inv date amount :as row]]
{:vendor-code "Worldwide Produce"
{:vendor-code "Worldwide Produce"
:customer-identifier customer-name
:invoice-number (str/replace inv #"[=\"]" "")
:date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"]))
:total (str/replace amount #"[\$,]" "")
:text (str/join " " row)
:full-text (str/join " " row)})))
:invoice-number (str/replace inv #"[=\"]" "")
:date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"]))
:total (str/replace amount #"[\$,]" "")
:text (str/join " " row)
:full-text (str/join " " row)})))
conj
[]
(drop 1 rows)))
(defmethod parse-csv :sysco-invoices-list
[rows]
(let [header (first rows)]
(->> (drop 1 rows)
(map (fn [row] (into {} (map vector header row))))
(filter (fn [row]
(let [qty (get row "Current Quantity")]
(not (or (str/blank? qty) (= qty "0"))))))
(group-by (fn [row] (get row "Invoice")))
(map (fn [[_ invoice-rows]]
(let [first-row (first invoice-rows)]
{:invoice-number (get first-row "Invoice")
:date (parse-date-fallover (get first-row "Invoice Date") ["yyyy-MM-dd"])
:total (str/replace (get first-row "Amount Due") #"[,\$]" "")
:customer-identifier (get first-row "Ship To Name")
:vendor-code "Sysco"
:text (str/join " " (mapcat vals invoice-rows))
:full-text (str/join " " (mapcat vals invoice-rows))
:line-items (mapv (fn [row]
{:description (get row "Item Description")
:amount (Double/parseDouble (str/replace (get row "Total Amount") #"[,\$]" ""))})
invoice-rows)})))
vec)))
#_{:clj-kondo/ignore [:unused-binding]}
(defmethod parse-csv nil
[rows]

View File

@@ -8,6 +8,7 @@
[auto-ap.datomic.vendors :as d-vendors]
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
[auto-ap.import.manual :as manual]
[auto-ap.jobs.sysco :as sysco]
[auto-ap.logging :as alog]
[auto-ap.parse :as parse]
[auto-ap.routes.utils :refer [wrap-secure]]
@@ -49,7 +50,7 @@
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
{:vendor-code vendor-code})))))
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}]
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status line-items]}]
(let [matching-client (cond
account-number (:db/id (d-clients/exact-match account-number))
customer-identifier (:db/id (d-clients/best-match customer-identifier))
@@ -71,18 +72,19 @@
matching-client)
text
full-text))]
#:invoice {:db/id (random-tempid)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid}))
(cond-> #:invoice {:db/id (random-tempid)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid}
(seq line-items) (assoc :line-items line-items))))
(defn validate-invoice [invoice user]
(when-not (:invoice/client invoice)
@@ -111,6 +113,7 @@
(map #(validate-invoice % user))
admin-only-if-multiple-clients
(mapv d-invoices/code-invoice)
(mapv sysco/maybe-code-line-items)
(mapv (fn [i] [:propose-invoice i])))]
(alog/info ::creating-invoice :invoices potential-invoices)

View File

@@ -11,6 +11,7 @@
[auto-ap.graphql.utils :refer [assert-can-see-client assert-not-locked
can-see-client? exception->notification
extract-client-ids]]
[auto-ap.jobs.sysco :as sysco]
[auto-ap.logging :as alog]
[auto-ap.parse :as parse]
[auto-ap.permissions :refer [can? wrap-must]]
@@ -597,7 +598,7 @@
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
{:vendor-code vendor-code})))))
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-search vendor-override location-override import-status]} user]
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-search vendor-override location-override import-status line-items]} user]
(when-not total
(throw (Exception. "Couldn't parse total from file.")))
(when-not date
@@ -624,19 +625,20 @@
matching-client)
text
full-text))]
#:invoice {:db/id (random-tempid)
:invoice/uploader (-> user :db/id)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid}))
(cond-> #:invoice {:db/id (random-tempid)
:invoice/uploader (-> user :db/id)
:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid}
(seq line-items) (assoc :line-items line-items))))
(defn validate-invoice [invoice user]
(let [missing-keys (for [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date]
@@ -681,6 +683,7 @@
proposed-invoices (->> potential-invoices
(filter #(not (:error-message %)))
(mapv d-invoices/code-invoice)
(mapv sysco/maybe-code-line-items)
(mapv (fn [i] [:propose-invoice i])))]
(alog/info ::creating-invoice :invoices proposed-invoices)