diff --git a/resources/sysco_line_item_mapping.csv b/resources/sysco_line_item_mapping.csv index e821eda6..3bcf2b79 100644 --- a/resources/sysco_line_item_mapping.csv +++ b/resources/sysco_line_item_mapping.csv @@ -1759,4 +1759,38 @@ Id,Sysco Category,Sysco Description,Integreat Account,Integreat Account Code,Nic 1758,MEATS,PORK BELLY SKIN ON P12 COV,Beef/Pork Costs,51110, 1759,MEATS,PORK SHANK BONE KUROBUTA PR12,Beef/Pork Costs,51110, 1760,CANNED AND DRY,SEASONING ITALIAN WHL,Food Costs,50000, -1761,PRODUCE,MUSHROOM PORTABELLA CAP 4-5,Produce Costs,51200, \ No newline at end of file +1761,PRODUCE,MUSHROOM PORTABELLA CAP 4-5,Produce Costs,51200, +1762,PAPER & DISP,BAG PAPER 250 CT,Paper Costs,55000, +1763,MEATS,BEEF SHLDR TERES MAJOR SEL,Beef/Pork Costs,51110, +1764,PAPER & DISP,BOWL PLASTIC COATING 42 OZ,Paper Costs,55000, +1765,PAPER & DISP,BOX CATERING 21X13X4.25 LOGO,Paper Costs,55000, +1766,CANNED AND DRY,CANDY MILK CHOC SHELLS,Food Costs,50000, +1767,CANNED AND DRY,CHOCOLATE DUBAI PISTCHO KUNFEH,Food Costs,50000, +1768,PAPER & DISP,CONTAINER PAPER 1/30 OZ NTG,Paper Costs,55000, +1769,PAPER & DISP,CONTAINER PAPER 4/110OZ NTG,Paper Costs,55000, +1770,PAPER & DISP,CUP PAPER COLD 22 OZ LOGO NTG,Paper Costs,55000, +1771,PAPER & DISP,CUP PORTION PLAS CLR 1.50 OZ,Paper Costs,55000, +1772,CANNED AND DRY,DESSERT CUP,Food Costs,50000, +1773,FROZEN,DESSERT MINI PLAIN BEIGNET,Food Costs,50000, +1774,CANNED AND DRY,DIP GARLIC TOUM,Food Costs,50000, +1775,CANNED AND DRY,DRINK ENERGY ORANGE SPRKLNG,Soft Beverage Costs,52000, +1776,CANNED AND DRY,DRINK ENERGY PEACH VIBE SPRKLG,Soft Beverage Costs,52000, +1777,CANNED AND DRY,DRINK ENERGY TROPICAL VIBE,Soft Beverage Costs,52000, +1778,PAPER & DISP,FILM PVC 18X2000 ROLL,Paper Costs,55000, +1779,CANNED AND DRY,JUICE CONC MANDARIN CARDAMOM,Food Costs,50000, +1780,CANNED AND DRY,JUICE CONC STRAWB DRAGON,Food Costs,50000, +1781,PAPER & DISP,LID CLEAR PET 42 OZ,Paper Costs,55000, +1782,PAPER & DISP,LID DOME DESSERT CUP,Paper Costs,55000, +1783,PAPER & DISP,NAPKIN 2PLY INTR FOLD 6.3X8.26,Paper Costs,55000, +1784,CANNED AND DRY,PASTE HERB HARISSA MOROCCAN,Food Costs,50000, +1785,CANNED AND DRY,PASTE TAHINI DRESSING,Food Costs,50000, +1786,FROZEN,PASTRY BEIGNET MN FLD CHOCCRML,Food Costs,50000, +1787,CANNED AND DRY,PEPPER BANANA MILD RING,Food Costs,50000, +1788,CANNED AND DRY,RICE MIX NICKS,Food Costs,50000, +1789,CANNED AND DRY,SODA CHERRY VISSINADA GREEK,Soft Beverage Costs,52000, +1790,CANNED AND DRY,SODA COLA PEPSI ZERO SUGAR,Soft Beverage Costs,52000, +1791,CANNED AND DRY,SODA PEPSI COLA,Soft Beverage Costs,52000, +1792,FROZEN,SPANAKOPITA SPINACH COOKED,Food Costs,50000, +1793,PAPER & DISP,SPOON PLAS TEA PP X-HVY BLK,Paper Costs,55000, +1794,PAPER & DISP,WRAP PAPER 14X14 LOGO VER2,Paper Costs,55000, +1795,DAIRY PRODUCTS,YOGURT FRZN NF NICK THE GREEK,Dairy Costs,51300, diff --git a/src/clj/auto_ap/jobs/sysco.clj b/src/clj/auto_ap/jobs/sysco.clj index c8465ce0..6023d18c 100644 --- a/src/clj/auto_ap/jobs/sysco.clj +++ b/src/clj/auto_ap/jobs/sysco.clj @@ -18,7 +18,7 @@ [datomic.api :as dc]) (:import (java.util UUID))) -(def sysco-name->line (atom nil)) +(def sysco-name->line (atom nil)) (defn get-sysco->line [] (when (nil? @sysco-name->line) (reset! sysco-name->line @@ -90,16 +90,60 @@ tax) updated-invoice (assoc invoice :invoice/expense-accounts (for [[account amount] items-with-tax] - #:invoice-expense-account {:db/id (random-tempid) - :account account + #:invoice-expense-account {:db/id (random-tempid) + :account account :location (:invoice/location invoice) - :amount amount}))] + :amount amount}))] (if (check-okay-amount? updated-invoice) updated-invoice (do (alog/warn ::itemized-expenses-not-adding-up :invoice updated-invoice) invoice)))) +(defn code-invoices-list-items [invoice] + (with-precision 2 + (let [line-items (:line-items invoice) + invoice-total (if (string? (:invoice/total invoice)) + (Double/parseDouble (:invoice/total invoice)) + (:invoice/total invoice)) + abs-total (Math/abs invoice-total) + expense-accounts (reduce + (fn [acc {:keys [description amount]}] + (let [account (get-line-account description)] + (update acc account (fnil + 0.0) amount))) + {} + line-items) + total-line-amount (reduce + 0.0 (vals expense-accounts)) + accounts (if (zero? total-line-amount) + [] + (vec (for [[account amount] expense-accounts] + (let [ratio (/ amount total-line-amount) + cents (int (Math/round (* ratio abs-total 100)))] + #:invoice-expense-account {:db/id (random-tempid) + :account account + :location (:invoice/location invoice) + :amount (* 0.01 cents)})))) + accounts (mapv + (fn [a] + (update a :invoice-expense-account/amount + #(with-precision 2 + (double (.setScale (bigdec %) 2 java.math.RoundingMode/HALF_UP))))) + accounts) + leftover (with-precision 2 (.round (bigdec (- abs-total + (reduce + 0.0 (map :invoice-expense-account/amount accounts)))) + *math-context*)) + accounts (if (seq accounts) + (update-in accounts [(dec (count accounts)) :invoice-expense-account/amount] #(+ % (double leftover))) + [])] + (dissoc (assoc invoice :invoice/expense-accounts accounts) :line-items)))) + +(defn maybe-code-line-items [invoice] + (if (and (seq (:line-items invoice)) + (= "Sysco" (-> (dc/pull (dc/db conn) [:vendor/name] (:invoice/vendor invoice)) + :vendor/name))) + (code-invoices-list-items invoice) + (dissoc invoice :line-items))) + (defn extract-invoice-details [csv-rows sysco-vendor] (let [[header-row & csv-rows] csv-rows header-row (into {} (map vector header-keys header-row)) @@ -139,31 +183,31 @@ :invoice-number (header-row "InvoiceNumber") :customer-name (header-row "CustomerName")) - (cond-> #:invoice {:invoice-number (header-row "InvoiceNumber") + (cond-> #:invoice {:invoice-number (header-row "InvoiceNumber") :db/id (random-tempid) - :total (+ total tax) + :total (+ total tax) :outstanding-balance (+ total tax) - :location (parse/best-location-match (dc/pull (dc/db conn) - [{:client/location-matches [:location-match/location :location-match/matches]} - :client/default-location - :client/locations] - (:db/id matching-client)) - location-hint - location-hint) - :date (coerce/to-date date) - :vendor (:db/id sysco-vendor) - :client (:db/id matching-client) - :import-status :import-status/imported - :status :invoice-status/unpaid - :client-identifier customer-identifier} + :location (parse/best-location-match (dc/pull (dc/db conn) + [{:client/location-matches [:location-match/location :location-match/matches]} + :client/default-location + :client/locations] + (:db/id matching-client)) + location-hint + location-hint) + :date (coerce/to-date date) + :vendor (:db/id sysco-vendor) + :client (:db/id matching-client) + :import-status :import-status/imported + :status :invoice-status/unpaid + :client-identifier customer-identifier} true (code-invoice) code-items (code-individual-items csv-rows tax)))) (defn mark-key [k] - (s3/copy-object {:source-bucket-name bucket-name + (s3/copy-object {:source-bucket-name bucket-name :destination-bucket-name bucket-name - :destination-key (str/replace-first k "pending" "imported") - :source-key k}) + :destination-key (str/replace-first k "pending" "imported") + :source-key k}) (s3/delete-object {:bucket-name bucket-name :key k})) @@ -180,7 +224,7 @@ ([] (get-test-invoice-file 999)) ([i] (nth (->> (s3/list-objects-v2 {:bucket-name "data.prod.app.integreatconsult.com" - :prefix "sysco/imported"}) + :prefix "sysco/imported"}) :object-summaries (map :key)) i))) @@ -205,10 +249,10 @@ (defn import-sysco [] (let [sysco-vendor (get-sysco-vendor) - keys (->> (s3/list-objects-v2 {:bucket-name bucket-name - :prefix "sysco/pending"}) - :object-summaries - (map :key))] + keys (->> (s3/list-objects-v2 {:bucket-name bucket-name + :prefix "sysco/pending"}) + :object-summaries + (map :key))] (alog/info ::importing-sysco :count (count keys) @@ -219,10 +263,10 @@ (try (let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ; invoice-url (str "https://" (:data-bucket env) "/" invoice-key)] - (s3/copy-object {:source-bucket-name (:data-bucket env) + (s3/copy-object {:source-bucket-name (:data-bucket env) :destination-bucket-name (:data-bucket env) - :source-key k - :destination-key invoice-key}) + :source-key k + :destination-key invoice-key}) [[:propose-invoice (-> k read-sysco-csv @@ -232,13 +276,13 @@ (alog/error ::cant-load-file :file k :error e e) - (s3/copy-object {:source-bucket-name (:data-bucket env) + (s3/copy-object {:source-bucket-name (:data-bucket env) :destination-bucket-name (:data-bucket env) - :source-key k - :destination-key (str "sysco/error/" - (.getName (io/file k)))}) + :source-key k + :destination-key (str "sysco/error/" + (.getName (io/file k)))}) []))))) - result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})]) + result (audit-transact transaction {:user/name "sysco importer" :user/role "admin"})]) (doseq [k keys] (mark-key k)))) diff --git a/src/clj/auto_ap/parse/csv.clj b/src/clj/auto_ap/parse/csv.clj index 63aca950..89820c3b 100644 --- a/src/clj/auto_ap/parse/csv.clj +++ b/src/clj/auto_ap/parse/csv.clj @@ -26,6 +26,9 @@ (str/includes? (str header) "DISCOUNT_MESSAGE") :wismettac + (str/includes? (str header) "Item Description") + :sysco-invoices-list + (str/includes? (str header) "Status") :ledyard @@ -34,7 +37,6 @@ (str/includes? (str header) "PARENT CUSTOMER NAME") :worldwide - :else nil)] (alog/info ::csv-type-determined :type csv-type) @@ -108,7 +110,7 @@ {:vendor-code "Mama Lu's Foods" :customer-identifier customer :invoice-number (str po-number "-" invoice-number) - :date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"]) + :date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"]) :total (str/replace value #"," "") :text (str/join " " row) :full-text (str/join " " row)}))) @@ -124,7 +126,7 @@ {:vendor-code "Mama Lu's Foods" :customer-identifier customer :invoice-number (str po-number "-" invoice-number) - :date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"]) + :date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"]) :total (str/replace value #"," "") :text (str/join " " row) :full-text (str/join " " row)}))) @@ -171,13 +173,13 @@ (transduce (comp (map (fn [[invoice-number date due amount standard :as row]] - {:vendor-code "Performance Food Group - LEDYARD" + {:vendor-code "Performance Food Group - LEDYARD" :invoice-number invoice-number - :date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"])) + :date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"])) :due (some-> due not-empty (parse-date-fallover ["MM/dd/yy"])) - :total (str/replace amount #"[\$,]" "") - :text (str/join " " row) - :full-text (str/join " " row)}))) + :total (str/replace amount #"[\$,]" "") + :text (str/join " " row) + :full-text (str/join " " row)}))) conj [] (drop 1 rows))) @@ -187,17 +189,41 @@ (transduce (comp (map (fn [[_ customer-name _ inv date amount :as row]] - {:vendor-code "Worldwide Produce" + {:vendor-code "Worldwide Produce" :customer-identifier customer-name - :invoice-number (str/replace inv #"[=\"]" "") - :date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"])) - :total (str/replace amount #"[\$,]" "") - :text (str/join " " row) - :full-text (str/join " " row)}))) + :invoice-number (str/replace inv #"[=\"]" "") + :date (some-> date not-empty (parse-date-fallover ["MM/dd/yy"])) + :total (str/replace amount #"[\$,]" "") + :text (str/join " " row) + :full-text (str/join " " row)}))) conj [] (drop 1 rows))) +(defmethod parse-csv :sysco-invoices-list + [rows] + (let [header (first rows)] + (->> (drop 1 rows) + (map (fn [row] (into {} (map vector header row)))) + (filter (fn [row] + (let [qty (get row "Current Quantity")] + (not (or (str/blank? qty) (= qty "0")))))) + (group-by (fn [row] (get row "Invoice"))) + (map (fn [[_ invoice-rows]] + (let [first-row (first invoice-rows)] + {:invoice-number (get first-row "Invoice") + :date (parse-date-fallover (get first-row "Invoice Date") ["yyyy-MM-dd"]) + :total (str/replace (get first-row "Amount Due") #"[,\$]" "") + :customer-identifier (get first-row "Ship To Name") + :vendor-code "Sysco" + :text (str/join " " (mapcat vals invoice-rows)) + :full-text (str/join " " (mapcat vals invoice-rows)) + :line-items (mapv (fn [row] + {:description (get row "Item Description") + :amount (Double/parseDouble (str/replace (get row "Total Amount") #"[,\$]" ""))}) + invoice-rows)}))) + vec))) + #_{:clj-kondo/ignore [:unused-binding]} (defmethod parse-csv nil [rows] diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 9accd9d3..fdbc30a3 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -8,6 +8,7 @@ [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] [auto-ap.import.manual :as manual] + [auto-ap.jobs.sysco :as sysco] [auto-ap.logging :as alog] [auto-ap.parse :as parse] [auto-ap.routes.utils :refer [wrap-secure]] @@ -49,7 +50,7 @@ (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) -(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}] +(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status line-items]}] (let [matching-client (cond account-number (:db/id (d-clients/exact-match account-number)) customer-identifier (:db/id (d-clients/best-match customer-identifier)) @@ -71,18 +72,19 @@ matching-client) text full-text))] - #:invoice {:db/id (random-tempid) - :invoice/client matching-client - :invoice/client-identifier (or account-number customer-identifier) - :invoice/vendor (:db/id matching-vendor) - :invoice/source-url source-url - :invoice/invoice-number invoice-number - :invoice/total (Double/parseDouble total) - :invoice/date (to-date date) - :invoice/location matching-location - :invoice/import-status (or import-status :import-status/pending) - :invoice/outstanding-balance (Double/parseDouble total) - :invoice/status :invoice-status/unpaid})) + (cond-> #:invoice {:db/id (random-tempid) + :invoice/client matching-client + :invoice/client-identifier (or account-number customer-identifier) + :invoice/vendor (:db/id matching-vendor) + :invoice/source-url source-url + :invoice/invoice-number invoice-number + :invoice/total (Double/parseDouble total) + :invoice/date (to-date date) + :invoice/location matching-location + :invoice/import-status (or import-status :import-status/pending) + :invoice/outstanding-balance (Double/parseDouble total) + :invoice/status :invoice-status/unpaid} + (seq line-items) (assoc :line-items line-items)))) (defn validate-invoice [invoice user] (when-not (:invoice/client invoice) @@ -111,6 +113,7 @@ (map #(validate-invoice % user)) admin-only-if-multiple-clients (mapv d-invoices/code-invoice) + (mapv sysco/maybe-code-line-items) (mapv (fn [i] [:propose-invoice i])))] (alog/info ::creating-invoice :invoices potential-invoices) diff --git a/src/clj/auto_ap/ssr/invoice/import.clj b/src/clj/auto_ap/ssr/invoice/import.clj index dfabd346..098a0250 100644 --- a/src/clj/auto_ap/ssr/invoice/import.clj +++ b/src/clj/auto_ap/ssr/invoice/import.clj @@ -11,6 +11,7 @@ [auto-ap.graphql.utils :refer [assert-can-see-client assert-not-locked can-see-client? exception->notification extract-client-ids]] + [auto-ap.jobs.sysco :as sysco] [auto-ap.logging :as alog] [auto-ap.parse :as parse] [auto-ap.permissions :refer [can? wrap-must]] @@ -597,7 +598,7 @@ (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) -(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-search vendor-override location-override import-status]} user] +(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-search vendor-override location-override import-status line-items]} user] (when-not total (throw (Exception. "Couldn't parse total from file."))) (when-not date @@ -624,19 +625,20 @@ matching-client) text full-text))] - #:invoice {:db/id (random-tempid) - :invoice/uploader (-> user :db/id) - :invoice/client matching-client - :invoice/client-identifier (or account-number customer-identifier) - :invoice/vendor (:db/id matching-vendor) - :invoice/source-url source-url - :invoice/invoice-number invoice-number - :invoice/total (Double/parseDouble total) - :invoice/date (to-date date) - :invoice/location matching-location - :invoice/import-status (or import-status :import-status/pending) - :invoice/outstanding-balance (Double/parseDouble total) - :invoice/status :invoice-status/unpaid})) + (cond-> #:invoice {:db/id (random-tempid) + :invoice/uploader (-> user :db/id) + :invoice/client matching-client + :invoice/client-identifier (or account-number customer-identifier) + :invoice/vendor (:db/id matching-vendor) + :invoice/source-url source-url + :invoice/invoice-number invoice-number + :invoice/total (Double/parseDouble total) + :invoice/date (to-date date) + :invoice/location matching-location + :invoice/import-status (or import-status :import-status/pending) + :invoice/outstanding-balance (Double/parseDouble total) + :invoice/status :invoice-status/unpaid} + (seq line-items) (assoc :line-items line-items)))) (defn validate-invoice [invoice user] (let [missing-keys (for [k [:invoice/invoice-number :invoice/client :invoice/vendor :invoice/total :invoice/outstanding-balance :invoice/date] @@ -681,6 +683,7 @@ proposed-invoices (->> potential-invoices (filter #(not (:error-message %))) (mapv d-invoices/code-invoice) + (mapv sysco/maybe-code-line-items) (mapv (fn [i] [:propose-invoice i])))] (alog/info ::creating-invoice :invoices proposed-invoices)