notid/integreat
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(sysco): fall back to PAPER & DISP category when description is unmapped

Browse Source 
The Sysco importer codes each line item by exact description match against
resources/sysco_line_item_mapping.csv, silently defaulting to GL 50000 (Food
Costs) when the description is absent. Every new or renamed Sysco SKU
therefore leaks into Food Costs until someone hand-patches the CSV, which is
what 38575aa5 did for 34 descriptions.

Add a category-level fallback consulted after the description map and before
the 50000 default, enabled for PAPER & DISP only. The description mapping
still wins wherever it exists, so nothing already mapped changes.

PAPER & DISP is safe to generalize: all 454 mapped PAPER & DISP rows point at
55000, with no exceptions. Of the 852 distinct descriptions ever invoiced
under that category, only 3 resolved elsewhere, each because a row with a
different category shared the description and won the later-wins (into {}).
One of those, DESSERT CUP, was simply mis-categorized -- it is paper, and its
own lid (id 1782 LID DOME DESSERT CUP) was already 55000 -- so correct id 1772
to PAPER & DISP / 55000. The remaining two stay at 50000 on purpose, since
they are not paper: PAD SCRUB S-S 35 GRAM 1.25 OZ (SUPP & EQUIP) and TEST
STRIP SANITIZER QUAT (CHEMICAL/JANTRL).

Verified by replaying both changes over all 1,022,732 DET lines in the 56,010
CSVs under sysco-poller/: every resulting transition is 50000 -> 55000 (10,994
lines, $728,106.62). No line that already resolved to a non-default account
moved.

Note this only affects clients carrying the code-sysco-items feature flag, and
only on import -- already-imported invoices need a separate recode.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Bryce
2026-07-31 07:43:58 -07:00
parent 2b5fbaca00
commit 7a0e256f07
2 changed files with 72 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
src/clj/auto_ap/jobs/sysco.clj
View File

@@ -35,20 +35,34 @@
(into {}))))))
@sysco-name->line)
(defn get-line-account [item-name]
(get (get-sysco->line)
item-name
(ffirst (dc/q '[:find ?a
:in $ ?an
:where [?a :account/numeric-code ?an]]
(dc/db conn)
50000))))
(defn get-account-by-code [numeric-code]
(ffirst (dc/q '[:find ?a
:in $ ?an
:where [?a :account/numeric-code ?an]]
(dc/db conn)
numeric-code)))
;; Sysco categories that are unambiguous at the category level, so a line item
;; whose description is missing from sysco_line_item_mapping.csv still codes
;; correctly instead of silently defaulting to Food Costs. Only PAPER & DISP
;; qualifies: all 454 mapped PAPER & DISP descriptions point at 55000, and the
;; description mapping still wins where it exists.
(def category->numeric-code {"PAPER & DISP" 55000})
(def default-numeric-code 50000)
(defn get-line-account
([item-name] (get-line-account item-name nil))
([item-name sysco-category]
(or (get (get-sysco->line) item-name)
(some-> (category->numeric-code sysco-category) get-account-by-code)
(get-account-by-code default-numeric-code))))
(def ^:dynamic bucket-name (:data-bucket env))
(def header-keys ["TransCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "AccountName" "AccountDunsNo" "InvoiceDate" "AccountDate" "CustomerPONo" "PaymentTerms" "TermsDescription" "StoreNumber" "CustomerName" "AddressLine1" "AddressLine2" "City1" "State1" "Zip1" "Phone1" "Duns1" "Hin1" "Dea1" "TIDCustomer" "ChainNumber" "BidNumber" "ContractNumber" "CompanyNumber" "BriefName" "Address" "Address2" "City2" "State2" "Zip2" "Phone2" "Duns2" "Hin2" "Dea2" "Tid_OPCO" "ObligationIndicator" "Manifest" "Route" "Stop" "TermsDiscountPercent" "TermsDiscountDueDate" "TermsNetDueDate" "TermsDiscountAmount" "TermsDiscountCode" "OrderDate" "DepartmentCode"])
(def item-price-index 15)
(def item-category-index 25)
(def item-name-index 29)
(def summary-keys ["TranCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "TotalLines" "TotalQtyInvoice" "TotalQty" "TotalQtySplit" "TotalQtyPounds" "TotalExtendedPrice" "TotalTaxAmount" "TotalInvoiceAmount" "AccountDate"])
@@ -56,14 +70,13 @@
(defn get-sysco-vendor []
(let [db (dc/db conn)]
(->
(dc/q '[:find (pull ?v r)
:in $ r
:where [?v :vendor/name "Sysco"]]
db
d-vendors/default-read)
first
first)))
(dc/q '[:find (pull ?v r)
:in $ r
:where [?v :vendor/name "Sysco"]]
db
d-vendors/default-read)
first
first)))
(defn read-sysco-csv [k]
(-> (s3/get-object {:bucket-name bucket-name
@@ -73,34 +86,33 @@
csv/read-csv))
(defn check-okay-amount? [i]
(dollars=
(dollars=
(:invoice/total i)
(reduce + 0.0 (map :invoice-expense-account/amount (:invoice/expense-accounts i)))))
(defn code-individual-items [invoice csv-rows tax]
(let [items (->> csv-rows
butlast
(reduce
(fn [acc row]
(update acc (get-line-account (nth row item-name-index))
(fnil + 0.0)
(Double/parseDouble (nth row item-price-index))
)
)
{})
)
(reduce
(fn [acc row]
(update acc (get-line-account (nth row item-name-index)
(nth row item-category-index))
(fnil + 0.0)
(Double/parseDouble (nth row item-price-index))))
{}))
items-with-tax (update items (get-line-account "TAX")
(fnil + 0.0)
(fnil + 0.0)
tax)
updated-invoice (assoc invoice :invoice/expense-accounts
(for [[account amount] items-with-tax]
#:invoice-expense-account {:db/id (random-tempid)
:account account
:location (:invoice/location invoice)
:amount amount}))]
updated-invoice (assoc invoice :invoice/expense-accounts
(for [[account amount] items-with-tax]
#:invoice-expense-account {:db/id (random-tempid)
:account account
:location (:invoice/location invoice)
:amount amount}))]
(if (check-okay-amount? updated-invoice)
updated-invoice
(do (alog/warn ::itemized-expenses-not-adding-up
(do (alog/warn ::itemized-expenses-not-adding-up
:invoice updated-invoice)
invoice))))
@@ -122,11 +134,11 @@
(header-row "AddressLine2")
(header-row "City1")
(header-row "City2")])
account-number (some-> account-number Long/parseLong str)
matching-client (and account-number
(d-clients/exact-match account-number))
_ (when-not matching-client
(throw (ex-info "cannot find matching client"
{:account-number account-number
@@ -153,9 +165,9 @@
:client/locations]
(:db/id matching-client))
location-hint
location-hint )
location-hint)
:date (coerce/to-date date)
:vendor (:db/id sysco-vendor )
:vendor (:db/id sysco-vendor)
:client (:db/id matching-client)
:import-status :import-status/imported
:status :invoice-status/unpaid
@@ -180,64 +192,54 @@
(s3/delete-object {:bucket-name bucket-name
:key k}))
(defn get-test-invoice-file
(defn get-test-invoice-file
([] (get-test-invoice-file 999))
( [i]
([i]
(nth (->> (s3/list-objects-v2 {:bucket-name "data.prod.app.integreatconsult.com"
:prefix "sysco/imported"})
:object-summaries
(map :key)
)
(map :key))
i)))
(comment
(with-bindings { #'bucket-name "data.prod.app.integreatconsult.com"}
(doall
(for [n (range 930 940 )
:let [result (-> (get-test-invoice-file n)
read-sysco-csv
(extract-invoice-details (get-sysco-vendor))
)]
#_#_:when (not (check-okay-amount? result))]
(comment
(with-bindings {#'bucket-name "data.prod.app.integreatconsult.com"}
(doall
(for [n (range 930 940)
:let [result (-> (get-test-invoice-file n)
read-sysco-csv
(extract-invoice-details (get-sysco-vendor)))]
#_#_:when (not (check-okay-amount? result))]
result)))
(with-bindings { #'bucket-name "data.prod.app.integreatconsult.com"}
(let [result (-> "sysco/error/SYSCO050_00175962_20241010122639019.csv"
(with-bindings {#'bucket-name "data.prod.app.integreatconsult.com"}
(let [result (-> "sysco/error/SYSCO050_00175962_20241010122639019.csv"
read-sysco-csv
(extract-invoice-details (get-sysco-vendor))
)]
(extract-invoice-details (get-sysco-vendor)))]
result))
)
result)))
(defn import-sysco []
(let [sysco-vendor (get-sysco-vendor)
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "sysco/pending"})
:object-summaries
(map :key))]
:object-summaries
(map :key))]
(alog/info ::importing-sysco
:count (count keys)
:keys (pr-str keys))
(let [transaction (->> keys
(mapcat (fn [k]
(try
(try
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
invoice-url (str "https://" (:data-bucket env) "/" invoice-key)]
(s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key invoice-key})
[[:propose-invoice
[[:propose-invoice
(-> k
read-sysco-csv
(extract-invoice-details sysco-vendor)
@@ -246,7 +248,7 @@
(alog/error ::cant-load-file
:file k
:error e e)
(s3/copy-object {:source-bucket-name (:data-bucket env)
(s3/copy-object {:source-bucket-name (:data-bucket env)
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key (str "sysco/error/"
@@ -256,6 +258,5 @@
(doseq [k keys]
(mark-key k))))
(defn -main [& _]
(execute "sysco" import-sysco))