diff --git a/config/prod-background-worker.edn b/config/prod-background-worker.edn index 7aa8c468..9b6841e2 100644 --- a/config/prod-background-worker.edn +++ b/config/prod-background-worker.edn @@ -173,4 +173,4 @@ :token "EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"}} :plaid {:base-url "https://production.plaid.com" :client-id "61bfab05f7e762001b323f79" - :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}} + :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}} diff --git a/config/prod-cloud-background-worker.edn b/config/prod-cloud-background-worker.edn index 966dd855..4a77e7b5 100644 --- a/config/prod-cloud-background-worker.edn +++ b/config/prod-cloud-background-worker.edn @@ -31,5 +31,5 @@ :yodlee2-proxy-port 8888 :plaid {:base-url "https://production.plaid.com" :client-id "61bfab05f7e762001b323f79" - :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"} + :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"} } diff --git a/config/prod-cloud.edn b/config/prod-cloud.edn index db5449f8..739219ce 100644 --- a/config/prod-cloud.edn +++ b/config/prod-cloud.edn @@ -34,5 +34,5 @@ :yodlee2-proxy-port 8888 :plaid {:base-url "https://production.plaid.com" :client-id "61bfab05f7e762001b323f79" - :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"} + :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"} } diff --git a/dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf b/dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf new file mode 100644 index 00000000..ffd6e5c0 Binary files /dev/null and b/dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf differ diff --git a/hours.txt b/hours.txt index 8b137891..96944906 100644 --- a/hours.txt +++ b/hours.txt @@ -1 +1 @@ - +1` diff --git a/src/clj/auto_ap/jobs/backfill_olo_processors.clj b/src/clj/auto_ap/jobs/backfill_olo_processors.clj new file mode 100644 index 00000000..eee95b52 --- /dev/null +++ b/src/clj/auto_ap/jobs/backfill_olo_processors.clj @@ -0,0 +1,375 @@ +(ns auto-ap.jobs.backfill-olo-processors + "One-off backfill for the Olo third-party-source fix (commit fa25620b, \"olo fixes\"). + + That commit replaced the exact-match `condp` on `(:name (:source order))` in + `square3/tender->charge` with substring matching, so Olo-brokered orders that + arrive from Square with source names like \"Olo - DoorDash\" or + \"olo-ubereats\" now classify as the real delivery processor instead of + falling through to `:ccp-processor/na`. + + Orders imported *before* the fix shipped still carry the old + `:charge/processor`, which means `sales-summaries` bucketed them as + \"Unknown\" instead of \"Food App Payments\". This job recomputes + `:charge/processor` for already-imported Square charges over a date range + (default: 2026-07-07 -> today) and marks the affected sales summaries dirty + so `auto-ap.jobs.sales-summaries` recalculates them. + + IMPORTANT: this backfill never re-implements the classifier. It feeds the + stored shape of each charge back through the real production function + (`square3/tender->charge`), so it cannot drift from the importer. + + No Square API calls are made -- every input the classifier needs + (`:sales-order/source`, `:charge/note`, `:charge/type-name`) is already in + Datomic. See the `(comment ...)` block at the bottom for the API-based + fallback if you need to repair charges that have no parent sales order. + + Dry run by default. Pass `:apply? true` to write." + (:require + [auto-ap.datomic :refer [conn]] + [auto-ap.jobs.core :refer [execute]] + [auto-ap.jobs.sales-summaries :as summaries] + [auto-ap.logging :as alog] + [auto-ap.square.core3 :as square3] + [auto-ap.time :as atime] + [clj-time.coerce :as coerce] + [clj-time.core :as time] + [clj-time.format :as f] + [clj-time.periodic :as per] + [clojure.string :as str] + [config.core :refer [env]] + [datomic.api :as dc] + ;; the datalog below calls iol-ion.query/scan-charges by fully-qualified + ;; symbol, so the namespace has to be loaded. + [iol-ion.query])) + +;; --------------------------------------------------------------------------- +;; Date handling +;; --------------------------------------------------------------------------- + +(def pacific (time/time-zone-for-id "America/Los_Angeles")) + +(def default-start-date + "The Olo fix landed 2026-07-22; the user-visible bad data starts 7/7." + "2026-07-07") + +(defn parse-day + "\"2026-07-07\" -> local (Pacific) midnight DateTime. Passes DateTimes through." + [d] + (cond + (nil? d) nil + (string? d) (f/parse (f/with-zone (f/formatter "yyyy-MM-dd") pacific) d) + :else (coerce/to-date-time d))) + +(defn local-midnight [dt] + (.toDateMidnight (atime/localize dt))) + +;; --------------------------------------------------------------------------- +;; Classification -- delegates to the production importer +;; --------------------------------------------------------------------------- + +(defn recompute-processor + "Runs the *live* classifier over a charge's stored inputs. + + `tender->charge` only reads `(:name (:source order))` from the order and + `:type`/`:note` from the tender when deciding `:charge/processor`, so a + synthetic order/tender carrying the stored values yields exactly what a + fresh import would produce today. `:created_at` and `:id` are placeholders + purely to keep the unrelated `:charge/date` and `:charge/reference-link` + branches from blowing up; we only read `:charge/processor` back out. + + Note on nil sources: the importer stores `(or (:name (:source order)) + \"Square\")`, so a source that was originally nil comes back as \"Square\". + Both nil and \"Square\" classify to `:ccp-processor/na`, so the round trip + is faithful." + [{:keys [source note type-name]}] + (:charge/processor + (square3/tender->charge {:source {:name source} + :created_at "1970-01-01T00:00:00Z"} + {} ; client -> only :charge/client + {} ; location -> only :charge/location + {:id "backfill" + :type type-name + :note note}))) + +;; --------------------------------------------------------------------------- +;; Scanning +;; --------------------------------------------------------------------------- + +(def charge-pull + [:db/id + :charge/external-id + :charge/type-name + :charge/note + :charge/date + :charge/total + {:charge/processor [:db/ident]} + {:sales-order/_charges [:db/id + :sales-order/external-id + :sales-order/source + {:sales-order/vendor [:db/ident]}]}]) + +(defn client-charges + "All charges for `client-eid` between `start-inst` and `end-inst` (both + inclusive by day), via the :charge/client+date index." + [db client-eid start-inst end-inst] + (->> (dc/q '[:find (pull ?c pull-pattern) + :in $ pull-pattern [?clients ?start ?end] + :where + [(iol-ion.query/scan-charges $ ?clients ?start ?end) [[?c _ _] ...]]] + db + charge-pull + [[client-eid] start-inst end-inst]) + (map first))) + +(defn parent-order + "Reverse refs on component attributes come back as a single map from pull, + but tolerate a collection in case that ever changes." + [charge] + (let [o (:sales-order/_charges charge)] + (if (sequential? o) (first o) o))) + +(defn square-charge? [charge] + (= :vendor/ccp-square (get-in (parent-order charge) [:sales-order/vendor :db/ident]))) + +;; --------------------------------------------------------------------------- +;; Planning +;; --------------------------------------------------------------------------- + +(defn charge->change + "nil when the charge is already correct (or isn't ours to touch)." + [charge] + (when (square-charge? charge) + (let [order (parent-order charge) + current (get-in charge [:charge/processor :db/ident]) + next (recompute-processor {:source (:sales-order/source order) + :note (:charge/note charge) + :type-name (:charge/type-name charge)})] + (when (not= current next) + {:charge (:db/id charge) + :external-id (:charge/external-id charge) + :order (:sales-order/external-id order) + :date (:charge/date charge) + :source (:sales-order/source order) + :note (:charge/note charge) + :type-name (:charge/type-name charge) + :total (:charge/total charge) + :from current + :to next})))) + +(defn downgrade? + "A change that *loses* processor information. The Olo fix can only ever widen + matching, so these should not exist -- if they do, something else changed + and we'd rather report than clobber." + [{:keys [from to]}] + (and (= :ccp-processor/na to) + (not= :ccp-processor/na from) + (some? from))) + +(defn plan-for-client + [db {:keys [db/id client/code]} start-inst end-inst allow-downgrade?] + (let [charges (client-charges db id start-inst end-inst) + square (filter square-charge? charges) + orphans (->> charges + (remove (comp some? parent-order)) + (filter #(some-> (:charge/external-id %) + (str/starts-with? "square/charge/")))) + all (keep charge->change square) + [skipped changes] (if allow-downgrade? + [[] all] + [(filter downgrade? all) (remove downgrade? all)])] + {:client id + :code code + :scanned (count square) + :orphan-charges (count orphans) + :changes (vec changes) + :skipped (vec skipped)})) + +(defn build-plan + [db clients start-inst end-inst allow-downgrade?] + (->> clients + (map #(plan-for-client db % start-inst end-inst allow-downgrade?)) + (remove #(and (zero? (count (:changes %))) + (zero? (count (:skipped %))) + (zero? (:orphan-charges %)))) + vec)) + +;; --------------------------------------------------------------------------- +;; Reporting +;; --------------------------------------------------------------------------- + +(defn- transitions [changes] + (->> changes + (group-by (juxt :from :to)) + (map (fn [[[from to] cs]] + [from to (count cs) (reduce + 0.0 (keep :total cs))])) + (sort-by #(- (nth % 2))))) + +(defn print-report [plan] + (println) + (println "=== Olo processor backfill ===") + (doseq [{:keys [code scanned changes skipped orphan-charges]} plan] + (println) + (printf "%-8s scanned %d square charges, %d to update%n" + (str code) scanned (count changes)) + (doseq [[from to n total] (transitions changes)] + (printf " %-24s -> %-24s %5d $%.2f%n" + (str from) (str to) n total)) + (when (seq skipped) + (printf " !! %d downgrade(s) to :ccp-processor/na SKIPPED (pass :allow-downgrade? true to force)%n" + (count skipped)) + (doseq [{:keys [external-id order source note from]} (take 10 skipped)] + (printf " %s (order %s, source %s, note %s) was %s%n" + external-id order (pr-str source) (pr-str note) from))) + (when (pos? orphan-charges) + (printf " note: %d square charge(s) have no parent sales order (custom-amount tenders).%n" + orphan-charges) + (println " Their order source is not stored, so they cannot be reclassified") + (println " locally -- use the re-import fallback in the comment block."))) + (let [total (reduce + 0 (map (comp count :changes) plan))] + (println) + (printf "TOTAL: %d charge(s) across %d client(s)%n" total (count plan)) + (println) + total)) + +;; --------------------------------------------------------------------------- +;; Applying +;; --------------------------------------------------------------------------- + +(defn apply-plan! + "Transacts the processor corrections in batches. Returns the number written." + [plan] + (let [tx (for [{:keys [changes]} plan + {:keys [charge to]} changes] + {:db/id charge :charge/processor to})] + (doseq [batch (partition-all 100 tx)] + (alog/info ::updating-charges :count (count batch)) + @(dc/transact-async conn batch)) + (count tx))) + +(defn mark-summaries-dirty! + "Processor drives the Card / Food App / Unknown split in + `sales-summaries/get-payment-items`, so every day we touched has to be + recalculated. `periodic-seq` is end-exclusive, hence the +1 day." + [plan start end] + (doseq [{:keys [client code changes]} plan + :when (seq changes)] + (alog/info ::marking-summaries-dirty :client code) + (summaries/mark-dirty client + (local-midnight start) + (time/plus (local-midnight end) (time/days 1))))) + +;; --------------------------------------------------------------------------- +;; Entry point +;; --------------------------------------------------------------------------- + +(defn backfill! + "Recompute :charge/processor for imported Square charges. + + Options: + :start \"2026-07-07\" (default) or a DateTime -- inclusive + :end \"2026-07-29\" or a DateTime -- inclusive, default today + :codes seq of client codes; default every Square client + :apply? false (default) = dry run, print only + :allow-downgrade? false (default) = refuse changes that drop a known + processor back to :ccp-processor/na + :mark-dirty? true (default when applying) = flag sales summaries for + recalculation + + Returns the plan so you can inspect individual charges in the REPL." + [& {:keys [start end codes apply? allow-downgrade? mark-dirty?] + :or {start default-start-date + apply? false + allow-downgrade? false}}] + (let [start-dt (parse-day start) + end-dt (or (parse-day end) (atime/localize (time/now))) + start-inst (coerce/to-date (local-midnight start-dt)) + end-inst (coerce/to-date (local-midnight end-dt)) + db (dc/db conn) + clients (if (seq codes) + (apply square3/get-square-clients codes) + (square3/get-square-clients)) + _ (alog/info ::scanning + :start start-inst + :end end-inst + :clients (count clients) + :dry-run? (not apply?)) + plan (build-plan db clients start-inst end-inst allow-downgrade?) + total (print-report plan)] + (if-not apply? + (do (println "DRY RUN -- nothing written. Re-run with :apply? true to commit.") + (alog/info ::dry-run-complete :change-count total)) + (do + (alog/info ::applying :change-count total) + (apply-plan! plan) + (when (not= false mark-dirty?) + (mark-summaries-dirty! plan start-dt end-dt) + (println "Sales summaries marked dirty. Run auto-ap.jobs.sales-summaries") + (println "(or `(auto-ap.jobs.sales-summaries/sales-summaries-v2)`) to rebuild them.")) + (alog/info ::done :change-count total))) + plan)) + +(defn -main [& _] + (execute "backfill-olo-processors" + (fn [] + (let [{:keys [start end codes apply allow-downgrade mark-dirty]} (:args env)] + (backfill! :start (or start default-start-date) + :end end + :codes (cond-> codes (string? codes) + (str/split #",")) + :apply? (boolean apply) + :allow-downgrade? (boolean allow-downgrade) + :mark-dirty? (if (nil? mark-dirty) true (boolean mark-dirty))))))) + +(comment + ;; --------------------------------------------------------------------- + ;; 1. Dry run everything from 7/7 -- read only, prints what would change. + ;; --------------------------------------------------------------------- + (def plan (backfill!)) + + ;; One client at a time while you sanity check. + (backfill! :codes ["NGCL"]) + + ;; Eyeball the actual charges behind a transition. + (->> plan + (mapcat :changes) + (filter #(= :ccp-processor/doordash (:to %))) + (map (juxt :date :source :note :total :from :to)) + (take 20)) + + ;; Every distinct source string we saw reclassified -- confirms the Olo + ;; variants ("Olo - DoorDash", "olo-ubereats", ...) are what moved. + (->> plan (mapcat :changes) (map :source) frequencies) + + ;; --------------------------------------------------------------------- + ;; 2. Commit, then rebuild the summaries. + ;; --------------------------------------------------------------------- + (backfill! :apply? true) + (auto-ap.jobs.sales-summaries/sales-summaries-v2) + + ;; --------------------------------------------------------------------- + ;; 3. Verify: no square charge in the window disagrees with the classifier. + ;; --------------------------------------------------------------------- + (->> (backfill!) (mapcat :changes) count) ; => 0 + + ;; --------------------------------------------------------------------- + ;; Fallback: charges with no parent sales order (custom-amount tenders, + ;; see `is-order-only-for-charge?` in square3/order->sales-order) do not + ;; store the order source, so they cannot be fixed locally. Re-import them + ;; through the real pipeline instead -- `upsert` is idempotent on + ;; :charge/external-id / :sales-order/external-id, so re-running a day is + ;; safe and rewrites the processor from live Square data. + ;; + ;; This hits the Square API for every day x location; the existing + ;; auto-ap.jobs.load-historical-sales job does the same thing by day count. + ;; --------------------------------------------------------------------- + (doseq [client (square3/get-square-clients) + location (:client/square-locations client) + :when (:square-location/client-location location) + d (per/periodic-seq (parse-day "2026-07-07") + (time/plus (atime/localize (time/now)) (time/days 1)) + (time/days 1))] + (println (:client/code client) (:square-location/client-location location) (str d)) + @(square3/upsert client location d (time/plus d (time/days 1)))) + + ;; ...then mark dirty + rebuild summaries as in step 2. + ) diff --git a/src/clj/auto_ap/parse/templates.clj b/src/clj/auto_ap/parse/templates.clj index 1e9e315d..ab759252 100644 --- a/src/clj/auto_ap/parse/templates.clj +++ b/src/clj/auto_ap/parse/templates.clj @@ -742,6 +742,19 @@ :total [:trim-commas-and-negate nil]} :multi #"\n" :multi-match? #"^\d+"} + + ;; REEL PRODUCE STATEMENT (QuickBooks layout -- no "Reel Produce" text on the page) + {:vendor "Reel Produce" + :keywords [#"reelproduce\.com" #"Statement"] + :extract {:date #"^\s*([0-9]+/[0-9]+/[0-9]+)" + :customer-identifier #"To:(?:.*?)\n\s*(.*?)\s{2,}" + :invoice-number #"INV #(\d+)" + :total #"Orig\. Amount \$([\d\-,]+\.\d{2,2})"} + :parser {:date [:clj-time "MM/dd/yyyy"] + :total [:trim-commas-and-negate nil]} + :multi #"\n" + :multi-match? #"^\s*[0-9]+/[0-9]+/[0-9]+\s+INV #"} + {:vendor "Paulino's Bakery" :keywords [#"paulinosbakery"] :extract {:date #"\s*([0-9]+/[0-9]+/[0-9]+)" diff --git a/test/clj/auto_ap/parse/templates_test.clj b/test/clj/auto_ap/parse/templates_test.clj index fcdfe54b..4183f766 100644 --- a/test/clj/auto_ap/parse/templates_test.clj +++ b/test/clj/auto_ap/parse/templates_test.clj @@ -70,3 +70,24 @@ (is (= "NICK THE GREEK" (:customer-identifier result))) (is (= "600 VISTA WAY" (str/trim (:account-number result)))) (is (= "946.24" (:total result))))))) + +(deftest parse-reel-produce-statement-28676 + (testing "Should parse the Reel Produce statement layout that no longer prints 'Reel Produce' on the page" + (let [pdf-file (io/file "dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf") + pdf-text (:out (clojure.java.shell/sh "pdftotext" "-layout" (str pdf-file) "-")) + results (sut/parse pdf-text)] + (is (seq results) "Template should match and return results") + (is (= 7 (count results)) "Should parse 7 invoices from statement") + (doseq [result results] + (is (= "Reel Produce" (:vendor-code result))) + (is (= "Sushi Confidential - San Jose" (:customer-identifier result)))) + (is (= ["454379" "454826" "455120" "455683" "456654" "456774" "457171"] + (mapv :invoice-number results))) + (is (= ["1003.10" "530.85" "605.00" "1187.40" "164.00" "675.60" "265.75"] + (mapv :total results))) + ;; totals add up to the statement's $4,431.70 amount due + (is (= 4431.70 (->> results (map #(Double/parseDouble (:total %))) (reduce +)))) + (let [d (:date (first results))] + (is (= 2026 (time/year d))) + (is (= 6 (time/month d))) + (is (= 23 (time/day d)))))))