Merge branch 'master' into staging

config/prod-background-worker.edn

@@ -173,4 +173,4 @@
                    :token "EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"}}
  :plaid {:base-url   "https://production.plaid.com"
          :client-id  "61bfab05f7e762001b323f79"
-         :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}}
+         :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}}

config/prod-cloud-background-worker.edn

@@ -31,5 +31,5 @@
  :yodlee2-proxy-port 8888
  :plaid {:base-url   "https://production.plaid.com"
          :client-id  "61bfab05f7e762001b323f79"
-         :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}
+         :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}
  }

config/prod-cloud.edn

@@ -34,5 +34,5 @@
  :yodlee2-proxy-port 8888
  :plaid {:base-url   "https://production.plaid.com"
          :client-id  "61bfab05f7e762001b323f79"
-         :secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}
+         :secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}
  }

hours.txt

@@ -1 +1 @@
-
+1`

src/clj/auto_ap/jobs/backfill_olo_processors.clj

@@ -0,0 +1,375 @@
+(ns auto-ap.jobs.backfill-olo-processors
+  "One-off backfill for the Olo third-party-source fix (commit fa25620b, \"olo fixes\").
+
+   That commit replaced the exact-match `condp` on `(:name (:source order))` in
+   `square3/tender->charge` with substring matching, so Olo-brokered orders that
+   arrive from Square with source names like \"Olo - DoorDash\" or
+   \"olo-ubereats\" now classify as the real delivery processor instead of
+   falling through to `:ccp-processor/na`.
+
+   Orders imported *before* the fix shipped still carry the old
+   `:charge/processor`, which means `sales-summaries` bucketed them as
+   \"Unknown\" instead of \"Food App Payments\". This job recomputes
+   `:charge/processor` for already-imported Square charges over a date range
+   (default: 2026-07-07 -> today) and marks the affected sales summaries dirty
+   so `auto-ap.jobs.sales-summaries` recalculates them.
+
+   IMPORTANT: this backfill never re-implements the classifier. It feeds the
+   stored shape of each charge back through the real production function
+   (`square3/tender->charge`), so it cannot drift from the importer.
+
+   No Square API calls are made -- every input the classifier needs
+   (`:sales-order/source`, `:charge/note`, `:charge/type-name`) is already in
+   Datomic. See the `(comment ...)` block at the bottom for the API-based
+   fallback if you need to repair charges that have no parent sales order.
+
+   Dry run by default. Pass `:apply? true` to write."
+  (:require
+   [auto-ap.datomic :refer [conn]]
+   [auto-ap.jobs.core :refer [execute]]
+   [auto-ap.jobs.sales-summaries :as summaries]
+   [auto-ap.logging :as alog]
+   [auto-ap.square.core3 :as square3]
+   [auto-ap.time :as atime]
+   [clj-time.coerce :as coerce]
+   [clj-time.core :as time]
+   [clj-time.format :as f]
+   [clj-time.periodic :as per]
+   [clojure.string :as str]
+   [config.core :refer [env]]
+   [datomic.api :as dc]
+   ;; the datalog below calls iol-ion.query/scan-charges by fully-qualified
+   ;; symbol, so the namespace has to be loaded.
+   [iol-ion.query]))
+
+;; ---------------------------------------------------------------------------
+;; Date handling
+;; ---------------------------------------------------------------------------
+
+(def pacific (time/time-zone-for-id "America/Los_Angeles"))
+
+(def default-start-date
+  "The Olo fix landed 2026-07-22; the user-visible bad data starts 7/7."
+  "2026-07-07")
+
+(defn parse-day
+  "\"2026-07-07\" -> local (Pacific) midnight DateTime. Passes DateTimes through."
+  [d]
+  (cond
+    (nil? d)     nil
+    (string? d)  (f/parse (f/with-zone (f/formatter "yyyy-MM-dd") pacific) d)
+    :else        (coerce/to-date-time d)))
+
+(defn local-midnight [dt]
+  (.toDateMidnight (atime/localize dt)))
+
+;; ---------------------------------------------------------------------------
+;; Classification -- delegates to the production importer
+;; ---------------------------------------------------------------------------
+
+(defn recompute-processor
+  "Runs the *live* classifier over a charge's stored inputs.
+
+   `tender->charge` only reads `(:name (:source order))` from the order and
+   `:type`/`:note` from the tender when deciding `:charge/processor`, so a
+   synthetic order/tender carrying the stored values yields exactly what a
+   fresh import would produce today. `:created_at` and `:id` are placeholders
+   purely to keep the unrelated `:charge/date` and `:charge/reference-link`
+   branches from blowing up; we only read `:charge/processor` back out.
+
+   Note on nil sources: the importer stores `(or (:name (:source order))
+   \"Square\")`, so a source that was originally nil comes back as \"Square\".
+   Both nil and \"Square\" classify to `:ccp-processor/na`, so the round trip
+   is faithful."
+  [{:keys [source note type-name]}]
+  (:charge/processor
+   (square3/tender->charge {:source     {:name source}
+                            :created_at "1970-01-01T00:00:00Z"}
+                           {}                       ; client -> only :charge/client
+                           {}                       ; location -> only :charge/location
+                           {:id   "backfill"
+                            :type type-name
+                            :note note})))
+
+;; ---------------------------------------------------------------------------
+;; Scanning
+;; ---------------------------------------------------------------------------
+
+(def charge-pull
+  [:db/id
+   :charge/external-id
+   :charge/type-name
+   :charge/note
+   :charge/date
+   :charge/total
+   {:charge/processor [:db/ident]}
+   {:sales-order/_charges [:db/id
+                           :sales-order/external-id
+                           :sales-order/source
+                           {:sales-order/vendor [:db/ident]}]}])
+
+(defn client-charges
+  "All charges for `client-eid` between `start-inst` and `end-inst` (both
+   inclusive by day), via the :charge/client+date index."
+  [db client-eid start-inst end-inst]
+  (->> (dc/q '[:find (pull ?c pull-pattern)
+               :in $ pull-pattern [?clients ?start ?end]
+               :where
+               [(iol-ion.query/scan-charges $ ?clients ?start ?end) [[?c _ _] ...]]]
+             db
+             charge-pull
+             [[client-eid] start-inst end-inst])
+       (map first)))
+
+(defn parent-order
+  "Reverse refs on component attributes come back as a single map from pull,
+   but tolerate a collection in case that ever changes."
+  [charge]
+  (let [o (:sales-order/_charges charge)]
+    (if (sequential? o) (first o) o)))
+
+(defn square-charge? [charge]
+  (= :vendor/ccp-square (get-in (parent-order charge) [:sales-order/vendor :db/ident])))
+
+;; ---------------------------------------------------------------------------
+;; Planning
+;; ---------------------------------------------------------------------------
+
+(defn charge->change
+  "nil when the charge is already correct (or isn't ours to touch)."
+  [charge]
+  (when (square-charge? charge)
+    (let [order   (parent-order charge)
+          current (get-in charge [:charge/processor :db/ident])
+          next    (recompute-processor {:source    (:sales-order/source order)
+                                        :note      (:charge/note charge)
+                                        :type-name (:charge/type-name charge)})]
+      (when (not= current next)
+        {:charge      (:db/id charge)
+         :external-id (:charge/external-id charge)
+         :order       (:sales-order/external-id order)
+         :date        (:charge/date charge)
+         :source      (:sales-order/source order)
+         :note        (:charge/note charge)
+         :type-name   (:charge/type-name charge)
+         :total       (:charge/total charge)
+         :from        current
+         :to          next}))))
+
+(defn downgrade?
+  "A change that *loses* processor information. The Olo fix can only ever widen
+   matching, so these should not exist -- if they do, something else changed
+   and we'd rather report than clobber."
+  [{:keys [from to]}]
+  (and (= :ccp-processor/na to)
+       (not= :ccp-processor/na from)
+       (some? from)))
+
+(defn plan-for-client
+  [db {:keys [db/id client/code]} start-inst end-inst allow-downgrade?]
+  (let [charges  (client-charges db id start-inst end-inst)
+        square   (filter square-charge? charges)
+        orphans  (->> charges
+                      (remove (comp some? parent-order))
+                      (filter #(some-> (:charge/external-id %)
+                                       (str/starts-with? "square/charge/"))))
+        all      (keep charge->change square)
+        [skipped changes] (if allow-downgrade?
+                            [[] all]
+                            [(filter downgrade? all) (remove downgrade? all)])]
+    {:client         id
+     :code           code
+     :scanned        (count square)
+     :orphan-charges (count orphans)
+     :changes        (vec changes)
+     :skipped        (vec skipped)}))
+
+(defn build-plan
+  [db clients start-inst end-inst allow-downgrade?]
+  (->> clients
+       (map #(plan-for-client db % start-inst end-inst allow-downgrade?))
+       (remove #(and (zero? (count (:changes %)))
+                     (zero? (count (:skipped %)))
+                     (zero? (:orphan-charges %))))
+       vec))
+
+;; ---------------------------------------------------------------------------
+;; Reporting
+;; ---------------------------------------------------------------------------
+
+(defn- transitions [changes]
+  (->> changes
+       (group-by (juxt :from :to))
+       (map (fn [[[from to] cs]]
+              [from to (count cs) (reduce + 0.0 (keep :total cs))]))
+       (sort-by #(- (nth % 2)))))
+
+(defn print-report [plan]
+  (println)
+  (println "=== Olo processor backfill ===")
+  (doseq [{:keys [code scanned changes skipped orphan-charges]} plan]
+    (println)
+    (printf "%-8s scanned %d square charges, %d to update%n"
+            (str code) scanned (count changes))
+    (doseq [[from to n total] (transitions changes)]
+      (printf "         %-24s -> %-24s %5d  $%.2f%n"
+              (str from) (str to) n total))
+    (when (seq skipped)
+      (printf "         !! %d downgrade(s) to :ccp-processor/na SKIPPED (pass :allow-downgrade? true to force)%n"
+              (count skipped))
+      (doseq [{:keys [external-id order source note from]} (take 10 skipped)]
+        (printf "            %s (order %s, source %s, note %s) was %s%n"
+                external-id order (pr-str source) (pr-str note) from)))
+    (when (pos? orphan-charges)
+      (printf "         note: %d square charge(s) have no parent sales order (custom-amount tenders).%n"
+              orphan-charges)
+      (println  "               Their order source is not stored, so they cannot be reclassified")
+      (println  "               locally -- use the re-import fallback in the comment block.")))
+  (let [total (reduce + 0 (map (comp count :changes) plan))]
+    (println)
+    (printf "TOTAL: %d charge(s) across %d client(s)%n" total (count plan))
+    (println)
+    total))
+
+;; ---------------------------------------------------------------------------
+;; Applying
+;; ---------------------------------------------------------------------------
+
+(defn apply-plan!
+  "Transacts the processor corrections in batches. Returns the number written."
+  [plan]
+  (let [tx (for [{:keys [changes]} plan
+                 {:keys [charge to]} changes]
+             {:db/id charge :charge/processor to})]
+    (doseq [batch (partition-all 100 tx)]
+      (alog/info ::updating-charges :count (count batch))
+      @(dc/transact-async conn batch))
+    (count tx)))
+
+(defn mark-summaries-dirty!
+  "Processor drives the Card / Food App / Unknown split in
+   `sales-summaries/get-payment-items`, so every day we touched has to be
+   recalculated. `periodic-seq` is end-exclusive, hence the +1 day."
+  [plan start end]
+  (doseq [{:keys [client code changes]} plan
+          :when (seq changes)]
+    (alog/info ::marking-summaries-dirty :client code)
+    (summaries/mark-dirty client
+                          (local-midnight start)
+                          (time/plus (local-midnight end) (time/days 1)))))
+
+;; ---------------------------------------------------------------------------
+;; Entry point
+;; ---------------------------------------------------------------------------
+
+(defn backfill!
+  "Recompute :charge/processor for imported Square charges.
+
+   Options:
+     :start            \"2026-07-07\" (default) or a DateTime -- inclusive
+     :end              \"2026-07-29\" or a DateTime -- inclusive, default today
+     :codes            seq of client codes; default every Square client
+     :apply?           false (default) = dry run, print only
+     :allow-downgrade? false (default) = refuse changes that drop a known
+                       processor back to :ccp-processor/na
+     :mark-dirty?      true (default when applying) = flag sales summaries for
+                       recalculation
+
+   Returns the plan so you can inspect individual charges in the REPL."
+  [& {:keys [start end codes apply? allow-downgrade? mark-dirty?]
+      :or   {start            default-start-date
+             apply?           false
+             allow-downgrade? false}}]
+  (let [start-dt   (parse-day start)
+        end-dt     (or (parse-day end) (atime/localize (time/now)))
+        start-inst (coerce/to-date (local-midnight start-dt))
+        end-inst   (coerce/to-date (local-midnight end-dt))
+        db         (dc/db conn)
+        clients    (if (seq codes)
+                     (apply square3/get-square-clients codes)
+                     (square3/get-square-clients))
+        _          (alog/info ::scanning
+                              :start start-inst
+                              :end end-inst
+                              :clients (count clients)
+                              :dry-run? (not apply?))
+        plan       (build-plan db clients start-inst end-inst allow-downgrade?)
+        total      (print-report plan)]
+    (if-not apply?
+      (do (println "DRY RUN -- nothing written. Re-run with :apply? true to commit.")
+          (alog/info ::dry-run-complete :change-count total))
+      (do
+        (alog/info ::applying :change-count total)
+        (apply-plan! plan)
+        (when (not= false mark-dirty?)
+          (mark-summaries-dirty! plan start-dt end-dt)
+          (println "Sales summaries marked dirty. Run auto-ap.jobs.sales-summaries")
+          (println "(or `(auto-ap.jobs.sales-summaries/sales-summaries-v2)`) to rebuild them."))
+        (alog/info ::done :change-count total)))
+    plan))
+
+(defn -main [& _]
+  (execute "backfill-olo-processors"
+           (fn []
+             (let [{:keys [start end codes apply allow-downgrade mark-dirty]} (:args env)]
+               (backfill! :start            (or start default-start-date)
+                          :end              end
+                          :codes            (cond-> codes (string? codes)
+                                                    (str/split #","))
+                          :apply?           (boolean apply)
+                          :allow-downgrade? (boolean allow-downgrade)
+                          :mark-dirty?      (if (nil? mark-dirty) true (boolean mark-dirty)))))))
+
+(comment
+  ;; ---------------------------------------------------------------------
+  ;; 1. Dry run everything from 7/7 -- read only, prints what would change.
+  ;; ---------------------------------------------------------------------
+  (def plan (backfill!))
+
+  ;; One client at a time while you sanity check.
+  (backfill! :codes ["NGCL"])
+
+  ;; Eyeball the actual charges behind a transition.
+  (->> plan
+       (mapcat :changes)
+       (filter #(= :ccp-processor/doordash (:to %)))
+       (map (juxt :date :source :note :total :from :to))
+       (take 20))
+
+  ;; Every distinct source string we saw reclassified -- confirms the Olo
+  ;; variants ("Olo - DoorDash", "olo-ubereats", ...) are what moved.
+  (->> plan (mapcat :changes) (map :source) frequencies)
+
+  ;; ---------------------------------------------------------------------
+  ;; 2. Commit, then rebuild the summaries.
+  ;; ---------------------------------------------------------------------
+  (backfill! :apply? true)
+  (auto-ap.jobs.sales-summaries/sales-summaries-v2)
+
+  ;; ---------------------------------------------------------------------
+  ;; 3. Verify: no square charge in the window disagrees with the classifier.
+  ;; ---------------------------------------------------------------------
+  (->> (backfill!) (mapcat :changes) count)   ; => 0
+
+  ;; ---------------------------------------------------------------------
+  ;; Fallback: charges with no parent sales order (custom-amount tenders,
+  ;; see `is-order-only-for-charge?` in square3/order->sales-order) do not
+  ;; store the order source, so they cannot be fixed locally. Re-import them
+  ;; through the real pipeline instead -- `upsert` is idempotent on
+  ;; :charge/external-id / :sales-order/external-id, so re-running a day is
+  ;; safe and rewrites the processor from live Square data.
+  ;;
+  ;; This hits the Square API for every day x location; the existing
+  ;; auto-ap.jobs.load-historical-sales job does the same thing by day count.
+  ;; ---------------------------------------------------------------------
+  (doseq [client (square3/get-square-clients)
+          location (:client/square-locations client)
+          :when (:square-location/client-location location)
+          d (per/periodic-seq (parse-day "2026-07-07")
+                              (time/plus (atime/localize (time/now)) (time/days 1))
+                              (time/days 1))]
+    (println (:client/code client) (:square-location/client-location location) (str d))
+    @(square3/upsert client location d (time/plus d (time/days 1))))
+
+  ;; ...then mark dirty + rebuild summaries as in step 2.
+  )

src/clj/auto_ap/parse/templates.clj

@@ -742,6 +742,19 @@
                :total [:trim-commas-and-negate nil]}
     :multi #"\n"
     :multi-match? #"^\d+"}
+
+   ;; REEL PRODUCE STATEMENT (QuickBooks layout -- no "Reel Produce" text on the page)
+   {:vendor   "Reel Produce"
+    :keywords [#"reelproduce\.com" #"Statement"]
+    :extract  {:date                #"^\s*([0-9]+/[0-9]+/[0-9]+)"
+               :customer-identifier #"To:(?:.*?)\n\s*(.*?)\s{2,}"
+               :invoice-number      #"INV #(\d+)"
+               :total               #"Orig\. Amount \$([\d\-,]+\.\d{2,2})"}
+    :parser   {:date  [:clj-time "MM/dd/yyyy"]
+               :total [:trim-commas-and-negate nil]}
+    :multi #"\n"
+    :multi-match? #"^\s*[0-9]+/[0-9]+/[0-9]+\s+INV #"}
+
    {:vendor   "Paulino's Bakery"
     :keywords [#"paulinosbakery"]
     :extract  {:date                #"\s*([0-9]+/[0-9]+/[0-9]+)"

test/clj/auto_ap/parse/templates_test.clj

@@ -70,3 +70,24 @@
         (is (= "NICK THE GREEK" (:customer-identifier result)))
         (is (= "600 VISTA WAY" (str/trim (:account-number result))))
         (is (= "946.24" (:total result)))))))
+
+(deftest parse-reel-produce-statement-28676
+  (testing "Should parse the Reel Produce statement layout that no longer prints 'Reel Produce' on the page"
+    (let [pdf-file (io/file "dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf")
+          pdf-text (:out (clojure.java.shell/sh "pdftotext" "-layout" (str pdf-file) "-"))
+          results (sut/parse pdf-text)]
+      (is (seq results) "Template should match and return results")
+      (is (= 7 (count results)) "Should parse 7 invoices from statement")
+      (doseq [result results]
+        (is (= "Reel Produce" (:vendor-code result)))
+        (is (= "Sushi Confidential - San Jose" (:customer-identifier result))))
+      (is (= ["454379" "454826" "455120" "455683" "456654" "456774" "457171"]
+             (mapv :invoice-number results)))
+      (is (= ["1003.10" "530.85" "605.00" "1187.40" "164.00" "675.60" "265.75"]
+             (mapv :total results)))
+      ;; totals add up to the statement's $4,431.70 amount due
+      (is (= 4431.70 (->> results (map #(Double/parseDouble (:total %))) (reduce +))))
+      (let [d (:date (first results))]
+        (is (= 2026 (time/year d)))
+        (is (= 6 (time/month d)))
+        (is (= 23 (time/day d)))))))