notid/integreat
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into staging

Browse Source
This commit is contained in:
Bryce
2026-07-29 10:25:04 -07:00
parent 9d007ee8e2 2b5fbaca00
commit 7155e199e8
8 changed files with 413 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/prod-background-worker.edn
View File

@@ -173,4 +173,4 @@
:token "EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7"}}
:plaid {:base-url "https://production.plaid.com"
:client-id "61bfab05f7e762001b323f79"
:secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}}
:secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}}

2
config/prod-cloud-background-worker.edn
View File

@@ -31,5 +31,5 @@
:yodlee2-proxy-port 8888
:plaid {:base-url "https://production.plaid.com"
:client-id "61bfab05f7e762001b323f79"
:secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}
:secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}
}

2
config/prod-cloud.edn
View File

@@ -34,5 +34,5 @@
:yodlee2-proxy-port 8888
:plaid {:base-url "https://production.plaid.com"
:client-id "61bfab05f7e762001b323f79"
:secret-key "2be026ca5e7f7e9f23f2fb4d7c914d"}
:secret-key "44a05fbe9f33a2975b3b3ac06b0b62"}
}

BIN
dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf Normal file
View File

Binary file not shown.

2
hours.txt
View File

@@ -1 +1 @@
1`

375
src/clj/auto_ap/jobs/backfill_olo_processors.clj Normal file
View File

@@ -0,0 +1,375 @@
(ns auto-ap.jobs.backfill-olo-processors
"One-off backfill for the Olo third-party-source fix (commit fa25620b, \"olo fixes\").
That commit replaced the exact-match `condp` on `(:name (:source order))` in
`square3/tender->charge` with substring matching, so Olo-brokered orders that
arrive from Square with source names like \"Olo - DoorDash\" or
\"olo-ubereats\" now classify as the real delivery processor instead of
falling through to `:ccp-processor/na`.
Orders imported *before* the fix shipped still carry the old
`:charge/processor`, which means `sales-summaries` bucketed them as
\"Unknown\" instead of \"Food App Payments\". This job recomputes
`:charge/processor` for already-imported Square charges over a date range
(default: 2026-07-07 -> today) and marks the affected sales summaries dirty
so `auto-ap.jobs.sales-summaries` recalculates them.
IMPORTANT: this backfill never re-implements the classifier. It feeds the
stored shape of each charge back through the real production function
(`square3/tender->charge`), so it cannot drift from the importer.
No Square API calls are made -- every input the classifier needs
(`:sales-order/source`, `:charge/note`, `:charge/type-name`) is already in
Datomic. See the `(comment ...)` block at the bottom for the API-based
fallback if you need to repair charges that have no parent sales order.
Dry run by default. Pass `:apply? true` to write."
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.jobs.sales-summaries :as summaries]
[auto-ap.logging :as alog]
[auto-ap.square.core3 :as square3]
[auto-ap.time :as atime]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clj-time.format :as f]
[clj-time.periodic :as per]
[clojure.string :as str]
[config.core :refer [env]]
[datomic.api :as dc]
;; the datalog below calls iol-ion.query/scan-charges by fully-qualified
;; symbol, so the namespace has to be loaded.
[iol-ion.query]))
;; ---------------------------------------------------------------------------
;; Date handling
;; ---------------------------------------------------------------------------
(def pacific (time/time-zone-for-id "America/Los_Angeles"))
(def default-start-date
"The Olo fix landed 2026-07-22; the user-visible bad data starts 7/7."
"2026-07-07")
(defn parse-day
"\"2026-07-07\" -> local (Pacific) midnight DateTime. Passes DateTimes through."
[d]
(cond
(nil? d) nil
(string? d) (f/parse (f/with-zone (f/formatter "yyyy-MM-dd") pacific) d)
:else (coerce/to-date-time d)))
(defn local-midnight [dt]
(.toDateMidnight (atime/localize dt)))
;; ---------------------------------------------------------------------------
;; Classification -- delegates to the production importer
;; ---------------------------------------------------------------------------
(defn recompute-processor
"Runs the *live* classifier over a charge's stored inputs.
`tender->charge` only reads `(:name (:source order))` from the order and
`:type`/`:note` from the tender when deciding `:charge/processor`, so a
synthetic order/tender carrying the stored values yields exactly what a
fresh import would produce today. `:created_at` and `:id` are placeholders
purely to keep the unrelated `:charge/date` and `:charge/reference-link`
branches from blowing up; we only read `:charge/processor` back out.
Note on nil sources: the importer stores `(or (:name (:source order))
\"Square\")`, so a source that was originally nil comes back as \"Square\".
Both nil and \"Square\" classify to `:ccp-processor/na`, so the round trip
is faithful."
[{:keys [source note type-name]}]
(:charge/processor
(square3/tender->charge {:source {:name source}
:created_at "1970-01-01T00:00:00Z"}
{} ; client -> only :charge/client
{} ; location -> only :charge/location
{:id "backfill"
:type type-name
:note note})))
;; ---------------------------------------------------------------------------
;; Scanning
;; ---------------------------------------------------------------------------
(def charge-pull
[:db/id
:charge/external-id
:charge/type-name
:charge/note
:charge/date
:charge/total
{:charge/processor [:db/ident]}
{:sales-order/_charges [:db/id
:sales-order/external-id
:sales-order/source
{:sales-order/vendor [:db/ident]}]}])
(defn client-charges
"All charges for `client-eid` between `start-inst` and `end-inst` (both
inclusive by day), via the :charge/client+date index."
[db client-eid start-inst end-inst]
(->> (dc/q '[:find (pull ?c pull-pattern)
:in $ pull-pattern [?clients ?start ?end]
:where
[(iol-ion.query/scan-charges $ ?clients ?start ?end) [[?c _ _] ...]]]
db
charge-pull
[[client-eid] start-inst end-inst])
(map first)))
(defn parent-order
"Reverse refs on component attributes come back as a single map from pull,
but tolerate a collection in case that ever changes."
[charge]
(let [o (:sales-order/_charges charge)]
(if (sequential? o) (first o) o)))
(defn square-charge? [charge]
(= :vendor/ccp-square (get-in (parent-order charge) [:sales-order/vendor :db/ident])))
;; ---------------------------------------------------------------------------
;; Planning
;; ---------------------------------------------------------------------------
(defn charge->change
"nil when the charge is already correct (or isn't ours to touch)."
[charge]
(when (square-charge? charge)
(let [order (parent-order charge)
current (get-in charge [:charge/processor :db/ident])
next (recompute-processor {:source (:sales-order/source order)
:note (:charge/note charge)
:type-name (:charge/type-name charge)})]
(when (not= current next)
{:charge (:db/id charge)
:external-id (:charge/external-id charge)
:order (:sales-order/external-id order)
:date (:charge/date charge)
:source (:sales-order/source order)
:note (:charge/note charge)
:type-name (:charge/type-name charge)
:total (:charge/total charge)
:from current
:to next}))))
(defn downgrade?
"A change that *loses* processor information. The Olo fix can only ever widen
matching, so these should not exist -- if they do, something else changed
and we'd rather report than clobber."
[{:keys [from to]}]
(and (= :ccp-processor/na to)
(not= :ccp-processor/na from)
(some? from)))
(defn plan-for-client
[db {:keys [db/id client/code]} start-inst end-inst allow-downgrade?]
(let [charges (client-charges db id start-inst end-inst)
square (filter square-charge? charges)
orphans (->> charges
(remove (comp some? parent-order))
(filter #(some-> (:charge/external-id %)
(str/starts-with? "square/charge/"))))
all (keep charge->change square)
[skipped changes] (if allow-downgrade?
[[] all]
[(filter downgrade? all) (remove downgrade? all)])]
{:client id
:code code
:scanned (count square)
:orphan-charges (count orphans)
:changes (vec changes)
:skipped (vec skipped)}))
(defn build-plan
[db clients start-inst end-inst allow-downgrade?]
(->> clients
(map #(plan-for-client db % start-inst end-inst allow-downgrade?))
(remove #(and (zero? (count (:changes %)))
(zero? (count (:skipped %)))
(zero? (:orphan-charges %))))
vec))
;; ---------------------------------------------------------------------------
;; Reporting
;; ---------------------------------------------------------------------------
(defn- transitions [changes]
(->> changes
(group-by (juxt :from :to))
(map (fn [[[from to] cs]]
[from to (count cs) (reduce + 0.0 (keep :total cs))]))
(sort-by #(- (nth % 2)))))
(defn print-report [plan]
(println)
(println "=== Olo processor backfill ===")
(doseq [{:keys [code scanned changes skipped orphan-charges]} plan]
(println)
(printf "%-8s scanned %d square charges, %d to update%n"
(str code) scanned (count changes))
(doseq [[from to n total] (transitions changes)]
(printf " %-24s -> %-24s %5d $%.2f%n"
(str from) (str to) n total))
(when (seq skipped)
(printf " !! %d downgrade(s) to :ccp-processor/na SKIPPED (pass :allow-downgrade? true to force)%n"
(count skipped))
(doseq [{:keys [external-id order source note from]} (take 10 skipped)]
(printf " %s (order %s, source %s, note %s) was %s%n"
external-id order (pr-str source) (pr-str note) from)))
(when (pos? orphan-charges)
(printf " note: %d square charge(s) have no parent sales order (custom-amount tenders).%n"
orphan-charges)
(println " Their order source is not stored, so they cannot be reclassified")
(println " locally -- use the re-import fallback in the comment block.")))
(let [total (reduce + 0 (map (comp count :changes) plan))]
(println)
(printf "TOTAL: %d charge(s) across %d client(s)%n" total (count plan))
(println)
total))
;; ---------------------------------------------------------------------------
;; Applying
;; ---------------------------------------------------------------------------
(defn apply-plan!
"Transacts the processor corrections in batches. Returns the number written."
[plan]
(let [tx (for [{:keys [changes]} plan
{:keys [charge to]} changes]
{:db/id charge :charge/processor to})]
(doseq [batch (partition-all 100 tx)]
(alog/info ::updating-charges :count (count batch))
@(dc/transact-async conn batch))
(count tx)))
(defn mark-summaries-dirty!
"Processor drives the Card / Food App / Unknown split in
`sales-summaries/get-payment-items`, so every day we touched has to be
recalculated. `periodic-seq` is end-exclusive, hence the +1 day."
[plan start end]
(doseq [{:keys [client code changes]} plan
:when (seq changes)]
(alog/info ::marking-summaries-dirty :client code)
(summaries/mark-dirty client
(local-midnight start)
(time/plus (local-midnight end) (time/days 1)))))
;; ---------------------------------------------------------------------------
;; Entry point
;; ---------------------------------------------------------------------------
(defn backfill!
"Recompute :charge/processor for imported Square charges.
Options:
:start \"2026-07-07\" (default) or a DateTime -- inclusive
:end \"2026-07-29\" or a DateTime -- inclusive, default today
:codes seq of client codes; default every Square client
:apply? false (default) = dry run, print only
:allow-downgrade? false (default) = refuse changes that drop a known
processor back to :ccp-processor/na
:mark-dirty? true (default when applying) = flag sales summaries for
recalculation
Returns the plan so you can inspect individual charges in the REPL."
[& {:keys [start end codes apply? allow-downgrade? mark-dirty?]
:or {start default-start-date
apply? false
allow-downgrade? false}}]
(let [start-dt (parse-day start)
end-dt (or (parse-day end) (atime/localize (time/now)))
start-inst (coerce/to-date (local-midnight start-dt))
end-inst (coerce/to-date (local-midnight end-dt))
db (dc/db conn)
clients (if (seq codes)
(apply square3/get-square-clients codes)
(square3/get-square-clients))
_ (alog/info ::scanning
:start start-inst
:end end-inst
:clients (count clients)
:dry-run? (not apply?))
plan (build-plan db clients start-inst end-inst allow-downgrade?)
total (print-report plan)]
(if-not apply?
(do (println "DRY RUN -- nothing written. Re-run with :apply? true to commit.")
(alog/info ::dry-run-complete :change-count total))
(do
(alog/info ::applying :change-count total)
(apply-plan! plan)
(when (not= false mark-dirty?)
(mark-summaries-dirty! plan start-dt end-dt)
(println "Sales summaries marked dirty. Run auto-ap.jobs.sales-summaries")
(println "(or `(auto-ap.jobs.sales-summaries/sales-summaries-v2)`) to rebuild them."))
(alog/info ::done :change-count total)))
plan))
(defn -main [& _]
(execute "backfill-olo-processors"
(fn []
(let [{:keys [start end codes apply allow-downgrade mark-dirty]} (:args env)]
(backfill! :start (or start default-start-date)
:end end
:codes (cond-> codes (string? codes)
(str/split #","))
:apply? (boolean apply)
:allow-downgrade? (boolean allow-downgrade)
:mark-dirty? (if (nil? mark-dirty) true (boolean mark-dirty)))))))
(comment
;; ---------------------------------------------------------------------
;; 1. Dry run everything from 7/7 -- read only, prints what would change.
;; ---------------------------------------------------------------------
(def plan (backfill!))
;; One client at a time while you sanity check.
(backfill! :codes ["NGCL"])
;; Eyeball the actual charges behind a transition.
(->> plan
(mapcat :changes)
(filter #(= :ccp-processor/doordash (:to %)))
(map (juxt :date :source :note :total :from :to))
(take 20))
;; Every distinct source string we saw reclassified -- confirms the Olo
;; variants ("Olo - DoorDash", "olo-ubereats", ...) are what moved.
(->> plan (mapcat :changes) (map :source) frequencies)
;; ---------------------------------------------------------------------
;; 2. Commit, then rebuild the summaries.
;; ---------------------------------------------------------------------
(backfill! :apply? true)
(auto-ap.jobs.sales-summaries/sales-summaries-v2)
;; ---------------------------------------------------------------------
;; 3. Verify: no square charge in the window disagrees with the classifier.
;; ---------------------------------------------------------------------
(->> (backfill!) (mapcat :changes) count) ; => 0
;; ---------------------------------------------------------------------
;; Fallback: charges with no parent sales order (custom-amount tenders,
;; see `is-order-only-for-charge?` in square3/order->sales-order) do not
;; store the order source, so they cannot be fixed locally. Re-import them
;; through the real pipeline instead -- `upsert` is idempotent on
;; :charge/external-id / :sales-order/external-id, so re-running a day is
;; safe and rewrites the processor from live Square data.
;;
;; This hits the Square API for every day x location; the existing
;; auto-ap.jobs.load-historical-sales job does the same thing by day count.
;; ---------------------------------------------------------------------
(doseq [client (square3/get-square-clients)
location (:client/square-locations client)
:when (:square-location/client-location location)
d (per/periodic-seq (parse-day "2026-07-07")
(time/plus (atime/localize (time/now)) (time/days 1))
(time/days 1))]
(println (:client/code client) (:square-location/client-location location) (str d))
@(square3/upsert client location d (time/plus d (time/days 1))))
;; ...then mark dirty + rebuild summaries as in step 2.
)

13
src/clj/auto_ap/parse/templates.clj
View File

@@ -742,6 +742,19 @@
:total [:trim-commas-and-negate nil]}
:multi #"\n"
:multi-match? #"^\d+"}
;; REEL PRODUCE STATEMENT (QuickBooks layout -- no "Reel Produce" text on the page)
{:vendor "Reel Produce"
:keywords [#"reelproduce\.com" #"Statement"]
:extract {:date #"^\s*([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"To:(?:.*?)\n\s*(.*?)\s{2,}"
:invoice-number #"INV #(\d+)"
:total #"Orig\. Amount \$([\d\-,]+\.\d{2,2})"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}
:multi #"\n"
:multi-match? #"^\s*[0-9]+/[0-9]+/[0-9]+\s+INV #"}
{:vendor "Paulino's Bakery"
:keywords [#"paulinosbakery"]
:extract {:date #"\s*([0-9]+/[0-9]+/[0-9]+)"

21
test/clj/auto_ap/parse/templates_test.clj
View File

@@ -70,3 +70,24 @@
(is (= "NICK THE GREEK" (:customer-identifier result)))
(is (= "600 VISTA WAY" (str/trim (:account-number result))))
(is (= "946.24" (:total result)))))))
(deftest parse-reel-produce-statement-28676
(testing "Should parse the Reel Produce statement layout that no longer prints 'Reel Produce' on the page"
(let [pdf-file (io/file "dev-resources/Statement1_from_REEL_Produce_Inc.28676.pdf")
pdf-text (:out (clojure.java.shell/sh "pdftotext" "-layout" (str pdf-file) "-"))
results (sut/parse pdf-text)]
(is (seq results) "Template should match and return results")
(is (= 7 (count results)) "Should parse 7 invoices from statement")
(doseq [result results]
(is (= "Reel Produce" (:vendor-code result)))
(is (= "Sushi Confidential - San Jose" (:customer-identifier result))))
(is (= ["454379" "454826" "455120" "455683" "456654" "456774" "457171"]
(mapv :invoice-number results)))
(is (= ["1003.10" "530.85" "605.00" "1187.40" "164.00" "675.60" "265.75"]
(mapv :total results)))
;; totals add up to the statement's $4,431.70 amount due
(is (= 4431.70 (->> results (map #(Double/parseDouble (:total %))) (reduce +))))
(let [d (:date (first results))]
(is (= 2026 (time/year d)))
(is (= 6 (time/month d)))
(is (= 23 (time/day d)))))))