feat(transactions): port manual bank-transaction import to SSR
Implement the SSR/alpine/htmx manual transaction import, wiring the already-declared but unhandled ::external-import-page/parse/import routes. Mirrors the SSR ledger import: paste the exact master-branch Yodlee positional-column TSV, review parsed rows in an editable grid with per-row error/warning badges, and import. Every master validation is preserved and the existing import.transactions engine is reused unchanged (via import.manual/import-batch), so core components are untouched. - New ns auto-ap.ssr.transaction.import (page, paste/parse, editable grid, two-tier validation, import handler) + admin-only transactions Import nav. - Two-tier validation: fixable problems (bad date/amount, unknown client or bank-account code, missing fields) are hard errors that block the whole batch; inherent skip-conditions (non-POSTED, before start-date/locked, already-imported) are warnings computed from the engine's own categorize-transaction so the grid preview matches the import result. - Tests: failing-first Playwright e2e (e2e/transaction-import.spec.ts) plus unit/integration coverage (ssr/transaction/import_test.clj, 10 tests). - Deterministic bank-account code in the e2e seed. Plan: docs/plans/2026-06-01-001-feat-manual-transaction-import-ssr-plan.md Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -304,6 +304,12 @@
|
||||
:hx-boost "true"
|
||||
:hx-include "#transaction-filters"}
|
||||
"Approved")
|
||||
(when (is-admin? (:identity request))
|
||||
(menu-button- {:href (bidi/path-for ssr-routes/only-routes
|
||||
::transaction-routes/external-import-page)
|
||||
:active? (= ::transaction-routes/external-import-page (:matched-route request))
|
||||
:hx-boost "true"}
|
||||
"Import"))
|
||||
(when (can? (:identity request)
|
||||
{:subject :transaction :activity :insights})
|
||||
(menu-button- {:href (bidi/path-for ssr-routes/only-routes
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
grid-page query-schema
|
||||
wrap-status-from-source]]
|
||||
[auto-ap.ssr.transaction.edit :as edit :refer [transaction-account-row*]]
|
||||
[auto-ap.ssr.transaction.import :as t-import]
|
||||
[auto-ap.ssr.utils
|
||||
:refer [apply-middleware-to-all-handlers entity-id html-response
|
||||
many-entity modal-response percentage ref->enum-schema
|
||||
@@ -101,6 +102,7 @@
|
||||
(def key->handler
|
||||
(merge edit/key->handler
|
||||
bulk-code/key->handler
|
||||
t-import/key->handler
|
||||
(apply-middleware-to-all-handlers
|
||||
{::route/page page
|
||||
::route/approved-page (-> page (wrap-implied-route-param :status :transaction-approval-status/approved))
|
||||
|
||||
301
src/clj/auto_ap/ssr/transaction/import.clj
Normal file
301
src/clj/auto_ap/ssr/transaction/import.clj
Normal file
@@ -0,0 +1,301 @@
|
||||
(ns auto-ap.ssr.transaction.import
|
||||
"SSR manual bank-transaction import. Mirrors the SSR ledger import
|
||||
(auto-ap.ssr.ledger) but accepts the exact master-branch Yodlee
|
||||
positional-column TSV and drives the existing
|
||||
auto-ap.import.transactions engine (via auto-ap.import.manual/import-batch)
|
||||
unchanged. Two-stage flow: paste -> editable review grid -> import."
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.graphql.utils :refer [assert-admin]]
|
||||
[auto-ap.import.manual :as manual]
|
||||
[auto-ap.import.transactions :as t]
|
||||
[auto-ap.routes.transactions :as route]
|
||||
[auto-ap.ssr-routes :as ssr-routes]
|
||||
[auto-ap.ssr.components :as com]
|
||||
[auto-ap.ssr.form-cursor :as fc]
|
||||
[auto-ap.ssr.hx :as hx]
|
||||
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
|
||||
[auto-ap.ssr.ui :refer [base-page]]
|
||||
[auto-ap.ssr.utils
|
||||
:refer [apply-middleware-to-all-handlers html-response strip
|
||||
wrap-form-4xx-2 wrap-schema-decode wrap-schema-enforce]]
|
||||
[auto-ap.permissions :refer [wrap-must]]
|
||||
[auto-ap.routes.utils :refer [wrap-client-redirect-unauthenticated]]
|
||||
[bidi.bidi :as bidi]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[datomic.api :as dc]
|
||||
[digest :as di]
|
||||
[malli.core :as mc]
|
||||
[slingshot.slingshot :refer [throw+]]))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Parsing (positional Yodlee columns, identical to master)
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn tsv->rows
|
||||
"Decode a pasted tab-separated Yodlee export into a vector of raw column
|
||||
vectors. Drops the header row (like auto-ap.import.manual/tabulate-data) and
|
||||
skips blank lines. No-op when already decoded."
|
||||
[data]
|
||||
(if (string? data)
|
||||
(with-open [r (io/reader (char-array data))]
|
||||
(into []
|
||||
(comp (drop 1)
|
||||
(filter (fn [row] (some (fn [c] (seq (str/trim (or c "")))) row))))
|
||||
(csv/read-csv r :separator \tab)))
|
||||
data))
|
||||
|
||||
(defn vector->row
|
||||
"Map a raw column vector onto the master positional column keys."
|
||||
[t]
|
||||
(if (vector? t)
|
||||
(into {} (filter first (map vector manual/columns t)))
|
||||
t))
|
||||
|
||||
(def parse-form-schema
|
||||
(mc/schema
|
||||
[:map
|
||||
[:table {:min 1
|
||||
:error/message "Paste should contain at least one row to import"
|
||||
:decode/string tsv->rows}
|
||||
[:vector {:coerce? true}
|
||||
[:map {:decode/arbitrary vector->row}
|
||||
[:status {:optional true} [:maybe :string]]
|
||||
[:raw-date {:optional true} [:maybe :string]]
|
||||
[:description-original {:optional true} [:maybe :string]]
|
||||
[:high-level-category {:optional true} [:maybe :string]]
|
||||
[:amount {:optional true} [:maybe :string]]
|
||||
[:bank-account-code {:optional true} [:maybe :string]]
|
||||
[:client-code {:optional true} [:maybe :string]]]]]]))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Validation (two-tier, preserving every master validation)
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn- bank-account-code->client [db]
|
||||
(into {} (dc/q '[:find ?bac ?c
|
||||
:where
|
||||
[?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/code ?bac]]
|
||||
db)))
|
||||
|
||||
(defn- bank-account-code->bank-account [db]
|
||||
(into {} (dc/q '[:find ?bac ?ba
|
||||
:where [?ba :bank-account/code ?bac]]
|
||||
db)))
|
||||
|
||||
(defn warn-message
|
||||
"Map a non-:import engine categorization to a [message :warn] pair, or nil
|
||||
when the row will import cleanly."
|
||||
[action]
|
||||
(case action
|
||||
:extant ["Already imported — skipped" :warn]
|
||||
:not-ready ["Not ready (before account start date, client locked, or not posted) — skipped" :warn]
|
||||
:suppressed ["Suppressed — skipped" :warn]
|
||||
nil))
|
||||
|
||||
(defn classify-table
|
||||
"Given parsed row maps, return {:form-errors {:table {idx [[msg status]...]}}
|
||||
:has-errors? bool}. Hard (fixable) errors come from
|
||||
manual/manual->transaction; warnings come from the engine's own
|
||||
categorize-transaction so the grid preview matches what the import will do."
|
||||
[rows]
|
||||
(let [db (dc/db conn)
|
||||
client-lookup (bank-account-code->client db)
|
||||
ba-lookup (bank-account-code->bank-account db)
|
||||
indexed (map-indexed
|
||||
(fn [i row]
|
||||
(assoc (manual/manual->transaction row ba-lookup client-lookup)
|
||||
::idx i))
|
||||
rows)
|
||||
with-ids (t/apply-synthetic-ids indexed)
|
||||
ba-cache (atom {})
|
||||
existing-cache (atom {})
|
||||
entries (->> with-ids
|
||||
(map (fn [txn]
|
||||
(let [idx (::idx txn)
|
||||
hard (mapv (fn [e] [(:info e) :error]) (:errors txn))
|
||||
warn (when (and (empty? hard)
|
||||
(:transaction/bank-account txn))
|
||||
(let [ba-id (:transaction/bank-account txn)
|
||||
ba (or (get @ba-cache ba-id)
|
||||
(get (swap! ba-cache assoc ba-id
|
||||
(dc/pull db t/bank-account-pull ba-id))
|
||||
ba-id))
|
||||
existing (or (get @existing-cache ba-id)
|
||||
(get (swap! existing-cache assoc ba-id
|
||||
(t/get-existing ba-id))
|
||||
ba-id))]
|
||||
(warn-message (t/categorize-transaction txn ba existing))))]
|
||||
[idx (cond-> hard warn (conj warn))])))
|
||||
(sort-by first))
|
||||
form-errors {:table (into {} (filter (fn [[_ errs]] (seq errs)) entries))}
|
||||
has-errors? (boolean (some (fn [[_ errs]] (some (fn [[_ s]] (= :error s)) errs)) entries))]
|
||||
{:form-errors form-errors
|
||||
:has-errors? has-errors?}))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Views
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn- row-badge [errors]
|
||||
(when (seq errors)
|
||||
[:div.p-1.flex.flex-col.gap-1
|
||||
(for [[m s] errors]
|
||||
[:div.text-xs {:class (if (= :error s) "text-red-600" "text-yellow-600")} m])]))
|
||||
|
||||
(defn- parsed-banner [request]
|
||||
(let [errs (->> (:form-errors request) :table vals (mapcat identity))
|
||||
n-err (count (filter (fn [[_ s]] (= :error s)) errs))
|
||||
n-warn (count (filter (fn [[_ s]] (= :warn s)) errs))
|
||||
n-rows (count (:table (:form-params request)))]
|
||||
[:div.bg-green-50.text-green-700.rounded.p-3.my-2
|
||||
(format "%,d rows parsed. " n-rows)
|
||||
(when (pos? n-err)
|
||||
[:span.text-red-700.font-semibold (format "%d error(s) must be fixed. " n-err)])
|
||||
(when (pos? n-warn)
|
||||
[:span.text-yellow-700.font-semibold (format "%d warning row(s) will be skipped. " n-warn)])]))
|
||||
|
||||
(defn external-import-text-form* [request]
|
||||
(fc/start-form
|
||||
(or (:form-params request) {}) (:form-errors request)
|
||||
[:form#parse-form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-parse)
|
||||
:hx-target "#forms"
|
||||
:hx-swap "outerHTML"}
|
||||
(fc/with-field :table
|
||||
[:div.flex.flex-col.gap-2
|
||||
(com/errors {:errors (when (string? (fc/field-errors)) (fc/field-errors))})
|
||||
(com/text-area {:name (fc/field-name)
|
||||
:rows 6
|
||||
:class "w-full font-mono text-xs"
|
||||
:placeholder "Paste your Yodlee transaction export (tab-separated, including the header row) here"})])
|
||||
(com/button {:color :primary :type "submit"} "Parse")]))
|
||||
|
||||
(defn external-import-table-form* [request]
|
||||
(fc/start-form
|
||||
(:form-params request) (:form-errors request)
|
||||
(fc/with-field :table
|
||||
(when (seq (fc/field-value))
|
||||
[:div.mt-4 {:x-data (hx/json {"showTable" true})}
|
||||
(when (:just-parsed? request)
|
||||
(parsed-banner request))
|
||||
[:form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-import)
|
||||
:hx-target "#forms"
|
||||
:hx-swap "outerHTML"
|
||||
:autocomplete "off"}
|
||||
[:div.flex.gap-4.items-center.my-2
|
||||
(com/checkbox {"@click" "showTable=!showTable"} "Show table")
|
||||
(com/button {:color :primary :type "submit"} "Import")]
|
||||
[:div {:x-show "showTable"}
|
||||
(com/data-grid-card
|
||||
{:id "transaction-import-data"
|
||||
:route nil
|
||||
:title "Transactions to import"
|
||||
:paginate? false
|
||||
:headers [(com/data-grid-header {} "Date")
|
||||
(com/data-grid-header {} "Description")
|
||||
(com/data-grid-header {} "Amount")
|
||||
(com/data-grid-header {} "Bank Account")
|
||||
(com/data-grid-header {} "Client")
|
||||
(com/data-grid-header {} "Status")
|
||||
(com/data-grid-header {} "")]
|
||||
:rows
|
||||
(fc/cursor-map
|
||||
(fn [_]
|
||||
(let [row-errors (fc/field-errors)]
|
||||
(com/data-grid-row
|
||||
{}
|
||||
(com/data-grid-cell {} (fc/with-field :raw-date
|
||||
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
||||
(com/data-grid-cell {} (fc/with-field :description-original
|
||||
(com/text-input {:value (fc/field-value) :name (fc/field-name)})))
|
||||
(com/data-grid-cell {} (fc/with-field :amount
|
||||
(com/money-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
||||
(com/data-grid-cell {} (fc/with-field :bank-account-code
|
||||
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
||||
(com/data-grid-cell {} (fc/with-field :client-code
|
||||
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-24"})))
|
||||
(com/data-grid-cell {} [:span.text-xs.text-gray-500 (fc/with-field :status (fc/field-value))])
|
||||
(com/data-grid-cell {:class "align-top"} (row-badge row-errors))))))}
|
||||
nil)]]]))))
|
||||
|
||||
(defn external-import-form* [request]
|
||||
[:div#forms
|
||||
(external-import-text-form* request)
|
||||
(external-import-table-form* request)])
|
||||
|
||||
(defn external-import-page [request]
|
||||
(base-page
|
||||
request
|
||||
(com/page {:nav com/main-aside-nav
|
||||
:client-selection (:client-selection request)
|
||||
:clients (:clients request)
|
||||
:client (:client request)
|
||||
:identity (:identity request)
|
||||
:request request}
|
||||
(com/breadcrumbs {}
|
||||
[:a {:href (bidi/path-for ssr-routes/only-routes ::route/page)} "Transactions"]
|
||||
[:a {:href (bidi/path-for ssr-routes/only-routes ::route/external-import-page)} "Import"])
|
||||
(external-import-form* request))
|
||||
"Import Transactions"))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Handlers
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn external-import-parse [request]
|
||||
(let [{:keys [form-errors]} (classify-table (:table (:form-params request)))]
|
||||
(html-response
|
||||
(external-import-form* (assoc request :form-errors form-errors :just-parsed? true)))))
|
||||
|
||||
(defn import-transactions
|
||||
"Validate the (possibly edited) rows. Block the whole batch when any hard
|
||||
error remains; otherwise run the existing import engine on the rows. Returns
|
||||
the engine stats."
|
||||
[request]
|
||||
(assert-admin (:identity request))
|
||||
(let [rows (:table (:form-params request))
|
||||
{:keys [form-errors has-errors?]} (classify-table rows)]
|
||||
(when has-errors?
|
||||
(throw+ {:type :field-validation
|
||||
:form-errors form-errors
|
||||
:form-params (:form-params request)}))
|
||||
(let [user (or (:user/name (:identity request))
|
||||
(:user (:identity request))
|
||||
"SSR import")]
|
||||
(manual/import-batch rows user))))
|
||||
|
||||
(defn external-import-import [request]
|
||||
(let [stats (import-transactions request)
|
||||
imported (:import-batch/imported stats 0)
|
||||
extant (:import-batch/extant stats 0)
|
||||
not-ready (:import-batch/not-ready stats 0)
|
||||
errored (+ (:import-batch/error stats 0) (:failed-validation stats 0))]
|
||||
(html-response
|
||||
(external-import-form* (assoc request :form-params {} :form-errors {}))
|
||||
:headers {"hx-trigger"
|
||||
(hx/json {"notification"
|
||||
(format "%d imported, %d already imported, %d not ready, %d errored."
|
||||
imported extant not-ready errored)})})))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Routing
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(def key->handler
|
||||
(apply-middleware-to-all-handlers
|
||||
{::route/external-import-page external-import-page
|
||||
::route/external-import-parse (-> external-import-parse
|
||||
(wrap-schema-enforce :form-schema parse-form-schema)
|
||||
(wrap-form-4xx-2 external-import-parse)
|
||||
(wrap-schema-decode :form-schema parse-form-schema))
|
||||
::route/external-import-import (-> external-import-import
|
||||
(wrap-schema-enforce :form-schema parse-form-schema)
|
||||
(wrap-form-4xx-2 external-import-parse)
|
||||
(wrap-nested-form-params))}
|
||||
(fn [h]
|
||||
(-> h
|
||||
(wrap-must {:activity :import :subject :transaction})
|
||||
(wrap-client-redirect-unauthenticated)))))
|
||||
Reference in New Issue
Block a user