Bryce
9cc3418b1b
- Alphabetize the import.clj :require block (AGENTS.md Import Formatting). - Remove unused imports (digest, strip) flagged by clj-kondo. - Make the client-not-found classify-table test independent: it previously reused the bank-account-not-found input and added zero marginal coverage; now seeds an orphan bank account so only the client error fires. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
301 lines
14 KiB
Clojure
301 lines
14 KiB
Clojure
(ns auto-ap.ssr.transaction.import
|
|
"SSR manual bank-transaction import. Mirrors the SSR ledger import
|
|
(auto-ap.ssr.ledger) but accepts the exact master-branch Yodlee
|
|
positional-column TSV and drives the existing
|
|
auto-ap.import.transactions engine (via auto-ap.import.manual/import-batch)
|
|
unchanged. Two-stage flow: paste -> editable review grid -> import."
|
|
(:require
|
|
[auto-ap.datomic :refer [conn]]
|
|
[auto-ap.graphql.utils :refer [assert-admin]]
|
|
[auto-ap.import.manual :as manual]
|
|
[auto-ap.import.transactions :as t]
|
|
[auto-ap.permissions :refer [wrap-must]]
|
|
[auto-ap.routes.transactions :as route]
|
|
[auto-ap.routes.utils :refer [wrap-client-redirect-unauthenticated]]
|
|
[auto-ap.ssr-routes :as ssr-routes]
|
|
[auto-ap.ssr.components :as com]
|
|
[auto-ap.ssr.form-cursor :as fc]
|
|
[auto-ap.ssr.hx :as hx]
|
|
[auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]]
|
|
[auto-ap.ssr.ui :refer [base-page]]
|
|
[auto-ap.ssr.utils
|
|
:refer [apply-middleware-to-all-handlers html-response
|
|
wrap-form-4xx-2 wrap-schema-decode wrap-schema-enforce]]
|
|
[bidi.bidi :as bidi]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.java.io :as io]
|
|
[clojure.string :as str]
|
|
[datomic.api :as dc]
|
|
[malli.core :as mc]
|
|
[slingshot.slingshot :refer [throw+]]))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Parsing (positional Yodlee columns, identical to master)
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(defn tsv->rows
|
|
"Decode a pasted tab-separated Yodlee export into a vector of raw column
|
|
vectors. Drops the header row (like auto-ap.import.manual/tabulate-data) and
|
|
skips blank lines. No-op when already decoded."
|
|
[data]
|
|
(if (string? data)
|
|
(with-open [r (io/reader (char-array data))]
|
|
(into []
|
|
(comp (drop 1)
|
|
(filter (fn [row] (some (fn [c] (seq (str/trim (or c "")))) row))))
|
|
(csv/read-csv r :separator \tab)))
|
|
data))
|
|
|
|
(defn vector->row
|
|
"Map a raw column vector onto the master positional column keys."
|
|
[t]
|
|
(if (vector? t)
|
|
(into {} (filter first (map vector manual/columns t)))
|
|
t))
|
|
|
|
(def parse-form-schema
|
|
(mc/schema
|
|
[:map
|
|
[:table {:min 1
|
|
:error/message "Paste should contain at least one row to import"
|
|
:decode/string tsv->rows}
|
|
[:vector {:coerce? true}
|
|
[:map {:decode/arbitrary vector->row}
|
|
[:status {:optional true} [:maybe :string]]
|
|
[:raw-date {:optional true} [:maybe :string]]
|
|
[:description-original {:optional true} [:maybe :string]]
|
|
[:high-level-category {:optional true} [:maybe :string]]
|
|
[:amount {:optional true} [:maybe :string]]
|
|
[:bank-account-code {:optional true} [:maybe :string]]
|
|
[:client-code {:optional true} [:maybe :string]]]]]]))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Validation (two-tier, preserving every master validation)
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(defn- bank-account-code->client [db]
|
|
(into {} (dc/q '[:find ?bac ?c
|
|
:where
|
|
[?c :client/bank-accounts ?ba]
|
|
[?ba :bank-account/code ?bac]]
|
|
db)))
|
|
|
|
(defn- bank-account-code->bank-account [db]
|
|
(into {} (dc/q '[:find ?bac ?ba
|
|
:where [?ba :bank-account/code ?bac]]
|
|
db)))
|
|
|
|
(defn warn-message
|
|
"Map a non-:import engine categorization to a [message :warn] pair, or nil
|
|
when the row will import cleanly."
|
|
[action]
|
|
(case action
|
|
:extant ["Already imported — skipped" :warn]
|
|
:not-ready ["Not ready (before account start date, client locked, or not posted) — skipped" :warn]
|
|
:suppressed ["Suppressed — skipped" :warn]
|
|
nil))
|
|
|
|
(defn classify-table
|
|
"Given parsed row maps, return {:form-errors {:table {idx [[msg status]...]}}
|
|
:has-errors? bool}. Hard (fixable) errors come from
|
|
manual/manual->transaction; warnings come from the engine's own
|
|
categorize-transaction so the grid preview matches what the import will do."
|
|
[rows]
|
|
(let [db (dc/db conn)
|
|
client-lookup (bank-account-code->client db)
|
|
ba-lookup (bank-account-code->bank-account db)
|
|
indexed (map-indexed
|
|
(fn [i row]
|
|
(assoc (manual/manual->transaction row ba-lookup client-lookup)
|
|
::idx i))
|
|
rows)
|
|
with-ids (t/apply-synthetic-ids indexed)
|
|
ba-cache (atom {})
|
|
existing-cache (atom {})
|
|
entries (->> with-ids
|
|
(map (fn [txn]
|
|
(let [idx (::idx txn)
|
|
hard (mapv (fn [e] [(:info e) :error]) (:errors txn))
|
|
warn (when (and (empty? hard)
|
|
(:transaction/bank-account txn))
|
|
(let [ba-id (:transaction/bank-account txn)
|
|
ba (or (get @ba-cache ba-id)
|
|
(get (swap! ba-cache assoc ba-id
|
|
(dc/pull db t/bank-account-pull ba-id))
|
|
ba-id))
|
|
existing (or (get @existing-cache ba-id)
|
|
(get (swap! existing-cache assoc ba-id
|
|
(t/get-existing ba-id))
|
|
ba-id))]
|
|
(warn-message (t/categorize-transaction txn ba existing))))]
|
|
[idx (cond-> hard warn (conj warn))])))
|
|
(sort-by first))
|
|
form-errors {:table (into {} (filter (fn [[_ errs]] (seq errs)) entries))}
|
|
has-errors? (boolean (some (fn [[_ errs]] (some (fn [[_ s]] (= :error s)) errs)) entries))]
|
|
{:form-errors form-errors
|
|
:has-errors? has-errors?}))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Views
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(defn- row-badge [errors]
|
|
(when (seq errors)
|
|
[:div.p-1.flex.flex-col.gap-1
|
|
(for [[m s] errors]
|
|
[:div.text-xs {:class (if (= :error s) "text-red-600" "text-yellow-600")} m])]))
|
|
|
|
(defn- parsed-banner [request]
|
|
(let [errs (->> (:form-errors request) :table vals (mapcat identity))
|
|
n-err (count (filter (fn [[_ s]] (= :error s)) errs))
|
|
n-warn (count (filter (fn [[_ s]] (= :warn s)) errs))
|
|
n-rows (count (:table (:form-params request)))]
|
|
[:div.bg-green-50.text-green-700.rounded.p-3.my-2
|
|
(format "%,d rows parsed. " n-rows)
|
|
(when (pos? n-err)
|
|
[:span.text-red-700.font-semibold (format "%d error(s) must be fixed. " n-err)])
|
|
(when (pos? n-warn)
|
|
[:span.text-yellow-700.font-semibold (format "%d warning row(s) will be skipped. " n-warn)])]))
|
|
|
|
(defn external-import-text-form* [request]
|
|
(fc/start-form
|
|
(or (:form-params request) {}) (:form-errors request)
|
|
[:form#parse-form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-parse)
|
|
:hx-target "#forms"
|
|
:hx-swap "outerHTML"}
|
|
(fc/with-field :table
|
|
[:div.flex.flex-col.gap-2
|
|
(com/errors {:errors (when (string? (fc/field-errors)) (fc/field-errors))})
|
|
(com/text-area {:name (fc/field-name)
|
|
:rows 6
|
|
:class "w-full font-mono text-xs"
|
|
:placeholder "Paste your Yodlee transaction export (tab-separated, including the header row) here"})])
|
|
(com/button {:color :primary :type "submit"} "Parse")]))
|
|
|
|
(defn external-import-table-form* [request]
|
|
(fc/start-form
|
|
(:form-params request) (:form-errors request)
|
|
(fc/with-field :table
|
|
(when (seq (fc/field-value))
|
|
[:div.mt-4 {:x-data (hx/json {"showTable" true})}
|
|
(when (:just-parsed? request)
|
|
(parsed-banner request))
|
|
[:form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-import)
|
|
:hx-target "#forms"
|
|
:hx-swap "outerHTML"
|
|
:autocomplete "off"}
|
|
[:div.flex.gap-4.items-center.my-2
|
|
(com/checkbox {"@click" "showTable=!showTable"} "Show table")
|
|
(com/button {:color :primary :type "submit"} "Import")]
|
|
[:div {:x-show "showTable"}
|
|
(com/data-grid-card
|
|
{:id "transaction-import-data"
|
|
:route nil
|
|
:title "Transactions to import"
|
|
:paginate? false
|
|
:headers [(com/data-grid-header {} "Date")
|
|
(com/data-grid-header {} "Description")
|
|
(com/data-grid-header {} "Amount")
|
|
(com/data-grid-header {} "Bank Account")
|
|
(com/data-grid-header {} "Client")
|
|
(com/data-grid-header {} "Status")
|
|
(com/data-grid-header {} "")]
|
|
:rows
|
|
(fc/cursor-map
|
|
(fn [_]
|
|
(let [row-errors (fc/field-errors)]
|
|
(com/data-grid-row
|
|
{}
|
|
(com/data-grid-cell {} (fc/with-field :raw-date
|
|
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
|
(com/data-grid-cell {} (fc/with-field :description-original
|
|
(com/text-input {:value (fc/field-value) :name (fc/field-name)})))
|
|
(com/data-grid-cell {} (fc/with-field :amount
|
|
(com/money-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
|
(com/data-grid-cell {} (fc/with-field :bank-account-code
|
|
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"})))
|
|
(com/data-grid-cell {} (fc/with-field :client-code
|
|
(com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-24"})))
|
|
(com/data-grid-cell {} [:span.text-xs.text-gray-500 (fc/with-field :status (fc/field-value))])
|
|
(com/data-grid-cell {:class "align-top"} (row-badge row-errors))))))}
|
|
nil)]]]))))
|
|
|
|
(defn external-import-form* [request]
|
|
[:div#forms
|
|
(external-import-text-form* request)
|
|
(external-import-table-form* request)])
|
|
|
|
(defn external-import-page [request]
|
|
(base-page
|
|
request
|
|
(com/page {:nav com/main-aside-nav
|
|
:client-selection (:client-selection request)
|
|
:clients (:clients request)
|
|
:client (:client request)
|
|
:identity (:identity request)
|
|
:request request}
|
|
(com/breadcrumbs {}
|
|
[:a {:href (bidi/path-for ssr-routes/only-routes ::route/page)} "Transactions"]
|
|
[:a {:href (bidi/path-for ssr-routes/only-routes ::route/external-import-page)} "Import"])
|
|
(external-import-form* request))
|
|
"Import Transactions"))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Handlers
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(defn external-import-parse [request]
|
|
(let [{:keys [form-errors]} (classify-table (:table (:form-params request)))]
|
|
(html-response
|
|
(external-import-form* (assoc request :form-errors form-errors :just-parsed? true)))))
|
|
|
|
(defn import-transactions
|
|
"Validate the (possibly edited) rows. Block the whole batch when any hard
|
|
error remains; otherwise run the existing import engine on the rows. Returns
|
|
the engine stats."
|
|
[request]
|
|
(assert-admin (:identity request))
|
|
(let [rows (:table (:form-params request))
|
|
{:keys [form-errors has-errors?]} (classify-table rows)]
|
|
(when has-errors?
|
|
(throw+ {:type :field-validation
|
|
:form-errors form-errors
|
|
:form-params (:form-params request)}))
|
|
(let [user (or (:user/name (:identity request))
|
|
(:user (:identity request))
|
|
"SSR import")]
|
|
(manual/import-batch rows user))))
|
|
|
|
(defn external-import-import [request]
|
|
(let [stats (import-transactions request)
|
|
imported (:import-batch/imported stats 0)
|
|
extant (:import-batch/extant stats 0)
|
|
not-ready (:import-batch/not-ready stats 0)
|
|
errored (+ (:import-batch/error stats 0) (:failed-validation stats 0))]
|
|
(html-response
|
|
(external-import-form* (assoc request :form-params {} :form-errors {}))
|
|
:headers {"hx-trigger"
|
|
(hx/json {"notification"
|
|
(format "%d imported, %d already imported, %d not ready, %d errored."
|
|
imported extant not-ready errored)})})))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Routing
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(def key->handler
|
|
(apply-middleware-to-all-handlers
|
|
{::route/external-import-page external-import-page
|
|
::route/external-import-parse (-> external-import-parse
|
|
(wrap-schema-enforce :form-schema parse-form-schema)
|
|
(wrap-form-4xx-2 external-import-parse)
|
|
(wrap-schema-decode :form-schema parse-form-schema))
|
|
::route/external-import-import (-> external-import-import
|
|
(wrap-schema-enforce :form-schema parse-form-schema)
|
|
(wrap-form-4xx-2 external-import-parse)
|
|
(wrap-nested-form-params))}
|
|
(fn [h]
|
|
(-> h
|
|
(wrap-must {:activity :import :subject :transaction})
|
|
(wrap-client-redirect-unauthenticated)))))
|