(ns auto-ap.ssr.transaction.import "SSR manual bank-transaction import. Mirrors the SSR ledger import (auto-ap.ssr.ledger) but accepts the exact master-branch Yodlee positional-column TSV and drives the existing auto-ap.import.transactions engine (via auto-ap.import.manual/import-batch) unchanged. Two-stage flow: paste -> editable review grid -> import." (:require [auto-ap.datomic :refer [conn]] [auto-ap.graphql.utils :refer [assert-admin]] [auto-ap.import.manual :as manual] [auto-ap.import.transactions :as t] [auto-ap.permissions :refer [wrap-must]] [auto-ap.routes.transactions :as route] [auto-ap.routes.utils :refer [wrap-client-redirect-unauthenticated]] [auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr.components :as com] [auto-ap.ssr.form-cursor :as fc] [auto-ap.ssr.hx :as hx] [auto-ap.ssr.nested-form-params :refer [wrap-nested-form-params]] [auto-ap.ssr.ui :refer [base-page]] [auto-ap.ssr.utils :refer [apply-middleware-to-all-handlers html-response wrap-form-4xx-2 wrap-schema-decode wrap-schema-enforce]] [bidi.bidi :as bidi] [clojure.data.csv :as csv] [clojure.java.io :as io] [clojure.string :as str] [datomic.api :as dc] [malli.core :as mc] [slingshot.slingshot :refer [throw+]])) ;; --------------------------------------------------------------------------- ;; Parsing (positional Yodlee columns, identical to master) ;; --------------------------------------------------------------------------- (defn tsv->rows "Decode a pasted tab-separated Yodlee export into a vector of raw column vectors. Drops the header row (like auto-ap.import.manual/tabulate-data) and skips blank lines. No-op when already decoded." [data] (if (string? data) (with-open [r (io/reader (char-array data))] (into [] (comp (drop 1) (filter (fn [row] (some (fn [c] (seq (str/trim (or c "")))) row)))) (csv/read-csv r :separator \tab))) data)) (defn vector->row "Map a raw column vector onto the master positional column keys." [t] (if (vector? t) (into {} (filter first (map vector manual/columns t))) t)) (def parse-form-schema (mc/schema [:map [:table {:min 1 :error/message "Paste should contain at least one row to import" :decode/string tsv->rows} [:vector {:coerce? true} [:map {:decode/arbitrary vector->row} [:status {:optional true} [:maybe :string]] [:raw-date {:optional true} [:maybe :string]] [:description-original {:optional true} [:maybe :string]] [:high-level-category {:optional true} [:maybe :string]] [:amount {:optional true} [:maybe :string]] [:bank-account-code {:optional true} [:maybe :string]] [:client-code {:optional true} [:maybe :string]]]]]])) ;; --------------------------------------------------------------------------- ;; Validation (two-tier, preserving every master validation) ;; --------------------------------------------------------------------------- (defn- bank-account-code->client [db] (into {} (dc/q '[:find ?bac ?c :where [?c :client/bank-accounts ?ba] [?ba :bank-account/code ?bac]] db))) (defn- bank-account-code->bank-account [db] (into {} (dc/q '[:find ?bac ?ba :where [?ba :bank-account/code ?bac]] db))) (defn warn-message "Map a non-:import engine categorization to a [message :warn] pair, or nil when the row will import cleanly." [action] (case action :extant ["Already imported — skipped" :warn] :not-ready ["Not ready (before account start date, client locked, or not posted) — skipped" :warn] :suppressed ["Suppressed — skipped" :warn] nil)) (defn classify-table "Given parsed row maps, return {:form-errors {:table {idx [[msg status]...]}} :has-errors? bool}. Hard (fixable) errors come from manual/manual->transaction; warnings come from the engine's own categorize-transaction so the grid preview matches what the import will do." [rows] (let [db (dc/db conn) client-lookup (bank-account-code->client db) ba-lookup (bank-account-code->bank-account db) indexed (map-indexed (fn [i row] (assoc (manual/manual->transaction row ba-lookup client-lookup) ::idx i)) rows) with-ids (t/apply-synthetic-ids indexed) ba-cache (atom {}) existing-cache (atom {}) entries (->> with-ids (map (fn [txn] (let [idx (::idx txn) hard (mapv (fn [e] [(:info e) :error]) (:errors txn)) warn (when (and (empty? hard) (:transaction/bank-account txn)) (let [ba-id (:transaction/bank-account txn) ba (or (get @ba-cache ba-id) (get (swap! ba-cache assoc ba-id (dc/pull db t/bank-account-pull ba-id)) ba-id)) existing (or (get @existing-cache ba-id) (get (swap! existing-cache assoc ba-id (t/get-existing ba-id)) ba-id))] (warn-message (t/categorize-transaction txn ba existing))))] [idx (cond-> hard warn (conj warn))]))) (sort-by first)) form-errors {:table (into {} (filter (fn [[_ errs]] (seq errs)) entries))} has-errors? (boolean (some (fn [[_ errs]] (some (fn [[_ s]] (= :error s)) errs)) entries))] {:form-errors form-errors :has-errors? has-errors?})) ;; --------------------------------------------------------------------------- ;; Views ;; --------------------------------------------------------------------------- (defn- row-badge [errors] (when (seq errors) [:div.p-1.flex.flex-col.gap-1 (for [[m s] errors] [:div.text-xs {:class (if (= :error s) "text-red-600" "text-yellow-600")} m])])) (defn- parsed-banner [request] (let [errs (->> (:form-errors request) :table vals (mapcat identity)) n-err (count (filter (fn [[_ s]] (= :error s)) errs)) n-warn (count (filter (fn [[_ s]] (= :warn s)) errs)) n-rows (count (:table (:form-params request)))] [:div.bg-green-50.text-green-700.rounded.p-3.my-2 (format "%,d rows parsed. " n-rows) (when (pos? n-err) [:span.text-red-700.font-semibold (format "%d error(s) must be fixed. " n-err)]) (when (pos? n-warn) [:span.text-yellow-700.font-semibold (format "%d warning row(s) will be skipped. " n-warn)])])) (defn external-import-text-form* [request] (fc/start-form (or (:form-params request) {}) (:form-errors request) [:form#parse-form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-parse) :hx-target "#forms" :hx-swap "outerHTML"} (fc/with-field :table [:div.flex.flex-col.gap-2 (com/errors {:errors (when (string? (fc/field-errors)) (fc/field-errors))}) (com/text-area {:name (fc/field-name) :rows 6 :class "w-full font-mono text-xs" :placeholder "Paste your Yodlee transaction export (tab-separated, including the header row) here"})]) (com/button {:color :primary :type "submit"} "Parse")])) (defn external-import-table-form* [request] (fc/start-form (:form-params request) (:form-errors request) (fc/with-field :table (when (seq (fc/field-value)) [:div.mt-4 {:x-data (hx/json {"showTable" true})} (when (:just-parsed? request) (parsed-banner request)) [:form {:hx-post (bidi/path-for ssr-routes/only-routes ::route/external-import-import) :hx-target "#forms" :hx-swap "outerHTML" :autocomplete "off"} [:div.flex.gap-4.items-center.my-2 (com/checkbox {"@click" "showTable=!showTable"} "Show table") (com/button {:color :primary :type "submit"} "Import")] [:div {:x-show "showTable"} (com/data-grid-card {:id "transaction-import-data" :route nil :title "Transactions to import" :paginate? false :headers [(com/data-grid-header {} "Date") (com/data-grid-header {} "Description") (com/data-grid-header {} "Amount") (com/data-grid-header {} "Bank Account") (com/data-grid-header {} "Client") (com/data-grid-header {} "Status") (com/data-grid-header {} "")] :rows (fc/cursor-map (fn [_] (let [row-errors (fc/field-errors)] (com/data-grid-row {} (com/data-grid-cell {} (fc/with-field :raw-date (com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"}))) (com/data-grid-cell {} (fc/with-field :description-original (com/text-input {:value (fc/field-value) :name (fc/field-name)}))) (com/data-grid-cell {} (fc/with-field :amount (com/money-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"}))) (com/data-grid-cell {} (fc/with-field :bank-account-code (com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-28"}))) (com/data-grid-cell {} (fc/with-field :client-code (com/text-input {:value (fc/field-value) :name (fc/field-name) :class "w-24"}))) (com/data-grid-cell {} [:span.text-xs.text-gray-500 (fc/with-field :status (fc/field-value))]) (com/data-grid-cell {:class "align-top"} (row-badge row-errors))))))} nil)]]])))) (defn external-import-form* [request] [:div#forms (external-import-text-form* request) (external-import-table-form* request)]) (defn external-import-page [request] (base-page request (com/page {:nav com/main-aside-nav :client-selection (:client-selection request) :clients (:clients request) :client (:client request) :identity (:identity request) :request request} (com/breadcrumbs {} [:a {:href (bidi/path-for ssr-routes/only-routes ::route/page)} "Transactions"] [:a {:href (bidi/path-for ssr-routes/only-routes ::route/external-import-page)} "Import"]) (external-import-form* request)) "Import Transactions")) ;; --------------------------------------------------------------------------- ;; Handlers ;; --------------------------------------------------------------------------- (defn external-import-parse [request] (let [{:keys [form-errors]} (classify-table (:table (:form-params request)))] (html-response (external-import-form* (assoc request :form-errors form-errors :just-parsed? true))))) (defn import-transactions "Validate the (possibly edited) rows. Block the whole batch when any hard error remains; otherwise run the existing import engine on the rows. Returns the engine stats." [request] (assert-admin (:identity request)) (let [rows (:table (:form-params request)) {:keys [form-errors has-errors?]} (classify-table rows)] (when has-errors? (throw+ {:type :field-validation :form-errors form-errors :form-params (:form-params request)})) (let [user (or (:user/name (:identity request)) (:user (:identity request)) "SSR import")] (manual/import-batch rows user)))) (defn external-import-import [request] (let [stats (import-transactions request) imported (:import-batch/imported stats 0) extant (:import-batch/extant stats 0) not-ready (:import-batch/not-ready stats 0) errored (+ (:import-batch/error stats 0) (:failed-validation stats 0))] (html-response (external-import-form* (assoc request :form-params {} :form-errors {})) :headers {"hx-trigger" (hx/json {"notification" (format "%d imported, %d already imported, %d not ready, %d errored." imported extant not-ready errored)})}))) ;; --------------------------------------------------------------------------- ;; Routing ;; --------------------------------------------------------------------------- (def key->handler (apply-middleware-to-all-handlers {::route/external-import-page external-import-page ::route/external-import-parse (-> external-import-parse (wrap-schema-enforce :form-schema parse-form-schema) (wrap-form-4xx-2 external-import-parse) (wrap-schema-decode :form-schema parse-form-schema)) ::route/external-import-import (-> external-import-import (wrap-schema-enforce :form-schema parse-form-schema) (wrap-form-4xx-2 external-import-parse) (wrap-nested-form-params))} (fn [h] (-> h (wrap-must {:activity :import :subject :transaction}) (wrap-client-redirect-unauthenticated)))))