notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(cloud) adds transaction insights

Browse Source
This commit is contained in:
Bryce Covert
2023-04-18 11:32:58 -07:00
parent ed37b19d6a
commit 69eaf42eda
11 changed files with 518 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
resources/cloud-migration-schema.edn
View File

@@ -25,4 +25,28 @@
:db/doc "Used to find sales orders quickly",
:db/noHistory true
}
{:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one
:db/ident :transaction/recommended-account
:db/doc "The recommended account from the ML alogorithm"
:db/noHistory true}
{:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one
:db/ident :transaction/recommended-vendor
:db/doc "The recommended vendor from the ML alogorithm"
:db/noHistory true}
{:db/valueType :db.type/double
:db/cardinality :db.cardinality/one
:db/ident :transaction/vendor-confidence
:db/doc "The confidence from the ML alogorithm"
:db/noHistory true}
{:db/valueType :db.type/double
:db/cardinality :db.cardinality/one
:db/ident :transaction/account-confidence
:db/doc "The confidence from the ML alogorithm"
:db/noHistory true}
]

4
resources/public/css/bulma.min.css vendored
View File

@@ -11060,4 +11060,8 @@ tbody tr.live-added {
opacity: 1 !important;
}
.htmx-request .htmx-indicator {
opacity: 1 !important;
}
/*# sourceMappingURL=bulma.min.css.map */

2
resources/public/css/bulma.min.css.map
View File

File diff suppressed because one or more lines are too long

4
resources/sass/bulma.scss vendored
View File

@@ -187,6 +187,10 @@ tbody tr.live-added {
.htmx-indicator.button {
opacity: 1.0 !important
}
.htmx-request .htmx-indicator {
opacity: 1.0 !important
}
.htmx-request .button.htmx-indicator {
@extend .is-loading
}

172
scratch-sessions/build_ml.clj Normal file
View File

@@ -0,0 +1,172 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(ns build-ml
(:require [datomic.client.api :as dc]
[clojure.java.io :as io]
[clojure.data.csv :as csv]
[auto-ap.datomic :refer [conn]]))
(println "hi")
(defn boot []
(user/init-repl)
(user/start-db))
(defn write-vendor-training-data []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-training.csv")]
(csv/write-csv f
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount" "vendor"]]
(->>
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v
:in $ ?start
:where
[?t :transaction/date ?d]
[(>= ?d ?start)]
[?t :transaction/approval-status :transaction-approval-status/approved]
(not [?t :transaction/payment])
[?t :transaction/client ?c]
[?t :transaction/vendor ?v]
[?v :vendor/name ?v-name]
[?t :transaction/description-original ?do]
[?t :transaction/amount ?amt]
[?t :transaction/bank-account ?b]
[?b :bank-account/name ?bn]
[?b :bank-account/type ?bat]
[?bat :db/ident ?bt]
[(name ?bt) ?bt2]
]
(dc/db auto-ap.datomic/conn)
#inst "2021-01-01")
(map (fn [[t code bn bat d date amt v]]
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v]))))
:quote? (constantly true))))
(defn write-account-training-data []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/account-training.csv")]
(csv/write-csv f
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount" "vendor" "account"]]
(->>
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v ?a
:in $ ?start
:where
[?t :transaction/date ?d]
[(>= ?d ?start)]
[?t :transaction/approval-status :transaction-approval-status/approved]
[?t :transaction/accounts ?ta]
[?ta :transaction-account/account ?a]
[?t :transaction/vendor ?v]
[?v :vendor/name ?v-name]
[?t :transaction/client ?c]
[?t :transaction/description-original ?do]
[?t :transaction/amount ?amt]
[?t :transaction/bank-account ?b]
[?b :bank-account/name ?bn]
[?b :bank-account/type ?bat]
[?bat :db/ident ?bt]
[(name ?bt) ?bt2]]
(dc/db auto-ap.datomic/conn)
#inst "2022-01-01")
(map (fn [[t code bn bat d date amt v a]]
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v a]))))
:quote? (constantly true))))
(defn write-inference []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/inference.csv")]
(csv/write-csv f
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]]
(->>
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt
:in $ ?start
:where
[?t :transaction/date ?d]
[(>= ?d ?start)]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/matched-rule])
(not [?t :transaction/payment])
[?t :transaction/client ?c]
[?c :client/code ?code]
(not [?t :transaction/vendor])
(not [?t :transaction/accounts])
[?t :transaction/description-original ?do]
[?t :transaction/amount ?amt]
[?t :transaction/bank-account ?b]
[?b :bank-account/name ?bn]
[?b :bank-account/type ?bat]
[?bat :db/ident ?bt]
[(name ?bt) ?bt2]
]
(dc/db auto-ap.datomic/conn)
#inst "2023-01-01")
(map (fn [[t code bn bat d date amt]]
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt]))
))
:quote? (constantly true))))
(defn write-vendor-lookup []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendors.csv")]
(csv/write-csv f
(into [["vendor" "vendor_name"]]
(->>
(dc/q '[:find ?v ?v-name
:where [?v :vendor/name ?v-name]
[_ :transaction/vendor ?v]
]
(dc/db conn))))
:separator \,)))
(defn write-vendor-client-lookup []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-client-usage.csv")]
(csv/write-csv f
(into [["vendor" "client" "vendor_client_count"]]
(->>
(dc/q '[:find ?v ?c (count ?t)
:where
[?t :transaction/client ?c]
[?t :transaction/vendor ?v]
]
(dc/db conn))))
:separator \,)))
(defn write-account-lookup []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/accounts.csv")]
(csv/write-csv f
(into [["account" "numeric_code"]]
(->>
(dc/q '[:find ?a ?nm
:where [?a :account/numeric-code ?nm]]
(dc/db conn))))
:separator \,)))
(defn write-account-client-lookup []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/account-client-usage.csv")]
(csv/write-csv f
(into [["client" "account" "client_account_count"]]
(->>
(dc/q '[:find ?c ?a (count ?t)
:where
[?t :transaction/client ?c]
[?t :transaction/accounts ?ta]
[?ta :transaction-account/account ?a]]
(dc/db conn))))
:separator \,)))
(defn write-all-training []
(write-account-lookup)
(write-vendor-lookup)
(write-account-client-lookup)
(write-vendor-client-lookup)
(write-account-training-data)
(write-vendor-training-data))

71
scratch-sessions/ingest_ml.clj Normal file
View File

@@ -0,0 +1,71 @@
(ns ingest-ml
(:require [datomic.client.api :as dc]
[clojure.java.io :as io]
[clojure.data.csv :as csv]
[iol-ion.tx :refer [upsert-entity]]
[auto-ap.datomic :refer [conn]]))
(println "hi")
(defn boot []
(user/init-repl)
(user/start-db))
(defn reset-inference []
(doseq [p (->>
(dc/q '[:find ?t
:where [?t :transaction/recommended-account]]
(dc/db conn))
(map (fn [[t]]
`(upsert-entity
~{:db/id t
:transaction/recommended-account nil
:tranasction/vendor-confidence nil
:transaction/account-confidence nil
:transaction/recommended-vendor nil})))
(partition-all 100))]
(dc/transact conn {:tx-data p})))
(defn read-inference []
(with-open [reader (io/reader "/mnt/data/dev2/ml-test/inference-outcome.csv")]
(->> (csv/read-csv reader)
(into []
(comp
(drop 1)
(map (fn [[_ transaction best-vendor best-account account-confidence]]
{:db/id (Long/parseLong transaction)
:transaction/recommended-account (Long/parseLong best-account)
:transaction/account-confidence (Double/parseDouble account-confidence)
:transaction/recommended-vendor (Long/parseLong best-vendor)}
)))))))
(defn apply-inference [inference]
(doseq [p (->> inference (partition-all 100))]
(dc/transact conn {:tx-data p})))
(defn check-applied-inference []
(clojure.pprint/pprint
(->>
(dc/q '[:find ?code ?bac ?do ?amount ?vc ?rvn ?ac ?ran
:in $
:where
[?t :transaction/recommended-account ?ra]
[?ra :account/name ?ran]
[?t :transaction/account-confidence ?ac]
[?t :transaction/recommended-vendor ?rv]
[?rv :vendor/name ?rvn]
[?t :transaction/description-original ?do]
[?t :transaction/client ?c]
[?c :client/code ?code]
[?t :transaction/bank-account ?ba]
[?ba :bank-account/code ?bac]
[?t :transaction/amount ?amount]
]
(dc/db conn))
(shuffle)
(take 10))))

8
src/clj/auto_ap/ssr/core.clj
View File

@@ -4,6 +4,7 @@
:refer [wrap-admin wrap-client-redirect-unauthenticated wrap-secure]]
[auto-ap.ssr.admin :as admin]
[auto-ap.ssr.auth :as auth]
[auto-ap.ssr.transaction.insights :as insights]
[auto-ap.ssr.company.company-1099 :as company-1099]
[auto-ap.ssr.company-dropdown :as company-dropdown]))
@@ -19,5 +20,10 @@
:company-1099 (wrap-client-redirect-unauthenticated (wrap-secure company-1099/page))
:company-1099-vendor-table (wrap-client-redirect-unauthenticated (wrap-secure company-1099/vendor-table))
:company-1099-vendor-dialog (wrap-client-redirect-unauthenticated (wrap-secure company-1099/vendor-dialog))
:company-1099-vendor-save (wrap-client-redirect-unauthenticated (wrap-secure company-1099/vendor-save))})
:company-1099-vendor-save (wrap-client-redirect-unauthenticated (wrap-secure company-1099/vendor-save))
:transaction-insights (wrap-client-redirect-unauthenticated (wrap-secure insights/page))
:transaction-insight-table (wrap-client-redirect-unauthenticated (wrap-secure insights/insight-table))
:transaction-insight-rows (wrap-client-redirect-unauthenticated (wrap-secure insights/transaction-rows))
:transaction-insight-approve (wrap-client-redirect-unauthenticated (wrap-secure insights/approve))
:transaction-insight-explain (wrap-client-redirect-unauthenticated (wrap-secure insights/explain))})

218
src/clj/auto_ap/ssr/transaction/insights.clj Normal file
View File

@@ -0,0 +1,218 @@
(ns auto-ap.ssr.transaction.insights
(:require
[auto-ap.datomic :refer [conn visible-clients]]
[auto-ap.shared-views.company.sidebar :refer [company-side-bar]]
[auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.ui :refer [base-page]]
[auto-ap.ssr.utils :refer [html-response]]
[auto-ap.time :as atime]
[bidi.bidi :as bidi]
[clj-time.coerce :as coerce]
[datomic.client.api :as dc]
[hiccup2.core :as hiccup]
[clj-time.core :as time]))
(def pull-expr [:transaction/description-original
:db/id
:transaction/amount
{:transaction/client [:client/code]
:transaction/bank-account [:bank-account/code]
:transaction/recommended-vendor [:vendor/name :db/id]
:transaction/recommended-account [:account/name :account/numeric-code :db/id]}
:transaction/account-confidence
:transaction/date
])
(defn transaction-recommendations [identity selected-client & {:keys [after]}]
(let [visible-clients (visible-clients identity)]
(->>
(dc/q '[:find (pull ?t pull-expr)
:in $ [?c ...] pull-expr
:where [?t :transaction/recommended-account]
[?t :transaction/client ?c]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/vendor])]
(dc/db conn)
(if selected-client
[selected-client]
visible-clients)
pull-expr)
(map first)
(sort-by :transaction/date)
(reverse)
(drop-while (fn [x]
(if after
(not= (Long/parseLong after) (:db/id x))
false)))
(#(if after
(drop 1 %)
%))
(take 10)
(into []))))
(defn transaction-row [r & {:keys [hide-actions? class last?]}]
[:tr (cond-> {:class class}
last? (assoc :hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-rows
:after (:db/id r))
:hx-trigger "intersect once"
:hx-indicator "#insight-table"
:hx-swap "afterend"))
[:td {:style {:width "8em"}}(:client/code (:transaction/client r))]
[:td {:style {:width "10em"}} (:bank-account/code (:transaction/bank-account r))]
[:td {:style {:width "12em"}} (some-> (:transaction/date r) coerce/to-date-time (atime/unparse-local atime/normal-date))]
[:td {:style {:width "30em" :max-width "30em"}} (str (:transaction/description-original r))]
[:td {:style {:width "10em"}}
(if (> (:transaction/amount r) 0.0 )
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])]
[:td {:style {:width "12em"}}
[:div [:div.tag (:vendor/name (:transaction/recommended-vendor r))]]
[:div [:div.tag (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r)))]]
[:div [:div.tag
{:class (cond
(> (:transaction/account-confidence r) 0.90)
"is-success is-light"
(> (:transaction/account-confidence r) 0.80)
"is-info is-light"
:else
"is-warning is-light")}
(str "%" (Math/round (* 100.0 (:transaction/account-confidence r))))]]]
[:td
(when-not hide-actions?
[:div.buttons
[:button.button {:hx-post (bidi/path-for ssr-routes/only-routes
:transaction-insight-approve
:transaction-id (:db/id r))
:hx-target "closest tr"}
[:i.fa.fa-thumbs-up ]]
[:button.button
[:i.fa.fa-thumbs-down ]]
[:a.button {:hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-explain
:transaction-id (:db/id r))
:hx-target "#modal-holder"
:hx-swap "beforeend"}
[:i.fa.fa-question ]]])]])
(defn approve [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(html-response (transaction-row
(dc/pull (dc/db conn)
pull-expr
(Long/parseLong transaction-id))
:auto-remove? true
:hide-actions? true
:class "live-added")))
(defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(let [r (dc/pull (dc/db conn)
pull-expr
(Long/parseLong transaction-id))
similar (->> {:query '[:find ?date ?do ?amt
:in $ ?tr
:where
[(iol-ion.query/recent-date 180) ?start-date]
[?tr :transaction/client ?c]
[?tr :transaction/recommended-account ?a ]
[?tr :transaction/recommended-vendor ?v ]
[?t2 :transaction/client ?c]
[?t2 :transaction/date ?date]
[(>= ?date ?start-date)]
[?t2 :transaction/vendor ?v]
[?t2 :transaction/accounts ?a2]
[?a2 :transaction-account/account ?a]
[?t2 :transaction/description-original ?do]
[?t2 :transaction/amount ?amt]]
:args [(dc/db conn)
(Long/parseLong transaction-id)]
:limit 5}
dc/q
sort
reverse)]
(html-response [:div.modal.is-active.wide
[:div.modal-background {"_" (hiccup/raw "on click remove <#modal-holder div/>")}]
[:div.modal-card
[:div.modal-card-head
[:h1.title "Similar transactions"]
[:div.tags
[:div.tag.is-large.is-info.is-light (:vendor/name (:transaction/recommended-vendor r))]
[:div.tag.is-large.is-info.is-light (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r)))]]]
[:div.modal-card-body
[:table.table
[:thead
[:tr
[:td "Date"]
[:td "Description"]
[:td "Amount"]]]
[:tbody
[:tr
[:th (some-> r :transaction/date coerce/to-date-time (atime/unparse-local atime/normal-date))]
[:th (-> r :transaction/description-original)]
[:th (if (> (-> r :transaction/amount) 0.0 )
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])]]
(for [[date description amt] similar]
[:tr
[:td (some-> date coerce/to-date-time (atime/unparse-local atime/normal-date))]
[:td description]
[:td (if (> amt 0.0 )
[:div.tag.is-success.is-light (str "$" (Math/round amt))]
[:div.tag.is-danger.is-light (str "$" (Math/round amt))])]])]]
]]
[:button.modal-close.is-large {"_" (hiccup/raw "on click remove <#modal-holder div/>")}]])))
(defn transaction-rows* [{:keys [selected-client identity after]}]
(let [recommendations (transaction-recommendations identity selected-client :after after)]
(if (seq recommendations)
(for [r recommendations
:let [last? (= r (last recommendations))]]
(transaction-row r :last? last?))
[:tr [:td.has-text-centered.has-text-gray {:colspan 7 }
[:i "That's the last of 'em!"]]])))
(defn transaction-rows [{:keys [session identity route-params]}]
(html-response (transaction-rows* {:selected-client (-> session :client :db/id)
:identity identity
:after (:after route-params)})))
(defn insight-table* [{:keys [selected-client identity]}]
[:div#insight-table {:hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-table
:request-method :get)
:hx-trigger "clientSelected from:body"
:hx-swap "outerHTML swap:100ms"}
[:table.table
[:thead
[:tr
[:td "Client"]
[:td "Account"]
[:td "Date"]
[:td "Description"]
[:td "Amount"]
[:td "Vendor / Account"]
[:td "action"]]]
[:tbody
(transaction-rows* {:selected-client selected-client
:identity identity})]]
[:div.container.htmx-indicator
[:div.column.is-4.is-offset-4.has-text-centered
[:div.loader.is-loading.is-active.big.is-centered]]]])
(defn insight-table [{:keys [session identity]}]
(html-response (insight-table* {:selected-client
(-> session :client :db/id)
:identity identity})))
(defn page [{:keys [identity matched-route session] :as request}]
(base-page
request
[:div
[:h1.title "Transaction Insights"]
(insight-table* {:selected-client
(-> session :client :db/id)
:identity identity})]
[:div (company-side-bar matched-route)]))

5
src/clj/auto_ap/ssr/ui.clj
View File

@@ -35,9 +35,12 @@
[:link {:rel "stylesheet", :href "https://unpkg.com/placeholder-loading/dist/css/placeholder-loading.min.css"}]
#_[:link {:rel "stylesheet", :href "https://cdn.jsdelivr.net/npm/@tarekraafat/autocomplete.js@10.2.7/dist/css/autoComplete.min.css"}]
[:script {:src "https://unpkg.com/hyperscript.org@0.9.7"}]
[:script {:src "https://unpkg.com/htmx.org@1.8.4"
[:script {:src "https://unpkg.com/@popperjs/core@2"}]
#_[:script {:src "https://unpkg.com/htmx.org@1.8.4"
:integrity "sha384-wg5Y/JwF7VxGk4zLsJEcAojRtlVp1FKKdGy1qN+OMtdq72WRvX/EdRdqg/LOhYeV"
:crossorigin= "anonymous"}]
[:script {:src "https://unpkg.com/htmx.org@1.9.0/dist/htmx.js"
:crossorigin= "anonymous"}]
[:script {:type "text/javascript", :src "https://cdn.yodlee.com/fastlink/v4/initialize.js", :async "async" }]]
[:script {:type "text/javascript", :src "https://cdn.jsdelivr.net/npm/@tarekraafat/autocomplete.js@10.2.7/dist/autoComplete.min.js"}]
[:body

5
src/cljc/auto_ap/ssr_routes.cljc
View File

@@ -6,6 +6,11 @@
#"/search/?" :admin-history-search
["/" [#"\d+" :entity-id] #"/?"] :admin-history-search
["/inspect/" [#"\d+" :entity-id] #"/?"] :admin-history-inspect}}
"transaction" {"/insights" {"" :transaction-insights
"/table" :transaction-insight-table
["/approve/" [#"\d+" :transaction-id]] {:post :transaction-insight-approve}
["/rows/" [#"\d+" :after]] {:get :transaction-insight-rows}
["/explain/" [#"\d+" :transaction-id]] {:get :transaction-insight-explain}}}
"company" {"/dropdown" :company-dropdown-contents
"/active" {:put :active-client}
"/1099" :company-1099

8
src/cljs/auto_ap/views/pages/transactions/side_bar.cljs
View File

@@ -1,6 +1,7 @@
(ns auto-ap.views.pages.transactions.side-bar
(:require
[auto-ap.routes :as routes]
[auto-ap.ssr-routes :as ssr-routes]
[auto-ap.subs :as subs]
[auto-ap.views.components.bank-account-filter
:refer [bank-account-filter]]
@@ -53,6 +54,13 @@
[:span {:class "icon icon-task-list-disable" :style {:font-size "25px"}}]
[:span {:class "name"} "Excluded"]]]
[:li.menu-item
[:a.item {:href (bidi/path-for ssr-routes/only-routes :transaction-insights)
:class [(active-when ap = :transaction-insights)]}
[:span {:class "icon icon-task-list-disable" :style {:font-size "25px"}}]
[:span {:class "name"} "Insights"]]]
]]
[:p.menu-label "Bank Account"]
[:div