notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Able to find nearby transactions

Browse Source
This commit is contained in:
Bryce
2023-08-13 15:57:15 -07:00
parent 83ab75a4de
commit 043b72ea76
2 changed files with 116 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
scratch-sessions/build_ml.clj
View File

@@ -8,7 +8,8 @@
(:require [datomic.api :as dc] (:require [datomic.api :as dc]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[auto-ap.datomic :refer [conn]])) [auto-ap.datomic :refer [conn]]
[auto-ap.datomic :refer [visible-clients]]))
(println "hi") (println "hi")
@@ -18,6 +19,46 @@
(defn write-recent-transactions-to-embed []
(let [visible-clients (visible-clients identity)]
(->>
(dc/qseq {:query '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v
:in $ ?starting [?c ...] pull-expr
:where
[?t :transaction/client ?c]
;; [?t :transaction/approval-status :transaction-approval-status/unapproved]
;; [?t :transaction/vendor] ;; should be not
[(>= ?d ?starting)]
[?t :transaction/description-original ?do]
[?t :transaction/amount ?amt]
[?t :transaction/date ?d]
[?t :transaction/bank-account ?b]
[?b :bank-account/name ?bn]
[?b :bank-account/name ?bn]
[?b :bank-account/type ?bat]
[?bat :db/ident ?bt]
[(name ?bt) ?bt2]
]
:args [(dc/db conn)
#inst "2021-01-01"
visible-clients
pull-expr]})
(map first)
(sort-by :transaction/date)
(reverse)
(drop-while (fn [x]
(if after
(not= (Long/parseLong after) (:db/id x))
false)))
(#(if after
(drop 1 %)
%))
(take 200)
(into [])))
)
(defn write-vendor-training-data [] (defn write-vendor-training-data []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-training.csv")] (with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-training.csv")]
(csv/write-csv f (csv/write-csv f
@@ -73,7 +114,7 @@
[?bat :db/ident ?bt] [?bat :db/ident ?bt]
[(name ?bt) ?bt2]] [(name ?bt) ?bt2]]
(dc/db auto-ap.datomic/conn) (dc/db auto-ap.datomic/conn)
#inst "2022-01-01") #inst "2020-01-01")
(map (fn [[t code bn bat d date amt v a]] (map (fn [[t code bn bat d date amt v a]]
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v a])))) [t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v a]))))
:quote? (constantly true)))) :quote? (constantly true))))

108
src/clj/auto_ap/ssr/transaction/insights.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.ssr.transaction.insights (ns auto-ap.ssr.transaction.insights
(:require (:require
[auto-ap.client-routes :as client-routes] [auto-ap.client-routes :as client-routes]
[auto-ap.datomic :refer [conn visible-clients]] [auto-ap.datomic :refer [conn pull-attr visible-clients]]
[auto-ap.rule-matching :refer [spread-cents]] [auto-ap.rule-matching :refer [spread-cents]]
[auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.components :as com] [auto-ap.ssr.components :as com]
@@ -10,6 +10,8 @@
[auto-ap.ssr.utils :refer [html-response]] [auto-ap.ssr.utils :refer [html-response]]
[auto-ap.time :as atime] [auto-ap.time :as atime]
[bidi.bidi :as bidi] [bidi.bidi :as bidi]
[cemerick.url :as url]
[clj-http.client :as http]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[datomic.api :as dc] [datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]])) [iol-ion.tx :refer [random-tempid]]))
@@ -29,19 +31,22 @@
(let [visible-clients (visible-clients identity)] (let [visible-clients (visible-clients identity)]
(->> (->>
(dc/qseq {:query '[:find (pull ?t pull-expr) (dc/qseq {:query '[:find (pull ?t pull-expr)
:in $ [?c ...] pull-expr :in $ ?starting [?c ...] pull-expr
:where [?t :transaction/recommended-account] :where
[?t :transaction/client ?c] [?t :transaction/client ?c]
[?t :transaction/approval-status :transaction-approval-status/unapproved] [?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/vendor])] ;; [?t :transaction/vendor] ;; should be not
[?t :transaction/date ?d]
[(>= ?d ?starting)]
]
:args [(dc/db conn) :args [(dc/db conn)
(iol-ion.query/recent-date 120)
(if selected-client (if selected-client
[selected-client] [selected-client]
visible-clients) visible-clients)
pull-expr]}) pull-expr]})
(map first) (map first)
(sort-by :transaction/date)
(reverse)
(drop-while (fn [x] (drop-while (fn [x]
(if after (if after
(not= (Long/parseLong after) (:db/id x)) (not= (Long/parseLong after) (:db/id x))
@@ -49,7 +54,7 @@
(#(if after (#(if after
(drop 1 %) (drop 1 %)
%)) %))
(take 10) (take 200)
(into [])))) (into []))))
@@ -73,9 +78,9 @@
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])) [:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))]))
(com/data-grid-cell {:style {:width "12em"}} (com/data-grid-cell {:style {:width "12em"}}
[:div.flex.gap-2.flex-wrap {:style {:width "12em"}} [:div.flex.gap-2.flex-wrap {:style {:width "12em"}}
(com/pill {:color :primary} (:vendor/name (:transaction/recommended-vendor r))) #_(com/pill {:color :primary} (:vendor/name (:transaction/recommended-vendor r)))
(com/pill {:color :secondary} (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r)))) #_(com/pill {:color :secondary} (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r))))
(com/pill {:class (cond #_(com/pill {:class (cond
(> (:transaction/account-confidence r) 0.90) (> (:transaction/account-confidence r) 0.90)
"is-success is-light" "is-success is-light"
(> (:transaction/account-confidence r) 0.80) (> (:transaction/account-confidence r) 0.80)
@@ -149,30 +154,44 @@
:hide-actions? true :hide-actions? true
:class "live-removed")))) :class "live-removed"))))
(defn get-pinecone [transaction-id]
(->
(http/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
url/url
(assoc :query {:ids transaction-id})
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:as :json
:keywordize? false})
:body
:vectors
((keyword (str transaction-id)))
:values))
(defn get-pinecone-similarities [transaction-id]
(filter
(fn [{:keys [score]}]
(> score 0.95)
)
(->
(http/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
url/url
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:form-params {"vector" (get-pinecone transaction-id)
"topK" 100,
"includeMetadata" true
"namespace" ""}
:content-type :json
:as :json})
:body
:matches)))
(defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}] (defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(let [r (dc/pull (dc/db conn) (let [r (dc/pull (dc/db conn)
pull-expr pull-expr
(Long/parseLong transaction-id)) (Long/parseLong transaction-id))
similar (->> (dc/q '[:find ?date ?do ?amt similar (get-pinecone-similarities transaction-id)]
:in $ ?tr
:where
[(iol-ion.query/recent-date 180) ?start-date]
[?tr :transaction/client ?c]
[?tr :transaction/recommended-account ?a]
[?tr :transaction/recommended-vendor ?v]
[?t2 :transaction/client ?c]
[?t2 :transaction/date ?date]
[(>= ?date ?start-date)]
[?t2 :transaction/vendor ?v]
[?t2 :transaction/accounts ?a2]
[?a2 :transaction-account/account ?a]
[?t2 :transaction/description-original ?do]
[?t2 :transaction/amount ?amt]]
(dc/db conn)
(Long/parseLong transaction-id))
(take 5)
sort
reverse)]
(html-response (html-response
(com/modal {} (com/modal {}
(com/modal-card {:style {:width "900px"}} (com/modal-card {:style {:width "900px"}}
@@ -182,21 +201,36 @@
[:tr [:tr
[:td "Date"] [:td "Date"]
[:td "Description"] [:td "Description"]
[:td "Amount"]]] [:td "Amount"]
[:td "Vendor"]
[:td "Account"]
[:td "Score"]]]
[:tbody [:tbody
[:tr [:tr
[:th.text-left (some-> r :transaction/date coerce/to-date-time (atime/unparse-local atime/normal-date))] [:th.text-left (some-> r :transaction/date coerce/to-date-time (atime/unparse-local atime/normal-date))]
[:th.text-left (-> r :transaction/description-original)] [:th.text-left (-> r :transaction/description-original)]
[:th.text-left (if (> (-> r :transaction/amount) 0.0) [:th.text-left (if (> (-> r :transaction/amount) 0.0)
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))] [:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])]] [:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])]
(for [[date description amt] similar] [:th]
[:th]
[:th.text-left ]]
(take 10
(for [{{:keys [amount date description vendor]} :metadata score :score id :id} similar
:let [vendor-name (:vendor/name (:transaction/vendor (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name]} ] (Long/parseLong id))))
account-code (-> (dc/pull (dc/db conn) [{:transaction/accounts [{:transaction-account/account [:account/numeric-code]}]} ] (Long/parseLong id))
:transaction/accounts
first
:transaction-account/account
:account/numeric-code)]
:when (or vendor-name account-code)]
[:tr [:tr
[:td (some-> date coerce/to-date-time (atime/unparse-local atime/normal-date))] [:td (subs date 0 10)]
[:td description] [:td description]
[:td (if (> amt 0.0) [:td amount]
[:div.tag.is-success.is-light (str "$" (Math/round amt))] [:td vendor-name]
[:div.tag.is-danger.is-light (str "$" (Math/round amt))])]])]] [:td account-code]
[:td (format "%.1f%%" (* 100 (double score)))]]))]]
[:div]))))) [:div])))))
(defn transaction-rows* [{:keys [selected-client identity after]}] (defn transaction-rows* [{:keys [selected-client identity after]}]