From 043b72ea76c143755f442966fe92d085fa459986 Mon Sep 17 00:00:00 2001 From: Bryce Date: Sun, 13 Aug 2023 15:57:15 -0700 Subject: [PATCH] Able to find nearby transactions --- scratch-sessions/build_ml.clj | 45 +++++++- src/clj/auto_ap/ssr/transaction/insights.clj | 112 ++++++++++++------- 2 files changed, 116 insertions(+), 41 deletions(-) diff --git a/scratch-sessions/build_ml.clj b/scratch-sessions/build_ml.clj index 257b952e..23e77be3 100644 --- a/scratch-sessions/build_ml.clj +++ b/scratch-sessions/build_ml.clj @@ -8,7 +8,8 @@ (:require [datomic.api :as dc] [clojure.java.io :as io] [clojure.data.csv :as csv] - [auto-ap.datomic :refer [conn]])) + [auto-ap.datomic :refer [conn]] + [auto-ap.datomic :refer [visible-clients]])) (println "hi") @@ -18,6 +19,46 @@ +(defn write-recent-transactions-to-embed [] + (let [visible-clients (visible-clients identity)] + (->> + (dc/qseq {:query '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v + :in $ ?starting [?c ...] pull-expr + :where + + [?t :transaction/client ?c] + ;; [?t :transaction/approval-status :transaction-approval-status/unapproved] + ;; [?t :transaction/vendor] ;; should be not + [(>= ?d ?starting)] + [?t :transaction/description-original ?do] + [?t :transaction/amount ?amt] + [?t :transaction/date ?d] + [?t :transaction/bank-account ?b] + [?b :bank-account/name ?bn] + + [?b :bank-account/name ?bn] + [?b :bank-account/type ?bat] + [?bat :db/ident ?bt] + [(name ?bt) ?bt2] + ] + :args [(dc/db conn) + #inst "2021-01-01" + visible-clients + pull-expr]}) + (map first) + (sort-by :transaction/date) + (reverse) + (drop-while (fn [x] + (if after + (not= (Long/parseLong after) (:db/id x)) + false))) + (#(if after + (drop 1 %) + %)) + (take 200) + (into []))) + ) + (defn write-vendor-training-data [] (with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-training.csv")] (csv/write-csv f @@ -73,7 +114,7 @@ [?bat :db/ident ?bt] [(name ?bt) ?bt2]] (dc/db auto-ap.datomic/conn) - #inst "2022-01-01") + #inst "2020-01-01") (map (fn [[t code bn bat d date amt v a]] [t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v a])))) :quote? (constantly true)))) diff --git a/src/clj/auto_ap/ssr/transaction/insights.clj b/src/clj/auto_ap/ssr/transaction/insights.clj index 77ee295b..afe935a6 100644 --- a/src/clj/auto_ap/ssr/transaction/insights.clj +++ b/src/clj/auto_ap/ssr/transaction/insights.clj @@ -1,7 +1,7 @@ (ns auto-ap.ssr.transaction.insights (:require [auto-ap.client-routes :as client-routes] - [auto-ap.datomic :refer [conn visible-clients]] + [auto-ap.datomic :refer [conn pull-attr visible-clients]] [auto-ap.rule-matching :refer [spread-cents]] [auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr.components :as com] @@ -10,6 +10,8 @@ [auto-ap.ssr.utils :refer [html-response]] [auto-ap.time :as atime] [bidi.bidi :as bidi] + [cemerick.url :as url] + [clj-http.client :as http] [clj-time.coerce :as coerce] [datomic.api :as dc] [iol-ion.tx :refer [random-tempid]])) @@ -29,19 +31,22 @@ (let [visible-clients (visible-clients identity)] (->> (dc/qseq {:query '[:find (pull ?t pull-expr) - :in $ [?c ...] pull-expr - :where [?t :transaction/recommended-account] + :in $ ?starting [?c ...] pull-expr + :where + [?t :transaction/client ?c] [?t :transaction/approval-status :transaction-approval-status/unapproved] - (not [?t :transaction/vendor])] + ;; [?t :transaction/vendor] ;; should be not + [?t :transaction/date ?d] + [(>= ?d ?starting)] + ] :args [(dc/db conn) + (iol-ion.query/recent-date 120) (if selected-client [selected-client] visible-clients) pull-expr]}) (map first) - (sort-by :transaction/date) - (reverse) (drop-while (fn [x] (if after (not= (Long/parseLong after) (:db/id x)) @@ -49,7 +54,7 @@ (#(if after (drop 1 %) %)) - (take 10) + (take 200) (into [])))) @@ -73,9 +78,9 @@ [:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])) (com/data-grid-cell {:style {:width "12em"}} [:div.flex.gap-2.flex-wrap {:style {:width "12em"}} - (com/pill {:color :primary} (:vendor/name (:transaction/recommended-vendor r))) - (com/pill {:color :secondary} (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r)))) - (com/pill {:class (cond + #_(com/pill {:color :primary} (:vendor/name (:transaction/recommended-vendor r))) + #_(com/pill {:color :secondary} (str (:account/numeric-code (:transaction/recommended-account r)) " - " (:account/name (:transaction/recommended-account r)))) + #_(com/pill {:class (cond (> (:transaction/account-confidence r) 0.90) "is-success is-light" (> (:transaction/account-confidence r) 0.80) @@ -149,30 +154,44 @@ :hide-actions? true :class "live-removed")))) +(defn get-pinecone [transaction-id] + (-> + (http/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch" + url/url + (assoc :query {:ids transaction-id}) + str) + {:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"} + :as :json + :keywordize? false}) + :body + :vectors + ((keyword (str transaction-id))) + :values)) + +(defn get-pinecone-similarities [transaction-id] + (filter + (fn [{:keys [score]}] + (> score 0.95) + ) + (-> + (http/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query" + url/url + str) + {:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"} + :form-params {"vector" (get-pinecone transaction-id) + "topK" 100, + "includeMetadata" true + "namespace" ""} + :content-type :json + :as :json}) + :body + :matches))) + (defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}] (let [r (dc/pull (dc/db conn) pull-expr (Long/parseLong transaction-id)) - similar (->> (dc/q '[:find ?date ?do ?amt - :in $ ?tr - :where - [(iol-ion.query/recent-date 180) ?start-date] - [?tr :transaction/client ?c] - [?tr :transaction/recommended-account ?a] - [?tr :transaction/recommended-vendor ?v] - [?t2 :transaction/client ?c] - [?t2 :transaction/date ?date] - [(>= ?date ?start-date)] - [?t2 :transaction/vendor ?v] - [?t2 :transaction/accounts ?a2] - [?a2 :transaction-account/account ?a] - [?t2 :transaction/description-original ?do] - [?t2 :transaction/amount ?amt]] - (dc/db conn) - (Long/parseLong transaction-id)) - (take 5) - sort - reverse)] + similar (get-pinecone-similarities transaction-id)] (html-response (com/modal {} (com/modal-card {:style {:width "900px"}} @@ -182,21 +201,36 @@ [:tr [:td "Date"] [:td "Description"] - [:td "Amount"]]] + [:td "Amount"] + [:td "Vendor"] + [:td "Account"] + [:td "Score"]]] [:tbody [:tr [:th.text-left (some-> r :transaction/date coerce/to-date-time (atime/unparse-local atime/normal-date))] [:th.text-left (-> r :transaction/description-original)] [:th.text-left (if (> (-> r :transaction/amount) 0.0) [:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))] - [:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])]] - (for [[date description amt] similar] - [:tr - [:td (some-> date coerce/to-date-time (atime/unparse-local atime/normal-date))] - [:td description] - [:td (if (> amt 0.0) - [:div.tag.is-success.is-light (str "$" (Math/round amt))] - [:div.tag.is-danger.is-light (str "$" (Math/round amt))])]])]] + [:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))])] + [:th] + [:th] + [:th.text-left ]] + (take 10 + (for [{{:keys [amount date description vendor]} :metadata score :score id :id} similar + :let [vendor-name (:vendor/name (:transaction/vendor (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name]} ] (Long/parseLong id)))) + account-code (-> (dc/pull (dc/db conn) [{:transaction/accounts [{:transaction-account/account [:account/numeric-code]}]} ] (Long/parseLong id)) + :transaction/accounts + first + :transaction-account/account + :account/numeric-code)] + :when (or vendor-name account-code)] + [:tr + [:td (subs date 0 10)] + [:td description] + [:td amount] + [:td vendor-name] + [:td account-code] + [:td (format "%.1f%%" (* 100 (double score)))]]))]] [:div]))))) (defn transaction-rows* [{:keys [selected-client identity after]}]