notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Pregenerates recommendatios

Browse Source
This commit is contained in:
Bryce
2023-08-19 13:52:44 -07:00
parent 750a147f1c
commit 486c428917
11 changed files with 820 additions and 1042 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/clj/auto_ap/jobs/insight_outcome_recommendation.clj Normal file
View File

@@ -0,0 +1,97 @@
(ns auto-ap.jobs.insght-outcome-recommendation
(:require
[auto-ap.datomic :refer [audit-transact-batch conn]]
[auto-ap.jobs.core :refer [execute]]
[cemerick.url :as url]
[clj-http.client :as http2]
[datomic.api :as dc]))
(defn get-recent-transactions []
(->> conn
dc/db
(dc/q '[:find ?t ?c
:where
[(iol-ion.query/recent-date 90) ?start]
[?t :transaction/date ?d]
[(>= ?d ?start)]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/outcome-recommendation])
[?t :transaction/description-original ?do]
(not [(clojure.string/includes? ?do "CHECK")])
(not [(clojure.string/includes? ?do "Check")])
[?t :transaction/client ?c]])))
(defn get-pinecone [transaction-id]
(->
(http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
url/url
(assoc :query {:ids transaction-id})
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:as :json
:keywordize? false})
:body
:vectors
((keyword (str transaction-id)))
:values))
(defn get-pinecone-similarities [transaction-id]
(filter
(fn [{:keys [score]}]
(> score 0.95)
)
(->
(http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
url/url
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:form-params {"vector" (get-pinecone transaction-id)
"topK" 200,
"includeMetadata" true
"namespace" ""}
:content-type :json
:as :json})
:body
:matches)))
(defn pinecone-similarity-list [transaction-id]
(for [{{:keys [amount date description vendor]} :metadata score :score id :id} (get-pinecone-similarities transaction-id)
:let [similar-transaction (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name :db/id]
:transaction/accounts [{:transaction-account/account [:account/numeric-code :db/id]}]
:transaction/client [:db/id]}] (Long/parseLong id))]
:when (or (:transaction/vendor similar-transaction) (seq (:transaction/accounts similar-transaction)))]
(assoc similar-transaction :score score)))
(defn similar->recommendation [txs client]
(->> txs
(reduce (fn [acc t]
(-> acc
(update-in [[(:db/id (:transaction/vendor t))
(:db/id (:transaction-account/account (first (:transaction/accounts t))))]
:count]
(fnil inc 0))
(update-in [[(:db/id (:transaction/vendor t))
(:db/id (:transaction-account/account (first (:transaction/accounts t))))]
:seen-by-client?]
(fn [seen-by-client?] (or seen-by-client? (= (:db/id (:transaction/client t))
client))))))
{})
(map (fn [[k v]]
(-> k
(conj (:count v))
(conj (:seen-by-client? v)))))))
(defn get-outcome-recommendations []
(for [[transaction client] (get-recent-transactions)
:let [similarity (pinecone-similarity-list transaction)]
:when (seq similarity)]
{:db/id transaction
:transaction/outcome-recommendation
(similar->recommendation similarity client)}))
(defn make-outcome-recommendations []
(audit-transact-batch (get-outcome-recommendations) {:user/name "Outcome recommendations from pinecone"}))
(defn -main [& _]
(execute "insight-outcome-recommendation" make-outcome-recommendations))

93
src/clj/auto_ap/server.clj
View File

@@ -26,7 +26,8 @@
[mount.core :as mount]
[nrepl.server :refer [start-server]]
[ring.adapter.jetty :refer [run-jetty]]
[yang.scheduler :as scheduler])
[yang.scheduler :as scheduler]
[auto-ap.jobs.insight-outcome-recommendation :as insight-outcome-recommendation])
(:import
(org.eclipse.jetty.server.handler StatisticsHandler)
(org.eclipse.jetty.server.handler.gzip GzipHandler)))
@@ -89,70 +90,72 @@
(mount/stop))
(defn -main [& _]
(let [job (System/getenv "INTEGREAT_JOB")]
(println "JOB is" job)
(cond (= job "square-import-job")
(job-square/-main)
(let [job (System/getenv "INTEGREAT_JOB")]
(println "JOB is" job)
(cond (= job "square-import-job")
(job-square/-main)
(= job "square2-import-job")
(job-square2/-main)
(= job "square2-import-job")
(job-square2/-main)
(= job "reconcile-ledger")
(job-reconcile-ledger/-main)
(= job "reconcile-ledger")
(job-reconcile-ledger/-main)
(= job "current-balance-cache")
(job-current-balance-cache/-main)
(= job "current-balance-cache")
(job-current-balance-cache/-main)
(= job "yodlee2")
(job-yodlee2/-main)
(= job "yodlee2")
(job-yodlee2/-main)
(= job "yodlee2-accounts")
(job-yodlee2/accounts-only)
(= job "yodlee2-accounts")
(job-yodlee2/accounts-only)
(= job "plaid")
(job-plaid/-main)
(= job "plaid")
(job-plaid/-main)
(= job "intuit")
(job-intuit/-main)
(= job "intuit")
(job-intuit/-main)
(= job "vendor-usages")
(job-vendor-usages/-main)
(= job "vendor-usages")
(job-vendor-usages/-main)
(= job "import-uploaded-invoices")
(job-import-uploaded-invoices/-main)
(= job "import-uploaded-invoices")
(job-import-uploaded-invoices/-main)
(= job "sysco")
(job-sysco/-main)
(= job "sysco")
(job-sysco/-main)
(= job "close-auto-invoices")
(job-close-auto-invoices/-main)
(= job "close-auto-invoices")
(job-close-auto-invoices/-main)
(= job "ezcater-upsert")
(job-ezcater-upsert/-main)
(= job "ezcater-upsert")
(job-ezcater-upsert/-main)
(= job "register-invoice-import")
(job-register-invoice-import/-main)
(= job "register-invoice-import")
(job-register-invoice-import/-main)
(= job "load-historical-sales")
(job-load-historical-sales/-main)
(= job "load-historical-sales")
(job-load-historical-sales/-main)
(= job "bulk-journal-import")
(job-bulk-journal-import/-main)
(= job "bulk-journal-import")
(job-bulk-journal-import/-main)
(= job "restore-from-backup")
(job-restore-from-backup/-main)
(= job "restore-from-backup")
(job-restore-from-backup/-main)
(= job "insight-outcome-recommendation")
(insight-outcome-recommendation/-main)
;; (= job "export-backup")
;; (backup/-main)
(= job "ntg")
(job-ntg/-main)
(= job "ntg")
(job-ntg/-main)
:else
(do
(add-shutdown-hook! shutdown-mount)
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
(mount/start)
#_(alter-var-root #'nrepl.middleware.print/*print-fn* (constantly clojure.pprint/pprint))))))
:else
(do
(add-shutdown-hook! shutdown-mount)
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
(mount/start)
#_(alter-var-root #'nrepl.middleware.print/*print-fn* (constantly clojure.pprint/pprint))))))

1
src/clj/auto_ap/ssr/components.clj
View File

@@ -38,6 +38,7 @@
(def page page/page-)
(def pill tags/pill-)
(def badge tags/badge-)
(def data-grid data-grid/data-grid-)
(def data-grid-header data-grid/header-)

3
src/clj/auto_ap/ssr/components/tags.clj
View File

@@ -18,3 +18,6 @@
(= :red (:color params))
(update :class str " bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300"))]
children))
(defn badge- [params & children]
[:div {:class "absolute inline-flex items-center justify-center w-6 h-6 text-xs font-bold text-white bg-red-500 border-2 border-white rounded-full -top-2 -right-2 dark:border-gray-900"} children])

2
src/clj/auto_ap/ssr/core.clj
View File

@@ -49,7 +49,7 @@
:transaction-insights (wrap-client-redirect-unauthenticated (wrap-admin insights/page))
:transaction-insight-table (wrap-client-redirect-unauthenticated (wrap-admin insights/insight-table))
:transaction-insight-rows (wrap-client-redirect-unauthenticated (wrap-admin insights/transaction-rows))
:transaction-insight-approve (wrap-client-redirect-unauthenticated (wrap-admin insights/approve))
:transaction-insight-code (wrap-client-redirect-unauthenticated (wrap-admin insights/code))
:transaction-insight-disapprove (wrap-client-redirect-unauthenticated (wrap-admin insights/disapprove))
:transaction-insight-explain (wrap-client-redirect-unauthenticated (wrap-admin insights/explain))
:admin-ezcater-xls (wrap-client-redirect-unauthenticated (wrap-admin ezcater-xls/page))

192
src/clj/auto_ap/ssr/transaction/insights.clj
View File

@@ -14,18 +14,18 @@
[clj-http.client :as http]
[clj-time.coerce :as coerce]
[datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]]))
[iol-ion.tx :refer [random-tempid]]
[hiccup2.core :as hiccup]))
(def pull-expr [:transaction/description-original
:db/id
:transaction/outcome-recommendation
:transaction/amount
{:transaction/client [:client/code]
:transaction/bank-account [:bank-account/code]
:transaction/recommended-vendor [:vendor/name :db/id]
:transaction/recommended-account [:account/name :account/numeric-code :db/id]}
:transaction/bank-account [:bank-account/code]}
:transaction/account-confidence
:transaction/date
])
:transaction/date])
(defn transaction-recommendations [identity selected-client & {:keys [after]}]
(let [visible-clients (visible-clients identity)]
@@ -33,13 +33,13 @@
(dc/qseq {:query '[:find (pull ?t pull-expr)
:in $ ?starting [?c ...] pull-expr
:where
[?t :transaction/outcome-recommendation]
[?t :transaction/client ?c]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
;; [?t :transaction/vendor] ;; should be not
[?t :transaction/date ?d]
[(>= ?d ?starting)]
]
[(>= ?d ?starting)]]
:args [(dc/db conn)
(iol-ion.query/recent-date 120)
(if selected-client
@@ -54,6 +54,20 @@
(#(if after
(drop 1 %)
%))
(map (fn [tx]
(update tx :transaction/outcome-recommendation
(fn [ors]
(map
(fn [[v a c s]]
{:vendor (dc/pull (dc/db conn)
[:vendor/name :db/id]
v)
:account (dc/pull (dc/db conn)
[:account/name :db/id]
a)
:count c
:seen-by-client? s})
ors)))))
(take 50)
(into []))))
@@ -108,97 +122,94 @@
:score score}))
(defn transaction-row [r & {:keys [hide-actions? class last?]}]
(let [simality-list (try (pinecone-similarity-list (:db/id r)) (catch Exception e
[]))]
(when (seq simality-list)
(com/data-grid-row
(cond-> {:class class}
last? (assoc :hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-rows
:after (:db/id r))
:hx-trigger "intersect once"
:hx-indicator "#insight-table"
:hx-swap "afterend"))
(com/data-grid-cell {} (:client/code (:transaction/client r)))
(com/data-grid-cell {} (:bank-account/code (:transaction/bank-account r)))
(com/data-grid-cell {} (some-> (:transaction/date r) coerce/to-date-time (atime/unparse-local atime/normal-date)))
(defn transaction-row [r & {:keys [hide-actions? class last?] hs "_"}]
(com/data-grid-row
(cond-> {:class class}
hs (assoc "_" hs)
(com/data-grid-cell {} (str (:transaction/description-original r)))
(com/data-grid-cell {}
(if (> (:transaction/amount r) 0.0)
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))]))
(com/data-grid-cell {:style {:width "12em"}}
[:div.flex.gap-2.flex-wrap {:style {:width "12em"}}
(when-let [vendor-name (:vendor-name (first simality-list))]
(com/pill {:color :primary} vendor-name))
(when-let [numeric-code (:numeric-code (first simality-list))]
(com/pill {:color :secondary} numeric-code))
#_(com/pill {:class (cond
(> (:transaction/account-confidence r) 0.90)
"is-success is-light"
(> (:transaction/account-confidence r) 0.80)
"is-info is-light"
last? (assoc :hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-rows
:after (:db/id r))
:hx-trigger "intersect once"
:hx-indicator "#insight-table"
:hx-swap "afterend"))
(com/data-grid-cell {} (:client/code (:transaction/client r)))
(com/data-grid-cell {} (:bank-account/code (:transaction/bank-account r)))
(com/data-grid-cell {} (some-> (:transaction/date r) coerce/to-date-time (atime/unparse-local atime/normal-date)))
:else
"is-warning is-light")} (str "%" (Math/round (* 100.0 (:transaction/account-confidence r)))))])
(com/data-grid-right-stack-cell {}
(when-not hide-actions?
[:form.flex.gap-2
[:input {:type :hidden :name "id" :value (:db/id r)}]
(com/icon-button {:hx-post (bidi/path-for ssr-routes/only-routes
:transaction-insight-approve
:transaction-id (:db/id r))
:hx-target "closest tr"
:color :primary-light}
svg/thumbs-up)
(com/icon-button {:hx-delete (bidi/path-for ssr-routes/only-routes
:transaction-insight-disapprove
:transaction-id (:db/id r))
:hx-target "closest tr"
:color :danger-light}
svg/thumbs-down)
(com/icon-button {:hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-explain
:transaction-id (:db/id r))
:hx-target "#modal-holder"
:hx-swap "outerHTML"}
svg/question)]))))))
(com/data-grid-cell {} (str (:transaction/description-original r)))
(com/data-grid-cell {}
(if (> (:transaction/amount r) 0.0)
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))]))
(defn approve [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(com/data-grid-right-stack-cell {}
(when-not hide-actions?
[:div.flex.gap-2.flex-col {:style {:width "25em"}}
(for [or (sort-by (comp - :count)
(:transaction/outcome-recommendation r))]
[:form {:hx-post (bidi/path-for ssr-routes/only-routes
:transaction-insight-code
:transaction-id (:db/id r))
:hx-target "closest tr"}
(when-let [vendor-id (:db/id (:vendor or))]
[:input {:type :hidden :value vendor-id :name "vendor"}])
(when-let [account-id (:db/id (:account or))]
[:input {:type :hidden :value account-id :name "account"}])
(com/button {:color (if (:seen-by-client? or)
:primary
:secondary)
:style {:position "relative"}}
(:vendor/name (:vendor or))
(when (:vendor/name (:vendor or))
" | ")
(:account/name (:account or))
(com/badge {:color :secondary}
(:count or)))])
(com/icon-button {:hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-explain
:transaction-id (:db/id r))
:hx-target "#modal-holder"
:hx-swap "outerHTML"}
svg/question)]))))
(defn code [{:keys [identity session] {:keys [transaction-id]} :route-params {:strs [vendor account]} :form-params}]
(let [approval-details (dc/pull (dc/db conn) [{:transaction/recommended-account [:account/location :db/id]}
:transaction/recommended-vendor
:transaction/amount
:db/id
{:transaction/client [:client/locations]} ]
{:transaction/client [:client/locations]}]
(cond-> transaction-id
string? (Long/parseLong)))
account (dc/pull (dc/db conn) [:account/location :db/id]
(cond-> account
string? (Long/parseLong)))
cents-to-distribute (int (Math/round (Math/abs (* (:transaction/amount approval-details) 100))))
valid-locations (or
(some-> approval-details :transaction/recommended-account :account/location vector)
(->> approval-details
:transaction/client
:client/locations))
(some-> approval-details :transaction/recommended-account :account/location vector)
(->> approval-details
:transaction/client
:client/locations))
updated-transaction [:upsert-transaction {:db/id (:db/id approval-details)
:transaction/approval-status :transaction-approval-status/approved
:transaction/vendor (:transaction/recommended-vendor approval-details)
:transaction/vendor (some-> vendor not-empty (Long/parseLong))
:transaction/accounts (->> valid-locations
(map
(fn [cents location]
{:db/id (random-tempid)
:transaction-account/account (-> approval-details :transaction/recommended-account :db/id)
:transaction-account/amount (* 0.01 cents)
:transaction-account/location location})
(spread-cents cents-to-distribute (count valid-locations))))}]]
(map
(fn [cents location]
{:db/id (random-tempid)
:transaction-account/account (-> account :db/id)
:transaction-account/amount (* 0.01 cents)
:transaction-account/location location})
(spread-cents cents-to-distribute (count valid-locations))))}]]
@(dc/transact conn [updated-transaction])
(html-response (transaction-row
(dc/pull (dc/db conn)
pull-expr
(Long/parseLong transaction-id))
:auto-remove? true
:hide-actions? true
:class "live-added"))))
(dc/pull (dc/db conn)
pull-expr
(Long/parseLong transaction-id))
:auto-remove? true
:hide-actions? true
:class "live-added"))))
(defn disapprove [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(let [transaction-id (cond-> transaction-id string? (Long/parseLong))]
@@ -207,7 +218,8 @@
(dc/pull (dc/db conn) pull-expr transaction-id)
:auto-remove? true
:hide-actions? true
:class "live-removed"))))
:class "live-removed"
"_" (hiccup/raw "init transition opacity to 0 then remove me")))))
(defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(let [r (dc/pull (dc/db conn)
pull-expr
@@ -272,13 +284,11 @@
:rows (for [r recommendations
:let [last? (= r (last recommendations))]]
(transaction-row r :last? last?))
:headers [(com/data-grid-header {} "Client")
(com/data-grid-header {} "Account")
(com/data-grid-header {} "Date")
:headers [(com/data-grid-header {:style {:width "10em"}} "Client")
(com/data-grid-header {:style {:width "15em"}} "Account")
(com/data-grid-header {:style {:width "8em"}} "Date")
(com/data-grid-header {} "Description")
(com/data-grid-header {} "Amount")
(com/data-grid-header {:style {:width "4em"}} "Vendor / Account")
(com/data-grid-header {})]})))
(com/data-grid-header {:style {:width "8em"}} "Amount")]})))
(defn insight-table [{:keys [session identity]}]
(html-response (insight-table* {:selected-client

9
src/cljc/auto_ap/ssr_routes.cljc
View File

@@ -15,7 +15,7 @@
"/ezcater-xls" :admin-ezcater-xls}
"transaction" {"/insights" {"" :transaction-insights
"/table" :transaction-insight-table
["/approve/" [#"\d+" :transaction-id]] {:post :transaction-insight-approve}
["/code/" [#"\d+" :transaction-id]] {:post :transaction-insight-code}
["/disapprove/" [#"\d+" :transaction-id]] {:delete :transaction-insight-disapprove}
["/rows/" [#"\d+" :after]] {:get :transaction-insight-rows}
["/explain/" [#"\d+" :transaction-id]] {:get :transaction-insight-explain}}}
@@ -36,11 +36,10 @@
"/reauthenticate" {:put :company-yodlee-provider-account-reauthenticate}}
"/plaid" {"" {:get :company-plaid}
"/table" {:get :company-plaid-table}
"/table" {:get :company-plaid-table}
"/link" {:post :company-plaid-link}
#_#_"/fastlink" {:get :company-yodlee-fastlink-dialog}
#_#_"/refresh" {:put :company-yodlee-provider-account-refresh}}
}})
#_#_"/fastlink" {:get :company-yodlee-fastlink-dialog}
#_#_"/refresh" {:put :company-yodlee-provider-account-refresh}}}})
(def only-routes ["/" routes])