notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pregenerates recommendatios

Browse Source
This commit is contained in:
Bryce
2023-08-19 13:52:44 -07:00
parent 750a147f1c
commit 486c428917
11 changed files with 820 additions and 1042 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
resources/public/output.css
View File

@@ -1196,6 +1196,10 @@ input:checked + .toggle-bg {
margin-left: 0.75rem;
}
.ml-auto {
margin-left: auto;
}
.mr-10 {
margin-right: 2.5rem;
}
@@ -1236,10 +1240,6 @@ input:checked + .toggle-bg {
margin-top: 1.25rem;
}
.ml-auto {
margin-left: auto;
}
.block {
display: block;
}
@@ -1376,6 +1376,10 @@ input:checked + .toggle-bg {
max-width: 32rem;
}
.max-w-md {
max-width: 28rem;
}
.max-w-screen-2xl {
max-width: 1536px;
}
@@ -1384,18 +1388,6 @@ input:checked + .toggle-bg {
max-width: 1024px;
}
.max-w-sm {
max-width: 24rem;
}
.max-w-md {
max-width: 28rem;
}
.max-w-xl {
max-width: 36rem;
}
.flex-1 {
flex: 1 1 0%;
}
@@ -1412,18 +1404,10 @@ input:checked + .toggle-bg {
flex-shrink: 1;
}
.basis-1\/2 {
flex-basis: 50%;
}
.basis-1\/4 {
flex-basis: 25%;
}
.basis-1\/3 {
flex-basis: 33.333333%;
}
.-translate-x-full {
--tw-translate-x: -100%;
transform: translate(var(--tw-translate-x), var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));
@@ -1524,10 +1508,6 @@ input:checked + .toggle-bg {
flex-wrap: wrap;
}
.content-center {
align-content: center;
}
.items-start {
align-items: flex-start;
}
@@ -1576,11 +1556,6 @@ input:checked + .toggle-bg {
gap: 2rem;
}
.gap-x-2 {
-moz-column-gap: 0.5rem;
column-gap: 0.5rem;
}
.gap-x-4 {
-moz-column-gap: 1rem;
column-gap: 1rem;
@@ -1645,10 +1620,6 @@ input:checked + .toggle-bg {
border-color: rgb(243 244 246 / var(--tw-divide-opacity));
}
.justify-self-end {
justify-self: end;
}
.overflow-auto {
overflow: auto;
}
@@ -1718,10 +1689,6 @@ input:checked + .toggle-bg {
border-width: 2px;
}
.border-4 {
border-width: 4px;
}
.border-b {
border-bottom-width: 1px;
}
@@ -1793,9 +1760,9 @@ input:checked + .toggle-bg {
background-color: rgb(153 215 247 / var(--tw-bg-opacity));
}
.bg-blue-300 {
.bg-blue-50 {
--tw-bg-opacity: 1;
background-color: rgb(102 196 242 / var(--tw-bg-opacity));
background-color: rgb(230 245 253 / var(--tw-bg-opacity));
}
.bg-blue-500 {
@@ -1873,6 +1840,11 @@ input:checked + .toggle-bg {
background-color: rgb(255 230 230 / var(--tw-bg-opacity));
}
.bg-red-500 {
--tw-bg-opacity: 1;
background-color: rgb(255 3 3 / var(--tw-bg-opacity));
}
.bg-white {
--tw-bg-opacity: 1;
background-color: rgb(255 255 255 / var(--tw-bg-opacity));
@@ -1887,11 +1859,6 @@ input:checked + .toggle-bg {
background-color: rgb(253 246 178 / var(--tw-bg-opacity));
}
.bg-blue-50 {
--tw-bg-opacity: 1;
background-color: rgb(230 245 253 / var(--tw-bg-opacity));
}
.bg-opacity-50 {
--tw-bg-opacity: 0.5;
}
@@ -2106,6 +2073,11 @@ input:checked + .toggle-bg {
line-height: 1.25;
}
.text-blue-400 {
--tw-text-opacity: 1;
color: rgb(51 176 238 / var(--tw-text-opacity));
}
.text-blue-600 {
--tw-text-opacity: 1;
color: rgb(0 125 187 / var(--tw-text-opacity));
@@ -2176,11 +2148,6 @@ input:checked + .toggle-bg {
color: rgb(114 59 19 / var(--tw-text-opacity));
}
.text-blue-400 {
--tw-text-opacity: 1;
color: rgb(51 176 238 / var(--tw-text-opacity));
}
.underline {
text-decoration-line: underline;
}
@@ -2707,6 +2674,11 @@ input:checked + .toggle-bg {
color: rgb(102 196 242 / var(--tw-text-opacity));
}
:is(.dark .dark\:text-blue-400) {
--tw-text-opacity: 1;
color: rgb(51 176 238 / var(--tw-text-opacity));
}
:is(.dark .dark\:text-blue-500) {
--tw-text-opacity: 1;
color: rgb(0 156 234 / var(--tw-text-opacity));
@@ -2767,11 +2739,6 @@ input:checked + .toggle-bg {
color: rgb(250 202 21 / var(--tw-text-opacity));
}
:is(.dark .dark\:text-blue-400) {
--tw-text-opacity: 1;
color: rgb(51 176 238 / var(--tw-text-opacity));
}
:is(.dark .dark\:placeholder-gray-400)::-moz-placeholder {
--tw-placeholder-opacity: 1;
color: rgb(156 163 175 / var(--tw-placeholder-opacity));

1276
resources/schema.edn
View File

File diff suppressed because it is too large Load Diff

93
scratch-sessions/investigate.repl Normal file
View File

@@ -0,0 +1,93 @@
;; This buffer is for Clojure experiments and evaluation.
;; Press C-j to evaluate the last expression.
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
(def is-set (set (map first (dc/q '[:find ?ext
:where
[?so :sales-order/client [:client/code "N-30001"]]
[?so :sales-order/date ?d]
[(>= ?d #inst "2023-06-03T00:00:00-08:00")]
[(<= ?d #inst "2023-06-06T00:00:00-08:00")]
[(iol-ion.query/excel-date ?d) ?d4]
[(= ?d4 "06/04/2023")]
[?so :sales-order/external-id ?ext]
]
(dc/db conn)))))
(def sales
(first (for [n (get-square-clients "N-30001")
l (:client/square-locations n)
:when (:square-location/client-location l)]
(->> @(daily-results n l (coerce/to-date-time #inst "2023-06-03T00:00:00-08:00") (coerce/to-date-time #inst "2023-06-06T00:00:00-08:00"))
(filter #(= "06/04/2023" (some->> %
:sales-order/date
iol-ion.query/excel-date)))
))))
(->> sales
(filter (fn [o]
(seq (filter (comp #{"Desserts"} :order-line-item/category )
(:sales-order/line-items o)))))
)
(->> sales
(mapcat :sales-order/line-items)
(filter (comp #{"Loukoumades"} :order-line-item/item-name ))
count
)
(->> sales
(filter (fn [o]
(seq (filter (comp #{"Loukoumades"} :order-line-item/item-name )
(:sales-order/line-items o)))))
(map :sales-order/date)
)
(->> sales
(map :sales-order/date))
(def should-be-set *1)
(count should-be-set)
(def should-not-be (set/difference is-set should-be-set))
(first should-not-be)
(first (for [n (get-square-clients "N-30001")
l (:client/square-locations n)
:when (:square-location/client-location l)]
(get-order n l "qfAWtK0NJJ8679nx6dVuy0UzbNHZY")
#_(get-payment n "5EIEhiyv8c7ivU7sDQckAcREmURZY")))
(count should-be-set)
(get-payment )
(seq (dc/q '[:find ?d4 (sum ?total) (sum ?tax) (sum ?tip) (sum ?service-charge) (sum ?discount) (sum ?returns)
:with ?s
:in $
:where
[(ground (iol-ion.query/recent-date)) ?min-d]
[(ground #inst "2040-01-01") ?max-d]
[?c :client/code "N-30001"]
[(iol-ion.query/sales-orders-in-range $ ?c ?min-d ?max-d) [?s ...]]
[?s :sales-order/date ?d]
[?s :sales-order/total ?total]
[?s :sales-order/tax ?tax]
[?s :sales-order/tip ?tip]
[?s :sales-order/service-charge ?service-charge]
[?s :sales-order/returns ?returns]
[?s :sales-order/discount ?discount]
[(iol-ion.query/excel-date ?d) ?d4]
]
(dc/db conn)))

97
src/clj/auto_ap/jobs/insight_outcome_recommendation.clj Normal file
View File

@@ -0,0 +1,97 @@
(ns auto-ap.jobs.insght-outcome-recommendation
(:require
[auto-ap.datomic :refer [audit-transact-batch conn]]
[auto-ap.jobs.core :refer [execute]]
[cemerick.url :as url]
[clj-http.client :as http2]
[datomic.api :as dc]))
(defn get-recent-transactions []
(->> conn
dc/db
(dc/q '[:find ?t ?c
:where
[(iol-ion.query/recent-date 90) ?start]
[?t :transaction/date ?d]
[(>= ?d ?start)]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/outcome-recommendation])
[?t :transaction/description-original ?do]
(not [(clojure.string/includes? ?do "CHECK")])
(not [(clojure.string/includes? ?do "Check")])
[?t :transaction/client ?c]])))
(defn get-pinecone [transaction-id]
(->
(http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
url/url
(assoc :query {:ids transaction-id})
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:as :json
:keywordize? false})
:body
:vectors
((keyword (str transaction-id)))
:values))
(defn get-pinecone-similarities [transaction-id]
(filter
(fn [{:keys [score]}]
(> score 0.95)
)
(->
(http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
url/url
str)
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
:form-params {"vector" (get-pinecone transaction-id)
"topK" 200,
"includeMetadata" true
"namespace" ""}
:content-type :json
:as :json})
:body
:matches)))
(defn pinecone-similarity-list [transaction-id]
(for [{{:keys [amount date description vendor]} :metadata score :score id :id} (get-pinecone-similarities transaction-id)
:let [similar-transaction (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name :db/id]
:transaction/accounts [{:transaction-account/account [:account/numeric-code :db/id]}]
:transaction/client [:db/id]}] (Long/parseLong id))]
:when (or (:transaction/vendor similar-transaction) (seq (:transaction/accounts similar-transaction)))]
(assoc similar-transaction :score score)))
(defn similar->recommendation [txs client]
(->> txs
(reduce (fn [acc t]
(-> acc
(update-in [[(:db/id (:transaction/vendor t))
(:db/id (:transaction-account/account (first (:transaction/accounts t))))]
:count]
(fnil inc 0))
(update-in [[(:db/id (:transaction/vendor t))
(:db/id (:transaction-account/account (first (:transaction/accounts t))))]
:seen-by-client?]
(fn [seen-by-client?] (or seen-by-client? (= (:db/id (:transaction/client t))
client))))))
{})
(map (fn [[k v]]
(-> k
(conj (:count v))
(conj (:seen-by-client? v)))))))
(defn get-outcome-recommendations []
(for [[transaction client] (get-recent-transactions)
:let [similarity (pinecone-similarity-list transaction)]
:when (seq similarity)]
{:db/id transaction
:transaction/outcome-recommendation
(similar->recommendation similarity client)}))
(defn make-outcome-recommendations []
(audit-transact-batch (get-outcome-recommendations) {:user/name "Outcome recommendations from pinecone"}))
(defn -main [& _]
(execute "insight-outcome-recommendation" make-outcome-recommendations))

7
src/clj/auto_ap/server.clj
View File

@@ -26,7 +26,8 @@
[mount.core :as mount]
[nrepl.server :refer [start-server]]
[ring.adapter.jetty :refer [run-jetty]]
[yang.scheduler :as scheduler])
[yang.scheduler :as scheduler]
[auto-ap.jobs.insight-outcome-recommendation :as insight-outcome-recommendation])
(:import
(org.eclipse.jetty.server.handler StatisticsHandler)
(org.eclipse.jetty.server.handler.gzip GzipHandler)))
@@ -130,7 +131,6 @@
(= job "ezcater-upsert")
(job-ezcater-upsert/-main)
(= job "register-invoice-import")
(job-register-invoice-import/-main)
@@ -143,6 +143,9 @@
(= job "restore-from-backup")
(job-restore-from-backup/-main)
(= job "insight-outcome-recommendation")
(insight-outcome-recommendation/-main)
;; (= job "export-backup")
;; (backup/-main)

1
src/clj/auto_ap/ssr/components.clj
View File

@@ -38,6 +38,7 @@
(def page page/page-)
(def pill tags/pill-)
(def badge tags/badge-)
(def data-grid data-grid/data-grid-)
(def data-grid-header data-grid/header-)

3
src/clj/auto_ap/ssr/components/tags.clj
View File

@@ -18,3 +18,6 @@
(= :red (:color params))
(update :class str " bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300"))]
children))
(defn badge- [params & children]
[:div {:class "absolute inline-flex items-center justify-center w-6 h-6 text-xs font-bold text-white bg-red-500 border-2 border-white rounded-full -top-2 -right-2 dark:border-gray-900"} children])

2
src/clj/auto_ap/ssr/core.clj
View File

@@ -49,7 +49,7 @@
:transaction-insights (wrap-client-redirect-unauthenticated (wrap-admin insights/page))
:transaction-insight-table (wrap-client-redirect-unauthenticated (wrap-admin insights/insight-table))
:transaction-insight-rows (wrap-client-redirect-unauthenticated (wrap-admin insights/transaction-rows))
:transaction-insight-approve (wrap-client-redirect-unauthenticated (wrap-admin insights/approve))
:transaction-insight-code (wrap-client-redirect-unauthenticated (wrap-admin insights/code))
:transaction-insight-disapprove (wrap-client-redirect-unauthenticated (wrap-admin insights/disapprove))
:transaction-insight-explain (wrap-client-redirect-unauthenticated (wrap-admin insights/explain))
:admin-ezcater-xls (wrap-client-redirect-unauthenticated (wrap-admin ezcater-xls/page))

110
src/clj/auto_ap/ssr/transaction/insights.clj
View File

@@ -14,18 +14,18 @@
[clj-http.client :as http]
[clj-time.coerce :as coerce]
[datomic.api :as dc]
[iol-ion.tx :refer [random-tempid]]))
[iol-ion.tx :refer [random-tempid]]
[hiccup2.core :as hiccup]))
(def pull-expr [:transaction/description-original
:db/id
:transaction/outcome-recommendation
:transaction/amount
{:transaction/client [:client/code]
:transaction/bank-account [:bank-account/code]
:transaction/recommended-vendor [:vendor/name :db/id]
:transaction/recommended-account [:account/name :account/numeric-code :db/id]}
:transaction/bank-account [:bank-account/code]}
:transaction/account-confidence
:transaction/date
])
:transaction/date])
(defn transaction-recommendations [identity selected-client & {:keys [after]}]
(let [visible-clients (visible-clients identity)]
@@ -33,13 +33,13 @@
(dc/qseq {:query '[:find (pull ?t pull-expr)
:in $ ?starting [?c ...] pull-expr
:where
[?t :transaction/outcome-recommendation]
[?t :transaction/client ?c]
[?t :transaction/approval-status :transaction-approval-status/unapproved]
;; [?t :transaction/vendor] ;; should be not
[?t :transaction/date ?d]
[(>= ?d ?starting)]
]
[(>= ?d ?starting)]]
:args [(dc/db conn)
(iol-ion.query/recent-date 120)
(if selected-client
@@ -54,6 +54,20 @@
(#(if after
(drop 1 %)
%))
(map (fn [tx]
(update tx :transaction/outcome-recommendation
(fn [ors]
(map
(fn [[v a c s]]
{:vendor (dc/pull (dc/db conn)
[:vendor/name :db/id]
v)
:account (dc/pull (dc/db conn)
[:account/name :db/id]
a)
:count c
:seen-by-client? s})
ors)))))
(take 50)
(into []))))
@@ -108,12 +122,11 @@
:score score}))
(defn transaction-row [r & {:keys [hide-actions? class last?]}]
(let [simality-list (try (pinecone-similarity-list (:db/id r)) (catch Exception e
[]))]
(when (seq simality-list)
(defn transaction-row [r & {:keys [hide-actions? class last?] hs "_"}]
(com/data-grid-row
(cond-> {:class class}
hs (assoc "_" hs)
last? (assoc :hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-rows
:after (:db/id r))
@@ -129,44 +142,39 @@
(if (> (:transaction/amount r) 0.0)
[:div.tag.is-success.is-light (str "$" (Math/round (:transaction/amount r)))]
[:div.tag.is-danger.is-light (str "$" (Math/round (:transaction/amount r)))]))
(com/data-grid-cell {:style {:width "12em"}}
[:div.flex.gap-2.flex-wrap {:style {:width "12em"}}
(when-let [vendor-name (:vendor-name (first simality-list))]
(com/pill {:color :primary} vendor-name))
(when-let [numeric-code (:numeric-code (first simality-list))]
(com/pill {:color :secondary} numeric-code))
#_(com/pill {:class (cond
(> (:transaction/account-confidence r) 0.90)
"is-success is-light"
(> (:transaction/account-confidence r) 0.80)
"is-info is-light"
:else
"is-warning is-light")} (str "%" (Math/round (* 100.0 (:transaction/account-confidence r)))))])
(com/data-grid-right-stack-cell {}
(when-not hide-actions?
[:form.flex.gap-2
[:input {:type :hidden :name "id" :value (:db/id r)}]
(com/icon-button {:hx-post (bidi/path-for ssr-routes/only-routes
:transaction-insight-approve
[:div.flex.gap-2.flex-col {:style {:width "25em"}}
(for [or (sort-by (comp - :count)
(:transaction/outcome-recommendation r))]
[:form {:hx-post (bidi/path-for ssr-routes/only-routes
:transaction-insight-code
:transaction-id (:db/id r))
:hx-target "closest tr"
:color :primary-light}
svg/thumbs-up)
(com/icon-button {:hx-delete (bidi/path-for ssr-routes/only-routes
:transaction-insight-disapprove
:transaction-id (:db/id r))
:hx-target "closest tr"
:color :danger-light}
svg/thumbs-down)
:hx-target "closest tr"}
(when-let [vendor-id (:db/id (:vendor or))]
[:input {:type :hidden :value vendor-id :name "vendor"}])
(when-let [account-id (:db/id (:account or))]
[:input {:type :hidden :value account-id :name "account"}])
(com/button {:color (if (:seen-by-client? or)
:primary
:secondary)
:style {:position "relative"}}
(:vendor/name (:vendor or))
(when (:vendor/name (:vendor or))
" | ")
(:account/name (:account or))
(com/badge {:color :secondary}
(:count or)))])
(com/icon-button {:hx-get (bidi/path-for ssr-routes/only-routes
:transaction-insight-explain
:transaction-id (:db/id r))
:hx-target "#modal-holder"
:hx-swap "outerHTML"}
svg/question)]))))))
svg/question)]))))
(defn approve [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(defn code [{:keys [identity session] {:keys [transaction-id]} :route-params {:strs [vendor account]} :form-params}]
(let [approval-details (dc/pull (dc/db conn) [{:transaction/recommended-account [:account/location :db/id]}
:transaction/recommended-vendor
:transaction/amount
@@ -174,6 +182,9 @@
{:transaction/client [:client/locations]}]
(cond-> transaction-id
string? (Long/parseLong)))
account (dc/pull (dc/db conn) [:account/location :db/id]
(cond-> account
string? (Long/parseLong)))
cents-to-distribute (int (Math/round (Math/abs (* (:transaction/amount approval-details) 100))))
valid-locations (or
(some-> approval-details :transaction/recommended-account :account/location vector)
@@ -182,12 +193,12 @@
:client/locations))
updated-transaction [:upsert-transaction {:db/id (:db/id approval-details)
:transaction/approval-status :transaction-approval-status/approved
:transaction/vendor (:transaction/recommended-vendor approval-details)
:transaction/vendor (some-> vendor not-empty (Long/parseLong))
:transaction/accounts (->> valid-locations
(map
(fn [cents location]
{:db/id (random-tempid)
:transaction-account/account (-> approval-details :transaction/recommended-account :db/id)
:transaction-account/account (-> account :db/id)
:transaction-account/amount (* 0.01 cents)
:transaction-account/location location})
(spread-cents cents-to-distribute (count valid-locations))))}]]
@@ -207,7 +218,8 @@
(dc/pull (dc/db conn) pull-expr transaction-id)
:auto-remove? true
:hide-actions? true
:class "live-removed"))))
:class "live-removed"
"_" (hiccup/raw "init transition opacity to 0 then remove me")))))
(defn explain [{:keys [identity session] {:keys [transaction-id]} :route-params}]
(let [r (dc/pull (dc/db conn)
pull-expr
@@ -272,13 +284,11 @@
:rows (for [r recommendations
:let [last? (= r (last recommendations))]]
(transaction-row r :last? last?))
:headers [(com/data-grid-header {} "Client")
(com/data-grid-header {} "Account")
(com/data-grid-header {} "Date")
:headers [(com/data-grid-header {:style {:width "10em"}} "Client")
(com/data-grid-header {:style {:width "15em"}} "Account")
(com/data-grid-header {:style {:width "8em"}} "Date")
(com/data-grid-header {} "Description")
(com/data-grid-header {} "Amount")
(com/data-grid-header {:style {:width "4em"}} "Vendor / Account")
(com/data-grid-header {})]})))
(com/data-grid-header {:style {:width "8em"}} "Amount")]})))
(defn insight-table [{:keys [session identity]}]
(html-response (insight-table* {:selected-client

5
src/cljc/auto_ap/ssr_routes.cljc
View File

@@ -15,7 +15,7 @@
"/ezcater-xls" :admin-ezcater-xls}
"transaction" {"/insights" {"" :transaction-insights
"/table" :transaction-insight-table
["/approve/" [#"\d+" :transaction-id]] {:post :transaction-insight-approve}
["/code/" [#"\d+" :transaction-id]] {:post :transaction-insight-code}
["/disapprove/" [#"\d+" :transaction-id]] {:delete :transaction-insight-disapprove}
["/rows/" [#"\d+" :after]] {:get :transaction-insight-rows}
["/explain/" [#"\d+" :transaction-id]] {:get :transaction-insight-explain}}}
@@ -39,8 +39,7 @@
"/table" {:get :company-plaid-table}
"/link" {:post :company-plaid-link}
#_#_"/fastlink" {:get :company-yodlee-fastlink-dialog}
#_#_"/refresh" {:put :company-yodlee-provider-account-refresh}}
}})
#_#_"/refresh" {:put :company-yodlee-provider-account-refresh}}}})
(def only-routes ["/" routes])

13
terraform/deploy.tf
View File

@@ -458,3 +458,16 @@ module "ntg_job" {
memory = 4096
cpu = 1024
}
module "insight_outcome_recommendation_job" {
schedule = "rate(6 hours)"
source = "./background-job/"
ecs_cluster = var.ecs_cluster
task_role_arn = var.task_role_arn
stage = var.stage
job_name = "insight-outcome-recommendation"
execution_role_arn = var.execution_role_arn
use_schedule = true
memory = 4096
cpu = 2048
}