Pregenerates recommendatios

2023-08-19 13:52:44 -07:00
parent 750a147f1c
commit 486c428917
11 changed files with 820 additions and 1042 deletions
--- a/src/clj/auto_ap/jobs/insight_outcome_recommendation.clj
+++ b/src/clj/auto_ap/jobs/insight_outcome_recommendation.clj
@@ -0,0 +1,97 @@
+(ns auto-ap.jobs.insght-outcome-recommendation
+  (:require
+   [auto-ap.datomic :refer [audit-transact-batch conn]]
+   [auto-ap.jobs.core :refer [execute]]
+   [cemerick.url :as url]
+   [clj-http.client :as http2]
+   [datomic.api :as dc]))
+
+(defn get-recent-transactions []
+  (->> conn
+       dc/db
+       (dc/q '[:find ?t ?c
+               :where
+               [(iol-ion.query/recent-date 90) ?start]
+               [?t :transaction/date ?d]
+               [(>= ?d ?start)]
+               [?t :transaction/approval-status :transaction-approval-status/unapproved]
+               (not [?t :transaction/outcome-recommendation])
+               [?t :transaction/description-original ?do]
+               (not [(clojure.string/includes? ?do "CHECK")])
+               (not [(clojure.string/includes? ?do "Check")])
+               [?t :transaction/client ?c]])))
+
+(defn get-pinecone [transaction-id]
+  (-> 
+    (http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
+                  url/url
+                  (assoc :query {:ids transaction-id})
+                  str)
+              {:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
+               :as      :json
+               :keywordize? false})
+    :body
+    :vectors
+    ((keyword (str transaction-id)))
+    :values))
+
+(defn get-pinecone-similarities [transaction-id]
+  (filter 
+    (fn [{:keys [score]}]
+      (> score 0.95)
+      )
+    (-> 
+      (http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
+                     url/url
+                     str)
+                 {:headers      {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
+                  :form-params  {"vector"          (get-pinecone transaction-id)
+                                 "topK"            200,
+                                 "includeMetadata" true
+                                 "namespace"       ""}
+                  :content-type :json
+                  :as           :json})
+      :body
+      :matches)))
+
+(defn pinecone-similarity-list [transaction-id]
+  (for [{{:keys [amount date description vendor]} :metadata score :score id :id} (get-pinecone-similarities transaction-id)
+        :let                                                                     [similar-transaction (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name :db/id]
+                                                                                                                              :transaction/accounts [{:transaction-account/account [:account/numeric-code :db/id]}]
+                                                                                                                              :transaction/client [:db/id]}] (Long/parseLong id))]
+        :when                                                                    (or (:transaction/vendor similar-transaction) (seq (:transaction/accounts similar-transaction)))]
+
+    (assoc similar-transaction :score score)))
+
+(defn similar->recommendation [txs client]
+  (->> txs
+       (reduce (fn [acc t]
+                 (-> acc
+                     (update-in [[(:db/id (:transaction/vendor t))
+                                  (:db/id (:transaction-account/account (first (:transaction/accounts t))))]
+                                 :count]
+                                (fnil inc 0))
+                     (update-in [[(:db/id (:transaction/vendor t))
+                                  (:db/id (:transaction-account/account (first (:transaction/accounts t))))]
+                                 :seen-by-client?]
+                                (fn [seen-by-client?] (or seen-by-client? (= (:db/id (:transaction/client t))
+                                                                            client))))))
+               {})
+       (map (fn [[k v]]
+              (-> k
+                  (conj (:count v))
+                  (conj (:seen-by-client? v)))))))
+
+(defn get-outcome-recommendations []
+  (for [[transaction client] (get-recent-transactions)
+        :let [similarity (pinecone-similarity-list transaction)]
+        :when (seq similarity)]
+    {:db/id transaction
+     :transaction/outcome-recommendation
+     (similar->recommendation similarity client)}))
+
+(defn make-outcome-recommendations []
+  (audit-transact-batch (get-outcome-recommendations)  {:user/name "Outcome recommendations from pinecone"}))
+
+(defn -main [& _]
+  (execute "insight-outcome-recommendation" make-outcome-recommendations))