(ns auto-ap.jobs.insight-outcome-recommendation (:require [auto-ap.datomic :refer [audit-transact-batch conn]] [auto-ap.jobs.core :refer [execute]] [auto-ap.logging :as alog] [cemerick.url :as url] [clj-http.client :as http2] [datomic.api :as dc])) (defn get-recent-transactions [] (->> conn dc/db (dc/q '[:find ?t ?c :where [(iol-ion.query/recent-date 14) ?start] [?t :transaction/date ?d] [(>= ?d ?start)] [?t :transaction/approval-status :transaction-approval-status/unapproved] (not [?t :transaction/outcome-recommendation]) [?t :transaction/description-original ?do] (not [(clojure.string/includes? ?do "CHECK")]) (not [(clojure.string/includes? ?do "Check")]) [?t :transaction/client ?c]]))) (defn get-pinecone [transaction-id] (-> (http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch" url/url (assoc :query {:ids transaction-id}) str) {:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"} :as :json :keywordize? false}) :body :vectors ((keyword (str transaction-id))) :values)) (defn get-pinecone-similarities [transaction-id] (if-let [vector (get-pinecone transaction-id)] (filter (fn [{:keys [score]}] (> score 0.95)) (-> (http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query" url/url str) {:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"} :form-params {"vector" vector "topK" 200, "includeMetadata" true "namespace" ""} :content-type :json :as :json}) :body :matches (doto (#(alog/info ::similarities-found :transaction transaction-id :count (count %) :sample (take 3 %)))))) (do (alog/info ::no-matches-for :transaction transaction-id) []))) (defn pinecone-similarity-list [transaction-id] (for [{{:keys [amount date description vendor]} :metadata score :score id :id} (get-pinecone-similarities transaction-id) :let [similar-transaction (dc/pull (dc/db conn) [{:transaction/vendor [:vendor/name :db/id] :transaction/accounts [{:transaction-account/account [:account/numeric-code :db/id]}] :transaction/client [:db/id]}] (Long/parseLong id))] :when (or (:transaction/vendor similar-transaction) (seq (:transaction/accounts similar-transaction)))] (assoc similar-transaction :score score))) (defn similar->recommendation [txs client] (try (->> txs (reduce (fn [acc t] (-> acc (update-in [[(:db/id (:transaction/vendor t)) (:db/id (:transaction-account/account (first (:transaction/accounts t))))] :count] (fnil inc 0)) (update-in [[(:db/id (:transaction/vendor t)) (:db/id (:transaction-account/account (first (:transaction/accounts t))))] :seen-by-client?] (fn [seen-by-client?] (or seen-by-client? (= (:db/id (:transaction/client t)) client)))))) {}) (map (fn [[k v]] (-> k (conj (:count v)) (conj (:seen-by-client? v)))))) (catch Exception e (alog/warn ::similarities-failed :error e) []))) (defn get-outcome-recommendations [] (let [recent (get-recent-transactions)] (alog/info ::recent-transactions-fetched :count (count recent) :sample (take 3 recent)) (for [[transaction client] recent :let [similarity (pinecone-similarity-list transaction)] :when (seq similarity)] {:db/id transaction :transaction/outcome-recommendation (similar->recommendation similarity client)}))) (defn make-outcome-recommendations [] (audit-transact-batch (get-outcome-recommendations) {:user/name "Outcome recommendations from pinecone"})) (defn -main [& _] (execute "insight-outcome-recommendation" make-outcome-recommendations))