(cloud) adds transaction insights
This commit is contained in:
172
scratch-sessions/build_ml.clj
Normal file
172
scratch-sessions/build_ml.clj
Normal file
@@ -0,0 +1,172 @@
|
||||
;; This buffer is for Clojure experiments and evaluation.
|
||||
|
||||
;; Press C-j to evaluate the last expression.
|
||||
|
||||
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
|
||||
|
||||
(ns build-ml
|
||||
(:require [datomic.client.api :as dc]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.data.csv :as csv]
|
||||
[auto-ap.datomic :refer [conn]]))
|
||||
|
||||
(println "hi")
|
||||
|
||||
(defn boot []
|
||||
(user/init-repl)
|
||||
(user/start-db))
|
||||
|
||||
|
||||
|
||||
(defn write-vendor-training-data []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-training.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount" "vendor"]]
|
||||
(->>
|
||||
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v
|
||||
:in $ ?start
|
||||
:where
|
||||
[?t :transaction/date ?d]
|
||||
[(>= ?d ?start)]
|
||||
[?t :transaction/approval-status :transaction-approval-status/approved]
|
||||
(not [?t :transaction/payment])
|
||||
[?t :transaction/client ?c]
|
||||
[?t :transaction/vendor ?v]
|
||||
[?v :vendor/name ?v-name]
|
||||
[?t :transaction/description-original ?do]
|
||||
[?t :transaction/amount ?amt]
|
||||
[?t :transaction/bank-account ?b]
|
||||
|
||||
[?b :bank-account/name ?bn]
|
||||
[?b :bank-account/type ?bat]
|
||||
[?bat :db/ident ?bt]
|
||||
[(name ?bt) ?bt2]
|
||||
|
||||
]
|
||||
(dc/db auto-ap.datomic/conn)
|
||||
#inst "2021-01-01")
|
||||
(map (fn [[t code bn bat d date amt v]]
|
||||
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v]))))
|
||||
:quote? (constantly true))))
|
||||
|
||||
(defn write-account-training-data []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/account-training.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount" "vendor" "account"]]
|
||||
(->>
|
||||
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt ?v ?a
|
||||
:in $ ?start
|
||||
:where
|
||||
[?t :transaction/date ?d]
|
||||
[(>= ?d ?start)]
|
||||
[?t :transaction/approval-status :transaction-approval-status/approved]
|
||||
[?t :transaction/accounts ?ta]
|
||||
[?ta :transaction-account/account ?a]
|
||||
[?t :transaction/vendor ?v]
|
||||
[?v :vendor/name ?v-name]
|
||||
[?t :transaction/client ?c]
|
||||
[?t :transaction/description-original ?do]
|
||||
[?t :transaction/amount ?amt]
|
||||
[?t :transaction/bank-account ?b]
|
||||
[?b :bank-account/name ?bn]
|
||||
[?b :bank-account/type ?bat]
|
||||
[?bat :db/ident ?bt]
|
||||
[(name ?bt) ?bt2]]
|
||||
(dc/db auto-ap.datomic/conn)
|
||||
#inst "2022-01-01")
|
||||
(map (fn [[t code bn bat d date amt v a]]
|
||||
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt v a]))))
|
||||
:quote? (constantly true))))
|
||||
|
||||
(defn write-inference []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/inference.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]]
|
||||
(->>
|
||||
(dc/q '[:find ?t ?c ?bn ?bt2 ?do ?d ?amt
|
||||
:in $ ?start
|
||||
:where
|
||||
[?t :transaction/date ?d]
|
||||
[(>= ?d ?start)]
|
||||
[?t :transaction/approval-status :transaction-approval-status/unapproved]
|
||||
(not [?t :transaction/matched-rule])
|
||||
(not [?t :transaction/payment])
|
||||
[?t :transaction/client ?c]
|
||||
[?c :client/code ?code]
|
||||
(not [?t :transaction/vendor])
|
||||
(not [?t :transaction/accounts])
|
||||
[?t :transaction/description-original ?do]
|
||||
[?t :transaction/amount ?amt]
|
||||
[?t :transaction/bank-account ?b]
|
||||
|
||||
[?b :bank-account/name ?bn]
|
||||
[?b :bank-account/type ?bat]
|
||||
[?bat :db/ident ?bt]
|
||||
[(name ?bt) ?bt2]
|
||||
|
||||
]
|
||||
(dc/db auto-ap.datomic/conn)
|
||||
#inst "2023-01-01")
|
||||
(map (fn [[t code bn bat d date amt]]
|
||||
[t code bn bat d (auto-ap.time/unparse-local (clj-time.coerce/to-date-time date) auto-ap.time/iso-date) amt]))
|
||||
))
|
||||
:quote? (constantly true))))
|
||||
|
||||
|
||||
|
||||
(defn write-vendor-lookup []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendors.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["vendor" "vendor_name"]]
|
||||
(->>
|
||||
(dc/q '[:find ?v ?v-name
|
||||
:where [?v :vendor/name ?v-name]
|
||||
[_ :transaction/vendor ?v]
|
||||
]
|
||||
(dc/db conn))))
|
||||
:separator \,)))
|
||||
|
||||
(defn write-vendor-client-lookup []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/vendor-client-usage.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["vendor" "client" "vendor_client_count"]]
|
||||
(->>
|
||||
(dc/q '[:find ?v ?c (count ?t)
|
||||
:where
|
||||
[?t :transaction/client ?c]
|
||||
[?t :transaction/vendor ?v]
|
||||
]
|
||||
(dc/db conn))))
|
||||
:separator \,)))
|
||||
|
||||
(defn write-account-lookup []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/accounts.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["account" "numeric_code"]]
|
||||
(->>
|
||||
(dc/q '[:find ?a ?nm
|
||||
:where [?a :account/numeric-code ?nm]]
|
||||
(dc/db conn))))
|
||||
:separator \,)))
|
||||
|
||||
(defn write-account-client-lookup []
|
||||
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/account-client-usage.csv")]
|
||||
(csv/write-csv f
|
||||
(into [["client" "account" "client_account_count"]]
|
||||
(->>
|
||||
(dc/q '[:find ?c ?a (count ?t)
|
||||
:where
|
||||
[?t :transaction/client ?c]
|
||||
[?t :transaction/accounts ?ta]
|
||||
[?ta :transaction-account/account ?a]]
|
||||
(dc/db conn))))
|
||||
:separator \,)))
|
||||
|
||||
|
||||
(defn write-all-training []
|
||||
(write-account-lookup)
|
||||
(write-vendor-lookup)
|
||||
(write-account-client-lookup)
|
||||
(write-vendor-client-lookup)
|
||||
(write-account-training-data)
|
||||
(write-vendor-training-data))
|
||||
71
scratch-sessions/ingest_ml.clj
Normal file
71
scratch-sessions/ingest_ml.clj
Normal file
@@ -0,0 +1,71 @@
|
||||
(ns ingest-ml
|
||||
(:require [datomic.client.api :as dc]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.data.csv :as csv]
|
||||
[iol-ion.tx :refer [upsert-entity]]
|
||||
[auto-ap.datomic :refer [conn]]))
|
||||
|
||||
(println "hi")
|
||||
|
||||
(defn boot []
|
||||
(user/init-repl)
|
||||
(user/start-db))
|
||||
|
||||
|
||||
(defn reset-inference []
|
||||
(doseq [p (->>
|
||||
(dc/q '[:find ?t
|
||||
:where [?t :transaction/recommended-account]]
|
||||
(dc/db conn))
|
||||
(map (fn [[t]]
|
||||
`(upsert-entity
|
||||
~{:db/id t
|
||||
:transaction/recommended-account nil
|
||||
:tranasction/vendor-confidence nil
|
||||
:transaction/account-confidence nil
|
||||
:transaction/recommended-vendor nil})))
|
||||
|
||||
(partition-all 100))]
|
||||
(dc/transact conn {:tx-data p})))
|
||||
|
||||
(defn read-inference []
|
||||
(with-open [reader (io/reader "/mnt/data/dev2/ml-test/inference-outcome.csv")]
|
||||
(->> (csv/read-csv reader)
|
||||
(into []
|
||||
(comp
|
||||
(drop 1)
|
||||
(map (fn [[_ transaction best-vendor best-account account-confidence]]
|
||||
{:db/id (Long/parseLong transaction)
|
||||
:transaction/recommended-account (Long/parseLong best-account)
|
||||
:transaction/account-confidence (Double/parseDouble account-confidence)
|
||||
:transaction/recommended-vendor (Long/parseLong best-vendor)}
|
||||
)))))))
|
||||
|
||||
(defn apply-inference [inference]
|
||||
(doseq [p (->> inference (partition-all 100))]
|
||||
(dc/transact conn {:tx-data p})))
|
||||
|
||||
|
||||
(defn check-applied-inference []
|
||||
(clojure.pprint/pprint
|
||||
(->>
|
||||
(dc/q '[:find ?code ?bac ?do ?amount ?vc ?rvn ?ac ?ran
|
||||
:in $
|
||||
:where
|
||||
[?t :transaction/recommended-account ?ra]
|
||||
[?ra :account/name ?ran]
|
||||
[?t :transaction/account-confidence ?ac]
|
||||
|
||||
[?t :transaction/recommended-vendor ?rv]
|
||||
[?rv :vendor/name ?rvn]
|
||||
|
||||
[?t :transaction/description-original ?do]
|
||||
[?t :transaction/client ?c]
|
||||
[?c :client/code ?code]
|
||||
[?t :transaction/bank-account ?ba]
|
||||
[?ba :bank-account/code ?bac]
|
||||
[?t :transaction/amount ?amount]
|
||||
]
|
||||
(dc/db conn))
|
||||
(shuffle)
|
||||
(take 10))))
|
||||
Reference in New Issue
Block a user