notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

tweaks.

Browse Source
This commit is contained in:
Bryce
2023-06-14 16:19:30 -07:00
parent 85ce806dfc
commit 4d4678f6f7
2 changed files with 71 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scratch-sessions/build_ml.clj
View File

@@ -79,7 +79,7 @@
:quote? (constantly true)))) :quote? (constantly true))))
(defn write-inference [] (defn write-inference []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/inference.csv")] (with-open [f (io/writer "~/dev/transaction-training/input/inference.csv")]
(csv/write-csv f (csv/write-csv f
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]] (into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]]
(->> (->>

98
scratch-sessions/ingest_ml.clj
View File

@@ -1,8 +1,8 @@
(ns ingest-ml (ns ingest-ml
(:require [datomic.client.api :as dc] (:require [datomic.api :as dc]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[iol-ion.tx :refer [upsert-entity]] #_[iol-ion.tx :refer [upsert-entity]]
[auto-ap.datomic :refer [conn]])) [auto-ap.datomic :refer [conn]]))
(println "hi") (println "hi")
@@ -13,37 +13,60 @@
(defn reset-inference [] (defn reset-inference []
(doseq [p (->> (doseq [p (->>
(dc/q '[:find ?t (dc/q '[:find ?t
:where [?t :transaction/recommended-account]] :where [?t :transaction/recommended-account]]
(dc/db conn)) (dc/db conn))
(map (fn [[t]] (map (fn [[t]]
`(upsert-entity [:upsert-entity {:db/id t
~{:db/id t :transaction/recommended-account nil
:transaction/recommended-account nil :tranasction/vendor-confidence nil
:tranasction/vendor-confidence nil :transaction/account-confidence nil
:transaction/account-confidence nil :transaction/recommended-vendor nil}]))
:transaction/recommended-vendor nil})))
(partition-all 100))]
(partition-all 100))] @(dc/transact conn p)))
(dc/transact conn {:tx-data p})))
(defn slurp-csv [c]
(with-open [s (io/reader c)]
(into [] (csv/read-csv s))))
(defn read-inference [] (defn read-inference []
(with-open [reader (io/reader "/mnt/data/dev2/ml-test/inference-outcome.csv")] (let [numeric-code->account (into {} (dc/q '[:find ?nc ?a
(->> (csv/read-csv reader) :in $
(into [] :where [?a :account/numeric-code ?nc]]
(comp (dc/db conn)))
(drop 1) inference-account->account (into {}
(map (fn [[_ transaction best-vendor best-account account-confidence]] (comp
{:db/id (Long/parseLong transaction) (drop 1)
:transaction/recommended-account (Long/parseLong best-account) (map (fn [[a n]]
:transaction/account-confidence (Double/parseDouble account-confidence) [(Long/parseLong a) (numeric-code->account (Long/parseLong n))])))
:transaction/recommended-vendor (Long/parseLong best-vendor)} (slurp-csv "/home/notid/dev/transaction-training/input/accounts.csv"))
))))))) vendor-name->vendor (into {} (dc/q '[:find ?vn ?v
:in $
:where [?v :vendor/name ?vn]]
(dc/db conn)))
inference-vendor->vendor (into {}
(comp
(drop 1)
(map (fn [[a n]]
[(Long/parseLong a) (vendor-name->vendor (Long/parseLong n))])))
(slurp-csv "/home/notid/dev/transaction-training/input/vendors.csv"))]
inference-vendor->vendor
#_(with-open [reader (io/reader "/home/notid/dev/transaction-training/inference-outcome.csv")]
(->> (csv/read-csv reader)
(into []
(comp
(drop 1)
(map (fn [[_ transaction best-vendor best-account account-confidence]]
{:db/id (Long/parseLong transaction)
:transaction/recommended-account (Long/parseLong best-account)
:transaction/account-confidence (Double/parseDouble account-confidence)
:transaction/recommended-vendor (Long/parseLong best-vendor)}))))))))
(defn apply-inference [inference] (defn apply-inference [inference]
(doseq [p (->> inference (partition-all 100))] (doseq [p (->> inference (partition-all 100))]
(dc/transact conn {:tx-data p}))) @(dc/transact conn p)))
(defn check-applied-inference [] (defn check-applied-inference []
@@ -69,3 +92,22 @@
(dc/db conn)) (dc/db conn))
(shuffle) (shuffle)
(take 10)))) (take 10))))
(defn random-infer []
(let [n (rand-int 1000)
vs (into [] (map first (dc/q '[:find ?v :in $ :where [?v :vendor/name]] (dc/db conn))))
as (into [] (map first (dc/q '[:find ?v :in $ :where [?v :account/name]] (dc/db conn))))]
(->>
(dc/qseq {:query '[:find ?t
:where
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/vendor])]
:args [(dc/db conn)]})
(map first)
(take n)
(map (fn [t]
{:db/id t
:transaction/recommended-account (rand-nth as)
:transaction/account-confidence (double (/ (double (rand-int 100)) 100.0))
:transaction/recommended-vendor (rand-nth vs)})))))