notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merged.

Browse Source
This commit is contained in:
Bryce
2023-06-14 16:20:40 -07:00
parent 1c245c6427 4d4678f6f7
commit 2b93118b84
3 changed files with 40 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scratch-sessions/build_ml.clj
View File

@@ -79,7 +79,7 @@
:quote? (constantly true)))) :quote? (constantly true))))
(defn write-inference [] (defn write-inference []
(with-open [f (io/writer "/mnt/data/dev2/ml-test/input/inference.csv")] (with-open [f (io/writer "~/dev/transaction-training/input/inference.csv")]
(csv/write-csv f (csv/write-csv f
(into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]] (into [["transaction" "client" "bank" "bank_type" "description" "date" "amount"]]
(->> (->>

50
scratch-sessions/ingest_ml.clj
View File

@@ -27,18 +27,42 @@
(partition-all 100))] (partition-all 100))]
@(dc/transact conn p))) @(dc/transact conn p)))
(defn slurp-csv [c]
(with-open [s (io/reader c)]
(into [] (csv/read-csv s))))
(defn read-inference [] (defn read-inference []
(with-open [reader (io/reader "data/inference-outcome.csv")] (let [numeric-code->account (into {} (dc/q '[:find ?nc ?a
(->> (csv/read-csv reader) :in $
(into [] :where [?a :account/numeric-code ?nc]]
(comp (dc/db conn)))
(drop 1) inference-account->account (into {}
(map (fn [[_ transaction best-vendor best-account account-confidence]] (comp
{:db/id (Long/parseLong transaction) (drop 1)
:transaction/recommended-account (Long/parseLong best-account) (map (fn [[a n]]
:transaction/account-confidence (Double/parseDouble account-confidence) [(Long/parseLong a) (numeric-code->account (Long/parseLong n))])))
:transaction/recommended-vendor (Long/parseLong best-vendor)} (slurp-csv "/home/notid/dev/transaction-training/input/accounts.csv"))
))))))) vendor-name->vendor (into {} (dc/q '[:find ?vn ?v
:in $
:where [?v :vendor/name ?vn]]
(dc/db conn)))
inference-vendor->vendor (into {}
(comp
(drop 1)
(map (fn [[a n]]
[(Long/parseLong a) (vendor-name->vendor (Long/parseLong n))])))
(slurp-csv "/home/notid/dev/transaction-training/input/vendors.csv"))]
inference-vendor->vendor
#_(with-open [reader (io/reader "/home/notid/dev/transaction-training/inference-outcome.csv")]
(->> (csv/read-csv reader)
(into []
(comp
(drop 1)
(map (fn [[_ transaction best-vendor best-account account-confidence]]
{:db/id (Long/parseLong transaction)
:transaction/recommended-account (Long/parseLong best-account)
:transaction/account-confidence (Double/parseDouble account-confidence)
:transaction/recommended-vendor (Long/parseLong best-vendor)}))))))))
(defn apply-inference [inference] (defn apply-inference [inference]
(doseq [p (->> inference (partition-all 100))] (doseq [p (->> inference (partition-all 100))]
@@ -76,7 +100,9 @@
as (into [] (map first (dc/q '[:find ?v :in $ :where [?v :account/name]] (dc/db conn))))] as (into [] (map first (dc/q '[:find ?v :in $ :where [?v :account/name]] (dc/db conn))))]
(->> (->>
(dc/qseq {:query '[:find ?t (dc/qseq {:query '[:find ?t
:where [?t :transaction/client]] :where
[?t :transaction/approval-status :transaction-approval-status/unapproved]
(not [?t :transaction/vendor])]
:args [(dc/db conn)]}) :args [(dc/db conn)]})
(map first) (map first)
(take n) (take n)

2
src/clj/auto_ap/parse/templates.clj
View File

@@ -605,7 +605,7 @@
:keywords [#"reelproduce.com"] :keywords [#"reelproduce.com"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)" :extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}" :customer-identifier #"Bill To(?:.*?)\n\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice #\n.*?([\d\-]+)\n" :invoice-number #"Invoice #\n.*?\n.*?([\d\-]+)\n"
:total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"} :total #"Total\s*\n\s+\$([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}]) :total [:trim-commas-and-negate nil]}}])