710 lines
23 KiB
Clojure
710 lines
23 KiB
Clojure
(ns debug-sccb-transactions
|
|
(:require [datomic.api :as d]
|
|
[auto-ap.datomic :refer [conn]]
|
|
[auto-ap.utils :refer [dollars=]]
|
|
[auto-ap.time :as atime :refer [iso-date]]
|
|
[clj-time.core :as t]
|
|
[clj-time.coerce :as coerce]
|
|
[clj-time.periodic :as periodic]
|
|
[clojure.data.csv :as csv]
|
|
|
|
[auto-ap.yodlee.core :as y]
|
|
[auto-ap.yodlee.core2 :as y2]))
|
|
|
|
(d/pull (d/db conn) '[*] [:bank-account/code "SCCB-9128CB"])
|
|
|
|
(def trans (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header)))
|
|
|
|
(user/init-repl)
|
|
|
|
[(->> trans
|
|
(sort-by :date)
|
|
(map (fn [t]
|
|
(if (= "DEBIT" (:baseType t))
|
|
(- (:amount (:amount t)))
|
|
(:amount (:amount t)))))
|
|
(reduce + 0.0))
|
|
(:amount (:runningBalance (first trans)))
|
|
(:amount (:runningBalance (last trans)))
|
|
(- (:amount (:runningBalance (last trans)))
|
|
(:amount (:runningBalance (first trans))))]
|
|
|
|
|
|
(defn y-tx->$ [b]
|
|
(if (= "DEBIT" (:baseType b))
|
|
(- (:amount (:amount b)))
|
|
(:amount (:amount b))))
|
|
|
|
|
|
(do
|
|
|
|
(doseq [bank-account #_ (d/q '[:find [?yai ...]
|
|
:in $
|
|
:where [_ :bank-account/yodlee-account-id ?yai]]
|
|
(d/db conn))
|
|
:when (not= 0 bank-account)]
|
|
(try
|
|
(println
|
|
[bank-account (->> (y/get-specific-transactions-with-date bank-account "2021-01-01" "2021-12-31" (y/get-auth-header))
|
|
(reverse)
|
|
(filter :runningBalance)
|
|
(partition 2 1)
|
|
(map (fn [[a b]]
|
|
(try
|
|
[(:amount (:runningBalance a))
|
|
|
|
(- (:amount (:runningBalance b))
|
|
(if (= "DEBIT" (:baseType b))
|
|
(- (:amount (:amount b)))
|
|
(:amount (:amount b))))
|
|
]
|
|
(catch Exception e
|
|
(println a b)
|
|
(println e)
|
|
(throw e)
|
|
))
|
|
))
|
|
(filter (fn [[a b]]
|
|
(not (dollars= a b))))
|
|
count
|
|
|
|
)])
|
|
(catch Exception e
|
|
(println "please retry" bank-account)))
|
|
)
|
|
(println "done"))
|
|
|
|
|
|
|
|
(take 10 trans)
|
|
|
|
(first trans)
|
|
(last trans)
|
|
|
|
|
|
(csv/write-csv *out*
|
|
(->> trans
|
|
(sort-by :date)
|
|
(map (fn [z]
|
|
(update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
|
|
(map (fn [z]
|
|
[(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
|
|
(- (:amount (:amount z)))
|
|
(:amount (:amount z)))]
|
|
))
|
|
|
|
)
|
|
|
|
:separator \tab
|
|
)
|
|
|
|
(def deep-dive (y/get-specific-transactions-with-date 27995674 "2021-12-03" "2021-12-03" (y/get-auth-header)))
|
|
|
|
|
|
(csv/write-csv *out*
|
|
(->> (->> (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header))
|
|
(group-by (fn [d]
|
|
((juxt :description
|
|
:amount
|
|
:date
|
|
:baseType
|
|
:type
|
|
:isManual
|
|
:merchantType
|
|
:status
|
|
:CONTAINER
|
|
:runningBalance
|
|
:subType
|
|
:isPhysical
|
|
:accountId
|
|
:postDate) d)))
|
|
(map (fn [[k v]]
|
|
(first (sort-by :id v)))))
|
|
(sort-by :date)
|
|
(map (fn [z]
|
|
(update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
|
|
(map (fn [z]
|
|
[(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
|
|
(- (:amount (:amount z)))
|
|
(:amount (:amount z)))]
|
|
))
|
|
|
|
)
|
|
|
|
:separator \tab
|
|
)
|
|
|
|
(->> (y/get-specific-transactions-with-date 27995674 "2021-12-30" "2021-12-30" (y/get-auth-header))
|
|
(group-by (fn [d]
|
|
((juxt :description
|
|
:amount
|
|
:date
|
|
:baseType
|
|
:type
|
|
:isManual
|
|
:merchantType
|
|
:status
|
|
:CONTAINER
|
|
:runningBalance
|
|
:subType
|
|
:isPhysical
|
|
:accountId
|
|
:postDate) d)))
|
|
(map (fn [[k v]]
|
|
[k (count v)]
|
|
#_(clojure.data/diff a b))))
|
|
|
|
(y2/get-accounts "DEMO2")
|
|
|
|
|
|
(->> (y2/get-specific-transactions "DEMO2" 15565801)
|
|
(group-by (fn [d]
|
|
((juxt :description
|
|
:amount
|
|
:date
|
|
:baseType
|
|
:type
|
|
:isManual
|
|
:merchantType
|
|
:status
|
|
:CONTAINER
|
|
:runningBalance
|
|
:subType
|
|
:isPhysical
|
|
:accountId
|
|
:postDate) d)))
|
|
(map (fn [[k v]]
|
|
[k (count v)]
|
|
#_(clojure.data/diff a b)))
|
|
count)
|
|
|
|
(csv/write-csv *out*
|
|
(->> (y2/get-specific-transactions "DEMO2" 15565801)
|
|
(sort-by :date)
|
|
(map (fn [z]
|
|
(update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
|
|
(map (fn [z]
|
|
[(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
|
|
(- (:amount (:amount z)))
|
|
(:amount (:amount z)))]
|
|
)))
|
|
:separator \tab)
|
|
|
|
|
|
|
|
(def bad-as [24265567
|
|
16422563
|
|
27327396
|
|
16279665
|
|
16279668
|
|
16279669
|
|
16279670
|
|
16279671
|
|
24230113
|
|
16279664
|
|
16421944
|
|
24287315
|
|
27723398
|
|
27723397
|
|
16422358
|
|
18685498
|
|
16422285
|
|
24370905
|
|
18911886
|
|
27995674
|
|
19935858
|
|
16422708])
|
|
|
|
|
|
(def all-as [
|
|
[18409209 0]
|
|
[18409210 0]
|
|
[27040638 0]
|
|
[18409211 0]
|
|
[27040639 0]
|
|
[27040640 0]
|
|
[27398018 0]
|
|
[24249175 0]
|
|
[16551129 0]
|
|
[24781151 0]
|
|
[24265567 2433]
|
|
[25829735 0]
|
|
[25976674 0]
|
|
[25829733 0]
|
|
[25829734 0]
|
|
[26403171 0]
|
|
[26403172 0]
|
|
[26403169 0]
|
|
[26403170 0]
|
|
[19679001 0]
|
|
[20772618 0]
|
|
[26403173 0]
|
|
[27509142 0]
|
|
[19679000 0]
|
|
[18937120 0]
|
|
[22730534 0]
|
|
[25640840 0]
|
|
[16551102 0]
|
|
[20186867 0]
|
|
[27784044 0]
|
|
[26216350 0]
|
|
[27784043 0]
|
|
[27784042 0]
|
|
[21961051 0]
|
|
[16422563 48]
|
|
[22981959 0]
|
|
[16422564 0]
|
|
[18356995 0]
|
|
[16422562 0]
|
|
[27040641 0]
|
|
[27040642 0]
|
|
[27040643 0]
|
|
[26997692 0]
|
|
[26997691 0]
|
|
[26793947 0]
|
|
[27327396 64]
|
|
[16279665 83]
|
|
[16279666 0]
|
|
[26988499 0]
|
|
[16279667 0]
|
|
[16279668 12]
|
|
[16279669 16]
|
|
[16279670 55]
|
|
[16279671 4]
|
|
[16279672 0]
|
|
[24230113 80]
|
|
[25624533 0]
|
|
[25624532 0]
|
|
[16279663 0]
|
|
[16279664 441]
|
|
[26991555 0]
|
|
[27798450 0]
|
|
[27798451 0]
|
|
[26607050 0]
|
|
[16422491 0]
|
|
[16422492 0]
|
|
[16422495 0]
|
|
[16422493 0]
|
|
[27847951 0]
|
|
[16422494 0]
|
|
[25646832 0]
|
|
[25646831 0]
|
|
[16421963 0]
|
|
[16422473 0]
|
|
[26920161 0]
|
|
[16422471 0]
|
|
[16422472 0]
|
|
[16421945 0]
|
|
[16422463 0]
|
|
[16422462 0]
|
|
[24591293 0]
|
|
[16421943 0]
|
|
[16421944 8]
|
|
[24566177 0]
|
|
[24434091 0]
|
|
[24434092 0]
|
|
[24104846 0]
|
|
[24104845 0]
|
|
[24104844 0]
|
|
[24411026 0]
|
|
[24104839 0]
|
|
[24104838 0]
|
|
[21754236 0]
|
|
[27576149 0]
|
|
[18099105 0]
|
|
[24499586 0]
|
|
[18937304 0]
|
|
[18937305 0]
|
|
[25308479 0]
|
|
[25308480 0]
|
|
[24104849 0]
|
|
[23662442 0]
|
|
[25308484 0]
|
|
[25308481 0]
|
|
[16876498 0]
|
|
[24265569 0]
|
|
[16876499 0]
|
|
[24265570 0]
|
|
[24287315 317]
|
|
[26983548 0]
|
|
[24287316 0]
|
|
[27723399 0]
|
|
[27723398 21]
|
|
[27723397 110]
|
|
[24287319 0]
|
|
[27576450 0]
|
|
[24287317 0]
|
|
[27576451 0]
|
|
[24287318 0]
|
|
[26983549 0]
|
|
[27576452 0]
|
|
[26794100 0]
|
|
[24287326 0]
|
|
[16422358 13]
|
|
[17303033 0]
|
|
[26942060 0]
|
|
[24776257 0]
|
|
[16876551 0]
|
|
[22883931 0]
|
|
[17970207 0]
|
|
[27361904 0]
|
|
[27361903 0]
|
|
[19941347 0]
|
|
[27345526 0]
|
|
[16417200 0]
|
|
[16428453 0]
|
|
[19756531 0]
|
|
[16428443 0]
|
|
[18685498 18]
|
|
[27465416 0]
|
|
[27465415 0]
|
|
[27465417 0]
|
|
[16422296 0]
|
|
[27847900 0]
|
|
[27847902 0]
|
|
[16422285 11]
|
|
[25350341 0]
|
|
[16428403 0]
|
|
[16551286 0]
|
|
[27282619 0]
|
|
[27282620 0]
|
|
[19942033 0]
|
|
[19942034 0]
|
|
[26220018 0]
|
|
[27780614 0]
|
|
[27793421 0]
|
|
[17303150 0]
|
|
[27793422 0]
|
|
[16428372 0]
|
|
[24370905 450]
|
|
[16428375 0]
|
|
[18911886 44]
|
|
[16428373 0]
|
|
[16428374 0]
|
|
[18911888 0]
|
|
[27995676 0]
|
|
[27995675 0]
|
|
[27995674 861]
|
|
[19942029 0]
|
|
[17408629 0]
|
|
[19942030 0]
|
|
[22219439 0]
|
|
[19942031 0]
|
|
[19942032 0]
|
|
[25543675 0]
|
|
[19935858 606]
|
|
[16422707 0]
|
|
[16422708 320]
|
|
[27116551 0]
|
|
[16428317 0]
|
|
[16428318 0]
|
|
[24510110 0]
|
|
[27847749 0]
|
|
[654654654 0]
|
|
[21489791 0]
|
|
[21489790 0]
|
|
[23139427 0]
|
|
[16422643 0]
|
|
[16422644 0]
|
|
[16422642 0]
|
|
[16428279 0]
|
|
[25247822 0]
|
|
[18409174 0]
|
|
[16422645 0]
|
|
[18948276 0]
|
|
[18948275 0]
|
|
])
|
|
(filter (fn [[_ x]] (not= 0 x)) all-as)
|
|
|
|
(->> (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
|
|
:in $ [?yai ...]
|
|
:where [?ba :bank-account/yodlee-account-id ?yai]]
|
|
(d/db conn)
|
|
|
|
)
|
|
(map (juxt :bank-account/name :bank-account/code)))
|
|
|
|
|
|
(def trans (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header)))
|
|
|
|
(def starting (:amount (:runningBalance (last trans))))
|
|
|
|
(def dates (sort (set (map :date trans))))
|
|
|
|
|
|
(defn find-mistakes [trans-set]
|
|
(let [sequences (->> trans-set
|
|
(filter :runningBalance)
|
|
(map (fn [index t]
|
|
{:index index
|
|
:id (:id t)
|
|
:starting (- (:amount (:runningBalance t))
|
|
(if (= "DEBIT" (:baseType t))
|
|
(- (:amount (:amount t)))
|
|
(:amount (:amount t))))
|
|
:amount (if (= "DEBIT" (:baseType t))
|
|
(- (:amount (:amount t)))
|
|
(:amount (:amount t)))
|
|
:after (:amount (:runningBalance t))
|
|
:original t}
|
|
) (range)))
|
|
graph (reduce
|
|
(fn [graph s]
|
|
(assoc graph s
|
|
(filter (fn [s2]
|
|
(dollars= (:starting s2)
|
|
(:after s))
|
|
) sequences)
|
|
)
|
|
)
|
|
{}
|
|
sequences)
|
|
mistakes (filter
|
|
(fn [[k v]]
|
|
(> (count v) 1)
|
|
)
|
|
graph)]
|
|
|
|
(->> mistakes
|
|
(map second)
|
|
(map (fn [duplicates]
|
|
(map :original duplicates)))
|
|
)))
|
|
|
|
(csv/write-csv *out*
|
|
(for [a bad-as
|
|
:let [{:bank-account/keys [name code]} (first (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
|
|
:in $ ?yai
|
|
:where [?ba :bank-account/yodlee-account-id ?yai]]
|
|
(d/db conn)
|
|
a))
|
|
transactions (y/get-specific-transactions-with-date a "2021-01-01" "2021-12-31" (y/get-auth-header))]
|
|
|
|
|
|
mistakes (->> transactions
|
|
(group-by :date)
|
|
vals
|
|
(mapcat find-mistakes))
|
|
]
|
|
|
|
[code
|
|
name
|
|
(:date mistakes)
|
|
(:amount (:amount mistakes))])
|
|
:separator \tab)
|
|
|
|
(def mistake-set
|
|
(doall
|
|
(for [a bad-as
|
|
:let [{:bank-account/keys [name code]} (first (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
|
|
:in $ ?yai
|
|
:where [?ba :bank-account/yodlee-account-id ?yai]]
|
|
(d/db conn)
|
|
a))
|
|
transactions (y/get-specific-transactions-with-date a "2021-01-01" "2021-12-31" (y/get-auth-header))]
|
|
|
|
|
|
]
|
|
|
|
[code
|
|
(->> transactions
|
|
(group-by :date)
|
|
vals
|
|
(map find-mistakes)
|
|
(filter seq))])))
|
|
|
|
(user/init-repl)
|
|
|
|
mistake-set
|
|
|
|
|
|
(csv/write-csv *out*
|
|
(mapcat identity
|
|
(for [[[client mistake-set] i] (map vector mistake-set (range))
|
|
[mistake-set i2] (map vector mistake-set (range))
|
|
[mistake-set i3] (map vector mistake-set (range))
|
|
:let [already-fixed? (= (->> mistake-set
|
|
(map #(d/pull (d/db conn)
|
|
[:db/id {:transaction/approval-status [:db/ident]}]
|
|
[:transaction/id
|
|
(digest/sha-256 (str(:id %)))])
|
|
)
|
|
(filter :db/id)
|
|
(filter #(and (not= :transaction-approval-status/suppressed (:db/ident (:transaction/approval-status %)) )
|
|
(not= :transaction-approval-status/exclude-from-ledger (:db/ident (:transaction/approval-status %)))))
|
|
count)
|
|
1)]]
|
|
|
|
(mapv
|
|
(fn [m]
|
|
(let [iol-transaction (d/pull (d/db conn)
|
|
[:db/id {:transaction/approval-status [:db/ident]
|
|
:transaction/bank-account [:bank-account/code]}]
|
|
[:transaction/id
|
|
(digest/sha-256 (str(:id m)))])]
|
|
[(str i "-" i2 "-" i3)
|
|
(or (:bank-account/code (:transaction/bank-account iol-transaction))
|
|
"not found")
|
|
(:date m)
|
|
(:amount (:amount m))
|
|
(:id m)
|
|
(or (:db/id iol-transaction) "not found")
|
|
(or (some-> (:db/ident (:transaction/approval-status iol-transaction))
|
|
name)
|
|
"not found")
|
|
already-fixed?
|
|
]
|
|
))
|
|
mistake-set)))
|
|
|
|
:separator \tab)
|
|
|
|
(->> trans
|
|
(reverse)
|
|
(filter :runningBalance)
|
|
(partition 2 1)
|
|
(map (fn [[a b]]
|
|
(try
|
|
[
|
|
(- starting
|
|
(:amount (:runningBalance a)))
|
|
|
|
|
|
(- starting
|
|
(:amount (:runningBalance b))
|
|
(if (= "DEBIT" (:baseType b))
|
|
(- (:amount (:amount b)))
|
|
(:amount (:amount b))))
|
|
]
|
|
(catch Exception e
|
|
(println a b)
|
|
(println e)
|
|
(throw e)
|
|
))
|
|
))
|
|
(filter (fn [[a b]]
|
|
(not (dollars= a b))))
|
|
)
|
|
|
|
(csv/write-csv *out*
|
|
(->> trans
|
|
(reverse)
|
|
(map (fn [t]
|
|
[(:date t)
|
|
(:amount (:runningBalance t))
|
|
(if (= "DEBIT" (:baseType t))
|
|
(- (:amount (:amount t)))
|
|
(:amount (:amount t)))]
|
|
)))
|
|
:separator \tab)
|
|
|
|
|
|
|
|
(def ts (auto-ap.intuit.core/get-transactions "2022-02-01" "2022-02-05" "BCFM - Heritage Main 7362 ---BCFM-H7362"))
|
|
|
|
(count ts)
|
|
|
|
|
|
|
|
(defn check-transactions [db cc bac ba yba]
|
|
(let [all-transactions (d/q '[:find ?d ?a ?tx
|
|
:in $ ?ba
|
|
:where [?tx :transaction/bank-account ?ba]
|
|
[?tx :transaction/date ?d]
|
|
[(>= ?d #inst "2021-01-01T00:00:00-08:00")]
|
|
[(<= ?d #inst "2021-12-31T00:00:00-08:00")]
|
|
(not [?tx :transaction/approval-status :transaction-approval-status/suppressed])
|
|
(not [?tx :transaction/approval-status :transaction-approval-status/excluded])
|
|
[?tx :transaction/amount ?a]]
|
|
db ba)
|
|
not-excluded (fn [t]
|
|
(not (y/known-bad-yodlee-ids (:id t))))
|
|
all-yodlee-transactions (->> (y/get-specific-transactions-with-date yba "2021-01-01" "2021-12-31" (y/get-auth-header))
|
|
(filter not-excluded))
|
|
transactions-by-date (->> all-transactions
|
|
(group-by first)
|
|
(map (fn [[k v]]
|
|
[(atime/unparse-local (coerce/to-date-time k) iso-date) v]))
|
|
(into {}))
|
|
yodlee-transactions-by-date (->> all-yodlee-transactions
|
|
(group-by :date)
|
|
(into {}))
|
|
yodlee-last-updated (-> (y/get-account yba (y/get-auth-header)) first :dataset first :lastUpdated)
|
|
|
|
]
|
|
(for [d (->>
|
|
(periodic/periodic-seq (coerce/to-date-time #inst "2021-01-01T00:00:00-08:00") (t/days 1))
|
|
(map (fn [d]
|
|
(atime/unparse d iso-date)))
|
|
(take 365))
|
|
|
|
:let [txs (transactions-by-date d)
|
|
y-txes (yodlee-transactions-by-date d)
|
|
y-total (reduce + 0.0 (map y-tx->$ y-txes))
|
|
t-total (reduce + 0.0 (map (fn [[_ a]
|
|
]
|
|
a)
|
|
txs))]]
|
|
|
|
(into
|
|
[cc bac
|
|
yba d
|
|
(count txs ) (count y-txes)
|
|
t-total y-total]
|
|
|
|
(cond
|
|
(and
|
|
(= (count txs ) (count y-txes) 0)
|
|
(>= (.compareTo d yodlee-last-updated) 0))
|
|
["Match but yodlee feed stopped"
|
|
(format "Date %s is after yodlee feed ended (%s)", d yodlee-last-updated)]
|
|
|
|
(and
|
|
(= (count txs ) (count y-txes))
|
|
(dollars= t-total y-total))
|
|
["Perfect Match"
|
|
""]
|
|
|
|
|
|
(>= (.compareTo d yodlee-last-updated) 0)
|
|
["Mismatched because yodlee feed stopped"
|
|
(format "Date %s is after yodlee feed ended (%s)", d yodlee-last-updated)]
|
|
|
|
(not= (count txs ) (count y-txes))
|
|
["Mismatched number of transactions"
|
|
""]
|
|
|
|
(not (dollars= t-total y-total))
|
|
["Transactions don't add up"
|
|
""]
|
|
|
|
:else
|
|
["Mismatch"
|
|
"Mismatch, unknown cause"]
|
|
))
|
|
)))
|
|
|
|
|
|
(do ;;find iol duplicates
|
|
|
|
#_(check-transactions (d/db auto-ap.datomic/conn) 17592188489475 18911886 )
|
|
|
|
(csv/write-csv *out*
|
|
(let [db (d/db auto-ap.datomic/conn)]
|
|
(for [[ba yba bac cc] (d/q '[:find ?ba ?yba ?bac ?cc
|
|
:in $
|
|
:where [?ba :bank-account/yodlee-account-id ?yba]
|
|
[(not= ?yba 0)]
|
|
[?ba :bank-account/code ?bac]
|
|
[?c :client/bank-accounts ?ba]
|
|
[?c :client/code ?cc]]
|
|
db
|
|
)
|
|
row (try (check-transactions db cc bac ba yba)
|
|
(catch Exception _
|
|
[]))]
|
|
row))
|
|
:separator \tab)
|
|
|
|
)
|