(ns debug-sccb-transactions
  (:require [datomic.api :as d]
            [auto-ap.datomic :refer [conn]]
            [auto-ap.utils :refer [dollars=]]
            [auto-ap.time :as atime :refer [iso-date]]
            [clj-time.core :as t]
            [clj-time.coerce :as coerce]
            [clj-time.periodic :as periodic]
            [clojure.data.csv :as csv]

            [auto-ap.yodlee.core :as y]
            [auto-ap.yodlee.core2 :as y2]))

(d/pull (d/db conn) '[*] [:bank-account/code "SCCB-9128CB"])

(def trans (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header)))

(user/init-repl)

[(->> trans
      (sort-by :date)
      (map (fn [t]
             (if (= "DEBIT" (:baseType t))
               (-  (:amount (:amount t)))
               (:amount (:amount t)))))
      (reduce + 0.0))
 (:amount (:runningBalance (first trans)))
 (:amount (:runningBalance (last trans)))
 (- (:amount (:runningBalance (last trans)))
    (:amount (:runningBalance (first trans))))]


(defn y-tx->$ [b]
  (if (= "DEBIT" (:baseType b))
    (-  (:amount (:amount b)))
    (:amount (:amount b))))


(do 
  
  (doseq [bank-account #_ (d/q '[:find [?yai ...]
                                 :in $
                                 :where [_ :bank-account/yodlee-account-id ?yai]]
                               (d/db conn))
          :when        (not= 0 bank-account)]
    (try 
      (println 
        [bank-account  (->> (y/get-specific-transactions-with-date bank-account "2021-01-01" "2021-12-31" (y/get-auth-header))
                            (reverse)
                            (filter :runningBalance)
                            (partition 2 1)
                            (map (fn [[a b]]
                                   (try 
                                     [(:amount (:runningBalance a))
                                      
                                      (- (:amount (:runningBalance b))
                                         (if (= "DEBIT" (:baseType b))
                                           (-  (:amount (:amount b)))
                                           (:amount (:amount b))))
                                      ]
                                     (catch Exception e
                                       (println a b)
                                       (println e)
                                       (throw e)
                                       ))
                                   ))
                            (filter (fn [[a b]]
                                      (not (dollars= a b))))
                            count
                            
                            )])
      (catch Exception e
        (println "please retry" bank-account)))
    )
  (println "done"))



(take 10 trans)

(first trans)
(last trans)


(csv/write-csv *out*
               (->> trans
                    (sort-by :date)
                    (map (fn [z]
                           (update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
                    (map (fn [z]
                           [(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
                                                                                                     (-  (:amount (:amount z)))
                                                                                                     (:amount (:amount z)))]
                           ))
                    
                    )

               :separator \tab
               )

(def deep-dive (y/get-specific-transactions-with-date 27995674 "2021-12-03" "2021-12-03" (y/get-auth-header)))


(csv/write-csv *out*
               (->> (->> (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header))
                         (group-by (fn [d]
                                     ((juxt :description
                                            :amount
                                            :date
                                            :baseType
                                            :type
                                            :isManual
                                            :merchantType
                                            :status
                                            :CONTAINER
                                            :runningBalance 
                                            :subType
                                            :isPhysical
                                            :accountId
                                            :postDate) d)))
                         (map (fn [[k v]]
                                (first (sort-by :id v)))))
                    (sort-by :date)
                    (map (fn [z]
                           (update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
                    (map (fn [z]
                           [(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
                                                                     (-  (:amount (:amount z)))
                                                                     (:amount (:amount z)))]
                           ))
                    
                    )

               :separator \tab
               )

(->> (y/get-specific-transactions-with-date 27995674 "2021-12-30" "2021-12-30" (y/get-auth-header))
                         (group-by (fn [d]
                                     ((juxt :description
                                            :amount
                                            :date
                                            :baseType
                                            :type
                                            :isManual
                                            :merchantType
                                            :status
                                            :CONTAINER
                                            :runningBalance 
                                            :subType
                                            :isPhysical
                                            :accountId
                                            :postDate) d)))
                         (map (fn [[k v]]
                                [k (count v)]
                                  #_(clojure.data/diff a b))))

(y2/get-accounts "DEMO2")


(->> (y2/get-specific-transactions "DEMO2" 15565801)
                         (group-by (fn [d]
                                     ((juxt :description
                                            :amount
                                            :date
                                            :baseType
                                            :type
                                            :isManual
                                            :merchantType
                                            :status
                                            :CONTAINER
                                            :runningBalance 
                                            :subType
                                            :isPhysical
                                            :accountId
                                            :postDate) d)))
                         (map (fn [[k v]]
                                [k (count v)]
                                #_(clojure.data/diff a b)))
                         count)

(csv/write-csv *out*
               (->> (y2/get-specific-transactions "DEMO2" 15565801)
                    (sort-by :date)
                    (map (fn [z]
                           (update z :date #(auto-ap.time/unparse (auto-ap.time/parse % auto-ap.time/iso-date) auto-ap.time/normal-date))))
                    (map (fn [z]
                           [(:date z) (:original (:description z)) (if (= "DEBIT" (:baseType z))
                                                                     (-  (:amount (:amount z)))
                                                                     (:amount (:amount z)))]
                           )))
               :separator \tab)



(def bad-as [24265567
             16422563
             27327396
             16279665
             16279668
             16279669
             16279670
             16279671
             24230113
             16279664
             16421944
             24287315
             27723398
             27723397
             16422358
             18685498
             16422285
             24370905
             18911886
             27995674
             19935858
             16422708])


(def all-as  [
[18409209 0]
[18409210 0]
[27040638 0]
[18409211 0]
[27040639 0]
[27040640 0]
[27398018 0]
[24249175 0]
[16551129 0]
[24781151 0]
[24265567 2433]
[25829735 0]
[25976674 0]
[25829733 0]
[25829734 0]
[26403171 0]
[26403172 0]
[26403169 0]
[26403170 0]
[19679001 0]
[20772618 0]
[26403173 0]
[27509142 0]
[19679000 0]
[18937120 0]
[22730534 0]
[25640840 0]
[16551102 0]
[20186867 0]
[27784044 0]
[26216350 0]
[27784043 0]
[27784042 0]
[21961051 0]
[16422563 48]
[22981959 0]
[16422564 0]
[18356995 0]
[16422562 0]
[27040641 0]
[27040642 0]
[27040643 0]
[26997692 0]
[26997691 0]
[26793947 0]
[27327396 64]
[16279665 83]
[16279666 0]
[26988499 0]
[16279667 0]
[16279668 12]
[16279669 16]
[16279670 55]
[16279671 4]
[16279672 0]
[24230113 80]
[25624533 0]
[25624532 0]
[16279663 0]
[16279664 441]
[26991555 0]
[27798450 0]
[27798451 0]
[26607050 0]
[16422491 0]
[16422492 0]
[16422495 0]
[16422493 0]
[27847951 0]
[16422494 0]
[25646832 0]
[25646831 0]
[16421963 0]
[16422473 0]
[26920161 0]
[16422471 0]
[16422472 0]
[16421945 0]
[16422463 0]
[16422462 0]
[24591293 0]
[16421943 0]
[16421944 8]
[24566177 0]
[24434091 0]
[24434092 0]
[24104846 0]
[24104845 0]
[24104844 0]
[24411026 0]
[24104839 0]
[24104838 0]
[21754236 0]
[27576149 0]
[18099105 0]
[24499586 0]
[18937304 0]
[18937305 0]
[25308479 0]
[25308480 0]
[24104849 0]
[23662442 0]
[25308484 0]
[25308481 0]
[16876498 0]
[24265569 0]
[16876499 0]
[24265570 0]
[24287315 317]
[26983548 0]
[24287316 0]
[27723399 0]
[27723398 21]
[27723397 110]
[24287319 0]
[27576450 0]
[24287317 0]
[27576451 0]
[24287318 0]
[26983549 0]
[27576452 0]
[26794100 0]
[24287326 0]
[16422358 13]
[17303033 0]
[26942060 0]
[24776257 0]
[16876551 0]
[22883931 0]
[17970207 0]
[27361904 0]
[27361903 0]
[19941347 0]
[27345526 0]
[16417200 0]
[16428453 0]
[19756531 0]
[16428443 0]
[18685498 18]
[27465416 0]
[27465415 0]
[27465417 0]
[16422296 0]
[27847900 0]
[27847902 0]
[16422285 11]
[25350341 0]
[16428403 0]
[16551286 0]
[27282619 0]
[27282620 0]
[19942033 0]
[19942034 0]
[26220018 0]
[27780614 0]
[27793421 0]
[17303150 0]
[27793422 0]
[16428372 0]
[24370905 450]
[16428375 0]
[18911886 44]
[16428373 0]
[16428374 0]
[18911888 0]
[27995676 0]
[27995675 0]
[27995674 861]
[19942029 0]
[17408629 0]
[19942030 0]
[22219439 0]
[19942031 0]
[19942032 0]
[25543675 0]
[19935858 606]
[16422707 0]
[16422708 320]
[27116551 0]
[16428317 0]
[16428318 0]
[24510110 0]
[27847749 0]
[654654654 0]
[21489791 0]
[21489790 0]
[23139427 0]
[16422643 0]
[16422644 0]
[16422642 0]
[16428279 0]
[25247822 0]
[18409174 0]
[16422645 0]
[18948276 0]
[18948275 0]
])
(filter (fn [[_ x]] (not= 0 x)) all-as)

(->> (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
            :in $ [?yai ...]
            :where [?ba :bank-account/yodlee-account-id ?yai]]
          (d/db conn)
          
          )
     (map (juxt :bank-account/name :bank-account/code)))


(def trans (y/get-specific-transactions-with-date 27995674 "2021-01-01" "2021-12-31" (y/get-auth-header)))

(def starting (:amount (:runningBalance (last trans))))

(def dates (sort (set (map :date trans))))


(defn find-mistakes [trans-set]
  (let [sequences (->> trans-set
                       (filter :runningBalance)
                       (map (fn [index t]
                              {:index index
                               :id (:id t)
                               :starting (- (:amount (:runningBalance t))
                                            (if (= "DEBIT" (:baseType t))
                                              (-  (:amount (:amount t)))
                                              (:amount (:amount t))))
                               :amount   (if (= "DEBIT" (:baseType t))
                                         (-  (:amount (:amount t)))
                                         (:amount (:amount t)))
                               :after    (:amount (:runningBalance t))
                               :original t}
                              ) (range)))
        graph (reduce 
                (fn [graph s]
                  (assoc graph s
                         (filter (fn [s2]
                                   (dollars= (:starting s2)
                                             (:after s))
                                   ) sequences)
                         )
                  )
                {}
                sequences)
        mistakes (filter
                   (fn [[k v]]
                     (> (count v) 1)
                     )
                   graph)]
    
    (->> mistakes
         (map second)
         (map (fn [duplicates]
                   (map :original duplicates)))
         )))

(csv/write-csv *out* 
               (for [a    bad-as
                     :let [{:bank-account/keys [name code]} (first (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
                                                                          :in $ ?yai
                                                                          :where [?ba :bank-account/yodlee-account-id ?yai]]
                                                                        (d/db conn)
                                                                        a))
                           transactions  (y/get-specific-transactions-with-date a "2021-01-01" "2021-12-31" (y/get-auth-header))]

                     
                     mistakes (->> transactions
                                   (group-by :date)
                                   vals
                                   (mapcat find-mistakes))
                     ]
                 
                 [code
                  name
                  (:date mistakes)
                  (:amount (:amount mistakes))])
               :separator \tab)

(def mistake-set 
  (doall 
    (for [a    bad-as
          :let [{:bank-account/keys [name code]} (first (d/q '[:find [(pull ?ba [:bank-account/name :bank-account/code]) ...]
                                                               :in $ ?yai
                                                               :where [?ba :bank-account/yodlee-account-id ?yai]]
                                                             (d/db conn)
                                                             a))
                transactions  (y/get-specific-transactions-with-date a "2021-01-01" "2021-12-31" (y/get-auth-header))]

          
          ]
      
      [code 
       (->> transactions
            (group-by :date)
            vals
            (map find-mistakes)
            (filter seq))])))

(user/init-repl)

mistake-set


(csv/write-csv *out* 
               (mapcat identity 
                       (for [[[client mistake-set] i] (map vector mistake-set (range))
                             [mistake-set i2]          (map vector mistake-set (range))
                             [mistake-set i3]          (map vector mistake-set (range))
                             :let                 [already-fixed? (= (->> mistake-set
                                                                          (map #(d/pull (d/db conn)
                                                                                        [:db/id {:transaction/approval-status [:db/ident]}]
                                                                                        [:transaction/id
                                                                                         (digest/sha-256 (str(:id %)))])
                                                                               )
                                                                          (filter :db/id)
                                                                          (filter #(and (not= :transaction-approval-status/suppressed (:db/ident  (:transaction/approval-status %)) )
                                                                                        (not= :transaction-approval-status/exclude-from-ledger (:db/ident  (:transaction/approval-status %)))))
                                                                          count)
                                                                     1)]]

                         (mapv
                           (fn [m]
                             (let [iol-transaction (d/pull (d/db conn)
                                                           [:db/id {:transaction/approval-status [:db/ident]
                                                                    :transaction/bank-account    [:bank-account/code]}]
                                                           [:transaction/id
                                                            (digest/sha-256 (str(:id m)))])]
                               [(str i "-" i2 "-" i3)
                                (or (:bank-account/code (:transaction/bank-account iol-transaction))
                                    "not found")
                                (:date m)
                                (:amount (:amount m))
                                (:id m)
                                (or (:db/id iol-transaction) "not found")
                                (or (some-> (:db/ident (:transaction/approval-status iol-transaction))
                                            name)
                                    "not found")
                                already-fixed?
                                ]
                               ))
                           mistake-set)))

               :separator \tab)

(->> trans 
     (reverse)
     (filter :runningBalance)
     (partition 2 1)
     (map (fn [[a b]]
            (try 
              [
               (- starting
                  (:amount (:runningBalance a)))
               
               
               (- starting
                 (:amount (:runningBalance b))
                  (if (= "DEBIT" (:baseType b))
                    (-  (:amount (:amount b)))
                    (:amount (:amount b))))
               ]
              (catch Exception e
                (println a b)
                (println e)
                (throw e)
                ))
            ))
     (filter (fn [[a b]]
               (not (dollars= a b))))
     )

(csv/write-csv *out* 
               (->> trans
                    (reverse)
                    (map (fn [t]
                           [(:date t)
                            (:amount (:runningBalance t))
                            (if (= "DEBIT" (:baseType t))
                              (-  (:amount (:amount t)))
                              (:amount (:amount t)))]
                           )))
               :separator \tab)



(def ts (auto-ap.intuit.core/get-transactions "2022-02-01" "2022-02-05" "BCFM - Heritage Main 7362 ---BCFM-H7362"))

(count ts)



(defn check-transactions [db cc bac ba yba]
    (let [all-transactions (d/q '[:find ?d ?a ?tx
                                  :in $ ?ba
                                  :where [?tx :transaction/bank-account ?ba]
                                  [?tx :transaction/date ?d]
                                  [(>= ?d #inst "2021-01-01T00:00:00-08:00")]
                                  [(<= ?d #inst "2021-12-31T00:00:00-08:00")]
                                  (not [?tx :transaction/approval-status :transaction-approval-status/suppressed])
                                  (not [?tx :transaction/approval-status :transaction-approval-status/excluded])
                                  [?tx :transaction/amount ?a]]
                                db ba)
          not-excluded (fn [t]
                         (not (y/known-bad-yodlee-ids (:id t))))
          all-yodlee-transactions (->> (y/get-specific-transactions-with-date yba "2021-01-01" "2021-12-31" (y/get-auth-header))
                                       (filter not-excluded))
          transactions-by-date (->> all-transactions
                                    (group-by first)
                                    (map (fn [[k v]]
                                           [(atime/unparse-local (coerce/to-date-time k) iso-date) v]))
                                    (into {}))
          yodlee-transactions-by-date (->> all-yodlee-transactions
                                           (group-by :date)
                                           (into {}))
          yodlee-last-updated (-> (y/get-account yba (y/get-auth-header)) first :dataset first :lastUpdated)

          ]
      (for [d (->> 
                (periodic/periodic-seq (coerce/to-date-time #inst "2021-01-01T00:00:00-08:00") (t/days 1))
                (map (fn [d]
                       (atime/unparse d iso-date)))
                (take 365))
            
            :let    [txs (transactions-by-date d)
                     y-txes (yodlee-transactions-by-date d)
                     y-total (reduce + 0.0 (map y-tx->$ y-txes))
                     t-total (reduce + 0.0 (map (fn [[_ a]
                                                     ]
                                                  a)
                                                txs))]]
        
        (into 
          [cc bac
           yba d
           (count txs ) (count y-txes)
           t-total y-total]

          (cond
            (and 
              (= (count txs ) (count y-txes) 0)
              (>= (.compareTo d yodlee-last-updated) 0))
            ["Match but yodlee feed stopped"
             (format "Date %s is after yodlee feed ended (%s)", d yodlee-last-updated)]

            (and 
              (= (count txs ) (count y-txes))
              (dollars= t-total y-total))
            ["Perfect Match"
             ""]
            

            (>= (.compareTo d yodlee-last-updated) 0)
            ["Mismatched because yodlee feed stopped"
             (format "Date %s is after yodlee feed ended (%s)", d yodlee-last-updated)]

            (not= (count txs ) (count y-txes))
            ["Mismatched number of transactions"
             ""]

            (not (dollars= t-total y-total))
            ["Transactions don't add up"
             ""]

            :else
            ["Mismatch"
             "Mismatch, unknown cause"]
            ))
        )))


(do ;;find iol duplicates
  
  #_(check-transactions (d/db auto-ap.datomic/conn) 17592188489475 18911886 )

  (csv/write-csv *out* 
                 (let [db (d/db auto-ap.datomic/conn)]
                   (for [[ba yba bac cc] (d/q '[:find ?ba ?yba ?bac ?cc
                                                :in $
                                                :where [?ba :bank-account/yodlee-account-id ?yba]
                                                [(not= ?yba 0)]
                                                [?ba :bank-account/code ?bac]
                                                [?c :client/bank-accounts ?ba]
                                                [?c :client/code ?cc]]
                                              db
                                              )
                         row             (try (check-transactions db cc bac ba yba)
                                              (catch Exception _
                                                []))]
                     row))
                 :separator \tab)

  )