notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dry running intuit

Browse Source
This commit is contained in:
Bryce Covert
2021-12-14 11:18:24 -08:00
parent 1178e723d1
commit b7324af538
3 changed files with 80 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/clj/auto_ap/intuit/core.clj
View File

@@ -15,7 +15,8 @@
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]
[clojure.core.async :as async])
[clojure.core.async :as async]
[clojure.core.memoize :as m])
(:import [org.apache.commons.codec.binary Base64]))
@@ -119,27 +120,29 @@
(defn get-transactions []
(client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=2021-11-30&end_date=2021-11-30")
{:headers prod-base-headers
:as :json}))
(defn get-transactions [start end external-id]
(defn get-all-transactions [start end]
(let [token (get-fresh-access-token)]
(let [body (:body (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=" start "&end_date=" end)
{:headers (with-auth prod-base-headers token)
:as :json}))
headers (map :ColTitle (:Column (:Columns body)))
rows (map :ColData (:Row (:Rows body)))]
(->> rows
(map
(fn [row]
(into {}
(map
(fn [h r]
[(keyword h) (:value r)])
headers
row))))
(filter #(str/includes? (:Account %) external-id))))))
:as :json}))]
body)))
(def memoize-get-all-transactions (m/ttl get-all-transactions :ttl/threshold 60000))
(defn get-transactions [start end external-id]
(let [body (memoize-get-all-transactions start end)
headers (map :ColTitle (:Column (:Columns body)))
rows (map :ColData (:Row (:Rows body)))]
(->> rows
(map
(fn [row]
(into {}
(map
(fn [h r]
[(keyword h) (:value r)])
headers
row))))
(filter #(str/includes? (:Account %) external-id)))))

46
src/clj/auto_ap/intuit/import.clj
View File

@@ -14,18 +14,19 @@
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
(defn get-intuit-bank-accounts [db]
(-> (d/q '[:find ?ba ?external-id
:in $
:where [?ba :bank-account/intuit-bank-account ?iab]
[?iab :intuit-bank-account/external-id ?external-id]]
db)))
(defn upsert-transactions []
(let [db (d/db conn)]
(doseq [[bank-account external-id] (-> (d/q '[:find ?ba ?external-id
:in $
:where [?ba :bank-account/intuit-bank-account ?iab]
[?iab :intuit-bank-account/external-id ?external-id]]
db))
(doseq [[bank-account external-id] (get-intuit-bank-accounts db)
:let [bank-account (d/entity db bank-account)
end (auto-ap.time/local-now)
start (time/plus end (time/days -90))]
start (time/plus end (time/days -30))]
]
(log/infof "importing from %s to %s for %s" start end external-id)
(let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
@@ -39,7 +40,36 @@
:date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
:status "posted"})))]
(log/infof "%d transactions found" (count transactions))
(y/grouped-import transactions)))))
#_(y/grouped-import transactions)
transactions))))
(defn dry-run-upsert-transactions []
(let [db (d/db conn)]
(clojure.data.csv/write-csv
*out*
(->>
(for [[bank-account external-id] (get-intuit-bank-accounts db)
:let [bank-account (d/entity db bank-account)
end (auto-ap.time/local-now)
start (time/plus end (time/days -30))]
r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
(auto-ap.time/unparse end auto-ap.time/iso-date)
external-id)
]
{:client-id (:db/id (:client/_bank-accounts bank-account))
:client-code (:client/code (:client/_bank-accounts bank-account))
:bank-account-id (:db/id bank-account)
:bank-account-code (:bank-account/code bank-account)
:external-id external-id
:description-original (:Memo/Description r)
:amount (Double/parseDouble (:Amount r))
:date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
:status "posted"})
(y/grouped-new)
(map (juxt :client-code :bank-account-code :external-id :description-original :amount #(auto-ap.time/unparse-local (:date %) auto-ap.time/normal-date) :new?)))
:separator \tab)))
(defn upsert-accounts []

21
src/clj/auto_ap/yodlee/import.clj
View File

@@ -334,11 +334,30 @@
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
(log/info "Importing " (count transformed-transactions) " grouped transactions")
(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
#_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
(audit-transact tx {:user/name "Yodlee import"
:user/role ":admin"}))
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))
(log/info "Imported grouped transactions"))))
(defn grouped-new [manual-transactions]
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount]))
(vals)
(mapcat (fn [transaction-group]
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
(assoc transaction :id
(str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)))
(range)
transaction-group))))
existing (get-existing)]
(map #(if (existing (sha-256 (str (:id %))))
(assoc % :new? false)
(assoc % :new? true))
transformed-transactions)))
(defn do-import
([]
(do-import (client/get-transactions)))