From b7324af5387c4e05eaed04567efabd7f7609369c Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 14 Dec 2021 11:18:24 -0800 Subject: [PATCH] Dry running intuit --- src/clj/auto_ap/intuit/core.clj | 41 ++++++++++++++------------- src/clj/auto_ap/intuit/import.clj | 46 +++++++++++++++++++++++++------ src/clj/auto_ap/yodlee/import.clj | 21 +++++++++++++- 3 files changed, 80 insertions(+), 28 deletions(-) diff --git a/src/clj/auto_ap/intuit/core.clj b/src/clj/auto_ap/intuit/core.clj index 12b12b9a..8a55a4f3 100644 --- a/src/clj/auto_ap/intuit/core.clj +++ b/src/clj/auto_ap/intuit/core.clj @@ -15,7 +15,8 @@ [mount.core :as mount] [unilog.context :as lc] [yang.scheduler :as scheduler] - [clojure.core.async :as async]) + [clojure.core.async :as async] + [clojure.core.memoize :as m]) (:import [org.apache.commons.codec.binary Base64])) @@ -119,27 +120,29 @@ -(defn get-transactions [] - (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=2021-11-30&end_date=2021-11-30") - {:headers prod-base-headers - :as :json})) -(defn get-transactions [start end external-id] +(defn get-all-transactions [start end] (let [token (get-fresh-access-token)] (let [body (:body (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=" start "&end_date=" end) {:headers (with-auth prod-base-headers token) - :as :json})) - headers (map :ColTitle (:Column (:Columns body))) - rows (map :ColData (:Row (:Rows body)))] - (->> rows - (map - (fn [row] - (into {} - (map - (fn [h r] - [(keyword h) (:value r)]) - headers - row)))) - (filter #(str/includes? (:Account %) external-id)))))) + :as :json}))] + body))) + +(def memoize-get-all-transactions (m/ttl get-all-transactions :ttl/threshold 60000)) + +(defn get-transactions [start end external-id] + (let [body (memoize-get-all-transactions start end) + headers (map :ColTitle (:Column (:Columns body))) + rows (map :ColData (:Row (:Rows body)))] + (->> rows + (map + (fn [row] + (into {} + (map + (fn [h r] + [(keyword h) (:value r)]) + headers + row)))) + (filter #(str/includes? (:Account %) external-id))))) diff --git a/src/clj/auto_ap/intuit/import.clj b/src/clj/auto_ap/intuit/import.clj index 6bcbab6c..6aec5663 100644 --- a/src/clj/auto_ap/intuit/import.clj +++ b/src/clj/auto_ap/intuit/import.clj @@ -14,18 +14,19 @@ [unilog.context :as lc] [yang.scheduler :as scheduler])) +(defn get-intuit-bank-accounts [db] + (-> (d/q '[:find ?ba ?external-id + :in $ + :where [?ba :bank-account/intuit-bank-account ?iab] + [?iab :intuit-bank-account/external-id ?external-id]] + db))) (defn upsert-transactions [] (let [db (d/db conn)] - (doseq [[bank-account external-id] (-> (d/q '[:find ?ba ?external-id - :in $ - :where [?ba :bank-account/intuit-bank-account ?iab] - [?iab :intuit-bank-account/external-id ?external-id]] - - db)) + (doseq [[bank-account external-id] (get-intuit-bank-accounts db) :let [bank-account (d/entity db bank-account) end (auto-ap.time/local-now) - start (time/plus end (time/days -90))] + start (time/plus end (time/days -30))] ] (log/infof "importing from %s to %s for %s" start end external-id) (let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) @@ -39,7 +40,36 @@ :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) :status "posted"})))] (log/infof "%d transactions found" (count transactions)) - (y/grouped-import transactions))))) + #_(y/grouped-import transactions) + transactions)))) + + +(defn dry-run-upsert-transactions [] + (let [db (d/db conn)] + (clojure.data.csv/write-csv + *out* + (->> + (for [[bank-account external-id] (get-intuit-bank-accounts db) + :let [bank-account (d/entity db bank-account) + end (auto-ap.time/local-now) + start (time/plus end (time/days -30))] + r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date) + (auto-ap.time/unparse end auto-ap.time/iso-date) + external-id) + + ] + {:client-id (:db/id (:client/_bank-accounts bank-account)) + :client-code (:client/code (:client/_bank-accounts bank-account)) + :bank-account-id (:db/id bank-account) + :bank-account-code (:bank-account/code bank-account) + :external-id external-id + :description-original (:Memo/Description r) + :amount (Double/parseDouble (:Amount r)) + :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) + :status "posted"}) + (y/grouped-new) + (map (juxt :client-code :bank-account-code :external-id :description-original :amount #(auto-ap.time/unparse-local (:date %) auto-ap.time/normal-date) :new?))) + :separator \tab))) (defn upsert-accounts [] diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 58fd8688..6558048e 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -334,11 +334,30 @@ transaction->bank-account (comp all-bank-accounts :bank-account-id)] (log/info "Importing " (count transformed-transactions) " grouped transactions") - (doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + #_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] (audit-transact tx {:user/name "Yodlee import" :user/role ":admin"})) + (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)) (log/info "Imported grouped transactions")))) +(defn grouped-new [manual-transactions] + (let [transformed-transactions (->> manual-transactions + (filter #(= "posted" (:status %))) + (group-by #(select-keys % [:date :description-original :amount])) + (vals) + (mapcat (fn [transaction-group] + (map + (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] + (assoc transaction :id + (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id))) + (range) + transaction-group)))) + existing (get-existing)] + (map #(if (existing (sha-256 (str (:id %)))) + (assoc % :new? false) + (assoc % :new? true)) + transformed-transactions))) + (defn do-import ([] (do-import (client/get-transactions)))