Dry running intuit

src/clj/auto_ap/intuit/core.clj

@@ -15,7 +15,8 @@
             [mount.core :as mount]
             [unilog.context :as lc]
             [yang.scheduler :as scheduler]
-            [clojure.core.async :as async])
+            [clojure.core.async :as async]
+            [clojure.core.memoize :as m])
   (:import [org.apache.commons.codec.binary Base64]))
 
 
@@ -119,27 +120,29 @@
 
 
 
-(defn get-transactions []
-  (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=2021-11-30&end_date=2021-11-30") 
-              {:headers prod-base-headers
-               :as :json}))
 
-(defn get-transactions [start end external-id]
+(defn get-all-transactions [start end]
   (let [token (get-fresh-access-token)]
     (let [body (:body (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=" start "&end_date=" end) 
                                   {:headers (with-auth prod-base-headers token)
-                                   :as :json}))
+                                   :as :json}))]
-          headers (map :ColTitle (:Column (:Columns body)))
+      body)))
-          rows (map :ColData (:Row (:Rows body)))]
+
-      (->> rows
+(def memoize-get-all-transactions (m/ttl get-all-transactions :ttl/threshold 60000))
-           (map
+
-            (fn [row]
+(defn get-transactions [start end external-id]
-              (into {}
+  (let [body (memoize-get-all-transactions start end)
-                    (map
+        headers (map :ColTitle (:Column (:Columns body)))
-                     (fn [h r]
+        rows (map :ColData (:Row (:Rows body)))]
-                       [(keyword h) (:value r)])
+    (->> rows
-                     headers
+         (map
-                     row))))
+          (fn [row]
-           (filter #(str/includes? (:Account %) external-id))))))
+            (into {}
+                  (map
+                   (fn [h r]
+                     [(keyword h) (:value r)])
+                   headers
+                   row))))
+         (filter #(str/includes? (:Account %) external-id)))))
 
 

src/clj/auto_ap/intuit/import.clj

@@ -14,18 +14,19 @@
             [unilog.context :as lc]
             [yang.scheduler :as scheduler]))
 
+(defn get-intuit-bank-accounts [db]
+  (-> (d/q '[:find ?ba ?external-id
+             :in $
+             :where [?ba :bank-account/intuit-bank-account ?iab]
+             [?iab :intuit-bank-account/external-id ?external-id]]
+           db)))
 
 (defn upsert-transactions []
   (let [db (d/db conn)]
-    (doseq [[bank-account external-id] (-> (d/q '[:find ?ba ?external-id
+    (doseq [[bank-account external-id] (get-intuit-bank-accounts db)
-                                                  :in $
-                                                  :where [?ba :bank-account/intuit-bank-account ?iab]
-                                                  [?iab :intuit-bank-account/external-id ?external-id]]
-
-                                                db))
             :let [bank-account (d/entity db bank-account)
                   end (auto-ap.time/local-now)
-                  start (time/plus end (time/days -90))]
+                  start (time/plus end (time/days -30))]
             ]
       (log/infof "importing from %s to %s for %s" start end external-id)
       (let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
@@ -39,7 +40,36 @@
                                        :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
                                        :status "posted"})))]
         (log/infof "%d transactions found" (count transactions))
-        (y/grouped-import transactions)))))
+        #_(y/grouped-import transactions)
+        transactions))))
+
+
+(defn dry-run-upsert-transactions []
+  (let [db (d/db conn)]
+    (clojure.data.csv/write-csv
+     *out*
+     (->> 
+      (for [[bank-account external-id]  (get-intuit-bank-accounts db)
+            :let [bank-account (d/entity db bank-account)
+                  end (auto-ap.time/local-now)
+                  start (time/plus end (time/days -30))]
+            r (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
+                                  (auto-ap.time/unparse end auto-ap.time/iso-date)
+                                  external-id)
+
+            ]
+        {:client-id (:db/id (:client/_bank-accounts bank-account))
+         :client-code (:client/code (:client/_bank-accounts bank-account))
+         :bank-account-id (:db/id bank-account)
+         :bank-account-code (:bank-account/code bank-account)
+         :external-id external-id
+         :description-original (:Memo/Description r)
+         :amount (Double/parseDouble (:Amount r))
+         :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
+         :status "posted"})
+      (y/grouped-new)
+      (map (juxt :client-code :bank-account-code :external-id :description-original :amount #(auto-ap.time/unparse-local (:date %) auto-ap.time/normal-date) :new?)))
+     :separator \tab)))
 
 
 (defn upsert-accounts []

src/clj/auto_ap/yodlee/import.clj

@@ -334,11 +334,30 @@
           transaction->bank-account (comp all-bank-accounts :bank-account-id)]
       (log/info "Importing " (count transformed-transactions) " grouped transactions")
       
-      (doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
+      #_(doseq [tx (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))]
         (audit-transact tx {:user/name "Yodlee import"
                             :user/role ":admin"}))
+      (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))
       (log/info "Imported  grouped transactions"))))
 
+(defn grouped-new [manual-transactions]
+  (let [transformed-transactions (->> manual-transactions
+                                      (filter #(= "posted" (:status %)))
+                                      (group-by #(select-keys % [:date :description-original :amount]))
+                                      (vals)
+                                      (mapcat (fn [transaction-group]
+                                                (map
+                                                 (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
+                                                   (assoc transaction :id 
+                                                          (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)))
+                                                 (range)
+                                                 transaction-group))))
+        existing (get-existing)]
+    (map #(if (existing (sha-256 (str (:id %))))
+            (assoc % :new? false)
+            (assoc % :new? true))
+         transformed-transactions)))
+
 (defn do-import
   ([]
    (do-import (client/get-transactions)))