beginnings of better import.

src/clj/auto_ap/intuit/core.clj

@@ -0,0 +1,145 @@
+(ns auto-ap.intuit.core
+  (:require [auto-ap.datomic :refer [conn remove-nils]]
+            [amazonica.aws.s3 :as s3]
+            [auto-ap.utils :refer [by]]
+            [clj-http.client :as client]
+            [clj-time.coerce :as coerce]
+            [clj-time.core :as time]
+            [clj-time.format :as f]
+            [config.core :refer [env] :as cfg ]
+            [clojure.string :as str]
+            [clojure.data.json :as json]
+            [clojure.java.io :as io]
+            [clojure.tools.logging :as log]
+            [datomic.api :as d]
+            [mount.core :as mount]
+            [unilog.context :as lc]
+            [yang.scheduler :as scheduler]
+            [clojure.core.async :as async])
+  (:import [org.apache.commons.codec.binary Base64]))
+
+
+(def authorization-code "AB11638463964I0tYPR3A1inog2HL407u2bZBXHg6LEqCbILRO")
+(def realm-id "4620816365202617680")
+(def access-token "eyJlbmMiOiJBMTI4Q0JDLUhTMjU2IiwiYWxnIjoiZGlyIn0..PUPVn2EYxMCGTIqIjaAm5Q.jV43shT64akeaLZ0JDV_B9kW-rpDH6G-UHqOQgK6Tsupju9noyW6ct1XZxOq6m866kksisdTALzRtojdHhXGSArpk9aql6eSdzr0tq8KvxFNDfURw8GYY7cwbElkD6F0DAfotssFMXYTpGma6EfQz2u4q7UshHDnLe3Fk8EfRn4wob1FrcjB4KcuVNw86F959TMcMQ78N7joYs9PP5OwEnvQ5_Eg9HRMlKcFjs5YQJJW3t6i3j10gn0P7iXnZbIKHF_h_67cd2XJMS0bfyW-X0TfPVuHH0THNEeDM7al86L98fINtUeOKijDY7iJSDWOz0Kma41PM2fvWQbg0JVkpHqaK6mABRzMfK9XuX2pyskKc2AaODtPcB-Uv6WHcgGfND9BHlULweNeI1CsX0NZFFoH8P59XfdmYy92Ul5kHH_ND3D60e4v6mgy3TSSyPkCx6rNZdPCzxMwAUT86k-VXW1tapPmPLdDOiOfnS9UI_tc51YeExWRpDMnHRbIa2TRlaAchG9h41ZrgUiuKl9QoLRhPXP1yg8O5MCQbPzB23sQWN9OngO62opwvnqQg2kbn1S5wHxmpI0a3QPmegFCF9idy8lQVlUXp2myw_WuCZI6SZa5gbbKD33yo2crQX5-gg7ImyGT7tYGe-C440lQC8BWcL2gPr1gS3vHt9knja4EtWDfRCW278IQ2jibH3mr-XjQkWz9Rto7lPhYNLcyrUrlNmS800ZyQN5f01pRr-TrSMqiT5H0bTs7hGDGRdRp.jSDjETBLUrBxmvIiT3sOqg")
+(def company-id "4620816365202617680")
+
+(def base-url "https://sandbox-quickbooks.api.intuit.com/v3")
+(def base-headers {"Authorization" (str "Bearer " access-token)
+                   "Accept" "application/json"
+                   "Content-Type" "application/json"})
+
+(def prod-client-id "ABFRwAiOqQiLN66HKplXfyRE3ipD390DHsrUquflRCiOa81mxa")
+(def prod-client-secret "xDUj04GeQXpLvrhxep1jjDYwjJWbzzOPrirUQTKF")
+(def prod-redirect-uri "https://developer.intuit.com/v2/OAuth2Playground/RedirectUrl")
+(def prod-authorization-code "AB11638464998wYuapsEGtIEnRqphrw0H97XUnvEG2dK4cGUyL")
+(def prod-realm-id "123146163906404")
+;;  "refreshToken": "AB11647191065B0olWYQ61wfq8uszBusfe6Jpn7Au7qY5exkLL",
+;; "accessToken":,
+;;
+(def prod-access-token "eyJlbmMiOiJBMTI4Q0JDLUhTMjU2IiwiYWxnIjoiZGlyIn0..zbZ9E2iJQfn8cFNV9ROo-A.Md9HAuz1Vv08pANm-4tJSrZVGwhuBuElF8lrOFH_5mO2pBXUeYLra1ag6AdUBP7fdBgJvV53aXSWx2m7tnDygnaEHm95Wxr091HsawIAslzg59QkmAyC1UdbHCE7RX8q2pBeQAt2EBg8G-kmtFxtIIWlFuvt49vHPb1DpTG5z_Mf369OhjEBT0DrKfiCa9Jo1hcppkub6_aHt_aEFKLAshlvpGGV_peYImBiQH-Z2rM5ya7j1leuBdkmVzJH1MeH3mYJDKWoL1v7BTVPwE-4KTo72qCp-6dPvQPnVsHk1hpbvI2qjR9m5CvM3SmuwZqirpmOPGp9IL6gOJnEjGRf1mrk7MJqQ5DEr2T9nhCp9TWDoOle26hoE1dlNi83qFx0I5VwKcGHNmJz7BJxHhyidOcmi4V2RZLQ3CmA032Kc6AL8cgRKRuuhiyFcPBinlgFKzM-a82hp0lwpw6KeaaRSlR3i57W5SBmDoUaelyv9iVaIScs5tgjKAIPFv1Wu-6OnyUIAFwmhi_2dSNSj7NzXlEzDQ9ByoHJYG_DCN_2H-MwOpavNSz-rrGnqocBcTQYNH9P9D61tthS4NCxLLVtiLYHtT0ioJoS5esKuk8wM_jSz5oDNoR364CjEw0Ij9vIZ6eANXVf8Qu_IE9S8O4_G4__Wc4pi4nLKk5q3_kUj414m0ASE8Cm4Qph2b8i7wv1WsuBbQrdMOVQxuE2xFvuHYvT7lNuxO2SbV10iqKlepI.Sa9o2pE3xsVnuEMuuhXl2Q")
+
+
+
+(def prod-company-id "123146163906404")
+
+
+(def prod-base-url "https://quickbooks.api.intuit.com/v3")
+
+(defn set-access-token [t]
+  (s3/put-object :bucket-name (:data-bucket env) 
+                 :key (str "intuit/access-token")
+                 :input-stream (io/make-input-stream (.getBytes t) {})
+                 :metadata {:content-type "application/text"}))
+(defn set-refresh-token [t]
+  (s3/put-object :bucket-name (:data-bucket env) 
+                 :key (str "intuit/refresh-token")
+                 :input-stream (io/make-input-stream (.getBytes t) {})
+                 :metadata {:content-type "application/text"}))
+
+(defn init-tokens [access refresh]
+  (set-access-token access)
+  (set-refresh-token refresh))
+
+(defn lookup-access-token []
+  (slurp  (:object-content (s3/get-object
+                            :bucket-name (:data-bucket env)
+                            :key "intuit/access-token"))))
+
+(defn lookup-refresh-token []
+  (slurp  (:object-content (s3/get-object
+                            :bucket-name (:data-bucket env)
+                            :key "intuit/refresh-token"))))
+
+(defn get-basic-auth []
+  (Base64/encodeBase64String (.getBytes (str prod-client-id ":" prod-client-secret))))
+
+
+(defn get-fresh-access-token []
+  (let [refresh-token (lookup-refresh-token)
+        response (:body (client/post  (str "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer" )
+
+                             {:headers {"Accept" "application/json"
+                                        "Content-Type" "application/x-www-form-urlencoded"
+                                        "Authorization" (str "Basic " (get-basic-auth))}
+                              :form-params {"grant_type" "refresh_token"
+                                            "refresh_token" refresh-token}
+                              :as :json}))]
+    (set-access-token (:access_token response))
+    (set-refresh-token (:refresh_token response))
+    (:access_token response)))
+
+(def prod-base-headers {"Authorization" (str "Bearer " prod-access-token)
+                        "Accept" "application/json"
+                        "Content-Type" "application/json"})
+(defn with-auth [t token]
+  (assoc t "Authorization" (str "Bearer " token)))
+
+#_(client/get (str base-url "/company/4620816365202617680") 
+            {:headers base-headers
+             :as :json})
+
+
+
+(defn get-bank-accounts [token]
+  (->> (:body (client/get (str prod-base-url "/company/" prod-company-id "/query" ) 
+                          {:headers
+                           (with-auth prod-base-headers token)
+                           :as :json
+                           :query-params {"query" "SELECT * From Account"}}))
+       :QueryResponse
+       :Account
+       (filter
+        #(#{"Bank" "Credit Card"} (:AccountType %)))
+       (map (juxt :Id :Name))
+       (map (fn [[id name]]
+              {:id id
+               :name name}))))
+
+
+
+(defn get-transactions []
+  (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=2021-11-30&end_date=2021-11-30") 
+              {:headers prod-base-headers
+               :as :json}))
+
+(defn get-transactions [start end external-id]
+  (let [token (get-fresh-access-token)]
+    (let [body (:body (client/get (str prod-base-url "/company/" prod-company-id "/reports/TransactionList" "?minorversion=63&start_date=" start "&end_date=" end) 
+                                  {:headers (with-auth prod-base-headers token)
+                                   :as :json}))
+          headers (map :ColTitle (:Column (:Columns body)))
+          rows (map :ColData (:Row (:Rows body)))]
+      (->> rows
+           (map
+            (fn [row]
+              (into {}
+                    (map
+                     (fn [h r]
+                       [(keyword h) (:value r)])
+                     headers
+                     row))))
+           (filter #(= external-id
+                       (:Account %)))))))
+
+

src/clj/auto_ap/intuit/import.clj

@@ -0,0 +1,52 @@
+(ns auto-ap.intuit.import
+  (:require [amazonica.aws.s3 :as s3]
+            [auto-ap.datomic :refer [conn remove-nils]]
+            [auto-ap.intuit.core :as i]
+            [auto-ap.utils :refer [by]]
+            [auto-ap.yodlee.import :as y]
+            [clj-time.coerce :as coerce]
+            [clj-time.core :as time]
+            [clj-time.format :as f]
+            [clojure.string :as str]
+            [clojure.tools.logging :as log]
+            [datomic.api :as d]
+            [mount.core :as mount]
+            [unilog.context :as lc]
+            [yang.scheduler :as scheduler]))
+
+
+(defn upsert-transactions []
+  (let [db (d/db conn)]
+    (doseq [[bank-account external-id] (-> (d/q '[:find ?ba ?external-id
+                                                  :in $
+                                                  :where [?ba :bank-account/intuit-bank-account ?iab]
+                                                  [?iab :intuit-bank-account/external-id ?external-id]]
+
+                                                db))
+            :let [bank-account (d/entity db bank-account)
+                  end (auto-ap.time/local-now)
+                  start (time/plus end (time/days -10))]
+            ]
+      (log/infof "importing from %s to %s for %s" start end external-id)
+      (let [transactions (->> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
+                                                  (auto-ap.time/unparse end auto-ap.time/iso-date)
+                                                  external-id)
+                              (mapv (fn [r]
+                                      {:client-id (:db/id (:client/_bank-accounts bank-account))
+                                       :bank-account-id (:db/id bank-account)
+                                       :description-original (:Memo/Description r)
+                                       :amount (Double/parseDouble (:Amount r))
+                                       :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date)
+                                       :status "posted"})))]
+        (log/infof "%d transactions found" (count transactions))
+        (y/grouped-import transactions)))))
+
+
+(defn upsert-accounts []
+  (let [token (i/get-fresh-access-token)
+        bank-accounts (i/get-bank-accounts token)]
+    @(d/transact conn (mapv
+                      (fn [ba]
+                        {:intuit-bank-account/external-id (:name ba)
+                         :intuit-bank-account/name (:name ba)})
+                      bank-accounts))))