From d110755939119c693ca87e317dafc6215eded29e Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 14 Jun 2018 17:26:54 -0700 Subject: [PATCH] ability to import yodlee data. --- config/dev.edn | 3 + config/local.edn | 3 + config/prod.edn | 3 + config/staging.edn | 3 + .../1529005616-DOWN-add-yodleee-staging.sql | 2 + .../1529005616-UP-add-yodleee-staging.sql | 15 ++++ project.clj | 1 + src/clj/auto_ap/db/yodlee_imports.clj | 14 +++ src/clj/auto_ap/yodlee/core.clj | 89 +++++++++++++++++++ src/clj/auto_ap/yodlee/import.clj | 30 +++++++ 10 files changed, 163 insertions(+) create mode 100644 migrator/migrations/1529005616-DOWN-add-yodleee-staging.sql create mode 100644 migrator/migrations/1529005616-UP-add-yodleee-staging.sql create mode 100644 src/clj/auto_ap/db/yodlee_imports.clj create mode 100644 src/clj/auto_ap/yodlee/core.clj create mode 100644 src/clj/auto_ap/yodlee/import.clj diff --git a/config/dev.edn b/config/dev.edn index f9fc21f9..e34d4fc8 100644 --- a/config/dev.edn +++ b/config/dev.edn @@ -7,4 +7,7 @@ :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :invoice-email "invoices-staging@mail.app.integreatconsult.com" :data-bucket "data.staging.app.integreatconsult.com" + + :yodlee-cobrand-login "sbCobda48aa19712a83c3ca4e935dd5e5d46b1a" + :yodlee-cobrand-password "0a07ea32-1b5d-461b-ad0f-2752cdd77602" } diff --git a/config/local.edn b/config/local.edn index 6843f8fa..bcd4cd71 100644 --- a/config/local.edn +++ b/config/local.edn @@ -6,4 +6,7 @@ :aws-region "us-east-1" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :data-bucket "data.staging.app.integreatconsult.com" + + :yodlee-cobrand-login "sbCobda48aa19712a83c3ca4e935dd5e5d46b1a" + :yodlee-cobrand-password "0a07ea32-1b5d-461b-ad0f-2752cdd77602" } diff --git a/config/prod.edn b/config/prod.edn index 3f9e5ea6..9dc55dd2 100644 --- a/config/prod.edn +++ b/config/prod.edn @@ -7,4 +7,7 @@ :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" :invoice-email "invoices@mail.app.integreatconsult.com" :data-bucket "data.prod.app.integreatconsult.com" + + :yodlee-cobrand-login "sbCobda48aa19712a83c3ca4e935dd5e5d46b1a" + :yodlee-cobrand-password "0a07ea32-1b5d-461b-ad0f-2752cdd77602" } diff --git a/config/staging.edn b/config/staging.edn index c67235b3..2a2861d1 100644 --- a/config/staging.edn +++ b/config/staging.edn @@ -7,4 +7,7 @@ :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" :invoice-email "invoices-staging@mail.app.integreatconsult.com" :data-bucket "data.staging.app.integreatconsult.com" + + :yodlee-cobrand-login "sbCobda48aa19712a83c3ca4e935dd5e5d46b1a" + :yodlee-cobrand-password "0a07ea32-1b5d-461b-ad0f-2752cdd77602" } diff --git a/migrator/migrations/1529005616-DOWN-add-yodleee-staging.sql b/migrator/migrations/1529005616-DOWN-add-yodleee-staging.sql new file mode 100644 index 00000000..8ece9cc4 --- /dev/null +++ b/migrator/migrations/1529005616-DOWN-add-yodleee-staging.sql @@ -0,0 +1,2 @@ +-- 1529005616 DOWN add-yodleee-staging +drop table yodlee_imports; diff --git a/migrator/migrations/1529005616-UP-add-yodleee-staging.sql b/migrator/migrations/1529005616-UP-add-yodleee-staging.sql new file mode 100644 index 00000000..34da9a2d --- /dev/null +++ b/migrator/migrations/1529005616-UP-add-yodleee-staging.sql @@ -0,0 +1,15 @@ +-- 1529005616 UP add-yodleee-staging +create table yodlee_imports ( +amount decimal, +description_original varchar(255), +description_simple varchar (255), +merchant_id varchar(255), +merchant_name varchar(255), +id int primary key, +date timestamp, +post_date timestamp, +type varchar(255), +account_id int, +status varchar(255)); + + diff --git a/project.clj b/project.clj index ebb001e5..430d5b8a 100644 --- a/project.clj +++ b/project.clj @@ -19,6 +19,7 @@ ;; https://mvnrepository.com/artifact/postgresql/postgresql [vincit/venia "0.2.5"] [postgresql/postgresql "9.3-1102.jdbc41"] + [nilenso/honeysql-postgres "0.2.4"] [cljs-http "0.1.44"] [clj-http "3.7.0"] [clj-pdf "2.2.31"] diff --git a/src/clj/auto_ap/db/yodlee_imports.clj b/src/clj/auto_ap/db/yodlee_imports.clj new file mode 100644 index 00000000..db0cc780 --- /dev/null +++ b/src/clj/auto_ap/db/yodlee_imports.clj @@ -0,0 +1,14 @@ +(ns auto-ap.db.yodlee-imports + (:require [clojure.java.jdbc :as j] + [honeysql.core :as sql] + [honeysql.helpers :as helpers] + [honeysql-postgres.format :as postgres-format] + [honeysql-postgres.helpers :as postgres-helpers] + [auto-ap.db.utils :refer [clj->db kebab->snake db->clj get-conn query] :as utils])) + +(defn upsert! [row] + (j/db-do-prepared (get-conn) + (sql/format (-> (helpers/insert-into :yodlee-imports) + (helpers/values [row]) + (postgres-helpers/upsert (-> (postgres-helpers/on-conflict :id) + (postgres-helpers/do-update-set :amount :status))))))) diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj new file mode 100644 index 00000000..b5dd65f4 --- /dev/null +++ b/src/clj/auto_ap/yodlee/core.clj @@ -0,0 +1,89 @@ +(ns auto-ap.yodlee.core + (:require [clj-http.client :as client] + [clojure.data.json :as json] + [config.core :refer [env]])) + +(defn auth-header + ([cob-session] (str "{cobSession=" cob-session "}")) + ([cob-session user-session] (str "{cobSession=" cob-session ",userSession=" user-session "}"))) + +(def base-headers {"Api-Version" "1.1" + "Cobrand-Name" "restserver" + "Content-Type" "application/json"}) + +(defn login-cobrand [] + (-> "https://developer.api.yodlee.com/ysl/cobrand/login" + (client/post {:headers base-headers + :body + (json/write-str {:cobrand {:cobrandLogin (:yodlee-cobrand-login env) + :cobrandPassword (:yodlee-cobrand-password env) + :locale "en_US"}}) + :as :json} + ) + :body + :session + :cobSession)) + + +(defn login-user [cob-session] + (-> "https://developer.api.yodlee.com/ysl/user/login" + (client/post {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) + :body + (json/write-str {:user {:loginName "sbMemda48aa19712a83c3ca4e935dd5e5d46b1a1" + :password "sbMemda48aa19712a83c3ca4e935dd5e5d46b1a1#123" + :locale "en_US"}}) + :as :json}) + :body + :user + :session + :userSession)) + +(defn get-accounts [] + (let [cob-session (login-cobrand) + user-session (login-user cob-session)] + (-> "https://developer.api.yodlee.com/ysl/accounts" + (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) + :as :json})))) + +(defn get-provider-accounts [] + (let [cob-session (login-cobrand) + user-session (login-user cob-session)] + (-> "https://developer.api.yodlee.com/ysl/providerAccounts/11049210" + (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) + :as :json})))) + +(defn get-transactions [] + (let [cob-session (login-cobrand) + user-session (login-user cob-session) + batch-size 500 + get-transaction-batch (fn [skip] + (-> (str "https://developer.api.yodlee.com/ysl/transactions?top=" batch-size "&skip=" skip) + (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) + :as :json}) + :body + :transaction + ))] + + (loop [transactions [] + skip 0] + (let [transaction-batch (get-transaction-batch skip)] + (if (seq transaction-batch) + (recur (concat transactions transaction-batch) (+ batch-size skip)) + transactions))))) + +#_(defn create-user [] + (let [cob-session (login-cobrand)] + (-> "https://developer.api.yodlee.com/ysl/user/register" + (client/post {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) + :body (json/write-str {:user {:loginName "brycesPersoonal2" + :password "kV@mdv3TU11" + :email "yodleepersonal2@brycecovertoperations.com"}}) + :as :json}) + :body))) + +#_(defn get-users [] + (let [cob-session (login-cobrand)] + (-> "https://developer.api.yodlee.com/ysl/user" + (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) + :as :json}) + :body))) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj new file mode 100644 index 00000000..12fd737c --- /dev/null +++ b/src/clj/auto_ap/yodlee/import.clj @@ -0,0 +1,30 @@ +(ns auto-ap.yodlee.import + (:require [auto-ap.yodlee.core :as client] + [auto-ap.db.yodlee-imports :as yodlee-imports] + [auto-ap.time :as time])) + +(defn do-import [] + (doseq [transaction (client/get-transactions) + :let [{post-date :postDate + account-id :accountId + date :date + id :id + {amount :amount} :amount + {description-original :original + description-simple :simple} :description + {merchant-id :i + merchant-name :name} :merchant + type :type + status :status + } + transaction]] + (yodlee-imports/upsert! + {:post-date (time/parse post-date "YYYY-MM-dd") + :id id + :account-id account-id + :date (time/parse date "YYYY-MM-dd") + :amount amount + :description-original description-original + :description-simple description-simple + :type type + :status status})))