From 216429bfdbe6cb4cb05f02acf7b4813fbff2e790 Mon Sep 17 00:00:00 2001 From: BC Date: Thu, 28 Jun 2018 21:20:51 -0700 Subject: [PATCH] checks are actually matched --- .../1529684190-UP-reset-database.sql | 6 +- .../1530242355-DOWN-add-check-number.sql | 3 + .../1530242355-UP-add-check-number.sql | 3 + src/clj/auto_ap/db/checks.clj | 5 +- src/clj/auto_ap/yodlee/core.clj | 4 +- src/clj/auto_ap/yodlee/import.clj | 108 +++++++++++------- 6 files changed, 79 insertions(+), 50 deletions(-) create mode 100644 migrator/migrations/1530242355-DOWN-add-check-number.sql create mode 100644 migrator/migrations/1530242355-UP-add-check-number.sql diff --git a/migrator/migrations/1529684190-UP-reset-database.sql b/migrator/migrations/1529684190-UP-reset-database.sql index ac6c73d3..676c6ea3 100644 --- a/migrator/migrations/1529684190-UP-reset-database.sql +++ b/migrator/migrations/1529684190-UP-reset-database.sql @@ -10,10 +10,10 @@ delete from companies; INSERT INTO companies (code, name, data) VALUES -('BCBC', 'Brown Chicken Brown Cow', '{:locations ["CB"]}'), -('BSG', 'Bella Saratoga', '{:locations ["SG"]}'), +('BCBC', 'Brown Chicken Brown Cow', '{:locations ["CB"] }'), +('BSG', 'Bella Saratoga', '{:locations ["SG"] }'), ('CBC', 'Campbell Brewing Co', '{:locations ["CB"]}'), -('IBC', 'Iguanas Burritozilla', '{:locations ["DT" "EG" "SC" "EG" "SG" "CB" "BH"]}'), +('IBC', 'Iguanas Burritozilla', '{:locations ["DT" "EG" "SC" "EG" "SG" "CB" "BH"] :bank-accounts [{:number "123456789" :id 1 :check-number 6789 :bank-name "Bank of America" :bank-code "90-4149/1211" :routing "12345678" :name "BOA-6789" :yodlee-account-id 11703936} {:number "987654321" :id 2 :check-number 1234 :bank-name "Bank of America" :bank-code "90-4149/1211" :routing "123456" :name "BOA-4321"}]}'), ('IBCBC', 'Shared CBC-IBC Expenses', '{:locations ["CB"]}'), ('KOG', 'Knock Out Grill & Bar', '{:locations ["SV"]}'), ('LFT', 'Lefty''s East Coast Pizzeria', '{:locations ["SB"]}'), diff --git a/migrator/migrations/1530242355-DOWN-add-check-number.sql b/migrator/migrations/1530242355-DOWN-add-check-number.sql new file mode 100644 index 00000000..65d571f8 --- /dev/null +++ b/migrator/migrations/1530242355-DOWN-add-check-number.sql @@ -0,0 +1,3 @@ +-- 1530242355 DOWN add-check-number +alter table transactions drop column check_number; +alter table transactions drop column bank_account_id; diff --git a/migrator/migrations/1530242355-UP-add-check-number.sql b/migrator/migrations/1530242355-UP-add-check-number.sql new file mode 100644 index 00000000..8fc0912e --- /dev/null +++ b/migrator/migrations/1530242355-UP-add-check-number.sql @@ -0,0 +1,3 @@ +-- 1530242355 UP add-check-number +alter table transactions add column check_number INT; +alter table transactions add column bank_account_id int; diff --git a/src/clj/auto_ap/db/checks.clj b/src/clj/auto_ap/db/checks.clj index ca5d95f4..16335823 100644 --- a/src/clj/auto_ap/db/checks.clj +++ b/src/clj/auto_ap/db/checks.clj @@ -68,10 +68,11 @@ :else q))) -(defn base-graphql [{:keys [company-id vendor-id]}] +(defn base-graphql [{:keys [company-id vendor-id check-number]}] (cond-> base-query (not (nil? company-id)) (helpers/merge-where [:= :company-id company-id]) - (not (nil? vendor-id)) (helpers/merge-where [:= :vendor-id vendor-id]))) + (not (nil? vendor-id)) (helpers/merge-where [:= :vendor-id vendor-id]) + (not (nil? check-number)) (helpers/merge-where [:= :check-number check-number]))) (defn get-graphql [{:keys [start sort-by asc] :as args}] (query diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj index b5dd65f4..40610e56 100644 --- a/src/clj/auto_ap/yodlee/core.clj +++ b/src/clj/auto_ap/yodlee/core.clj @@ -55,7 +55,7 @@ (defn get-transactions [] (let [cob-session (login-cobrand) user-session (login-user cob-session) - batch-size 500 + batch-size 100 get-transaction-batch (fn [skip] (-> (str "https://developer.api.yodlee.com/ysl/transactions?top=" batch-size "&skip=" skip) (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)}) @@ -71,7 +71,7 @@ (recur (concat transactions transaction-batch) (+ batch-size skip)) transactions))))) -#_(defn create-user [] +(defn create-user [] (let [cob-session (login-cobrand)] (-> "https://developer.api.yodlee.com/ysl/user/register" (client/post {:headers (merge base-headers {"Authorization" (auth-header cob-session)}) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index 684d59ae..9d16e941 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -2,58 +2,80 @@ (:require [auto-ap.yodlee.core :as client] [auto-ap.db.transactions :as transactions] [auto-ap.db.vendors :as vendors] + [auto-ap.utils :refer [by]] [auto-ap.db.companies :as companies] [auto-ap.db.checks :as checks] [auto-ap.time :as time])) -(defn account->company-id [account-id] - (-> (companies/get-all) - first - :id)) (defn transaction->vendor-id [_] (-> (vendors/get-all) first :id)) -(defn transaction->check-id [_ company-id vendor-id] - (when (= 0 (rand-int 2)) - (-> (checks/get-graphql {:company-id company-id - :vendor-id vendor-id}) - first - :id))) +(defn transaction->check-id [_ check-number company-id vendor-id] + (when check-number + (-> (checks/get-graphql {:company-id company-id + :check-number check-number}) + first + :id))) + +(defn extract-check-number [{{description-original :original} :description}] + (if-let [[_ check-number] (re-find #"(?i)check[^0-9]+([0-9]*)" description-original)] + (Integer/parseInt check-number) + nil)) (defn do-import [] - (doseq [transaction (client/get-transactions) - :let [{post-date :postDate - account-id :accountId - date :date - id :id - {amount :amount} :amount - {description-original :original - description-simple :simple} :description - {merchant-id :i - merchant-name :name} :merchant - type :type - status :status} - transaction - company-id (account->company-id account-id) - vendor-id (transaction->vendor-id transaction)]] - - (try - (transactions/upsert! - {:post-date (time/parse post-date "YYYY-MM-dd") - :id id - :account-id account-id - :date (time/parse date "YYYY-MM-dd") - :amount amount - :description-original description-original - :description-simple description-simple - :type type - :status status - :company-id company-id - :vendor-id vendor-id - :check-id (transaction->check-id transaction company-id vendor-id) - }) - (catch Exception e - (println e))))) + (let [transactions (client/get-transactions) + all-bank-accounts (mapcat (fn [c] (map + (fn [{:keys [id yodlee-account-id]}] + (when (and id yodlee-account-id) + {:bank-account-id id + :company-id (:id c) + :yodlee-account-id yodlee-account-id})) + (:bank-accounts c))) + (companies/get-all)) + account->company (by :yodlee-account-id :company-id all-bank-accounts) + yodlee-account-id->bank-account-id (by :yodlee-account-id :bank-account-id all-bank-accounts)] + (println "importing " (count transactions) "transactions") + (println "yodlee account->company" account->company) + (println "yodlee account->bank-account-id" yodlee-account-id->bank-account-id) + (doseq [transaction transactions + :let [{post-date :postDate + account-id :accountId + date :date + id :id + {amount :amount} :amount + {description-original :original + description-simple :simple} :description + {merchant-id :i + merchant-name :name} :merchant + type :type + status :status} + transaction + transaction (if (= 0 (rand-int 3)) + (assoc-in transaction [:description :original] (str "check xxx" (rand-nth ["6789" "1234" "6790"]))) + transaction) + check-number (extract-check-number transaction) + company-id (account->company account-id) + vendor-id (transaction->vendor-id transaction)]] + + (try + (transactions/upsert! + {:post-date (time/parse post-date "YYYY-MM-dd") + :id id + :account-id account-id + :date (time/parse date "YYYY-MM-dd") + :amount amount + :description-original description-original + :description-simple description-simple + :type type + :status status + :company-id company-id + :vendor-id vendor-id + :check-number check-number + :bank-account-id (yodlee-account-id->bank-account-id account-id) + :check-id (transaction->check-id transaction check-number company-id vendor-id) + }) + (catch Exception e + (println e))))))