diff --git a/docker-compose.yml b/docker-compose.yml index 85ebcecf..a9bf08c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,8 @@ services: POSTGRES_USER: ap POSTGRES_PASSWORD: fifteen-invoices-imported! POSTGRES_DB: autoap + ports: + - "5432:5432" volumes: - ./data/var/lib/postgresql/data:/var/lib/postgresql/data migrator: diff --git a/migrator/migrations/1531452173-DOWN-change-yodlee-transaction-id.sql b/migrator/migrations/1531452173-DOWN-change-yodlee-transaction-id.sql new file mode 100644 index 00000000..ab20a585 --- /dev/null +++ b/migrator/migrations/1531452173-DOWN-change-yodlee-transaction-id.sql @@ -0,0 +1 @@ +-- 1531452173 DOWN change-yodlee-transaction-id \ No newline at end of file diff --git a/migrator/migrations/1531452173-UP-change-yodlee-transaction-id.sql b/migrator/migrations/1531452173-UP-change-yodlee-transaction-id.sql new file mode 100644 index 00000000..b4b71f2c --- /dev/null +++ b/migrator/migrations/1531452173-UP-change-yodlee-transaction-id.sql @@ -0,0 +1,3 @@ +-- 1531452173 UP change-yodlee-transaction-id + +alter table transactions alter column id type varchar(255); diff --git a/project.clj b/project.clj index 6dcf2947..9c7a8c13 100644 --- a/project.clj +++ b/project.clj @@ -19,6 +19,7 @@ ;; https://mvnrepository.com/artifact/postgresql/postgresql [vincit/venia "0.2.5"] [postgresql/postgresql "9.3-1102.jdbc41"] + [digest "1.4.8"] [nilenso/honeysql-postgres "0.2.4"] [cljs-http "0.1.44"] [clj-http "3.7.0"] diff --git a/src/clj/auto_ap/db/transactions.clj b/src/clj/auto_ap/db/transactions.clj index f24933b8..7fa18604 100644 --- a/src/clj/auto_ap/db/transactions.clj +++ b/src/clj/auto_ap/db/transactions.clj @@ -8,10 +8,13 @@ (defn upsert! [row] (j/db-do-prepared (get-conn) - (sql/format (-> (helpers/insert-into :transactions) - (helpers/values [row]) - (postgres-helpers/upsert (-> (postgres-helpers/on-conflict :id) - (postgres-helpers/do-update-set :post_date :status :check_id))))))) + (doto (sql/format (-> (helpers/insert-into :transactions) + (helpers/values [row]) + (postgres-helpers/upsert (-> (postgres-helpers/on-conflict :id) + + (postgres-helpers/do-update-set :post_date :status :check_id) + (helpers/where [:<> :EXCLUDED.check_id nil]))))) + println))) (def base-query (sql/build :select :* :from :transactions)) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index f984b2d3..ee5f2ac0 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -100,7 +100,7 @@ :resolve :get-checks-invoices} }} - :transaction {:fields {:id {:type 'Int} + :transaction {:fields {:id {:type 'String} :amount {:type 'String} :description_original {:type 'String} :description_simple {:type 'String} diff --git a/src/clj/auto_ap/time.clj b/src/clj/auto_ap/time.clj index dfdcfa5c..bbf0148a 100644 --- a/src/clj/auto_ap/time.clj +++ b/src/clj/auto_ap/time.clj @@ -14,3 +14,9 @@ (time/time-zone-for-id "America/Los_Angeles")) (catch Exception e nil))) + +(defn unparse [v format] + (try + (f/unparse (f/formatter format) v) + (catch Exception e + nil))) diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index b6be983c..9c0cb085 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -4,6 +4,7 @@ [auto-ap.db.vendors :as vendors] [auto-ap.utils :refer [by]] [auto-ap.db.companies :as companies] + [digest :refer [sha-256]] [auto-ap.db.checks :as checks] [auto-ap.time :as time])) @@ -36,6 +37,70 @@ (Integer/parseInt check-number) nil)) +(defn import-transactions [transactions yodlee-account-id->company yodlee-account-id->bank-account-id] + (doseq [transaction transactions + :let [{post-date :postDate + account-id :accountId + date :date + id :id + {amount :amount} :amount + {description-original :original + description-simple :simple} :description + {merchant-id :i + merchant-name :name} :merchant + type :type + status :status} + transaction + check-number (extract-check-number transaction) + company-id (yodlee-account-id->company account-id) + bank-account-id (yodlee-account-id->bank-account-id account-id) + check-id (transaction->check-id transaction check-number company-id bank-account-id amount) + ]] + + (try + (transactions/upsert! + {:post-date (time/parse post-date "YYYY-MM-dd") + :id (sha-256 (str id)) + :account-id account-id + :date (time/parse date "YYYY-MM-dd") + :amount amount + :description-original description-original + :description-simple description-simple + :type type + :status status + :company-id company-id + :check-number check-number + :bank-account-id (yodlee-account-id->bank-account-id account-id) + :check-id check-id + }) + (when check-id + (checks/update! {:id check-id :status "cleared"})) + + (catch Exception e + (println e))))) + +(defn manual-import [manual-transactions company-id bank-account-id] + (let [transformed-transactions (->> manual-transactions + (filter #(= "posted" (:status %))) + (group-by #(select-keys % [:date :original-description :amount])) + + (vals) + + (mapcat (fn [transaction-group] + (map + (fn [index {:keys [date description-original high-level-category amount] :as transaction}] + {:id (str date "-" description-original "-" amount "-" index) + :date (time/unparse date "YYYY-MM-dd") + :amount {:amount amount} + :description {:original description-original + :simple high-level-category} + :status "POSTED"}) + (range) + + transaction-group))))] + (println "importing manual transactions" transformed-transactions) + (import-transactions transformed-transactions (constantly company-id) (constantly bank-account-id)))) + (defn do-import [] (let [transactions (client/get-transactions) all-bank-accounts (mapcat (fn [c] (map @@ -46,48 +111,9 @@ :yodlee-account-id yodlee-account-id})) (:bank-accounts c))) (companies/get-all)) - account->company (by :yodlee-account-id :company-id all-bank-accounts) + yodlee-account-id->company (by :yodlee-account-id :company-id all-bank-accounts) yodlee-account-id->bank-account-id (by :yodlee-account-id :bank-account-id all-bank-accounts)] - (doseq [transaction transactions - :let [{post-date :postDate - account-id :accountId - date :date - id :id - {amount :amount} :amount - {description-original :original - description-simple :simple} :description - {merchant-id :i - merchant-name :name} :merchant - type :type - status :status} - transaction - check-number (extract-check-number transaction) - company-id (account->company account-id) - bank-account-id (yodlee-account-id->bank-account-id account-id) - check-id (transaction->check-id transaction check-number company-id bank-account-id amount) - ]] - - (try - (transactions/upsert! - {:post-date (time/parse post-date "YYYY-MM-dd") - :id id - :account-id account-id - :date (time/parse date "YYYY-MM-dd") - :amount amount - :description-original description-original - :description-simple description-simple - :type type - :status status - :company-id company-id - :check-number check-number - :bank-account-id (yodlee-account-id->bank-account-id account-id) - :check-id check-id - }) - (when check-id - (checks/update! {:id check-id :status "cleared"})) - - (catch Exception e - (println e)))))) + (import-transactions transactions yodlee-account-id->company yodlee-account-id->bank-account-id))) #_(defn match-old-checks [] (let [transactions (transactions/get-unmatched)]