(ns auto-ap.yodlee.import (:require [auto-ap.yodlee.core :as client] [auto-ap.utils :refer [by]] [datomic.api :as d] [auto-ap.datomic :refer [uri remove-nils]] [auto-ap.datomic.accounts :as a] [clj-time.coerce :as coerce] [digest :refer [sha-256]] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.datomic.clients :as d-clients] [auto-ap.time :as time] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.rule-matching :as rm] [clojure.string :as str])) (defn transaction->payment [_ check-number client-id bank-account-id amount id] (cond (not (and client-id bank-account-id)) nil (:transaction/payment (d-transactions/get-by-id [:transaction/id (sha-256 (str id))])) nil check-number (-> (d-checks/get-graphql {:client-id client-id :bank-account-id bank-account-id :check-number check-number :amount (- amount) :status :payment-status/pending}) first first) (and client-id bank-account-id amount) (let [[matching-checks] (d-checks/get-graphql {:client-id client-id :bank-account-id bank-account-id :amount (- amount) :status :payment-status/pending})] (if (= 1 (count matching-checks)) (first matching-checks) nil)) :else nil)) (defn extract-check-number [{{description-original :original} :description}] (if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)] (try (Integer/parseInt check-number) (catch NumberFormatException e nil)) nil)) (defn transactions->txs [transactions transaction->bank-account apply-rules existing] (into [] (for [transaction transactions :let [{post-date :postDate account-id :accountId date :date id :id {amount :amount} :amount {description-original :original description-simple :simple} :description {merchant-id :id merchant-name :name} :merchant base-type :baseType type :type status :status} transaction amount (if (= "DEBIT" base-type) (- amount) amount) check-number (extract-check-number transaction) bank-account (transaction->bank-account transaction) bank-account-id (:db/id bank-account) client (:client/_bank-accounts bank-account) client-id (:db/id client) valid-locations (or (:bank-account/locations bank-account) (:client/locations client)) check (transaction->payment transaction check-number client-id bank-account-id amount id)] :when (and client-id (not (existing (sha-256 (str id)))) (= "POSTED" status) )] (-> #:transaction {:post-date (coerce/to-date (time/parse post-date "YYYY-MM-dd")) :id (sha-256 (str id)) :account-id account-id :date (coerce/to-date (time/parse date "YYYY-MM-dd")) :amount (double amount) :description-original (some-> description-original (str/replace #"\s+" " ")) :description-simple (some-> description-simple (str/replace #"\s+" " ")) :approval-status (if check :transaction-approval-status/approved :transaction-approval-status/unapproved) :type type :status status :client client-id :check-number check-number :bank-account bank-account-id :payment (when check {:db/id (:db/id check) :payment/status :payment-status/cleared}) :vendor (when check (:db/id (:payment/vendor check))) :location (when check "A") :accounts (when check [#:transaction-account {:account (:db/id (a/get-account-by-numeric-code-and-sets 21000 ["default"])) :location "A" :amount (Math/abs (double amount))}])} (apply-rules valid-locations) remove-nils)))) (defn batch-transact [transactions] (reduce (fn [acc batch] (into acc (->> batch (d/transact (d/connect uri) ) (deref) :tempids vals))) [] (partition-all 100 transactions))) (defn get-existing [] (transduce (map first) conj #{} (d/query {:query {:find ['?tid] :in ['$] :where ['[_ :transaction/id ?tid]]} :args [(d/db (d/connect uri))]}))) (defn get-all-bank-accounts [] (->> (d-clients/get-all) (mapcat (fn [client] (->> client :client/bank-accounts #_(filter :bank-account/yodlee-account-id) (filter :db/id) (map (fn [{:keys [:db/id :bank-account/yodlee-account-id] :as bank-account}] (assoc bank-account :client/_bank-accounts client)))))))) (defn manual-import [manual-transactions] (let [transformed-transactions (->> manual-transactions (filter #(= "posted" (:status %))) (group-by #(select-keys % [:date :description-original :amount])) (vals) (mapcat (fn [transaction-group] (map (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) :bank-account-id bank-account-id :date (time/unparse date "YYYY-MM-dd") :amount {:amount amount} :description {:original description-original :simple high-level-category} :status "POSTED"}) (range) transaction-group)))) all-rules (tr/get-all) all-bank-accounts (by :db/id (get-all-bank-accounts)) transaction->bank-account (comp all-bank-accounts :bank-account-id)] (println "importing manual transactions" transformed-transactions) (let [result (batch-transact (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))] (println "imported " (count result))))) (defn do-import ([] (do-import (client/get-transactions))) ([transactions] (let [all-bank-accounts (get-all-bank-accounts) transaction->bank-account (comp (by :bank-account/yodlee-account-id all-bank-accounts) :accountId) all-rules (tr/get-all)] (batch-transact (transactions->txs transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))))))