From de8563a60bdcde0e615fd4dcffc8b8945f1314f0 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Fri, 9 Nov 2018 10:47:47 -0800 Subject: [PATCH] Making manual import work in datomic --- src/clj/auto_ap/datomic.clj | 7 +- src/clj/auto_ap/datomic/checks.clj | 11 ++- src/clj/auto_ap/datomic/transactions.clj | 2 +- src/clj/auto_ap/graphql.clj | 2 +- src/clj/auto_ap/graphql/checks.clj | 1 + src/clj/auto_ap/routes/invoices.clj | 26 +++--- src/clj/auto_ap/server.clj | 2 +- src/clj/auto_ap/yodlee/import.clj | 90 ++++++++++--------- .../components/expense_accounts_dialog.cljs | 2 +- .../auto_ap/views/components/paginator.cljs | 2 +- .../auto_ap/views/pages/transactions.cljs | 2 +- 11 files changed, 80 insertions(+), 67 deletions(-) diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index 58e18920..b4d136d4 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -618,7 +618,8 @@ :date (coerce/to-date date) :customer-identifier customer-identifier :client [:client/original-id company-id] - :vendor [:vendor/original-id vendor-id] + :vendor (when vendor-id [:vendor/original-id vendor-id]) + :default-location default-location :default-expense-account default-expense-account :total (double total) @@ -646,7 +647,7 @@ :s3-uuid s3-uuid :s3-key s3-key :s3-url s3-url - :vendor [:vendor/original-id vendor-id] + :vendor (when vendor-id [:vendor/original-id vendor-id]) :client [:client/original-id company-id] :bank-account (when bank-account-id [:bank-account/original-id (str company-id "-" bank-account-id)]) :check-number check-number @@ -752,7 +753,7 @@ @(load-vendors all-vendors)) (let [all-clients (c/get-all)] @(load-clients all-clients)) - @(load-invoices (i/get-all)) + @(load-invoices (i/get-all)) @(load-payments (checks/get-all)) @(load-invoices-payments (ic/get-all)) @(load-invoices-expense-accounts (iea/get-all)) diff --git a/src/clj/auto_ap/datomic/checks.clj b/src/clj/auto_ap/datomic/checks.clj index f8949683..c9913ec9 100644 --- a/src/clj/auto_ap/datomic/checks.clj +++ b/src/clj/auto_ap/datomic/checks.clj @@ -27,6 +27,7 @@ {:payment/type [:db/ident]}])) (defn raw-graphql [args] + (println "ARGS" args) (let [query (cond-> {:query {:find [default-read] :in ['$] :where ['[?e :payment/client]]} @@ -36,7 +37,15 @@ '[?e :payment/client ?client-id]) (:original-id args) (add-arg '?original-id (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong ) '[?e :payment/client ?c] - '[?c :client/original-id ?original-id]))] + '[?c :client/original-id ?original-id]) + (:check-number args) (add-arg '?check-number (:check-number args) + '[?e :payment/check-number ?check-number]) + (:bank-account-id args) (add-arg '?bank-account-id (:bank-account-id args) + '[?e :payment/bank-account ?bank-account-id]) + (:amount args) (add-arg '?amount (:amount args) + '[?e :payment/amount ?amount]) + (:status args) (add-arg '?status (:status args) + '[?e :payment/status ?status]))] (->> (d/query query) diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj index 82a6b4b2..f5218128 100644 --- a/src/clj/auto_ap/datomic/transactions.clj +++ b/src/clj/auto_ap/datomic/transactions.clj @@ -16,7 +16,7 @@ (cond-> {:query {:find ['(pull ?e [* {:transaction/client [:client/name :db/id] :transaction/bank-account [:bank-account/name :bank-account/yodlee-account-id]}])] :in ['$] - :where ['[?e :transaction/original-id]]} + :where ['[?e :transaction/id]]} :args [(d/db (d/connect uri))]} (:client-id args) (add-arg '?client-id (Long/parseLong (:client-id args)) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 6ea17000..b3aac920 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -222,7 +222,7 @@ :resolve :get-all-payments} :transaction_page {:type '(list :transaction_page) - :args {:company_id {:type 'String} + :args {:client_id {:type 'String} :start {:type 'Int} :sort_by {:type 'String} :asc {:type 'Boolean}} diff --git a/src/clj/auto_ap/graphql/checks.clj b/src/clj/auto_ap/graphql/checks.clj index b84ae0f0..45e166ef 100644 --- a/src/clj/auto_ap/graphql/checks.clj +++ b/src/clj/auto_ap/graphql/checks.clj @@ -325,6 +325,7 @@ bank-account :payment-type/check 0 {(Long/parseLong (:invoice_id args)) (:amount args)})] @(d/transact (d/connect uri) [(assoc base-payment + :payment/status :payment-status/pending :payment/check-number (:check_number args) :payment/date (c/to-date (parse (:date args) iso-date)) :payment/amount (:amount args)) diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index 45e25d07..7f7ca397 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -66,17 +66,17 @@ (defn parse-account-id [i] (try - (Integer/parseInt (second - (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:account-id i)))) + (Long/parseLong (second + (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i)))) (catch Exception e - (throw (Exception. (str "Could not parse account from value '" (:account-id i) "'") e))))) + (throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e))))) -(defn parse-company-id [i] +(defn parse-client-id [i] (try - (Integer/parseInt (second - (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:company-id i)))) + (Long/parseLong (second + (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:client-id i)))) (catch Exception e - (throw (Exception. (str "Could not parse company from value '" (:company-id i) "'") e))))) + (throw (Exception. (str "Could not parse client from value '" (:client-id i) "'") e))))) (defn parse-date [{:keys [raw-date]}] (try @@ -99,28 +99,28 @@ (POST "/batch-upload" {{:keys [data]} :edn-params user :identity} (assert-admin user) - (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :account-id :company-id] + (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-id :client-id] rows (->> (str/split data #"\n" ) (drop 1) (map #(str/split % #"\t")) (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) (map (parse-or-error :amount parse-amount)) - (map (parse-or-error :account-id parse-account-id)) - (map (parse-or-error :company-id parse-company-id)) + (map (parse-or-error :bank-account-id parse-account-id)) + (map (parse-or-error :client-id parse-client-id)) (map (parse-or-error :date parse-date))) error-rows (filter :errors rows) raw-transactions (vec (->> rows (filter #(not (seq (:errors %))) ) - (map (fn [{:keys [description-original company-id status high-level-category amount account-id date]}] + (map (fn [{:keys [description-original client-id status high-level-category amount bank-account-id date]}] {:description-original description-original :date date :status status :high-level-category high-level-category :amount amount - :company-id company-id - :account-id account-id}))))] + :client-id client-id + :bank-account-id bank-account-id}))))] (manual-import raw-transactions) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index eaab2bb9..3b6ccf2c 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -1,5 +1,5 @@ (ns auto-ap.server - (:require [auto-ap.background.mail :refer [always-process-sqs]] + (:require #_[auto-ap.background.mail :refer [always-process-sqs]] [auto-ap.handler :refer [app]] [clojure.tools.nrepl.server :refer [start-server stop-server]] [config.core :refer [env]] diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index c1faa4fe..59ad6a9f 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -3,31 +3,35 @@ [auto-ap.db.transactions :as transactions] [auto-ap.db.vendors :as vendors] [auto-ap.utils :refer [by]] + [datomic.api :as d] + [auto-ap.datomic :refer [uri remove-nils]] [auto-ap.db.companies :as companies] + [clj-time.coerce :as coerce] [digest :refer [sha-256]] - [auto-ap.db.checks :as checks] + [auto-ap.datomic.checks :as d-checks] [auto-ap.time :as time])) -(defn transaction->check-id [_ check-number company-id bank-account-id amount] - (cond (and check-number company-id bank-account-id) - (-> (checks/get-graphql {:company-id company-id - :bank-account-id bank-account-id - :check-number check-number - :amount (- amount) - :status "pending"}) +(defn transaction->check-id [_ check-number client-id bank-account-id amount] + (cond (and check-number client-id bank-account-id) + (-> (d-checks/get-graphql {:client-id client-id + :bank-account-id bank-account-id + :check-number check-number + :amount (- amount) + :status :payment-status/pending}) first - :id) + (doto println) + :db/id) - (and company-id bank-account-id amount) + (and client-id bank-account-id amount) - (let [matching-checks (checks/get-graphql {:company-id company-id + (let [matching-checks (d-checks/get-graphql {:client-id client-id :bank-account-id bank-account-id :amount (- amount) - :status "pending"})] + :status :payment-status/pending})] (if (= 1 (count matching-checks)) - (:id (first matching-checks)) + (:db/id (first matching-checks)) nil)) :else @@ -61,31 +65,32 @@ (- amount) amount) check-number (extract-check-number transaction) - company-id (transaction->company transaction) + client-id (transaction->company transaction) bank-account-id (transaction->bank-account-id transaction) - check-id (transaction->check-id transaction check-number company-id bank-account-id amount)]] - - - + check-id (transaction->check-id transaction check-number client-id bank-account-id amount)]] (try - (when company-id - (transactions/upsert! - {:post-date (time/parse post-date "YYYY-MM-dd") - :id (sha-256 (str id)) - :account-id account-id - :date (time/parse date "YYYY-MM-dd") - :amount amount - :description-original description-original - :description-simple description-simple - :type type - :status status - :company-id company-id - :check-number check-number - :bank-account-id (transaction->bank-account-id transaction) - :check-id check-id - }) - (when check-id - (checks/update! {:id check-id :status "cleared"}))) + (when client-id + @(->> [(remove-nils #:transaction + {:post-date (time/parse post-date "YYYY-MM-dd") + :id (sha-256 (str id)) + :account-id account-id + :date (coerce/to-date (time/parse date "YYYY-MM-dd")) + :amount amount + :description-original description-original + :description-simple description-simple + :type type + :status status + :client client-id + :check-number check-number + :bank-account (transaction->bank-account-id transaction) + :payment (when check-id + {:db/id check-id + :payment/status :payment-status/cleared} + ) + })] + + + (d/transact (d/connect uri)))) (catch Exception e (println e))))) @@ -94,25 +99,22 @@ (let [transformed-transactions (->> manual-transactions (filter #(= "posted" (:status %))) (group-by #(select-keys % [:date :description-original :amount])) - (vals) - (mapcat (fn [transaction-group] (map - (fn [index {:keys [date description-original high-level-category amount account-id company-id] :as transaction}] - {:id (str date "-" account-id "-" description-original "-" amount "-" index "-" company-id) - :company-id company-id - :bank-account-id account-id + (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] + {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) + :client-id client-id + :bank-account-id bank-account-id :date (time/unparse date "YYYY-MM-dd") :amount {:amount amount} :description {:original description-original :simple high-level-category} :status "POSTED"}) (range) - transaction-group))))] (println "importing manual transactions" transformed-transactions) - (import-transactions transformed-transactions :company-id :bank-account-id))) + (import-transactions transformed-transactions :client-id :bank-account-id))) (defn do-import [] (let [transactions (client/get-transactions) diff --git a/src/cljs/auto_ap/views/components/expense_accounts_dialog.cljs b/src/cljs/auto_ap/views/components/expense_accounts_dialog.cljs index 37a36f9c..19aded25 100644 --- a/src/cljs/auto_ap/views/components/expense_accounts_dialog.cljs +++ b/src/cljs/auto_ap/views/components/expense_accounts_dialog.cljs @@ -82,7 +82,7 @@ (map js/parseFloat) (map #(or % 0)) (reduce + 0)) - does-add-up? (< (Math/abs (- expense-accounts-total (js/parseFloat total))) 0.01)]] + does-add-up? (< (Math/abs (- expense-accounts-total (js/parseFloat total))) 0.01)] [action-modal {:id ::change-expense-accounts :title "Change expense accounts" :action-text "Save" diff --git a/src/cljs/auto_ap/views/components/paginator.cljs b/src/cljs/auto_ap/views/components/paginator.cljs index 042dd7e6..c6e9fdc4 100644 --- a/src/cljs/auto_ap/views/components/paginator.cljs +++ b/src/cljs/auto_ap/views/components/paginator.cljs @@ -19,7 +19,7 @@ y)) (defn paginator [{:keys [start end count total on-change]}] - (let [per-page 10 + (let [per-page 20 max-buttons 5 buttons-before (Math/floor (/ max-buttons 2)) total-pages (Math/ceil (/ total per-page)) diff --git a/src/cljs/auto_ap/views/pages/transactions.cljs b/src/cljs/auto_ap/views/pages/transactions.cljs index cebb8a11..1ad7a85a 100644 --- a/src/cljs/auto_ap/views/pages/transactions.cljs +++ b/src/cljs/auto_ap/views/pages/transactions.cljs @@ -33,7 +33,7 @@ (assoc-in [::params] params)) :graphql {:token (-> cofx :db :user) :query-obj {:venia/queries [[:transaction_page - (assoc params :company-id (:id @(re-frame/subscribe [::subs/company]))) + (assoc params :client-id (:id @(re-frame/subscribe [::subs/company]))) [[:transactions [:id :amount :date