notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Making manual import work in datomic

Browse Source
This commit is contained in:
Bryce Covert
2018-11-09 10:47:47 -08:00
parent 5a25462c5f
commit de8563a60b
11 changed files with 80 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/clj/auto_ap/datomic.clj
View File

@@ -618,7 +618,8 @@
:date (coerce/to-date date) :date (coerce/to-date date)
:customer-identifier customer-identifier :customer-identifier customer-identifier
:client [:client/original-id company-id] :client [:client/original-id company-id]
:vendor [:vendor/original-id vendor-id] :vendor (when vendor-id [:vendor/original-id vendor-id])
:default-location default-location :default-location default-location
:default-expense-account default-expense-account :default-expense-account default-expense-account
:total (double total) :total (double total)
@@ -646,7 +647,7 @@
:s3-uuid s3-uuid :s3-uuid s3-uuid
:s3-key s3-key :s3-key s3-key
:s3-url s3-url :s3-url s3-url
:vendor [:vendor/original-id vendor-id] :vendor (when vendor-id [:vendor/original-id vendor-id])
:client [:client/original-id company-id] :client [:client/original-id company-id]
:bank-account (when bank-account-id [:bank-account/original-id (str company-id "-" bank-account-id)]) :bank-account (when bank-account-id [:bank-account/original-id (str company-id "-" bank-account-id)])
:check-number check-number :check-number check-number

11
src/clj/auto_ap/datomic/checks.clj
View File

@@ -27,6 +27,7 @@
{:payment/type [:db/ident]}])) {:payment/type [:db/ident]}]))
(defn raw-graphql [args] (defn raw-graphql [args]
(println "ARGS" args)
(let [query (cond-> {:query {:find [default-read] (let [query (cond-> {:query {:find [default-read]
:in ['$] :in ['$]
:where ['[?e :payment/client]]} :where ['[?e :payment/client]]}
@@ -36,7 +37,15 @@
'[?e :payment/client ?client-id]) '[?e :payment/client ?client-id])
(:original-id args) (add-arg '?original-id (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong ) (:original-id args) (add-arg '?original-id (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong )
'[?e :payment/client ?c] '[?e :payment/client ?c]
'[?c :client/original-id ?original-id]))] '[?c :client/original-id ?original-id])
(:check-number args) (add-arg '?check-number (:check-number args)
'[?e :payment/check-number ?check-number])
(:bank-account-id args) (add-arg '?bank-account-id (:bank-account-id args)
'[?e :payment/bank-account ?bank-account-id])
(:amount args) (add-arg '?amount (:amount args)
'[?e :payment/amount ?amount])
(:status args) (add-arg '?status (:status args)
'[?e :payment/status ?status]))]
(->> (d/query (->> (d/query
query) query)

2
src/clj/auto_ap/datomic/transactions.clj
View File

@@ -16,7 +16,7 @@
(cond-> {:query {:find ['(pull ?e [* {:transaction/client [:client/name :db/id] (cond-> {:query {:find ['(pull ?e [* {:transaction/client [:client/name :db/id]
:transaction/bank-account [:bank-account/name :bank-account/yodlee-account-id]}])] :transaction/bank-account [:bank-account/name :bank-account/yodlee-account-id]}])]
:in ['$] :in ['$]
:where ['[?e :transaction/original-id]]} :where ['[?e :transaction/id]]}
:args [(d/db (d/connect uri))]} :args [(d/db (d/connect uri))]}
(:client-id args) (add-arg '?client-id (Long/parseLong (:client-id args)) (:client-id args) (add-arg '?client-id (Long/parseLong (:client-id args))

2
src/clj/auto_ap/graphql.clj
View File

@@ -222,7 +222,7 @@
:resolve :get-all-payments} :resolve :get-all-payments}
:transaction_page {:type '(list :transaction_page) :transaction_page {:type '(list :transaction_page)
:args {:company_id {:type 'String} :args {:client_id {:type 'String}
:start {:type 'Int} :start {:type 'Int}
:sort_by {:type 'String} :sort_by {:type 'String}
:asc {:type 'Boolean}} :asc {:type 'Boolean}}

1
src/clj/auto_ap/graphql/checks.clj
View File

@@ -325,6 +325,7 @@
bank-account :payment-type/check 0 {(Long/parseLong (:invoice_id args)) (:amount args)})] bank-account :payment-type/check 0 {(Long/parseLong (:invoice_id args)) (:amount args)})]
@(d/transact (d/connect uri) @(d/transact (d/connect uri)
[(assoc base-payment [(assoc base-payment
:payment/status :payment-status/pending
:payment/check-number (:check_number args) :payment/check-number (:check_number args)
:payment/date (c/to-date (parse (:date args) iso-date)) :payment/date (c/to-date (parse (:date args) iso-date))
:payment/amount (:amount args)) :payment/amount (:amount args))

26
src/clj/auto_ap/routes/invoices.clj
View File

@@ -66,17 +66,17 @@
(defn parse-account-id [i] (defn parse-account-id [i]
(try (try
(Integer/parseInt (second (Long/parseLong (second
(re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:account-id i)))) (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:bank-account-id i))))
(catch Exception e (catch Exception e
(throw (Exception. (str "Could not parse account from value '" (:account-id i) "'") e))))) (throw (Exception. (str "Could not parse account from value '" (:bank-account-id i) "'") e)))))
(defn parse-company-id [i] (defn parse-client-id [i]
(try (try
(Integer/parseInt (second (Long/parseLong (second
(re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:company-id i)))) (re-matches #"[^0-9,\\-]*([0-9,\\-]+)[^0-9,]*" (:client-id i))))
(catch Exception e (catch Exception e
(throw (Exception. (str "Could not parse company from value '" (:company-id i) "'") e))))) (throw (Exception. (str "Could not parse client from value '" (:client-id i) "'") e)))))
(defn parse-date [{:keys [raw-date]}] (defn parse-date [{:keys [raw-date]}]
(try (try
@@ -99,28 +99,28 @@
(POST "/batch-upload" (POST "/batch-upload"
{{:keys [data]} :edn-params user :identity} {{:keys [data]} :edn-params user :identity}
(assert-admin user) (assert-admin user)
(let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :account-id :company-id] (let [columns [:status :raw-date :description-original :high-level-category nil nil :amount nil nil nil nil nil :bank-account-id :client-id]
rows (->> (str/split data #"\n" ) rows (->> (str/split data #"\n" )
(drop 1) (drop 1)
(map #(str/split % #"\t")) (map #(str/split % #"\t"))
(map #(into {} (filter identity (map (fn [c k] [k c] ) % columns)))) (map #(into {} (filter identity (map (fn [c k] [k c] ) % columns))))
(map (parse-or-error :amount parse-amount)) (map (parse-or-error :amount parse-amount))
(map (parse-or-error :account-id parse-account-id)) (map (parse-or-error :bank-account-id parse-account-id))
(map (parse-or-error :company-id parse-company-id)) (map (parse-or-error :client-id parse-client-id))
(map (parse-or-error :date parse-date))) (map (parse-or-error :date parse-date)))
error-rows (filter :errors rows) error-rows (filter :errors rows)
raw-transactions (vec (->> rows raw-transactions (vec (->> rows
(filter #(not (seq (:errors %))) ) (filter #(not (seq (:errors %))) )
(map (fn [{:keys [description-original company-id status high-level-category amount account-id date]}] (map (fn [{:keys [description-original client-id status high-level-category amount bank-account-id date]}]
{:description-original description-original {:description-original description-original
:date date :date date
:status status :status status
:high-level-category high-level-category :high-level-category high-level-category
:amount amount :amount amount
:company-id company-id :client-id client-id
:account-id account-id}))))] :bank-account-id bank-account-id}))))]
(manual-import raw-transactions) (manual-import raw-transactions)

2
src/clj/auto_ap/server.clj
View File

@@ -1,5 +1,5 @@
(ns auto-ap.server (ns auto-ap.server
(:require [auto-ap.background.mail :refer [always-process-sqs]] (:require #_[auto-ap.background.mail :refer [always-process-sqs]]
[auto-ap.handler :refer [app]] [auto-ap.handler :refer [app]]
[clojure.tools.nrepl.server :refer [start-server stop-server]] [clojure.tools.nrepl.server :refer [start-server stop-server]]
[config.core :refer [env]] [config.core :refer [env]]

90
src/clj/auto_ap/yodlee/import.clj
View File

@@ -3,31 +3,35 @@
[auto-ap.db.transactions :as transactions] [auto-ap.db.transactions :as transactions]
[auto-ap.db.vendors :as vendors] [auto-ap.db.vendors :as vendors]
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
[datomic.api :as d]
[auto-ap.datomic :refer [uri remove-nils]]
[auto-ap.db.companies :as companies] [auto-ap.db.companies :as companies]
[clj-time.coerce :as coerce]
[digest :refer [sha-256]] [digest :refer [sha-256]]
[auto-ap.db.checks :as checks] [auto-ap.datomic.checks :as d-checks]
[auto-ap.time :as time])) [auto-ap.time :as time]))
(defn transaction->check-id [_ check-number company-id bank-account-id amount] (defn transaction->check-id [_ check-number client-id bank-account-id amount]
(cond (and check-number company-id bank-account-id) (cond (and check-number client-id bank-account-id)
(-> (checks/get-graphql {:company-id company-id (-> (d-checks/get-graphql {:client-id client-id
:bank-account-id bank-account-id :bank-account-id bank-account-id
:check-number check-number :check-number check-number
:amount (- amount) :amount (- amount)
:status "pending"}) :status :payment-status/pending})
first first
:id) (doto println)
:db/id)
(and company-id bank-account-id amount) (and client-id bank-account-id amount)
(let [matching-checks (checks/get-graphql {:company-id company-id (let [matching-checks (d-checks/get-graphql {:client-id client-id
:bank-account-id bank-account-id :bank-account-id bank-account-id
:amount (- amount) :amount (- amount)
:status "pending"})] :status :payment-status/pending})]
(if (= 1 (count matching-checks)) (if (= 1 (count matching-checks))
(:id (first matching-checks)) (:db/id (first matching-checks))
nil)) nil))
:else :else
@@ -61,31 +65,32 @@
(- amount) (- amount)
amount) amount)
check-number (extract-check-number transaction) check-number (extract-check-number transaction)
company-id (transaction->company transaction) client-id (transaction->company transaction)
bank-account-id (transaction->bank-account-id transaction) bank-account-id (transaction->bank-account-id transaction)
check-id (transaction->check-id transaction check-number company-id bank-account-id amount)]] check-id (transaction->check-id transaction check-number client-id bank-account-id amount)]]
(try (try
(when company-id (when client-id
(transactions/upsert! @(->> [(remove-nils #:transaction
{:post-date (time/parse post-date "YYYY-MM-dd") {:post-date (time/parse post-date "YYYY-MM-dd")
:id (sha-256 (str id)) :id (sha-256 (str id))
:account-id account-id :account-id account-id
:date (time/parse date "YYYY-MM-dd") :date (coerce/to-date (time/parse date "YYYY-MM-dd"))
:amount amount :amount amount
:description-original description-original :description-original description-original
:description-simple description-simple :description-simple description-simple
:type type :type type
:status status :status status
:company-id company-id :client client-id
:check-number check-number :check-number check-number
:bank-account-id (transaction->bank-account-id transaction) :bank-account (transaction->bank-account-id transaction)
:check-id check-id :payment (when check-id
}) {:db/id check-id
(when check-id :payment/status :payment-status/cleared}
(checks/update! {:id check-id :status "cleared"}))) )
})]
(d/transact (d/connect uri))))
(catch Exception e (catch Exception e
(println e))))) (println e)))))
@@ -94,25 +99,22 @@
(let [transformed-transactions (->> manual-transactions (let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %))) (filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount])) (group-by #(select-keys % [:date :description-original :amount]))
(vals) (vals)
(mapcat (fn [transaction-group] (mapcat (fn [transaction-group]
(map (map
(fn [index {:keys [date description-original high-level-category amount account-id company-id] :as transaction}] (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" account-id "-" description-original "-" amount "-" index "-" company-id) {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:company-id company-id :client-id client-id
:bank-account-id account-id :bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd") :date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount} :amount {:amount amount}
:description {:original description-original :description {:original description-original
:simple high-level-category} :simple high-level-category}
:status "POSTED"}) :status "POSTED"})
(range) (range)
transaction-group))))] transaction-group))))]
(println "importing manual transactions" transformed-transactions) (println "importing manual transactions" transformed-transactions)
(import-transactions transformed-transactions :company-id :bank-account-id))) (import-transactions transformed-transactions :client-id :bank-account-id)))
(defn do-import [] (defn do-import []
(let [transactions (client/get-transactions) (let [transactions (client/get-transactions)

2
src/cljs/auto_ap/views/components/expense_accounts_dialog.cljs
View File

@@ -82,7 +82,7 @@
(map js/parseFloat) (map js/parseFloat)
(map #(or % 0)) (map #(or % 0))
(reduce + 0)) (reduce + 0))
does-add-up? (< (Math/abs (- expense-accounts-total (js/parseFloat total))) 0.01)]] does-add-up? (< (Math/abs (- expense-accounts-total (js/parseFloat total))) 0.01)]
[action-modal {:id ::change-expense-accounts [action-modal {:id ::change-expense-accounts
:title "Change expense accounts" :title "Change expense accounts"
:action-text "Save" :action-text "Save"

2
src/cljs/auto_ap/views/components/paginator.cljs
View File

@@ -19,7 +19,7 @@
y)) y))
(defn paginator [{:keys [start end count total on-change]}] (defn paginator [{:keys [start end count total on-change]}]
(let [per-page 10 (let [per-page 20
max-buttons 5 max-buttons 5
buttons-before (Math/floor (/ max-buttons 2)) buttons-before (Math/floor (/ max-buttons 2))
total-pages (Math/ceil (/ total per-page)) total-pages (Math/ceil (/ total per-page))

2
src/cljs/auto_ap/views/pages/transactions.cljs
View File

@@ -33,7 +33,7 @@
(assoc-in [::params] params)) (assoc-in [::params] params))
:graphql {:token (-> cofx :db :user) :graphql {:token (-> cofx :db :user)
:query-obj {:venia/queries [[:transaction_page :query-obj {:venia/queries [[:transaction_page
(assoc params :company-id (:id @(re-frame/subscribe [::subs/company]))) (assoc params :client-id (:id @(re-frame/subscribe [::subs/company])))
[[:transactions [:id [[:transactions [:id
:amount :amount
:date :date