diff --git a/Dockerfile b/Dockerfile
index d48a107d..6fc8df41 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,4 +4,4 @@ RUN apk add poppler
RUN apk add poppler-utils
COPY target/auto-ap.jar /usr/local/
COPY config /usr/local/config/
-CMD java -jar /usr/local/auto-ap.jar
+CMD java -Dlogback.configurationFile=logback-prod.xml -jar /usr/local/auto-ap.jar
diff --git a/project.clj b/project.clj
index 7d337f84..933f3340 100644
--- a/project.clj
+++ b/project.clj
@@ -45,6 +45,11 @@
[buddy/buddy-auth "2.1.0"]
[buddy/buddy-sign "2.1.0"]
[nrepl "0.6.0" :exclusions [org.clojure/tools.logging]]
+ [org.clojure/tools.logging "1.1.0"]
+ [ch.qos.logback/logback-classic "1.2.3"]
+ [ch.qos.logback.contrib/logback-jackson "0.1.5"]
+ [ch.qos.logback.contrib/logback-json-classic "0.1.5"]
+ [spootnik/unilog "0.7.24"]
[clj-time "0.14.3"]
[io.forward/clojure-mail "1.0.7"]
[ring/ring-json "0.4.0" :exclusions [cheshire]]
@@ -82,6 +87,7 @@
{:resource-paths ["resources" "target"]
:dependencies [[binaryage/devtools "0.9.4"]
+ #_[refactor-nrepl "2.5.0"]
[com.bhauman/figwheel-main "0.2.3" :exclusions [org.clojure/clojurescript
ring/ring-core
ring/ring-codec
@@ -134,7 +140,7 @@
:main auto-ap.server
- :aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format]
+ :aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format clojure.tools.logging.impl]
:uberjar-name "auto-ap.jar"
diff --git a/resources/logback-prod.xml b/resources/logback-prod.xml
new file mode 100644
index 00000000..4c9806d3
--- /dev/null
+++ b/resources/logback-prod.xml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+
diff --git a/resources/logback.xml b/resources/logback.xml
new file mode 100644
index 00000000..0e36ab38
--- /dev/null
+++ b/resources/logback.xml
@@ -0,0 +1,14 @@
+
+
+
+
+ %d{HH:mm:ss.SSS} [%thread] %highlight(%-5level) %logger{36} - %green(%msg) %gray(%mdc) %n
+
+
+
+
+
+
+
+
+
diff --git a/resources/logback_.xml b/resources/logback_.xml
new file mode 100644
index 00000000..5505f960
--- /dev/null
+++ b/resources/logback_.xml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+
diff --git a/src/clj/auto_ap/background/invoices.clj b/src/clj/auto_ap/background/invoices.clj
index 326b5e46..55d93136 100644
--- a/src/clj/auto_ap/background/invoices.clj
+++ b/src/clj/auto_ap/background/invoices.clj
@@ -5,30 +5,33 @@
[auto-ap.time :as time]
[clj-time.coerce :as coerce]
[mount.core :as mount]
- [yang.scheduler :as scheduler]))
+ [yang.scheduler :as scheduler]
+ [unilog.context :as lc]
+ [clojure.tools.logging :as log]))
(defn close-auto-invoices []
- (try
-
- (let [invoices-to-close (d/query {:query {:find ['?e]
- :in ['$ '?today]
- :where ['[?e :invoice/automatically-paid-when-due true]
- '[?e :invoice/status :invoice-status/unpaid]
- '[?e :invoice/due ?d]
- '[(<= ?d ?today)]]}
- :args [(d/db conn) (coerce/to-date (time/local-now))]})]
- (println "Closing " (count invoices-to-close) "automatic invoices")
- (some->> invoices-to-close
- seq
+ (lc/with-context {:source "close-auto-invoices"}
+ (try
+
+ (let [invoices-to-close (d/query {:query {:find ['?e]
+ :in ['$ '?today]
+ :where ['[?e :invoice/automatically-paid-when-due true]
+ '[?e :invoice/status :invoice-status/unpaid]
+ '[?e :invoice/due ?d]
+ '[(<= ?d ?today)]]}
+ :args [(d/db conn) (coerce/to-date (time/local-now))]})]
+ (log/info "Closing " (count invoices-to-close) "automatic invoices")
+ (some->> invoices-to-close
+ seq
- (mapv (fn [[i]] {:db/id i
- :invoice/outstanding-balance 0.0
- :invoice/status :invoice-status/paid}))
- (d/transact conn)
- deref)
- (println "Closed " (count invoices-to-close) "automatic invoices"))
- (catch Exception e
- (println (.toString e)))))
+ (mapv (fn [[i]] {:db/id i
+ :invoice/outstanding-balance 0.0
+ :invoice/status :invoice-status/paid}))
+ (d/transact conn)
+ deref)
+ (log/info "Closed " (count invoices-to-close) "automatic invoices"))
+ (catch Exception e
+ (log/error e)))))
(mount/defstate close-auto-invoices-worker
:start (scheduler/every 60000 close-auto-invoices)
diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj
index 46ae7fdb..918635e7 100644
--- a/src/clj/auto_ap/datomic/invoices.clj
+++ b/src/clj/auto_ap/datomic/invoices.clj
@@ -156,8 +156,7 @@
[(->> (graphql-results ids-to-retrieve db args))
matching-count
- (doto
- outstanding println)]))
+ outstanding]))
(defn get-by-id [id]
(-> (d/db (d/connect uri))
@@ -186,8 +185,7 @@
[(not= ?e ?invoice-id)]
]}
- :args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}
- true (doto println)))
+ :args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}))
(map first)
(map <-datomic)))
diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj
index 92b92649..34cabfbb 100644
--- a/src/clj/auto_ap/datomic/transactions.clj
+++ b/src/clj/auto_ap/datomic/transactions.clj
@@ -115,8 +115,7 @@
"status" ['[?e :transaction/status ?sort-status]]}
args)
true
- (merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}}))
- _ (println query)]
+ (merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}}))]
(cond->> query
true (d/query)
true (apply-sort-3 args)
diff --git a/src/clj/auto_ap/datomic/users.clj b/src/clj/auto_ap/datomic/users.clj
index 5b23c7b1..e9d113b2 100644
--- a/src/clj/auto_ap/datomic/users.clj
+++ b/src/clj/auto_ap/datomic/users.clj
@@ -41,7 +41,6 @@
first
first
(update :user/role :db/ident))]
- (println "USER" user)
(if user
user
(let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user
diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj
index a76aac2a..dd324d82 100644
--- a/src/clj/auto_ap/graphql.clj
+++ b/src/clj/auto_ap/graphql.clj
@@ -7,6 +7,7 @@
[com.walmartlabs.lacinia.resolve :as resolve]
[buddy.auth :refer [throw-unauthorized]]
[auto-ap.utils :refer [by]]
+ [auto-ap.logging :refer [info-event warn-event error-event]]
[auto-ap.graphql.utils :refer [assert-admin can-see-client? assert-can-see-client]]
[auto-ap.datomic :refer [uri merge-query]]
[datomic.api :as d]
@@ -29,7 +30,10 @@
[auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.time :as time]
[clojure.walk :as walk]
- [clojure.string :as str])
+ [clojure.string :as str]
+ [clojure.tools.logging :as log]
+ [yang.time :refer [time-it]]
+ [unilog.context :as lc])
(:import
(clojure.lang IPersistentMap)))
@@ -945,7 +949,6 @@
(filter (fn [[potential-id potential-date]]
(let [date (coerce/to-date-time date)
potential-date (coerce/to-date-time potential-date)]
- #_(println "HERE" id potential-id potential-date date)
(and (= id potential-id)
(<= (t/in-days (apply t/interval (sort [date potential-date]))) 10)))))
conj
@@ -1010,7 +1013,6 @@
:amount amount
:date (coerce/to-date-time next)})
is-week-a? (fn [d]
- (println d)
(= 0 (mod (t/in-weeks (t/interval first-week-a d)) 2)))]
{:beginning_balance total-cash
@@ -1110,20 +1112,24 @@
([id q]
(query id q nil ))
([id q v]
- (println "executing graphql query" id q v)
- (try
- (let [result (time (simplify (execute schema q v {:id id})))]
- (when (seq (:errors result))
- (throw (ex-info "GraphQL error" {:result result})))
- result)
+
+ (lc/with-context {:query q}
+ (log/info "Executing query" q)
+
+ (try
+ (let [[result time] (time-it (simplify (execute schema q v {:id id})))]
+
+ (info-event "Query completed"
+ {:time (:time time)
+ :errors (seq (:errors result))})
+ (when (seq (:errors result))
+ (throw (ex-info "GraphQL error" {:result result})))
+ result)
- (catch Exception e
- (if-let [v (:validation-error (ex-data e))]
- (println "validation error" v)
- (do
- (.printStackTrace e)
- (println e )))
+ (catch Exception e
+ (if-let [v (:validation-error (ex-data e))]
+ (warn-event "validation error" {:validation-error v
+ :data (ex-data e)})
+ (error-event "query error" {:error e}))
- (throw e)))))
-
-#_(query nil "{ ledger_page { count }}" nil)
+ (throw e))))))
diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj
index 142607a2..259954bb 100644
--- a/src/clj/auto_ap/graphql/clients.clj
+++ b/src/clj/auto_ap/graphql/clients.clj
@@ -19,8 +19,6 @@
(let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client)))
id (or (:db/id client) "new-client")
- _ (println id)
- _ (println edit_client)
_ (when client
@(d/transact (d/connect uri)
(into
@@ -79,7 +77,6 @@
)
(:forecasted_transactions edit_client))]]
result @(d/transact (d/connect uri) transactions)]
- (println result "ID" id)
(-> result :tempids (get id) (or id) d-clients/get-by-id
(update :client/location-matches
(fn [lms]
diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj
index 0bc73b95..e10f9c24 100644
--- a/src/clj/auto_ap/graphql/invoices.clj
+++ b/src/clj/auto_ap/graphql/invoices.clj
@@ -58,7 +58,6 @@
:location location}))
(defn add-invoice-transaction [{:keys [total invoice_number location automatically_paid_when_due client_id vendor_id vendor_name date due expense_accounts] :as in}]
- (println date)
(let [vendor (d-vendors/get-by-id vendor_id)
account (:vendor/default-account vendor)
_ (when-not (:db/id account)
@@ -119,11 +118,10 @@
(defn edit-invoice [context {{:keys [id due invoice_number total vendor_id date client_id expense_accounts automatically_paid_when_due] :as in} :invoice} value]
(let [invoice (d-invoices/get-by-id id)
- _ (when (seq (doto (d-invoices/find-conflicting {:db/id id
- :invoice/invoice-number invoice_number
- :invoice/vendor (:db/id (:invoice/vendor invoice))
- :invoice/client (:db/id (:invoice/client invoice))})
- println))
+ _ (when (seq (d-invoices/find-conflicting {:db/id id
+ :invoice/invoice-number invoice_number
+ :invoice/vendor (:db/id (:invoice/vendor invoice))
+ :invoice/client (:db/id (:invoice/client invoice))}))
(throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number})))
expense-account-total (reduce + 0 (map (fn [x] (Double/parseDouble (:amount x))) expense_accounts))
@@ -151,7 +149,6 @@
(defn void-invoice [context {id :invoice_id} value]
(let [invoice (d-invoices/get-by-id id)
- _ (println invoice)
_ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
updated-invoice (d-invoices/update {:db/id id
:invoice/total 0.0
diff --git a/src/clj/auto_ap/graphql/ledger.clj b/src/clj/auto_ap/graphql/ledger.clj
index fcd5b409..c7498e58 100644
--- a/src/clj/auto_ap/graphql/ledger.clj
+++ b/src/clj/auto_ap/graphql/ledger.clj
@@ -12,7 +12,9 @@
[auto-ap.parse.util :as parse]
[datomic.api :as d]
[auto-ap.parse.templates :as t]
- [auto-ap.datomic.clients :as d-clients]))
+ [auto-ap.datomic.clients :as d-clients]
+ [clojure.tools.logging :as log]
+ [unilog.context :as lc]))
(defn get-ledger-page [context args value]
(let [args (assoc args :id (:id context))
@@ -106,15 +108,12 @@
vals
(mapcat (fn [a]
(map (fn [o]
- #_(println "override" (:db/id a) (:db/id (:account-client-override/client o)))
- #_(print [[(:db/id a) (:db/id (:account-client-override/client o))]])
[[(:db/id a) (:db/id (:account-client-override/client o))]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
- #_(println a client-id (keys overrides-by-client))
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
@@ -158,7 +157,6 @@
(defn get-profit-and-loss [context args value]
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
- _ (println args)
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
@@ -179,108 +177,113 @@
(defn import-ledger [context args value]
(assert-admin (:id context))
- (let [all-vendors (by :vendor/name (d-vendors/get-graphql {}))
- all-clients (by :client/code (d-clients/get-all ))
- all-client-bank-accounts (reduce
- (fn [acc client]
- (assoc acc (:client/code client)
- (set (->> (:client/bank-accounts client)
- (map :bank-account/code)
- ))))
- {}
- (d-clients/get-all))
- all-client-locations (reduce
- (fn [acc client]
- (assoc acc (:client/code client)
- (-> (set (:client/locations client))
- (conj "HQ")
- (conj "A"))))
+ (lc/with-context {:area "import ledger"}
+ (let [all-vendors (by :vendor/name (d-vendors/get-graphql {}))
+ all-clients (by :client/code (d-clients/get-all ))
+ all-client-bank-accounts (reduce
+ (fn [acc client]
+ (assoc acc (:client/code client)
+ (set (->> (:client/bank-accounts client)
+ (map :bank-account/code)
+ ))))
+ {}
+ (d-clients/get-all))
+ all-client-locations (reduce
+ (fn [acc client]
+ (assoc acc (:client/code client)
+ (-> (set (:client/locations client))
+ (conj "HQ")
+ (conj "A"))))
+ {}
+ (d-clients/get-all))
+ new-hidden-vendors (reduce
+ (fn [new-vendors {:keys [vendor_name line_items]}]
+ (if (or (all-vendors vendor_name)
+ (new-vendors vendor_name))
+ new-vendors
+ (assoc new-vendors vendor_name
+ {:vendor/name vendor_name
+ :vendor/hidden true
+ :db/id vendor_name})))
{}
- (d-clients/get-all))
- new-hidden-vendors (reduce
- (fn [new-vendors {:keys [vendor_name line_items]}]
- (if (or (all-vendors vendor_name)
- (new-vendors vendor_name))
- new-vendors
- (assoc new-vendors vendor_name
- {:vendor/name vendor_name
- :vendor/hidden true
- :db/id vendor_name})))
- {}
- (:entries args))
- all-vendors (into all-vendors new-hidden-vendors)
- all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
- transaction (doall (map
- (assoc-error (fn [entry]
- (let [entry (-> entry
- (update :amount #(Double/parseDouble %))
- (update :line_items
- (fn [lis]
- (mapv
- (fn [li ]
- (-> li
- (update :debit #(Double/parseDouble (if (str/blank? %) "0" %)))
- (update :credit #(Double/parseDouble (if (str/blank? %) "0" %)))))
- lis))))]
- (let [vendor (all-vendors (:vendor_name entry))]
- (when-not (all-clients (:client_code entry))
- (throw (Exception. (str "Client '" (:client_code entry )"' not found.")) ))
- (when-not vendor
- (throw (Exception. (str "Vendor '" (:vendor_name entry) "' not found."))))
- (when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
- (throw (Exception. (str "Date must be MM/dd/yyyy"))))
- (when-not (dollars= (doto (reduce + 0.0 (map :debit (:line_items entry))))
- (reduce + 0.0 (map :credit (:line_items entry))))
- (throw (Exception. (str "Debits '"
- (reduce + 0 (map :debit (:line_items entry)))
- "' and credits '"
- (reduce + 0 (map :credit (:line_items entry)))
- "' do not add up."))))
- (remove-nils
- {:journal-entry/source (:source entry)
- :journal-entry/client [:client/code (:client_code entry)]
- :journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
- :journal-entry/external-id (:external_id entry)
- :journal-entry/vendor (all-vendors (:vendor_name entry))
- :journal-entry/amount (:amount entry)
- :journal-entry/note (:note entry)
- :journal-entry/cleared-against (:cleared_against entry)
+ (:entries args))
+ all-vendors (into all-vendors new-hidden-vendors)
+ all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
+ transaction (doall (map
+ (assoc-error (fn [entry]
+ (let [entry (-> entry
+ (update :amount #(Double/parseDouble %))
+ (update :line_items
+ (fn [lis]
+ (mapv
+ (fn [li ]
+ (-> li
+ (update :debit #(Double/parseDouble (if (str/blank? %) "0" %)))
+ (update :credit #(Double/parseDouble (if (str/blank? %) "0" %)))))
+ lis))))]
+ (let [vendor (all-vendors (:vendor_name entry))]
+ (when-not (all-clients (:client_code entry))
+ (throw (Exception. (str "Client '" (:client_code entry )"' not found.")) ))
+ (when-not vendor
+ (throw (Exception. (str "Vendor '" (:vendor_name entry) "' not found."))))
+ (when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
+ (throw (Exception. (str "Date must be MM/dd/yyyy"))))
+ (when-not (dollars= (doto (reduce + 0.0 (map :debit (:line_items entry))))
+ (reduce + 0.0 (map :credit (:line_items entry))))
+ (throw (Exception. (str "Debits '"
+ (reduce + 0 (map :debit (:line_items entry)))
+ "' and credits '"
+ (reduce + 0 (map :credit (:line_items entry)))
+ "' do not add up."))))
+ (remove-nils
+ {:journal-entry/source (:source entry)
+ :journal-entry/client [:client/code (:client_code entry)]
+ :journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
+ :journal-entry/external-id (:external_id entry)
+ :journal-entry/vendor (all-vendors (:vendor_name entry))
+ :journal-entry/amount (:amount entry)
+ :journal-entry/note (:note entry)
+ :journal-entry/cleared-against (:cleared_against entry)
- :journal-entry/line-items
- (mapv (fn [ea]
- (when-not (get
- (get all-client-locations (:client_code entry))
- (:location ea))
- (throw (Exception. (str "Location '" (:location ea) "' not found."))))
- (when (< (or (:debit ea) (:credit ea)) 0.0)
- (throw (Exception. (str (or (:debit ea) (:credit ea)) "must be greater than 0."))))
- (when (and (not (all-accounts (:account_identifier ea)))
- (not (get
- (get all-client-bank-accounts (:client_code entry))
- (:account_identifier ea))))
- (throw (Exception. (str "Account '" (:account_identifier ea) "' not found."))))
- (remove-nils {:journal-entry-line/account
- (if (re-matches #"^[0-9]+$" (:account_identifier ea))
- (:db/id (a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))
- [:bank-account/code (:account_identifier ea)])
- :journal-entry-line/location (:location ea)
- :journal-entry-line/debit (when (> (:debit ea) 0)
- (:debit ea))
- :journal-entry-line/credit (when (> (:credit ea) 0)
- (:credit ea))}))
- (:line_items entry))
-
- :journal-entry/cleared true})))))
- (:entries args)))
- errors (filter :error transaction)
- success (filter (comp not :error) transaction)
- retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]])
- success)]
- (println (take 4 success))
- (run! (fn [batch] (println "transacting retraction batch") @(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
- (run! (fn [batch] (println "transacting success batch") @(d/transact (d/connect uri) batch)) (partition-all 100 success))
- {:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
- :existing []
- :errors (map (fn [x] {:external_id (:external_id x)
- :error (:error x)}) errors)}))
+ :journal-entry/line-items
+ (mapv (fn [ea]
+ (when-not (get
+ (get all-client-locations (:client_code entry))
+ (:location ea))
+ (throw (Exception. (str "Location '" (:location ea) "' not found."))))
+ (when (< (or (:debit ea) (:credit ea)) 0.0)
+ (throw (Exception. (str (or (:debit ea) (:credit ea)) "must be greater than 0."))))
+ (when (and (not (all-accounts (:account_identifier ea)))
+ (not (get
+ (get all-client-bank-accounts (:client_code entry))
+ (:account_identifier ea))))
+ (throw (Exception. (str "Account '" (:account_identifier ea) "' not found."))))
+ (remove-nils {:journal-entry-line/account
+ (if (re-matches #"^[0-9]+$" (:account_identifier ea))
+ (:db/id (a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))
+ [:bank-account/code (:account_identifier ea)])
+ :journal-entry-line/location (:location ea)
+ :journal-entry-line/debit (when (> (:debit ea) 0)
+ (:debit ea))
+ :journal-entry-line/credit (when (> (:credit ea) 0)
+ (:credit ea))}))
+ (:line_items entry))
+
+ :journal-entry/cleared true})))))
+ (:entries args)))
+ errors (filter :error transaction)
+ success (filter (comp not :error) transaction)
+ retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]])
+ success)]
+ (log/info "manual ledger import has " (count success) " new rows")
+ (run! (fn [batch]
+ (log/info "transacting retraction batch")
+ @(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
+ (run! (fn [batch]
+ (log/info "transacting success batch")
+ @(d/transact (d/connect uri) batch)) (partition-all 100 success))
+ {:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
+ :existing []
+ :errors (map (fn [x] {:external_id (:external_id x)
+ :error (:error x)}) errors)})))
diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj
index 732f4575..4cfe23b7 100644
--- a/src/clj/auto_ap/graphql/transactions.clj
+++ b/src/clj/auto_ap/graphql/transactions.clj
@@ -18,10 +18,6 @@
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm]
[clj-time.coerce :as coerce]))
-(defn prn-each [xs]
- (doseq [x xs]
- (println x))
- xs)
(def approval-status->graphql (ident->enum-f :transaction/approval-status))
diff --git a/src/clj/auto_ap/graphql/users.clj b/src/clj/auto_ap/graphql/users.clj
index d6e4304d..bc0f5804 100644
--- a/src/clj/auto_ap/graphql/users.clj
+++ b/src/clj/auto_ap/graphql/users.clj
@@ -10,7 +10,6 @@
":user" :user-role/user})
(defn edit-user [context {:keys [edit_user] :as args} value]
- (println args edit_user)
(assert-admin (:id context))
(let [user (d-users/get-by-id (:id edit_user))
new-clients (set (map #(Long/parseLong %) (:clients edit_user)))
@@ -21,11 +20,10 @@
@(d/transact (d/connect uri)
- (doto (-> [{:db/id (:db/id user)
- :user/role (role->datomic-role (:role edit_user))
- :user/clients new-clients}]
- (into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove)))
- clojure.pprint/pprint))
+ (-> [{:db/id (:db/id user)
+ :user/role (role->datomic-role (:role edit_user))
+ :user/clients new-clients}]
+ (into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove))))
(->graphql
(d-users/get-by-id (:id edit_user)))))
diff --git a/src/clj/auto_ap/graphql/utils.clj b/src/clj/auto_ap/graphql/utils.clj
index e6b75157..6699906e 100644
--- a/src/clj/auto_ap/graphql/utils.clj
+++ b/src/clj/auto_ap/graphql/utils.clj
@@ -1,7 +1,8 @@
(ns auto-ap.graphql.utils
(:require [clojure.string :as str]
[buddy.auth :refer [throw-unauthorized]]
- [clojure.walk :as walk]))
+ [clojure.walk :as walk]
+ [clojure.tools.logging :as log]))
(defn snake->kebab [s]
@@ -43,24 +44,23 @@
m))
(defn is-admin? [id]
- (println "role" id)
(= "admin" (:user/role id)))
(defn assert-admin [id]
- (println "role" id)
(when-not (= "admin" (:user/role id))
+ (log/warn "user " id " not an admin!")
(throw-unauthorized)))
(defn can-see-client? [identity client]
(when (not client)
- (println "WARNING - permission checking for null client"))
+ (log/warn "WARNING - permission checking for null client"))
(or (= "admin" (:user/role identity))
((set (map :db/id (:user/clients identity))) (:db/id client))
((set (map :db/id (:user/clients identity))) client)))
(defn assert-can-see-client [identity client]
(when-not (can-see-client? identity client)
- (println "IDENTITY " identity " can not see company " client)
+ (log/warn "IDENTITY " identity " can not see company " client)
(throw-unauthorized)))
(defn limited-clients [id]
diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj
index e8a30c6b..61714f12 100644
--- a/src/clj/auto_ap/graphql/vendors.clj
+++ b/src/clj/auto_ap/graphql/vendors.clj
@@ -72,19 +72,15 @@
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/account-overrides account-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/terms-overrides terms-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/automatically-paid-when-due
- (doto (mapv
- (fn [apwd]
- {:db/id apwd})
- (:automatically_paid_when_due in))
- println)]))
+ (mapv
+ (fn [apwd]
+ {:db/id apwd})
+ (:automatically_paid_when_due in))]))
- _ (println transaction)
transaction-result @(d/transact (d/connect uri) transaction)]
(-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
id))
-
- (doto println)
(->graphql))))
(defn merge-vendors [context {:keys [from to]} value]
diff --git a/src/clj/auto_ap/handler.clj b/src/clj/auto_ap/handler.clj
index 6ae319d0..0935677e 100644
--- a/src/clj/auto_ap/handler.clj
+++ b/src/clj/auto_ap/handler.clj
@@ -16,6 +16,8 @@
[compojure.core :refer :all]
[compojure.route :as route]
[config.core :refer [env]]
+ [clojure.tools.logging :as log]
+ [unilog.context :as lc]
[ring.middleware.edn :refer [wrap-edn-params]]
[ring.middleware.multipart-params :as mp]
[ring.middleware.params :refer [wrap-params]]
@@ -55,10 +57,20 @@
static-routes))
+(defn wrap-logging [handler]
+ (fn [request]
+ (lc/with-context {:uri (:uri request)
+ :source "request"
+ :user-role (:user/role (:identity request))
+ :user-name (:user/name (:identity request))}
+ (log/info "Beginning request" (:uri request))
+ (handler request))))
+
(def app
(-> #'app-routes
+ (wrap-logging)
(wrap-authorization auth-backend)
(wrap-authentication auth-backend)
(wrap-reload)
diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj
index 129a8ed9..174a719c 100644
--- a/src/clj/auto_ap/ledger.clj
+++ b/src/clj/auto_ap/ledger.clj
@@ -4,7 +4,10 @@
[mount.core :as mount]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic :refer [uri remove-nils]]
- [clojure.spec.alpha :as s]))
+ [clojure.spec.alpha :as s]
+ [clojure.tools.logging :as log]
+ [auto-ap.logging :refer [info-event]]
+ [unilog.context :as lc]))
(defn datums->impacted-entity [db [e changes]]
@@ -129,66 +132,75 @@
(defn process-one []
- (let [transaction (.take tx-report-queue)
- _ (println "processing transaction")
- db (:db-after transaction)
- affected-entities (->> (:tx-data transaction)
- (map (fn [^datomic.db.Datum x]
- {:e (:e x)
- :a (d/ident db (:a x))
- :v (:v x)
- :added (:added x)}))
- (group-by :e)
- (mapcat #(datums->impacted-entity db %))
- (set))
- _ (println "processing transaction affected" (count affected-entities))
- d-txs (->> affected-entities
- (map #(entity-change->ledger db %))
- (filter seq))
- retractions (map (fn [[_ e]] [:db/retractEntity [:journal-entry/original-entity e]]) affected-entities)]
+ (lc/with-context {:source "process-txes"}
+ (try
+ (let [transaction (.take tx-report-queue)
+ _ (log/info "Converting tranasction to ledger")
+ db (:db-after transaction)
+ affected-entities (->> (:tx-data transaction)
+ (map (fn [^datomic.db.Datum x]
+ {:e (:e x)
+ :a (d/ident db (:a x))
+ :v (:v x)
+ :added (:added x)}))
+ (group-by :e)
+ (mapcat #(datums->impacted-entity db %))
+ (set))
+ _ (info-event (str "Found " (count affected-entities) " affected entities")
+ {:affected-entities (count affected-entities)})
+ d-txs (->> affected-entities
+ (map #(entity-change->ledger db %))
+ (filter seq))
+ retractions (map (fn [[_ e]] [:db/retractEntity [:journal-entry/original-entity e]]) affected-entities)]
- (when (seq retractions)
- @(d/transact conn retractions))
+ (when (seq retractions)
+ @(d/transact conn retractions))
- (doseq [d-tx d-txs]
- @(d/transact conn [d-tx]))))
+ (doseq [d-tx d-txs]
+ @(d/transact conn [d-tx]))
+ (log/info "Succesfully process transaction"))
+ (catch Exception e
+ (log/error e)))))
(mount/defstate process-txes-worker
:start (scheduler/run-fun process-one 1)
:stop (-> process-txes-worker :running? (reset! false)))
(defn reconcile-ledger []
- (try
- (println "Attempting to reconcile the ledger")
- (let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
- :in ['$]
- :where ['[?t :transaction/date]
- '(not [?t :transaction/approval-status :transaction-approval-status/excluded])
- '(not-join [?t] [?e :journal-entry/original-entity ?t])]}
- :args [(d/db conn)]})
- (map first)
- (mapv #(entity-change->ledger (d/db conn) [:transaction %])))
+ (lc/with-context {:source "reconcile-ledger"}
+ (try
+ (log/info "Attempting to reconcile the ledger")
+ (let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
+ :in ['$]
+ :where ['[?t :transaction/date]
+ '(not [?t :transaction/approval-status :transaction-approval-status/excluded])
+ '(not-join [?t] [?e :journal-entry/original-entity ?t])]}
+ :args [(d/db conn)]})
+ (map first)
+ (mapv #(entity-change->ledger (d/db conn) [:transaction %])))
- invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
- :in ['$]
- :where ['[?t :invoice/date]
- '(not [?t :invoice/status :invoice-status/voided])
- '(not [?t :invoice/import-status :import-status/pending])
- '(not [?t :invoice/exclude-from-ledger true])
- '(not-join [?t] [?e :journal-entry/original-entity ?t])]}
- :args [(d/db conn)]})
- (map first)
- (mapv #(entity-change->ledger (d/db conn) [:invoice %])))
- repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
-
- (when (seq repairs)
- (println "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
+ invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
+ :in ['$]
+ :where ['[?t :invoice/date]
+ '(not [?t :invoice/status :invoice-status/voided])
+ '(not [?t :invoice/import-status :import-status/pending])
+ '(not [?t :invoice/exclude-from-ledger true])
+ '(not-join [?t] [?e :journal-entry/original-entity ?t])]}
+ :args [(d/db conn)]})
+ (map first)
+ (mapv #(entity-change->ledger (d/db conn) [:invoice %])))
+ repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
- @(d/transact conn repairs)))
- (catch Exception e
- (println e))))
+ (when (seq repairs)
+ (log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
+
+
+ @(d/transact conn repairs))
+ (log/info "Finished reconciling ledger"))
+ (catch Exception e
+ (log/error e)))))
(mount/defstate reconciliation-frequency :start 60000)
diff --git a/src/clj/auto_ap/logging.clj b/src/clj/auto_ap/logging.clj
new file mode 100644
index 00000000..c429ae7e
--- /dev/null
+++ b/src/clj/auto_ap/logging.clj
@@ -0,0 +1,15 @@
+(ns auto-ap.logging
+ (:require [clojure.tools.logging :as log]
+ [unilog.context :as lc]))
+(defn info-event [message context]
+ (lc/with-context context
+ (log/info message)))
+
+(defn warn-event [message context]
+ (lc/with-context context
+ (log/warn message)))
+
+
+(defn error-event [message context]
+ (lc/with-context context
+ (log/warn message)))
diff --git a/src/clj/auto_ap/parse.clj b/src/clj/auto_ap/parse.clj
index 83e67cd6..ea47257f 100644
--- a/src/clj/auto_ap/parse.clj
+++ b/src/clj/auto_ap/parse.clj
@@ -1,15 +1,16 @@
(ns auto-ap.parse
- (:require [auto-ap.parse.excel :as excel]
+ (:require [auto-ap.parse.csv :as csv]
+ [auto-ap.parse.excel :as excel]
[auto-ap.parse.templates :as t]
[auto-ap.parse.util :as u]
- [auto-ap.parse.csv :as csv]
[clj-fuzzy.metrics :as m]
- [clojure.java.shell :as sh]
- [clojure.string :as str]
- [clj-time.format :as f]
[clj-time.core :as time]
- [clojure.set :as set]))
-
+ [clj-time.format :as f]
+ [clojure.java.shell :as sh]
+ [clojure.set :as set]
+ [clojure.string :as str]
+ [clojure.tools.logging :as log]
+ [auto-ap.logging :refer [info-event]]))
(def last-text (atom nil))
@@ -19,7 +20,8 @@
(defn extract-template
([text template]
- (println "template" template)
+ (log/info "Template was determined to be" template)
+
(if (:multi template)
(mapcat
#(extract-template % text (dissoc template :multi))
@@ -39,16 +41,15 @@
str/trim )
[value-parser parser-params] (-> template :parser k)]
(assoc result k (try
- (println "applying parser" value-parser "to value" value)
(u/parse-value value-parser parser-params value)
(catch Exception e
- (println e))))))
+ (log/warn e))))))
{:vendor-code (:vendor template)
:text text
:full-text full-text}))])))
(defn parse [text]
- (println "Parsing PDF " text)
+ (info-event "Parsing pdf text" {:raw text})
(reset! last-text text)
(->> t/pdf-templates
(filter (partial template-applies? text))
diff --git a/src/clj/auto_ap/parse/csv.clj b/src/clj/auto_ap/parse/csv.clj
index 7771d0c2..6c1adc36 100644
--- a/src/clj/auto_ap/parse/csv.clj
+++ b/src/clj/auto_ap/parse/csv.clj
@@ -2,29 +2,29 @@
(:require [auto-ap.parse.util :as u]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
- [clojure.string :as str]))
+ [clojure.string :as str]
+ [clojure.tools.logging :as log]))
(defn determine
[[header :as z]]
- (prn header)
- (doto (cond (str/includes? (second header) "Customer's PO No.")
- :mama-lus
+ (let [csv-type (cond (str/includes? (second header) "Customer's PO No.")
+ :mama-lus
- (str/includes? (str header) "Ship-To Number")
- :sysco-style-2
+ (str/includes? (str header) "Ship-To Number")
+ :sysco-style-2
- (str/includes? (str header) "Closed Date")
- :sysco-style-1
+ (str/includes? (str header) "Closed Date")
+ :sysco-style-1
- (str/includes? (str header) "Business Unit")
- :mission
+ (str/includes? (str header) "Business Unit")
+ :mission
- (str/includes? (str header) "Document Number")
- :philz
+ (str/includes? (str header) "Document Number")
+ :philz
- :else
- nil)
- println))
+ :else
+ nil)]
+ (log/info "csv type was determined to be" csv-type)))
(defmulti parse-csv
determine
@@ -44,7 +44,6 @@
(defmethod parse-csv :sysco-style-1
[rows]
- (println "Importing Sysco-styled 1")
(let [header (first rows)]
(transduce
(comp (drop 1)
@@ -68,7 +67,7 @@
(defmethod parse-csv :sysco-style-2
[rows]
- (println "Importing Sysco-styled 1")
+
(let [header (first rows)]
(transduce
(comp (drop 1)
@@ -125,9 +124,7 @@
(comp
(filter (fn [[dt _ doc-number name _ status _ _ amount :as row]]
(= status "Billed")))
-
(map (fn [[dt _ doc-number name _ _ _ _ amount :as row]]
- (print name)
{:vendor-code "PHILZ COFFEE, INC"
:customer-identifier name
:invoice-number doc-number
diff --git a/src/clj/auto_ap/parse/util.clj b/src/clj/auto_ap/parse/util.clj
index 154bd86d..617aff8a 100644
--- a/src/clj/auto_ap/parse/util.clj
+++ b/src/clj/auto_ap/parse/util.clj
@@ -3,7 +3,8 @@
[clojure.java.shell :as sh]
[clojure.string :as str]
[clj-time.format :as f]
- [clj-time.core :as time]))
+ [clj-time.core :as time]
+ [clojure.tools.logging :as log]))
(defmulti parse-value (fn [method _ _]
method))
@@ -39,7 +40,7 @@
(reduced (time/from-time-zone (f/parse (f/formatter format) value)
(time/time-zone-for-id "America/Los_Angeles")))
(catch Exception e
- (println (.getMessage e))
+ (log/warn e)
nil)))
nil
format)
diff --git a/src/clj/auto_ap/routes/auth.clj b/src/clj/auto_ap/routes/auth.clj
index d6e9dca1..3281863c 100644
--- a/src/clj/auto_ap/routes/auth.clj
+++ b/src/clj/auto_ap/routes/auth.clj
@@ -4,7 +4,8 @@
[clj-http.client :as http]
[clj-time.core :as time]
[compojure.core :refer [GET defroutes]]
- [config.core :refer [env]]))
+ [config.core :refer [env]]
+ [clojure.tools.logging :as log]))
(def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com")
(def google-client-secret "OC-WemHurPXYpuIw5cT-B90g")
@@ -19,7 +20,6 @@
(defroutes routes
(GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers}
- (println "Authenticating with" r "..." code)
(try
(let [auth (-> "https://accounts.google.com/o/oauth2/token"
(http/post
@@ -30,37 +30,34 @@
"grant_type" "authorization_code"}
:as :json})
:body)
- _ (println auth)
token (:access_token auth)
profile (-> (http/get "https://www.googleapis.com/oauth2/v1/userinfo"
{:headers {"Authorization" (str "Bearer " token)} :as :json})
- :body
- (doto println))
+ :body)
user (users/find-or-insert! {:user/provider "google"
:user/provider-id (:id profile)
:user/role :user-role/none
:user/name (:name profile)})
]
- (println "authenticated as user" user)
+ (log/info "authenticated as user" user)
;; TODO - these namespaces are not being transmitted/deserialized properly
(if (and token user)
- (let [jwt (jwt/sign (doto {:user (:name profile)
- :exp (time/plus (time/now) (time/days 30))
- :user/clients (map (fn [c]
- (dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address))
- (:user/clients user))
- :user/role (name (:user/role user))
- :user/name (:name profile)}
- println)
+ (let [jwt (jwt/sign {:user (:name profile)
+ :exp (time/plus (time/now) (time/days 30))
+ :user/clients (map (fn [c]
+ (dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address))
+ (:user/clients user))
+ :user/role (name (:user/role user))
+ :user/name (:name profile)}
(:jwt-secret env)
{:alg :hs512})]
- (println "authenticated. using jwt" jwt)
+
{:status 301
:headers {"Location" (str "/?jwt=" jwt)}})
{:status 401
:body "Couldn't authenticate"}))
(catch Exception e
-
+ (log/warn e )
{:status 401
:body (str "Couldn't authenticate " (.toString e))}))))
diff --git a/src/clj/auto_ap/routes/events.clj b/src/clj/auto_ap/routes/events.clj
index c53dde0e..ad6ca5be 100644
--- a/src/clj/auto_ap/routes/events.clj
+++ b/src/clj/auto_ap/routes/events.clj
@@ -9,24 +9,27 @@
[clj-time.predicates :as pred]
[clojure.data.json :as json]
[compojure.core :refer [GET PUT POST context defroutes
- wrap-routes]])
+ wrap-routes]]
+ [clojure.tools.logging :as log]
+ [unilog.context :as lc])
(:import (org.joda.time DateTime)))
(defroutes routes
(context "/events" []
(POST "/yodlee-import" {:keys [query-params headers body] :as x}
- (let [notification-type (get headers "x-amz-sns-message-type")]
- (println "Received notification " notification-type)
- (if (= "SubscriptionConfirmation" notification-type)
- (do
- (println "Responding to confirmation" )
- (let [json (json/read-str (slurp body))]
- (println json)
- (http/get (get json "SubscribeURL"))))
- (do
- (println "importing from yodlee")
- (yodlee-import/do-import))))
+ (lc/with-context {:source "Import yodlee transactions"}
+ (let [notification-type (get headers "x-amz-sns-message-type")]
+ (log/info "Received notification " notification-type)
+ (if (= "SubscriptionConfirmation" notification-type)
+ (do
+ (log/info "Responding to confirmation" )
+ (let [json (json/read-str (slurp body))]
+ (log/info json)
+ (http/get (get json "SubscribeURL"))))
+ (do
+ (log/info "importing from yodlee")
+ (yodlee-import/do-import)))))
{:status 200
:body "{}"
diff --git a/src/clj/auto_ap/routes/graphql.clj b/src/clj/auto_ap/routes/graphql.clj
index 0589a65b..900f8bca 100644
--- a/src/clj/auto_ap/routes/graphql.clj
+++ b/src/clj/auto_ap/routes/graphql.clj
@@ -1,10 +1,12 @@
(ns auto-ap.routes.graphql
(:require [auto-ap.routes.utils :refer [wrap-secure wrap-spec]]
[auto-ap.graphql :as ql]
+ [auto-ap.logging :refer [warn-event]]
[buddy.auth :refer [throw-unauthorized]]
[clojure.edn :as edn]
[compojure.core :refer [GET POST PUT context defroutes
- wrap-routes]]))
+ wrap-routes]]
+ [clojure.tools.logging :as log]))
(defn handle-graphql [{:keys [request-method query-params body edn-params method] :as r}]
(when (= "none" (:user/role (:identity r)))
(throw-unauthorized))
@@ -14,19 +16,23 @@
edn/read-string)
body (some-> r :body slurp)]
{:status 200
- :body (pr-str (ql/query (:identity r) (doto (if (= request-method :get) (query-params "query") body) println) variables ))
+ :body (pr-str (ql/query (:identity r) (if (= request-method :get) (query-params "query") body) variables ))
:headers {"Content-Type" "application/edn"}})
(catch Throwable e
(if-let [result (:result (ex-data e))]
- {:status 400
- :body (pr-str result)
- :headers {"Content-Type" "application/edn"}}
- (if-let [message (:validation-error (ex-data e) )]
+ (do (log/warn "Graphql Result error" e)
{:status 400
- :body (pr-str {:errors [(merge {:message message} (ex-data e))]})
- :headers {"Content-Type" "application/edn"}}
- (do (println e)
+ :body (pr-str result)
+ :headers {"Content-Type" "application/edn"}})
+ (if-let [message (:validation-error (ex-data e) )]
+ (do
+ (warn-event "GraphQL Validation error" {:message message
+ :data e})
+ {:status 400
+ :body (pr-str {:errors [(merge {:message message} (ex-data e))]})
+ :headers {"Content-Type" "application/edn"}})
+ (do (log/error "GraphQL error" e)
{:status 500
:body (pr-str {:errors [(merge {:message (.getMessage e)} (ex-data e))]})
:headers {"Content-Type" "application/edn"}}))))))
diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj
index 1c3297bb..74427f5b 100644
--- a/src/clj/auto_ap/routes/invoices.clj
+++ b/src/clj/auto_ap/routes/invoices.clj
@@ -19,7 +19,10 @@
wrap-routes]]
[clojure.string :as str]
[clojure.java.io :as io]
- [clojure.data.csv :as csv]))
+ [clojure.data.csv :as csv]
+ [unilog.context :as lc]
+ [clojure.tools.logging :as log]
+ [auto-ap.logging :refer [info-event]]))
(defn reset-id [i]
(update i :invoice-number
@@ -203,88 +206,91 @@
{:vendor-code vendor-code})))))
(defn import-uploaded-invoice [client forced-location forced-vendor imports]
- (let [clients (d-clients/get-all)
- transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}]
- (println "searching for" vendor-code)
- (let [ _ (println "matching" customer-identifier)
- matching-client (or (and account-number
- (parse/best-match clients account-number 0.0))
- (and customer-identifier
- (parse/best-match clients customer-identifier))
- (if client
- (first (filter (fn [c]
- (= (:db/id c) (Long/parseLong client)))
- clients))))
- _ (when-not matching-client
- (throw (ex-info (str "Searched clients for '" customer-identifier "'. No client found in file. Select a client first.")
- {:invoice-number invoice-number
- :customer-identifier customer-identifier
- :vendor-code vendor-code})))
-
- matching-vendor (match-vendor vendor-code forced-vendor)
-
-
- _ (println "invoice \"" invoice-number "\" matches client " (:client/name matching-client) " (" (:db/id matching-client) ")")
- matching-location (or (when-not (str/blank? forced-location)
- forced-location)
- (parse/best-location-match matching-client text full-text))
- [existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
- (try
- (->> (d/query
- (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
- :in ['$ '?invoice-number '?vendor '?client]
- :where '[[?e :invoice/invoice-number ?invoice-number]
- [?e :invoice/vendor ?vendor]
- [?e :invoice/client ?client]
- [?e :invoice/outstanding-balance ?outstanding-balance]
- [?e :invoice/status ?status]
- [?e :invoice/import-status ?import-status]
- [?import-status :db/ident ?import-status2]]}
- :args [(d/db (d/connect uri)) invoice-number (:db/id matching-vendor) (:db/id matching-client)]}))
- first)
- (catch Exception e
- (throw (ex-info (str "Failed to find potential matching invoice with"
- " invoice " invoice-number
- " vendor " matching-vendor
- " client " (:client/name matching-client)
- ". "
- (.toString e))
- {:args [ invoice-number matching-vendor (:db/id matching-client)]})))
- ))
- automatically-paid-for (set (map :db/id (:vendor/automatically-paid-when-due matching-vendor)))]
+ (lc/with-context {:area "upload-invoice"}
+ (let [clients (d-clients/get-all)
+ transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}]
- (cond
- (not (and matching-location matching-client))
- result
+ (let [
+ matching-client (or (and account-number
+ (parse/best-match clients account-number 0.0))
+ (and customer-identifier
+ (parse/best-match clients customer-identifier))
+ (if client
+ (first (filter (fn [c]
+ (= (:db/id c) (Long/parseLong client)))
+ clients))))
+ _ (when-not matching-client
+ (throw (ex-info (str "Searched clients for '" customer-identifier "'. No client found in file. Select a client first.")
+ {:invoice-number invoice-number
+ :customer-identifier customer-identifier
+ :vendor-code vendor-code})))
- (= :import-status/imported import-status)
- result
+ matching-vendor (match-vendor vendor-code forced-vendor)
- :else
- (conj result (cond-> (remove-nils #:invoice {:invoice/client (:db/id matching-client)
- :invoice/client-identifier customer-identifier
- :invoice/vendor (:db/id matching-vendor)
- :invoice/invoice-number invoice-number
- :invoice/automatically-paid-when-due (boolean (automatically-paid-for (:db/id matching-client)))
- :invoice/total (Double/parseDouble total)
- :invoice/date (to-date date)
- :invoice/import-status :import-status/pending
- :invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total))
- :invoice/status (or existing-status :invoice-status/unpaid)
- :invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:account (d-vendors/account-for-client-id matching-vendor (:db/id matching-client))
- :location matching-location
- :amount (Double/parseDouble total)}])
- :db/id existing-id
- })
- (:vendor/terms matching-vendor) (assoc :invoice/due (coerce/to-date
- (time/plus date (time/days (d-vendors/terms-for-client-id matching-vendor (:db/id matching-client)))))))))
- ))
- []
- imports)]
- (when-not (seq transactions)
- (throw (ex-info "No invoices found."
- {:imports (str imports)})))
- @(d/transact (d/connect uri) (vec (set transactions)))))
+
+ _ (info-event "Found match for invoice" {:invoice-number invoice-number
+ :vendor (select-keys matching-vendor [:vendor/name :db/id])
+ :client (select-keys matching-client [:client/name :db/id])})
+ matching-location (or (when-not (str/blank? forced-location)
+ forced-location)
+ (parse/best-location-match matching-client text full-text))
+ [existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
+ (try
+ (->> (d/query
+ (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
+ :in ['$ '?invoice-number '?vendor '?client]
+ :where '[[?e :invoice/invoice-number ?invoice-number]
+ [?e :invoice/vendor ?vendor]
+ [?e :invoice/client ?client]
+ [?e :invoice/outstanding-balance ?outstanding-balance]
+ [?e :invoice/status ?status]
+ [?e :invoice/import-status ?import-status]
+ [?import-status :db/ident ?import-status2]]}
+ :args [(d/db (d/connect uri)) invoice-number (:db/id matching-vendor) (:db/id matching-client)]}))
+ first)
+ (catch Exception e
+ (throw (ex-info (str "Failed to find potential matching invoice with"
+ " invoice " invoice-number
+ " vendor " matching-vendor
+ " client " (:client/name matching-client)
+ ". "
+ (.toString e))
+ {:args [ invoice-number matching-vendor (:db/id matching-client)]})))
+ ))
+ automatically-paid-for (set (map :db/id (:vendor/automatically-paid-when-due matching-vendor)))]
+
+ (cond
+ (not (and matching-location matching-client))
+ result
+
+ (= :import-status/imported import-status)
+ result
+
+ :else
+ (conj result (cond-> (remove-nils #:invoice {:invoice/client (:db/id matching-client)
+ :invoice/client-identifier customer-identifier
+ :invoice/vendor (:db/id matching-vendor)
+ :invoice/invoice-number invoice-number
+ :invoice/automatically-paid-when-due (boolean (automatically-paid-for (:db/id matching-client)))
+ :invoice/total (Double/parseDouble total)
+ :invoice/date (to-date date)
+ :invoice/import-status :import-status/pending
+ :invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total))
+ :invoice/status (or existing-status :invoice-status/unpaid)
+ :invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:account (d-vendors/account-for-client-id matching-vendor (:db/id matching-client))
+ :location matching-location
+ :amount (Double/parseDouble total)}])
+ :db/id existing-id
+ })
+ (:vendor/terms matching-vendor) (assoc :invoice/due (coerce/to-date
+ (time/plus date (time/days (d-vendors/terms-for-client-id matching-vendor (:db/id matching-client)))))))))
+ ))
+ []
+ imports)]
+ (when-not (seq transactions)
+ (throw (ex-info "No invoices found."
+ {:imports (str imports)})))
+ @(d/transact (d/connect uri) (vec (set transactions))))))
(defn validate-account-rows [rows code->existing-account]
@@ -318,7 +324,6 @@
:in ['$ '?z]
:where [['?e :client/code '?z]]}
:args [(d/db (d/connect uri)) customer]})))
- _ (println client-id)
headers (map read-string header)
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
{:account/applicability [:db/ident]}
@@ -427,7 +432,7 @@
(map (parse-or-error :amount parse-amount))
(map (parse-or-error :date parse-date)))
error-rows (filter :errors rows)
- _ (println "importing raw transactions" rows)
+ _ (log/info "Importing " (count rows) "raw transactions")
raw-transactions (vec (->> rows
(filter #(not (seq (:errors %))) )
(map (fn [{:keys [description-original client-code status high-level-category amount bank-account-code date]}]
@@ -445,7 +450,7 @@
:errors (map #(dissoc % :date) error-rows)})
:headers {"Content-Type" "application/edn"}})
(catch Exception e
- (println e)
+ (log/error e)
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
@@ -469,18 +474,18 @@
vendor (some-> (or vendor vendor-2)
(Long/parseLong))
{:keys [filename tempfile]} files]
- #_(println params (.getPath tempfile) filename)
- (try
- (import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename))
- {:status 200
- :body (pr-str {})
- :headers {"Content-Type" "application/edn"}}
- (catch Exception e
- {:status 500
- :body (pr-str {:message (.getMessage e)
- :error (.toString e)
- :data (ex-data e)})
- :headers {"Content-Type" "application/edn"}}))))
+ (lc/with-context {:parsing-file filename}
+ (try
+ (import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename))
+ {:status 200
+ :body (pr-str {})
+ :headers {"Content-Type" "application/edn"}}
+ (catch Exception e
+ {:status 500
+ :body (pr-str {:message (.getMessage e)
+ :error (.toString e)
+ :data (ex-data e)})
+ :headers {"Content-Type" "application/edn"}})))))
(POST "/upload-integreat"
{{:keys [excel-rows]} :edn-params user :identity}
@@ -515,7 +520,6 @@
files-2 "file"} :params :as params
user :identity}
(let [files (or files files-2)
- _ (println files)
{:keys [filename tempfile]} files]
(assert-admin user)
(try
@@ -524,7 +528,7 @@
:body (pr-str {})
:headers {"Content-Type" "application/edn"}}
(catch Exception e
- (println e)
+ (log/error e)
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
@@ -545,7 +549,7 @@
:body (import-account-overrides client (.getPath tempfile))
:headers {"Content-Type" "application/json"}}
(catch Exception e
- (println e)
+ (log/error e)
{:status 500
:body {:message (.getMessage e)
:data (ex-data e)}
diff --git a/src/clj/auto_ap/routes/yodlee.clj b/src/clj/auto_ap/routes/yodlee.clj
index 857c1ece..ba3c3e96 100644
--- a/src/clj/auto_ap/routes/yodlee.clj
+++ b/src/clj/auto_ap/routes/yodlee.clj
@@ -1,17 +1,18 @@
(ns auto-ap.routes.yodlee
(:require
- [auto-ap.graphql :as graphql]
- [clj-http.client :as http]
+ [auto-ap.graphql :as graphql]
+ [clj-http.client :as http]
- [auto-ap.yodlee.core :as yodlee]
- [auto-ap.graphql.utils :refer [->graphql assert-admin]]
- [auto-ap.routes.utils :refer [wrap-secure]]
- [clj-time.coerce :refer [to-date]]
- [ring.middleware.json :refer [wrap-json-response]]
- [compojure.core :refer [GET POST context defroutes wrap-routes]]
- [clojure.string :as str]
- [config.core :refer [env]]
- ))
+ [auto-ap.yodlee.core :as yodlee]
+ [auto-ap.graphql.utils :refer [->graphql assert-admin]]
+ [auto-ap.routes.utils :refer [wrap-secure]]
+ [clj-time.coerce :refer [to-date]]
+ [ring.middleware.json :refer [wrap-json-response]]
+ [compojure.core :refer [GET POST context defroutes wrap-routes]]
+ [clojure.string :as str]
+ [config.core :refer [env]]
+
+ [clojure.tools.logging :as log]))
(defroutes routes
(wrap-routes
@@ -53,7 +54,7 @@
:headers {"Content-Type" "application/edn"}
:body (pr-str (yodlee/reauthenticate (Long/parseLong id) data)) })
(catch Exception e
- (println e)
+ (log/error e)
{:status 500
:headers {"Content-Type" "application/edn"}
:body (pr-str {:message (.getMessage e)
diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj
index 6a34e384..12bec0db 100644
--- a/src/clj/auto_ap/server.clj
+++ b/src/clj/auto_ap/server.clj
@@ -7,14 +7,19 @@
[nrepl.server :refer [start-server stop-server]]
[config.core :refer [env]]
[ring.adapter.jetty :refer [run-jetty]]
+ [clojure.tools.logging :as log]
+ [unilog.config]
[mount.core :as mount])
(:gen-class))
+
+
+(mount/defstate port :start (Integer/parseInt (or (env :port) "3000")))
+(mount/defstate jetty
+ :start (run-jetty app {:port port :join? false})
+ :stop (.stop jetty)
+ )
+
(defn -main [& args]
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
- (let [port (Integer/parseInt (or (env :port) "3000"))]
-
-
- (mount/start)
- #_(future (always-process-sqs))
- (run-jetty app {:port port :join? false})))
+ (mount/start))
diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj
index 814009bb..0fa63c0a 100644
--- a/src/clj/auto_ap/yodlee/core.clj
+++ b/src/clj/auto_ap/yodlee/core.clj
@@ -2,7 +2,10 @@
(:require [clj-http.client :as client]
[auto-ap.utils :refer [by]]
[cemerick.url :as u]
+ [unilog.context :as lc]
+ [clojure.tools.logging :as log]
[clojure.data.json :as json]
+ [clojure.core.async :as async]
[config.core :refer [env]]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
@@ -92,9 +95,7 @@
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip)
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -114,9 +115,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts")
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount
@@ -131,9 +130,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount)))
@@ -145,9 +142,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id )
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:query-params {"include" "credentials,questions,preferences"}
:as :json})
:body
@@ -161,9 +156,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
- (client/put {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/put {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json}))))
@@ -179,9 +172,7 @@
get-transaction-batch (fn [skip]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -199,11 +190,8 @@
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
- (doto println)
- (client/get {:headers (doto
- (merge base-headers {"Authorization" (auth-header cob-session user-session)})
- println)
+ (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -236,34 +224,55 @@
:as :json})
:body)))
+
+
(defn get-provider-accounts-with-details []
(let [provider-accounts (get-provider-accounts)]
- (reduce
- (fn [pas pa]
- (conj pas (try (get-provider-account-detail (:id pa))
- (catch Exception e
- pa))))
- []
- provider-accounts)))
+ (let [concurrent 20
+ output-chan (async/chan)]
+ (async/pipeline-blocking concurrent
+ output-chan
+ (map (fn [provider-account]
+ (lc/with-context {:provider-account-id (:id provider-account)}
+ (log/info "fetching details for provider" (:id provider-account))
+ (get-provider-account-detail (:id provider-account)))))
+ (async/to-chan provider-accounts))
+ (async/> accounts
(reduce
- (fn [provider-accounts a]
- (update-in provider-accounts [(:providerAccountId a) :accounts] conj a)) provider-accounts)
+ (fn [provider-accounts [which accounts]]
+ (assoc-in provider-accounts [which :accounts] accounts))
+ provider-accounts)
vals)))
(mount/defstate in-memory-cache
- :start (doto (atom (get-provider-accounts-with-accounts)) println))
+ :start (atom []))
(defn refresh-in-memory-cache []
- (try
- (println "Refreshing Yodlee in memory cache")
- (reset! in-memory-cache (get-provider-accounts-with-accounts))
- (catch Exception e
- (println e))))
+ (lc/with-context {:source "refreshing-in-memory-cache"}
+ (try
+ (log/info "Refreshing Yodlee in memory cache")
+ (reset! in-memory-cache (get-provider-accounts-with-accounts))
+
+ (catch Exception e
+ (log/error e)))))
(mount/defstate in-memory-cache-worker
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)
diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj
index d37f4759..3b9d1706 100644
--- a/src/clj/auto_ap/yodlee/import.clj
+++ b/src/clj/auto_ap/yodlee/import.clj
@@ -12,7 +12,9 @@
[auto-ap.time :as time]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm]
- [clojure.string :as str]))
+ [clojure.string :as str]
+ [unilog.context :as lc]
+ [clojure.tools.logging :as log]))
@@ -149,29 +151,31 @@
:client/_bank-accounts client))))))))
(defn manual-import [manual-transactions]
- (let [transformed-transactions (->> manual-transactions
- (filter #(= "posted" (:status %)))
- (group-by #(select-keys % [:date :description-original :amount]))
- (vals)
- (mapcat (fn [transaction-group]
- (map
- (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
- {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
- :bank-account-id bank-account-id
- :date (time/unparse date "YYYY-MM-dd")
- :amount {:amount amount}
- :description {:original description-original
- :simple high-level-category}
- :status "POSTED"})
- (range)
- transaction-group))))
- all-rules (tr/get-all)
- all-bank-accounts (by :db/id (get-all-bank-accounts))
- transaction->bank-account (comp all-bank-accounts :bank-account-id)]
- (println "importing manual transactions" transformed-transactions)
- (let [result (batch-transact
- (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
- (println "imported " (count result)))))
+ (lc/with-context {:source "manual import"}
+ (let [transformed-transactions (->> manual-transactions
+ (filter #(= "posted" (:status %)))
+ (group-by #(select-keys % [:date :description-original :amount]))
+ (vals)
+ (mapcat (fn [transaction-group]
+ (map
+ (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
+ {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
+ :bank-account-id bank-account-id
+ :date (time/unparse date "YYYY-MM-dd")
+ :amount {:amount amount}
+ :description {:original description-original
+ :simple high-level-category}
+ :status "POSTED"})
+ (range)
+ transaction-group))))
+ all-rules (tr/get-all)
+ all-bank-accounts (by :db/id (get-all-bank-accounts))
+ transaction->bank-account (comp all-bank-accounts :bank-account-id)]
+ (log/info "Importing " (count transformed-transactions) " manual transactions")
+
+ (let [result (batch-transact
+ (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
+ (log/info "Imported " (count result) " manual transactions")))))
(defn do-import
([]
diff --git a/src/clj/user.clj b/src/clj/user.clj
index 6acd73d3..94322540 100644
--- a/src/clj/user.clj
+++ b/src/clj/user.clj
@@ -3,6 +3,8 @@
[auto-ap.utils :refer [by]]
#_[auto-ap.ledger :as l]
+ [mount.core :as mount]
+ [auto-ap.server ]
[datomic.api :as d]
[clojure.data.csv :as csv]
[clj-time.coerce :as c]
@@ -360,3 +362,8 @@
'(not-join [?i] [?e :journal-entry/original-entity ?i])]}
:args [(d/db (d/connect uri))]})])
+
+(defn go []
+ (require '[mount.core :as mount])
+ (require '[auto-ap.server])
+ (mount/start-without #'auto-ap.server/jetty))
diff --git a/src/cljs/auto_ap/views/pages/ledger.cljs b/src/cljs/auto_ap/views/pages/ledger.cljs
index 81a508ce..daddbc23 100644
--- a/src/cljs/auto_ap/views/pages/ledger.cljs
+++ b/src/cljs/auto_ap/views/pages/ledger.cljs
@@ -52,7 +52,6 @@
::params-change
[with-user (re-frame/inject-cofx ::inject/sub [::params])]
(fn [{:keys [user ::params db]} [_]]
- (println "execute")
{:db (-> db
(assoc-in [::last-params] params)
(assoc-in [:status :loading] true))