notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

revamped logging!

Browse Source
This commit is contained in:
Bryce Covert
2020-08-01 17:07:17 -07:00
parent 49f98522c0
commit 51b097766f
33 changed files with 598 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -4,4 +4,4 @@ RUN apk add poppler
RUN apk add poppler-utils RUN apk add poppler-utils
COPY target/auto-ap.jar /usr/local/ COPY target/auto-ap.jar /usr/local/
COPY config /usr/local/config/ COPY config /usr/local/config/
CMD java -jar /usr/local/auto-ap.jar CMD java -Dlogback.configurationFile=logback-prod.xml -jar /usr/local/auto-ap.jar

8
project.clj
View File

@@ -45,6 +45,11 @@
[buddy/buddy-auth "2.1.0"] [buddy/buddy-auth "2.1.0"]
[buddy/buddy-sign "2.1.0"] [buddy/buddy-sign "2.1.0"]
[nrepl "0.6.0" :exclusions [org.clojure/tools.logging]] [nrepl "0.6.0" :exclusions [org.clojure/tools.logging]]
[org.clojure/tools.logging "1.1.0"]
[ch.qos.logback/logback-classic "1.2.3"]
[ch.qos.logback.contrib/logback-jackson "0.1.5"]
[ch.qos.logback.contrib/logback-json-classic "0.1.5"]
[spootnik/unilog "0.7.24"]
[clj-time "0.14.3"] [clj-time "0.14.3"]
[io.forward/clojure-mail "1.0.7"] [io.forward/clojure-mail "1.0.7"]
[ring/ring-json "0.4.0" :exclusions [cheshire]] [ring/ring-json "0.4.0" :exclusions [cheshire]]
@@ -82,6 +87,7 @@
{:resource-paths ["resources" "target"] {:resource-paths ["resources" "target"]
:dependencies [[binaryage/devtools "0.9.4"] :dependencies [[binaryage/devtools "0.9.4"]
#_[refactor-nrepl "2.5.0"]
[com.bhauman/figwheel-main "0.2.3" :exclusions [org.clojure/clojurescript [com.bhauman/figwheel-main "0.2.3" :exclusions [org.clojure/clojurescript
ring/ring-core ring/ring-core
ring/ring-codec ring/ring-codec
@@ -134,7 +140,7 @@
:main auto-ap.server :main auto-ap.server
:aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format] :aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format clojure.tools.logging.impl]
:uberjar-name "auto-ap.jar" :uberjar-name "auto-ap.jar"

14
resources/logback-prod.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
<jsonFormatter class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter"/>
<appendLineSeparator>true</appendLineSeparator>
</layout>
</encoder>
</appender>
<root level="info">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

14
resources/logback.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %highlight(%-5level) %logger{36} - %green(%msg) %gray(%mdc) %n</pattern>
</encoder>
</appender>
<logger level="info" name="auto-ap" additivity="false">
<appender-ref ref="CONSOLE" />
</logger>
<root level="error">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

14
resources/logback_.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
<jsonFormatter class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter"/>
<appendLineSeparator>true</appendLineSeparator>
</layout>
</encoder>
</appender>
<root level="debug">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

11
src/clj/auto_ap/background/invoices.clj
View File

@@ -5,9 +5,12 @@
[auto-ap.time :as time] [auto-ap.time :as time]
[clj-time.coerce :as coerce] [clj-time.coerce :as coerce]
[mount.core :as mount] [mount.core :as mount]
[yang.scheduler :as scheduler])) [yang.scheduler :as scheduler]
[unilog.context :as lc]
[clojure.tools.logging :as log]))
(defn close-auto-invoices [] (defn close-auto-invoices []
(lc/with-context {:source "close-auto-invoices"}
(try (try
(let [invoices-to-close (d/query {:query {:find ['?e] (let [invoices-to-close (d/query {:query {:find ['?e]
@@ -17,7 +20,7 @@
'[?e :invoice/due ?d] '[?e :invoice/due ?d]
'[(<= ?d ?today)]]} '[(<= ?d ?today)]]}
:args [(d/db conn) (coerce/to-date (time/local-now))]})] :args [(d/db conn) (coerce/to-date (time/local-now))]})]
(println "Closing " (count invoices-to-close) "automatic invoices") (log/info "Closing " (count invoices-to-close) "automatic invoices")
(some->> invoices-to-close (some->> invoices-to-close
seq seq
@@ -26,9 +29,9 @@
:invoice/status :invoice-status/paid})) :invoice/status :invoice-status/paid}))
(d/transact conn) (d/transact conn)
deref) deref)
(println "Closed " (count invoices-to-close) "automatic invoices")) (log/info "Closed " (count invoices-to-close) "automatic invoices"))
(catch Exception e (catch Exception e
(println (.toString e))))) (log/error e)))))
(mount/defstate close-auto-invoices-worker (mount/defstate close-auto-invoices-worker
:start (scheduler/every 60000 close-auto-invoices) :start (scheduler/every 60000 close-auto-invoices)

6
src/clj/auto_ap/datomic/invoices.clj
View File

@@ -156,8 +156,7 @@
[(->> (graphql-results ids-to-retrieve db args)) [(->> (graphql-results ids-to-retrieve db args))
matching-count matching-count
(doto outstanding]))
outstanding println)]))
(defn get-by-id [id] (defn get-by-id [id]
(-> (d/db (d/connect uri)) (-> (d/db (d/connect uri))
@@ -186,8 +185,7 @@
[(not= ?e ?invoice-id)] [(not= ?e ?invoice-id)]
]} ]}
:args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]} :args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}))
true (doto println)))
(map first) (map first)
(map <-datomic))) (map <-datomic)))

3
src/clj/auto_ap/datomic/transactions.clj
View File

@@ -115,8 +115,7 @@
"status" ['[?e :transaction/status ?sort-status]]} "status" ['[?e :transaction/status ?sort-status]]}
args) args)
true true
(merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}})) (merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}}))]
_ (println query)]
(cond->> query (cond->> query
true (d/query) true (d/query)
true (apply-sort-3 args) true (apply-sort-3 args)

1
src/clj/auto_ap/datomic/users.clj
View File

@@ -41,7 +41,6 @@
first first
first first
(update :user/role :db/ident))] (update :user/role :db/ident))]
(println "USER" user)
(if user (if user
user user
(let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user (let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user

30
src/clj/auto_ap/graphql.clj
View File

@@ -7,6 +7,7 @@
[com.walmartlabs.lacinia.resolve :as resolve] [com.walmartlabs.lacinia.resolve :as resolve]
[buddy.auth :refer [throw-unauthorized]] [buddy.auth :refer [throw-unauthorized]]
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
[auto-ap.logging :refer [info-event warn-event error-event]]
[auto-ap.graphql.utils :refer [assert-admin can-see-client? assert-can-see-client]] [auto-ap.graphql.utils :refer [assert-admin can-see-client? assert-can-see-client]]
[auto-ap.datomic :refer [uri merge-query]] [auto-ap.datomic :refer [uri merge-query]]
[datomic.api :as d] [datomic.api :as d]
@@ -29,7 +30,10 @@
[auto-ap.graphql.transaction-rules :as gq-transaction-rules] [auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.time :as time] [auto-ap.time :as time]
[clojure.walk :as walk] [clojure.walk :as walk]
[clojure.string :as str]) [clojure.string :as str]
[clojure.tools.logging :as log]
[yang.time :refer [time-it]]
[unilog.context :as lc])
(:import (:import
(clojure.lang IPersistentMap))) (clojure.lang IPersistentMap)))
@@ -945,7 +949,6 @@
(filter (fn [[potential-id potential-date]] (filter (fn [[potential-id potential-date]]
(let [date (coerce/to-date-time date) (let [date (coerce/to-date-time date)
potential-date (coerce/to-date-time potential-date)] potential-date (coerce/to-date-time potential-date)]
#_(println "HERE" id potential-id potential-date date)
(and (= id potential-id) (and (= id potential-id)
(<= (t/in-days (apply t/interval (sort [date potential-date]))) 10))))) (<= (t/in-days (apply t/interval (sort [date potential-date]))) 10)))))
conj conj
@@ -1010,7 +1013,6 @@
:amount amount :amount amount
:date (coerce/to-date-time next)}) :date (coerce/to-date-time next)})
is-week-a? (fn [d] is-week-a? (fn [d]
(println d)
(= 0 (mod (t/in-weeks (t/interval first-week-a d)) 2)))] (= 0 (mod (t/in-weeks (t/interval first-week-a d)) 2)))]
{:beginning_balance total-cash {:beginning_balance total-cash
@@ -1110,20 +1112,24 @@
([id q] ([id q]
(query id q nil )) (query id q nil ))
([id q v] ([id q v]
(println "executing graphql query" id q v)
(lc/with-context {:query q}
(log/info "Executing query" q)
(try (try
(let [result (time (simplify (execute schema q v {:id id})))] (let [[result time] (time-it (simplify (execute schema q v {:id id})))]
(info-event "Query completed"
{:time (:time time)
:errors (seq (:errors result))})
(when (seq (:errors result)) (when (seq (:errors result))
(throw (ex-info "GraphQL error" {:result result}))) (throw (ex-info "GraphQL error" {:result result})))
result) result)
(catch Exception e (catch Exception e
(if-let [v (:validation-error (ex-data e))] (if-let [v (:validation-error (ex-data e))]
(println "validation error" v) (warn-event "validation error" {:validation-error v
(do :data (ex-data e)})
(.printStackTrace e) (error-event "query error" {:error e}))
(println e )))
(throw e))))) (throw e))))))
#_(query nil "{ ledger_page { count }}" nil)

3
src/clj/auto_ap/graphql/clients.clj
View File

@@ -19,8 +19,6 @@
(let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client))) (let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client)))
id (or (:db/id client) "new-client") id (or (:db/id client) "new-client")
_ (println id)
_ (println edit_client)
_ (when client _ (when client
@(d/transact (d/connect uri) @(d/transact (d/connect uri)
(into (into
@@ -79,7 +77,6 @@
) )
(:forecasted_transactions edit_client))]] (:forecasted_transactions edit_client))]]
result @(d/transact (d/connect uri) transactions)] result @(d/transact (d/connect uri) transactions)]
(println result "ID" id)
(-> result :tempids (get id) (or id) d-clients/get-by-id (-> result :tempids (get id) (or id) d-clients/get-by-id
(update :client/location-matches (update :client/location-matches
(fn [lms] (fn [lms]

7
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -58,7 +58,6 @@
:location location})) :location location}))
(defn add-invoice-transaction [{:keys [total invoice_number location automatically_paid_when_due client_id vendor_id vendor_name date due expense_accounts] :as in}] (defn add-invoice-transaction [{:keys [total invoice_number location automatically_paid_when_due client_id vendor_id vendor_name date due expense_accounts] :as in}]
(println date)
(let [vendor (d-vendors/get-by-id vendor_id) (let [vendor (d-vendors/get-by-id vendor_id)
account (:vendor/default-account vendor) account (:vendor/default-account vendor)
_ (when-not (:db/id account) _ (when-not (:db/id account)
@@ -119,11 +118,10 @@
(defn edit-invoice [context {{:keys [id due invoice_number total vendor_id date client_id expense_accounts automatically_paid_when_due] :as in} :invoice} value] (defn edit-invoice [context {{:keys [id due invoice_number total vendor_id date client_id expense_accounts automatically_paid_when_due] :as in} :invoice} value]
(let [invoice (d-invoices/get-by-id id) (let [invoice (d-invoices/get-by-id id)
_ (when (seq (doto (d-invoices/find-conflicting {:db/id id _ (when (seq (d-invoices/find-conflicting {:db/id id
:invoice/invoice-number invoice_number :invoice/invoice-number invoice_number
:invoice/vendor (:db/id (:invoice/vendor invoice)) :invoice/vendor (:db/id (:invoice/vendor invoice))
:invoice/client (:db/id (:invoice/client invoice))}) :invoice/client (:db/id (:invoice/client invoice))}))
println))
(throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number}))) (throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number})))
expense-account-total (reduce + 0 (map (fn [x] (Double/parseDouble (:amount x))) expense_accounts)) expense-account-total (reduce + 0 (map (fn [x] (Double/parseDouble (:amount x))) expense_accounts))
@@ -151,7 +149,6 @@
(defn void-invoice [context {id :invoice_id} value] (defn void-invoice [context {id :invoice_id} value]
(let [invoice (d-invoices/get-by-id id) (let [invoice (d-invoices/get-by-id id)
_ (println invoice)
_ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice))) _ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
updated-invoice (d-invoices/update {:db/id id updated-invoice (d-invoices/update {:db/id id
:invoice/total 0.0 :invoice/total 0.0

21
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -12,7 +12,9 @@
[auto-ap.parse.util :as parse] [auto-ap.parse.util :as parse]
[datomic.api :as d] [datomic.api :as d]
[auto-ap.parse.templates :as t] [auto-ap.parse.templates :as t]
[auto-ap.datomic.clients :as d-clients])) [auto-ap.datomic.clients :as d-clients]
[clojure.tools.logging :as log]
[unilog.context :as lc]))
(defn get-ledger-page [context args value] (defn get-ledger-page [context args value]
(let [args (assoc args :id (:id context)) (let [args (assoc args :id (:id context))
@@ -106,15 +108,12 @@
vals vals
(mapcat (fn [a] (mapcat (fn [a]
(map (fn [o] (map (fn [o]
#_(println "override" (:db/id a) (:db/id (:account-client-override/client o)))
#_(print [[(:db/id a) (:db/id (:account-client-override/client o))]])
[[(:db/id a) (:db/id (:account-client-override/client o))] [[(:db/id a) (:db/id (:account-client-override/client o))]
(:account-client-override/name o)]) (:account-client-override/name o)])
(:account/client-overrides a)) (:account/client-overrides a))
) ) ) )
(into {} ))] (into {} ))]
(fn [a] (fn [a]
#_(println a client-id (keys overrides-by-client))
{:name (or (:bank-account/name (bank-accounts a)) {:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id]) (overrides-by-client [a client-id])
(:account/name (accounts a))) (:account/name (accounts a)))
@@ -158,7 +157,6 @@
(defn get-profit-and-loss [context args value] (defn get-profit-and-loss [context args value]
(let [client-id (:client_id args) (let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id) _ (assert-can-see-client (:id context) client-id)
_ (println args)
all-ledger-entries (full-ledger-for-client client-id) all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)] lookup-account (build-account-lookup client-id)]
@@ -179,6 +177,7 @@
(defn import-ledger [context args value] (defn import-ledger [context args value]
(assert-admin (:id context)) (assert-admin (:id context))
(lc/with-context {:area "import ledger"}
(let [all-vendors (by :vendor/name (d-vendors/get-graphql {})) (let [all-vendors (by :vendor/name (d-vendors/get-graphql {}))
all-clients (by :client/code (d-clients/get-all )) all-clients (by :client/code (d-clients/get-all ))
all-client-bank-accounts (reduce all-client-bank-accounts (reduce
@@ -276,11 +275,15 @@
success (filter (comp not :error) transaction) success (filter (comp not :error) transaction)
retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]]) retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]])
success)] success)]
(println (take 4 success)) (log/info "manual ledger import has " (count success) " new rows")
(run! (fn [batch] (println "transacting retraction batch") @(d/transact (d/connect uri) batch)) (partition-all 100 retraction)) (run! (fn [batch]
(run! (fn [batch] (println "transacting success batch") @(d/transact (d/connect uri) batch)) (partition-all 100 success)) (log/info "transacting retraction batch")
@(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
(run! (fn [batch]
(log/info "transacting success batch")
@(d/transact (d/connect uri) batch)) (partition-all 100 success))
{:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success) {:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
:existing [] :existing []
:errors (map (fn [x] {:external_id (:external_id x) :errors (map (fn [x] {:external_id (:external_id x)
:error (:error x)}) errors)})) :error (:error x)}) errors)})))

4
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -18,10 +18,6 @@
[auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm] [auto-ap.rule-matching :as rm]
[clj-time.coerce :as coerce])) [clj-time.coerce :as coerce]))
(defn prn-each [xs]
(doseq [x xs]
(println x))
xs)
(def approval-status->graphql (ident->enum-f :transaction/approval-status)) (def approval-status->graphql (ident->enum-f :transaction/approval-status))

6
src/clj/auto_ap/graphql/users.clj
View File

@@ -10,7 +10,6 @@
":user" :user-role/user}) ":user" :user-role/user})
(defn edit-user [context {:keys [edit_user] :as args} value] (defn edit-user [context {:keys [edit_user] :as args} value]
(println args edit_user)
(assert-admin (:id context)) (assert-admin (:id context))
(let [user (d-users/get-by-id (:id edit_user)) (let [user (d-users/get-by-id (:id edit_user))
new-clients (set (map #(Long/parseLong %) (:clients edit_user))) new-clients (set (map #(Long/parseLong %) (:clients edit_user)))
@@ -21,11 +20,10 @@
@(d/transact (d/connect uri) @(d/transact (d/connect uri)
(doto (-> [{:db/id (:db/id user) (-> [{:db/id (:db/id user)
:user/role (role->datomic-role (:role edit_user)) :user/role (role->datomic-role (:role edit_user))
:user/clients new-clients}] :user/clients new-clients}]
(into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove))) (into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove))))
clojure.pprint/pprint))
(->graphql (->graphql
(d-users/get-by-id (:id edit_user))))) (d-users/get-by-id (:id edit_user)))))

10
src/clj/auto_ap/graphql/utils.clj
View File

@@ -1,7 +1,8 @@
(ns auto-ap.graphql.utils (ns auto-ap.graphql.utils
(:require [clojure.string :as str] (:require [clojure.string :as str]
[buddy.auth :refer [throw-unauthorized]] [buddy.auth :refer [throw-unauthorized]]
[clojure.walk :as walk])) [clojure.walk :as walk]
[clojure.tools.logging :as log]))
(defn snake->kebab [s] (defn snake->kebab [s]
@@ -43,24 +44,23 @@
m)) m))
(defn is-admin? [id] (defn is-admin? [id]
(println "role" id)
(= "admin" (:user/role id))) (= "admin" (:user/role id)))
(defn assert-admin [id] (defn assert-admin [id]
(println "role" id)
(when-not (= "admin" (:user/role id)) (when-not (= "admin" (:user/role id))
(log/warn "user " id " not an admin!")
(throw-unauthorized))) (throw-unauthorized)))
(defn can-see-client? [identity client] (defn can-see-client? [identity client]
(when (not client) (when (not client)
(println "WARNING - permission checking for null client")) (log/warn "WARNING - permission checking for null client"))
(or (= "admin" (:user/role identity)) (or (= "admin" (:user/role identity))
((set (map :db/id (:user/clients identity))) (:db/id client)) ((set (map :db/id (:user/clients identity))) (:db/id client))
((set (map :db/id (:user/clients identity))) client))) ((set (map :db/id (:user/clients identity))) client)))
(defn assert-can-see-client [identity client] (defn assert-can-see-client [identity client]
(when-not (can-see-client? identity client) (when-not (can-see-client? identity client)
(println "IDENTITY " identity " can not see company " client) (log/warn "IDENTITY " identity " can not see company " client)
(throw-unauthorized))) (throw-unauthorized)))
(defn limited-clients [id] (defn limited-clients [id]

8
src/clj/auto_ap/graphql/vendors.clj
View File

@@ -72,19 +72,15 @@
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/account-overrides account-overrides]) (is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/account-overrides account-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/terms-overrides terms-overrides]) (is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/terms-overrides terms-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/automatically-paid-when-due (is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/automatically-paid-when-due
(doto (mapv (mapv
(fn [apwd] (fn [apwd]
{:db/id apwd}) {:db/id apwd})
(:automatically_paid_when_due in)) (:automatically_paid_when_due in))]))
println)]))
_ (println transaction)
transaction-result @(d/transact (d/connect uri) transaction)] transaction-result @(d/transact (d/connect uri) transaction)]
(-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor")) (-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
id)) id))
(doto println)
(->graphql)))) (->graphql))))
(defn merge-vendors [context {:keys [from to]} value] (defn merge-vendors [context {:keys [from to]} value]

12
src/clj/auto_ap/handler.clj
View File

@@ -16,6 +16,8 @@
[compojure.core :refer :all] [compojure.core :refer :all]
[compojure.route :as route] [compojure.route :as route]
[config.core :refer [env]] [config.core :refer [env]]
[clojure.tools.logging :as log]
[unilog.context :as lc]
[ring.middleware.edn :refer [wrap-edn-params]] [ring.middleware.edn :refer [wrap-edn-params]]
[ring.middleware.multipart-params :as mp] [ring.middleware.multipart-params :as mp]
[ring.middleware.params :refer [wrap-params]] [ring.middleware.params :refer [wrap-params]]
@@ -55,10 +57,20 @@
static-routes)) static-routes))
(defn wrap-logging [handler]
(fn [request]
(lc/with-context {:uri (:uri request)
:source "request"
:user-role (:user/role (:identity request))
:user-name (:user/name (:identity request))}
(log/info "Beginning request" (:uri request))
(handler request))))
(def app (def app
(-> #'app-routes (-> #'app-routes
(wrap-logging)
(wrap-authorization auth-backend) (wrap-authorization auth-backend)
(wrap-authentication auth-backend) (wrap-authentication auth-backend)
(wrap-reload) (wrap-reload)

28
src/clj/auto_ap/ledger.clj
View File

@@ -4,7 +4,10 @@
[mount.core :as mount] [mount.core :as mount]
[auto-ap.datomic.accounts :as a] [auto-ap.datomic.accounts :as a]
[auto-ap.datomic :refer [uri remove-nils]] [auto-ap.datomic :refer [uri remove-nils]]
[clojure.spec.alpha :as s])) [clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]
[unilog.context :as lc]))
(defn datums->impacted-entity [db [e changes]] (defn datums->impacted-entity [db [e changes]]
@@ -129,8 +132,10 @@
(defn process-one [] (defn process-one []
(lc/with-context {:source "process-txes"}
(try
(let [transaction (.take tx-report-queue) (let [transaction (.take tx-report-queue)
_ (println "processing transaction") _ (log/info "Converting tranasction to ledger")
db (:db-after transaction) db (:db-after transaction)
affected-entities (->> (:tx-data transaction) affected-entities (->> (:tx-data transaction)
(map (fn [^datomic.db.Datum x] (map (fn [^datomic.db.Datum x]
@@ -141,7 +146,8 @@
(group-by :e) (group-by :e)
(mapcat #(datums->impacted-entity db %)) (mapcat #(datums->impacted-entity db %))
(set)) (set))
_ (println "processing transaction affected" (count affected-entities)) _ (info-event (str "Found " (count affected-entities) " affected entities")
{:affected-entities (count affected-entities)})
d-txs (->> affected-entities d-txs (->> affected-entities
(map #(entity-change->ledger db %)) (map #(entity-change->ledger db %))
(filter seq)) (filter seq))
@@ -151,15 +157,19 @@
@(d/transact conn retractions)) @(d/transact conn retractions))
(doseq [d-tx d-txs] (doseq [d-tx d-txs]
@(d/transact conn [d-tx])))) @(d/transact conn [d-tx]))
(log/info "Succesfully process transaction"))
(catch Exception e
(log/error e)))))
(mount/defstate process-txes-worker (mount/defstate process-txes-worker
:start (scheduler/run-fun process-one 1) :start (scheduler/run-fun process-one 1)
:stop (-> process-txes-worker :running? (reset! false))) :stop (-> process-txes-worker :running? (reset! false)))
(defn reconcile-ledger [] (defn reconcile-ledger []
(lc/with-context {:source "reconcile-ledger"}
(try (try
(println "Attempting to reconcile the ledger") (log/info "Attempting to reconcile the ledger")
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ] (let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$] :in ['$]
:where ['[?t :transaction/date] :where ['[?t :transaction/date]
@@ -169,6 +179,7 @@
(map first) (map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %]))) (mapv #(entity-change->ledger (d/db conn) [:transaction %])))
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ] invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$] :in ['$]
:where ['[?t :invoice/date] :where ['[?t :invoice/date]
@@ -183,12 +194,13 @@
(when (seq repairs) (when (seq repairs)
(println "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries") (log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs))) @(d/transact conn repairs))
(log/info "Finished reconciling ledger"))
(catch Exception e (catch Exception e
(println e)))) (log/error e)))))
(mount/defstate reconciliation-frequency :start 60000) (mount/defstate reconciliation-frequency :start 60000)

15
src/clj/auto_ap/logging.clj Normal file
View File

@@ -0,0 +1,15 @@
(ns auto-ap.logging
(:require [clojure.tools.logging :as log]
[unilog.context :as lc]))
(defn info-event [message context]
(lc/with-context context
(log/info message)))
(defn warn-event [message context]
(lc/with-context context
(log/warn message)))
(defn error-event [message context]
(lc/with-context context
(log/warn message)))

23
src/clj/auto_ap/parse.clj
View File

@@ -1,15 +1,16 @@
(ns auto-ap.parse (ns auto-ap.parse
(:require [auto-ap.parse.excel :as excel] (:require [auto-ap.parse.csv :as csv]
[auto-ap.parse.excel :as excel]
[auto-ap.parse.templates :as t] [auto-ap.parse.templates :as t]
[auto-ap.parse.util :as u] [auto-ap.parse.util :as u]
[auto-ap.parse.csv :as csv]
[clj-fuzzy.metrics :as m] [clj-fuzzy.metrics :as m]
[clojure.java.shell :as sh]
[clojure.string :as str]
[clj-time.format :as f]
[clj-time.core :as time] [clj-time.core :as time]
[clojure.set :as set])) [clj-time.format :as f]
[clojure.java.shell :as sh]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]))
(def last-text (atom nil)) (def last-text (atom nil))
@@ -19,7 +20,8 @@
(defn extract-template (defn extract-template
([text template] ([text template]
(println "template" template) (log/info "Template was determined to be" template)
(if (:multi template) (if (:multi template)
(mapcat (mapcat
#(extract-template % text (dissoc template :multi)) #(extract-template % text (dissoc template :multi))
@@ -39,16 +41,15 @@
str/trim ) str/trim )
[value-parser parser-params] (-> template :parser k)] [value-parser parser-params] (-> template :parser k)]
(assoc result k (try (assoc result k (try
(println "applying parser" value-parser "to value" value)
(u/parse-value value-parser parser-params value) (u/parse-value value-parser parser-params value)
(catch Exception e (catch Exception e
(println e)))))) (log/warn e))))))
{:vendor-code (:vendor template) {:vendor-code (:vendor template)
:text text :text text
:full-text full-text}))]))) :full-text full-text}))])))
(defn parse [text] (defn parse [text]
(println "Parsing PDF " text) (info-event "Parsing pdf text" {:raw text})
(reset! last-text text) (reset! last-text text)
(->> t/pdf-templates (->> t/pdf-templates
(filter (partial template-applies? text)) (filter (partial template-applies? text))

15
src/clj/auto_ap/parse/csv.clj
View File

@@ -2,12 +2,12 @@
(:require [auto-ap.parse.util :as u] (:require [auto-ap.parse.util :as u]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.string :as str])) [clojure.string :as str]
[clojure.tools.logging :as log]))
(defn determine (defn determine
[[header :as z]] [[header :as z]]
(prn header) (let [csv-type (cond (str/includes? (second header) "Customer's PO No.")
(doto (cond (str/includes? (second header) "Customer's PO No.")
:mama-lus :mama-lus
(str/includes? (str header) "Ship-To Number") (str/includes? (str header) "Ship-To Number")
@@ -23,8 +23,8 @@
:philz :philz
:else :else
nil) nil)]
println)) (log/info "csv type was determined to be" csv-type)))
(defmulti parse-csv (defmulti parse-csv
determine determine
@@ -44,7 +44,6 @@
(defmethod parse-csv :sysco-style-1 (defmethod parse-csv :sysco-style-1
[rows] [rows]
(println "Importing Sysco-styled 1")
(let [header (first rows)] (let [header (first rows)]
(transduce (transduce
(comp (drop 1) (comp (drop 1)
@@ -68,7 +67,7 @@
(defmethod parse-csv :sysco-style-2 (defmethod parse-csv :sysco-style-2
[rows] [rows]
(println "Importing Sysco-styled 1")
(let [header (first rows)] (let [header (first rows)]
(transduce (transduce
(comp (drop 1) (comp (drop 1)
@@ -125,9 +124,7 @@
(comp (comp
(filter (fn [[dt _ doc-number name _ status _ _ amount :as row]] (filter (fn [[dt _ doc-number name _ status _ _ amount :as row]]
(= status "Billed"))) (= status "Billed")))
(map (fn [[dt _ doc-number name _ _ _ _ amount :as row]] (map (fn [[dt _ doc-number name _ _ _ _ amount :as row]]
(print name)
{:vendor-code "PHILZ COFFEE, INC" {:vendor-code "PHILZ COFFEE, INC"
:customer-identifier name :customer-identifier name
:invoice-number doc-number :invoice-number doc-number

5
src/clj/auto_ap/parse/util.clj
View File

@@ -3,7 +3,8 @@
[clojure.java.shell :as sh] [clojure.java.shell :as sh]
[clojure.string :as str] [clojure.string :as str]
[clj-time.format :as f] [clj-time.format :as f]
[clj-time.core :as time])) [clj-time.core :as time]
[clojure.tools.logging :as log]))
(defmulti parse-value (fn [method _ _] (defmulti parse-value (fn [method _ _]
method)) method))
@@ -39,7 +40,7 @@
(reduced (time/from-time-zone (f/parse (f/formatter format) value) (reduced (time/from-time-zone (f/parse (f/formatter format) value)
(time/time-zone-for-id "America/Los_Angeles"))) (time/time-zone-for-id "America/Los_Angeles")))
(catch Exception e (catch Exception e
(println (.getMessage e)) (log/warn e)
nil))) nil)))
nil nil
format) format)

17
src/clj/auto_ap/routes/auth.clj
View File

@@ -4,7 +4,8 @@
[clj-http.client :as http] [clj-http.client :as http]
[clj-time.core :as time] [clj-time.core :as time]
[compojure.core :refer [GET defroutes]] [compojure.core :refer [GET defroutes]]
[config.core :refer [env]])) [config.core :refer [env]]
[clojure.tools.logging :as log]))
(def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com") (def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com")
(def google-client-secret "OC-WemHurPXYpuIw5cT-B90g") (def google-client-secret "OC-WemHurPXYpuIw5cT-B90g")
@@ -19,7 +20,6 @@
(defroutes routes (defroutes routes
(GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers} (GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers}
(println "Authenticating with" r "..." code)
(try (try
(let [auth (-> "https://accounts.google.com/o/oauth2/token" (let [auth (-> "https://accounts.google.com/o/oauth2/token"
(http/post (http/post
@@ -30,37 +30,34 @@
"grant_type" "authorization_code"} "grant_type" "authorization_code"}
:as :json}) :as :json})
:body) :body)
_ (println auth)
token (:access_token auth) token (:access_token auth)
profile (-> (http/get "https://www.googleapis.com/oauth2/v1/userinfo" profile (-> (http/get "https://www.googleapis.com/oauth2/v1/userinfo"
{:headers {"Authorization" (str "Bearer " token)} :as :json}) {:headers {"Authorization" (str "Bearer " token)} :as :json})
:body :body)
(doto println))
user (users/find-or-insert! {:user/provider "google" user (users/find-or-insert! {:user/provider "google"
:user/provider-id (:id profile) :user/provider-id (:id profile)
:user/role :user-role/none :user/role :user-role/none
:user/name (:name profile)}) :user/name (:name profile)})
] ]
(println "authenticated as user" user) (log/info "authenticated as user" user)
;; TODO - these namespaces are not being transmitted/deserialized properly ;; TODO - these namespaces are not being transmitted/deserialized properly
(if (and token user) (if (and token user)
(let [jwt (jwt/sign (doto {:user (:name profile) (let [jwt (jwt/sign {:user (:name profile)
:exp (time/plus (time/now) (time/days 30)) :exp (time/plus (time/now) (time/days 30))
:user/clients (map (fn [c] :user/clients (map (fn [c]
(dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address)) (dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address))
(:user/clients user)) (:user/clients user))
:user/role (name (:user/role user)) :user/role (name (:user/role user))
:user/name (:name profile)} :user/name (:name profile)}
println)
(:jwt-secret env) (:jwt-secret env)
{:alg :hs512})] {:alg :hs512})]
(println "authenticated. using jwt" jwt)
{:status 301 {:status 301
:headers {"Location" (str "/?jwt=" jwt)}}) :headers {"Location" (str "/?jwt=" jwt)}})
{:status 401 {:status 401
:body "Couldn't authenticate"})) :body "Couldn't authenticate"}))
(catch Exception e (catch Exception e
(log/warn e )
{:status 401 {:status 401
:body (str "Couldn't authenticate " (.toString e))})))) :body (str "Couldn't authenticate " (.toString e))}))))

15
src/clj/auto_ap/routes/events.clj
View File

@@ -9,24 +9,27 @@
[clj-time.predicates :as pred] [clj-time.predicates :as pred]
[clojure.data.json :as json] [clojure.data.json :as json]
[compojure.core :refer [GET PUT POST context defroutes [compojure.core :refer [GET PUT POST context defroutes
wrap-routes]]) wrap-routes]]
[clojure.tools.logging :as log]
[unilog.context :as lc])
(:import (org.joda.time DateTime))) (:import (org.joda.time DateTime)))
(defroutes routes (defroutes routes
(context "/events" [] (context "/events" []
(POST "/yodlee-import" {:keys [query-params headers body] :as x} (POST "/yodlee-import" {:keys [query-params headers body] :as x}
(lc/with-context {:source "Import yodlee transactions"}
(let [notification-type (get headers "x-amz-sns-message-type")] (let [notification-type (get headers "x-amz-sns-message-type")]
(println "Received notification " notification-type) (log/info "Received notification " notification-type)
(if (= "SubscriptionConfirmation" notification-type) (if (= "SubscriptionConfirmation" notification-type)
(do (do
(println "Responding to confirmation" ) (log/info "Responding to confirmation" )
(let [json (json/read-str (slurp body))] (let [json (json/read-str (slurp body))]
(println json) (log/info json)
(http/get (get json "SubscribeURL")))) (http/get (get json "SubscribeURL"))))
(do (do
(println "importing from yodlee") (log/info "importing from yodlee")
(yodlee-import/do-import)))) (yodlee-import/do-import)))))
{:status 200 {:status 200
:body "{}" :body "{}"

16
src/clj/auto_ap/routes/graphql.clj
View File

@@ -1,10 +1,12 @@
(ns auto-ap.routes.graphql (ns auto-ap.routes.graphql
(:require [auto-ap.routes.utils :refer [wrap-secure wrap-spec]] (:require [auto-ap.routes.utils :refer [wrap-secure wrap-spec]]
[auto-ap.graphql :as ql] [auto-ap.graphql :as ql]
[auto-ap.logging :refer [warn-event]]
[buddy.auth :refer [throw-unauthorized]] [buddy.auth :refer [throw-unauthorized]]
[clojure.edn :as edn] [clojure.edn :as edn]
[compojure.core :refer [GET POST PUT context defroutes [compojure.core :refer [GET POST PUT context defroutes
wrap-routes]])) wrap-routes]]
[clojure.tools.logging :as log]))
(defn handle-graphql [{:keys [request-method query-params body edn-params method] :as r}] (defn handle-graphql [{:keys [request-method query-params body edn-params method] :as r}]
(when (= "none" (:user/role (:identity r))) (when (= "none" (:user/role (:identity r)))
(throw-unauthorized)) (throw-unauthorized))
@@ -14,19 +16,23 @@
edn/read-string) edn/read-string)
body (some-> r :body slurp)] body (some-> r :body slurp)]
{:status 200 {:status 200
:body (pr-str (ql/query (:identity r) (doto (if (= request-method :get) (query-params "query") body) println) variables )) :body (pr-str (ql/query (:identity r) (if (= request-method :get) (query-params "query") body) variables ))
:headers {"Content-Type" "application/edn"}}) :headers {"Content-Type" "application/edn"}})
(catch Throwable e (catch Throwable e
(if-let [result (:result (ex-data e))] (if-let [result (:result (ex-data e))]
(do (log/warn "Graphql Result error" e)
{:status 400 {:status 400
:body (pr-str result) :body (pr-str result)
:headers {"Content-Type" "application/edn"}} :headers {"Content-Type" "application/edn"}})
(if-let [message (:validation-error (ex-data e) )] (if-let [message (:validation-error (ex-data e) )]
(do
(warn-event "GraphQL Validation error" {:message message
:data e})
{:status 400 {:status 400
:body (pr-str {:errors [(merge {:message message} (ex-data e))]}) :body (pr-str {:errors [(merge {:message message} (ex-data e))]})
:headers {"Content-Type" "application/edn"}} :headers {"Content-Type" "application/edn"}})
(do (println e) (do (log/error "GraphQL error" e)
{:status 500 {:status 500
:body (pr-str {:errors [(merge {:message (.getMessage e)} (ex-data e))]}) :body (pr-str {:errors [(merge {:message (.getMessage e)} (ex-data e))]})
:headers {"Content-Type" "application/edn"}})))))) :headers {"Content-Type" "application/edn"}}))))))

30
src/clj/auto_ap/routes/invoices.clj
View File

@@ -19,7 +19,10 @@
wrap-routes]] wrap-routes]]
[clojure.string :as str] [clojure.string :as str]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.data.csv :as csv])) [clojure.data.csv :as csv]
[unilog.context :as lc]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]))
(defn reset-id [i] (defn reset-id [i]
(update i :invoice-number (update i :invoice-number
@@ -203,10 +206,11 @@
{:vendor-code vendor-code}))))) {:vendor-code vendor-code})))))
(defn import-uploaded-invoice [client forced-location forced-vendor imports] (defn import-uploaded-invoice [client forced-location forced-vendor imports]
(lc/with-context {:area "upload-invoice"}
(let [clients (d-clients/get-all) (let [clients (d-clients/get-all)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}] transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}]
(println "searching for" vendor-code)
(let [ _ (println "matching" customer-identifier) (let [
matching-client (or (and account-number matching-client (or (and account-number
(parse/best-match clients account-number 0.0)) (parse/best-match clients account-number 0.0))
(and customer-identifier (and customer-identifier
@@ -224,7 +228,9 @@
matching-vendor (match-vendor vendor-code forced-vendor) matching-vendor (match-vendor vendor-code forced-vendor)
_ (println "invoice \"" invoice-number "\" matches client " (:client/name matching-client) " (" (:db/id matching-client) ")") _ (info-event "Found match for invoice" {:invoice-number invoice-number
:vendor (select-keys matching-vendor [:vendor/name :db/id])
:client (select-keys matching-client [:client/name :db/id])})
matching-location (or (when-not (str/blank? forced-location) matching-location (or (when-not (str/blank? forced-location)
forced-location) forced-location)
(parse/best-location-match matching-client text full-text)) (parse/best-location-match matching-client text full-text))
@@ -284,7 +290,7 @@
(when-not (seq transactions) (when-not (seq transactions)
(throw (ex-info "No invoices found." (throw (ex-info "No invoices found."
{:imports (str imports)}))) {:imports (str imports)})))
@(d/transact (d/connect uri) (vec (set transactions))))) @(d/transact (d/connect uri) (vec (set transactions))))))
(defn validate-account-rows [rows code->existing-account] (defn validate-account-rows [rows code->existing-account]
@@ -318,7 +324,6 @@
:in ['$ '?z] :in ['$ '?z]
:where [['?e :client/code '?z]]} :where [['?e :client/code '?z]]}
:args [(d/db (d/connect uri)) customer]}))) :args [(d/db (d/connect uri)) customer]})))
_ (println client-id)
headers (map read-string header) headers (map read-string header)
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
{:account/applicability [:db/ident]} {:account/applicability [:db/ident]}
@@ -427,7 +432,7 @@
(map (parse-or-error :amount parse-amount)) (map (parse-or-error :amount parse-amount))
(map (parse-or-error :date parse-date))) (map (parse-or-error :date parse-date)))
error-rows (filter :errors rows) error-rows (filter :errors rows)
_ (println "importing raw transactions" rows) _ (log/info "Importing " (count rows) "raw transactions")
raw-transactions (vec (->> rows raw-transactions (vec (->> rows
(filter #(not (seq (:errors %))) ) (filter #(not (seq (:errors %))) )
(map (fn [{:keys [description-original client-code status high-level-category amount bank-account-code date]}] (map (fn [{:keys [description-original client-code status high-level-category amount bank-account-code date]}]
@@ -445,7 +450,7 @@
:errors (map #(dissoc % :date) error-rows)}) :errors (map #(dissoc % :date) error-rows)})
:headers {"Content-Type" "application/edn"}}) :headers {"Content-Type" "application/edn"}})
(catch Exception e (catch Exception e
(println e) (log/error e)
{:status 500 {:status 500
:body (pr-str {:message (.getMessage e) :body (pr-str {:message (.getMessage e)
:error (.toString e) :error (.toString e)
@@ -469,7 +474,7 @@
vendor (some-> (or vendor vendor-2) vendor (some-> (or vendor vendor-2)
(Long/parseLong)) (Long/parseLong))
{:keys [filename tempfile]} files] {:keys [filename tempfile]} files]
#_(println params (.getPath tempfile) filename) (lc/with-context {:parsing-file filename}
(try (try
(import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename)) (import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename))
{:status 200 {:status 200
@@ -480,7 +485,7 @@
:body (pr-str {:message (.getMessage e) :body (pr-str {:message (.getMessage e)
:error (.toString e) :error (.toString e)
:data (ex-data e)}) :data (ex-data e)})
:headers {"Content-Type" "application/edn"}})))) :headers {"Content-Type" "application/edn"}})))))
(POST "/upload-integreat" (POST "/upload-integreat"
{{:keys [excel-rows]} :edn-params user :identity} {{:keys [excel-rows]} :edn-params user :identity}
@@ -515,7 +520,6 @@
files-2 "file"} :params :as params files-2 "file"} :params :as params
user :identity} user :identity}
(let [files (or files files-2) (let [files (or files files-2)
_ (println files)
{:keys [filename tempfile]} files] {:keys [filename tempfile]} files]
(assert-admin user) (assert-admin user)
(try (try
@@ -524,7 +528,7 @@
:body (pr-str {}) :body (pr-str {})
:headers {"Content-Type" "application/edn"}} :headers {"Content-Type" "application/edn"}}
(catch Exception e (catch Exception e
(println e) (log/error e)
{:status 500 {:status 500
:body (pr-str {:message (.getMessage e) :body (pr-str {:message (.getMessage e)
:error (.toString e) :error (.toString e)
@@ -545,7 +549,7 @@
:body (import-account-overrides client (.getPath tempfile)) :body (import-account-overrides client (.getPath tempfile))
:headers {"Content-Type" "application/json"}} :headers {"Content-Type" "application/json"}}
(catch Exception e (catch Exception e
(println e) (log/error e)
{:status 500 {:status 500
:body {:message (.getMessage e) :body {:message (.getMessage e)
:data (ex-data e)} :data (ex-data e)}

5
src/clj/auto_ap/routes/yodlee.clj
View File

@@ -11,7 +11,8 @@
[compojure.core :refer [GET POST context defroutes wrap-routes]] [compojure.core :refer [GET POST context defroutes wrap-routes]]
[clojure.string :as str] [clojure.string :as str]
[config.core :refer [env]] [config.core :refer [env]]
))
[clojure.tools.logging :as log]))
(defroutes routes (defroutes routes
(wrap-routes (wrap-routes
@@ -53,7 +54,7 @@
:headers {"Content-Type" "application/edn"} :headers {"Content-Type" "application/edn"}
:body (pr-str (yodlee/reauthenticate (Long/parseLong id) data)) }) :body (pr-str (yodlee/reauthenticate (Long/parseLong id) data)) })
(catch Exception e (catch Exception e
(println e) (log/error e)
{:status 500 {:status 500
:headers {"Content-Type" "application/edn"} :headers {"Content-Type" "application/edn"}
:body (pr-str {:message (.getMessage e) :body (pr-str {:message (.getMessage e)

17
src/clj/auto_ap/server.clj
View File

@@ -7,14 +7,19 @@
[nrepl.server :refer [start-server stop-server]] [nrepl.server :refer [start-server stop-server]]
[config.core :refer [env]] [config.core :refer [env]]
[ring.adapter.jetty :refer [run-jetty]] [ring.adapter.jetty :refer [run-jetty]]
[clojure.tools.logging :as log]
[unilog.config]
[mount.core :as mount]) [mount.core :as mount])
(:gen-class)) (:gen-class))
(mount/defstate port :start (Integer/parseInt (or (env :port) "3000")))
(mount/defstate jetty
:start (run-jetty app {:port port :join? false})
:stop (.stop jetty)
)
(defn -main [& args] (defn -main [& args]
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler)) (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
(let [port (Integer/parseInt (or (env :port) "3000"))] (mount/start))
(mount/start)
#_(future (always-process-sqs))
(run-jetty app {:port port :join? false})))

79
src/clj/auto_ap/yodlee/core.clj
View File

@@ -2,7 +2,10 @@
(:require [clj-http.client :as client] (:require [clj-http.client :as client]
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
[cemerick.url :as u] [cemerick.url :as u]
[unilog.context :as lc]
[clojure.tools.logging :as log]
[clojure.data.json :as json] [clojure.data.json :as json]
[clojure.core.async :as async]
[config.core :refer [env]] [config.core :refer [env]]
[mount.core :as mount] [mount.core :as mount]
[yang.scheduler :as scheduler])) [yang.scheduler :as scheduler]))
@@ -92,9 +95,7 @@
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip) (-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip)
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json}) :as :json})
:body :body
:transaction :transaction
@@ -114,9 +115,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts") (-> (str (:yodlee-base-url env) "/providerAccounts")
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json}) :as :json})
:body :body
:providerAccount :providerAccount
@@ -131,9 +130,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id) (-> (str (:yodlee-base-url env) "/providerAccounts/" id)
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json}) :as :json})
:body :body
:providerAccount))) :providerAccount)))
@@ -145,9 +142,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id ) (-> (str (:yodlee-base-url env) "/providerAccounts/" id )
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:query-params {"include" "credentials,questions,preferences"} :query-params {"include" "credentials,questions,preferences"}
:as :json}) :as :json})
:body :body
@@ -161,9 +156,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa) (-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put {:headers (doto (client/put {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}" :body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json})))) :as :json}))))
@@ -179,9 +172,7 @@
get-transaction-batch (fn [skip] get-transaction-batch (fn [skip]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account) (-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json}) :as :json})
:body :body
:transaction :transaction
@@ -199,11 +190,8 @@
user-session (login-user cob-session)] user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account) (-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
(doto println)
(client/get {:headers (doto (client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
:as :json}) :as :json})
:body :body
:transaction :transaction
@@ -236,34 +224,55 @@
:as :json}) :as :json})
:body))) :body)))
(defn get-provider-accounts-with-details [] (defn get-provider-accounts-with-details []
(let [provider-accounts (get-provider-accounts)] (let [provider-accounts (get-provider-accounts)]
(reduce (let [concurrent 20
(fn [pas pa] output-chan (async/chan)]
(conj pas (try (get-provider-account-detail (:id pa)) (async/pipeline-blocking concurrent
(catch Exception e output-chan
pa)))) (map (fn [provider-account]
[] (lc/with-context {:provider-account-id (:id provider-account)}
provider-accounts))) (log/info "fetching details for provider" (:id provider-account))
(get-provider-account-detail (:id provider-account)))))
(async/to-chan provider-accounts))
(async/<!! (async/into [] output-chan)))))
(defn concurrent-get-accounts-for-providers [provider-account-ids]
(let [concurrent 20
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account-id]
(lc/with-context {:provider-account-id provider-account-id}
(log/info "fetching accounts for provider" provider-account-id)
[provider-account-id
(get-accounts-for-provider-account provider-account-id)])))
(async/to-chan provider-account-ids))
(async/<!! (async/into {} output-chan))))
(defn get-provider-accounts-with-accounts [] (defn get-provider-accounts-with-accounts []
(let [provider-accounts (by :id (get-provider-accounts-with-details)) (let [provider-accounts (by :id (get-provider-accounts-with-details))
accounts (get-accounts)] accounts (concurrent-get-accounts-for-providers (keys provider-accounts))]
(->> accounts (->> accounts
(reduce (reduce
(fn [provider-accounts a] (fn [provider-accounts [which accounts]]
(update-in provider-accounts [(:providerAccountId a) :accounts] conj a)) provider-accounts) (assoc-in provider-accounts [which :accounts] accounts))
provider-accounts)
vals))) vals)))
(mount/defstate in-memory-cache (mount/defstate in-memory-cache
:start (doto (atom (get-provider-accounts-with-accounts)) println)) :start (atom []))
(defn refresh-in-memory-cache [] (defn refresh-in-memory-cache []
(lc/with-context {:source "refreshing-in-memory-cache"}
(try (try
(println "Refreshing Yodlee in memory cache") (log/info "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts)) (reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e (catch Exception e
(println e)))) (log/error e)))))
(mount/defstate in-memory-cache-worker (mount/defstate in-memory-cache-worker
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache) :start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)

10
src/clj/auto_ap/yodlee/import.clj
View File

@@ -12,7 +12,9 @@
[auto-ap.time :as time] [auto-ap.time :as time]
[auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm] [auto-ap.rule-matching :as rm]
[clojure.string :as str])) [clojure.string :as str]
[unilog.context :as lc]
[clojure.tools.logging :as log]))
@@ -149,6 +151,7 @@
:client/_bank-accounts client)))))))) :client/_bank-accounts client))))))))
(defn manual-import [manual-transactions] (defn manual-import [manual-transactions]
(lc/with-context {:source "manual import"}
(let [transformed-transactions (->> manual-transactions (let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %))) (filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount])) (group-by #(select-keys % [:date :description-original :amount]))
@@ -168,10 +171,11 @@
all-rules (tr/get-all) all-rules (tr/get-all)
all-bank-accounts (by :db/id (get-all-bank-accounts)) all-bank-accounts (by :db/id (get-all-bank-accounts))
transaction->bank-account (comp all-bank-accounts :bank-account-id)] transaction->bank-account (comp all-bank-accounts :bank-account-id)]
(println "importing manual transactions" transformed-transactions) (log/info "Importing " (count transformed-transactions) " manual transactions")
(let [result (batch-transact (let [result (batch-transact
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))] (transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
(println "imported " (count result))))) (log/info "Imported " (count result) " manual transactions")))))
(defn do-import (defn do-import
([] ([]

7
src/clj/user.clj
View File

@@ -3,6 +3,8 @@
[auto-ap.utils :refer [by]] [auto-ap.utils :refer [by]]
#_[auto-ap.ledger :as l] #_[auto-ap.ledger :as l]
[mount.core :as mount]
[auto-ap.server ]
[datomic.api :as d] [datomic.api :as d]
[clojure.data.csv :as csv] [clojure.data.csv :as csv]
[clj-time.coerce :as c] [clj-time.coerce :as c]
@@ -360,3 +362,8 @@
'(not-join [?i] [?e :journal-entry/original-entity ?i])]} '(not-join [?i] [?e :journal-entry/original-entity ?i])]}
:args [(d/db (d/connect uri))]})]) :args [(d/db (d/connect uri))]})])
(defn go []
(require '[mount.core :as mount])
(require '[auto-ap.server])
(mount/start-without #'auto-ap.server/jetty))

1
src/cljs/auto_ap/views/pages/ledger.cljs
View File

@@ -52,7 +52,6 @@
::params-change ::params-change
[with-user (re-frame/inject-cofx ::inject/sub [::params])] [with-user (re-frame/inject-cofx ::inject/sub [::params])]
(fn [{:keys [user ::params db]} [_]] (fn [{:keys [user ::params db]} [_]]
(println "execute")
{:db (-> db {:db (-> db
(assoc-in [::last-params] params) (assoc-in [::last-params] params)
(assoc-in [:status :loading] true)) (assoc-in [:status :loading] true))