notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revamped logging!

Browse Source
This commit is contained in:
Bryce Covert
2020-08-01 17:07:17 -07:00
parent 49f98522c0
commit 51b097766f
33 changed files with 598 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -4,4 +4,4 @@ RUN apk add poppler
RUN apk add poppler-utils
COPY target/auto-ap.jar /usr/local/
COPY config /usr/local/config/
CMD java -jar /usr/local/auto-ap.jar
CMD java -Dlogback.configurationFile=logback-prod.xml -jar /usr/local/auto-ap.jar

8
project.clj
View File

@@ -45,6 +45,11 @@
[buddy/buddy-auth "2.1.0"]
[buddy/buddy-sign "2.1.0"]
[nrepl "0.6.0" :exclusions [org.clojure/tools.logging]]
[org.clojure/tools.logging "1.1.0"]
[ch.qos.logback/logback-classic "1.2.3"]
[ch.qos.logback.contrib/logback-jackson "0.1.5"]
[ch.qos.logback.contrib/logback-json-classic "0.1.5"]
[spootnik/unilog "0.7.24"]
[clj-time "0.14.3"]
[io.forward/clojure-mail "1.0.7"]
[ring/ring-json "0.4.0" :exclusions [cheshire]]
@@ -82,6 +87,7 @@
{:resource-paths ["resources" "target"]
:dependencies [[binaryage/devtools "0.9.4"]
#_[refactor-nrepl "2.5.0"]
[com.bhauman/figwheel-main "0.2.3" :exclusions [org.clojure/clojurescript
ring/ring-core
ring/ring-codec
@@ -134,7 +140,7 @@
:main auto-ap.server
:aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format]
:aot [auto-ap.server auto-ap.datomic.migrate auto-ap.time clj-time.core clj-time.coerce clj-time.format clojure.tools.logging.impl]
:uberjar-name "auto-ap.jar"

14
resources/logback-prod.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
<jsonFormatter class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter"/>
<appendLineSeparator>true</appendLineSeparator>
</layout>
</encoder>
</appender>
<root level="info">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

14
resources/logback.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %highlight(%-5level) %logger{36} - %green(%msg) %gray(%mdc) %n</pattern>
</encoder>
</appender>
<logger level="info" name="auto-ap" additivity="false">
<appender-ref ref="CONSOLE" />
</logger>
<root level="error">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

14
resources/logback_.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
<jsonFormatter class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter"/>
<appendLineSeparator>true</appendLineSeparator>
</layout>
</encoder>
</appender>
<root level="debug">
<appender-ref ref="CONSOLE" />
</root>
</configuration>

45
src/clj/auto_ap/background/invoices.clj
View File

@@ -5,30 +5,33 @@
[auto-ap.time :as time]
[clj-time.coerce :as coerce]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
[yang.scheduler :as scheduler]
[unilog.context :as lc]
[clojure.tools.logging :as log]))
(defn close-auto-invoices []
(try
(let [invoices-to-close (d/query {:query {:find ['?e]
:in ['$ '?today]
:where ['[?e :invoice/automatically-paid-when-due true]
'[?e :invoice/status :invoice-status/unpaid]
'[?e :invoice/due ?d]
'[(<= ?d ?today)]]}
:args [(d/db conn) (coerce/to-date (time/local-now))]})]
(println "Closing " (count invoices-to-close) "automatic invoices")
(some->> invoices-to-close
seq
(lc/with-context {:source "close-auto-invoices"}
(try
(let [invoices-to-close (d/query {:query {:find ['?e]
:in ['$ '?today]
:where ['[?e :invoice/automatically-paid-when-due true]
'[?e :invoice/status :invoice-status/unpaid]
'[?e :invoice/due ?d]
'[(<= ?d ?today)]]}
:args [(d/db conn) (coerce/to-date (time/local-now))]})]
(log/info "Closing " (count invoices-to-close) "automatic invoices")
(some->> invoices-to-close
seq
(mapv (fn [[i]] {:db/id i
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}))
(d/transact conn)
deref)
(println "Closed " (count invoices-to-close) "automatic invoices"))
(catch Exception e
(println (.toString e)))))
(mapv (fn [[i]] {:db/id i
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}))
(d/transact conn)
deref)
(log/info "Closed " (count invoices-to-close) "automatic invoices"))
(catch Exception e
(log/error e)))))
(mount/defstate close-auto-invoices-worker
:start (scheduler/every 60000 close-auto-invoices)

6
src/clj/auto_ap/datomic/invoices.clj
View File

@@ -156,8 +156,7 @@
[(->> (graphql-results ids-to-retrieve db args))
matching-count
(doto
outstanding println)]))
outstanding]))
(defn get-by-id [id]
(-> (d/db (d/connect uri))
@@ -186,8 +185,7 @@
[(not= ?e ?invoice-id)]
]}
:args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}
true (doto println)))
:args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}))
(map first)
(map <-datomic)))

3
src/clj/auto_ap/datomic/transactions.clj
View File

@@ -115,8 +115,7 @@
"status" ['[?e :transaction/status ?sort-status]]}
args)
true
(merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}}))
_ (println query)]
(merge-query {:query {:find ['?e] :where ['[?e :transaction/id]]}}))]
(cond->> query
true (d/query)
true (apply-sort-3 args)

1
src/clj/auto_ap/datomic/users.clj
View File

@@ -41,7 +41,6 @@
first
first
(update :user/role :db/ident))]
(println "USER" user)
(if user
user
(let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user

42
src/clj/auto_ap/graphql.clj
View File

@@ -7,6 +7,7 @@
[com.walmartlabs.lacinia.resolve :as resolve]
[buddy.auth :refer [throw-unauthorized]]
[auto-ap.utils :refer [by]]
[auto-ap.logging :refer [info-event warn-event error-event]]
[auto-ap.graphql.utils :refer [assert-admin can-see-client? assert-can-see-client]]
[auto-ap.datomic :refer [uri merge-query]]
[datomic.api :as d]
@@ -29,7 +30,10 @@
[auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.time :as time]
[clojure.walk :as walk]
[clojure.string :as str])
[clojure.string :as str]
[clojure.tools.logging :as log]
[yang.time :refer [time-it]]
[unilog.context :as lc])
(:import
(clojure.lang IPersistentMap)))
@@ -945,7 +949,6 @@
(filter (fn [[potential-id potential-date]]
(let [date (coerce/to-date-time date)
potential-date (coerce/to-date-time potential-date)]
#_(println "HERE" id potential-id potential-date date)
(and (= id potential-id)
(<= (t/in-days (apply t/interval (sort [date potential-date]))) 10)))))
conj
@@ -1010,7 +1013,6 @@
:amount amount
:date (coerce/to-date-time next)})
is-week-a? (fn [d]
(println d)
(= 0 (mod (t/in-weeks (t/interval first-week-a d)) 2)))]
{:beginning_balance total-cash
@@ -1110,20 +1112,24 @@
([id q]
(query id q nil ))
([id q v]
(println "executing graphql query" id q v)
(try
(let [result (time (simplify (execute schema q v {:id id})))]
(when (seq (:errors result))
(throw (ex-info "GraphQL error" {:result result})))
result)
(lc/with-context {:query q}
(log/info "Executing query" q)
(try
(let [[result time] (time-it (simplify (execute schema q v {:id id})))]
(info-event "Query completed"
{:time (:time time)
:errors (seq (:errors result))})
(when (seq (:errors result))
(throw (ex-info "GraphQL error" {:result result})))
result)
(catch Exception e
(if-let [v (:validation-error (ex-data e))]
(println "validation error" v)
(do
(.printStackTrace e)
(println e )))
(catch Exception e
(if-let [v (:validation-error (ex-data e))]
(warn-event "validation error" {:validation-error v
:data (ex-data e)})
(error-event "query error" {:error e}))
(throw e)))))
#_(query nil "{ ledger_page { count }}" nil)
(throw e))))))

3
src/clj/auto_ap/graphql/clients.clj
View File

@@ -19,8 +19,6 @@
(let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client)))
id (or (:db/id client) "new-client")
_ (println id)
_ (println edit_client)
_ (when client
@(d/transact (d/connect uri)
(into
@@ -79,7 +77,6 @@
)
(:forecasted_transactions edit_client))]]
result @(d/transact (d/connect uri) transactions)]
(println result "ID" id)
(-> result :tempids (get id) (or id) d-clients/get-by-id
(update :client/location-matches
(fn [lms]

11
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -58,7 +58,6 @@
:location location}))
(defn add-invoice-transaction [{:keys [total invoice_number location automatically_paid_when_due client_id vendor_id vendor_name date due expense_accounts] :as in}]
(println date)
(let [vendor (d-vendors/get-by-id vendor_id)
account (:vendor/default-account vendor)
_ (when-not (:db/id account)
@@ -119,11 +118,10 @@
(defn edit-invoice [context {{:keys [id due invoice_number total vendor_id date client_id expense_accounts automatically_paid_when_due] :as in} :invoice} value]
(let [invoice (d-invoices/get-by-id id)
_ (when (seq (doto (d-invoices/find-conflicting {:db/id id
:invoice/invoice-number invoice_number
:invoice/vendor (:db/id (:invoice/vendor invoice))
:invoice/client (:db/id (:invoice/client invoice))})
println))
_ (when (seq (d-invoices/find-conflicting {:db/id id
:invoice/invoice-number invoice_number
:invoice/vendor (:db/id (:invoice/vendor invoice))
:invoice/client (:db/id (:invoice/client invoice))}))
(throw (ex-info (str "Invoice '" invoice_number "' already exists.") {:invoice-number invoice_number})))
expense-account-total (reduce + 0 (map (fn [x] (Double/parseDouble (:amount x))) expense_accounts))
@@ -151,7 +149,6 @@
(defn void-invoice [context {id :invoice_id} value]
(let [invoice (d-invoices/get-by-id id)
_ (println invoice)
_ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
updated-invoice (d-invoices/update {:db/id id
:invoice/total 0.0

217
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -12,7 +12,9 @@
[auto-ap.parse.util :as parse]
[datomic.api :as d]
[auto-ap.parse.templates :as t]
[auto-ap.datomic.clients :as d-clients]))
[auto-ap.datomic.clients :as d-clients]
[clojure.tools.logging :as log]
[unilog.context :as lc]))
(defn get-ledger-page [context args value]
(let [args (assoc args :id (:id context))
@@ -106,15 +108,12 @@
vals
(mapcat (fn [a]
(map (fn [o]
#_(println "override" (:db/id a) (:db/id (:account-client-override/client o)))
#_(print [[(:db/id a) (:db/id (:account-client-override/client o))]])
[[(:db/id a) (:db/id (:account-client-override/client o))]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
#_(println a client-id (keys overrides-by-client))
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
@@ -158,7 +157,6 @@
(defn get-profit-and-loss [context args value]
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
_ (println args)
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
@@ -179,108 +177,113 @@
(defn import-ledger [context args value]
(assert-admin (:id context))
(let [all-vendors (by :vendor/name (d-vendors/get-graphql {}))
all-clients (by :client/code (d-clients/get-all ))
all-client-bank-accounts (reduce
(fn [acc client]
(assoc acc (:client/code client)
(set (->> (:client/bank-accounts client)
(map :bank-account/code)
))))
{}
(d-clients/get-all))
all-client-locations (reduce
(fn [acc client]
(assoc acc (:client/code client)
(-> (set (:client/locations client))
(conj "HQ")
(conj "A"))))
(lc/with-context {:area "import ledger"}
(let [all-vendors (by :vendor/name (d-vendors/get-graphql {}))
all-clients (by :client/code (d-clients/get-all ))
all-client-bank-accounts (reduce
(fn [acc client]
(assoc acc (:client/code client)
(set (->> (:client/bank-accounts client)
(map :bank-account/code)
))))
{}
(d-clients/get-all))
all-client-locations (reduce
(fn [acc client]
(assoc acc (:client/code client)
(-> (set (:client/locations client))
(conj "HQ")
(conj "A"))))
{}
(d-clients/get-all))
new-hidden-vendors (reduce
(fn [new-vendors {:keys [vendor_name line_items]}]
(if (or (all-vendors vendor_name)
(new-vendors vendor_name))
new-vendors
(assoc new-vendors vendor_name
{:vendor/name vendor_name
:vendor/hidden true
:db/id vendor_name})))
{}
(d-clients/get-all))
new-hidden-vendors (reduce
(fn [new-vendors {:keys [vendor_name line_items]}]
(if (or (all-vendors vendor_name)
(new-vendors vendor_name))
new-vendors
(assoc new-vendors vendor_name
{:vendor/name vendor_name
:vendor/hidden true
:db/id vendor_name})))
{}
(:entries args))
all-vendors (into all-vendors new-hidden-vendors)
all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
transaction (doall (map
(assoc-error (fn [entry]
(let [entry (-> entry
(update :amount #(Double/parseDouble %))
(update :line_items
(fn [lis]
(mapv
(fn [li ]
(-> li
(update :debit #(Double/parseDouble (if (str/blank? %) "0" %)))
(update :credit #(Double/parseDouble (if (str/blank? %) "0" %)))))
lis))))]
(let [vendor (all-vendors (:vendor_name entry))]
(when-not (all-clients (:client_code entry))
(throw (Exception. (str "Client '" (:client_code entry )"' not found.")) ))
(when-not vendor
(throw (Exception. (str "Vendor '" (:vendor_name entry) "' not found."))))
(when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
(throw (Exception. (str "Date must be MM/dd/yyyy"))))
(when-not (dollars= (doto (reduce + 0.0 (map :debit (:line_items entry))))
(reduce + 0.0 (map :credit (:line_items entry))))
(throw (Exception. (str "Debits '"
(reduce + 0 (map :debit (:line_items entry)))
"' and credits '"
(reduce + 0 (map :credit (:line_items entry)))
"' do not add up."))))
(remove-nils
{:journal-entry/source (:source entry)
:journal-entry/client [:client/code (:client_code entry)]
:journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
:journal-entry/external-id (:external_id entry)
:journal-entry/vendor (all-vendors (:vendor_name entry))
:journal-entry/amount (:amount entry)
:journal-entry/note (:note entry)
:journal-entry/cleared-against (:cleared_against entry)
(:entries args))
all-vendors (into all-vendors new-hidden-vendors)
all-accounts (transduce (map (comp str :account/numeric-code)) conj #{} (a/get-accounts))
transaction (doall (map
(assoc-error (fn [entry]
(let [entry (-> entry
(update :amount #(Double/parseDouble %))
(update :line_items
(fn [lis]
(mapv
(fn [li ]
(-> li
(update :debit #(Double/parseDouble (if (str/blank? %) "0" %)))
(update :credit #(Double/parseDouble (if (str/blank? %) "0" %)))))
lis))))]
(let [vendor (all-vendors (:vendor_name entry))]
(when-not (all-clients (:client_code entry))
(throw (Exception. (str "Client '" (:client_code entry )"' not found.")) ))
(when-not vendor
(throw (Exception. (str "Vendor '" (:vendor_name entry) "' not found."))))
(when-not (re-find #"\d{1,2}/\d{1,2}/\d{4}" (:date entry))
(throw (Exception. (str "Date must be MM/dd/yyyy"))))
(when-not (dollars= (doto (reduce + 0.0 (map :debit (:line_items entry))))
(reduce + 0.0 (map :credit (:line_items entry))))
(throw (Exception. (str "Debits '"
(reduce + 0 (map :debit (:line_items entry)))
"' and credits '"
(reduce + 0 (map :credit (:line_items entry)))
"' do not add up."))))
(remove-nils
{:journal-entry/source (:source entry)
:journal-entry/client [:client/code (:client_code entry)]
:journal-entry/date (coerce/to-date (parse/parse-value :clj-time "MM/dd/yyyy" (:date entry)))
:journal-entry/external-id (:external_id entry)
:journal-entry/vendor (all-vendors (:vendor_name entry))
:journal-entry/amount (:amount entry)
:journal-entry/note (:note entry)
:journal-entry/cleared-against (:cleared_against entry)
:journal-entry/line-items
(mapv (fn [ea]
(when-not (get
(get all-client-locations (:client_code entry))
(:location ea))
(throw (Exception. (str "Location '" (:location ea) "' not found."))))
(when (< (or (:debit ea) (:credit ea)) 0.0)
(throw (Exception. (str (or (:debit ea) (:credit ea)) "must be greater than 0."))))
(when (and (not (all-accounts (:account_identifier ea)))
(not (get
(get all-client-bank-accounts (:client_code entry))
(:account_identifier ea))))
(throw (Exception. (str "Account '" (:account_identifier ea) "' not found."))))
(remove-nils {:journal-entry-line/account
(if (re-matches #"^[0-9]+$" (:account_identifier ea))
(:db/id (a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))
[:bank-account/code (:account_identifier ea)])
:journal-entry-line/location (:location ea)
:journal-entry-line/debit (when (> (:debit ea) 0)
(:debit ea))
:journal-entry-line/credit (when (> (:credit ea) 0)
(:credit ea))}))
(:line_items entry))
:journal-entry/cleared true})))))
(:entries args)))
errors (filter :error transaction)
success (filter (comp not :error) transaction)
retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]])
success)]
(println (take 4 success))
(run! (fn [batch] (println "transacting retraction batch") @(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
(run! (fn [batch] (println "transacting success batch") @(d/transact (d/connect uri) batch)) (partition-all 100 success))
{:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
:existing []
:errors (map (fn [x] {:external_id (:external_id x)
:error (:error x)}) errors)}))
:journal-entry/line-items
(mapv (fn [ea]
(when-not (get
(get all-client-locations (:client_code entry))
(:location ea))
(throw (Exception. (str "Location '" (:location ea) "' not found."))))
(when (< (or (:debit ea) (:credit ea)) 0.0)
(throw (Exception. (str (or (:debit ea) (:credit ea)) "must be greater than 0."))))
(when (and (not (all-accounts (:account_identifier ea)))
(not (get
(get all-client-bank-accounts (:client_code entry))
(:account_identifier ea))))
(throw (Exception. (str "Account '" (:account_identifier ea) "' not found."))))
(remove-nils {:journal-entry-line/account
(if (re-matches #"^[0-9]+$" (:account_identifier ea))
(:db/id (a/get-account-by-numeric-code-and-sets (Integer/parseInt (:account_identifier ea)) ["default"]))
[:bank-account/code (:account_identifier ea)])
:journal-entry-line/location (:location ea)
:journal-entry-line/debit (when (> (:debit ea) 0)
(:debit ea))
:journal-entry-line/credit (when (> (:credit ea) 0)
(:credit ea))}))
(:line_items entry))
:journal-entry/cleared true})))))
(:entries args)))
errors (filter :error transaction)
success (filter (comp not :error) transaction)
retraction (mapv (fn [x] [:db/retractEntity [:journal-entry/external-id (:journal-entry/external-id x)]])
success)]
(log/info "manual ledger import has " (count success) " new rows")
(run! (fn [batch]
(log/info "transacting retraction batch")
@(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
(run! (fn [batch]
(log/info "transacting success batch")
@(d/transact (d/connect uri) batch)) (partition-all 100 success))
{:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
:existing []
:errors (map (fn [x] {:external_id (:external_id x)
:error (:error x)}) errors)})))

4
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -18,10 +18,6 @@
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm]
[clj-time.coerce :as coerce]))
(defn prn-each [xs]
(doseq [x xs]
(println x))
xs)
(def approval-status->graphql (ident->enum-f :transaction/approval-status))

10
src/clj/auto_ap/graphql/users.clj
View File

@@ -10,7 +10,6 @@
":user" :user-role/user})
(defn edit-user [context {:keys [edit_user] :as args} value]
(println args edit_user)
(assert-admin (:id context))
(let [user (d-users/get-by-id (:id edit_user))
new-clients (set (map #(Long/parseLong %) (:clients edit_user)))
@@ -21,11 +20,10 @@
@(d/transact (d/connect uri)
(doto (-> [{:db/id (:db/id user)
:user/role (role->datomic-role (:role edit_user))
:user/clients new-clients}]
(into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove)))
clojure.pprint/pprint))
(-> [{:db/id (:db/id user)
:user/role (role->datomic-role (:role edit_user))
:user/clients new-clients}]
(into (map (fn [c] [:db/retract (:db/id user) :user/clients c]) clients-to-remove))))
(->graphql
(d-users/get-by-id (:id edit_user)))))

10
src/clj/auto_ap/graphql/utils.clj
View File

@@ -1,7 +1,8 @@
(ns auto-ap.graphql.utils
(:require [clojure.string :as str]
[buddy.auth :refer [throw-unauthorized]]
[clojure.walk :as walk]))
[clojure.walk :as walk]
[clojure.tools.logging :as log]))
(defn snake->kebab [s]
@@ -43,24 +44,23 @@
m))
(defn is-admin? [id]
(println "role" id)
(= "admin" (:user/role id)))
(defn assert-admin [id]
(println "role" id)
(when-not (= "admin" (:user/role id))
(log/warn "user " id " not an admin!")
(throw-unauthorized)))
(defn can-see-client? [identity client]
(when (not client)
(println "WARNING - permission checking for null client"))
(log/warn "WARNING - permission checking for null client"))
(or (= "admin" (:user/role identity))
((set (map :db/id (:user/clients identity))) (:db/id client))
((set (map :db/id (:user/clients identity))) client)))
(defn assert-can-see-client [identity client]
(when-not (can-see-client? identity client)
(println "IDENTITY " identity " can not see company " client)
(log/warn "IDENTITY " identity " can not see company " client)
(throw-unauthorized)))
(defn limited-clients [id]

12
src/clj/auto_ap/graphql/vendors.clj
View File

@@ -72,19 +72,15 @@
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/account-overrides account-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/terms-overrides terms-overrides])
(is-admin? (:id context)) (conj [:reset (if id id "vendor") :vendor/automatically-paid-when-due
(doto (mapv
(fn [apwd]
{:db/id apwd})
(:automatically_paid_when_due in))
println)]))
(mapv
(fn [apwd]
{:db/id apwd})
(:automatically_paid_when_due in))]))
_ (println transaction)
transaction-result @(d/transact (d/connect uri) transaction)]
(-> (d-vendors/get-by-id (or (-> transaction-result :tempids (get "vendor"))
id))
(doto println)
(->graphql))))
(defn merge-vendors [context {:keys [from to]} value]

12
src/clj/auto_ap/handler.clj
View File

@@ -16,6 +16,8 @@
[compojure.core :refer :all]
[compojure.route :as route]
[config.core :refer [env]]
[clojure.tools.logging :as log]
[unilog.context :as lc]
[ring.middleware.edn :refer [wrap-edn-params]]
[ring.middleware.multipart-params :as mp]
[ring.middleware.params :refer [wrap-params]]
@@ -55,10 +57,20 @@
static-routes))
(defn wrap-logging [handler]
(fn [request]
(lc/with-context {:uri (:uri request)
:source "request"
:user-role (:user/role (:identity request))
:user-name (:user/name (:identity request))}
(log/info "Beginning request" (:uri request))
(handler request))))
(def app
(-> #'app-routes
(wrap-logging)
(wrap-authorization auth-backend)
(wrap-authentication auth-backend)
(wrap-reload)

110
src/clj/auto_ap/ledger.clj
View File

@@ -4,7 +4,10 @@
[mount.core :as mount]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic :refer [uri remove-nils]]
[clojure.spec.alpha :as s]))
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]
[unilog.context :as lc]))
(defn datums->impacted-entity [db [e changes]]
@@ -129,66 +132,75 @@
(defn process-one []
(let [transaction (.take tx-report-queue)
_ (println "processing transaction")
db (:db-after transaction)
affected-entities (->> (:tx-data transaction)
(map (fn [^datomic.db.Datum x]
{:e (:e x)
:a (d/ident db (:a x))
:v (:v x)
:added (:added x)}))
(group-by :e)
(mapcat #(datums->impacted-entity db %))
(set))
_ (println "processing transaction affected" (count affected-entities))
d-txs (->> affected-entities
(map #(entity-change->ledger db %))
(filter seq))
retractions (map (fn [[_ e]] [:db/retractEntity [:journal-entry/original-entity e]]) affected-entities)]
(lc/with-context {:source "process-txes"}
(try
(let [transaction (.take tx-report-queue)
_ (log/info "Converting tranasction to ledger")
db (:db-after transaction)
affected-entities (->> (:tx-data transaction)
(map (fn [^datomic.db.Datum x]
{:e (:e x)
:a (d/ident db (:a x))
:v (:v x)
:added (:added x)}))
(group-by :e)
(mapcat #(datums->impacted-entity db %))
(set))
_ (info-event (str "Found " (count affected-entities) " affected entities")
{:affected-entities (count affected-entities)})
d-txs (->> affected-entities
(map #(entity-change->ledger db %))
(filter seq))
retractions (map (fn [[_ e]] [:db/retractEntity [:journal-entry/original-entity e]]) affected-entities)]
(when (seq retractions)
@(d/transact conn retractions))
(when (seq retractions)
@(d/transact conn retractions))
(doseq [d-tx d-txs]
@(d/transact conn [d-tx]))))
(doseq [d-tx d-txs]
@(d/transact conn [d-tx]))
(log/info "Succesfully process transaction"))
(catch Exception e
(log/error e)))))
(mount/defstate process-txes-worker
:start (scheduler/run-fun process-one 1)
:stop (-> process-txes-worker :running? (reset! false)))
(defn reconcile-ledger []
(try
(println "Attempting to reconcile the ledger")
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :transaction/date]
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
(lc/with-context {:source "reconcile-ledger"}
(try
(log/info "Attempting to reconcile the ledger")
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :transaction/date]
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :invoice/date]
'(not [?t :invoice/status :invoice-status/voided])
'(not [?t :invoice/import-status :import-status/pending])
'(not [?t :invoice/exclude-from-ledger true])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
(when (seq repairs)
(println "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
:in ['$]
:where ['[?t :invoice/date]
'(not [?t :invoice/status :invoice-status/voided])
'(not [?t :invoice/import-status :import-status/pending])
'(not [?t :invoice/exclude-from-ledger true])
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
:args [(d/db conn)]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
@(d/transact conn repairs)))
(catch Exception e
(println e))))
(when (seq repairs)
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs))
(log/info "Finished reconciling ledger"))
(catch Exception e
(log/error e)))))
(mount/defstate reconciliation-frequency :start 60000)

15
src/clj/auto_ap/logging.clj Normal file
View File

@@ -0,0 +1,15 @@
(ns auto-ap.logging
(:require [clojure.tools.logging :as log]
[unilog.context :as lc]))
(defn info-event [message context]
(lc/with-context context
(log/info message)))
(defn warn-event [message context]
(lc/with-context context
(log/warn message)))
(defn error-event [message context]
(lc/with-context context
(log/warn message)))

23
src/clj/auto_ap/parse.clj
View File

@@ -1,15 +1,16 @@
(ns auto-ap.parse
(:require [auto-ap.parse.excel :as excel]
(:require [auto-ap.parse.csv :as csv]
[auto-ap.parse.excel :as excel]
[auto-ap.parse.templates :as t]
[auto-ap.parse.util :as u]
[auto-ap.parse.csv :as csv]
[clj-fuzzy.metrics :as m]
[clojure.java.shell :as sh]
[clojure.string :as str]
[clj-time.format :as f]
[clj-time.core :as time]
[clojure.set :as set]))
[clj-time.format :as f]
[clojure.java.shell :as sh]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]))
(def last-text (atom nil))
@@ -19,7 +20,8 @@
(defn extract-template
([text template]
(println "template" template)
(log/info "Template was determined to be" template)
(if (:multi template)
(mapcat
#(extract-template % text (dissoc template :multi))
@@ -39,16 +41,15 @@
str/trim )
[value-parser parser-params] (-> template :parser k)]
(assoc result k (try
(println "applying parser" value-parser "to value" value)
(u/parse-value value-parser parser-params value)
(catch Exception e
(println e))))))
(log/warn e))))))
{:vendor-code (:vendor template)
:text text
:full-text full-text}))])))
(defn parse [text]
(println "Parsing PDF " text)
(info-event "Parsing pdf text" {:raw text})
(reset! last-text text)
(->> t/pdf-templates
(filter (partial template-applies? text))

35
src/clj/auto_ap/parse/csv.clj
View File

@@ -2,29 +2,29 @@
(:require [auto-ap.parse.util :as u]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.string :as str]))
[clojure.string :as str]
[clojure.tools.logging :as log]))
(defn determine
[[header :as z]]
(prn header)
(doto (cond (str/includes? (second header) "Customer's PO No.")
:mama-lus
(let [csv-type (cond (str/includes? (second header) "Customer's PO No.")
:mama-lus
(str/includes? (str header) "Ship-To Number")
:sysco-style-2
(str/includes? (str header) "Ship-To Number")
:sysco-style-2
(str/includes? (str header) "Closed Date")
:sysco-style-1
(str/includes? (str header) "Closed Date")
:sysco-style-1
(str/includes? (str header) "Business Unit")
:mission
(str/includes? (str header) "Business Unit")
:mission
(str/includes? (str header) "Document Number")
:philz
(str/includes? (str header) "Document Number")
:philz
:else
nil)
println))
:else
nil)]
(log/info "csv type was determined to be" csv-type)))
(defmulti parse-csv
determine
@@ -44,7 +44,6 @@
(defmethod parse-csv :sysco-style-1
[rows]
(println "Importing Sysco-styled 1")
(let [header (first rows)]
(transduce
(comp (drop 1)
@@ -68,7 +67,7 @@
(defmethod parse-csv :sysco-style-2
[rows]
(println "Importing Sysco-styled 1")
(let [header (first rows)]
(transduce
(comp (drop 1)
@@ -125,9 +124,7 @@
(comp
(filter (fn [[dt _ doc-number name _ status _ _ amount :as row]]
(= status "Billed")))
(map (fn [[dt _ doc-number name _ _ _ _ amount :as row]]
(print name)
{:vendor-code "PHILZ COFFEE, INC"
:customer-identifier name
:invoice-number doc-number

5
src/clj/auto_ap/parse/util.clj
View File

@@ -3,7 +3,8 @@
[clojure.java.shell :as sh]
[clojure.string :as str]
[clj-time.format :as f]
[clj-time.core :as time]))
[clj-time.core :as time]
[clojure.tools.logging :as log]))
(defmulti parse-value (fn [method _ _]
method))
@@ -39,7 +40,7 @@
(reduced (time/from-time-zone (f/parse (f/formatter format) value)
(time/time-zone-for-id "America/Los_Angeles")))
(catch Exception e
(println (.getMessage e))
(log/warn e)
nil)))
nil
format)

29
src/clj/auto_ap/routes/auth.clj
View File

@@ -4,7 +4,8 @@
[clj-http.client :as http]
[clj-time.core :as time]
[compojure.core :refer [GET defroutes]]
[config.core :refer [env]]))
[config.core :refer [env]]
[clojure.tools.logging :as log]))
(def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com")
(def google-client-secret "OC-WemHurPXYpuIw5cT-B90g")
@@ -19,7 +20,6 @@
(defroutes routes
(GET "/oauth" {{:strs [code]} :query-params :keys [scheme] :as r {:strs [host]} :headers}
(println "Authenticating with" r "..." code)
(try
(let [auth (-> "https://accounts.google.com/o/oauth2/token"
(http/post
@@ -30,37 +30,34 @@
"grant_type" "authorization_code"}
:as :json})
:body)
_ (println auth)
token (:access_token auth)
profile (-> (http/get "https://www.googleapis.com/oauth2/v1/userinfo"
{:headers {"Authorization" (str "Bearer " token)} :as :json})
:body
(doto println))
:body)
user (users/find-or-insert! {:user/provider "google"
:user/provider-id (:id profile)
:user/role :user-role/none
:user/name (:name profile)})
]
(println "authenticated as user" user)
(log/info "authenticated as user" user)
;; TODO - these namespaces are not being transmitted/deserialized properly
(if (and token user)
(let [jwt (jwt/sign (doto {:user (:name profile)
:exp (time/plus (time/now) (time/days 30))
:user/clients (map (fn [c]
(dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address))
(:user/clients user))
:user/role (name (:user/role user))
:user/name (:name profile)}
println)
(let [jwt (jwt/sign {:user (:name profile)
:exp (time/plus (time/now) (time/days 30))
:user/clients (map (fn [c]
(dissoc c :client/bank-accounts :client/location-matches :client/forecasted-transactions :client/matches :client/week-a-debits :client/week-a-credits :client/week-b-debits :client/week-b-credits :client/signature-file :client/address))
(:user/clients user))
:user/role (name (:user/role user))
:user/name (:name profile)}
(:jwt-secret env)
{:alg :hs512})]
(println "authenticated. using jwt" jwt)
{:status 301
:headers {"Location" (str "/?jwt=" jwt)}})
{:status 401
:body "Couldn't authenticate"}))
(catch Exception e
(log/warn e )
{:status 401
:body (str "Couldn't authenticate " (.toString e))}))))

27
src/clj/auto_ap/routes/events.clj
View File

@@ -9,24 +9,27 @@
[clj-time.predicates :as pred]
[clojure.data.json :as json]
[compojure.core :refer [GET PUT POST context defroutes
wrap-routes]])
wrap-routes]]
[clojure.tools.logging :as log]
[unilog.context :as lc])
(:import (org.joda.time DateTime)))
(defroutes routes
(context "/events" []
(POST "/yodlee-import" {:keys [query-params headers body] :as x}
(let [notification-type (get headers "x-amz-sns-message-type")]
(println "Received notification " notification-type)
(if (= "SubscriptionConfirmation" notification-type)
(do
(println "Responding to confirmation" )
(let [json (json/read-str (slurp body))]
(println json)
(http/get (get json "SubscribeURL"))))
(do
(println "importing from yodlee")
(yodlee-import/do-import))))
(lc/with-context {:source "Import yodlee transactions"}
(let [notification-type (get headers "x-amz-sns-message-type")]
(log/info "Received notification " notification-type)
(if (= "SubscriptionConfirmation" notification-type)
(do
(log/info "Responding to confirmation" )
(let [json (json/read-str (slurp body))]
(log/info json)
(http/get (get json "SubscribeURL"))))
(do
(log/info "importing from yodlee")
(yodlee-import/do-import)))))
{:status 200
:body "{}"

24
src/clj/auto_ap/routes/graphql.clj
View File

@@ -1,10 +1,12 @@
(ns auto-ap.routes.graphql
(:require [auto-ap.routes.utils :refer [wrap-secure wrap-spec]]
[auto-ap.graphql :as ql]
[auto-ap.logging :refer [warn-event]]
[buddy.auth :refer [throw-unauthorized]]
[clojure.edn :as edn]
[compojure.core :refer [GET POST PUT context defroutes
wrap-routes]]))
wrap-routes]]
[clojure.tools.logging :as log]))
(defn handle-graphql [{:keys [request-method query-params body edn-params method] :as r}]
(when (= "none" (:user/role (:identity r)))
(throw-unauthorized))
@@ -14,19 +16,23 @@
edn/read-string)
body (some-> r :body slurp)]
{:status 200
:body (pr-str (ql/query (:identity r) (doto (if (= request-method :get) (query-params "query") body) println) variables ))
:body (pr-str (ql/query (:identity r) (if (= request-method :get) (query-params "query") body) variables ))
:headers {"Content-Type" "application/edn"}})
(catch Throwable e
(if-let [result (:result (ex-data e))]
{:status 400
:body (pr-str result)
:headers {"Content-Type" "application/edn"}}
(if-let [message (:validation-error (ex-data e) )]
(do (log/warn "Graphql Result error" e)
{:status 400
:body (pr-str {:errors [(merge {:message message} (ex-data e))]})
:headers {"Content-Type" "application/edn"}}
(do (println e)
:body (pr-str result)
:headers {"Content-Type" "application/edn"}})
(if-let [message (:validation-error (ex-data e) )]
(do
(warn-event "GraphQL Validation error" {:message message
:data e})
{:status 400
:body (pr-str {:errors [(merge {:message message} (ex-data e))]})
:headers {"Content-Type" "application/edn"}})
(do (log/error "GraphQL error" e)
{:status 500
:body (pr-str {:errors [(merge {:message (.getMessage e)} (ex-data e))]})
:headers {"Content-Type" "application/edn"}}))))))

200
src/clj/auto_ap/routes/invoices.clj
View File

@@ -19,7 +19,10 @@
wrap-routes]]
[clojure.string :as str]
[clojure.java.io :as io]
[clojure.data.csv :as csv]))
[clojure.data.csv :as csv]
[unilog.context :as lc]
[clojure.tools.logging :as log]
[auto-ap.logging :refer [info-event]]))
(defn reset-id [i]
(update i :invoice-number
@@ -203,88 +206,91 @@
{:vendor-code vendor-code})))))
(defn import-uploaded-invoice [client forced-location forced-vendor imports]
(let [clients (d-clients/get-all)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}]
(println "searching for" vendor-code)
(let [ _ (println "matching" customer-identifier)
matching-client (or (and account-number
(parse/best-match clients account-number 0.0))
(and customer-identifier
(parse/best-match clients customer-identifier))
(if client
(first (filter (fn [c]
(= (:db/id c) (Long/parseLong client)))
clients))))
_ (when-not matching-client
(throw (ex-info (str "Searched clients for '" customer-identifier "'. No client found in file. Select a client first.")
{:invoice-number invoice-number
:customer-identifier customer-identifier
:vendor-code vendor-code})))
matching-vendor (match-vendor vendor-code forced-vendor)
_ (println "invoice \"" invoice-number "\" matches client " (:client/name matching-client) " (" (:db/id matching-client) ")")
matching-location (or (when-not (str/blank? forced-location)
forced-location)
(parse/best-location-match matching-client text full-text))
[existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
(try
(->> (d/query
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
[?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/status ?status]
[?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]}
:args [(d/db (d/connect uri)) invoice-number (:db/id matching-vendor) (:db/id matching-client)]}))
first)
(catch Exception e
(throw (ex-info (str "Failed to find potential matching invoice with"
" invoice " invoice-number
" vendor " matching-vendor
" client " (:client/name matching-client)
". "
(.toString e))
{:args [ invoice-number matching-vendor (:db/id matching-client)]})))
))
automatically-paid-for (set (map :db/id (:vendor/automatically-paid-when-due matching-vendor)))]
(lc/with-context {:area "upload-invoice"}
(let [clients (d-clients/get-all)
transactions (reduce (fn [result {:keys [invoice-number customer-identifier account-number total date vendor-code text full-text] :as info}]
(cond
(not (and matching-location matching-client))
result
(let [
matching-client (or (and account-number
(parse/best-match clients account-number 0.0))
(and customer-identifier
(parse/best-match clients customer-identifier))
(if client
(first (filter (fn [c]
(= (:db/id c) (Long/parseLong client)))
clients))))
_ (when-not matching-client
(throw (ex-info (str "Searched clients for '" customer-identifier "'. No client found in file. Select a client first.")
{:invoice-number invoice-number
:customer-identifier customer-identifier
:vendor-code vendor-code})))
(= :import-status/imported import-status)
result
matching-vendor (match-vendor vendor-code forced-vendor)
:else
(conj result (cond-> (remove-nils #:invoice {:invoice/client (:db/id matching-client)
:invoice/client-identifier customer-identifier
:invoice/vendor (:db/id matching-vendor)
:invoice/invoice-number invoice-number
:invoice/automatically-paid-when-due (boolean (automatically-paid-for (:db/id matching-client)))
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/import-status :import-status/pending
:invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total))
:invoice/status (or existing-status :invoice-status/unpaid)
:invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:account (d-vendors/account-for-client-id matching-vendor (:db/id matching-client))
:location matching-location
:amount (Double/parseDouble total)}])
:db/id existing-id
})
(:vendor/terms matching-vendor) (assoc :invoice/due (coerce/to-date
(time/plus date (time/days (d-vendors/terms-for-client-id matching-vendor (:db/id matching-client)))))))))
))
[]
imports)]
(when-not (seq transactions)
(throw (ex-info "No invoices found."
{:imports (str imports)})))
@(d/transact (d/connect uri) (vec (set transactions)))))
_ (info-event "Found match for invoice" {:invoice-number invoice-number
:vendor (select-keys matching-vendor [:vendor/name :db/id])
:client (select-keys matching-client [:client/name :db/id])})
matching-location (or (when-not (str/blank? forced-location)
forced-location)
(parse/best-location-match matching-client text full-text))
[existing-id existing-outstanding-balance existing-status import-status] (when (and matching-client matching-location)
(try
(->> (d/query
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
[?e :invoice/outstanding-balance ?outstanding-balance]
[?e :invoice/status ?status]
[?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]}
:args [(d/db (d/connect uri)) invoice-number (:db/id matching-vendor) (:db/id matching-client)]}))
first)
(catch Exception e
(throw (ex-info (str "Failed to find potential matching invoice with"
" invoice " invoice-number
" vendor " matching-vendor
" client " (:client/name matching-client)
". "
(.toString e))
{:args [ invoice-number matching-vendor (:db/id matching-client)]})))
))
automatically-paid-for (set (map :db/id (:vendor/automatically-paid-when-due matching-vendor)))]
(cond
(not (and matching-location matching-client))
result
(= :import-status/imported import-status)
result
:else
(conj result (cond-> (remove-nils #:invoice {:invoice/client (:db/id matching-client)
:invoice/client-identifier customer-identifier
:invoice/vendor (:db/id matching-vendor)
:invoice/invoice-number invoice-number
:invoice/automatically-paid-when-due (boolean (automatically-paid-for (:db/id matching-client)))
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/import-status :import-status/pending
:invoice/outstanding-balance (or existing-outstanding-balance (Double/parseDouble total))
:invoice/status (or existing-status :invoice-status/unpaid)
:invoice/expense-accounts (when-not existing-id [#:invoice-expense-account {:account (d-vendors/account-for-client-id matching-vendor (:db/id matching-client))
:location matching-location
:amount (Double/parseDouble total)}])
:db/id existing-id
})
(:vendor/terms matching-vendor) (assoc :invoice/due (coerce/to-date
(time/plus date (time/days (d-vendors/terms-for-client-id matching-vendor (:db/id matching-client)))))))))
))
[]
imports)]
(when-not (seq transactions)
(throw (ex-info "No invoices found."
{:imports (str imports)})))
@(d/transact (d/connect uri) (vec (set transactions))))))
(defn validate-account-rows [rows code->existing-account]
@@ -318,7 +324,6 @@
:in ['$ '?z]
:where [['?e :client/code '?z]]}
:args [(d/db (d/connect uri)) customer]})))
_ (println client-id)
headers (map read-string header)
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
{:account/applicability [:db/ident]}
@@ -427,7 +432,7 @@
(map (parse-or-error :amount parse-amount))
(map (parse-or-error :date parse-date)))
error-rows (filter :errors rows)
_ (println "importing raw transactions" rows)
_ (log/info "Importing " (count rows) "raw transactions")
raw-transactions (vec (->> rows
(filter #(not (seq (:errors %))) )
(map (fn [{:keys [description-original client-code status high-level-category amount bank-account-code date]}]
@@ -445,7 +450,7 @@
:errors (map #(dissoc % :date) error-rows)})
:headers {"Content-Type" "application/edn"}})
(catch Exception e
(println e)
(log/error e)
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
@@ -469,18 +474,18 @@
vendor (some-> (or vendor vendor-2)
(Long/parseLong))
{:keys [filename tempfile]} files]
#_(println params (.getPath tempfile) filename)
(try
(import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename))
{:status 200
:body (pr-str {})
:headers {"Content-Type" "application/edn"}}
(catch Exception e
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
:data (ex-data e)})
:headers {"Content-Type" "application/edn"}}))))
(lc/with-context {:parsing-file filename}
(try
(import-uploaded-invoice client location vendor (parse/parse-file (.getPath tempfile) filename))
{:status 200
:body (pr-str {})
:headers {"Content-Type" "application/edn"}}
(catch Exception e
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
:data (ex-data e)})
:headers {"Content-Type" "application/edn"}})))))
(POST "/upload-integreat"
{{:keys [excel-rows]} :edn-params user :identity}
@@ -515,7 +520,6 @@
files-2 "file"} :params :as params
user :identity}
(let [files (or files files-2)
_ (println files)
{:keys [filename tempfile]} files]
(assert-admin user)
(try
@@ -524,7 +528,7 @@
:body (pr-str {})
:headers {"Content-Type" "application/edn"}}
(catch Exception e
(println e)
(log/error e)
{:status 500
:body (pr-str {:message (.getMessage e)
:error (.toString e)
@@ -545,7 +549,7 @@
:body (import-account-overrides client (.getPath tempfile))
:headers {"Content-Type" "application/json"}}
(catch Exception e
(println e)
(log/error e)
{:status 500
:body {:message (.getMessage e)
:data (ex-data e)}

25
src/clj/auto_ap/routes/yodlee.clj
View File

@@ -1,17 +1,18 @@
(ns auto-ap.routes.yodlee
(:require
[auto-ap.graphql :as graphql]
[clj-http.client :as http]
[auto-ap.graphql :as graphql]
[clj-http.client :as http]
[auto-ap.yodlee.core :as yodlee]
[auto-ap.graphql.utils :refer [->graphql assert-admin]]
[auto-ap.routes.utils :refer [wrap-secure]]
[clj-time.coerce :refer [to-date]]
[ring.middleware.json :refer [wrap-json-response]]
[compojure.core :refer [GET POST context defroutes wrap-routes]]
[clojure.string :as str]
[config.core :refer [env]]
))
[auto-ap.yodlee.core :as yodlee]
[auto-ap.graphql.utils :refer [->graphql assert-admin]]
[auto-ap.routes.utils :refer [wrap-secure]]
[clj-time.coerce :refer [to-date]]
[ring.middleware.json :refer [wrap-json-response]]
[compojure.core :refer [GET POST context defroutes wrap-routes]]
[clojure.string :as str]
[config.core :refer [env]]
[clojure.tools.logging :as log]))
(defroutes routes
(wrap-routes
@@ -53,7 +54,7 @@
:headers {"Content-Type" "application/edn"}
:body (pr-str (yodlee/reauthenticate (Long/parseLong id) data)) })
(catch Exception e
(println e)
(log/error e)
{:status 500
:headers {"Content-Type" "application/edn"}
:body (pr-str {:message (.getMessage e)

17
src/clj/auto_ap/server.clj
View File

@@ -7,14 +7,19 @@
[nrepl.server :refer [start-server stop-server]]
[config.core :refer [env]]
[ring.adapter.jetty :refer [run-jetty]]
[clojure.tools.logging :as log]
[unilog.config]
[mount.core :as mount])
(:gen-class))
(mount/defstate port :start (Integer/parseInt (or (env :port) "3000")))
(mount/defstate jetty
:start (run-jetty app {:port port :join? false})
:stop (.stop jetty)
)
(defn -main [& args]
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
(let [port (Integer/parseInt (or (env :port) "3000"))]
(mount/start)
#_(future (always-process-sqs))
(run-jetty app {:port port :join? false})))
(mount/start))

85
src/clj/auto_ap/yodlee/core.clj
View File

@@ -2,7 +2,10 @@
(:require [clj-http.client :as client]
[auto-ap.utils :refer [by]]
[cemerick.url :as u]
[unilog.context :as lc]
[clojure.tools.logging :as log]
[clojure.data.json :as json]
[clojure.core.async :as async]
[config.core :refer [env]]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
@@ -92,9 +95,7 @@
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -114,9 +115,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts")
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount
@@ -131,9 +130,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:providerAccount)))
@@ -145,9 +142,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts/" id )
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:query-params {"include" "credentials,questions,preferences"}
:as :json})
:body
@@ -161,9 +156,7 @@
(-> (str (:yodlee-base-url env) "/providerAccounts?providerAccountIds=" pa)
(client/put {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/put {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:body "{\"dataSetName\": [\"BASIC_AGG_DATA\"]}"
:as :json}))))
@@ -179,9 +172,7 @@
get-transaction-batch (fn [skip]
(-> (str (:yodlee-base-url env) "/transactions?top=" batch-size "&skip=" skip "&accountId=" account)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -199,11 +190,8 @@
user-session (login-user cob-session)]
(-> (str (:yodlee-base-url env) "/transactions/count?accountId=" account)
(doto println)
(client/get {:headers (doto
(merge base-headers {"Authorization" (auth-header cob-session user-session)})
println)
(client/get {:headers (merge base-headers {"Authorization" (auth-header cob-session user-session)})
:as :json})
:body
:transaction
@@ -236,34 +224,55 @@
:as :json})
:body)))
(defn get-provider-accounts-with-details []
(let [provider-accounts (get-provider-accounts)]
(reduce
(fn [pas pa]
(conj pas (try (get-provider-account-detail (:id pa))
(catch Exception e
pa))))
[]
provider-accounts)))
(let [concurrent 20
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account]
(lc/with-context {:provider-account-id (:id provider-account)}
(log/info "fetching details for provider" (:id provider-account))
(get-provider-account-detail (:id provider-account)))))
(async/to-chan provider-accounts))
(async/<!! (async/into [] output-chan)))))
(defn concurrent-get-accounts-for-providers [provider-account-ids]
(let [concurrent 20
output-chan (async/chan)]
(async/pipeline-blocking concurrent
output-chan
(map (fn [provider-account-id]
(lc/with-context {:provider-account-id provider-account-id}
(log/info "fetching accounts for provider" provider-account-id)
[provider-account-id
(get-accounts-for-provider-account provider-account-id)])))
(async/to-chan provider-account-ids))
(async/<!! (async/into {} output-chan))))
(defn get-provider-accounts-with-accounts []
(let [provider-accounts (by :id (get-provider-accounts-with-details))
accounts (get-accounts)]
accounts (concurrent-get-accounts-for-providers (keys provider-accounts))]
(->> accounts
(reduce
(fn [provider-accounts a]
(update-in provider-accounts [(:providerAccountId a) :accounts] conj a)) provider-accounts)
(fn [provider-accounts [which accounts]]
(assoc-in provider-accounts [which :accounts] accounts))
provider-accounts)
vals)))
(mount/defstate in-memory-cache
:start (doto (atom (get-provider-accounts-with-accounts)) println))
:start (atom []))
(defn refresh-in-memory-cache []
(try
(println "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(println e))))
(lc/with-context {:source "refreshing-in-memory-cache"}
(try
(log/info "Refreshing Yodlee in memory cache")
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(log/error e)))))
(mount/defstate in-memory-cache-worker
:start (scheduler/every (* 5 60 1000) refresh-in-memory-cache)

52
src/clj/auto_ap/yodlee/import.clj
View File

@@ -12,7 +12,9 @@
[auto-ap.time :as time]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.rule-matching :as rm]
[clojure.string :as str]))
[clojure.string :as str]
[unilog.context :as lc]
[clojure.tools.logging :as log]))
@@ -149,29 +151,31 @@
:client/_bank-accounts client))))))))
(defn manual-import [manual-transactions]
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount]))
(vals)
(mapcat (fn [transaction-group]
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount}
:description {:original description-original
:simple high-level-category}
:status "POSTED"})
(range)
transaction-group))))
all-rules (tr/get-all)
all-bank-accounts (by :db/id (get-all-bank-accounts))
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
(println "importing manual transactions" transformed-transactions)
(let [result (batch-transact
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
(println "imported " (count result)))))
(lc/with-context {:source "manual import"}
(let [transformed-transactions (->> manual-transactions
(filter #(= "posted" (:status %)))
(group-by #(select-keys % [:date :description-original :amount]))
(vals)
(mapcat (fn [transaction-group]
(map
(fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}]
{:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id)
:bank-account-id bank-account-id
:date (time/unparse date "YYYY-MM-dd")
:amount {:amount amount}
:description {:original description-original
:simple high-level-category}
:status "POSTED"})
(range)
transaction-group))))
all-rules (tr/get-all)
all-bank-accounts (by :db/id (get-all-bank-accounts))
transaction->bank-account (comp all-bank-accounts :bank-account-id)]
(log/info "Importing " (count transformed-transactions) " manual transactions")
(let [result (batch-transact
(transactions->txs transformed-transactions transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing)))]
(log/info "Imported " (count result) " manual transactions")))))
(defn do-import
([]

7
src/clj/user.clj
View File

@@ -3,6 +3,8 @@
[auto-ap.utils :refer [by]]
#_[auto-ap.ledger :as l]
[mount.core :as mount]
[auto-ap.server ]
[datomic.api :as d]
[clojure.data.csv :as csv]
[clj-time.coerce :as c]
@@ -360,3 +362,8 @@
'(not-join [?i] [?e :journal-entry/original-entity ?i])]}
:args [(d/db (d/connect uri))]})])
(defn go []
(require '[mount.core :as mount])
(require '[auto-ap.server])
(mount/start-without #'auto-ap.server/jetty))

1
src/cljs/auto_ap/views/pages/ledger.cljs
View File

@@ -52,7 +52,6 @@
::params-change
[with-user (re-frame/inject-cofx ::inject/sub [::params])]
(fn [{:keys [user ::params db]} [_]]
(println "execute")
{:db (-> db
(assoc-in [::last-params] params)
(assoc-in [:status :loading] true))