notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

tons of bug fixes.

Browse Source
This commit is contained in:
Bryce Covert
2020-07-15 08:17:39 -07:00
parent d120b7e810
commit 08f12b8107
9 changed files with 496 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -5,7 +5,7 @@
[auto-ap.datomic.accounts :as a]
[auto-ap.utils :refer [by dollars=]]
[auto-ap.time :refer [parse iso-date]]
[auto-ap.graphql.utils :refer [->graphql <-graphql limited-clients assert-admin result->page]]
[auto-ap.graphql.utils :refer [->graphql <-graphql limited-clients assert-admin result->page assert-can-see-client]]
[clj-time.coerce :as coerce]
[clojure.string :as str]
[clj-time.core :as time]
@@ -51,98 +51,121 @@
(or overriden-name (:account/name account))))
(defn roll-up [client results]
(->> results
(mapcat :journal-entry/line-items)
(group-by (juxt :journal-entry-line/account :journal-entry-line/location))
(reduce-kv (fn [result [account location] line-items]
;; TODO fix
#_(when-not (or (:bank-account/name account) (:account/name account))
(println "WARNING " account line-items))
(conj result
{:name (str (or (:bank-account/name account) (account-name account client))
(when-not (#{"A" } location)
(str
"-" location)))
:location location
:id (str (:db/id account) "-" location)
:numeric-code (or (:account/numeric-code account)
(and (#{:bank-account-type/check} (:db/ident (:bank-account/type account)))
11100)
(and (#{:bank-account-type/credit} (:db/ident (:bank-account/type account)))
28000))
:account-type (or (:db/ident (:account/type account))
({:bank-account-type/check :asset
:bank-account-type/credit :liability}
(:db/ident (:bank-account/type account))))
:amount (reduce + 0 (map
(fn [line-item]
(cond
(and (credit-account? account) (:journal-entry-line/debit line-item))
(- (:journal-entry-line/debit line-item))
(defn roll-up-until
([lookup-account all-ledger-entries end-date]
(roll-up-until lookup-account all-ledger-entries end-date nil))
([lookup-account all-ledger-entries end-date start-date]
(->> all-ledger-entries
(filter (fn [[d]]
(if start-date
(and
(>= (compare d start-date) 0)
(<= (compare d end-date) 0))
(<= (compare d end-date) 0))))
(reduce
(fn [acc [_ _ account location debit credit]]
(-> acc
(update-in [[location account] :debit] (fnil + 0.0) debit)
(update-in [[location account] :credit] (fnil + 0.0) credit)
(update-in [[location account] :count] (fnil + 0) 1))
)
{})
(reduce-kv
(fn [acc [location account-id] {:keys [debit credit count]}]
(let [account (lookup-account account-id)
account-type (:account_type account)]
(conj acc (merge {:id (str account-id "-" location)
:location (or location "")
:amount (if account-type (if (#{:account-type/asset
:account-type/dividend
:account-type/expense} account-type)
(- debit credit)
(- credit debit)
)
0.0)}
account)))
)
(and (credit-account? account) (:journal-entry-line/credit line-item))
(:journal-entry-line/credit line-item)
[]))))
(and (debit-account? account) (:journal-entry-line/debit line-item))
(:journal-entry-line/debit line-item)
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db (d/connect uri) )]})))
(and (debit-account? account) (:journal-entry-line/credit line-item))
(- (:journal-entry-line/credit line-item))
bank-accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :bank-account/name {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(d/db (d/connect uri))]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
(map (fn [o]
[[[(:db/id a) (:db/id (:account-client-override/client o))]]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
:account_type (or (:db/ident (:account/type (accounts a)))
({:bank-account-type/check :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (bank-accounts a)))))
:numeric_code (or (:account/numeric-code (accounts a))
(and (#{:bank-account-type/check} (:db/ident (:bank-account/type (bank-accounts a))))
11100)
(and (#{:bank-account-type/credit} (:db/ident (:bank-account/type (bank-accounts a))))
28000))})))
:else
0.0))
line-items))}))
[])))
(defn full-ledger-for-client [client-id]
(->> (d/query
{:query {:find ['?d '?jel '?account '?location '?debit '?credit ]
:in ['$ '?client-id]
:where ['[?e :journal-entry/client ?client-id]
'[?e :journal-entry/date ?d]
'[?e :journal-entry/line-items ?jel]
'[(get-else $ ?jel :journal-entry-line/account :account/unknown) ?account]
'[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit ]
'[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
'[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db (d/connect uri)) client-id]})
(sort-by first)))
(defn get-balance-sheet [context args value]
(let [args (assoc args :id (:id context))
client (d-clients/get-by-id (:client_id args))
[results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date (:date args))}
:count Integer/MAX_VALUE})
[comparable-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date (time/minus (:date args) (time/years 1)))}
:count Integer/MAX_VALUE})]
(println "count" (take 3 comparable-results))
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
end-date (coerce/to-date (:date args))
comparable-date (coerce/to-date (time/minus (:date args) (time/years 1)))
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
(->graphql
{:balance-sheet-accounts (roll-up client results)
:comparable-balance-sheet-accounts (roll-up client comparable-results)})))
{:balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries end-date)
:comparable-balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries comparable-date)})))
(defn get-profit-and-loss [context args value]
(let [args (assoc args :id (:id context))
client (d-clients/get-by-id (:client_id args))
pnl (fn [from-date to-date]
(println "FROM" from-date to-date)
(let [[starting-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (-> from-date
(time/minus (time/seconds 1))
coerce/to-date)}
:count Integer/MAX_VALUE})
[ending-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date to-date)}
:count Integer/MAX_VALUE})
starting-accounts (by :id (roll-up client starting-results))
ending-accounts (by :id (roll-up client ending-results))]
(reduce-kv
(fn [results k v]
(conj results (update v :amount (fn [amt]
(- amt
(get-in starting-accounts [k :amount] 0))))))
[]
ending-accounts)))]
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
start-date (coerce/to-date (:start (:date_range args)))
end-date (coerce/to-date (:end (:date_range args)))
comparable-start-date (coerce/to-date (time/minus (:start (:date_range args)) (time/years 1)))
comparable-end-date (coerce/to-date (time/minus (:end (:date_range args)) (time/years 1)))
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
(->graphql
{:balance-sheet-accounts (pnl (:start (:date_range args))
(:end (:date_range args)))
:comparable-balance-sheet-accounts (pnl (time/minus (:start (:date_range args)) (time/years 1))
(time/minus (:end (:date_range args)) (time/years 1)))})))
{:balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries end-date start-date )
:comparable-balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries comparable-end-date comparable-start-date )})))
#_(get-profit-and-loss nil {:client_id [:client/code "CBC"]
:from_date "2018-01-01"
:to_date "2019-04-01"} nil)
(defn assoc-error [f]
(fn [entry]
@@ -254,7 +277,7 @@
(run! (fn [batch] (println "transacting retraction batch") @(d/transact (d/connect uri) batch)) (partition-all 100 retraction))
(run! (fn [batch] (println "transacting success batch") @(d/transact (d/connect uri) batch)) (partition-all 100 success))
{:successful (map (fn [x] {:external_id (:journal-entry/external-id x)}) success)
:existing []
:existing []
:errors (map (fn [x] {:external_id (:external_id x)
:error (:error x)}) errors)}))

44
src/clj/auto_ap/parse/csv.clj
View File

@@ -16,6 +16,11 @@
(str/includes? (str header) "Closed Date")
:sysco-style-1
(str/includes? (str header) "Business Unit")
:mission
(str/includes? (str header) "Document Number")
:philz
:else
nil)
@@ -100,6 +105,45 @@
[]
rows))
(defmethod parse-csv :mission
[rows]
(transduce
(comp (drop 1)
(map (fn [[ po-number despatch-number invoice-number invoice-date customer value :as row]]
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number )
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:total (str/replace value #"," "")
:text (str/join " " row)
:full-text (str/join " " row)})))
conj
[]
rows))
(defmethod parse-csv :philz
[rows]
(transduce
(comp
(filter (fn [[dt _ doc-number name _ status _ _ amount :as row]]
(= status "Billed")))
(map (fn [[dt _ doc-number name _ _ _ _ amount :as row]]
(print name)
{:vendor-code "PHILZ COFFEE, INC"
:customer-identifier name
:invoice-number doc-number
:date (some-> dt not-empty (parse-date-fallover ["MM/dd/yyyy"]))
:total (str/replace amount #"," "")
:text (str/join " " row)
:full-text (str/join " " row)}))
)
conj
[]
(drop 1 rows)))
(defmethod parse-csv nil
[rows]
nil)

40
src/clj/auto_ap/parse/templates.clj
View File

@@ -427,6 +427,46 @@
:invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)"
:total #"(?:INVOICE|TOTAL|CREDIT)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}}
;; ROMA BAKERY
{:vendor "Roma Bakery Inc."
:keywords [#"Roma Bakery Inc"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice (\d+)"
:total #"Total\s+([\d\-\.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
;; KAEL FOODS
{:vendor "Kael Foods"
:keywords [#"kaelfoods.com"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}"
:invoice-number #"INVOICE 0*(\d+)"
:total #"TOTAL:\s+\$([\d\-\.,]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}}
;; Starter Bakery
{:vendor "Starter Bakery"
:keywords [#"starterbakery.com"]
:extract {:date #"INVOICE DATE:\s+(.*?)\s{2,}"
:customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice.*?(\d+)"
:total #"Total:.*?([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MMMM dd, yyyy"]
:total [:trim-commas-and-negate nil]}}
;; Trimark
{:vendor "TriMark R.W. Smith"
:keywords [#"TriMark"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To\s+(.*?)\s{2,}"
:invoice-number #"Invoice #\n.*?([\d\-]+)\n"
:total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}])
(defn offset [c x y]

7
src/clj/auto_ap/yodlee/core.clj
View File

@@ -246,6 +246,8 @@
(defonce in-memory-cache (atom []))
(defonce break? (atom false))
(defn load-in-memory-cache []
(future
(loop []
@@ -253,8 +255,9 @@
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(println e)))
(Thread/sleep (* 60 * 1000 * 5))
(recur))))
(Thread/sleep (* 30 1000 5))
(when-not @break?
(recur)))))
(defn refresh-provider-account [id]
(swap! in-memory-cache