tons of bug fixes.

This commit is contained in:
Bryce Covert
2020-07-15 08:17:39 -07:00
parent d120b7e810
commit 08f12b8107
9 changed files with 496 additions and 186 deletions

View File

@@ -0,0 +1,142 @@
(ns make-ledger-fast-again
(:require [datomic.api :as d]
[auto-ap.datomic :refer[uri]]
[auto-ap.utils :refer [by]]
[clj-time.coerce :as coerce]
))
(time
(def old-results
(map (fn [o] (update o :amount int)) (:balance_sheet_accounts (auto-ap.graphql.ledger/get-balance-sheet nil {:client_id 17592186046483
:date (coerce/to-date-time #inst "2020-09-01")} nil)))))
#_(defn roll-up-2 [jes]
)
(def new-results
(let [client-id 17592186046483
brackets [#inst "2019-01-01" #inst "2020-01-01"]
accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db (d/connect uri) )]})))
bank-accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :bank-account/name {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(d/db (d/connect uri))]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
(map (fn [o]
[[[(:db/id a) (:db/id (:account-client-override/client o))]]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))
account->name (fn [a]
(or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a))))
account->numeric-code (fn [a]
(or (:account/numeric-code (accounts a))
(and (#{:bank-account-type/check} (:db/ident (:bank-account/type (bank-accounts a))))
11100)
(and (#{:bank-account-type/credit} (:db/ident (:bank-account/type (bank-accounts a))))
28000)))
account->type (fn [a]
(or (:db/ident (:account/type (accounts a)))
({:bank-account-type/check :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (bank-accounts a))))))]
(->> (d/query
{:query {:find ['?d '?jel '?account '?location '?debit '?credit ]
:in ['$ '?client-id]
:where ['[?e :journal-entry/client ?client-id]
'[?e :journal-entry/date ?d]
'[(<= ?d #inst "2020-09-01")]
'[?e :journal-entry/line-items ?jel]
'[(get-else $ ?jel :journal-entry-line/account :account/unknown) ?account]
'[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
'[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
'[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db (d/connect uri)) client-id]})
(sort-by first)
(reduce
(fn [acc [_ _ account location debit credit]]
(-> acc
(update-in [[location account] :debit] (fnil + 0.0) debit)
(update-in [[location account] :credit] (fnil + 0.0) credit)
(update-in [[location account] :count] (fnil + 0) 1))
)
{})
(reduce-kv
(fn [acc [location account] {:keys [debit credit count]}]
(let [account-type (account->type account)]
(conj acc {:name (if-not (= "A" location)
(str (account->name account) "-" location)
(account->name account))
:id (str account "-" location)
:numeric_code (account->numeric-code account)
:location (or location "")
:amount (if account-type (if (#{:account-type/asset
:account-type/dividend
:account-type/expense} account-type)
(- debit credit)
(- credit debit)
)
0.0)
:account_type account-type}))
)
[])
#_(map (juxt :name :count (comp int :amount)))
#_(sort-by first)
))
)
;; 9804
new-results
old-results
(for [[k1 a1] (by :id (map #(dissoc % :location :account_type ) (filter :numeric_code new-results)))
:let [a2 (get (by :id (map #(dissoc % :location :account_type ) (filter :numeric_code old-results)))
k1)]
:when (not= a1 a2)]
[a1 a2]
)
(clojure.data/diff (set new-results) (set old-results))
;;16275
#_(d/query
{:query {:find ['?account '?location '?debit '?credit ]
:in ['$]
:where ['[?e :journal-entry/client 17592186046483]
'[?e :journal-entry/date ?d]
'[(<= ?d #inst "2020-09-01")]
'[?e :journal-entry/line-items ?jel]
'[(get-else $ ?jel :journal-entry-line/account :account/unknown) ?account]
'[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
'[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
'[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db (d/connect uri))]})

View File

@@ -0,0 +1,44 @@
(ns remove-old-yodlee)
(def account->provider-account
(->> @in-memory-cache
(reduce
(fn [acc {:keys [id accounts]}]
(reduce
(fn [acc a]
(assoc acc (:id a) id))
acc
accounts)
)
{})))
(def used-provider-accounts
(->> (datomic.api/query {:query {:find ['?y]
:in ['$]
:where ['[_ :bank-account/yodlee-account-id ?y]]}
:args [(datomic.api/db (datomic.api/connect auto-ap.datomic/uri))]})
(map first)
(map account->provider-account)
set
))
(def all-provider-accounts (set (map :id @in-memory-cache)))
(clojure.set/difference all-provider-accounts used-provider-accounts)
(->> [17891145 14187174 14264733 14274237 16681522 18612051 17796307 18643735
14187073 17754737 16153006 14565664 15941583 18621649 14187279 14187177
18652162 17892740 14187420 15613426 14187503 16604611 18611904 14180564
16153014 15942027 17574983 14186984 14180087 18651698 18645515 18472838
15735320]
(map (by :id @in-memory-cache))
(mapcat :accounts)
(mapcat :dataset)
(map :additionalStatus)
#_(map :lastUpdated)
)

View File

@@ -5,7 +5,7 @@
[auto-ap.datomic.accounts :as a]
[auto-ap.utils :refer [by dollars=]]
[auto-ap.time :refer [parse iso-date]]
[auto-ap.graphql.utils :refer [->graphql <-graphql limited-clients assert-admin result->page]]
[auto-ap.graphql.utils :refer [->graphql <-graphql limited-clients assert-admin result->page assert-can-see-client]]
[clj-time.coerce :as coerce]
[clojure.string :as str]
[clj-time.core :as time]
@@ -51,98 +51,121 @@
(or overriden-name (:account/name account))))
(defn roll-up [client results]
(->> results
(mapcat :journal-entry/line-items)
(group-by (juxt :journal-entry-line/account :journal-entry-line/location))
(reduce-kv (fn [result [account location] line-items]
;; TODO fix
#_(when-not (or (:bank-account/name account) (:account/name account))
(println "WARNING " account line-items))
(conj result
{:name (str (or (:bank-account/name account) (account-name account client))
(when-not (#{"A" } location)
(str
"-" location)))
:location location
:id (str (:db/id account) "-" location)
:numeric-code (or (:account/numeric-code account)
(and (#{:bank-account-type/check} (:db/ident (:bank-account/type account)))
(defn roll-up-until
([lookup-account all-ledger-entries end-date]
(roll-up-until lookup-account all-ledger-entries end-date nil))
([lookup-account all-ledger-entries end-date start-date]
(->> all-ledger-entries
(filter (fn [[d]]
(if start-date
(and
(>= (compare d start-date) 0)
(<= (compare d end-date) 0))
(<= (compare d end-date) 0))))
(reduce
(fn [acc [_ _ account location debit credit]]
(-> acc
(update-in [[location account] :debit] (fnil + 0.0) debit)
(update-in [[location account] :credit] (fnil + 0.0) credit)
(update-in [[location account] :count] (fnil + 0) 1))
)
{})
(reduce-kv
(fn [acc [location account-id] {:keys [debit credit count]}]
(let [account (lookup-account account-id)
account-type (:account_type account)]
(conj acc (merge {:id (str account-id "-" location)
:location (or location "")
:amount (if account-type (if (#{:account-type/asset
:account-type/dividend
:account-type/expense} account-type)
(- debit credit)
(- credit debit)
)
0.0)}
account)))
)
[]))))
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db (d/connect uri) )]})))
bank-accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :bank-account/name {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(d/db (d/connect uri))]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
(map (fn [o]
[[[(:db/id a) (:db/id (:account-client-override/client o))]]
(:account-client-override/name o)])
(:account/client-overrides a))
) )
(into {} ))]
(fn [a]
{:name (or (:bank-account/name (bank-accounts a))
(overrides-by-client [a client-id])
(:account/name (accounts a)))
:account_type (or (:db/ident (:account/type (accounts a)))
({:bank-account-type/check :account-type/asset
:bank-account-type/credit :account-type/liability}
(:db/ident (:bank-account/type (bank-accounts a)))))
:numeric_code (or (:account/numeric-code (accounts a))
(and (#{:bank-account-type/check} (:db/ident (:bank-account/type (bank-accounts a))))
11100)
(and (#{:bank-account-type/credit} (:db/ident (:bank-account/type account)))
28000))
:account-type (or (:db/ident (:account/type account))
({:bank-account-type/check :asset
:bank-account-type/credit :liability}
(:db/ident (:bank-account/type account))))
:amount (reduce + 0 (map
(fn [line-item]
(cond
(and (credit-account? account) (:journal-entry-line/debit line-item))
(- (:journal-entry-line/debit line-item))
(and (#{:bank-account-type/credit} (:db/ident (:bank-account/type (bank-accounts a))))
28000))})))
(and (credit-account? account) (:journal-entry-line/credit line-item))
(:journal-entry-line/credit line-item)
(and (debit-account? account) (:journal-entry-line/debit line-item))
(:journal-entry-line/debit line-item)
(and (debit-account? account) (:journal-entry-line/credit line-item))
(- (:journal-entry-line/credit line-item))
:else
0.0))
line-items))}))
[])))
(defn full-ledger-for-client [client-id]
(->> (d/query
{:query {:find ['?d '?jel '?account '?location '?debit '?credit ]
:in ['$ '?client-id]
:where ['[?e :journal-entry/client ?client-id]
'[?e :journal-entry/date ?d]
'[?e :journal-entry/line-items ?jel]
'[(get-else $ ?jel :journal-entry-line/account :account/unknown) ?account]
'[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit ]
'[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
'[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db (d/connect uri)) client-id]})
(sort-by first)))
(defn get-balance-sheet [context args value]
(let [args (assoc args :id (:id context))
client (d-clients/get-by-id (:client_id args))
[results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date (:date args))}
:count Integer/MAX_VALUE})
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
end-date (coerce/to-date (:date args))
comparable-date (coerce/to-date (time/minus (:date args) (time/years 1)))
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
[comparable-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date (time/minus (:date args) (time/years 1)))}
:count Integer/MAX_VALUE})]
(println "count" (take 3 comparable-results))
(->graphql
{:balance-sheet-accounts (roll-up client results)
:comparable-balance-sheet-accounts (roll-up client comparable-results)})))
{:balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries end-date)
:comparable-balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries comparable-date)})))
(defn get-profit-and-loss [context args value]
(let [args (assoc args :id (:id context))
client (d-clients/get-by-id (:client_id args))
pnl (fn [from-date to-date]
(println "FROM" from-date to-date)
(let [[starting-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (-> from-date
(time/minus (time/seconds 1))
coerce/to-date)}
:count Integer/MAX_VALUE})
[ending-results] (l/get-graphql {:client-id (:client_id args)
:date-range {:end (coerce/to-date to-date)}
:count Integer/MAX_VALUE})
starting-accounts (by :id (roll-up client starting-results))
ending-accounts (by :id (roll-up client ending-results))]
(reduce-kv
(fn [results k v]
(conj results (update v :amount (fn [amt]
(- amt
(get-in starting-accounts [k :amount] 0))))))
[]
ending-accounts)))]
(->graphql
{:balance-sheet-accounts (pnl (:start (:date_range args))
(:end (:date_range args)))
:comparable-balance-sheet-accounts (pnl (time/minus (:start (:date_range args)) (time/years 1))
(time/minus (:end (:date_range args)) (time/years 1)))})))
(let [client-id (:client_id args)
_ (assert-can-see-client (:id context) client-id)
start-date (coerce/to-date (:start (:date_range args)))
end-date (coerce/to-date (:end (:date_range args)))
comparable-start-date (coerce/to-date (time/minus (:start (:date_range args)) (time/years 1)))
comparable-end-date (coerce/to-date (time/minus (:end (:date_range args)) (time/years 1)))
all-ledger-entries (full-ledger-for-client client-id)
lookup-account (build-account-lookup client-id)]
(->graphql
{:balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries end-date start-date )
:comparable-balance-sheet-accounts (roll-up-until lookup-account all-ledger-entries comparable-end-date comparable-start-date )})))
#_(get-profit-and-loss nil {:client_id [:client/code "CBC"]
:from_date "2018-01-01"
:to_date "2019-04-01"} nil)
(defn assoc-error [f]
(fn [entry]

View File

@@ -16,6 +16,11 @@
(str/includes? (str header) "Closed Date")
:sysco-style-1
(str/includes? (str header) "Business Unit")
:mission
(str/includes? (str header) "Document Number")
:philz
:else
nil)
@@ -100,6 +105,45 @@
[]
rows))
(defmethod parse-csv :mission
[rows]
(transduce
(comp (drop 1)
(map (fn [[ po-number despatch-number invoice-number invoice-date customer value :as row]]
{:vendor-code "Mama Lu's Foods"
:customer-identifier customer
:invoice-number (str po-number "-" invoice-number )
:date (parse-date-fallover invoice-date ["M/d/yyyy HH:ss" "M/d/yyyy HH:mm:ss aa" "M/d/yyyy"])
:total (str/replace value #"," "")
:text (str/join " " row)
:full-text (str/join " " row)})))
conj
[]
rows))
(defmethod parse-csv :philz
[rows]
(transduce
(comp
(filter (fn [[dt _ doc-number name _ status _ _ amount :as row]]
(= status "Billed")))
(map (fn [[dt _ doc-number name _ _ _ _ amount :as row]]
(print name)
{:vendor-code "PHILZ COFFEE, INC"
:customer-identifier name
:invoice-number doc-number
:date (some-> dt not-empty (parse-date-fallover ["MM/dd/yyyy"]))
:total (str/replace amount #"," "")
:text (str/join " " row)
:full-text (str/join " " row)}))
)
conj
[]
(drop 1 rows)))
(defmethod parse-csv nil
[rows]
nil)

View File

@@ -427,6 +427,46 @@
:invoice-number #"(\S+)\s+(?=[0-9]+/[0-9]+/[0-9]+)"
:total #"(?:INVOICE|TOTAL|CREDIT)\s+([\d\.,\-]+\.[\d\-]+( CR)?)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}}
;; ROMA BAKERY
{:vendor "Roma Bakery Inc."
:keywords [#"Roma Bakery Inc"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice (\d+)"
:total #"Total\s+([\d\-\.]+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}
;; KAEL FOODS
{:vendor "Kael Foods"
:keywords [#"kaelfoods.com"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To.*\n\s+(.*?)\s{2,}"
:invoice-number #"INVOICE 0*(\d+)"
:total #"TOTAL:\s+\$([\d\-\.,]+)"}
:parser {:date [:clj-time "MM/dd/yyyy"]
:total [:trim-commas-and-negate nil]}}
;; Starter Bakery
{:vendor "Starter Bakery"
:keywords [#"starterbakery.com"]
:extract {:date #"INVOICE DATE:\s+(.*?)\s{2,}"
:customer-identifier #"BILL TO:.*\n\s+(.*?)\s{2,}"
:invoice-number #"Invoice.*?(\d+)"
:total #"Total:.*?([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MMMM dd, yyyy"]
:total [:trim-commas-and-negate nil]}}
;; Trimark
{:vendor "TriMark R.W. Smith"
:keywords [#"TriMark"]
:extract {:date #"([0-9]+/[0-9]+/[0-9]+)"
:customer-identifier #"Bill To\s+(.*?)\s{2,}"
:invoice-number #"Invoice #\n.*?([\d\-]+)\n"
:total #"Invoice Total\s+([\d\-,]+\.\d{2,2}+)"}
:parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}])
(defn offset [c x y]

View File

@@ -246,6 +246,8 @@
(defonce in-memory-cache (atom []))
(defonce break? (atom false))
(defn load-in-memory-cache []
(future
(loop []
@@ -253,8 +255,9 @@
(reset! in-memory-cache (get-provider-accounts-with-accounts))
(catch Exception e
(println e)))
(Thread/sleep (* 60 * 1000 * 5))
(recur))))
(Thread/sleep (* 30 1000 5))
(when-not @break?
(recur)))))
(defn refresh-provider-account [id]
(swap! in-memory-cache

View File

@@ -4,7 +4,17 @@
[clojure.string :as str]))
(s/def ::client (s/nilable map?))
(s/def ::description (s/nilable string?))
(s/def ::description (s/nilable (s/and string?
#( #?@(:clj (try
(re-pattern %)
true
(catch Exception _
false))
:cljs (try
(re-pattern %)
true
(catch js/Error _
false)))))))
(s/def ::amount-gte (s/nilable double?))
(s/def ::amount-lte (s/nilable double?))
(s/def ::dom-gte (s/nilable int?))

View File

@@ -222,119 +222,120 @@
(defn form [{:keys [can-change-amount?] :as params}]
[layouts/side-bar {:on-close (dispatch-event [::forms/form-closing ::form ])}
(let [{:keys [data active? error id]} @(re-frame/subscribe [::forms/form ::form])
{:keys [form field raw-field error-notification submit-button ]} rule-form
{:keys [form-inline field raw-field error-notification submit-button ]} rule-form
default-note @(re-frame/subscribe [::default-note])
exists? (:id data)]
^{:key id}
[form (assoc params :title "New Transaction Rule")
(form-inline (assoc params :title "New Transaction Rule")
[:<>
[field "Client"
(field "Client"
[typeahead-entity {:matches @(re-frame/subscribe [::subs/clients])
:auto-focus true
:match->text :name
:type "typeahead-entity"
:field [:client]
:spec ::entity/client}]]
:spec ::entity/client}])
[field "Bank account"
(field "Bank account"
[typeahead-entity {:matches @(re-frame/subscribe [::subs/real-bank-accounts-for-client (:client data)])
:match->text :name
:type "typeahead-entity"
:field [:bank-account]
:spec ::entity/bank-account}]]
:spec ::entity/bank-account}])
[field "Yodlee Merchant"
(field "Yodlee Merchant"
[typeahead-entity {:matches @(re-frame/subscribe [::subs/yodlee-merchants])
:match->text #(do (println %) (str (:name %) " - " (:yodlee-id %)))
:match->text #(str (:name %) " - " (:yodlee-id %))
:type "typeahead-entity"
:field [:yodlee-merchant]}]]
:field [:yodlee-merchant]}])
[field [:span "Description (" [:a {:href "https://regex101.com" :target "_new"} "regex tester"] ")" ]
(field [:span "Description (" [:a {:href "https://regex101.com" :target "_new"} "regex tester"] ")" ]
[:input.input {:type "text"
:field [:description]
:spec ::entity/description}]]
:spec ::entity/description}])
[:div.field
[:p.help "Amount"]
[:div.control
[:div.columns
[:div.column
[raw-field
(raw-field
[:input.input {:type "number"
:placeholder ">="
:field [:amount-gte]
:spec ::entity/amount-gte
:step "0.01"}]]]
:step "0.01"}])]
[:div.column
[raw-field
(raw-field
[:input.input {:type "number"
:placeholder "<="
:field [:amount-lte]
:spec ::entity/amount-lte
:step "0.01"}]]]]]]
:step "0.01"}])]]]]
[:div.field
[:p.help "Day of Month"]
[:div.control
[:div.columns
[:div.column
[raw-field
(raw-field
[:input.input {:type "number"
:placeholder ">="
:field [:dom-gte]
:spec ::entity/dom-gte
:precision 0
:step "1"}]]]
:step "1"}])]
[:div.column
[raw-field
(raw-field
[:input.input {:type "number"
:placeholder "<="
:field [:dom-lte]
:spec ::entity/dom-lte
:precision 0
:step "1"}]]]]]]
:step "1"}])]]]]
[:h2.title.is-4 "Outcomes"]
[field "Assign Vendor"
(field "Assign Vendor"
[typeahead-entity {:matches @(re-frame/subscribe [::subs/all-vendors])
:match->text :name
:type "typeahead-entity"
:field [:vendor]
:spec ::entity/vendor}]]
:spec ::entity/vendor}])
[field nil
(field nil
[expense-accounts-field {:type "expense-accounts"
:descriptor "account asssignment"
:percentage-only? true
:client (:client data)
:locations (into ["Shared"] @(re-frame/subscribe [::subs/locations-for-client-or-bank-account (:id (:client data)) (:id (:bank-account data))]))
:max 100
:field [:accounts]}]]
:field [:accounts]}])
[field "Approval Status"
(field "Approval Status"
[button-radio
{:type "button-radio"
:field [:transaction-approval-status]
:options [[:unapproved "Unapproved"]
[:requires-feedback "Client Review"]
[:approved "Approved"]
[:excluded "Excluded from Ledger"]]}]]
[:excluded "Excluded from Ledger"]]}])
[field "Note"
(field "Note"
[:input.input {:type "text"
:field [:note]
:placeholder default-note
:spec (s/nilable ::entity/note)}]]
:spec (s/nilable ::entity/note)}])
[:div.is-divider]
[error-notification]
(error-notification)
[:div.columns
[:div.column
[:a.button.is-medium.is-fullwidth.is-outlined {:on-click (dispatch-event [::test-clicked])} "Test Rule"]]
[:div.column
[submit-button "Save"]]]])])
(submit-button "Save")]]]))])

View File

@@ -118,7 +118,10 @@
(re-frame/reg-event-fx
::authenticated
(fn [{:keys [db]} [_ authentication]]
{:dispatch [::mounted]}))
{:db (-> db
(assoc-in [::yodlee :authentication] authentication)
(assoc-in [::yodlee :loading?] false))}
))
(re-frame/reg-event-fx
::save-error