notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Makes integreat run on datomic cloud

Browse Source
This commit is contained in:
Bryce Covert
2022-08-23 12:13:12 -07:00
parent 58b9dcf272
commit d02fba2b44
58 changed files with 2163 additions and 1257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
src/clj/auto_ap/datomic.clj
View File

@@ -1,25 +1,31 @@
(ns auto-ap.datomic
(:require
[auto-ap.utils :refer [default-pagination-size]]
[auto-ap.utils :refer [default-pagination-size by]]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[datomic.api :as d]
[mount.core :as mount]))
[datomic.client.api :as dc]
[mount.core :as mount])
(:import
(java.util UUID)))
(def uri (:datomic-url env))
(mount/defstate client
:start (dc/client {:server-type :dev-local
:system "dev"})
:stop nil)
(mount/defstate conn
:start (d/connect uri)
:stop (d/release conn))
:start (dc/connect client {:db-name "prod-migration"})
:stop nil)
#_(def uri "datomic:mem://datomic-transactor:4334/invoice")
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn create-database []
#_(defn create-database []
(d/create-database uri))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn drop-database []
#_(defn drop-database []
(d/delete-database uri))
(defn remove-nils [m]
@@ -35,26 +41,6 @@
result
nil)))
(defn replace-nils-with-retract [updated original]
(let [result (reduce-kv
(fn [[m & retractions] k v]
(cond (and (nil? v)
(not (nil? (get original k))))
(into [m] (conj retractions [:db/retract (:db/id original) k (or (:db/id (get original k))
(get original k))]))
(nil? v)
(into [m] retractions)
:else
(into [(assoc m k v)] retractions)))
[{}]
updated)]
(if (seq result)
result
nil)))
(def vendor-schema
[{:db/ident :vendor/original-id
:db/valueType :db.type/long
@@ -101,8 +87,7 @@
:db/isComponent true
:db/cardinality :db.cardinality/one
:db/doc "The vendor's secondary contact"}
{:db/id #db/id[:db.part/db]
:db/ident :vendor/address
{:db/ident :vendor/address
:db/valueType :db.type/ref
:db/cardinality :db.cardinality/one
:db/isComponent true
@@ -588,6 +573,20 @@
q
(:sort args)))
(defn add-sorter-fields-2 [q sort-map args]
(log/info "sort-map" (pr-str sort-map))
(reduce
(fn [q {:keys [sort-key]}]
(merge-query q
{:query {:find [(last (last (sort-map
sort-key
(println "Warning, trying to sort by unsupported field" sort-key))))]
:where (sort-map
sort-key
(println "Warning, trying to sort by unsupported field" sort-key))}}))
q
(:sort args)))
(defn apply-sort-3 [args results]
(let [sort-bys (conj (:sort args)
{:sort-key "default" :asc (if (contains? args :default-asc?)
@@ -625,8 +624,7 @@
:audit/user (str (:user/role id) "-" (:user/name id))
:audit/batch batch-id})
_ (log/info "transacting batch " batch-id " " (count batch))
tx-result @(d/transact conn batch)
_ (Thread/sleep 100)]
tx-result (dc/transact conn {:tx-data batch})]
(cond-> full-tx
(:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result)))
@@ -639,5 +637,255 @@
(partition-all 50 txes))))
(defn audit-transact [txes id]
@(d/transact conn (conj txes {:db/id "datomic.tx"
:audit/user (str (:user/role id) "-" (:user/name id))})))
(dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx"
:audit/user (str (:user/role id) "-" (:user/name id))})}))
(defn pull-many [db read ids ]
(->> (dc/q '[:find (pull ?e r)
:in $ [?e ...] r]
db
ids
read)
(map first)))
(defn pull-many-by-id [db read ids ]
(into {}
(map (fn [[e]]
[(:db/id e) e]))
(dc/q '[:find (pull ?e r)
:in $ [?e ...] r]
db
ids
read)))
(defn random-tempid []
(str (UUID/randomUUID)))
(defn pull-attr [db k id]
(get (dc/pull db [k] id) k))
(defn pull-ref [db k id]
(:db/id (pull-attr db k id)))
(declare upsert-entity)
(defn reset-rels [db e a vs]
(assert (every? :db/id vs) (format "In order to reset attribute %s, every value must have :db/id" a))
(let [ids (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
new-id-set (set (map :db/id vs))
retract-ids (filter (complement new-id-set) ids)
{is-component? :db/isComponent} (dc/pull db [:db/isComponent] a)
new-rels (filter (complement (set ids)) (map :db/id vs))]
(-> []
(into (map (fn [i] (if is-component?
[:db/retractEntity i]
[:db/retract e a i ])) retract-ids))
(into (map (fn [i] [:db/add e a (:db/id i)]) new-rels))
(into (mapcat (fn [i] (upsert-entity db i)) vs)))))
(defn reset-scalars [db e a vs]
(let [extant (when-not (string? e)
(->> (dc/q '[:find ?z
:in $ ?e ?a
:where [?e ?a ?z]]
db e a)
(map first)))
retracts (filter (complement (set vs)) extant)
new (filter (complement (set extant)) vs)]
(-> []
(into (map (fn [i] [:db/retract e a i ]) retracts))
(into (map (fn [i] [:db/add e a i]) new)))))
(defn upsert-entity [db entity]
(assert (:db/id entity) "Cannot upsert without :db/id")
(let [e (:db/id entity)
is-new? (string? e)
extant-entity (when-not is-new?
(dc/pull db (keys entity) (:db/id entity)))
ident->value-type (by :db/ident (comp :db/ident
:db/valueType)
(pull-many
db
[:db/valueType :db/ident]
(keys entity)))
ops (->> entity
(reduce
(fn [ops [a v]]
(cond
(= :db/id a)
ops
(or (= v (a extant-entity))
(= v (:db/ident (a extant-entity) :nope))
(= v (:db/id (a extant-entity)) :nope))
ops
(and (nil? v)
(not (nil? (a extant-entity))))
(conj ops [:db/retract e a (cond-> (a extant-entity)
(:db/id (a extant-entity)) :db/id)])
(nil? v)
ops
;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq)
(and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v))
(into ops (reset-rels db e a v))
(and (sequential? v) (not= :db.type/ref (ident->value-type a)))
(into ops (reset-scalars db e a v))
(and (map? v)
(= :db.type/ref (ident->value-type a)))
(let [id (or (:db/id v) (random-tempid))]
(-> ops
(conj [:db/add e a id])
(into (upsert-entity db (assoc v :db/id id)))))
:else
(conj ops [:db/add e a v])
))
[]))]
ops))
(defn plus [db e a amount]
[[:db/add e a (-> (dc/pull db [a] e) a (+ amount))]])
(comment
(dc/pull (dc/db conn) '[*] 175921860633685)
(upsert-entity (dc/db conn) {:db/id 175921860633685 :invoice/invoice-number nil :invoice/date #inst "2021-01-01" :invoice/expense-accounts [:reset-rels [{:db/id "new" :invoice-expense-account/amount 1}]]})
(upsert-entity (dc/db conn) {:invoice/client #:db{:id 79164837221949},
:invoice/status #:db{:id 101155069755470, :ident :invoice-status/paid},
:invoice/due #inst "2020-12-23T08:00:00.000-00:00",
:invoice/invoice-number "12648",
:invoice/import-status
:import-status/imported,
:invoice/vendor nil,
:invoice/date #inst "2020-12-16T08:00:00.000-00:00",
:entity/migration-key 17592234924273,
:db/id 175921860633685,
:invoice/outstanding-balance 0.0,
:invoice/expense-accounts
[{:entity/migration-key 17592234924274,
:invoice-expense-account/location nil
:invoice-expense-account/amount 360.0,
:invoice-expense-account/account #:db{:id 92358976759248}}],})
#_(dc/pull (dc/db conn) auto-ap.datomic.clients 79164837221904)
(upsert-entity (dc/db conn) {:client/name "20Twenty - WG Development LLC",
:client/square-locations
[{:db/id 83562883711605,
:entity/migration-key 17592258901782,
:square-location/square-id "L2579ATQ0X1ET",
:square-location/name "20Twenty",
:square-location/client-location "WG"}],
:client/square-auth-token
"EAAAEEr749Ea6AdPTdngsmUPwIM3ETbPwcx3QQl_NS0KWuIL-JNzAg4f3W9DGQhb",
:client/bank-accounts
[{:bank-account/sort-order 2,
:bank-account/include-in-reports true,
:bank-account/number "3467",
:bank-account/code "20TY-WFCC3467",
:bank-account/locations ["WG"],
:entity/migration-key 17592245102834,
:bank-account/current-balance 11160.289999999979,
:bank-account/name "Wells Fargo CC - 3467",
:db/id 83562883732805,
:bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00",
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755504, :ident :bank-account-type/credit},
:bank-account/intuit-bank-account #:db{:id 105553116286744},
:bank-account/integration-status
{:db/id 74766790691480,
:entity/migration-key 17592267080690,
:integration-status/last-updated #inst "2022-08-23T03:47:44.892-00:00",
:integration-status/last-attempt #inst "2022-08-23T03:47:44.892-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}},
:bank-account/bank-name "Wells Fargo"}
{:bank-account/sort-order 0,
:bank-account/include-in-reports true,
:bank-account/numeric-code 11301,
:bank-account/check-number 301,
:bank-account/number "1734742859",
:bank-account/code "20TY-WF2882",
:bank-account/locations ["WG"],
:bank-account/bank-code "11-4288/1210 4285",
:entity/migration-key 17592241193004,
:bank-account/current-balance -47342.54000000085,
:bank-account/name "Wells Fargo Main - 2859",
:db/id 83562883732846,
:bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00",
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755468, :ident :bank-account-type/check},
:bank-account/intuit-bank-account #:db{:id 105553116286745},
:bank-account/routing "121042882",
:bank-account/integration-status
{:db/id 74766790691458,
:entity/migration-key 17592267080255,
:integration-status/last-updated #inst "2022-08-23T03:46:45.879-00:00",
:integration-status/last-attempt #inst "2022-08-23T03:46:45.879-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}},
:bank-account/bank-name "Wells Fargo"}
{:bank-account/sort-order 1,
:bank-account/include-in-reports true,
:bank-account/numeric-code 20101,
:bank-account/yodlee-account-id 27345526,
:bank-account/number "41006",
:bank-account/code "20TY-Amex41006",
:bank-account/locations ["WG"],
:entity/migration-key 17592241193006,
:bank-account/current-balance 9674.069999999963,
:bank-account/name "Amex - 41006",
:db/id 83562883732847,
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755504, :ident :bank-account-type/credit},
:bank-account/bank-name "American Express"}
{:bank-account/sort-order 3,
:bank-account/include-in-reports true,
:bank-account/numeric-code 11101,
:bank-account/code "20TY-0",
:bank-account/locations ["WG"],
:entity/migration-key 17592241193005,
:bank-account/current-balance 0.0,
:bank-account/name "CASH",
:db/id 83562883732848,
:bank-account/visible true,
:bank-account/type
#:db{:id 101155069755469, :ident :bank-account-type/cash}}],
:entity/migration-key 17592241193003,
:db/id 79164837221904,
:client/address
{:db/id 105553116285906,
:entity/migration-key 17592250661126,
:address/street1 "1389 Lincoln Ave",
:address/city "San Jose",
:address/state "CA",
:address/zip "95125"},
:client/code "NY",
:client/locations ["WE" "NG"],
:client/square-integration-status
{:db/id 74766790691447,
:entity/migration-key 17592267072653,
:integration-status/last-updated #inst "2022-08-23T13:09:16.082-00:00",
:integration-status/last-attempt #inst "2022-08-23T13:08:47.018-00:00",
:integration-status/state
#:db{:id 101155069755529, :ident :integration-state/success}}})
)

60
src/clj/auto_ap/datomic/accounts.clj
View File

@@ -1,9 +1,9 @@
(ns auto-ap.datomic.accounts
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]]
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query conn pull-many]]
[clojure.string :as str]
[datomic.api :as d]
[datomic.client.api :as dc]
[clojure.tools.logging :as log]))
(defn <-datomic [a]
@@ -37,21 +37,21 @@
(let [query (cond-> {:query {:find [(list 'pull '?e default-read)]
:in ['$]
:where [['?e :account/name]]}
:args [(d/db (d/connect uri))]}
:args [(dc/db conn)]}
(:account-set args) (merge-query {:query {:in ['?account-set]
:where [['?e :account/account-set '?account-set]]}
:args [(:account-set args)]}))]
(->>
(d/query query)
(dc/q query)
(map first)
(map <-datomic)))))
(defn get-by-id [id]
(let [query {:query {:find [(list 'pull '?e default-read)]
:in ['$ '?e]}
:args [(d/db (d/connect uri) ) id]}]
:args [(dc/db conn ) id]}]
(->>
(d/query query)
(dc/q query)
(map first)
(map <-datomic)
first)))
@@ -59,38 +59,40 @@
(defn get-for-vendor [vendor-id client-id]
(if client-id
(->>
(d/q [:find (list 'pull '?e default-read)
:in '$ '?v '?c
:where '(or-join [?v ?c ?e]
(and [?v :vendor/account-overrides ?ao]
[?ao :vendor-account-override/client ?c]
[?ao :vendor-account-override/account ?e])
(and [?v :vendor/account-overrides ?ao]
(not [?ao vendor-account-override/client ?c])
[?v :vendor/default-account ?e])
(and (not [?v :vendor/account-overrides])
[?v :vendor/default-account ?e]))]
(dc/q '[:find (pull ?e r)
:in $ ?v ?c r
:where (or-join [?v ?c ?e]
(and [?v :vendor/account-overrides ?ao]
[?ao :vendor-account-override/client ?c]
[?ao :vendor-account-override/account ?e])
(and [?v :vendor/account-overrides ?ao]
(not [?ao :vendor-account-override/client ?c])
[?v :vendor/default-account ?e])
(and (not [?v :vendor/account-overrides])
[?v :vendor/default-account ?e]))]
(d/db conn )
(dc/db conn )
vendor-id
client-id)
client-id
default-read)
(map first)
(map <-datomic)
first)
(d/q [:find (list 'pull '?e default-read)
:in '$ '?v
:where '[?v :vendor/default-account ?e]]
(<-datomic (dc/q '[:find (pull ?e r)
:in $ ?v r
:where [?v :vendor/default-account ?e]]
(d/db conn )
vendor-id)))
(dc/db conn )
vendor-id
default-read))))
(defn get-account-by-numeric-code-and-sets [numeric-code _]
(let [query (cond-> {:query {:find ['(pull ?e [* {:account/type [:db/ident :db/id]}])]
:in ['$ '?numeric-code]
:where ['[?e :account/numeric-code ?numeric-code]]}
:args [(d/db (d/connect uri)) numeric-code]})]
:args [(dc/db conn) numeric-code]})]
(->>
(d/query query)
(dc/q query)
(map first)
(map <-datomic)
(first))))
@@ -121,13 +123,13 @@
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
accounts (->> ids
(map results)
@@ -137,7 +139,7 @@
(defn get-graphql [args]
(log/info "ARGS" args)
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count]))

8
src/clj/auto_ap/datomic/bank_accounts.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.datomic.bank-accounts
(:require
[auto-ap.datomic :refer [uri]]
[datomic.api :as d]))
[auto-ap.datomic :refer [conn]]
[datomic.client.api :as dc]))
(defn add-arg [query name value where & rest]
(let [query (-> query
@@ -21,10 +21,10 @@
(defn get-by-id [id]
(->>
(d/query (-> {:query {:find [default-read]
(dc/q (-> {:query {:find [default-read]
:in ['$]
:where []}
:args [(d/db (d/connect uri))]}
:args [(dc/db conn)]}
(add-arg '?e id ['?e])))
(map first)
(<-datomic)

43
src/clj/auto_ap/datomic/checks.clj
View File

@@ -1,10 +1,17 @@
(ns auto-ap.datomic.checks
(:require [datomic.api :as d]
[auto-ap.datomic :refer [merge-query apply-sort-3 apply-pagination add-sorter-fields conn]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clojure.set :refer [rename-keys]]
[clj-time.coerce :as c]
[clojure.tools.logging :as log]))
(:require
[auto-ap.datomic
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-many]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[datomic.client.api :as dc]
[clojure.set :refer [rename-keys]]
[clojure.tools.logging :as log]))
(defn <-datomic [result]
(-> result
@@ -28,7 +35,7 @@
{:transaction/_payment [:db/id :transaction/date]}])
(defn raw-graphql-ids
([args] (raw-graphql-ids (d/db conn) args))
([args] (raw-graphql-ids (dc/db conn) args))
([db args]
(let [check-number-like (try (Long/parseLong (:check-number-like args)) (catch Exception _ nil))
query (cond-> {:query {:find []
@@ -148,12 +155,12 @@
(log/info query)
(cond->> query
true (d/query)
true (apply-sort-3 args)
true (dc/q)
true (apply-sort-3 (assoc args :default-asc? false))
true (apply-pagination args)))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
payments (->> ids
(map results)
@@ -163,7 +170,7 @@
(defn get-graphql [args]
(log/info (:id args))
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
@@ -174,13 +181,21 @@
(->> {:query {:find ['?e]
:in ['$ '[?e ...]]
:where ['[?e :payment/date]]}
:args [(d/db conn) ids]}
(d/query)
:args [(dc/db conn) ids]}
(dc/q)
(map first)
vec)
[]))
(defn get-by-id [id]
(->>
(d/pull (d/db conn) default-read id)
(dc/pull (dc/db conn) default-read id)
(<-datomic)))
(defn pay [db e amount]
(let [current-outstanding-balance (-> (dc/pull db [:invoice/outstanding-balance] e) :invoice/outstanding-balance)
new-outstanding-balance (- current-outstanding-balance amount)]
[[:db/add e :invoice/outstanding-balance new-outstanding-balance]
[:db/add e :invoice/status (if (> new-outstanding-balance 0)
:invoice-status/unpaid
:invoice-status/paid)]]))

36
src/clj/auto_ap/datomic/clients.clj
View File

@@ -1,8 +1,10 @@
(ns auto-ap.datomic.clients
(:require
[auto-ap.datomic :refer [conn uri]]
[auto-ap.datomic :refer [conn]]
[auto-ap.search :as search]
[clj-time.coerce :as coerce]
[datomic.api :as d]))
[clojure.string :as str]
[datomic.client.api :as dc]))
(defn cleanse [e]
(-> e
@@ -27,7 +29,7 @@
(update :bank-account/sort-order (fn [so] (or so i)))))
(range) bas)))))
(defn get-all []
(->> (d/q '[:find (pull ?e [*
(->> (dc/q '[:find (pull ?e [*
{:client/square-integration-status [:integration-status/message
:integration-status/last-attempt
:integration-status/last-updated
@@ -56,14 +58,13 @@
{:plaid-item/_client [*]}
{:client/emails [:db/id :email-contact/email :email-contact/description]}])
:where [?e :client/name]]
(d/db (d/connect uri)))
(dc/db conn))
(map first)
(map cleanse)))
(defn get-by-id [id]
(->>
(d/pull (d/db conn )
(dc/pull (dc/db conn )
'[* {:client/bank-accounts [* {:bank-account/type [*]
:bank-account/yodlee-account [:yodlee-account/name :yodlee-account/id :yodlee-account/number]
:bank-account/intuit-bank-account [:intuit-bank-account/name :intuit-bank-account/external-id :db/id]
@@ -79,9 +80,28 @@
(defn code->id [code]
(->>
(d/query (-> {:query {:find ['?e]
(dc/q (-> {:query {:find ['?e]
:in ['$ '?code]
:where [['?e :client/code '?code ]]}
:args [(d/db (d/connect uri)) code]}))
:args [(dc/db conn) code]}))
(first)
(first)))
(defn best-match [identifier]
(first (search/search-ids {:q (str/replace identifier #"[\(\)\-/\*\]\[\#:\&]" " ")} "client")))
(defn exact-match [identifier]
(first (search/search-ids {:exact-match (str/upper-case identifier)} "client")))
(defn rebuild-search-index []
(search/full-index-query
(for [result (map first (dc/qseq '[:find (pull ?v [:client/name :client/matches :db/id])
:in $
:where [?v :client/code]]
(dc/db conn)))
match (conj (or (:client/matches result) [])
(:client/name result))]
{:id (:db/id result)
:text match
:exact-match (str/upper-case match)})
"client"))

14
src/clj/auto_ap/datomic/expected_deposit.clj
View File

@@ -1,10 +1,10 @@
(ns auto-ap.datomic.expected-deposit
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]]
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn <-datomic [result]
(let [transaction (first (:transaction/_expected-deposit result))
@@ -88,12 +88,12 @@
:where ['[?e :expected-deposit/date ?sort-default]]}}))]
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
payments (->> ids
(map results)
@@ -101,7 +101,7 @@
(mapv <-datomic)
(map (fn get-totals [ed]
(assoc ed :totals
(->> (d/q '[:find ?d4 (count ?c) (sum ?a)
(->> (dc/q '[:find ?d4 (count ?c) (sum ?a)
:in $ ?ed
:where [?ed :expected-deposit/charges ?c]
[?c :charge/total ?a]
@@ -110,7 +110,7 @@
[(clj-time.coerce/from-date ?d) ?d2]
[(auto-ap.time/localize ?d2) ?d3]
[(clj-time.coerce/to-local-date ?d3) ?d4]]
(d/db conn)
(dc/db conn)
(:db/id ed))
(map (fn [[date count amount]]
{:date (c/to-date-time date)
@@ -119,7 +119,7 @@
payments))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))

110
src/clj/auto_ap/datomic/invoices.clj
View File

@@ -1,7 +1,13 @@
(ns auto-ap.datomic.invoices
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]]
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-many
remove-nils]]
[auto-ap.datomic.accounts :as d-accounts]
[auto-ap.datomic.vendors :as d-vendors]
[auto-ap.graphql.utils :refer [limited-clients]]
@@ -9,7 +15,7 @@
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.set :refer [rename-keys]]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(def default-read '[*
{:invoice/client [:client/name :db/id :client/locations :client/code]}
@@ -37,7 +43,7 @@
(defn raw-graphql-ids
([args]
(raw-graphql-ids (d/db conn) args))
(raw-graphql-ids (dc/db conn) args))
([db args]
(->> (cond-> {:query {:find []
:in ['$]
@@ -164,13 +170,13 @@
(merge-query {:query {:find ['?sort-default '?e ]
:where ['[?e :invoice/client]
'[?e :invoice/date ?sort-default]]}}) )
(d/query)
(dc/q)
(apply-sort-3 args)
(apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
invoices (->> ids
@@ -182,11 +188,11 @@
(defn sum-outstanding [ids]
(->>
(d/query {:query {:find ['?id '?o]
(dc/q {:query {:find ['?id '?o]
:in ['$ '[?id ...]]
:where ['[?id :invoice/outstanding-balance ?o]]
}
:args [(d/db conn)
:args [(dc/db conn)
ids]})
(map last)
(reduce
@@ -196,11 +202,11 @@
(defn sum-total-amount [ids]
(->>
(d/query {:query {:find ['?id '?o]
(dc/q {:query {:find ['?id '?o]
:in ['$ '[?id ...]]
:where ['[?id :invoice/total ?o]]
}
:args [(d/db conn)
:args [(dc/db conn)
ids]})
(map last)
(reduce
@@ -209,7 +215,7 @@
(defn get-graphql [args]
(let [db (d/db (d/connect uri))
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)
outstanding (sum-outstanding ids-to-retrieve)
total-amount (sum-total-amount ids-to-retrieve)]
@@ -221,34 +227,34 @@
total-amount]))
(defn get-by-id [id]
(-> (d/db (d/connect uri))
(d/pull default-read id)
(-> (dc/db conn)
(dc/pull default-read id)
(<-datomic)))
(defn get-multi [ids]
(map <-datomic
(-> (d/db (d/connect uri))
(d/pull-many default-read ids))))
(pull-many (dc/db conn) default-read ids )))
(defn find-conflicting [{:keys [:invoice/invoice-number :invoice/vendor :invoice/client :db/id]}]
(->> (d/query
(cond-> {:query {:find [(list 'pull '?e default-read)]
:in ['$ '?invoice-number '?vendor '?client '?invoice-id]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
(not [?e :invoice/status :invoice-status/voided])
[(not= ?e ?invoice-id)]]}
:args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]}))
(->> (dc/q
(cond-> {:query {:find [(list 'pull '?e default-read)]
:in ['$ '?invoice-number '?vendor '?client '?invoice-id]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
[?e :invoice/client ?client]
(not [?e :invoice/status :invoice-status/voided])
[(not= ?e ?invoice-id)]]}
:args [(dc/db conn) invoice-number vendor client (or id 0)]}))
(map first)
(map <-datomic)))
(defn get-existing-set []
(let [vendored-results (set (d/query {:query {:find ['?vendor '?client '?invoice-number]
(let [vendored-results (set (dc/q {:query {:find ['?vendor '?client '?invoice-number]
:in ['$]
:where '[[?e :invoice/invoice-number ?invoice-number]
[?e :invoice/vendor ?vendor]
@@ -256,8 +262,8 @@
(not [?e :invoice/status :invoice-status/voided])
]}
:args [(d/db (d/connect uri))]}))
vendorless-results (->> (d/query {:query {:find ['?client '?invoice-number]
:args [(dc/db conn)]}))
vendorless-results (->> (dc/q {:query {:find ['?client '?invoice-number]
:in ['$]
:where '[[?e :invoice/invoice-number ?invoice-number]
(not [?e :invoice/vendor])
@@ -265,7 +271,7 @@
(not [?e :invoice/status :invoice-status/voided])
]}
:args [(d/db (d/connect uri))]})
:args [(dc/db conn)]})
(mapv (fn [[client invoice-number]]
[nil client invoice-number]) )
set)]
@@ -276,37 +282,37 @@
(->> {:query {:find ['?e]
:in ['$ '[?e ...]]
:where ['[?e :invoice/date]]}
:args [(d/db conn) ids]}
(d/query)
:args [(dc/db conn) ids]}
(dc/q)
(map first)
vec)
[]))
(defn code-invoice [invoice]
(let [db (d/db auto-ap.datomic/conn)
(let [db (dc/db auto-ap.datomic/conn)
client-id (:invoice/client invoice)
vendor-id (:invoice/vendor invoice)
date (:invoice/date invoice)
vendor (d/pull db '[*] vendor-id)
vendor (dc/pull db '[*] vendor-id)
due (when (:vendor/terms vendor)
(-> date
(coerce/to-date-time)
(time/plus (time/days (d-vendors/terms-for-client-id vendor client-id)))
coerce/to-date))
automatically-paid? (boolean (seq (d/q '[:find [?c ...]
:in $ ?v ?c
:where [?v :vendor/automatically-paid-when-due ?c]]
automatically-paid? (boolean (seq (map first (dc/q '[:find ?c
:in $ ?v ?c
:where [?v :vendor/automatically-paid-when-due ?c]]
db
vendor-id
client-id))))
[schedule-payment-dom] (map first (dc/q '[:find ?dom
:in $ ?v ?c
:where [?v :vendor/schedule-payment-dom ?sp ]
[?sp :vendor-schedule-payment-dom/client ?c]
[?sp :vendor-schedule-payment-dom/dom ?dom]]
db
vendor-id
client-id)))
[schedule-payment-dom] (d/q '[:find [?dom ...]
:in $ ?v ?c
:where [?v :vendor/schedule-payment-dom ?sp ]
[?sp :vendor-schedule-payment-dom/client ?c]
[?sp :vendor-schedule-payment-dom/dom ?dom]]
db
vendor-id
client-id)
client-id))
scheduled-payment (cond automatically-paid?
due
@@ -324,3 +330,19 @@
true (assoc :invoice/expense-accounts [default-expense-account])
due (assoc :invoice/due due)
scheduled-payment (assoc :invoice/scheduled-payment scheduled-payment))))
(defn propose-invoice [db invoice]
(let [existing? (boolean (seq (dc/q '[:find ?i
:in $ ?invoice-number ?client ?vendor
:where
[?i :invoice/invoice-number ?invoice-number]
[?i :invoice/client ?client]
[?i :invoice/vendor ?vendor]
(not [?i :invoice/status :invoice-status/voided])]
db
(:invoice/invoice-number invoice)
(:invoice/client invoice)
(:invoice/vendor invoice))))]
(if existing?
[]
[(doto (remove-nils invoice) println)])))

27
src/clj/auto_ap/datomic/ledger.clj
View File

@@ -1,9 +1,16 @@
(ns auto-ap.datomic.ledger
(:require [datomic.api :as d]
[auto-ap.graphql.utils :refer [limited-clients]]
[auto-ap.datomic :refer [merge-query apply-sort-3 apply-pagination add-sorter-fields conn]]
[clj-time.coerce :as c]
[auto-ap.datomic.accounts :as d-accounts]))
(:require
[auto-ap.datomic
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-many]]
[auto-ap.datomic.accounts :as d-accounts]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[datomic.client.api :as dc]))
(defn raw-graphql-ids [db args]
(let [query (cond-> {:query {:find []
@@ -120,12 +127,12 @@
true
(merge-query {:query {:find ['?sort-default '?e] :where ['[?e :journal-entry/date ?sort-default]]}}))]
(->> query
(d/query)
(dc/q)
(apply-sort-3 (update args :sort conj {:sort-key "default-2" :asc true}))
(apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db '[* {:journal-entry/client [:client/name :client/code :db/id]
(let [results (->> (pull-many db '[* {:journal-entry/client [:client/name :client/code :db/id]
:journal-entry/vendor [:vendor/name :db/id]
:journal-entry/line-items [* {:journal-entry-line/account [*
{:account/type [*]}
@@ -154,7 +161,7 @@
(map first))))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count]))
@@ -164,8 +171,8 @@
(->> {:query {:find ['?e]
:in ['$ '[?e ...]]
:where ['[?e :journal-entry/date]]}
:args [(d/db conn) ids]}
(d/query)
:args [(dc/db conn) ids]}
(dc/q)
(map first)
vec)
[]))

15
src/clj/auto_ap/datomic/reports.clj
View File

@@ -1,10 +1,15 @@
(ns auto-ap.datomic.reports
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]]
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-many]]
[auto-ap.graphql.utils :refer [can-see-client? limited-clients]]
[clj-time.coerce :as c]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn raw-graphql-ids [db args]
(let [query (cond-> {:query {:find []
@@ -33,12 +38,12 @@
true
(merge-query {:query {:find ['?sort-default '?e] :where ['[?e :report/created ?sort-default]]}}))]
(->> query
(d/query)
(dc/q)
(apply-sort-3 (update args :sort conj {:sort-key "default-2" :asc true}))
(apply-pagination args))))
(defn graphql-results [ids db args]
(let [results (->> (d/pull-many db '[:db/id :report/client :report/created :report/url :report/name :report/creator]
(let [results (->> (pull-many db '[:db/id :report/client :report/created :report/url :report/name :report/creator]
ids)
(map #(update % :report/created c/from-date))
(group-by :db/id))]
@@ -53,7 +58,7 @@
(map :db/id (:report/client r))))))))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))

120
src/clj/auto_ap/datomic/sales_orders.clj
View File

@@ -1,10 +1,18 @@
(ns auto-ap.datomic.sales-orders
(:require [auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[datomic.api :as d]
[clojure.tools.logging :as log]
[clojure.set :as set]))
(:require
[auto-ap.datomic
:refer [add-sorter-fields-2
apply-pagination
apply-sort-3
merge-query
pull-many
conn]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[clojure.set :as set]
[clojure.tools.logging :as log]
[datomic.client.api :as dc]
))
(defn <-datomic [result]
(-> result
@@ -21,12 +29,96 @@
{:sales-order/client [:client/name :db/id :client/code]
:sales-order/charges [* {:charge/processor [:db/ident]} {:expected-deposit/_charges [:db/id]}]}])
#_(defn client-index-search [db attr client-ids [start end]]
(for [c client-ids
d
(dc/q '[:find ?e
:in $ ?start ?end
:where [?e :sales-order/client+date ?g]
[(>= ?g ?start)]
[(<= ?g ?end)]]
db [c start] [c end])
#_(dc/index-range db {:attrid attr
:start [c start]
:end [c end]})]
(first d)))
#_(comment
(do
(println "starting")
(doseq [n (partition-all 500 (dc/q '[:find ?s ?d
:where [?s :sales-order/date ?d]]
(dc/db conn)))]
(print ".")
(dc/transact conn {:tx-data (map (fn [[n d]] {:db/id n :sales-order/date d}) n)}))
(println "done"))
(time (count (client-index-search (dc/db conn)
:sales-order/client+date
[3575611821801781]
[#inst "2021-05-21T18:06:02.000-00:00"
#inst "2022-09-25T18:06:02.000-00:00"]
)))
(time (count (dc/q '[:find ?e
:in $ ?client ?start ?end
:where [?e :sales-order/client ?client]
[?e :sales-order/date ?date]
[(>= ?date ?start)]
[(<= ?date ?end)]]
(dc/db conn)
3575611821801781
#inst "2021-05-21T18:06:02.000-00:00"
#inst "2022-09-25T18:06:02.000-00:00"
)))
(time (do (count (dc/q '[:find ?e,
:in $ [[?s1 ?e1]]
:where
#_[(untuple ?period) [?s1 ?e1]]
[(q '[:find ?e
:in $ ?s1 ?e1
:where [?e :sales-order/client+date ?g]
[(>= ?g ?s1)]
[(<= ?g ?e1)]]
$ ?s1 ?e1) [?e ...]]
]
(dc/db conn) [[[3575611821801781 #inst "2022-08-21T18:06:02.000-00:00" ] [3575611821801781 #inst "2022-09-25T18:06:02.000-00:00"]]
[[3575611821801766 #inst "2022-08-21T18:06:02.000-00:00" ] [3575611821801766 #inst "2022-09-25T18:06:02.000-00:00"]]]))
(println "done")))
(dc/index-range (dc/db conn) {:attrid :sales-order/date+client3
:start [#inst "2022-08-25T18:06:02.000-00:00" ]
:end [#inst "2022-08-25T20:06:02.000-00:00" ]})
(dc/pull (dc/db conn) '[*] :sales-order/date+client3)
(dc/pull (dc/db conn) '[*] 5040161313201309)
#_(time (do (dc/q {:query '{:find [(count ?x)],
:in [$ ?client-id],
:where [[?e :sales-order/date+client3 ?x]]},
:args [(dc/db conn) 3575611821801781]})
(println "done")))
#_(datomic.dev-local.async/return-1)
(clojure.core.async/<!! (dca/transact conn {:tx-data [{:db/ident :sales-order/client+date
:db/valueType :db.type/tuple
:db/tupleAttrs [:sales-order/client :sales-order/date]
:db/cardinality :db.cardinality/one}]
:timeout 600000}))
)
(defn raw-graphql-ids [db args]
(let [query (cond-> {:query {:find []
:in ['$]
:where []}
:args [db]}
(:sort args) (add-sorter-fields {"client" ['[?e :sales-order/client ?c]
(:sort args) (add-sorter-fields-2 {"client" ['[?e :sales-order/client ?c]
'[?c :client/name ?sort-client]]
"location" ['[?e :sales-order/location ?sort-location]]
"source" ['[?e :sales-order/source ?sort-source]]
@@ -100,17 +192,17 @@
:args [(:total args)]})
true
(merge-query {:query {:find ['?sort-default '?e]
:where ['[?e :sales-order/date ?sort-default]]}}))]
(merge-query {:query {:find ['?date '?e]
:where ['[?e :sales-order/date ?date]]}}))]
(log/info "Sales query" query)
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 (assoc args :default-asc? false))
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
payments (->> ids
(map results)
@@ -121,19 +213,19 @@
(defn summarize-orders [ids]
(let [[total tax] (->>
(d/query {:query {:find ['(sum ?t) '(sum ?tax)]
(dc/q {:query {:find ['(sum ?t) '(sum ?tax)]
:with ['?id]
:in ['$ '[?id ...]]
:where ['[?id :sales-order/total ?t]
'[?id :sales-order/tax ?tax]]}
:args [(d/db (d/connect uri))
:args [(dc/db conn)
ids]})
first)]
{:total total
:tax tax}))
(defn get-graphql [args]
(let [db (d/db (d/connect uri))
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count

25
src/clj/auto_ap/datomic/transaction_rules.clj
View File

@@ -1,10 +1,15 @@
(ns auto-ap.datomic.transaction-rules
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]]
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-many]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clojure.string :as str]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn <-datomic [result]
result)
@@ -72,12 +77,12 @@
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
transaction-rules (->> ids
(map results)
@@ -86,19 +91,19 @@
transaction-rules))
(defn get-graphql [args]
(let [db (d/db (d/connect uri))
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count]))
(defn get-by-id [id]
(->>
(d/pull (d/db (d/connect uri)) default-read id)
(dc/pull (dc/db conn) default-read id)
(<-datomic)))
(defn get-all []
(mapv first
(d/query {:query {:find [(list 'pull '?e default-read )]
:in ['$]
:where ['[?e :transaction-rule/transaction-approval-status]]}
:args [(d/db (d/connect uri))]})))
(dc/q {:query {:find [(list 'pull '?e default-read )]
:in ['$]
:where ['[?e :transaction-rule/transaction-approval-status]]}
:args [(dc/db conn)]})))

54
src/clj/auto_ap/datomic/transactions.clj
View File

@@ -1,18 +1,18 @@
(ns auto-ap.datomic.transactions
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]]
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query conn pull-many]]
[auto-ap.datomic.accounts :as d-accounts]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as coerce]
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn potential-duplicate-ids [db args]
(when (and (:potential-duplicates args)
(:bank-account-id args))
(->> (d/q '[:find ?tx ?amount ?date
(->> (dc/q '[:find ?tx ?amount ?date
:in $ ?ba
:where
[?tx :transaction/bank-account ?ba]
@@ -33,7 +33,7 @@
(defn raw-graphql-ids
([args] (raw-graphql-ids (d/db conn) args))
([args] (raw-graphql-ids (dc/db conn) args))
([db args]
(let [potential-duplicates (potential-duplicate-ids db args)
query (cond-> {:query {:find []
@@ -169,28 +169,28 @@
'(not [?e :transaction/approval-status :transaction-approval-status/suppressed])]}}))]
(log/info "query is" query)
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 (assoc args :default-asc? false))
true (apply-pagination args)))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db '[* {:transaction/client [:client/name :db/id :client/code]
:transaction/approval-status [:db/ident :db/id]
:transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance]
:transaction/forecast-match [:db/id :forecasted-transaction/identifier]
:transaction/vendor [:db/id :vendor/name]
:transaction/matched-rule [:db/id :transaction-rule/note]
:transaction/payment [:db/id :payment/date]
:transaction/expected-deposit [:db/id :expected-deposit/date]
:transaction/accounts [:transaction-account/amount
:db/id
:transaction-account/location
{:transaction-account/account [:account/name :db/id
:account/location
{:account/client-overrides [:account-client-override/name
{:account-client-override/client [:db/id]}]}]}]
:transaction/yodlee-merchant [:db/id :yodlee-merchant/yodlee-id :yodlee-merchant/name]}]
ids)
(let [results (->> (pull-many db '[* {:transaction/client [:client/name :db/id :client/code]
:transaction/approval-status [:db/ident :db/id]
:transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance]
:transaction/forecast-match [:db/id :forecasted-transaction/identifier]
:transaction/vendor [:db/id :vendor/name]
:transaction/matched-rule [:db/id :transaction-rule/note]
:transaction/payment [:db/id :payment/date]
:transaction/expected-deposit [:db/id :expected-deposit/date]
:transaction/accounts [:transaction-account/amount
:db/id
:transaction-account/location
{:transaction-account/account [:account/name :db/id
:account/location
{:account/client-overrides [:account-client-override/name
{:account-client-override/client [:db/id]}]}]}]
:transaction/yodlee-merchant [:db/id :yodlee-merchant/yodlee-id :yodlee-merchant/name]}]
ids)
(map #(update % :transaction/date coerce/from-date))
(map #(update % :transaction/post-date coerce/from-date))
(map #(update % :transaction/accounts
@@ -203,7 +203,7 @@
(cond-> transaction
(:transaction/payment transaction) (update-in [:transaction/payment :payment/date] coerce/from-date)
(:transaction/expected-deposit transaction) (update-in [:transaction/expected-deposit :expected-deposit/date] coerce/from-date))
))
))
(map #(dissoc % :transaction/id))
(group-by :db/id))]
@@ -212,7 +212,7 @@
(map first))))
(defn get-graphql [args]
(let [db (d/db (d/connect uri))
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
@@ -223,15 +223,15 @@
(->> {:query {:find ['?e]
:in ['$ '[?e ...]]
:where ['[?e :transaction/date]]}
:args [(d/db conn) ids]}
(d/query)
:args [(dc/db conn) ids]}
(dc/q)
(map first)
vec)
[]))
(defn get-by-id [id]
(->
(d/pull (d/db (d/connect uri))
(dc/pull (dc/db conn)
'[* {:transaction/client [:client/name :db/id :client/code :client/locations]
:transaction/approval-status [:db/ident :db/id]
:transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance]

26
src/clj/auto_ap/datomic/users.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.datomic.users
(:require
[auto-ap.datomic :refer [uri]]
[datomic.api :as d]))
[auto-ap.datomic :refer [conn]]
[datomic.client.api :as dc]))
(defn add-arg [query name value where & rest]
(let [query (-> query
@@ -16,35 +16,35 @@
{:user/role [:db/ident]}])]
:in ['$]
:where []}
:args [(d/db (d/connect uri))]}
:args [(dc/db conn)]}
(add-arg '?e id ['?e]))]
(->> (d/query query)
(->> (dc/q query)
(map first)
(map #(update % :user/role :db/ident))
first)))
(defn find-or-insert! [{:keys [:user/provider :user/provider-id] :as new-user}]
(let [is-first-user? (not (seq (d/query {:query [:find '?e
(let [is-first-user? (not (seq (dc/q {:query [:find '?e
:in '$
:where '[?e :user/provider]]
:args [(d/db (d/connect uri))]})))
:args [(dc/db conn)]})))
user (some-> {:query [:find '(pull ?e [*
{:user/clients [*]}
{:user/role [:db/ident]}])
:in '$ '?provider '?provider-id
:where '[?e :user/provider ?provider]
'[?e :user/provider-id ?provider-id]]
:args [(d/db (d/connect uri)) provider provider-id]}
(d/query)
:args [(dc/db conn) provider provider-id]}
(dc/q)
first
first
(update :user/role :db/ident))]
(if user
user
(let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user
true (assoc :db/id "user")
is-first-user? (assoc :user/role :user-role/admin))])]
(let [new-user-trans (dc/transact conn {:tx-data [(cond-> new-user
true (assoc :db/id "user")
is-first-user? (assoc :user/role :user-role/admin))]})]
(get-by-id (-> new-user-trans :tempids (get "user")))))))
(defn raw-graphql [_]
@@ -53,9 +53,9 @@
{:user/role [:db/ident]}])]
:in ['$]
:where ['[?e :user/role]]}
:args [(d/db (d/connect uri))]})]
:args [(dc/db conn)]})]
(->> (d/query query)
(->> (dc/q query)
(map first)
(map #(update % :user/role :db/ident))
)))

20
src/clj/auto_ap/datomic/vendors.clj
View File

@@ -1,9 +1,9 @@
(ns auto-ap.datomic.vendors
(:require
[auto-ap.datomic :refer [conn merge-query uri add-sorter-fields apply-pagination merge-query apply-sort-3]]
[auto-ap.datomic :refer [conn merge-query add-sorter-fields apply-pagination merge-query apply-sort-3 pull-many]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clojure.string :as str]
[datomic.api :as d]
[datomic.client.api :as dc]
[auto-ap.datomic.accounts :as d-accounts]))
(defn <-datomic [a]
@@ -63,7 +63,7 @@
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
@@ -83,7 +83,7 @@
))
(defn graphql-results [ids db args]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))
vendors (->> ids
(map results)
@@ -93,7 +93,7 @@
vendors))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count])
@@ -104,8 +104,8 @@
(->> (cond-> {:query {:find [(list 'pull '?e default-read)]
:in ['$ '?e]
:where ['[?e :vendor/name]]}
:args [(d/db (d/connect uri)) id]})
(d/query)
:args [(dc/db conn) id]})
(dc/q)
(map first)
(map #(cleanse (:id args) %))
(map <-datomic)
@@ -114,7 +114,7 @@
(defn get-by-id [id]
(->> (d/q '[:find (pull ?e [*
(->> (dc/q '[:find (pull ?e [*
{:vendor/default-account [:account/name :db/id :account/location]
:vendor/legal-entity-tin-type [:db/ident :db/id]
:vendor/legal-entity-1099-type [:db/ident :db/id]
@@ -125,7 +125,7 @@
:vendor/automatically-paid-when-due [:db/id :client/name]}])
:in $ ?e
:where [?e]]
(d/db (d/connect uri))
(dc/db conn)
id)
(map first)
(map <-datomic)
@@ -165,3 +165,5 @@
client-id)))
first
boolean))

10
src/clj/auto_ap/datomic/yodlee2.clj
View File

@@ -1,10 +1,10 @@
(ns auto-ap.datomic.yodlee2
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]]
:refer [add-sorter-fields apply-pagination apply-sort-3 merge-query conn pull-many]]
[auto-ap.graphql.utils :refer [limited-clients]]
[clj-time.coerce :as c]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(def default-read '[*])
@@ -39,13 +39,13 @@
(merge-query {:query {:find ['?e ]
:where ['[?e :yodlee-provider-account/id]]}}) )
(d/query)
(dc/q)
(apply-sort-3 args)
(apply-pagination args)))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(let [results (->> (pull-many db default-read ids)
(group-by :db/id))]
(->> ids
(map results)
@@ -54,7 +54,7 @@
(defn get-graphql [args]
(let [db (d/db (d/connect uri))
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(->> (graphql-results ids-to-retrieve db args))
matching-count]))

8
src/clj/auto_ap/datomic/yodlee_merchants.clj
View File

@@ -1,14 +1,14 @@
(ns auto-ap.datomic.yodlee-merchants
(:require
[auto-ap.datomic :refer [uri]]
[datomic.api :as d]))
[auto-ap.datomic :refer [conn]]
[datomic.client.api :as dc]))
(defn get-merchants [_]
;; TODO admin?
(let [query {:query {:find ['(pull ?e [:yodlee-merchant/name :yodlee-merchant/yodlee-id :db/id])]
:in ['$]
:where [['?e :yodlee-merchant/name]]}
:args [(d/db (d/connect uri))]}]
:args [(dc/db conn)]}]
(->>
(d/query query)
(dc/q query)
(mapv first))))

26
src/clj/auto_ap/ezcater/core.clj
View File

@@ -35,13 +35,13 @@
:subscriptions))
(defn get-integrations []
(d/q '[:find [(pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid
:db/id
:ezcater-integration/integration-status [:db/id]]) ...]
:in $
:where [?i :ezcater-integration/api-key]]
(d/db conn)))
(map first (d/q '[:find (pull ?i [:ezcater-integration/api-key
:ezcater-integration/subscriber-uuid
:db/id
:ezcater-integration/integration-status [:db/id]])
:in $
:where [?i :ezcater-integration/api-key]]
(d/db conn))))
(defn mark-integration-status [integration integration-status]
@(d/transact conn
@@ -63,12 +63,12 @@
(defn upsert-used-subscriptions
([integration]
(let [extant (get-subscriptions integration)
to-ensure (set (d/q '[:find [?cu ...]
:in $
:where [_ :client/ezcater-locations ?el]
[?el :ezcater-location/caterer ?c]
[?c :ezcater-caterer/uuid ?cu]]
(d/db conn)))
to-ensure (set (map first (d/q '[:find ?cu
:in $
:where [_ :client/ezcater-locations ?el]
[?el :ezcater-location/caterer ?c]
[?c :ezcater-caterer/uuid ?cu]]
(d/db conn))))
to-create (set/difference
to-ensure
(set (map :parentId extant)))]

37
src/clj/auto_ap/graphql.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.graphql
(:require
[auto-ap.datomic :refer [merge-query uri]]
[auto-ap.datomic :refer [merge-query conn]]
[auto-ap.datomic.users :as d-users]
[auto-ap.graphql.accounts :as gq-accounts]
[auto-ap.graphql.checks :as gq-checks]
@@ -34,7 +34,8 @@
[com.walmartlabs.lacinia :refer [execute]]
[com.walmartlabs.lacinia.parser :as p]
[com.walmartlabs.lacinia.schema :as schema]
[datomic.api :as d]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.client.api :as dc]
[unilog.context :as lc]
[yang.time :refer [time-it]])
(:import
@@ -137,7 +138,8 @@
:address
{:fields {:street1 {:type 'String}
{:fields {:id {:type :id}
:street1 {:type 'String}
:street2 {:type 'String}
:city {:type 'String}
:state {:type 'String}
@@ -432,7 +434,8 @@
:email {:type 'String}
:phone {:type 'String}}}
:add_address
{:fields {:street1 {:type 'String}
{:fields {:id {:type :id}
:street1 {:type 'String}
:street2 {:type 'String}
:city {:type 'String}
:state {:type 'String}
@@ -629,7 +632,7 @@
(let [result (cond-> {:query {:find ['?account '?account-name '(sum ?amount)]
:in ['$]
:where []}
:args [(d/db (d/connect uri)) client_id]}
:args [(dc/db conn) client_id]}
client_id (merge-query {:query {:in ['?c]}
:args [client_id]})
@@ -641,7 +644,7 @@
'[?account :account/name ?account-name]
'[?expense-account :invoice-expense-account/amount ?amount]]}})
true (d/query))]
true (dc/q))]
(for [[account-id account-name total] result]
{:account {:id account-id :name account-name} :total total})))
@@ -649,7 +652,7 @@
(let [result (cond-> {:query {:find ['?name '(sum ?outstanding-balance) '(sum ?total)]
:in ['$]
:where []}
:args [(d/db (d/connect uri)) client_id]}
:args [(dc/db conn) client_id]}
client_id (merge-query {:query {:in ['?c]}
:args [client_id]})
(not client_id) (merge-query {:query {:where ['[?c :client/name]]}})
@@ -662,7 +665,7 @@
'[(.between java.time.temporal.ChronoUnit/DAYS (java.time.Instant/now) ?d2 ) ?d3]
'[(auto-ap.graphql/categorize ?d3) ?name]]}})
true (d/query))
true (dc/q))
result (group-by first result)]
(for [[id name] [[:due "Due"] [:due-30 "0-30 days"] [:due-60 "31-60 days"] [:due-later ">60 days"]]
@@ -689,13 +692,13 @@
(defn get-cash-flow [_ {:keys [client_id]} _]
(when client_id
(let [{:client/keys [week-a-credits week-a-debits week-b-credits week-b-debits forecasted-transactions ]} (d/pull (d/db (d/connect uri)) '[*] client_id)
(let [{:client/keys [week-a-credits week-a-debits week-b-credits week-b-debits forecasted-transactions ]} (dc/pull (dc/db conn) '[*] client_id)
total-cash (reduce
(fn [total [credit debit]]
(- (+ total credit)
debit))
0.0
(d/query {:query {:find '[?debit ?credit]
(dc/q {:query {:find '[?debit ?credit]
:in '[$ ?client]
:where ['[?j :journal-entry/client ?client]
'[?j :journal-entry/line-items ?je]
@@ -703,8 +706,8 @@
'[?ba :bank-account/type :bank-account-type/check]
'[(get-else $ ?je :journal-entry-line/debit 0.0) ?debit]
'[(get-else $ ?je :journal-entry-line/credit 0.0) ?credit]]}
:args [(d/db (d/connect uri)) client_id]}))
bills-due-soon (d/query {:query {:find '[?due ?outstanding ?invoice-number ?vendor-id ?vendor-name]
:args [(dc/db conn) client_id]}))
bills-due-soon (dc/q {:query {:find '[?due ?outstanding ?invoice-number ?vendor-id ?vendor-name]
:in '[$ ?client ?due-before]
:where ['[?i :invoice/client ?client]
'[?i :invoice/status :invoice-status/unpaid]
@@ -714,11 +717,11 @@
'[?i :invoice/invoice-number ?invoice-number]
'[?i :invoice/vendor ?vendor-id]
'[?vendor-id :vendor/name ?vendor-name]]}
:args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]})
:args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]})
outstanding-checks (reduce
+
0.0
(map first (d/query {:query {:find '[?amount]
(map first (dc/q {:query {:find '[?amount]
:in '[$ ?client ?due-before]
:where ['[?p :payment/client ?client]
'[?p :payment/status :payment-status/pending]
@@ -726,14 +729,14 @@
'(or
[?p :payment/type :payment-type/debit]
[?p :payment/type :payment-type/check])]}
:args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]})))
recent-fulfillments (d/query {:query {:find '[?f ?d]
:args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]})))
recent-fulfillments (dc/q {:query {:find '[?f ?d]
:in '[$ ?client ?min-date]
:where ['[?t :transaction/forecast-match ?f]
'[?t :transaction/date ?d]
'[?t :transaction/client ?client]
'[(>= ?d ?min-date)]]}
:args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/months -2)))]})
:args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/months -2)))]})
forecasted-transactions (for [{:forecasted-transaction/keys [amount identifier day-of-month]
:db/keys [id]} forecasted-transactions
month (range -1 7)

106
src/clj/auto_ap/graphql/accounts.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.graphql.accounts
(:require
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
[auto-ap.datomic :refer [audit-transact conn upsert-entity]]
[auto-ap.datomic.accounts :as d-accounts]
[auto-ap.graphql.utils
:refer [->graphql
@@ -11,7 +11,12 @@
enum->keyword
is-admin?
result->page]]
[datomic.api :as d]))
[auto-ap.search :as search]
[datomic.client.api :as dc]
[clojure.string :as str])
(:import
(org.apache.lucene.search TermInSetQuery)
(org.apache.lucene.util BytesRef)))
(defn get-graphql [context args _]
(assert-admin (:id context))
@@ -33,48 +38,38 @@
(defn upsert-account [context args _]
(let [{{:keys [id client-overrides numeric-code location applicability account-set name invoice-allowance vendor-allowance type]} :account} (<-graphql args)]
(when-not id
(when (seq (d/query {:query {:find ['?e]
:in '[$ ?account-set ?numeric-code]
:where ['[?e :account/account-set ?account-set]
'[?e :account/numeric-code ?numeric-code]]}
:args [(d/db conn) account-set numeric-code]}))
(when (seq (dc/q {:query {:find ['?e]
:in '[$ ?account-set ?numeric-code]
:where ['[?e :account/account-set ?account-set]
'[?e :account/numeric-code ?numeric-code]]}
:args [(dc/db conn) account-set numeric-code]}))
(throw (ex-info (str "Account set " account-set " already has an account for code " numeric-code)
{} ))))
(let [original (when id
(d/entity (d/db conn) id))
result (audit-transact (cond->
[(remove-nils
{:db/id (or id "new-account")
:account/name name
:account/search-terms name
:account/type (keyword "account-type" (clojure.core/name type))
:account/applicability (or (enum->keyword applicability "account-applicability")
:account-applicability/global)
:account/invoice-allowance (some-> invoice-allowance (enum->keyword "allowance"))
:account/vendor-allowance (some-> vendor-allowance (enum->keyword "allowance"))
:account/default-allowance :allowance/allowed
:account/account-set account-set
:account/location location
:account/numeric-code (when-not id
numeric-code)
:account/code (when-not id
(str numeric-code))})
[:reset (or id "new-account") :account/client-overrides
(mapv
(fn [client-override]
(remove-nils
{:db/id (:id client-override)
:account-client-override/client (:client-id client-override)
:account-client-override/name (:name client-override)
:account-client-override/search-terms (:name client-override)}))
client-overrides)]]
(and (not location) (:account/location original)) (conj [:db/retract (or id "new-account") :account/location (:account/location original)]))
(let [result (audit-transact [`(upsert-entity
~{:db/id (or id "new-account")
:account/name name
:account/search-terms name
:account/type (keyword "account-type" (clojure.core/name type))
:account/applicability (or (enum->keyword applicability "account-applicability")
:account-applicability/global)
:account/invoice-allowance (some-> invoice-allowance (enum->keyword "allowance"))
:account/vendor-allowance (some-> vendor-allowance (enum->keyword "allowance"))
:account/default-allowance :allowance/allowed
:account/account-set account-set
:account/location location
:account/numeric-code numeric-code
:account/code (str numeric-code)
:account/client-overrides (mapv
(fn [client-override]
{:db/id (:id client-override)
:account-client-override/client (:client-id client-override)
:account-client-override/name (:name client-override)
:account-client-override/search-terms (:name client-override)})
client-overrides)})]
(:id context))]
(->graphql
(d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))))))
(d-accounts/get-by-id (or id (get-in result [:tempids "new-account"])))))))
(def search-pattern [:db/id
:account/numeric-code
@@ -86,6 +81,7 @@
(when client
(assert-can-see-client (:id context) client))
(let [query (cleanse-query query)
_ (println query)
num (some-> (re-find #"([0-9]+)" query)
second
(not-empty )
@@ -160,3 +156,35 @@
(sequence xform)))
[])))
(defn rebuild-search-index []
(search/full-index-query
(for [result (map first (dc/qseq '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}])
:in $
:where [?aco :account-client-override/client ]
[?aco :account-client-override/search-terms ]]
(dc/db conn)))
:when (:account/numeric-code (:account/_client-overrides result))]
{:id (:db/id (:account/_client-overrides result))
:text (:account-client-override/search-terms result)
:client (str (:db/id (:account-client-override/client result)))
:numeric-code (:account/numeric-code (:account/_client-overrides result))
:location (:account/location (:account/_client-overrides result))})
"account-client-override")
(search/full-index-query
(for [result (map first (dc/qseq '[:find (pull ?a [:account/numeric-code
:account/search-terms
{:account/applicability [:db/ident]}
:db/id
:account/location])
:in $
:where [?a :account/search-terms ]]
(dc/db conn)))
:when (:account/search-terms result)
]
{:id (:db/id result)
:text (:account/search-terms result)
:numeric-code (:account/numeric-code result)
:location (:account/location result)
:applicability (name (:db/ident (:account/applicability result)))})
"account"))

18
src/clj/auto_ap/graphql/checks.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.graphql.checks
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.datomic :refer [conn remove-nils plus]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.bank-accounts :as d-bank-accounts]
[auto-ap.datomic.checks :as d-checks]
@@ -34,9 +34,9 @@
[com.brunobonacci.mulog :as mu]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[config.core :refer [env]]
[datomic.api :as d]
[digest]
[clj-time.coerce :as coerce])
[clj-time.coerce :as coerce]
[datomic.client.api :as dc]
[digest])
(:import
(java.io ByteArrayOutputStream)
(java.text DecimalFormat)
@@ -230,7 +230,7 @@
[{:invoice-payment/payment (-> invoice :invoice/vendor :db/id str)
:invoice-payment/amount invoice-amount
:invoice-payment/invoice (:db/id invoice)}
[:pay (:db/id invoice) invoice-amount]])
`(d-checks/pay ~(:db/id invoice) ~invoice-amount)])
(reduce into [])))
(defn base-payment [invoices vendor client bank-account _ _ invoice-amounts]
@@ -391,7 +391,6 @@
(let [type (keyword "payment-type" (name type))
invoices (d-invoices/get-multi (map :invoice-id invoice-payments))
client (d-clients/get-by-id client-id)
invoice-amounts (by :invoice-id :amount invoice-payments)
invoices-grouped-by-vendor (group-by (comp :db/id :invoice/vendor) invoices)
vendors (->> (d/pull-many (d/db conn) d-vendors/default-read (keys invoices-grouped-by-vendor))
@@ -409,14 +408,16 @@
(reduce into [])
doall))
checks (if (= type :payment-type/check)
(conj checks [:inc (:db/id bank-account) :bank-account/check-number (count invoices-grouped-by-vendor)])
(conj checks `(plus ~(:db/id bank-account) ~:bank-account/check-number ~(count invoices-grouped-by-vendor)))
checks)]
(when (= type :payment-type/check)
(mu/trace ::making-pdfs [:checks checks]
(make-pdfs (filter #(and (= :payment-type/check (:payment/type %))
(> (:payment/amount %) 0.0))
checks))))
(transact-with-ledger checks id)
(transact-with-ledger (map #(if (map? %)
(dissoc % :payment/pdf-data)
%) checks ) id)
{:invoices (d-invoices/get-multi (map :invoice-id invoice-payments))
@@ -565,7 +566,6 @@
(log/info "Voiding " (count all-ids) args)
(void-payments-internal all-ids (:id context))
{:message (str "Succesfully voided " (count all-ids))}))
(defn get-all-payments [context args _]

242
src/clj/auto_ap/graphql/clients.clj
View File

@@ -1,19 +1,19 @@
(ns auto-ap.graphql.clients
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
[auto-ap.datomic :refer [audit-transact conn upsert-entity random-tempid]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.square.core :as square]
[auto-ap.graphql.utils
:refer [->graphql assert-admin can-see-client? is-admin?]]
[auto-ap.routes.queries :as q]
[auto-ap.square.core :as square]
[clj-time.coerce :as coerce]
[clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]
[datomic.client.api :as dc]
[unilog.context :as lc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]
[com.brunobonacci.mulog :as mu])
@@ -22,10 +22,10 @@
(org.apache.commons.codec.binary Base64)))
(defn assert-client-code-is-unique [code]
(when (seq (d/query {:query {:find '[?id]
:in ['$ '?code]
:where ['[?id :client/code ?code]]}
:args [(d/db conn) code]}))
(when (seq (dc/q {:query {:find '[?id]
:in ['$ '?code]
:where ['[?id :client/code ?code]]}
:args [(dc/db conn) code]}))
(throw (ex-info "Client is not unique" {:validation-error (str "Client code '" code "' is not unique.")}))))
(defn upload-signature-data [signature-data]
@@ -44,9 +44,9 @@
(str "https://integreat-signature-images.s3.amazonaws.com/" signature-id ".jpg")))))
(defn assert-no-shared-transaction-sources [client-code txes]
(let [new-db (:db-after (d/with (d/db conn)
txes))]
(when (seq (->> (d/q '[:find ?src (count ?ba)
(let [new-db (:db-after (dc/with (dc/with-db conn)
{:tx-data txes}))]
(when (seq (->> (dc/q '[:find ?src (count ?ba)
:in $ ?c
:where [?c :client/bank-accounts ?ba]
(or
@@ -66,116 +66,84 @@
(let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client)))
id (or (:db/id client) "new-client")
signature-file (upload-signature-data (:signature_data edit_client))
_ (when client
(audit-transact (-> []
(into (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client)))
(into (mapv (fn [m] [:db/retract (:db/id client) :client/matches m]) (:client/matches client)))
(into (mapv (fn [m] [:db/retract (:db/id client) :client/feature-flags m]) (:client/feature-flags client))))
(:id context)))
reverts (-> []
(into (->> (:bank_accounts edit_client)
(filter #(nil? (:yodlee_account_id %)))
(filter #(:bank-account/yodlee-account-id (d/entity (d/db conn) (:id %))))
(map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account-id (:bank-account/yodlee-account-id (d/entity (d/db conn) (:id ba)))]))))
(into (->> (:bank_accounts edit_client)
(filter #(nil? (:yodlee_account %)))
(filter #(:bank-account/yodlee-account (d/entity (d/db conn) (:id %))))
(map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account (:db/id (:bank-account/yodlee-account (d/entity (d/db conn) (:id ba))))]))))
(into (->> (:bank_accounts edit_client)
(filter #(nil? (:intuit_bank_account %)))
(filter #(:bank-account/intuit-bank-account (d/entity (d/db conn) (:id %))))
(map (fn [ba] [:db/retract (:id ba) :bank-account/intuit-bank-account (:db/id (:bank-account/intuit-bank-account (d/entity (d/db conn) (:id ba))))]))))
(into (->> (:bank_accounts edit_client)
(filter #(nil? (:plaid_account %)))
(filter #(:bank-account/plaid-account (d/entity (d/db conn) (:id %))))
(map (fn [ba] [:db/retract (:id ba) :bank-account/plaid-account (:db/id (:bank-account/plaid-account (d/entity (d/db conn) (:id ba))))])))))
client-code (if (str/blank? (:client/code client))
(:code edit_client)
(:client/code client))
transactions (into [(remove-nils {:db/id id
:client/code client-code
:client/name (:name edit_client)
:client/matches (:matches edit_client)
:client/signature-file signature-file
:client/email (:email edit_client)
:client/locked-until (some-> (:locked_until edit_client) (coerce/to-date))
:client/locations (filter identity (:locations edit_client))
:client/week-a-debits (:week_a_debits edit_client)
:client/week-a-credits (:week_a_credits edit_client)
:client/week-b-debits (:week_b_debits edit_client)
:client/square-auth-token (:square_auth_token edit_client)
:client/square-locations (map
(fn [sl]
(remove-nils
{:db/id (:id sl)
:square-location/client-location (:client_location sl)}))
(:square_locations edit_client))
updated-entity {:db/id id
:client/code client-code
:client/name (:name edit_client)
:client/matches (:matches edit_client)
:client/signature-file signature-file
:client/email (:email edit_client)
:client/locked-until (some-> (:locked_until edit_client) (coerce/to-date))
:client/locations (filter identity (:locations edit_client))
:client/week-a-debits (:week_a_debits edit_client)
:client/week-a-credits (:week_a_credits edit_client)
:client/week-b-debits (:week_b_debits edit_client)
:client/square-auth-token (:square_auth_token edit_client)
:client/square-locations (map
(fn [sl]
{:db/id (or (:id sl) (random-tempid))
:square-location/client-location (:client_location sl)})
(:square_locations edit_client))
:client/ezcater-locations (map
(fn [el]
(remove-nils
{:db/id (:id el)
:ezcater-location/location (:location el)
:ezcater-location/caterer (:caterer el)}))
(:ezcater_locations edit_client))
:client/week-b-credits (:week_b_credits edit_client)
:client/location-matches (->> (:location_matches edit_client)
(filter (fn [lm] (and (:location lm) (:match lm))))
(map (fn [lm] {:location-match/location (:location lm)
:location-match/matches [(:match lm)]})))
:client/address (remove-nils {
:address/street1 (:street1 (:address edit_client))
:address/street2 (:street2 (:address edit_client))
:address/city (:city (:address edit_client))
:address/state (:state (:address edit_client))
:address/zip (:zip (:address edit_client))})
:client/feature-flags (:feature_flags edit_client)
:client/bank-accounts (map #(remove-nils
(cond-> {:db/id (:id %)
:bank-account/code (:code %)
:bank-account/bank-name (:bank_name %)
:bank-account/bank-code (:bank_code %)
:bank-account/start-date (-> (:start_date %) (coerce/to-date))
:bank-account/routing (:routing %)
:bank-account/include-in-reports (:include_in_reports %)
:client/emails (map (fn [e]
{:db/id (or (:id e)
(random-tempid))
:email-contact/email (:email e)
:email-contact/description (:description e)})
(:emails edit_client))
:bank-account/name (:name %)
:bank-account/visible (:visible %)
:bank-account/number (:number %)
:bank-account/check-number (:check_number %)
:bank-account/numeric-code (:numeric_code %)
:bank-account/sort-order (:sort_order %)
:bank-account/locations (:locations %)
:bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date %))
:client/feature-flags (:feature_flags edit_client)
:client/ezcater-locations (map
(fn [el]
{:db/id (or (:id el) (random-tempid))
:ezcater-location/location (:location el)
:ezcater-location/caterer (:caterer el)})
(:ezcater_locations edit_client))
:client/week-b-credits (:week_b_credits edit_client)
:client/location-matches (->> (:location_matches edit_client)
(filter (fn [lm] (and (:location lm) (:match lm))))
(map (fn [lm] {:db/id (or (:id lm ) (random-tempid))
:location-match/location (:location lm)
:location-match/matches [(:match lm)]})))
:client/address {:db/id (or (:id (:address edit_client)) (random-tempid))
:address/street1 (:street1 (:address edit_client))
:address/street2 (:street2 (:address edit_client))
:address/city (:city (:address edit_client))
:address/state (:state (:address edit_client))
:address/zip (:zip (:address edit_client))}
:client/bank-accounts (map (fn [ba]
{:db/id (:id ba)
:bank-account/code (:code ba)
:bank-account/bank-name (:bank_name ba)
:bank-account/bank-code (:bank_code ba)
:bank-account/start-date (-> (:start_date ba) (coerce/to-date))
:bank-account/routing (:routing ba)
:bank-account/include-in-reports (:include_in_reports ba)
:bank-account/yodlee-account-id (:yodlee_account_id %)
:bank-account/type (keyword "bank-account-type" (name (:type %)))}
(:yodlee_account %) (assoc :bank-account/yodlee-account [:yodlee-account/id (:yodlee_account %)])
(:plaid_account %) (assoc :bank-account/plaid-account (:plaid_account %))
(:intuit_bank_account %) (assoc :bank-account/intuit-bank-account (:intuit_bank_account %))))
(:bank_accounts edit_client))
:bank-account/name (:name ba)
:bank-account/visible (:visible ba)
:bank-account/number (:number ba)
:bank-account/check-number (:check_number ba)
:bank-account/numeric-code (:numeric_code ba)
:bank-account/sort-order (:sort_order ba)
:bank-account/locations (:locations ba)
:bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba))
})
[:reset id :client/emails (map #(remove-nils
{:db/id (or (:id %)
(str (UUID/randomUUID)))
:email-contact/email (:email %)
:email-contact/description (:description %)})
(:emails edit_client))]
[:reset id :client/forecasted-transactions (map #(remove-nils
{:db/id (:id %)
:forecasted-transaction/day-of-month (:day_of_month %)
:forecasted-transaction/identifier (:identifier %)
:forecasted-transaction/amount (:amount %)}
)
(:forecasted_transactions edit_client))]]
reverts)
_ (assert-no-shared-transaction-sources client-code transactions)
_ (log/info "upserting client" transactions)
result (audit-transact transactions (:id context))]
:bank-account/yodlee-account-id (:yodlee_account_id ba)
:bank-account/type (keyword "bank-account-type" (name (:type ba)))
:bank-account/yodlee-account (when (:yodlee_account ba)
[:yodlee-account/id (:yodlee_account ba)])
:bank-account/plaid-account (:plaid_account ba)
:bank-account/intuit-bank-account (:intuit_bank_account ba)})
(:bank_accounts edit_client))
}
_ (assert-no-shared-transaction-sources client-code [`(upsert-entity ~updated-entity)])
_ (log/info "upserting client" updated-entity)
result (audit-transact [`(upsert-entity ~updated-entity)] (:id context))]
(when (:square_auth_token edit_client)
(square/upsert-locations (-> result :tempids (get id) (or id) d-clients/get-by-id)))
(-> (-> result :tempids (get id) (or id) d-clients/get-by-id)
@@ -194,13 +162,13 @@
->graphql)))
(defn refresh-bank-account-current-balance [bank-account-id]
(let [all-transactions (d/query
(let [all-transactions (dc/q
{:query {:find ['?e '?debit '?credit]
:in ['$ '?bank-account-id]
:where '[[?e :journal-entry-line/account ?bank-account-id]
[(get-else $ ?e :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?e :journal-entry-line/credit 0.0) ?credit]]}
:args [(d/db conn) bank-account-id]})
:args [(dc/db conn) bank-account-id]})
debits (->> all-transactions
(map (fn [[_ debit _]]
debit))
@@ -209,46 +177,45 @@
(map (fn [[_ _ credit]]
credit))
(reduce + 0.0))
current-balance (if (= :bank-account-type/check (:bank-account/type (d/entity (d/db conn) bank-account-id)))
current-balance (if (= :bank-account-type/check (:db/ident (:bank-account/type (dc/pull (dc/db conn) [:bank-account/type] bank-account-id))))
(- debits credits)
(- credits debits))]
@(d/transact conn [{:db/id bank-account-id
:bank-account/current-balance current-balance}])))
(dc/transact conn {:tx-data [{:db/id bank-account-id
:bank-account/current-balance current-balance}]})))
(defn bank-accounts-needing-refresh []
(let [last-refreshed (->> (d/query
{:query {:find ['?ba '?tx]
(let [last-refreshed (->> (dc/q {:query {:find ['?ba '?tx]
:in ['$ ]
:where ['[?ba :bank-account/current-balance _ ?tx true]]}
:args [(d/history (d/db conn))]})
:args [(dc/history (dc/db conn))]})
(group-by first)
(map (fn [[ba balance-txes]]
[ba (->> balance-txes
(map second)
(sort-by #(- %))
first)])))
has-newer-transaction (->> (d/query
has-newer-transaction (->> (dc/q
{:query {:find ['?ba]
:in '[$ [[?ba ?last-refreshed] ...] ]
:where ['[_ :journal-entry-line/account ?ba ?tx]
'[(>= ?tx ?last-refreshed)]]}
:args [(d/history (d/db conn))
:args [(dc/history (dc/db conn))
last-refreshed]})
(map first)
(set))
no-current-balance (->> (d/query {:query {:find ['?ba]
no-current-balance (->> (dc/q {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]
'(not [?ba :bank-account/current-balance])
]}
:args [(d/db conn)]})
:args [(dc/db conn)]})
(map first))]
(into has-newer-transaction no-current-balance)))
(defn build-current-balance [bank-accounts]
(doseq [bank-account bank-accounts]
(log/info "Refreshing bank account" (-> (d/db conn) (d/entity bank-account) :bank-account/code))
(log/info "Refreshing bank account" (-> (dc/db conn) (dc/pull [:bank-account/code] bank-account)))
(try
(refresh-bank-account-current-balance bank-account)
(catch Exception e
@@ -257,10 +224,10 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn refresh-all-current-balance []
(lc/with-context {:source "current-balance-cache"}
(build-current-balance (->> (d/query {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]]}
:args [(d/db conn)]})
(build-current-balance (->> (dc/q {:query {:find ['?ba]
:in '[$]
:where ['[?ba :bank-account/code]]}
:args [(dc/db conn)]})
(map first)))))
(defn refresh-current-balance []
@@ -410,8 +377,8 @@
(defn setup-sales-queries [context args _]
(assert-admin (:id context))
(let [{client-code :client/code feature-flags :client/feature-flags} (d/pull (d/db conn) '[:client/code :client/feature-flags] (:client_id args))
is-new-square? ((set feature-flags) "new-square")]
(let [{client-code :client/code feature-flags :client/feature-flags} (dc/pull (dc/db conn) '[:client/code :client/feature-flags] (:client_id args))
is-new-square? ((set feature-flags) "new-square")]
(q/put-query (str (UUID/randomUUID))
(format sales-summary-query client-code)
(str "sales query for " client-code)
@@ -442,12 +409,11 @@
(str "refunds query for " client-code)
(str client-code "-refund")
[:client/code client-code])
(let [sales-summary-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-summary")]))
sales-category-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-category")]))
expected-deposit-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-expected-deposit")]))
tender-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-tender")]))
refund-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-refund")]))]
(let [sales-summary-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-summary")]))
sales-category-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-category")]))
expected-deposit-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-expected-deposit")]))
tender-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-tender")]))
refund-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-refund")]))]
{:message (str/join "\n"
[
(str "For " client-code ":")

15
src/clj/auto_ap/graphql/import_batch.clj
View File

@@ -1,14 +1,13 @@
(ns auto-ap.graphql.import-batch
(:require
[auto-ap.datomic
:refer
[add-sorter-fields apply-pagination apply-sort-3 conn merge-query]]
[auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many-by-id]]
[auto-ap.graphql.utils
:refer
[<-graphql assert-admin ident->enum-f result->page]]
[clj-time.coerce :as coerce]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]
[datomic.client.api :as dc]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
(def default-read '[:db/id
@@ -36,19 +35,17 @@
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(group-by :db/id))]
(let [results (pull-many-by-id db default-read ids)]
(->> ids
(map results)
(map first))))
(map results))))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(graphql-results ids-to-retrieve db args)
matching-count]))

6
src/clj/auto_ap/graphql/intuit_bank_accounts.clj
View File

@@ -2,11 +2,11 @@
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.graphql.utils :refer [->graphql assert-admin]]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn get-intuit-bank-accounts [context _ _]
(assert-admin (:id context))
(->graphql (map first (d/q '[:find (pull ?e [*])
(->graphql (map first (dc/q '[:find (pull ?e [*])
:in $
:where [?e :intuit-bank-account/external-id]]
(d/db conn)))))
(dc/db conn)))))

130
src/clj/auto_ap/graphql/invoices.clj
View File

@@ -1,6 +1,8 @@
(ns auto-ap.graphql.invoices
(:require
[auto-ap.datomic :refer [conn remove-nils uri]]
[auto-ap.datomic
:refer [conn remove-nils upsert-entity]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :as d-invoices]
[auto-ap.datomic.vendors :as d-vendors]
@@ -21,11 +23,12 @@
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.set :as set]
[clojure.tools.logging :as log]
[datomic.api :as d]
[manifold.deferred :as de]
[com.brunobonacci.mulog :as mu]))
[com.brunobonacci.mulog :as mu]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.client.api :as dc]))
(defn ->graphql [invoice user ]
(if (= "admin" (:user/role user))
@@ -61,7 +64,7 @@
(defn reject-invoices [context {:keys [invoices]} _]
(assert-power-user (:id context))
(doseq [i invoices]
(assert-can-see-client (:id context) (:db/id (:invoice/client (d/entity (d/db conn) i)))))
(assert-can-see-client (:id context) (:db/id (:invoice/client (dc/pull (dc/db conn) [{:invoice/client [:db/id]}] i)))))
(let [transactions (map (fn [i] [:db/retractEntity i]) invoices)]
(transact-with-ledger transactions (:id context))
invoices))
@@ -69,7 +72,9 @@
(defn approve-invoices [context {:keys [invoices]} _]
(assert-power-user (:id context))
(doseq [i invoices
:let [invoice (d/entity (d/db conn) i)]]
:let [invoice (dc/pull (dc/db conn) [{:invoice/client [:db/id]}
:invoice/date]
i)]]
(assert-can-see-client (:id context) (-> invoice :invoice/client :db/id))
(assert-not-locked (-> invoice :invoice/client :db/id) (-> invoice :invoice/date)))
(let [transactions (map (fn [i] {:db/id i :invoice/import-status :import-status/imported}) invoices)]
@@ -104,34 +109,25 @@
due))
_ (when-not (:db/id account)
(throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))]
(cond->
{:db/id "invoice"
:invoice/invoice-number invoice_number
:invoice/client client_id
:invoice/vendor vendor_id
:invoice/import-status :import-status/imported
:invoice/total total
:invoice/outstanding-balance total
:invoice/status :invoice-status/unpaid
:invoice/date (coerce/to-date date)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)}
due (assoc :invoice/due (coerce/to-date due))
scheduled_payment (assoc :invoice/scheduled-payment (coerce/to-date scheduled_payment)))))
(defn deleted-expense-accounts [invoice expense-accounts]
(let [current-expense-accounts (:invoice/expense-accounts invoice)
specified-ids (->> expense-accounts
(map :id)
set)
existing-ids (->> current-expense-accounts
(map :db/id)
set)]
(set/difference existing-ids specified-ids)))
`(upsert-entity ~{:db/id "invoice"
:invoice/invoice-number invoice_number
:invoice/client client_id
:invoice/vendor vendor_id
:invoice/import-status :import-status/imported
:invoice/total total
:invoice/outstanding-balance total
:invoice/status :invoice-status/unpaid
:invoice/date (coerce/to-date date)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)
:invoice/due (coerce/to-date due)
:invoice/scheduled-payment (coerce/to-date scheduled_payment)})))
(defn assert-valid-expense-accounts [expense_accounts vendor_id]
(doseq [expense-account expense_accounts
:let [account (d/entity (d/db conn) (:account_id expense-account))]]
:let [account (dc/pull (dc/db conn)
[:account/location]
(:account_id expense-account))]]
(when (empty? (:location expense-account))
(throw (ex-info "Expense account is missing location" {:validation-error "Expense account is missing location"})))
@@ -178,7 +174,7 @@
(assert-valid-expense-accounts expense_accounts vendor_id)
(assert-invoice-amounts-add-up in)
(let [transaction-result (transact-with-ledger [(add-invoice-transaction in)] (:id context))]
(let [transaction-result (transact-with-ledger (add-invoice-transaction in) (:id context))]
(-> (d-invoices/get-by-id (get-in transaction-result [:tempids "invoice"]))
(->graphql (:id context)))))
@@ -218,23 +214,21 @@
paid-amount (- (:invoice/total invoice) (:invoice/outstanding-balance invoice))
_ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
deleted (deleted-expense-accounts invoice expense_accounts)
_ (assert-not-locked (:db/id (:invoice/client invoice)) (:date in))
_ (assert-valid-expense-accounts expense_accounts vendor_id)
_ (assert-invoice-amounts-add-up in)
updated-invoice (cond-> {:db/id id
:invoice/invoice-number invoice_number
:invoice/date (coerce/to-date date)
updated-invoice {:db/id id
:invoice/invoice-number invoice_number
:invoice/date (coerce/to-date date)
:invoice/total total
:invoice/outstanding-balance (- total paid-amount)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)}
due (assoc :invoice/due (coerce/to-date due))
scheduled_payment (assoc :invoice/scheduled-payment (coerce/to-date scheduled_payment)))]
(transact-with-ledger (concat [updated-invoice]
(map (fn [d] [:db/retract id :invoice/expense-accounts d]) deleted))
:invoice/total total
:invoice/outstanding-balance (- total paid-amount)
:invoice/expense-accounts (map expense-account->entity
expense_accounts)
:invoice/due (coerce/to-date due)
:invoice/scheduled-payment (coerce/to-date scheduled_payment)}]
(transact-with-ledger [`(upsert-entity ~updated-invoice)]
(:id context))
(-> (d-invoices/get-by-id id)
(->graphql (:id context)))))
@@ -300,15 +294,15 @@
(log/info "Voiding " (count all-ids) args)
(transact-with-ledger
(->> all-ids
(d/q '[:find [(pull ?i [:db/id :invoice/date {:invoice/expense-accounts [:db/id]}
]) ...]
(dc/q '[:find (pull ?i [:db/id :invoice/date {:invoice/expense-accounts [:db/id]}])
:in $ [?i ...]
:where (not [_ :invoice-payment/invoice ?i])
[?i :invoice/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?i :invoice/date ?d]
[(>= ?d ?lu)]]
(d/db conn))
(dc/db conn))
(map first)
(mapcat
(fn [i]
(into
@@ -328,16 +322,15 @@
(let [invoice (d-invoices/get-by-id id)
_ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
_ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
conn (d/connect uri)
history (d/history (d/db conn))
txs (d/query {:query {:find ['?tx '?e '?original-status '?original-outstanding '?total '?ea '?ea-amount]
:where ['[?e :invoice/status :invoice-status/voided ?tx true]
'[?e :invoice/status ?original-status ?tx false]
'[?e :invoice/outstanding-balance ?original-outstanding ?tx false]
'[?e :invoice/total ?total ?tx false]
'[?ea :invoice-expense-account/amount ?ea-amount ?tx false]]
:in ['$ '?e]}
:args [history id]})
history (dc/history (dc/db conn))
txs (dc/q {:query {:find ['?tx '?e '?original-status '?original-outstanding '?total '?ea '?ea-amount]
:where ['[?e :invoice/status :invoice-status/voided ?tx true]
'[?e :invoice/status ?original-status ?tx false]
'[?e :invoice/outstanding-balance ?original-outstanding ?tx false]
'[?e :invoice/total ?total ?tx false]
'[?ea :invoice-expense-account/amount ?ea-amount ?tx false]]
:in ['$ '?e]}
:args [history id]})
[last-transaction] (->> txs (sort-by first) (last))]
(transact-with-ledger [(->> txs
(filter (fn [[tx]] (= tx last-transaction)))
@@ -354,7 +347,11 @@
(->graphql (:id context)))))
(defn unautopay-invoice [context {id :invoice_id} _]
(let [invoice (d/entity (d/db conn) id)]
(let [invoice (dc/pull (dc/db conn) [{:invoice/client [:db/id]}
:invoice-payment/_invoice
:invoice/total
:invoice/scheduled-payment
:invoice/date] id)]
(assert (:invoice/client invoice))
(assert-can-see-client (:id context) (:db/id (:invoice/client invoice)))
(assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
@@ -370,18 +367,15 @@
(defn edit-expense-accounts [context args _]
(assert-can-see-client (:id context) (:db/id (:invoice/client (d-invoices/get-by-id (:invoice_id args)))))
(let [invoice-id (:invoice_id args)
invoice (d-invoices/get-by-id invoice-id)
_ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
_ (assert-valid-expense-accounts (:expense_accounts args) (:db/id (:invoice/vendor invoice )))
deleted (deleted-expense-accounts invoice (:expense_accounts args))
updated {:db/id invoice-id
:invoice/expense-accounts (map
expense-account->entity
(:expense_accounts args))}]
invoice (d-invoices/get-by-id invoice-id)
_ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice))
_ (assert-valid-expense-accounts (:expense_accounts args) (:db/id (:invoice/vendor invoice )))]
(transact-with-ledger (concat [updated]
(map (fn [d] [:db/retract invoice-id :invoice/expense-accounts d]) deleted))
(:id context))
(transact-with-ledger [`(upsert-entity ~{:db/id invoice-id
:invoice/expense-accounts (map
expense-account->entity
(:expense_accounts args))})]
(:id context))
(->graphql
(d-invoices/get-by-id (:invoice_id args))
(:id context))))

49
src/clj/auto_ap/graphql/ledger.clj
View File

@@ -1,6 +1,7 @@
(ns auto-ap.graphql.ledger
(:require
[auto-ap.datomic :refer [audit-transact-batch conn remove-nils uri]]
[auto-ap.datomic
:refer [audit-transact-batch conn pull-attr pull-many remove-nils]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.ledger :as l]
@@ -17,7 +18,7 @@
[clojure.tools.logging :as log]
[clojure.data.csv :as csv]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]
[datomic.client.api :as dc]
[mount.core :as mount]
[com.brunobonacci.mulog :as mu]
[unilog.context :as lc]
@@ -135,19 +136,19 @@
[]))))
(defn build-account-lookup [client-id]
(let [accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :account/name
(let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name
:account/numeric-code
{:account/type [:db/ident]
:account/client-overrides [:account-client-override/client :account-client-override/name]}
])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db (d/connect uri) )]})))
:args [(dc/db conn )]})))
bank-accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])]
bank-accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])]
:in ['$]
:where ['[?e :bank-account/name]]}
:args [(d/db (d/connect uri))]})))
:args [(dc/db conn)]})))
overrides-by-client (->> accounts
vals
(mapcat (fn [a]
@@ -171,7 +172,7 @@
:client_id client-id})))
(defn full-ledger-for-client [client-id]
(->> (d/query
(->> (dc/q
{:query {:find ['?d '?jel '?account '?location '?debit '?credit]
:in ['$ '?client-id]
:where '[[?e :journal-entry/client ?client-id]
@@ -191,7 +192,7 @@
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db (d/connect uri)) client-id]})
:args [(dc/db conn) client-id]})
(sort-by first)))
(defn get-balance-sheet [context args _]
@@ -259,14 +260,15 @@
(defn all-ids-not-locked [all-ids]
(->> all-ids
(d/q '[:find [?t ...]
(dc/q '[:find ?t
:in $ [?t ...]
:where
[?t :journal-entry/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?t :journal-entry/date ?d]
[(>= ?d ?lu)]]
(d/db conn))))
(dc/db conn))
(map first)))
(defn delete-external-ledger [context args _]
(let [_ (assert-admin (:id context))
@@ -275,7 +277,7 @@
(assoc :only-external true)
(<-graphql)
(assoc :per-page Integer/MAX_VALUE)
(#(l/raw-graphql-ids (d/db conn) %))
(#(l/raw-graphql-ids (dc/db conn) %))
:ids)
specific-ids (l/filter-ids (:ids args))
all-ids (all-ids-not-locked (into (set ids) specific-ids))]
@@ -426,7 +428,6 @@
(:location ea)
"'")
{:status :error})))
(when (and matching-account
(not (:account/location matching-account))
(= "A" (:location ea)))
@@ -513,7 +514,7 @@
(defn running-balance-for [client-id]
(let [lookup-account (build-account-lookup client-id)]
(->> (d/query
(->> (dc/q
{:query {:find ['?d '?e '?jel '?account '?location '?debit '?credit]
:in ['$ '?client-id]
:where '[[?e :journal-entry/client ?client-id]
@@ -533,7 +534,7 @@
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
[(get-else $ ?jel :journal-entry-line/location "") ?location]]
}
:args [(d/db conn) client-id]})
:args [(dc/db conn) client-id]})
(sort-by (juxt first second))
(build-running-balance lookup-account))))
@@ -541,25 +542,23 @@
(defn build-running-balance-cache []
(let [clients-needing-refresh (if-let [last-run @last-run-running-balance]
(->> (d/query
(->> (dc/q
{:query {:find ['?v]
:in ['$ '?log '?since '?till]
:where ['[(tx-ids ?log ?since ?till) [?tx ...]]
'[$ _ :journal-entry/client ?v ?tx]]}
:args [(d/history (d/db conn))
(d/log conn)
last-run
(java.util.Date.)]})
:in ['$ '[?tx ...]]
:where ['[$ _ :journal-entry/client ?v ?tx]]}
:args [(dc/history (dc/db conn))
(map :t (mapcat :data (dc/tx-range conn {:start last-run
:end (java.util.Date.)})))]})
(map first)
(into #{}))
(into #{} (map :db/id (d-clients/get-all))))
(into #{}))
(into #{} (map :db/id (d-clients/get-all))))
starting (java.util.Date.)]
(log/info (count clients-needing-refresh) "Clients need their balance cache refreshed.")
(swap! running-balance-cache
merge
(reduce
(fn [acc client]
(log/info "Computing running balance cache for " (:client/code (d/entity (d/db conn) client)))
(log/info "Computing running balance cache for " (pull-attr (dc/db conn) :client/code client ))
(assoc acc client (running-balance-for client)))
{}
clients-needing-refresh))

47
src/clj/auto_ap/graphql/plaid.clj
View File

@@ -1,7 +1,13 @@
(ns auto-ap.graphql.plaid
(:require
[auto-ap.datomic
:refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]]
:refer [add-sorter-fields
apply-pagination
apply-sort-3
conn
merge-query
pull-attr
pull-many-by-id]]
[auto-ap.graphql.utils
:refer [->graphql
<-graphql
@@ -14,13 +20,15 @@
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.tools.logging :as log]
[datomic.client.api :as dc]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]))
(defn plaid-link-token [context value _]
(when-not (:client_id value)
(throw (ex-info "Client ID is required" {:validation-error "Client ID is required"})))
(assert-can-see-client (:id context) (:client_id value))
(let [client-code (:client/code (d/pull (d/db conn) [:client/code] (:client_id value)))]
(let [client-code (pull-attr (dc/db conn) :client/code (:client_id value))]
{:token (p/get-link-token client-code)}))
(defn link-plaid [context value _]
@@ -28,8 +36,8 @@
(throw (ex-info "Client not provided" {:validation-error "Client not provided."})))
(when-not (:public_token value)
(throw (ex-info "Public token not provided" {:validation-error "public token not provided"})))
(log/info (:id context) (:db/id (d/pull (d/db conn) [:db/id] [:client/code (:client_code value)])))
(assert-can-see-client (:id context) (:db/id (d/pull (d/db conn) [:db/id] [:client/code (:client_code value)])))
(log/info (:id context) (pull-attr (dc/db conn) :db/id [:client/code (:client_code value)]))
(assert-can-see-client (:id context) (pull-attr (dc/db conn) :db/id [:client/code (:client_code value)]))
(let [access-token (:access_token (p/exchange-public-token (:public_token value) (:client_code value)))
account-result (p/get-accounts access-token )
item {:plaid-item/client [:client/code (:client_code value)]
@@ -40,15 +48,16 @@
:plaid-item/last-updated (coerce/to-date (time/now))
:db/id "plaid-item"}]
@(d/transact conn (->> (:accounts account-result)
(map (fn [a]
(let [balance (some-> a :balances :current (* 0.01))]
(cond-> {:plaid-account/external-id (:account_id a)
:plaid-account/number (:mask a)
:plaid-account/name (str (:name a) " " (:mask a))
:plaid-item/_accounts "plaid-item"}
balance (assoc :plaid-account/balance balance)))))
(into [item])))
(dc/transact conn {:tx-data
(->> (:accounts account-result)
(map (fn [a]
(let [balance (some-> a :balances :current (* 0.01))]
(cond-> {:plaid-account/external-id (:account_id a)
:plaid-account/number (:mask a)
:plaid-account/name (str (:name a) " " (:mask a))
:plaid-item/_accounts "plaid-item"}
balance (assoc :plaid-account/balance balance)))))
(into [item]))})
(log/info "Access token was " access-token)
{:message (str "Plaid linked successfully.")}))
@@ -87,19 +96,17 @@
:where ['[?e :plaid-item/external-id]]}}))]
(cond->> query
true (d/query)
true (dc/q)
true (apply-sort-3 args)
true (apply-pagination args))))
(defn graphql-results [ids db _]
(let [results (->> (d/pull-many db default-read ids)
(group-by :db/id))]
(let [results (pull-many-by-id db default-read ids)]
(->> ids
(map results)
(map first))))
(map results))))
(defn get-graphql [args]
(let [db (d/db conn)
(let [db (dc/db conn)
{ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)]
[(graphql-results ids-to-retrieve db args)
matching-count]))
@@ -121,7 +128,7 @@
(defn delete-plaid-item [context args _]
(assert-admin (:id context))
(assert-present args :id)
@(d/transact conn [[:db/retractEntity (:id args)]])
(dc/transact conn {:tx-data [[:db/retractEntity (:id args)]]})
{:message "Item deleted."})
(defn attach [schema]

8
src/clj/auto_ap/graphql/reports.clj
View File

@@ -6,7 +6,7 @@
[auto-ap.graphql.utils
:refer [<-graphql assert-admin attach-tracing-resolvers result->page]]
[config.core :refer [env]]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn get-report-page [context args _]
(let [args (assoc args :id (:id context))
@@ -17,15 +17,15 @@
(defn delete-report [context args _]
(assert-admin (:id context))
(let [[id-to-delete key] (first (d/q '[:find ?i ?k
(let [[id-to-delete key] (first (dc/q '[:find ?i ?k
:in $ ?i
:where [?i :report/key ?k]]
(d/db conn)
(dc/db conn)
(:id args)))]
(when id-to-delete
(s3/delete-object :bucket-name (:data-bucket env)
:key key)
@(d/transact conn [[:db/retractEntity id-to-delete]]))
(dc/transact conn {:tx-data [[:db/retractEntity id-to-delete]]}))
{:message (format "deleted %s successfully" key)}))

64
src/clj/auto_ap/graphql/transaction_rules.clj
View File

@@ -1,11 +1,6 @@
(ns auto-ap.graphql.transaction-rules
(:require
[auto-ap.datomic
:refer [audit-transact
conn
merge-query
remove-nils
replace-nils-with-retract]]
[auto-ap.datomic :refer [audit-transact conn merge-query upsert-entity]]
[auto-ap.datomic.transaction-rules :as tr]
[auto-ap.datomic.transactions :as d-transactions]
[auto-ap.graphql.utils
@@ -20,7 +15,7 @@
[auto-ap.utils :refer [dollars=]]
[clj-time.coerce :as c]
[clojure.string :as str]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn get-transaction-rule-page [context args _]
(let [args (assoc args :id (:id context))
@@ -37,10 +32,10 @@
nil))
(defn transaction-rule-account->entity [{:keys [id account_id percentage location]}]
(remove-nils #:transaction-rule-account {:percentage percentage
:db/id id
:account account_id
:location location}))
#:transaction-rule-account {:percentage percentage
:db/id id
:account account_id
:location location})
(defn delete-transaction-rule [context {:keys [transaction_rule_id ]} _]
(assert-admin (:id context))
@@ -53,8 +48,7 @@
(defn upsert-transaction-rule [context {{:keys [id description yodlee_merchant_id note client_id bank_account_id amount_lte amount_gte vendor_id accounts transaction_approval_status dom_gte dom_lte]} :transaction_rule} _]
(assert-admin (:id context))
(let [existing-transaction (tr/get-by-id id)
account-total (reduce + 0 (map (fn [x] (:percentage x)) accounts))
(let [account-total (reduce + 0 (map (fn [x] (:percentage x)) accounts))
_ (try
(. java.util.regex.Pattern (compile description java.util.regex.Pattern/CASE_INSENSITIVE))
(catch Exception e
@@ -67,8 +61,8 @@
(let [error (str "You must provide a description or a yodlee merchant")]
(throw (ex-info error {:validation-error error}))))
_ (doseq [a accounts
:let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a))
client (d/entity (d/db conn) client_id)
:let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn) [:account/location :account/name] (:account_id a))
client (dc/pull (dc/db conn) [:client/locations] client_id)
]]
(when (and location (not= location (:location a)))
(let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)]
@@ -82,27 +76,25 @@
rule-id (if id
id
"transaction-rule")
transaction (replace-nils-with-retract #:transaction-rule {:db/id rule-id
:description description
:note note
:client client_id
:bank-account bank_account_id
:yodlee-merchant yodlee_merchant_id
:dom-lte dom_lte
:dom-gte dom_gte
:amount-lte amount_lte
:amount-gte amount_gte
:vendor vendor_id
:transaction-approval-status
(some->> transaction_approval_status
name
snake->kebab
(keyword "transaction-approval-status"))}
existing-transaction)
transaction [`(upsert-entity ~#:transaction-rule {:db/id rule-id
:description description
:note note
:client client_id
:bank-account bank_account_id
:yodlee-merchant yodlee_merchant_id
:dom-lte dom_lte
:dom-gte dom_gte
:amount-lte amount_lte
:amount-gte amount_gte
:vendor vendor_id
:transaction-approval-status
(some->> transaction_approval_status
name
snake->kebab
(keyword "transaction-approval-status"))
:transaction-rule/accounts (map transaction-rule-account->entity accounts)})]
transaction (conj transaction
[:reset rule-id :transaction-rule/accounts (map transaction-rule-account->entity accounts)])
transaction-result (audit-transact transaction (:id context))]
(-> (tr/get-by-id (or (-> transaction-result :tempids (get "transaction-rule"))
id))
@@ -111,14 +103,14 @@
(defn -test-transaction-rule [id {:keys [:transaction-rule/description :transaction-rule/client :transaction-rule/bank-account :transaction-rule/amount-lte :transaction-rule/amount-gte :transaction-rule/dom-lte :transaction-rule/dom-gte :transaction-rule/yodlee-merchant]} include-coded? count]
(->>
(d/query
(dc/q
(cond-> {:query {:find ['(pull ?e [* {:transaction/client [:client/name]
:transaction/bank-account [:bank-account/name]
:transaction/payment [:db/id]}
])]
:in ['$ ]
:where []}
:args [(d/db conn)]}
:args [(dc/db conn)]}
(limited-clients id)
(merge-query {:query {:in ['[?xx ...]]

141
src/clj/auto_ap/graphql/transactions.clj
View File

@@ -1,6 +1,14 @@
(ns auto-ap.graphql.transactions
(:require
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.datomic
:refer [audit-transact
audit-transact-batch
conn
pull-attr
pull-many
pull-ref
remove-nils
upsert-entity]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.invoices :as d-invoices]
@@ -27,6 +35,7 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.client.api :as dc]
[datomic.api :as d]
[auto-ap.graphql.utils :refer [attach-tracing-resolvers]]))
@@ -74,14 +83,15 @@
(defn all-ids-not-locked [all-ids]
(->> all-ids
(d/q '[:find [?t ...]
(dc/q '[:find ?t
:in $ [?t ...]
:where
[?t :transaction/client ?c]
[(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu]
[?t :transaction/date ?d]
[(>= ?d ?lu)]]
(d/db conn))))
(dc/db conn))
(map first)))
(defn bulk-change-status [context args _]
(let [_ (assert-admin (:id context))
args (assoc args :id (:id context))
@@ -140,11 +150,11 @@
(throw (ex-info "Client is required"
{:validation-error "client is required"})))
(let [args (assoc args :id (:id context))
locations (:client/locations (d/pull (d/db conn)
[:client/locations]
(:client_id args)))
locations (pull-attr (dc/db conn)
:client/locations
(:client_id args))
all-ids (all-ids-not-locked (get-ids-matching-filters args))
transactions (d/pull-many (d/db conn) '[:db/id :transaction/amount] (vec all-ids))
transactions (pull-many (dc/db conn) [:db/id :transaction/amount] (vec all-ids))
account-total (reduce + 0 (map (fn [x] (:percentage x)) (:accounts args)))]
(log/info "client is" locations)
@@ -156,7 +166,9 @@
(throw (ex-info error {:validation-error error}))))
(doseq [a (:accounts args)
:let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a))]]
:let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn)
[:account/location :account/name]
(:account_id a))]]
(when (and location (not= location (:location a)))
(let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)]
(throw (ex-info err {:validation-error err}) )))
@@ -184,12 +196,14 @@
(let [_ (assert-admin (:id context))
args (assoc args :id (:id context))
all-ids (all-ids-not-locked (get-ids-matching-filters args))
db (d/db conn)]
db (dc/db conn)]
(log/info "Deleting " (count all-ids) args)
(transact-batch-with-ledger
(mapcat (fn [i]
(let [transaction (d/entity db i)
(let [transaction (dc/pull db [:transaction/payment
:transaction/expected-deposit
:db/id] i)
payment-id (-> transaction :transaction/payment :db/id)
expected-deposit-id (-> transaction :transaction/expected-deposit :db/id)
transaction-tx (if (:suppress args)
@@ -233,7 +247,7 @@
(let [_ (assert-power-user (:id context))
args (assoc args :id (:id context))
transaction-id (:transaction_id args)
transaction (d/pull (d/db conn)
transaction (dc/pull (dc/db conn)
[:transaction/approval-status
:transaction/status
:transaction/date
@@ -249,12 +263,12 @@
(assert-not-locked (:db/id (:transaction/client transaction)) (-> transaction :transaction/payment :payment/date)))
_ (log/info "Unlinking" transaction)
payment (-> transaction :transaction/payment )
is-autopay-payment? (some->> (doto (d/query {:query {:find ['?sp]
is-autopay-payment? (some->> (doto (dc/q {:query {:find ['?sp]
:in ['$ '?payment]
:where ['[?ip :invoice-payment/payment ?payment]
'[?ip :invoice-payment/invoice ?i]
'[(get-else $ ?i :invoice/scheduled-payment "N/A") ?sp]]}
:args [(d/db conn) (:db/id payment)]})
:args [(dc/db conn) (:db/id payment)]})
log/info)
seq
(map first)
@@ -288,10 +302,10 @@
true
(into (map (fn [[invoice-payment]]
[:db/retractEntity invoice-payment])
(d/query {:query {:find ['?ip]
:in ['$ '?p]
:where ['[?ip :invoice-payment/payment ?p]]}
:args [(d/db conn) (:db/id payment)]} ))))
(dc/q {:query {:find ['?ip]
:in ['$ '?p]
:where ['[?ip :invoice-payment/payment ?p]]}
:args [(dc/db conn) (:db/id payment)]} ))))
(:id context))
(transact-with-ledger
(into (cond-> [{:db/id (:db/id payment)
@@ -314,25 +328,17 @@
(defn transaction-account->entity [{:keys [id account_id amount location]}]
(remove-nils #:transaction-account {:amount amount
:db/id id
:account account_id
:location location}))
#:transaction-account {:amount amount
:db/id id
:account account_id
:location location})
(defn deleted-accounts [transaction accounts]
(let [current-accounts (:transaction/accounts transaction)
specified-ids (->> accounts
(map :id)
set)
existing-ids (->> current-accounts
(map :db/id)
set)]
(set/difference existing-ids specified-ids)))
(defn assert-valid-expense-accounts [accounts]
(doseq [trans-account accounts
:let [account (d/entity (d/db conn) (:account_id trans-account))]]
:let [account (dc/pull (dc/db conn)
[:account/location]
(:account_id trans-account))]]
(when (empty? (:location trans-account))
(throw (ex-info "Account is missing location" {:validation-error "Account is missing location"})))
@@ -358,7 +364,6 @@
_ (assert-can-see-client (:id context) (:transaction/client existing-transaction) )
_ (assert-valid-expense-accounts accounts)
_ (assert-not-locked (:db/id (:transaction/client existing-transaction)) (:transaction/date existing-transaction))
deleted (deleted-accounts existing-transaction accounts)
account-total (reduce + 0 (map (fn [x] (:amount x)) accounts))
missing-locations (seq (set/difference
(->> accounts
@@ -376,27 +381,14 @@
(when missing-locations
(throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) )
(transact-with-ledger (concat [(remove-nils {:db/id id
:transaction/vendor vendor_id
:transaction/approval-status (some->> approval_status
name
snake->kebab
(keyword "transaction-approval-status"))
:transaction/accounts (map transaction-account->entity accounts)
})
]
(cond forecast_match
[[:db/add id :transaction/forecast-match forecast_match]]
(:db/id (:transaction/forecast-match existing-transaction))
[[:db/retract id :transaction/forecast-match (:db/id (:transaction/forecast-match existing-transaction))]]
:else
[])
(map (fn [d]
[:db/retract id :transaction/accounts d])
deleted))
(transact-with-ledger [`(upsert-entity ~{:db/id id
:transaction/vendor vendor_id
:transaction/approval-status (some->> approval_status
name
snake->kebab
(keyword "transaction-approval-status"))
:transaction/accounts (map transaction-account->entity accounts)
:transaction/forecast-match forecast_match})]
(:id context))
(-> (d-transactions/get-by-id id)
approval-status->graphql
@@ -440,9 +432,9 @@
(let [_ (assert-power-user (:id context))
transaction (d-transactions/get-by-id transaction_id)
_ (assert-can-see-client (:id context) (:transaction/client transaction) )
db (d/db conn)
invoice-clients (set (map (comp :db/id :invoice/client #(d/entity db %)) autopay_invoice_ids))
invoice-amount (reduce + 0.0 (map (comp :invoice/total #(d/entity db %)) autopay_invoice_ids))
db (dc/db conn)
invoice-clients (set (map #(pull-ref db :invoice/client %) autopay_invoice_ids))
invoice-amount (reduce + 0.0 (map #(pull-attr db :invoice/total %) autopay_invoice_ids))
_ (assert-not-locked (:db/id (:transaction/client transaction)) (:transaction/date transaction))]
(when (:transaction/payment transaction)
(throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"})))
@@ -454,22 +446,15 @@
(when-not (dollars= (- (:transaction/amount transaction))
invoice-amount)
(throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"})))
#_(log/info [#_(select-keys (d/entity db transaction_id) #{:transaction/amount :db/id})]
(->> autopay_invoice_ids
(map (fn [id]
(let [entity (d/entity db id)]
[(-> entity :invoice/vendor :db/id)
(-> entity :db/id)
(-> entity :invoice/total)])))))
(let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
(map (fn [id]
(let [entity (d/entity db id)]
[(-> entity :invoice/vendor :db/id)
(-> entity :db/id)
(-> entity :invoice/total)]))
autopay_invoice_ids)
(:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))]
(let [payment-tx (i-transactions/add-new-payment [(dc/pull db [:transaction/amount :transaction/date :db/id] transaction_id)]
(map (fn [id]
(let [entity (dc/pull db [:invoice/vendor :db/id :invoice/total] id)]
[(-> entity :invoice/vendor :db/id)
(-> entity :db/id)
(-> entity :invoice/total)]))
autopay_invoice_ids)
(:db/id (:transaction/bank-account transaction))
(:db/id (:transaction/client transaction)))]
(log/info "Adding a new payment" payment-tx)
(transact-with-ledger payment-tx (:id context)))
@@ -483,9 +468,9 @@
_ (assert-can-see-client (:id context) (:transaction/client transaction) )
_ (assert-not-locked (:db/id (:transaction/client transaction)) (:transaction/date transaction))
db (d/db conn)
invoice-clients (set (map (comp :db/id :invoice/client #(d/entity db %)) unpaid_invoice_ids))
invoice-amount (reduce + 0.0 (map (comp :invoice/outstanding-balance #(d/entity db %)) unpaid_invoice_ids))]
db (dc/db conn)
invoice-clients (set (map #(pull-ref db :invoice/client %) unpaid_invoice_ids))
invoice-amount (reduce + 0.0 (map #(pull-attr db :invoice/outstanding-balance %) unpaid_invoice_ids))]
(when (or (> (count invoice-clients) 1)
(not= (:db/id (:transaction/client transaction))
(first invoice-clients)))
@@ -498,9 +483,9 @@
(when (:transaction/payment transaction)
(throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"})))
(let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})]
(let [payment-tx (i-transactions/add-new-payment [(dc/pull db [:transaction/amount :transaction/date :db/id] transaction_id)]
(map (fn [id]
(let [entity (d/entity db id)]
(let [entity (dc/pull db [:invoice/vendor :db/id :invoice/total] id)]
[(-> entity :invoice/vendor :db/id)
(-> entity :db/id)
(-> entity :invoice/total)]))

4
src/clj/auto_ap/graphql/utils.clj
View File

@@ -4,7 +4,7 @@
[clj-time.coerce :as coerce]
[auto-ap.time :as atime]
[buddy.auth :refer [throw-unauthorized]]
[datomic.api :as d]
[datomic.client.api :as dc]
[clojure.walk :as walk]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[clojure.tools.logging :as log]
@@ -113,7 +113,7 @@
(some->> e name snake->kebab (keyword namespace)))
(defn get-locked-until [client-id]
(:client/locked-until (d/pull (d/db conn) [:client/locked-until] client-id)))
(:client/locked-until (dc/pull (dc/db conn) [:client/locked-until] client-id)))
(defn assert-not-locked [client-id date]
(let [locked-until (get-locked-until client-id)]

36
src/clj/auto_ap/graphql/vendors.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.graphql.vendors
(:require
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
[auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]]
[auto-ap.datomic.vendors :as d-vendors]
[auto-ap.graphql.utils
:refer [->graphql
@@ -14,20 +14,22 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.api :as d]))
[datomic.client.api :as dc]
[auto-ap.search :as search]))
(defn can-user-edit-vendor? [vendor-id id]
(if (is-admin? id)
true
(empty?
(set/difference (set (d/q '[:find [?c ...]
:in $ ?v
:where [?vu :vendor-usage/vendor ?v]
[?vu :vendor-usage/client ?c]
[?vu :vendor-usage/count ?d]
[(>= ?d 0)]]
(d/db conn)
vendor-id))
(set/difference (set (->> (dc/q '[:find ?c
:in $ ?v
:where [?vu :vendor-usage/vendor ?v]
[?vu :vendor-usage/client ?c]
[?vu :vendor-usage/count ?d]
[(>= ?d 0)]]
(dc/db conn)
vendor-id)
(map first)))
(set (map :db/id (:user/clients id)))))))
(defn upsert-vendor [context {{:keys [id name hidden terms code print_as primary_contact secondary_contact address default_account_id invoice_reminder_schedule schedule_payment_dom terms_overrides account_overrides] :as in} :vendor} _]
@@ -59,8 +61,7 @@
hidden (if (is-admin? (:id context))
hidden
false)
existing (when id
(d/pull (d/db conn) '[:vendor/name] id))
existing (pull-attr (dc/db conn) :vendor/name id)
terms-overrides (mapv
(fn [to]
(cond->
@@ -145,11 +146,11 @@
(->graphql))))
(defn merge-vendors [context {:keys [from to]} _]
(let [transaction (->> (d/query {:query {:find '[?x ?a2]
(let [transaction (->> (dc/q {:query {:find '[?x ?a2]
:in '[$ ?vendor-from ]
:where ['[?x ?a ?vendor-from]
'[?a :db/ident ?a2]]}
:args [(d/db conn) from]})
:args [(dc/db conn) from]})
(mapcat (fn [[src attr]]
[[:db/retract src attr from]
@@ -182,6 +183,13 @@
matches))
(defn search [context args _]
(comment (let [search-query (cleanse-query (:query args))]
(for [[id name] (search/search (cond-> {:q search-query}
(not (is-admin? (:id context))) (assoc :hidden false))
"vendors")]
{:name name
:id (Long/parseLong id)})
))
(if-let [search-query (cleanse-query (:query args))]
(let [data (if (is-admin? (:id context))
(d/q '[:find ?n ?i ?s

31
src/clj/auto_ap/import/common.clj
View File

@@ -1,28 +1,29 @@
(ns auto-ap.import.common
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic :refer [conn pull-ref random-tempid]]
[clojure.tools.logging :as log]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn bank-account->integration-id [bank-account]
(or (->> bank-account
(d/pull (d/db conn) [:bank-account/integration-status])
:bank-account/integration-status
:db/id)
#db/id[:db.part/user]))
(or (pull-ref (dc/db conn) :bank-account/integration-status bank-account)
(random-tempid)))
(defn wrap-integration [f bank-account]
(try
(let [result (f)]
@(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account)
:integration-status/state :integration-state/success
:integration-status/last-attempt (java.util.Date.)
:integration-status/last-updated (java.util.Date.)}}])
(dc/transact conn {:tx-data [{:db/id bank-account
:bank-account/integration-status
{:db/id (bank-account->integration-id bank-account)
:integration-status/state :integration-state/success
:integration-status/last-attempt (java.util.Date.)
:integration-status/last-updated (java.util.Date.)}}]})
result)
(catch Exception e
@(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account)
:integration-status/state :integration-state/failed
:integration-status/last-attempt (java.util.Date.)
:integration-status/message (.getMessage e)}}])
(dc/transact conn {:tx-data [{:db/id bank-account
:bank-account/integration-status
{:db/id (bank-account->integration-id bank-account)
:integration-status/state :integration-state/failed
:integration-status/last-attempt (java.util.Date.)
:integration-status/message (.getMessage e)}}]})
(log/warn e)
nil)))

18
src/clj/auto_ap/import/intuit.clj
View File

@@ -10,10 +10,12 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]))
[datomic.client.api :as dc]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
(defn get-intuit-bank-accounts [db]
(d/q '[:find ?external-id ?ba ?c
(dc/q '[:find ?external-id ?ba ?c
:in $
:where
[?c :client/bank-accounts ?ba]
@@ -49,7 +51,7 @@
:priority :low}
nil)
(let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
db (d/db conn)
db (dc/db conn)
end (auto-ap.time/local-now)
start (time/plus end (time/days -30))]
(try
@@ -76,8 +78,8 @@
(defn upsert-accounts []
(let [token (i/get-fresh-access-token)
bank-accounts (i/get-bank-accounts token)]
@(d/transact conn (mapv
(fn [ba]
{:intuit-bank-account/external-id (:name ba)
:intuit-bank-account/name (:name ba)})
bank-accounts))))
(dc/transact conn {:tx-data (mapv
(fn [ba]
{:intuit-bank-account/external-id (:name ba)
:intuit-bank-account/name (:name ba)})
bank-accounts)})))

10
src/clj/auto_ap/import/manual.clj
View File

@@ -5,7 +5,7 @@
[auto-ap.import.transactions :as t]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv]
[datomic.api :as d]
[datomic.client.api :as dc]
[unilog.context :as lc]))
@@ -35,17 +35,17 @@
(defn import-batch [transactions user]
(lc/with-context {:source "Manual import transactions"}
(let [bank-account-code->client (into {}
(d/q '[:find ?bac ?c
(dc/q '[:find ?bac ?c
:in $
:where
[?c :client/bank-accounts ?ba]
[?ba :bank-account/code ?bac]]
(d/db conn)))
(dc/db conn)))
bank-account-code->bank-account (into {}
(d/q '[:find ?bac ?ba
(dc/q '[:find ?bac ?ba
:in $
:where [?ba :bank-account/code ?bac]]
(d/db conn)))
(dc/db conn)))
import-batch (t/start-import-batch :import-source/manual user)
transactions (->> transactions
(map (fn [t]

6
src/clj/auto_ap/import/plaid.clj
View File

@@ -8,12 +8,12 @@
[auto-ap.utils :refer [allow-once by]]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[datomic.api :as d]
[datomic.client.api :as dc]
[digest :as di]
[unilog.context :as lc]))
(defn get-plaid-accounts [db]
(-> (d/q '[:find ?ba ?c ?external-id ?t
(-> (dc/q '[:find ?ba ?c ?external-id ?t
:in $
:where
[?c :client/bank-accounts ?ba]
@@ -42,7 +42,7 @@
end (atime/local-now)
start (time/plus end (time/days -30))]
(try
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn))
(doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (dc/db conn))
:let [transaction-result (wrap-integration #(p/get-transactions access-token external-id start end)
bank-account-id)
accounts-by-id (by :account_id (:accounts transaction-result))]

83
src/clj/auto_ap/import/transactions.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.import.transactions
(:require
[auto-ap.datomic :refer [conn remove-nils uri]]
[auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]]
[auto-ap.datomic.accounts :as a]
[auto-ap.datomic.checks :as d-checks]
[auto-ap.datomic.transaction-rules :as tr]
@@ -13,7 +13,7 @@
[clj-time.core :as t]
[clojure.core.cache :as cache]
[clojure.tools.logging :as log]
[datomic.api :as d]
[datomic.client.api :as dc]
[digest :as di]))
(defn rough-match [client-id bank-account-id amount]
@@ -56,19 +56,19 @@
(defn match-transaction-to-unfulfilled-autopayments [amount client-id]
(log/info "trying to find uncleared autopay invoices")
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd]
:in ['$ '?client-id]
:where ['[?e :invoice/client ?client-id]
'[?e :invoice/scheduled-payment ?sd]
'[?e :invoice/status :invoice-status/paid]
'(not [_ :invoice-payment/invoice ?e])
'[?e :invoice/vendor ?vendor-id]
'[?e :invoice/total ?total]]}
:args [(d/db conn) client-id]})
(let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?total '?sd]
:in ['$ '?client-id]
:where ['[?e :invoice/client ?client-id]
'[?e :invoice/scheduled-payment ?sd]
'[?e :invoice/status :invoice-status/paid]
'(not [_ :invoice-payment/invoice ?e])
'[?e :invoice/vendor ?vendor-id]
'[?e :invoice/total ?total]]}
:args [(dc/db conn) client-id]})
(sort-by last) ;; sort by scheduled payment date
(group-by first) ;; group by vendors
vals)
considerations (for [candidate-invoices candidate-invoices-vendor-groups
considerations (for [candidate-invoices candidate-invoices-vendor-groups
invoice-count (range 1 32)
consideration (partition invoice-count 1 candidate-invoices)
:when (dollars= (reduce (fn [acc [_ _ amount]]
@@ -81,19 +81,19 @@
(defn match-transaction-to-unpaid-invoices [amount client-id]
(log/info "trying to find unpaid invoices for " client-id amount)
(let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
:in ['$ '?client-id]
:where ['[?e :invoice/client ?client-id]
'[?e :invoice/status :invoice-status/unpaid]
'(not [_ :invoice-payment/invoice ?e])
'[?e :invoice/vendor ?vendor-id]
'[?e :invoice/outstanding-balance ?outstanding-balance]
'[?e :invoice/date ?d]]}
:args [(d/db conn) client-id]})
(let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?outstanding-balance '?d]
:in ['$ '?client-id]
:where ['[?e :invoice/client ?client-id]
'[?e :invoice/status :invoice-status/unpaid]
'(not [_ :invoice-payment/invoice ?e])
'[?e :invoice/vendor ?vendor-id]
'[?e :invoice/outstanding-balance ?outstanding-balance]
'[?e :invoice/date ?d]]}
:args [(dc/db conn) client-id]})
(sort-by last) ;; sort by scheduled payment date
(group-by first) ;; group by vendors
vals)
considerations (for [candidate-invoices candidate-invoices-vendor-groups
considerations (for [candidate-invoices candidate-invoices-vendor-groups
invoice-count (range 1 32)
consideration (partition invoice-count 1 candidate-invoices)
:when (dollars= (reduce (fn [acc [_ _ amount]]
@@ -111,7 +111,7 @@
(defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id]
(log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments)
(let [payment-id (d/tempid :db.part/user)]
(let [payment-id (random-tempid)]
(-> tx
(conj {:payment/bank-account bank-account-id
@@ -142,7 +142,6 @@
:amount (Math/abs (:transaction/amount transaction))}]]))))
(defn extract-check-number [{:transaction/keys [description-original]}]
(if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)]
(try
(Integer/parseInt check-number)
@@ -152,8 +151,8 @@
(defn find-expected-deposit [client-id amount date]
(when date
(-> (d/q
'[:find [(pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}]) ...]
(-> (dc/q
'[:find (pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}])
:in $ ?c ?a ?d-start
:where
[?ed :expected-deposit/client ?c]
@@ -163,8 +162,8 @@
[?ed :expected-deposit/total ?a2]
[(auto-ap.utils/dollars= ?a2 ?a)]
]
(d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
first)))
(dc/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10))))
ffirst)))
(defn categorize-transaction [transaction bank-account existing]
@@ -242,7 +241,7 @@
))))
(defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations]
(when-not (seq (match-transaction-to-unpaid-invoices amount client))
(when (seq (match-transaction-to-unpaid-invoices amount client))
(apply-rules transaction valid-locations)))
(defn transaction->txs [transaction bank-account apply-rules]
@@ -267,13 +266,13 @@
(defn get-existing [bank-account]
(log/info "looking up bank account data for" bank-account)
(into {}
(d/query {:query {:find ['?tid '?as2]
(dc/q {:query {:find ['?tid '?as2]
:in ['$ '?ba]
:where ['[?e :transaction/bank-account ?ba]
'[?e :transaction/id ?tid]
'[?e :transaction/approval-status ?as]
'[?as :db/ident ?as2]]}
:args [(d/db (d/connect uri)) bank-account]})))
:args [(dc/db conn) bank-account]})))
(defprotocol ImportBatch
(import-transaction! [this transaction])
@@ -288,17 +287,17 @@
:import-batch/not-ready 0
:import-batch/extant 0})
extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 ))
import-id (get (:tempids @(d/transact conn [{:db/id "import-batch"
:import-batch/date (coerce/to-date (t/now))
:import-batch/source source
:import-batch/status :import-status/started
:import-batch/user-name user}])) "import-batch")
import-id (get (:tempids (dc/transact conn {:tx-data [{:db/id "import-batch"
:import-batch/date (coerce/to-date (t/now))
:import-batch/source source
:import-batch/status :import-status/started
:import-batch/user-name user}]})) "import-batch")
rule-applying-function (rm/rule-applying-fn (tr/get-all))]
(log/info "Importing transactions from " source)
(reify ImportBatch
(import-transaction! [_ transaction]
(let [bank-account (d/pull (d/db conn)
(let [bank-account (dc/pull (dc/db conn)
[:bank-account/code
:db/id
:bank-account/locations
@@ -327,14 +326,14 @@
(fail! [_ error]
(log/errorf "Couldn't complete import %d with error." import-id)
(log/error error)
@(d/transact conn [(merge {:db/id import-id
:import-batch/status :import-status/completed
:import-batch/error-message (str error)}
@stats)]))
(dc/transact conn {:tx-data [(merge {:db/id import-id
:import-batch/status :import-status/completed
:import-batch/error-message (str error)}
@stats)]}))
(finish! [_]
(log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats))
@(d/transact conn [(merge {:db/id import-id
(dc/transact conn [(merge {:db/id import-id
:import-batch/status :import-status/completed}
@stats)])))))

11
src/clj/auto_ap/import/yodlee2.clj
View File

@@ -9,9 +9,10 @@
[clj-time.coerce :as coerce]
[clojure.string :as str]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]
[digest :as di]
[unilog.context :as lc]))
[datomic.client.api :as dc]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
#_{:clj-kondo/ignore [:unresolved-var]}
(defn yodlee->transaction [transaction use-date-instead-of-post-date?]
@@ -51,7 +52,7 @@
nil)
(let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
(try
(let [account-lookup (d/q '[:find ?ya ?ba ?cd ?ud
(let [account-lookup (dc/q '[:find ?ya ?ba ?cd ?ud
:in $
:where
[?ba :bank-account/yodlee-account ?y]
@@ -60,7 +61,7 @@
[?c :client/code ?cd]
[?y :yodlee-account/id ?ya]
]
(d/db conn))]
(dc/db conn))]
(doseq [[yodlee-account bank-account client-code use-date-instead-of-post-date?] account-lookup
transaction (wrap-integration #(client2/get-specific-transactions client-code yodlee-account)
bank-account)]

19
src/clj/auto_ap/jobs/close_auto_invoices.clj
View File

@@ -6,24 +6,23 @@
[auto-ap.time :as time]
[clj-time.coerce :as coerce]
[clojure.tools.logging :as log]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn close-auto-invoices []
(let [invoices-to-close (d/query {:query {:find ['?e]
(let [invoices-to-close (dc/q {:query {:find ['?e]
:in ['$ '?today]
:where ['[?e :invoice/scheduled-payment ?d]
'[?e :invoice/status :invoice-status/unpaid]
'[(<= ?d ?today)]]}
:args [(d/db conn) (coerce/to-date (time/local-now))]})]
:args [(dc/db conn) (coerce/to-date (time/local-now))]})]
(log/info "Closing " (count invoices-to-close) "scheduled invoices")
(some->> invoices-to-close
seq
(dc/transact conn {:tx-data (some->> invoices-to-close
seq
(mapv (fn [[i]] {:db/id i
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}))
(d/transact conn)
deref)
(mapv (fn [[i]] {:db/id i
:invoice/outstanding-balance 0.0
:invoice/status :invoice-status/paid}))
)})
(log/info "Closed " (count invoices-to-close) "scheduled invoices")))

51
src/clj/auto_ap/jobs/sysco.clj
View File

@@ -2,9 +2,9 @@
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.datomic.invoices :refer [code-invoice propose-invoice]]
[auto-ap.ledger :refer [transact-with-ledger]]
[auto-ap.parse :as parse]
[auto-ap.time :as t]
@@ -15,7 +15,8 @@
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[datomic.api :as d])
[datomic.client.api :as dc]
[auto-ap.datomic.vendors :as d-vendors])
(:import
(java.util UUID)))
@@ -26,16 +27,15 @@
(def summary-keys ["TranCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "TotalLines" "TotalQtyInvoice" "TotalQty" "TotalQtySplit" "TotalQtyPounds" "TotalExtendedPrice" "TotalTaxAmount" "TotalInvoiceAmount" "AccountDate"])
(defn get-sysco-vendor []
(let [db (d/db conn)]
(d/entity
db
(->
(d/q '[:find ?v
:in $
:where [?v :vendor/name "Sysco"]]
db)
(let [db (dc/db conn)]
(->
(dc/q '[:find (pull ?v r)
:in $ r
:where [?v :vendor/name "Sysco"]]
db
d-vendors/default-read)
first
first))))
first)))
(defn read-sysco-csv [k]
@@ -45,7 +45,7 @@
io/reader
csv/read-csv))
(defn extract-invoice-details [csv-rows clients sysco-vendor]
(defn extract-invoice-details [csv-rows sysco-vendor]
(let [[header-row & csv-rows] csv-rows
header-row (into {} (map vector header-keys header-row))
summary-row (->> csv-rows
@@ -65,8 +65,8 @@
(header-row "City2")])
account-number (some-> account-number Long/parseLong str)
[matching-client similarity] (and account-number
(parse/best-match clients account-number 0.0))
matching-client (and account-number
(d-clients/exact-match account-number))
_ (when-not matching-client
(throw (ex-info "cannot find matching client"
{:account-number account-number
@@ -81,14 +81,19 @@
(cond-> #:invoice {:invoice-number (header-row "InvoiceNumber")
:total (+ total tax)
:outstanding-balance (+ total tax)
:location (parse/best-location-match matching-client location-hint location-hint )
:location (parse/best-location-match (dc/pull (dc/db conn)
[{:client/location-matches [:location-match/location :location-match/matches]}
:client/default-location
:client/locations]
matching-client)
location-hint
location-hint )
:date (coerce/to-date date)
:vendor (:db/id sysco-vendor )
:client (:db/id matching-client)
:import-status :import-status/completed
:status :invoice-status/unpaid
:client-identifier customer-identifier}
similarity (assoc :invoice/similarity (- 1.0 (double similarity)))
true (code-invoice))))
(defn mark-key [k]
@@ -111,7 +116,6 @@
(defn import-sysco []
(let [sysco-vendor (get-sysco-vendor)
clients (d-clients/get-all)
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
:prefix "sysco/pending"})
:object-summaries
@@ -129,11 +133,11 @@
:destination-bucket-name (:data-bucket env)
:source-key k
:destination-key invoice-key})
[[:propose-invoice
(-> k
[`(propose-invoice
~(-> k
read-sysco-csv
(extract-invoice-details clients sysco-vendor)
(assoc :invoice/source-url invoice-url))]])
(extract-invoice-details sysco-vendor)
(assoc :invoice/source-url invoice-url)))])
(catch Exception e
(log/error (str "Cannot load file " k) e)
(log/info
@@ -144,8 +148,7 @@
(.getName (io/file k)))
println)}))
[])))))
result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]
#_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})])
(doseq [k keys]
(mark-key k))))

44
src/clj/auto_ap/jobs/vendor_usages.clj
View File

@@ -3,31 +3,29 @@
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn refresh-vendor-usages []
(->> {:query {:find ['?v '?c '(count ?e)]
:in ['$]
:where ['[?v :vendor/name]
'(or-join [?v ?c ?e]
(and
[?e :invoice/vendor ?v]
[?e :invoice/client ?c])
(and
[?e :transaction/vendor ?v]
[?e :transaction/client ?c])
(and
[?e :journal-entry/vendor ?v]
[?e :journal-entry/client ?c]))]}
:args [(d/db conn)]}
(d/query)
(map (fn [[v c cnt]]
#:vendor-usage {:vendor v
:client c
:key (str v "-" c)
:count cnt}))
(d/transact conn)
deref))
(dc/transact conn {:tx-data (->> (dc/q '[:find ?v ?c (count ?e)
:in $
:where
[?v :vendor/name]
(or-join [?v ?c ?e]
(and
[?e :invoice/vendor ?v]
[?e :invoice/client ?c])
(and
[?e :transaction/vendor ?v]
[?e :transaction/client ?c])
(and
[?e :journal-entry/vendor ?v]
[?e :journal-entry/client ?c]))]
(dc/db conn))
(map (fn [[v c cnt]]
#:vendor-usage {:vendor v
:client c
:key (str v "-" c)
:count cnt})))}))
(defn -main [& _]
(execute "vendor-usages" refresh-vendor-usages))

193
src/clj/auto_ap/ledger.clj
View File

@@ -1,21 +1,17 @@
(ns auto-ap.ledger
(:require
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
[auto-ap.logging :refer [info-event]]
[auto-ap.datomic :refer [conn remove-nils pull-ref audit-transact]]
[auto-ap.utils :refer [dollars-0? dollars=]]
[clj-time.coerce :as c]
[clj-time.core :as t]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
[datomic.client.api :as dc]))
(defn datums->impacted-entity [db [e changes]]
(let [entity (d/pull db '[* {:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
namespaces (->> changes
(map :a)
(map #(:db/ident (dc/pull db '[:db/ident] (:a %))))
(map namespace)
set)]
(cond (namespaces "invoice" ) [[:invoice e]]
@@ -31,7 +27,7 @@
(defmethod entity-change->ledger :invoice
[db [_ id]]
(when id
(let [entity (d/pull db ['* {:invoice/vendor '[*]
(let [entity (dc/pull db ['* {:invoice/vendor '[*]
:invoice/payment '[*]
:invoice/status '[:db/ident]
:invoice/import-status '[:db/ident]}] id)
@@ -71,7 +67,7 @@
(defmethod entity-change->ledger :transaction
[db [_ id]]
(when id
(let [entity (d/pull db ['* {:transaction/vendor '[*]
(let [entity (dc/pull db ['* {:transaction/vendor '[*]
:transaction/client '[*]
:transaction/approval-status '[*]
:transaction/bank-account '[* {:bank-account/type [:db/ident]}]
@@ -144,13 +140,12 @@
[_ _]
nil)
(defn reconcile-ledger
([] (reconcile-ledger (-> (t/now)
(t/plus (t/months -6))
(c/to-date))))
([start-date]
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
(let [txes-missing-ledger-entries (->> (dc/q {:query {:find ['?t ]
:in ['$ '?sd]
:where [
'[?t :transaction/date ?d]
@@ -160,12 +155,12 @@
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
'(not [?t :transaction/approval-status :transaction-approval-status/suppressed])
]}
:args [(d/db conn) start-date]})
:args [(dc/db conn) start-date]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
(mapv #(entity-change->ledger (dc/db conn) [:transaction %])))
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
invoices-missing-ledger-entries (->> (dc/q {:query {:find ['?t ]
:in ['$ '?sd]
:where ['[?t :invoice/date ?d]
'[(>= ?d ?sd)]
@@ -176,45 +171,71 @@
'(not [?t :invoice/import-status :import-status/pending])
'(not [?t :invoice/exclude-from-ledger true])
]}
:args [(d/db conn) start-date]})
:args [(dc/db conn) start-date]})
(map first)
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
(mapv #(entity-change->ledger (dc/db conn) [:invoice %])))
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
(when (seq repairs)
(log/info (take 3 repairs))
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
@(d/transact conn repairs)))))
(dc/transact conn {:tx-data repairs})))))
(defn touch-transaction [e]
@(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
(when-let [change (entity-change->ledger (d/db conn)
[:transaction e])]
@(d/transact conn [{:db/id "datomic.tx"
:db/doc "touching transaction to update ledger"}
change])))
(dc/transact conn {:tx-data [[:db/retractEntity [:journal-entry/original-entity e]]]})
(when-let [change (entity-change->ledger (d/db conn)
[:transaction e])]
(dc/transact conn {:tx-data [{:db/id "datomic.tx"
:db/doc "touching transaction to update ledger"}
(entity-change->ledger (dc/db conn)
[:transaction e])]}))
)
(defn touch-invoice [e]
@(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
(when-let [change (entity-change->ledger (d/db conn)
[:invoice e])]
@(d/transact conn [{:db/id "datomic.tx"
:db/doc "touching invoice to update ledger"}
change])))
(dc/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
(when-let [change (entity-change->ledger (d/db conn)
[:invoice e])]
(dc/transact conn [{:db/id "datomic.tx"
:db/doc "touching invoice to update ledger"}
(entity-change->ledger (dc/db conn)
[:invoice e])]))
)
(defn lazy-tx-range
([start end xf] (lazy-tx-range start end xf 0))
([start end xf o]
(let [next-results (dc/tx-range conn {:start start
:end end
:offset o
:limit 200})]
(lazy-seq
(if (seq next-results)
(concat (sequence (comp (mapcat :data)
xf) next-results) (lazy-tx-range start
end
xf
(+ (count next-results)
o)))
next-results)))))
(defn recently-changed-entities [start end]
(set (map (fn [d]
(:e d))
(mapcat :data (dc/tx-range conn {:start start
:end end})))))
(defn entities-since-last-ledger-entry []
(count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5)))
:end (coerce/to-date (time/now))})))
(defn mismatched-transactions
([]
(mismatched-transactions (c/to-date (t/minus (t/now) (t/days 7)))
(c/to-date (t/minus (t/now) (t/hours 1)))) )
([changed-between-start changed-between-end]
(let [entities-to-consider (d/q '[:find [?t ...]
:in $ ?log ?start ?end
:where
[(tx-ids ?log ?start ?end) [?tx ...]]
[(tx-data ?log ?tx) [[?t]]]
[?t :transaction/date]]
(d/db auto-ap.datomic/conn)
(d/log auto-ap.datomic/conn)
(let [entities-to-consider (recently-changed-entities
changed-between-start
changed-between-end)
_ (log/info "checking" (count entities-to-consider) "transactions looking for mismatches between" changed-between-start changed-between-end)
@@ -222,20 +243,21 @@
(fn [acc [e lia]]
(update acc e (fnil conj #{} ) lia))
{}
(d/q '[:find ?e ?lia
(dc/q '[:find ?e ?lia
:in $ [?e ...]
:where
[?je :journal-entry/original-entity ?e]
[?e :transaction/date]
[?je :journal-entry/line-items ?li]
[?li :journal-entry-line/account ?lia]
[?lia :account/name]]
(d/db auto-ap.datomic/conn)
(dc/db conn)
entities-to-consider))
transaction-accounts (reduce
(fn [acc [e lia]]
(update acc e (fnil conj #{} ) lia))
{}
(d/q '[:find ?e ?lia
(dc/q '[:find ?e ?lia
:in $ [?e ...]
:where
[?e :transaction/date ?d]
@@ -246,7 +268,7 @@
[?lia :account/name]
[?e :transaction/amount ?amt]
[(not= ?amt 0.0)]]
(d/db auto-ap.datomic/conn)
(dc/db conn)
entities-to-consider))]
(->> transaction-accounts
(filter
@@ -256,20 +278,9 @@
([] (unbalanced-transactions (c/to-date (t/minus (t/now) (t/days 7)))
(c/to-date (t/minus (t/now) (t/hours 1)))))
([changed-between-start changed-between-end]
(let [entities-to-consider (d/q '[:find [?je ...]
:in $ ?log ?start ?end
:where
[(tx-ids ?log ?start ?end) [?tx ...]]
[(tx-data ?log ?tx) [[?je]]]
[?je :journal-entry/amount]
[?je :journal-entry/original-entity ?i]
[?i :transaction/date]]
(d/db auto-ap.datomic/conn)
(d/log auto-ap.datomic/conn)
changed-between-start
changed-between-end)]
(let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end)]
(log/info "checking" (count entities-to-consider) "transaction journal entries looking for mismatches between" changed-between-start changed-between-end)
(->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit)
(->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit)
:with ?jel
:in $ [?je ...]
:where [?je :journal-entry/amount ?a]
@@ -277,52 +288,42 @@
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
]
(d/db auto-ap.datomic/conn)
(dc/db conn)
entities-to-consider)
(filter (fn [[_ a d c]]
(or (not (dollars= a d))
(not (dollars= a c)))))
(map first)
(map (fn [je]
(:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn)
je))))
(map :db/id)))))
(pull-ref (dc/db conn) :journal-entry/original-entity je)))))))
(defn unbalanced-invoices
([] (unbalanced-invoices (c/to-date (t/minus (t/now) (t/days 7)))
(c/to-date (t/minus (t/now) (t/hours 1)))))
([changed-between-start changed-between-end]
(let [entities-to-consider (d/q '[:find [?je ...]
:in $ ?log ?start ?end
:where
[(tx-ids ?log ?start ?end) [?tx ...]]
[(tx-data ?log ?tx) [[?je]]]
[?je :journal-entry/amount]
[?je :journal-entry/original-entity ?i]
[?i :invoice/date]]
(d/db auto-ap.datomic/conn)
(d/log auto-ap.datomic/conn)
changed-between-start
changed-between-end)]
(let [entities-to-consider (recently-changed-entities
changed-between-start
changed-between-end)]
(log/info "checking" (count entities-to-consider) "invoice journal entries looking for mismatches between" changed-between-start changed-between-end)
(->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit)
:with ?jel
:in $ [?je ...]
:where [?je :journal-entry/amount ?a]
[?je :journal-entry/line-items ?jel]
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
]
(d/db auto-ap.datomic/conn)
entities-to-consider)
(->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit)
:with ?jel
:in $ [?je ...]
:where [?je :journal-entry/amount ?a]
[?je :journal-entry/original-entity ?i]
[?i :invoice/date]
[?je :journal-entry/line-items ?jel]
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]]
(dc/db conn)
entities-to-consider)
(filter (fn [[_ a d c]]
(or (not (dollars= a d))
(not (dollars= a c)))))
(map first)
(map (fn [je]
(:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn)
je))))
(map :db/id)))))
(pull-ref (dc/db conn) :journal-entry/original-entity je)))))))
(defn mismatched-invoices
([]
@@ -330,36 +331,28 @@
(c/to-date (t/minus (t/now) (t/hours 1)))) )
([changed-between-start changed-between-end]
(let [entities-to-consider (d/q '[:find [?i ...]
:in $ ?log ?start ?end
:where
[(tx-ids ?log ?start ?end) [?tx ...]]
[(tx-data ?log ?tx) [[?i]]]
[?i :invoice/date]]
(d/db auto-ap.datomic/conn)
(d/log auto-ap.datomic/conn)
changed-between-start
changed-between-end)
(let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end)
_ (log/info (count entities-to-consider) "invoices have changed between" changed-between-start "and" changed-between-end)
jel-accounts (reduce
(fn [acc [e lia]]
(update acc e (fnil conj #{} ) lia))
{}
(d/q '[:find ?e ?lia
(dc/q '[:find ?e ?lia
:in $ [?e ...]
:where
[?je :journal-entry/original-entity ?e]
[?e :invoice/date]
[?je :journal-entry/line-items ?li]
[?li :journal-entry-line/account ?lia]
(not [?lia :account/numeric-code 21000])
[?lia :account/name]]
(d/db auto-ap.datomic/conn)
(dc/db conn)
entities-to-consider))
invoice-accounts (reduce
(fn [acc [e lia]]
(update acc e (fnil conj #{} ) lia))
{}
(d/q '[:find ?e ?lia
(dc/q '[:find ?e ?lia
:in $ [?e ...]
:where
[?e :invoice/expense-accounts ?li]
@@ -370,7 +363,7 @@
(not [?e :invoice/status :invoice-status/voided])
(not [?e :invoice/exclude-from-ledger true])
[?e :invoice/import-status :import-status/imported]]
(d/db auto-ap.datomic/conn)
(dc/db conn)
entities-to-consider))
]
(filter
@@ -427,12 +420,12 @@
nil))
(defn transact-with-ledger [transaction id]
(let [db (d/db conn)
(let [db (dc/db conn)
tx (audit-transact transaction id)
affected-entities (->> (:tx-data tx)
(map (fn [^datomic.db.Datum x]
{:e (:e x)
:a (d/ident db (:a x))
:a (:a x)
:v (:v x)
:added (:added x)}))
(group-by :e)

45
src/clj/auto_ap/pdf/ledger.clj
View File

@@ -1,17 +1,17 @@
(ns auto-ap.pdf.ledger
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.ledger.reports :as l-reports]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic :refer [conn pull-attr pull-many]]
[auto-ap.graphql.utils :refer [<-graphql]]
[auto-ap.ledger.reports :as l-reports]
[auto-ap.time :as atime]
[auto-ap.utils :refer [by dollars-0?]]
[clj-pdf.core :as pdf]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[datomic.api :as d]
[clojure.tools.logging :as log])
[datomic.client.api :as dc])
(:import
(java.io ByteArrayOutputStream)
(java.text DecimalFormat)
@@ -152,7 +152,7 @@
args (assoc args
:periods (filter identity (cond-> [(:date args)]
(:include-comparison args) (conj (:comparison-date args)))))
clients (d/pull-many (d/db conn) '[:client/code :client/name :db/id] [(:client-id args)])
clients (pull-many (dc/db conn) [:client/code :client/name :db/id] [(:client-id args)])
data (concat (->> (:balance-sheet-accounts data)
(map (fn [b]
(assoc b
@@ -183,7 +183,7 @@
(let [data (<-graphql data)
args (<-graphql args)
clients (d/pull-many (d/db conn) '[:client/code :client/name :db/id] (:client-ids args))
clients (pull-many (dc/db conn) [:client/code :client/name :db/id] (:client-ids args))
data (->> data
:periods
(mapcat (fn [p1 p2]
@@ -255,7 +255,7 @@
max-date (atime/unparse-local
(->> args :periods (map :end) last)
atime/iso-date)
names (str/replace (->> args :client_ids (d/pull-many (d/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
names (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )]
(format "Profit-and-loss-%s-to-%s-for-%s" min-date max-date names)))
(defn journal-detail-args->name [args]
@@ -273,7 +273,7 @@
(let [date (atime/unparse-local
(:date args)
atime/iso-date)
name (str/replace (->> args :client_id (d/pull (d/db conn) [:client/name]) :client/name ) #" " "_" )]
name (str/replace (->> args :client_id (pull-attr (dc/db conn) :client/name)) #" " "_" )]
(format "Balance-sheet-%s-for-%s" date name)))
(defn print-pnl [user args data]
@@ -287,13 +287,13 @@
:input-stream (io/make-input-stream pdf-data {})
:metadata {:content-length (count pdf-data)
:content-type "application/pdf"})
@(d/transact conn
[{:report/name name
:report/client (:client_ids args)
:report/key key
:report/url url
:report/creator (:user user)
:report/created (java.util.Date.)}])
(dc/transact conn
{:tx-data [{:report/name name
:report/client (:client_ids args)
:report/key key
:report/url url
:report/creator (:user user)
:report/created (java.util.Date.)}]})
{:report/name name
:report/url url }))
@@ -308,13 +308,14 @@
:input-stream (io/make-input-stream pdf-data {})
:metadata {:content-length (count pdf-data)
:content-type "application/pdf"})
@(d/transact conn
[{:report/name name
:report/client [(:client_id args)]
:report/key key
:report/url url
:report/creator (:user user)
:report/created (java.util.Date.)}])
(dc/transact conn
{:tx-data
[{:report/name name
:report/client [(:client_id args)]
:report/key key
:report/url url
:report/creator (:user user)
:report/created (java.util.Date.)}]})
{:report/name name
:report/url url }))

5
src/clj/auto_ap/routes/exports.clj
View File

@@ -20,6 +20,9 @@
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[datomic.api :as d]
[compojure.core :refer [context defroutes GET routes wrap-routes]]
[config.core :refer [env]]
[datomic.client.api :as dc]
[ring.middleware.json :refer [wrap-json-response]]
[venia.core :as venia]))
@@ -48,8 +51,6 @@
[(auto-ap.time/unparse ?d3 auto-ap.time/normal-date) ?d4]]}
:args [(d/db conn) client-id]}))))
(defn client-tag [params]
(when-let [code (or (params "client-code")
(:client-code params))]

190
src/clj/auto_ap/routes/invoices.clj
View File

@@ -54,8 +54,8 @@
(defn parse-invoice-number [{:keys [invoice-number]}]
(or invoice-number ""))
(defn parse-vendor [{:keys [vendor-name check]} vendors]
(let [v (vendors vendor-name)]
(defn parse-vendor [{:keys [vendor-name check]} vendor-name->vendor]
(let [v (vendor-name->vendor vendor-name)]
(cond v
v
@@ -65,9 +65,6 @@
:else
(throw (Exception. (str "Vendor '" vendor-name "' not found."))))))
(defn parse-vendor-id [{:keys [vendor]}]
(:db/id vendor))
(defn parse-automatically-paid-when-due [{:keys [vendor client-id]}]
(boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id)))
@@ -77,22 +74,31 @@
(defn parse-invoice-rows [excel-rows]
(let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code]
all-vendors (->> (d/q '[:find [?e ...]
:in $
:where [?e :vendor/name]]
(d/db conn))
(d/pull-many (d/db conn) d-vendors/default-read)
(by :vendor/name))
all-clients (d-clients/get-all)
all-clients (merge (by :client/code all-clients) (by :client/name all-clients))
rows (->> (str/split excel-rows #"\n")
(map #(str/split % #"\t"))
(map #(into {} (map (fn [c k] [k c]) % columns)))
tabulated (->> (str/split excel-rows #"\n")
(map #(str/split % #"\t"))
(map #(into {} (map (fn [c k] [k c]) % columns))))
vendor-name->vendor (->>
(set (map :vendor-name tabulated))
(dc/q '[:find ?n ?v
:in $ [?n ...]
:where [?v :vendor/name ?n]]
(dc/db conn)
)
(into {}))
all-clients (merge (into {}(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
:in $
:where [?v :client/name ?n]]
(dc/db conn)))
(into {}
(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
:in $
:where [?v :client/code ?n]]
(dc/db conn))))
rows (->> tabulated
(map reset-id)
(map assoc-client-code)
(map (c/parse-or-error :client-id #(parse-client % all-clients)))
(map (c/parse-or-error :vendor #(parse-vendor % all-vendors)))
(map (c/parse-or-error :vendor-id #(parse-vendor-id %)))
(map (c/parse-or-error :vendor-id #(parse-vendor % vendor-name->vendor)))
(map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
(map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
(map (c/parse-or-error :account-id c/parse-account-numeric-code))
@@ -107,51 +113,54 @@
(throw (ex-info (str "No vendor found. Please supply an forced vendor.")
{:vendor-code vendor-code})))
(let [vendor-id (or forced-vendor
(->> (d/query
(->> (dc/q
{:query {:find ['?vendor]
:in ['$ '?vendor-name]
:where ['[?vendor :vendor/name ?vendor-name]]}
:args [(d/db (d/connect uri)) vendor-code]})
:args [(dc/db conn) vendor-code]})
first
first))]
(when-not vendor-id
(throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.")
{:vendor-code vendor-code})))
(if-let [matching-vendor (->> (d/query
(if-let [matching-vendor (->> (dc/q
{:query {:find [(list 'pull '?vendor-id d-vendors/default-read)]
:in ['$ '?vendor-id]}
:args [(d/db (d/connect uri)) vendor-id]})
:args [(dc/db conn) vendor-id]})
first
first)]
matching-vendor
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
{:vendor-code vendor-code})))))
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]} clients]
(let [[matching-client similarity] (cond
account-number (parse/best-match clients account-number 0.0)
customer-identifier (parse/best-match clients customer-identifier)
client-override [(first (filter (fn [c]
(= (:db/id c) (Long/parseLong client-override)))
clients))
1.0])
matching-vendor (match-vendor vendor-code vendor-override)
matching-location (or (when-not (str/blank? location-override)
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}]
(let [matching-client (cond
account-number (d-clients/exact-match account-number)
customer-identifier (d-clients/best-match customer-identifier)
client-override (Long/parseLong client-override))
matching-vendor (match-vendor vendor-code vendor-override)
matching-location (or (when-not (str/blank? location-override)
location-override)
(parse/best-location-match matching-client text full-text))]
(remove-nils #:invoice {:invoice/client (:db/id matching-client)
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/similarity (some-> similarity double (#(- 1.0 %)))
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid})))
(parse/best-location-match (dc/pull (dc/db conn)
[{:client/location-matches [:location-match/location :location-match/matches]}
:client/default-location
:client/locations]
matching-client)
text
full-text))]
#:invoice {:invoice/client matching-client
:invoice/client-identifier (or account-number customer-identifier)
:invoice/vendor (:db/id matching-vendor)
:invoice/source-url source-url
:invoice/invoice-number invoice-number
:invoice/total (Double/parseDouble total)
:invoice/date (to-date date)
:invoice/location matching-location
:invoice/import-status (or import-status :import-status/pending)
:invoice/outstanding-balance (Double/parseDouble total)
:invoice/status :invoice-status/unpaid}))
(defn validate-invoice [invoice user]
(when-not (:invoice/client invoice)
@@ -168,7 +177,7 @@
(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}]
(try
(->> (d/query
(->> (dc/q
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
:in ['$ '?invoice-number '?vendor '?client]
:where '[[?e :invoice/invoice-number ?invoice-number]
@@ -178,7 +187,7 @@
[?e :invoice/status ?status]
[?e :invoice/import-status ?import-status]
[?import-status :db/ident ?import-status2]]}
:args [(d/db (d/connect uri)) invoice-number vendor client]}))
:args [(dc/db conn) invoice-number vendor client]}))
first
boolean)
(catch Exception e
@@ -222,13 +231,13 @@
:status :payment-status/cleared
:date (:invoice/date invoice)})})
transaction (when (= :invoice-status/paid (:invoice/status invoice))
(let [[bank-account] (d/q '[:find [?ba ...]
:in $ ?c
:where [?c :client/bank-accounts ?ba]
[?ba :bank-account/type :bank-account-type/cash]
]
(d/db conn)
client-id)]
(let [[[bank-account]] (dc/q '[:find ?ba
:in $ ?c
:where [?c :client/bank-accounts ?ba]
[?ba :bank-account/type :bank-account-type/cash]
]
(dc/db conn)
client-id)]
#:transaction {:amount (- (:invoice/total invoice))
:payment payment-id
:client (:invoice/client invoice)
@@ -244,8 +253,8 @@
:transaction-account/location "A"
:transaction-account/amount (Math/abs (:invoice/total invoice))}]}))
]
[[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice)
user))]
[`(d-invoices/propose-invoice ~(d-invoices/code-invoice (validate-invoice (remove-nils invoice)
user)))
(some-> payment remove-nils)
transaction])))
(filter identity)))
@@ -260,19 +269,23 @@
(defn import-uploaded-invoice [user imports]
(lc/with-context {:area "upload-invoice"}
(log/info "Number of invoices to import is" (count imports))
(let [clients (d-clients/get-all)
potential-invoices (->> imports
(mapv #(import->invoice % clients))
(mapv #(validate-invoice % user))
(let [potential-invoices (->> imports
(map import->invoice)
(map #(validate-invoice % user))
admin-only-if-multiple-clients
(filter #(not (extant-invoice? %)))
(mapv d-invoices/code-invoice)
(mapv (fn [i] [:propose-invoice i])))]
(when-not (seq potential-invoices)
(throw (ex-info "No new invoices found."
{})))
(mapv (fn [i] `(d-invoices/propose-invoice ~i))))]
(log/info "creating invoice" potential-invoices)
(transact-with-ledger potential-invoices user))))
(let [tx (transact-with-ledger potential-invoices user)]
(when-not (seq (dc/q '[:find ?i
:in $ [?i ...]
:where [?i :invoice/invoice-number]]
(:db-after tx)
(map :e (:tx-data tx))))
(throw (ex-info "No new invoices found."
{})))
tx))))
(defn validate-account-rows [rows code->existing-account]
(when-let [bad-types (seq (->> rows
@@ -296,23 +309,22 @@
{:rows duplicate-rows}))))
(defn import-account-overrides [customer filename]
(let [conn (d/connect uri)
[_ & rows] (-> filename (io/reader) csv/read-csv)
[client-id] (first (d/query (-> {:query {:find ['?e]
:in ['$ '?z]
:where [['?e :client/code '?z]]}
:args [(d/db (d/connect uri)) customer]})))
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
(let [[_ & rows] (-> filename (io/reader) csv/read-csv)
[client-id] (first (dc/q (-> {:query {:find ['?e]
:in ['$ '?z]
:where [['?e :client/code '?z]]}
:args [(dc/db conn) customer]})))
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
{:account/applicability [:db/ident]}
:db/id])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db conn)]})))
:args [(dc/db conn)]})))
existing-account-overrides (d/query (-> {:query {:find ['?e]
:in ['$ '?client-id]
:where [['?e :account-client-override/client '?client-id]]}
:args [(d/db (d/connect uri)) client-id]}))
existing-account-overrides (dc/q (-> {:query {:find ['?e]
:in ['$ '?client-id]
:where [['?e :account-client-override/client '?client-id]]}
:args [(dc/db conn) client-id]}))
rows (transduce (comp
(map (fn [[_ account account-name override-name _ type]]
[account account-name override-name type]))
@@ -367,21 +379,21 @@
existing-account-overrides)
rows)]
@(d/transact conn txes)
(dc/transact conn {:tx-data txes})
txes))
(defn import-transactions-cleared-against [file]
(let [[_ & rows] (-> file (io/reader) csv/read-csv)
txes (transduce
(comp
(filter (fn [[transaction-id _]]
(d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id))))
(map (fn [[transaction-id cleared-against]]
{:db/id (Long/parseLong transaction-id)
:transaction/cleared-against cleared-against})))
conj
[]
rows)]
txes (transduce
(comp
(filter (fn [[transaction-id _]]
(dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id))))
(map (fn [[transaction-id cleared-against]]
{:db/id (Long/parseLong transaction-id)
:transaction/cleared-against cleared-against})))
conj
[]
rows)]
(transact-with-ledger txes nil)))
(defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}]

24
src/clj/auto_ap/routes/queries.clj
View File

@@ -1,7 +1,7 @@
(ns auto-ap.routes.queries
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic :refer [conn pull-attr upsert-entity]]
[auto-ap.graphql.utils :refer [assert-admin]]
[clojure.data.csv :as csv]
[clojure.edn :as edn]
@@ -9,8 +9,11 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[config.core :refer [env]]
[datomic.client.api :as dc]
[datomic.api :as d]
[compojure.core
:refer [context defroutes GET POST PUT routes wrap-routes]]
[ring.middleware.json :refer [wrap-json-response]]
[ring.util.request :refer [body-string]]
[unilog.context :as lc])
@@ -34,21 +37,20 @@
(let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env)
:key (str "queries/" (:query-id params))))))]
(log/info "Executing query " query-string)
(into (list) (apply d/q (edn/read-string query-string)
(into [(d/db conn)] (edn/read-string (get query-params "args" "[]")))))))))
(into (list) (apply dc/q (edn/read-string query-string)
(into [(dc/db conn)] (edn/read-string (get query-params "args" "[]")))))))))
(defn put-query [guid body note & [lookup-key client]]
(let [guid (if lookup-key
(or (:saved-query/guid (d/pull (d/db conn) [:saved-query/guid] [:saved-query/lookup-key lookup-key]))
(or (pull-attr (dc/db conn) :saved-query/guid [:saved-query/lookup-key lookup-key])
guid)
guid)]
@(d/transact conn [(cond->
{:saved-query/guid guid
:saved-query/description note
:saved-query/key (str "queries/" guid)}
client (assoc :saved-query/client client)
lookup-key (assoc :saved-query/lookup-key lookup-key))])
(dc/transact conn [`(upsert-entity ~{:saved-query/guid guid
:saved-query/description note
:saved-query/key (str "queries/" guid)
:saved-query/client client
:saved-query/lookup-key lookup-key})])
(s3/put-object :bucket-name (:data-bucket env)
:key (str "queries/" guid)
:input-stream (io/make-input-stream (.getBytes body) {})

6
src/clj/auto_ap/routes/yodlee2.clj
View File

@@ -1,16 +1,16 @@
(ns auto-ap.routes.yodlee2
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic :refer [conn pull-attr]]
[auto-ap.datomic.clients :as d-clients]
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
[auto-ap.routes.utils :refer [wrap-secure]]
[auto-ap.yodlee.core2 :as yodlee]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[datomic.api :as d]))
[datomic.client.api :as dc]))
(defn fastlink [{:keys [query-params identity]}]
(assert-can-see-client identity (d/pull (d/db conn) [:db/id] [:client/code (get query-params "client")]))
(assert-can-see-client identity (pull-attr (dc/db conn) :db/id [:client/code (get query-params "client")]))
(let [token (if-let [client-id (get query-params "client-id")]
(-> client-id

72
src/clj/auto_ap/search.clj Normal file
View File

@@ -0,0 +1,72 @@
(ns auto-ap.search
(:import
(java.nio.file Paths)
(org.apache.lucene.analysis.standard StandardAnalyzer)
(org.apache.lucene.document Document Field$Store StoredField StringField TextField)
(org.apache.lucene.index DirectoryReader IndexWriter IndexWriterConfig Term)
(org.apache.lucene.queryparser.classic QueryParser)
(org.apache.lucene.search BooleanClause$Occur BooleanQuery$Builder IndexSearcher PhraseQuery$Builder Query TermQuery)
(org.apache.lucene.store FSDirectory)))
(defn full-index-query [results index-name]
(let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name))))
analyzer (StandardAnalyzer.)
index-writer-config (IndexWriterConfig. analyzer)
index-writer (IndexWriter. directory index-writer-config)]
(.deleteAll index-writer)
(try
(doseq [{:keys [text id] :as x} results
:let [doc (doto
(Document.)
(.add (TextField. "name" text Field$Store/YES))
(.add (StoredField. "id" (long id))))]]
(doseq [k (filter (complement #{:text :id}) (keys x))]
(println "K" (name k) (get x k))
(.add doc (StringField. (name k) (str (get x k)) Field$Store/YES)))
(println "adding" text)
(flush)
(.addDocument index-writer doc))
(finally
(.close index-writer)))))
(defn make-query [n]
(let [
text-query (when (:q n)
(.parse (QueryParser. "name" (StandardAnalyzer.)) (:q n)))
text-query-exact (when (:q-exact n)
(.build (doto (PhraseQuery$Builder. )
(.add (Term. "name" (:q-exact n)) 0))))
full-query (BooleanQuery$Builder.)
]
(when text-query
(.add full-query text-query BooleanClause$Occur/MUST))
(when text-query-exact
(.add full-query text-query-exact BooleanClause$Occur/MUST))
(doseq [[k v] (dissoc n :q :q-exact)]
(if (instance? Query v)
(.add full-query v BooleanClause$Occur/MUST)
(.add full-query (TermQuery. (Term. (name k) (str v))) BooleanClause$Occur/MUST)))
(.build full-query)))
(defn search
([n index-name]
(search n index-name []))
([n index-name other-keys]
(let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name))))
index-reader (DirectoryReader/open directory)
index-searcher (IndexSearcher. index-reader)]
(for [x (seq (.scoreDocs (.search index-searcher (make-query n) 10)))]
(into
[(.get (.doc index-searcher (.-doc x)) "id")
(.get (.doc index-searcher (.-doc x)) "name")]
(map (fn [o]
(.get (.doc index-searcher (.-doc x)) o))
other-keys)))))
)
(defn search-ids [n index-name]
(let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name))))
index-reader (DirectoryReader/open directory)
index-searcher (IndexSearcher. index-reader)]
(for [x (seq (.scoreDocs (.search index-searcher (make-query n) 100)))]
(Long/parseLong (.get (.doc index-searcher (.-doc x)) "id")))))

4
src/clj/auto_ap/server.clj
View File

@@ -86,7 +86,6 @@
(mount/stop))
(defn -main [& _]
(let [job (System/getenv "INTEGREAT_JOB")]
(cond (= job "square-import-job")
(job-square/-main)
@@ -138,12 +137,9 @@
(job-bulk-journal-import/-main)
:else
(do
(add-shutdown-hook! shutdown-mount)
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
(mount/start)
#_(alter-var-root #'nrepl.middleware.print/*print-fn* (constantly clojure.pprint/pprint))))))
(comment
)

78
src/clj/auto_ap/square/core.clj
View File

@@ -1,6 +1,6 @@
(ns auto-ap.square.core
(:require
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.datomic :refer [conn random-tempid remove-nils]]
[auto-ap.time :as atime]
[clj-http.client :as client]
[clj-time.coerce :as coerce]
@@ -12,7 +12,7 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.api :as d]
[datomic.client.api :as dc]
[slingshot.slingshot :refer [try+]]
[unilog.context :as lc]))
@@ -395,11 +395,11 @@
(upsert client square-location (time/plus (time/now) (time/days -3)) (time/now))))
([client location start end]
(lc/with-context {:source "Square loading"}
(let [existing (->> (d/query {:query {:find ['?external-id]
(let [existing (->> (dc/q {:query {:find ['?external-id]
:in ['$ '?client]
:where ['[?o :sales-order/client ?client]
'[?o :sales-order/external-id ?external-id]]}
:args [(d/db conn) (:db/id client)]})
:args [(dc/db conn) (:db/id client)]})
(map first)
set)
_ (log/info (count existing) "Sales orders already exist")
@@ -407,7 +407,7 @@
(daily-results client location start end))]
(doseq [x (partition-all 20 to-create)]
(log/info "Loading " (count x))
@(d/transact conn x))))))
(dc/transact conn {:tx-data x}))))))
(defn upsert-settlements
([client]
@@ -419,7 +419,7 @@
:client (:client/code client)}
(doseq [x (partition-all 20 (daily-settlements client location))]
(log/info "Loading expected deposit" (count x))
@(d/transact conn x))
(dc/transact conn {:tx-data x}))
(log/info "Done loading settlements"))))
(defn upsert-refunds
@@ -433,7 +433,7 @@
:location (:square-location/client-location client)}
(doseq [x (partition-all 20 (refunds client location))]
(log/info "Loading refund" (count x))
@(d/transact conn x))
(dc/transact conn {:tx-data x}))
(log/info "Done loading refunds"))))
(def square-read [:db/id
@@ -443,26 +443,26 @@
(defn get-square-clients
([]
(d/q '[:find [(pull ?c [:db/id
:client/square-integration-status
:client/code
:client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...]
:in $
:where [?c :client/square-auth-token]
(not [?c :client/feature-flags "new-square"])]
(d/db conn)))
(map first (dc/q '[:find (pull ?c [:db/id
:client/square-integration-status
:client/code
:client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}])
:in $
:where [?c :client/square-auth-token]
(not [?c :client/feature-flags "new-square"])]
(dc/db conn))))
([ & codes]
(d/q '[:find [(pull ?c [:db/id
:client/code
:client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...]
:in $ [?code ...]
:where [?c :client/square-auth-token]
(not [?c :client/feature-flags "new-square"])
[?c :client/code ?code]]
(d/db conn)
codes)))
(map first (dc/q '[:find (pull ?c [:db/id
:client/code
:client/square-auth-token
{:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}])
:in $ [?code ...]
:where [?c :client/square-auth-token]
(not [?c :client/feature-flags "new-square"])
[?c :client/code ?code]]
(dc/db conn)
codes))))
(defn upsert-locations
([] (doseq [client (get-square-clients)]
@@ -474,31 +474,29 @@
[(:square-location/square-id sl)
(:db/id sl)])
(:client/square-locations client)))]
(->> (for [square-location (client-locations client)]
{:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user))
:client/_square-locations (:db/id client)
:square-location/name (:name square-location)
:square-location/square-id (:id square-location)})
(d/transact conn)
deref))))
(dc/transact conn {:tx-data (for [square-location (client-locations client)]
{:db/id (or (square-id->id (:id square-location)) (random-tempid))
:client/_square-locations (:db/id client)
:square-location/name (:name square-location)
:square-location/square-id (:id square-location)})}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn reset []
(->>
(d/query {:query {:find ['?e]
(dc/q {:query {:find ['?e]
:in ['$]
:where ['(or [?e :sales-order/date]
[?e :expected-deposit/date])]}
:args [(d/db conn)]})
:args [(dc/db conn)]})
(map first)
(map (fn [x] [:db/retractEntity x]))))
(defn mark-integration-status [client integration-status]
@(d/transact conn
[{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}]))
(dc/transact conn
{:tx-data [{:db/id (:db/id client)
:client/square-integration-status (assoc integration-status
:db/id (or (-> client :client/square-integration-status :db/id)
#db/id [:db.part/user]))}]}))
(defn upsert-all [ & clients]
(doseq [client (apply get-square-clients clients)

10
src/clj/auto_ap/yodlee/core2.clj
View File

@@ -11,7 +11,7 @@
#_{:clj-kondo/ignore [:unused-namespace]}
[yang.scheduler :as scheduler]
[clj-time.coerce :as coerce]
[datomic.api :as d]
[datomic.client.api :as dc]
[auto-ap.datomic :refer [conn]]
[auto-ap.datomic.clients :as d-clients]))
;; switch all of this to use tokens instead of passing around client codes, particularly because the codes
@@ -293,7 +293,7 @@
:body
:providerAccount
first))
@(d/transact conn [[:db/retractEntity (:db/id (d/entity (d/db conn) [:yodlee-provider-account/id id]))]]))
(dc/transact conn {:tx-data [[:db/retractEntity [:yodlee-provider-account/id id]]]}))
(defn upsert-accounts-tx
([client-code]
@@ -322,8 +322,8 @@
(defn refresh-provider-account [client-code id]
(log/info "refreshing yodlee provider account id" id)
@(d/transact conn (upsert-accounts-tx client-code
[(get-provider-account client-code id)])))
(dc/transact conn {:tx-data (upsert-accounts-tx client-code
[(get-provider-account client-code id)])}))
(defn upsert-accounts []
(let [concurrent 20
@@ -341,7 +341,7 @@
(async/to-chan! (d-clients/get-all)))
(let [result (async/<!! (async/into [] output-chan))]
(log/info "Current yodlee state is " result)
@(d/transact conn result))))
(dc/transact conn {:tx-data result}))))

251
src/clj/user.clj
View File

@@ -1,7 +1,7 @@
(ns user
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [uri]]
[auto-ap.datomic :refer [conn pull-attr random-tempid]]
[auto-ap.ledger :as l :refer [transact-with-ledger]]
[auto-ap.server]
[auto-ap.square.core :as square]
@@ -17,7 +17,7 @@
[clojure.pprint]
[clojure.string :as str]
[config.core :refer [env]]
[datomic.api :as d]
[datomic.client.api :as dc]
[mount.core :as mount]
[nrepl.middleware.print]
[unilog.context :as lc]
@@ -70,71 +70,71 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn mark-until-date [client end]
(let [conn (d/connect uri)]
(doseq [p (->>
(d/query {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :invoice/client ?c]
'[?c :client/code ?client]
'[?e :invoice/date ?d ]
'[(<= ?d ?end) ]]}
:args [(d/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:invoice/exclude-from-ledger true}))
(partition-all 100))]
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :invoice/client ?c]
'[?c :client/code ?client]
'[?e :invoice/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:invoice/exclude-from-ledger true}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100"))
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100"))
(doseq [p (->>
(d/query {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :transaction/client ?c]
'[?c :client/code ?client]
'[?e :transaction/date ?d ]
'[(<= ?d ?end) ]]}
:args [(d/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:transaction/approval-status :transaction-approval-status/excluded}))
(partition-all 100))]
(doseq [p (->>
(dc/q {:query {:find '[?e]
:in '[$ ?client ?end ]
:where [
'[?e :transaction/client ?c]
'[?c :client/code ?client]
'[?e :transaction/date ?d ]
'[(<= ?d ?end) ]]}
:args [(dc/db conn)
client
(c/to-date end)]})
(mapv first)
(mapv (fn [i]
{:db/id i
:transaction/approval-status :transaction-approval-status/excluded}))
(partition-all 100))]
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) (println "process 100"))))
(transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"})
(println "process 100")))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn load-accounts [conn]
(let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
:db/id])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db conn)]})))
:args [(dc/db conn)]})))
also-merge-txes (fn [also-merge old-account-id]
(if old-account-id
(let [[sunset-account]
(first (d/query {:query {:find ['?a ]
:in ['$ '?ac ]
:where ['[?a :account/numeric-code ?ac]]}
:args [(d/db conn) also-merge ]}))]
(first (dc/q {:query {:find ['?a ]
:in ['$ '?ac ]
:where ['[?a :account/numeric-code ?ac]]}
:args [(dc/db conn) also-merge ]}))]
(into (mapv
(fn [[entity id _]]
[:db/add entity id old-account-id])
(d/query {:query {:find ['?e '?id '?a ]
(dc/q {:query {:find ['?e '?id '?a ]
:in ['$ '?ac ]
:where ['[?a :account/numeric-code ?ac]
'[?e ?at ?a]
'[?at :db/ident ?id]]}
:args [(d/db conn) also-merge ]}))
:args [(dc/db conn) also-merge ]}))
[[:db/retractEntity sunset-account]]))
[]))
@@ -186,32 +186,33 @@
conj
[]
rows)]
@(d/transact conn txes)))
(dc/transact conn {:tx-data txes})))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn find-bad-accounts []
(set (map second (d/query {:query {:find ['(pull ?x [*]) '?z]
(set (map second (dc/q {:query {:find ['(pull ?x [*]) '?z]
:in ['$]
:where ['[?e :account/numeric-code ?z]
'[(<= ?z 9999)]
'[?x ?a ?e]]}
:args [(d/db (d/connect uri))]}))))
:args [(dc/db conn)]}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn delete-4-digit-accounts []
@(d/transact (d/connect uri)
(transduce
(comp
(map first)
(map (fn [old-account-id]
[:db/retractEntity old-account-id])))
conj
[]
(d/query {:query {:find ['?e]
:in ['$]
:where ['[?e :account/numeric-code ?z]
'[(<= ?z 9999)]]}
:args [(d/db (d/connect uri))]})))
(dc/transact conn
{:tx-data
(transduce
(comp
(map first)
(map (fn [old-account-id]
[:db/retractEntity old-account-id])))
conj
[]
(dc/q {:query {:find ['?e]
:in ['$]
:where ['[?e :account/numeric-code ?z]
'[(<= ?z 9999)]]}
:args [(dc/db conn)]}))})
)
@@ -224,31 +225,30 @@
(fn [acc [e z]]
(update acc z conj e))
{}
(d/query {:query {:find ['?e '?z]
:in ['$]
:where ['[?e :account/numeric-code ?z]]}
:args [(d/db (d/connect uri))]}))))
(dc/q {:query {:find ['?e '?z]
:in ['$]
:where ['[?e :account/numeric-code ?z]]}
:args [(dc/db conn)]}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn customize-accounts [customer filename]
(let [conn (d/connect uri)
[_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
[client-id] (first (d/query (-> {:query {:find ['?e]
(let [[_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
[client-id] (first (dc/q (-> {:query {:find ['?e]
:in ['$ '?z]
:where [['?e :client/code '?z]]}
:args [(d/db (d/connect uri)) customer]})))
:args [(dc/db conn) customer]})))
_ (println client-id)
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
{:account/applicability [:db/ident]}
:db/id])]
:in ['$]
:where ['[?e :account/name]]}
:args [(d/db conn)]})))
:args [(dc/db conn)]})))
existing-account-overrides (d/query (-> {:query {:find ['?e]
existing-account-overrides (dc/q (-> {:query {:find ['?e]
:in ['$ '?client-id]
:where [['?e :account-client-override/client '?client-id]]}
:args [(d/db (d/connect uri)) client-id]}))
:args [(dc/db conn) client-id]}))
@@ -315,16 +315,16 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn fix-transactions-without-locations [client-code location]
(->>
(d/query {:query {:find ['(pull ?e [*])]
:in ['$ '?client-code]
:where ['[?e :transaction/accounts ?ta]
'[?e :transaction/matched-rule]
'[?e :transaction/approval-status :transaction-approval-status/approved]
'(not [?ta :transaction-account/location])
'[?e :transaction/client ?c]
'[?c :client/code ?client-code]
]}
:args [(d/db (d/connect uri)) client-code]})
(dc/q {:query {:find ['(pull ?e [*])]
:in ['$ '?client-code]
:where ['[?e :transaction/accounts ?ta]
'[?e :transaction/matched-rule]
'[?e :transaction/approval-status :transaction-approval-status/approved]
'(not [?ta :transaction-account/location])
'[?e :transaction/client ?c]
'[?c :client/code ?client-code]
]}
:args [(dc/db conn) client-code]})
(mapcat
(fn [[{:transaction/keys [accounts]}]]
(mapv
@@ -405,44 +405,48 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn entity-history [i]
(vec (sort-by first (d/query
(vec (sort-by first (dc/q
{:query {:find ['?tx '?z '?v ]
:in ['?i '$]
:where ['[?i ?a ?v ?tx ?ad]
'[?a :db/ident ?z]
'[(= ?ad true)]]}
:args [i (d/history (d/db (d/connect uri)))]}))))
:args [i (dc/history (dc/db conn))]}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn entity-history-with-revert [i]
(vec (sort-by first (d/query
(vec (sort-by first (dc/q
{:query {:find ['?tx '?z '?v '?ad ]
:in ['?i '$]
:where ['[?i ?a ?v ?tx ?ad]
'[?a :db/ident ?z]]}
:args [i (d/history (d/db (d/connect uri)))]}))))
:args [i (dc/history (dc/db conn))]}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn tx-detail [i]
(map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v :added)
(map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v)
(:data (first
(d/tx-range (d/log (d/connect uri))
i
(inc i))))))
(dc/tx-range conn
{:start i
:end (inc i)})))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn tx-range-detail [i]
(map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v)
(map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v)
(mapcat :data (d/tx-range (d/log (d/connect uri))
(- i 100)
(+ i 100)))))
(mapcat :data (dc/tx-range conn
{:start (- i 100)
:end (+ i 100)}))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn start-db []
(mu/start-publisher! {:type :dev})
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn})))
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
(defn restart-db []
(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})
(mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))
(start-db))
@@ -481,29 +485,15 @@
(async/<!! (async/into [] output-chan))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn load-sales-for-day [date]
(doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...]
:where ['?e :client/square-locations ]]
(d/db auto-ap.datomic/conn))
square-location (:client/square-locations client)
:when (:square-location/client-location square-location)]
(println client)
(println "orders")
(lc/with-context {:source "Historical loading data"}
(square/upsert client square-location (c/to-date-time date)
(t/plus (c/to-date-time date) (t/days 1))))))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn upsert-invoice-amounts [tsv]
(let [data (with-open [reader (io/reader (char-array tsv))]
(doall (csv/read-csv reader :separator \tab)))
db (d/db auto-ap.datomic/conn)
db (dc/db conn)
i->invoice-id (fn [i]
(try (Long/parseLong i)
(catch Exception e
(:db/id (d/pull db '[:db/id]
(:db/id (dc/pull db '[:db/id]
[:invoice/original-id (Long/parseLong (first (str/split i #"-")))])))))
invoice-totals (->> data
(drop 1)
@@ -521,7 +511,7 @@
:let [
invoice-id (i->invoice-id i)
invoice (d/entity db invoice-id)
invoice (dc/pull db '[FILL_IN] invoice-id)
current-total (:invoice/total invoice)
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
@@ -529,10 +519,9 @@
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
(:db/id (first (:invoice/expense-accounts invoice)))
(d/tempid :db.part/user))
(random-tempid))
invoice-expense-account (when-not new-account?
(or (d/entity db invoice-expense-account-id)
(d/entity db [:invoice-expense-account/original-id invoice-expense-account-id])))
(dc/pull db '[FILL_IN]invoice-expense-account-id))
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
target-account-id (Long/parseLong (str/trim target-account))
@@ -548,19 +537,19 @@
target-expense-account-location location
[[_ _ invoice-payment]] (vec (d/q
'[:find ?p ?a ?ip
:in $ ?i
:where [?ip :invoice-payment/invoice ?i]
[?ip :invoice-payment/amount ?a]
[?ip :invoice-payment/payment ?p]
]
db invoice-id))]
[[_ _ invoice-payment]] (vec (dc/q
'[:find ?p ?a ?ip
:in $ ?i
:where [?ip :invoice-payment/invoice ?i]
[?ip :invoice-payment/amount ?a]
[?ip :invoice-payment/payment ?p]
]
db invoice-id))]
:when current-total]
[
(when (not (auto-ap.utils/dollars= current-total target-total))
{:db/id invoice-id
{:db/id invoice-id
:invoice/total target-total})
(when new-account?
@@ -596,19 +585,19 @@
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-schema [prefix]
(->> (d/q '[:find ?i
(->> (dc/q '[:find ?i
:in $ ?p
:where [_ :db/ident ?i]
[(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix)
[(namespace ?i) ?p]] (dc/db auto-ap.datomic/conn) prefix)
(mapcat identity)
vec))
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
(defn get-idents []
(->> (d/q '[:find ?i
(->> (dc/q '[:find ?i
:in $
:where [_ :db/ident ?i]]
(d/db auto-ap.datomic/conn) )
(dc/db conn) )
(mapcat identity)
(map str)
(sort)

2
src/cljs/auto_ap/events.cljs
View File

@@ -30,7 +30,7 @@
[:intuit-bank-account [:name :id :external-id]]
:use-date-instead-of-post-date
:locations :include-in-reports :current-balance :yodlee-balance-old] ]
[:address [:street1 :street2 :city :state :zip]]
[:address [:id :street1 :street2 :city :state :zip]]
[:forecasted-transactions [:id :amount :identifier :day-of-month]]]
(= "admin" (or (get (jwt->data token) "role") (get (jwt->data token) "user/role")) ) (into [[:yodlee-provider-accounts [:id [:accounts [:id :name :number :available-balance]]]]
[:plaid-items [:id [:accounts [:id :name :number :balance]]]]])))

2
src/cljs/auto_ap/views/pages/admin/accounts.cljs
View File

@@ -16,7 +16,7 @@
[reagent.core :as reagent]
[vimsical.re-frame.fx.track :as track]))
(def default-read [:numeric-code :name :location :type :account_set :applicability :invoice-allowance :vendor-allowance :id [:client-overrides [:name [:client [:name :id]]]]])
(def default-read [:numeric-code :name :location :type :account_set :applicability :invoice-allowance :vendor-allowance :id [:client-overrides [:id :name [:client [:name :id]]]]])

3
src/cljs/auto_ap/views/pages/admin/clients/form.cljs
View File

@@ -169,7 +169,8 @@
:week-a-debits (:week-a-debits new-client-data)
:week-b-credits (:week-b-credits new-client-data)
:week-b-debits (:week-b-debits new-client-data)
:address {:street1 (:street1 (:address new-client-data))
:address {:id (:id (:address new-client-data))
:street1 (:street1 (:address new-client-data))
:street2 (:street2 (:address new-client-data)),
:city (:city (:address new-client-data))
:state (:state (:address new-client-data))