Makes integreat run on datomic cloud
This commit is contained in:
@@ -1,21 +1,17 @@
|
||||
(ns auto-ap.ledger
|
||||
(:require
|
||||
[auto-ap.datomic :refer [audit-transact conn remove-nils]]
|
||||
[auto-ap.logging :refer [info-event]]
|
||||
[auto-ap.datomic :refer [conn remove-nils pull-ref audit-transact]]
|
||||
[auto-ap.utils :refer [dollars-0? dollars=]]
|
||||
[clj-time.coerce :as c]
|
||||
[clj-time.core :as t]
|
||||
[clojure.tools.logging :as log]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[datomic.api :as d]
|
||||
[mount.core :as mount]
|
||||
[unilog.context :as lc]
|
||||
[yang.scheduler :as scheduler]))
|
||||
[datomic.client.api :as dc]))
|
||||
|
||||
(defn datums->impacted-entity [db [e changes]]
|
||||
(let [entity (d/pull db '[* {:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
|
||||
(let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
|
||||
namespaces (->> changes
|
||||
(map :a)
|
||||
(map #(:db/ident (dc/pull db '[:db/ident] (:a %))))
|
||||
(map namespace)
|
||||
set)]
|
||||
(cond (namespaces "invoice" ) [[:invoice e]]
|
||||
@@ -31,7 +27,7 @@
|
||||
(defmethod entity-change->ledger :invoice
|
||||
[db [_ id]]
|
||||
(when id
|
||||
(let [entity (d/pull db ['* {:invoice/vendor '[*]
|
||||
(let [entity (dc/pull db ['* {:invoice/vendor '[*]
|
||||
:invoice/payment '[*]
|
||||
:invoice/status '[:db/ident]
|
||||
:invoice/import-status '[:db/ident]}] id)
|
||||
@@ -71,7 +67,7 @@
|
||||
(defmethod entity-change->ledger :transaction
|
||||
[db [_ id]]
|
||||
(when id
|
||||
(let [entity (d/pull db ['* {:transaction/vendor '[*]
|
||||
(let [entity (dc/pull db ['* {:transaction/vendor '[*]
|
||||
:transaction/client '[*]
|
||||
:transaction/approval-status '[*]
|
||||
:transaction/bank-account '[* {:bank-account/type [:db/ident]}]
|
||||
@@ -144,13 +140,12 @@
|
||||
[_ _]
|
||||
nil)
|
||||
|
||||
|
||||
(defn reconcile-ledger
|
||||
([] (reconcile-ledger (-> (t/now)
|
||||
(t/plus (t/months -6))
|
||||
(c/to-date))))
|
||||
([start-date]
|
||||
(let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
|
||||
(let [txes-missing-ledger-entries (->> (dc/q {:query {:find ['?t ]
|
||||
:in ['$ '?sd]
|
||||
:where [
|
||||
'[?t :transaction/date ?d]
|
||||
@@ -160,12 +155,12 @@
|
||||
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
|
||||
'(not [?t :transaction/approval-status :transaction-approval-status/suppressed])
|
||||
]}
|
||||
:args [(d/db conn) start-date]})
|
||||
:args [(dc/db conn) start-date]})
|
||||
(map first)
|
||||
(mapv #(entity-change->ledger (d/db conn) [:transaction %])))
|
||||
(mapv #(entity-change->ledger (dc/db conn) [:transaction %])))
|
||||
|
||||
|
||||
invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ]
|
||||
invoices-missing-ledger-entries (->> (dc/q {:query {:find ['?t ]
|
||||
:in ['$ '?sd]
|
||||
:where ['[?t :invoice/date ?d]
|
||||
'[(>= ?d ?sd)]
|
||||
@@ -176,45 +171,71 @@
|
||||
'(not [?t :invoice/import-status :import-status/pending])
|
||||
'(not [?t :invoice/exclude-from-ledger true])
|
||||
]}
|
||||
:args [(d/db conn) start-date]})
|
||||
:args [(dc/db conn) start-date]})
|
||||
(map first)
|
||||
(mapv #(entity-change->ledger (d/db conn) [:invoice %])))
|
||||
(mapv #(entity-change->ledger (dc/db conn) [:invoice %])))
|
||||
repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))]
|
||||
(when (seq repairs)
|
||||
(log/info (take 3 repairs))
|
||||
(log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries")
|
||||
@(d/transact conn repairs)))))
|
||||
(dc/transact conn {:tx-data repairs})))))
|
||||
|
||||
|
||||
(defn touch-transaction [e]
|
||||
@(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
|
||||
(when-let [change (entity-change->ledger (d/db conn)
|
||||
[:transaction e])]
|
||||
@(d/transact conn [{:db/id "datomic.tx"
|
||||
:db/doc "touching transaction to update ledger"}
|
||||
change])))
|
||||
(dc/transact conn {:tx-data [[:db/retractEntity [:journal-entry/original-entity e]]]})
|
||||
(when-let [change (entity-change->ledger (d/db conn)
|
||||
[:transaction e])]
|
||||
(dc/transact conn {:tx-data [{:db/id "datomic.tx"
|
||||
:db/doc "touching transaction to update ledger"}
|
||||
(entity-change->ledger (dc/db conn)
|
||||
[:transaction e])]}))
|
||||
)
|
||||
|
||||
(defn touch-invoice [e]
|
||||
@(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
|
||||
(when-let [change (entity-change->ledger (d/db conn)
|
||||
[:invoice e])]
|
||||
@(d/transact conn [{:db/id "datomic.tx"
|
||||
:db/doc "touching invoice to update ledger"}
|
||||
change])))
|
||||
(dc/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]])
|
||||
(when-let [change (entity-change->ledger (d/db conn)
|
||||
[:invoice e])]
|
||||
(dc/transact conn [{:db/id "datomic.tx"
|
||||
:db/doc "touching invoice to update ledger"}
|
||||
(entity-change->ledger (dc/db conn)
|
||||
[:invoice e])]))
|
||||
)
|
||||
|
||||
(defn lazy-tx-range
|
||||
([start end xf] (lazy-tx-range start end xf 0))
|
||||
([start end xf o]
|
||||
(let [next-results (dc/tx-range conn {:start start
|
||||
:end end
|
||||
:offset o
|
||||
:limit 200})]
|
||||
(lazy-seq
|
||||
(if (seq next-results)
|
||||
(concat (sequence (comp (mapcat :data)
|
||||
xf) next-results) (lazy-tx-range start
|
||||
end
|
||||
xf
|
||||
(+ (count next-results)
|
||||
o)))
|
||||
next-results)))))
|
||||
|
||||
|
||||
|
||||
(defn recently-changed-entities [start end]
|
||||
(set (map (fn [d]
|
||||
(:e d))
|
||||
(mapcat :data (dc/tx-range conn {:start start
|
||||
:end end})))))
|
||||
|
||||
(defn entities-since-last-ledger-entry []
|
||||
(count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5)))
|
||||
:end (coerce/to-date (time/now))})))
|
||||
|
||||
(defn mismatched-transactions
|
||||
([]
|
||||
(mismatched-transactions (c/to-date (t/minus (t/now) (t/days 7)))
|
||||
(c/to-date (t/minus (t/now) (t/hours 1)))) )
|
||||
([changed-between-start changed-between-end]
|
||||
(let [entities-to-consider (d/q '[:find [?t ...]
|
||||
:in $ ?log ?start ?end
|
||||
:where
|
||||
[(tx-ids ?log ?start ?end) [?tx ...]]
|
||||
[(tx-data ?log ?tx) [[?t]]]
|
||||
[?t :transaction/date]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(d/log auto-ap.datomic/conn)
|
||||
(let [entities-to-consider (recently-changed-entities
|
||||
changed-between-start
|
||||
changed-between-end)
|
||||
_ (log/info "checking" (count entities-to-consider) "transactions looking for mismatches between" changed-between-start changed-between-end)
|
||||
@@ -222,20 +243,21 @@
|
||||
(fn [acc [e lia]]
|
||||
(update acc e (fnil conj #{} ) lia))
|
||||
{}
|
||||
(d/q '[:find ?e ?lia
|
||||
(dc/q '[:find ?e ?lia
|
||||
:in $ [?e ...]
|
||||
:where
|
||||
[?je :journal-entry/original-entity ?e]
|
||||
[?e :transaction/date]
|
||||
[?je :journal-entry/line-items ?li]
|
||||
[?li :journal-entry-line/account ?lia]
|
||||
[?lia :account/name]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(dc/db conn)
|
||||
entities-to-consider))
|
||||
transaction-accounts (reduce
|
||||
(fn [acc [e lia]]
|
||||
(update acc e (fnil conj #{} ) lia))
|
||||
{}
|
||||
(d/q '[:find ?e ?lia
|
||||
(dc/q '[:find ?e ?lia
|
||||
:in $ [?e ...]
|
||||
:where
|
||||
[?e :transaction/date ?d]
|
||||
@@ -246,7 +268,7 @@
|
||||
[?lia :account/name]
|
||||
[?e :transaction/amount ?amt]
|
||||
[(not= ?amt 0.0)]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(dc/db conn)
|
||||
entities-to-consider))]
|
||||
(->> transaction-accounts
|
||||
(filter
|
||||
@@ -256,20 +278,9 @@
|
||||
([] (unbalanced-transactions (c/to-date (t/minus (t/now) (t/days 7)))
|
||||
(c/to-date (t/minus (t/now) (t/hours 1)))))
|
||||
([changed-between-start changed-between-end]
|
||||
(let [entities-to-consider (d/q '[:find [?je ...]
|
||||
:in $ ?log ?start ?end
|
||||
:where
|
||||
[(tx-ids ?log ?start ?end) [?tx ...]]
|
||||
[(tx-data ?log ?tx) [[?je]]]
|
||||
[?je :journal-entry/amount]
|
||||
[?je :journal-entry/original-entity ?i]
|
||||
[?i :transaction/date]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(d/log auto-ap.datomic/conn)
|
||||
changed-between-start
|
||||
changed-between-end)]
|
||||
(let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end)]
|
||||
(log/info "checking" (count entities-to-consider) "transaction journal entries looking for mismatches between" changed-between-start changed-between-end)
|
||||
(->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit)
|
||||
(->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit)
|
||||
:with ?jel
|
||||
:in $ [?je ...]
|
||||
:where [?je :journal-entry/amount ?a]
|
||||
@@ -277,52 +288,42 @@
|
||||
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
|
||||
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
|
||||
]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(dc/db conn)
|
||||
entities-to-consider)
|
||||
(filter (fn [[_ a d c]]
|
||||
(or (not (dollars= a d))
|
||||
(not (dollars= a c)))))
|
||||
(map first)
|
||||
(map (fn [je]
|
||||
(:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn)
|
||||
je))))
|
||||
(map :db/id)))))
|
||||
(pull-ref (dc/db conn) :journal-entry/original-entity je)))))))
|
||||
|
||||
|
||||
|
||||
(defn unbalanced-invoices
|
||||
([] (unbalanced-invoices (c/to-date (t/minus (t/now) (t/days 7)))
|
||||
(c/to-date (t/minus (t/now) (t/hours 1)))))
|
||||
([changed-between-start changed-between-end]
|
||||
(let [entities-to-consider (d/q '[:find [?je ...]
|
||||
:in $ ?log ?start ?end
|
||||
:where
|
||||
[(tx-ids ?log ?start ?end) [?tx ...]]
|
||||
[(tx-data ?log ?tx) [[?je]]]
|
||||
[?je :journal-entry/amount]
|
||||
[?je :journal-entry/original-entity ?i]
|
||||
[?i :invoice/date]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(d/log auto-ap.datomic/conn)
|
||||
changed-between-start
|
||||
changed-between-end)]
|
||||
(let [entities-to-consider (recently-changed-entities
|
||||
changed-between-start
|
||||
changed-between-end)]
|
||||
(log/info "checking" (count entities-to-consider) "invoice journal entries looking for mismatches between" changed-between-start changed-between-end)
|
||||
(->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit)
|
||||
:with ?jel
|
||||
:in $ [?je ...]
|
||||
:where [?je :journal-entry/amount ?a]
|
||||
[?je :journal-entry/line-items ?jel]
|
||||
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
|
||||
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]
|
||||
]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
entities-to-consider)
|
||||
(->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit)
|
||||
:with ?jel
|
||||
:in $ [?je ...]
|
||||
:where [?je :journal-entry/amount ?a]
|
||||
[?je :journal-entry/original-entity ?i]
|
||||
[?i :invoice/date]
|
||||
[?je :journal-entry/line-items ?jel]
|
||||
[(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit]
|
||||
[(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]]
|
||||
(dc/db conn)
|
||||
entities-to-consider)
|
||||
(filter (fn [[_ a d c]]
|
||||
(or (not (dollars= a d))
|
||||
(not (dollars= a c)))))
|
||||
(map first)
|
||||
(map (fn [je]
|
||||
(:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn)
|
||||
je))))
|
||||
(map :db/id)))))
|
||||
(pull-ref (dc/db conn) :journal-entry/original-entity je)))))))
|
||||
|
||||
(defn mismatched-invoices
|
||||
([]
|
||||
@@ -330,36 +331,28 @@
|
||||
(c/to-date (t/minus (t/now) (t/hours 1)))) )
|
||||
|
||||
([changed-between-start changed-between-end]
|
||||
(let [entities-to-consider (d/q '[:find [?i ...]
|
||||
:in $ ?log ?start ?end
|
||||
:where
|
||||
[(tx-ids ?log ?start ?end) [?tx ...]]
|
||||
[(tx-data ?log ?tx) [[?i]]]
|
||||
[?i :invoice/date]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(d/log auto-ap.datomic/conn)
|
||||
changed-between-start
|
||||
changed-between-end)
|
||||
(let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end)
|
||||
_ (log/info (count entities-to-consider) "invoices have changed between" changed-between-start "and" changed-between-end)
|
||||
jel-accounts (reduce
|
||||
(fn [acc [e lia]]
|
||||
(update acc e (fnil conj #{} ) lia))
|
||||
{}
|
||||
(d/q '[:find ?e ?lia
|
||||
(dc/q '[:find ?e ?lia
|
||||
:in $ [?e ...]
|
||||
:where
|
||||
[?je :journal-entry/original-entity ?e]
|
||||
[?e :invoice/date]
|
||||
[?je :journal-entry/line-items ?li]
|
||||
[?li :journal-entry-line/account ?lia]
|
||||
(not [?lia :account/numeric-code 21000])
|
||||
[?lia :account/name]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(dc/db conn)
|
||||
entities-to-consider))
|
||||
invoice-accounts (reduce
|
||||
(fn [acc [e lia]]
|
||||
(update acc e (fnil conj #{} ) lia))
|
||||
{}
|
||||
(d/q '[:find ?e ?lia
|
||||
(dc/q '[:find ?e ?lia
|
||||
:in $ [?e ...]
|
||||
:where
|
||||
[?e :invoice/expense-accounts ?li]
|
||||
@@ -370,7 +363,7 @@
|
||||
(not [?e :invoice/status :invoice-status/voided])
|
||||
(not [?e :invoice/exclude-from-ledger true])
|
||||
[?e :invoice/import-status :import-status/imported]]
|
||||
(d/db auto-ap.datomic/conn)
|
||||
(dc/db conn)
|
||||
entities-to-consider))
|
||||
]
|
||||
(filter
|
||||
@@ -427,12 +420,12 @@
|
||||
nil))
|
||||
|
||||
(defn transact-with-ledger [transaction id]
|
||||
(let [db (d/db conn)
|
||||
(let [db (dc/db conn)
|
||||
tx (audit-transact transaction id)
|
||||
affected-entities (->> (:tx-data tx)
|
||||
(map (fn [^datomic.db.Datum x]
|
||||
{:e (:e x)
|
||||
:a (d/ident db (:a x))
|
||||
:a (:a x)
|
||||
:v (:v x)
|
||||
:added (:added x)}))
|
||||
(group-by :e)
|
||||
|
||||
Reference in New Issue
Block a user