572 lines
25 KiB
Clojure
572 lines
25 KiB
Clojure
(ns user
|
|
(:require
|
|
[amazonica.aws.s3 :as s3]
|
|
[auto-ap.datomic :refer [conn pull-attr random-tempid]]
|
|
[auto-ap.ledger :as l ]
|
|
[auto-ap.server]
|
|
[auto-ap.square.core :as square]
|
|
[auto-ap.square.core2 :as square2]
|
|
[auto-ap.time :as atime]
|
|
[auto-ap.utils :refer [by]]
|
|
[clj-time.coerce :as c]
|
|
[clj-time.core :as t]
|
|
[clj-time.periodic :as per]
|
|
[clojure.core.async :as async]
|
|
[clojure.data.csv :as csv]
|
|
[clojure.java.io :as io]
|
|
[clojure.pprint]
|
|
[clojure.string :as str]
|
|
[config.core :refer [env]]
|
|
[datomic.client.api :as dc]
|
|
[mount.core :as mount]
|
|
[nrepl.middleware.print]
|
|
[unilog.context :as lc]
|
|
[com.brunobonacci.mulog :as mu]
|
|
[com.brunobonacci.mulog.buffer :as rb])
|
|
(:import
|
|
(org.apache.commons.io.input BOMInputStream)))
|
|
|
|
(defn println-event [item]
|
|
(printf "%s - %s: %s\n" (:mulog/namespace item) (:mulog/event-name item)
|
|
(pr-str (reduce
|
|
(fn [acc [k v]]
|
|
(if (= "mulog" (namespace k))
|
|
acc
|
|
(assoc acc k v)))
|
|
{}
|
|
item))))
|
|
|
|
|
|
(deftype DevPublisher [config buffer transform]
|
|
|
|
com.brunobonacci.mulog.publisher.PPublisher
|
|
(agent-buffer [_]
|
|
buffer)
|
|
|
|
|
|
(publish-delay [_]
|
|
200)
|
|
|
|
|
|
(publish [_ buffer]
|
|
;; items are pairs [offset <item>]
|
|
(doseq [item (transform (map second (rb/items buffer)))]
|
|
(println-event item)
|
|
)
|
|
(flush)
|
|
(rb/clear buffer)))
|
|
|
|
|
|
|
|
(defn dev-publisher
|
|
[{:keys [transform pretty?] :as config}]
|
|
(DevPublisher. config (rb/agent-buffer 10000) (or transform identity)))
|
|
|
|
(defmethod com.brunobonacci.mulog.publisher/publisher-factory :dev
|
|
[config]
|
|
(dev-publisher config))
|
|
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn load-accounts [conn]
|
|
(let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
|
|
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
|
|
:db/id])]
|
|
:in ['$]
|
|
:where ['[?e :account/name]]}
|
|
:args [(dc/db conn)]})))
|
|
|
|
also-merge-txes (fn [also-merge old-account-id]
|
|
(if old-account-id
|
|
(let [[sunset-account]
|
|
(first (dc/q {:query {:find ['?a ]
|
|
:in ['$ '?ac ]
|
|
:where ['[?a :account/numeric-code ?ac]]}
|
|
:args [(dc/db conn) also-merge ]}))]
|
|
(into (mapv
|
|
(fn [[entity id _]]
|
|
[:db/add entity id old-account-id])
|
|
(dc/q {:query {:find ['?e '?id '?a ]
|
|
:in ['$ '?ac ]
|
|
:where ['[?a :account/numeric-code ?ac]
|
|
'[?e ?at ?a]
|
|
'[?at :db/ident ?id]]}
|
|
:args [(dc/db conn) also-merge ]}))
|
|
[[:db/retractEntity sunset-account]]))
|
|
[]))
|
|
|
|
txes (transduce
|
|
(comp
|
|
(map (fn ->map [r]
|
|
(into {} (map vector header r))))
|
|
(map (fn parse-map [r]
|
|
{:old-account-id (:db/id (code->existing-account
|
|
(or
|
|
(if (= (get r "IOL Account #")
|
|
"NEW")
|
|
nil
|
|
(Integer/parseInt (get r "IOL Account #")))
|
|
(Integer/parseInt (get r "Account #")))))
|
|
:new-account-number (Integer/parseInt (get r "Account #"))
|
|
:name (get r "Default Name")
|
|
:location (when-not (str/blank? (get r "Forced Location"))
|
|
(get r "Forced Location"))
|
|
:also-merge (when-not (str/blank? (get r "IOL # additional"))
|
|
(Integer/parseInt (get r "IOL # additional")))
|
|
:account-type (keyword "account-type"
|
|
(str/lower-case (get r "Account Type")))
|
|
:applicability (keyword "account-applicability"
|
|
(condp = (get r "Visiblity (Per-customer, Visible by default, hidden by default)")
|
|
"Visible by default"
|
|
"global"
|
|
|
|
"Hidden by default"
|
|
"optional"
|
|
|
|
"Per Customer"
|
|
"customized"))}))
|
|
(mapcat (fn ->tx [{:keys [old-account-id new-account-number name location also-merge account-type applicability]}]
|
|
(let [tx [(cond-> {:account/name name
|
|
:account/type account-type
|
|
:account/account-set "default"
|
|
:account/applicability applicability
|
|
:account/numeric-code new-account-number}
|
|
old-account-id (assoc :db/id old-account-id)
|
|
location (assoc :account/location location))]]
|
|
(if also-merge
|
|
(into tx
|
|
(also-merge-txes also-merge old-account-id))
|
|
tx)
|
|
))))
|
|
|
|
|
|
conj
|
|
[]
|
|
rows)]
|
|
(dc/transact conn {:tx-data txes})))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn find-bad-accounts []
|
|
(set (map second (dc/q {:query {:find ['(pull ?x [*]) '?z]
|
|
:in ['$]
|
|
:where ['[?e :account/numeric-code ?z]
|
|
'[(<= ?z 9999)]
|
|
'[?x ?a ?e]]}
|
|
:args [(dc/db conn)]}))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn delete-4-digit-accounts []
|
|
(dc/transact conn
|
|
{:tx-data
|
|
(transduce
|
|
(comp
|
|
(map first)
|
|
(map (fn [old-account-id]
|
|
[:db/retractEntity old-account-id])))
|
|
conj
|
|
[]
|
|
(dc/q {:query {:find ['?e]
|
|
:in ['$]
|
|
:where ['[?e :account/numeric-code ?z]
|
|
'[(<= ?z 9999)]]}
|
|
:args [(dc/db conn)]}))})
|
|
)
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn find-conflicting-accounts []
|
|
(filter
|
|
(fn [[_ v]]
|
|
(> (count v) 1))
|
|
(reduce
|
|
(fn [acc [e z]]
|
|
(update acc z conj e))
|
|
{}
|
|
(dc/q {:query {:find ['?e '?z]
|
|
:in ['$]
|
|
:where ['[?e :account/numeric-code ?z]]}
|
|
:args [(dc/db conn)]}))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn customize-accounts [customer filename]
|
|
(let [[_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv)
|
|
[client-id] (first (dc/q (-> {:query {:find ['?e]
|
|
:in ['$ '?z]
|
|
:where [['?e :client/code '?z]]}
|
|
:args [(dc/db conn) customer]})))
|
|
_ (println client-id)
|
|
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
|
|
{:account/applicability [:db/ident]}
|
|
:db/id])]
|
|
:in ['$]
|
|
:where ['[?e :account/name]]}
|
|
:args [(dc/db conn)]})))
|
|
|
|
existing-account-overrides (dc/q (-> {:query {:find ['?e]
|
|
:in ['$ '?client-id]
|
|
:where [['?e :account-client-override/client '?client-id]]}
|
|
:args [(dc/db conn) client-id]}))
|
|
|
|
|
|
|
|
_ (when-let [bad-rows (seq (->> rows
|
|
(group-by (fn [[_ account]]
|
|
account))
|
|
vals
|
|
(filter #(> (count %) 1))
|
|
(filter (fn [duplicates]
|
|
(apply not= (map rest duplicates))))
|
|
#_(map (fn [[[_ account]]]
|
|
account))
|
|
))]
|
|
(throw (Exception. (str "These accounts are duplicated:" (str bad-rows)))))
|
|
rows (vec (set (map rest rows)))
|
|
|
|
txes (transduce
|
|
(comp
|
|
(mapcat (fn parse-map [[account account-name override-name _ type]]
|
|
(let [code (some-> account
|
|
not-empty
|
|
Integer/parseInt)
|
|
existing (code->existing-account code)]
|
|
(cond (not code)
|
|
[]
|
|
|
|
(and existing (or (#{:account-applicability/optional :account-applicability/customized}
|
|
(:db/ident (:account/applicability existing)))
|
|
(and (not-empty override-name)
|
|
(not-empty account-name)
|
|
(not= override-name account-name)
|
|
)))
|
|
[{:db/id (:db/id existing)
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
|
|
(not existing)
|
|
[{:account/applicability :account-applicability/customized
|
|
:account/name account-name
|
|
:account/account-set "default"
|
|
:account/numeric-code code
|
|
:account/code (str code)
|
|
:account/type (if (str/blank? type)
|
|
:account-type/expense
|
|
(keyword "account-type" (str/lower-case type)))
|
|
:account/client-overrides [{:account-client-override/client client-id
|
|
:account-client-override/name (or (not-empty override-name)
|
|
(not-empty account-name))}]}]
|
|
:else
|
|
[])))))
|
|
|
|
conj
|
|
(mapv
|
|
(fn [[x]]
|
|
[:db/retractEntity x])
|
|
existing-account-overrides)
|
|
rows)]
|
|
|
|
txes
|
|
#_@(d/transact conn txes)))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn fix-transactions-without-locations [client-code location]
|
|
(->>
|
|
(dc/q {:query {:find ['(pull ?e [*])]
|
|
:in ['$ '?client-code]
|
|
:where ['[?e :transaction/accounts ?ta]
|
|
'[?e :transaction/matched-rule]
|
|
'[?e :transaction/approval-status :transaction-approval-status/approved]
|
|
'(not [?ta :transaction-account/location])
|
|
'[?e :transaction/client ?c]
|
|
'[?c :client/code ?client-code]
|
|
]}
|
|
:args [(dc/db conn) client-code]})
|
|
(mapcat
|
|
(fn [[{:transaction/keys [accounts]}]]
|
|
(mapv
|
|
(fn [a]
|
|
{:db/id (:db/id a)
|
|
:transaction-account/location location}
|
|
)
|
|
accounts)
|
|
)
|
|
)
|
|
vec))
|
|
|
|
;; TODO uncommenting the two below this makes lein build not work, probably related to the ledger
|
|
#_(defn patch-missing-ledger-entries []
|
|
@(d/transact
|
|
(d/connect uri)
|
|
(mapv
|
|
#(l/entity-change->ledger (d/db (d/connect uri)) [:transaction %])
|
|
(concat
|
|
(->>
|
|
(d/query {:query {:find ['?t ]
|
|
:in ['$]
|
|
:where ['[?t :transaction/date]
|
|
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
|
|
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
|
|
:args [(d/db (d/connect uri))]})
|
|
(map first))))))
|
|
|
|
|
|
|
|
#_(defn check-for-out-of-date-ledger [code]
|
|
[(d/query {:query {:find ['(count ?e)]
|
|
:in ['$ '?code]
|
|
:where ['[?e :transaction/accounts ?ta]
|
|
'[?e :transaction/matched-rule]
|
|
'[?e :transaction/approval-status :transaction-approval-status/approved]
|
|
'(not [?ta :transaction-account/location])
|
|
'[?e :transaction/client ?c]
|
|
'[?c :client/code ?code]
|
|
]}
|
|
:args [(d/db (d/connect uri)) code]})
|
|
|
|
(d/query {:query {:find ['?t ]
|
|
:in ['$]
|
|
:where ['[?t :transaction/date]
|
|
'[?t :transaction/client ?c]
|
|
'[?c :client/code ?code]
|
|
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
|
|
'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
|
|
:args [(d/db (d/connect uri))]})
|
|
|
|
(d/query {:query {:find ['?t ]
|
|
:in ['$]
|
|
:where ['[?t :transaction/date]
|
|
'[?t :transaction/client ?c]
|
|
'[?c :client/code ?code]
|
|
'(not [?t :transaction/approval-status :transaction-approval-status/excluded])
|
|
'[?t :transaction/vendor ?v]
|
|
'[?j :journal-entry/original-entity ?t]
|
|
'(not [?j :journal-entry/vendor ?v])
|
|
#_'(not-join [?t] [?e :journal-entry/original-entity ?t])]}
|
|
:args [(d/db (d/connect uri))]})
|
|
|
|
(d/query {:query {:find ['(count ?i) ]
|
|
:in ['$]
|
|
:where ['[?i :invoice/client ?c]
|
|
'(not [?i :invoice/status :invoice-status/voided])
|
|
'[?c :client/code ?code]
|
|
'(not-join [?i] [?e :journal-entry/original-entity ?i])]}
|
|
:args [(d/db (d/connect uri))]})])
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn go []
|
|
(require '[mount.core :as mount])
|
|
(require '[auto-ap.server])
|
|
(mount/start-without #'auto-ap.server/jetty))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn entity-history [i]
|
|
(vec (sort-by first (dc/q
|
|
{:query {:find ['?tx '?z '?v ]
|
|
:in ['?i '$]
|
|
:where ['[?i ?a ?v ?tx ?ad]
|
|
'[?a :db/ident ?z]
|
|
'[(= ?ad true)]]}
|
|
:args [i (dc/history (dc/db conn))]}))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn entity-history-with-revert [i]
|
|
(vec (sort-by first (dc/q
|
|
{:query {:find ['?tx '?z '?v '?ad ]
|
|
:in ['?i '$]
|
|
:where ['[?i ?a ?v ?tx ?ad]
|
|
'[?a :db/ident ?z]]}
|
|
:args [i (dc/history (dc/db conn))]}))))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn tx-detail [i]
|
|
(map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v)
|
|
(:data (first
|
|
(dc/tx-range conn
|
|
{:start i
|
|
:end (inc i)})))))
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn tx-range-detail [i]
|
|
(map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v)
|
|
|
|
(mapcat :data (dc/tx-range conn
|
|
{:start (- i 100)
|
|
:end (+ i 100)}))))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn start-db []
|
|
(mu/start-publisher! {:type :dev})
|
|
(mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})))
|
|
|
|
(defn start-search []
|
|
(mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer})))
|
|
|
|
(defn restart-db []
|
|
#_(require 'datomic.dev-local)
|
|
#_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"})
|
|
(mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))
|
|
(start-db))
|
|
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn spit-csv [columns data ]
|
|
(csv/write-csv *out*
|
|
(into [(map name columns)]
|
|
(for [r data]
|
|
((apply juxt columns) r )))))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn find-queries [words]
|
|
(let [obj (s3/list-objects-v2 :bucket-name (:data-bucket env)
|
|
:prefix (str "queries/"))
|
|
concurrent 30
|
|
output-chan (async/chan)]
|
|
(async/pipeline-blocking concurrent
|
|
output-chan
|
|
(comp
|
|
(map #(do
|
|
[(:key %)
|
|
(str (slurp (:object-content (s3/get-object
|
|
:bucket-name (:data-bucket env)
|
|
:key (:key %)))))]))
|
|
|
|
(filter #(->> words
|
|
(every? (fn [w] (str/includes? (second %) w)))))
|
|
(map first)
|
|
(map #(str/replace % #"queries/" ""))
|
|
)
|
|
(async/to-chan! (:object-summaries obj))
|
|
true
|
|
(fn [e]
|
|
(println "failed " e)))
|
|
(async/<!! (async/into [] output-chan))))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn upsert-invoice-amounts [tsv]
|
|
(let [data (with-open [reader (io/reader (char-array tsv))]
|
|
(doall (csv/read-csv reader :separator \tab)))
|
|
db (dc/db conn)
|
|
i->invoice-id (fn [i]
|
|
(try (Long/parseLong i)
|
|
(catch Exception e
|
|
(:db/id (dc/pull db '[:db/id]
|
|
[:invoice/original-id (Long/parseLong (first (str/split i #"-")))])))))
|
|
invoice-totals (->> data
|
|
(drop 1)
|
|
(group-by first)
|
|
(map (fn [[k values]]
|
|
[(i->invoice-id k)
|
|
(reduce + 0.0
|
|
(->> values
|
|
(map (fn [[_ _ _ _ amount]]
|
|
(- (Double/parseDouble amount))))))
|
|
]))
|
|
(into {}))]
|
|
(->>
|
|
(for [[i invoice-expense-account-id target-account target-date amount _ location] (drop 1 data)
|
|
:let [
|
|
invoice-id (i->invoice-id i)
|
|
|
|
invoice (dc/pull db '[FILL_IN] invoice-id)
|
|
current-total (:invoice/total invoice)
|
|
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
|
|
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
|
(:db/id (first (:invoice/expense-accounts invoice))))))
|
|
|
|
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
|
(:db/id (first (:invoice/expense-accounts invoice)))
|
|
(random-tempid))
|
|
invoice-expense-account (when-not new-account?
|
|
(dc/pull db '[FILL_IN]invoice-expense-account-id))
|
|
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
|
|
target-account-id (Long/parseLong (str/trim target-account))
|
|
|
|
target-date (clj-time.coerce/to-date (atime/parse target-date atime/normal-date))
|
|
current-date (:invoice/date invoice)
|
|
|
|
|
|
current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0)
|
|
target-expense-account-amount (- (Double/parseDouble amount))
|
|
|
|
|
|
current-expense-account-location (:invoice-expense-account/location invoice-expense-account)
|
|
target-expense-account-location location
|
|
|
|
|
|
[[_ _ invoice-payment]] (vec (dc/q
|
|
'[:find ?p ?a ?ip
|
|
:in $ ?i
|
|
:where [?ip :invoice-payment/invoice ?i]
|
|
[?ip :invoice-payment/amount ?a]
|
|
[?ip :invoice-payment/payment ?p]
|
|
]
|
|
db invoice-id))]
|
|
:when current-total]
|
|
|
|
[
|
|
(when (not (auto-ap.utils/dollars= current-total target-total))
|
|
{:db/id invoice-id
|
|
:invoice/total target-total})
|
|
|
|
(when new-account?
|
|
{:db/id invoice-id
|
|
:invoice/expense-accounts invoice-expense-account-id})
|
|
|
|
(when (and target-date (not= current-date target-date))
|
|
{:db/id invoice-id
|
|
:invoice/date target-date})
|
|
|
|
(when (and
|
|
(not (auto-ap.utils/dollars= current-total target-total))
|
|
invoice-payment)
|
|
[:db/retractEntity invoice-payment])
|
|
|
|
(when (or new-account?
|
|
(not (auto-ap.utils/dollars= current-expense-account-amount target-expense-account-amount)))
|
|
{:db/id invoice-expense-account-id
|
|
:invoice-expense-account/amount target-expense-account-amount})
|
|
|
|
(when (not= current-expense-account-location
|
|
target-expense-account-location)
|
|
{:db/id invoice-expense-account-id
|
|
:invoice-expense-account/location target-expense-account-location})
|
|
|
|
(when (not= current-account-id target-account-id )
|
|
{:db/id invoice-expense-account-id
|
|
:invoice-expense-account/account target-account-id})])
|
|
(mapcat identity)
|
|
(filter identity)
|
|
vec)))
|
|
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn get-schema [prefix]
|
|
(->> (dc/q '[:find ?i
|
|
:in $ ?p
|
|
:where [_ :db/ident ?i]
|
|
[(namespace ?i) ?p]] (dc/db auto-ap.datomic/conn) prefix)
|
|
(mapcat identity)
|
|
vec))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn get-idents []
|
|
(->> (dc/q '[:find ?i
|
|
:in $
|
|
:where [_ :db/ident ?i]]
|
|
(dc/db conn) )
|
|
(mapcat identity)
|
|
(map str)
|
|
(sort)
|
|
vec))
|
|
|
|
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
|
(defn init-repl []
|
|
(set! nrepl.middleware.print/*print-fn* clojure.pprint/pprint))
|
|
|