(ns user (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [conn pull-attr random-tempid]] [auto-ap.ledger :as l ] [clj-http.core :as http] [clj-http.client :as client] [auto-ap.server] [auto-ap.square.core :as square] [auto-ap.square.core2 :as square2] [auto-ap.time :as atime] [auto-ap.utils :refer [by]] [clj-time.coerce :as c] [clj-time.core :as t] [clj-time.periodic :as per] [clojure.core.async :as async] [clojure.data.csv :as csv] [clojure.java.io :as io] [clojure.pprint] [clojure.string :as str] [config.core :refer [env]] [datomic.api :as dc] [mount.core :as mount] [nrepl.middleware.print] [unilog.context :as lc] [com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog.buffer :as rb] [datomic.api :as d] [clojure.data.json :as json] [auto-ap.solr :as solr]) (:import (org.apache.commons.io.input BOMInputStream))) (defn println-event [item] (printf "%s - %s:%s %s\n" (:mulog/namespace item) (:mulog/event-name item) (if (:mulog/duration item) (str " " (int (/ (:mulog/duration item) 1000000)) "ms") "") (pr-str (reduce (fn [acc [k v]] (assoc acc k v)) {} item)))) (deftype DevPublisher [config buffer transform] com.brunobonacci.mulog.publisher.PPublisher (agent-buffer [_] buffer) (publish-delay [_] 200) (publish [_ buffer] ;; items are pairs [offset ] (doseq [item (transform (map second (rb/items buffer)))] (println-event item) ) (flush) (rb/clear buffer))) (defn dev-publisher [{:keys [transform pretty?] :as config}] (DevPublisher. config (rb/agent-buffer 10000) (or transform identity))) (defmethod com.brunobonacci.mulog.publisher/publisher-factory :dev [config] (dev-publisher config)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-accounts [conn] (let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) code->existing-account (by :account/numeric-code (map first (dc/q {:find ['(pull ?e [:account/numeric-code :db/id])] :in ['$] :where ['[?e :account/name]]} (dc/db conn)))) also-merge-txes (fn [also-merge old-account-id] (if old-account-id (let [[sunset-account] (first (dc/q {:find ['?a ] :in ['$ '?ac ] :where ['[?a :account/numeric-code ?ac]]} (dc/db conn) also-merge))] (into (mapv (fn [[entity id _]] [:db/add entity id old-account-id]) (dc/q {:find ['?e '?id '?a ] :in ['$ '?ac ] :where ['[?a :account/numeric-code ?ac] '[?e ?at ?a] '[?at :db/ident ?id]]} (dc/db conn) also-merge)) [[:db/retractEntity sunset-account]])) [])) txes (transduce (comp (map (fn ->map [r] (into {} (map vector header r)))) (map (fn parse-map [r] {:old-account-id (:db/id (code->existing-account (or (if (= (get r "IOL Account #") "NEW") nil (Integer/parseInt (get r "IOL Account #"))) (Integer/parseInt (get r "Account #"))))) :new-account-number (Integer/parseInt (get r "Account #")) :name (get r "Default Name") :location (when-not (str/blank? (get r "Forced Location")) (get r "Forced Location")) :also-merge (when-not (str/blank? (get r "IOL # additional")) (Integer/parseInt (get r "IOL # additional"))) :account-type (keyword "account-type" (str/lower-case (get r "Account Type"))) :applicability (keyword "account-applicability" (condp = (get r "Visiblity (Per-customer, Visible by default, hidden by default)") "Visible by default" "global" "Hidden by default" "optional" "Per Customer" "customized"))})) (mapcat (fn ->tx [{:keys [old-account-id new-account-number name location also-merge account-type applicability]}] (let [tx [(cond-> {:account/name name :account/type account-type :account/account-set "default" :account/applicability applicability :account/numeric-code new-account-number} old-account-id (assoc :db/id old-account-id) location (assoc :account/location location))]] (if also-merge (into tx (also-merge-txes also-merge old-account-id)) tx) )))) conj [] rows)] @(dc/transact conn txes))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-bad-accounts [] (set (map second (dc/q {:find ['(pull ?x [*]) '?z] :in ['$] :where ['[?e :account/numeric-code ?z] '[(<= ?z 9999)] '[?x ?a ?e]]} (dc/db conn))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn delete-4-digit-accounts [] @(dc/transact conn (transduce (comp (map first) (map (fn [old-account-id] [:db/retractEntity old-account-id]))) conj [] (dc/q {:find ['?e] :in ['$] :where ['[?e :account/numeric-code ?z] '[(<= ?z 9999)]]} (dc/db conn)))) ) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-conflicting-accounts [] (filter (fn [[_ v]] (> (count v) 1)) (reduce (fn [acc [e z]] (update acc z conj e)) {} (dc/q {:find ['?e '?z] :in ['$] :where ['[?e :account/numeric-code ?z]]} (dc/db conn))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn customize-accounts [customer filename] (let [[_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) [client-id] (first (dc/q (-> {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} (dc/db conn) customer))) _ (println client-id) code->existing-account (by :account/numeric-code (map first (dc/q {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} (dc/db conn)))) existing-account-overrides (dc/q {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} (dc/db conn) client-id) _ (when-let [bad-rows (seq (->> rows (group-by (fn [[_ account]] account)) vals (filter #(> (count %) 1)) (filter (fn [duplicates] (apply not= (map rest duplicates)))) #_(map (fn [[[_ account]]] account)) ))] (throw (Exception. (str "These accounts are duplicated:" (str bad-rows))))) rows (vec (set (map rest rows))) txes (transduce (comp (mapcat (fn parse-map [[account account-name override-name _ type]] (let [code (some-> account not-empty Integer/parseInt) existing (code->existing-account code)] (cond (not code) [] (and existing (or (#{:account-applicability/optional :account-applicability/customized} (:db/ident (:account/applicability existing))) (and (not-empty override-name) (not-empty account-name) (not= override-name account-name) ))) [{:db/id (:db/id existing) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] (not existing) [{:account/applicability :account-applicability/customized :account/name account-name :account/account-set "default" :account/numeric-code code :account/code (str code) :account/type (if (str/blank? type) :account-type/expense (keyword "account-type" (str/lower-case type))) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] :else []))))) conj (mapv (fn [[x]] [:db/retractEntity x]) existing-account-overrides) rows)] txes #_@(d/transact conn txes))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn fix-transactions-without-locations [client-code location] (->> (dc/q {:find ['(pull ?e [*])] :in ['$ '?client-code] :where ['[?e :transaction/accounts ?ta] '[?e :transaction/matched-rule] '[?e :transaction/approval-status :transaction-approval-status/approved] '(not [?ta :transaction-account/location]) '[?e :transaction/client ?c] '[?c :client/code ?client-code] ]} (dc/db conn) client-code) (mapcat (fn [[{:transaction/keys [accounts]}]] (mapv (fn [a] {:db/id (:db/id a) :transaction-account/location location} ) accounts) ) ) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn go [] (require '[mount.core :as mount]) (require '[auto-ap.server]) (mount/start-without #'auto-ap.server/jetty)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history [i] (vec (sort-by first (dc/q {:find ['?tx '?z '?v ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z] '[(= ?ad true)]]} i (dc/history (dc/db conn)))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history-with-revert [i] (vec (sort-by first (dc/q {:find ['?tx '?z '?v '?ad ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z]]} i (dc/history (dc/db conn)))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-detail [i] (map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v) (:data (first (dc/tx-range conn {:start i :end (inc i)}))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-range-detail [i] (map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v) (mapcat :data (dc/tx-range conn {:start (- i 100) :end (+ i 100)})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn start-db [] (mu/start-publisher! {:type :dev}) (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn }))) (defn start-search [] (mount.core/start (mount.core/only #{#'auto-ap.graphql.vendors/indexer #'auto-ap.graphql.accounts/indexer}))) (defn restart-db [] #_(require 'datomic.dev-local) #_(datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) (mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn })) (start-db)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn spit-csv [columns data ] (csv/write-csv *out* (into [(map name columns)] (for [r data] ((apply juxt columns) r ))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-queries [words] (let [obj (s3/list-objects-v2 :bucket-name (:data-bucket env) :prefix (str "queries/")) concurrent 30 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan (comp (map #(do [(:key %) (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env) :key (:key %)))))])) (filter #(->> words (every? (fn [w] (str/includes? (second %) w))))) (map first) (map #(str/replace % #"queries/" "")) ) (async/to-chan! (:object-summaries obj)) true (fn [e] (println "failed " e))) (async/invoice-id (fn [i] (try (Long/parseLong i) (catch Exception e (:db/id (dc/pull db '[:db/id] [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) invoice-totals (->> data (drop 1) (group-by first) (map (fn [[k values]] [(i->invoice-id k) (reduce + 0.0 (->> values (map (fn [[_ _ _ _ amount]] (- (Double/parseDouble amount)))))) ])) (into {}))] (->> (for [[i invoice-expense-account-id target-account target-date amount _ location] (drop 1 data) :let [ invoice-id (i->invoice-id i) invoice (dc/pull db '[FILL_IN] invoice-id) current-total (:invoice/total invoice) target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice)))))) invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice))) (random-tempid)) invoice-expense-account (when-not new-account? (dc/pull db '[FILL_IN]invoice-expense-account-id)) current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) target-account-id (Long/parseLong (str/trim target-account)) target-date (clj-time.coerce/to-date (atime/parse target-date atime/normal-date)) current-date (:invoice/date invoice) current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0) target-expense-account-amount (- (Double/parseDouble amount)) current-expense-account-location (:invoice-expense-account/location invoice-expense-account) target-expense-account-location location [[_ _ invoice-payment]] (vec (dc/q '[:find ?p ?a ?ip :in $ ?i :where [?ip :invoice-payment/invoice ?i] [?ip :invoice-payment/amount ?a] [?ip :invoice-payment/payment ?p] ] db invoice-id))] :when current-total] [ (when (not (auto-ap.utils/dollars= current-total target-total)) {:db/id invoice-id :invoice/total target-total}) (when new-account? {:db/id invoice-id :invoice/expense-accounts invoice-expense-account-id}) (when (and target-date (not= current-date target-date)) {:db/id invoice-id :invoice/date target-date}) (when (and (not (auto-ap.utils/dollars= current-total target-total)) invoice-payment) [:db/retractEntity invoice-payment]) (when (or new-account? (not (auto-ap.utils/dollars= current-expense-account-amount target-expense-account-amount))) {:db/id invoice-expense-account-id :invoice-expense-account/amount target-expense-account-amount}) (when (not= current-expense-account-location target-expense-account-location) {:db/id invoice-expense-account-id :invoice-expense-account/location target-expense-account-location}) (when (not= current-account-id target-account-id ) {:db/id invoice-expense-account-id :invoice-expense-account/account target-account-id})]) (mapcat identity) (filter identity) vec))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-schema [prefix] (->> (dc/q '[:find ?i :in $ ?p :where [_ :db/ident ?i] [(namespace ?i) ?p]] (dc/db auto-ap.datomic/conn) prefix) (mapcat identity) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-idents [] (->> (dc/q '[:find ?i :in $ :where [_ :db/ident ?i]] (dc/db conn) ) (mapcat identity) (map str) (sort) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn init-repl [] (set! nrepl.middleware.print/*print-fn* clojure.pprint/pprint)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn sample-ledger-import ([client-code] (sample-ledger-import client-code 10)) ([client-code n] (let [client-location (ffirst (d/q '[:find ?l :in $ ?c :where [?c :client/locations ?l]] (dc/db conn) [:client/code client-code]))] (clojure.data.csv/write-csv *out* (for [n (range n) :let [v (rand-nth (map first (d/q '[:find ?vn :where [_ :vendor/name ?vn]] (dc/db conn)))) [{a-1 :account/numeric-code a-1-location :account/location} {a-2 :account/numeric-code a-2-location :account/location}] (->> (d/q '[:find (pull ?a [:account/numeric-code :account/location]) :where [?a :account/numeric-code]] (dc/db conn)) (map first) (shuffle) (take 2)) amount (rand-int 2000) d (-> (t/now) (t/minus (t/days (rand-int 60))) (atime/unparse atime/normal-date)) id (rand-int 100000)] a [[(str id) client-code "synthetic" v d a-1 (or a-1-location client-location) 0 amount ] [(str id) client-code "synthetic" v d a-2 (or a-2-location client-location) amount 0]]] a) :separator \tab)))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn sample-manual-yodlee ([client-code] (sample-ledger-import client-code 10)) ([client-code n] (let [bank-accounts (map first (d/q '[:find ?bac :in $ ?c :where [?c :client/bank-accounts ?b] [?b :bank-account/code ?bac]] (dc/db conn) [:client/code client-code]))] (clojure.data.csv/write-csv *out* (for [n (range n) :let [amount (rand-int 2000) d (-> (t/now) (t/minus (t/days (rand-int 60))) (atime/unparse atime/normal-date)) id (rand-int 100000)]] ["posted" d (str "Random Description - " id) "Travel" nil nil (- amount) nil nil nil nil nil (rand-nth bank-accounts) client-code]) :separator \tab)))) (defn index-solr [] (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ :where [?i :invoice/invoice-number] (not [?i :invoice/status :invoice-status/voided])] :args [ (dc/db conn)]}) (map first) (partition-all 1000))] (print ".") (flush) (solr/index-documents solr/impl batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ :where [?i :payment/date] (not [?i :payment/status :payment-status/voided])] :args [(dc/db conn)]}) (map first) (partition-all 1000))] (print ".") (flush) (solr/index-documents solr/impl batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ :where [?i :transaction/description-original] (not [?i :transaction/approval-status :transaction-approval-status/suppressed])] :args [(dc/db conn)]}) (map first) (partition-all 1000))] (print ".") (flush) (solr/index-documents solr/impl batch)) (doseq [batch (->> (dc/qseq {:query '[:find ?i :in $ :where [?i :journal-entry/date]] :args [(dc/db conn)]}) (map first) (partition-all 1000))] (print ".") (flush) (solr/index-documents solr/impl batch))) (defn setup-sales-orders [] (doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]}) (map (fn [[s c]] {:db/id s :sales-order/client c})) (partition-all 1000))] (print ".") @(dc/transact auto-ap.datomic/conn n)))