(ns user (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [uri]] [auto-ap.ledger :as l :refer [transact-with-ledger]] [auto-ap.server] [auto-ap.square.core :as square] [auto-ap.time :as atime] [auto-ap.utils :refer [by]] [clj-time.coerce :as c] [clj-time.core :as t] [clj-time.periodic :as per] [clojure.core.async :as async] [clojure.data.csv :as csv] [clojure.java.io :as io] [clojure.pprint] [clojure.string :as str] [config.core :refer [env]] [datomic.api :as d] [mount.core :as mount] [nrepl.middleware.print] [unilog.context :as lc]) (:import (org.apache.commons.io.input BOMInputStream))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn mark-until-date [client end] (let [conn (d/connect uri)] (doseq [p (->> (d/query {:query {:find '[?e] :in '[$ ?client ?end ] :where [ '[?e :invoice/client ?c] '[?c :client/code ?client] '[?e :invoice/date ?d ] '[(<= ?d ?end) ]]} :args [(d/db conn) client (c/to-date end)]}) (mapv first) (mapv (fn [i] {:db/id i :invoice/exclude-from-ledger true})) (partition-all 100))] (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) (println "process 100")) (doseq [p (->> (d/query {:query {:find '[?e] :in '[$ ?client ?end ] :where [ '[?e :transaction/client ?c] '[?c :client/code ?client] '[?e :transaction/date ?d ] '[(<= ?d ?end) ]]} :args [(d/db conn) client (c/to-date end)]}) (mapv first) (mapv (fn [i] {:db/id i :transaction/approval-status :transaction-approval-status/excluded})) (partition-all 100))] (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) (println "process 100")))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-accounts [conn] (let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code :db/id])] :in ['$] :where ['[?e :account/name]]} :args [(d/db conn)]}))) also-merge-txes (fn [also-merge old-account-id] (if old-account-id (let [[sunset-account] (first (d/query {:query {:find ['?a ] :in ['$ '?ac ] :where ['[?a :account/numeric-code ?ac]]} :args [(d/db conn) also-merge ]}))] (into (mapv (fn [[entity id _]] [:db/add entity id old-account-id]) (d/query {:query {:find ['?e '?id '?a ] :in ['$ '?ac ] :where ['[?a :account/numeric-code ?ac] '[?e ?at ?a] '[?at :db/ident ?id]]} :args [(d/db conn) also-merge ]})) [[:db/retractEntity sunset-account]])) [])) txes (transduce (comp (map (fn ->map [r] (into {} (map vector header r)))) (map (fn parse-map [r] {:old-account-id (:db/id (code->existing-account (or (if (= (get r "IOL Account #") "NEW") nil (Integer/parseInt (get r "IOL Account #"))) (Integer/parseInt (get r "Account #"))))) :new-account-number (Integer/parseInt (get r "Account #")) :name (get r "Default Name") :location (when-not (str/blank? (get r "Forced Location")) (get r "Forced Location")) :also-merge (when-not (str/blank? (get r "IOL # additional")) (Integer/parseInt (get r "IOL # additional"))) :account-type (keyword "account-type" (str/lower-case (get r "Account Type"))) :applicability (keyword "account-applicability" (condp = (get r "Visiblity (Per-customer, Visible by default, hidden by default)") "Visible by default" "global" "Hidden by default" "optional" "Per Customer" "customized"))})) (mapcat (fn ->tx [{:keys [old-account-id new-account-number name location also-merge account-type applicability]}] (let [tx [(cond-> {:account/name name :account/type account-type :account/account-set "default" :account/applicability applicability :account/numeric-code new-account-number} old-account-id (assoc :db/id old-account-id) location (assoc :account/location location))]] (if also-merge (into tx (also-merge-txes also-merge old-account-id)) tx) )))) conj [] rows)] @(d/transact conn txes))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-bad-accounts [] (set (map second (d/query {:query {:find ['(pull ?x [*]) '?z] :in ['$] :where ['[?e :account/numeric-code ?z] '[(<= ?z 9999)] '[?x ?a ?e]]} :args [(d/db (d/connect uri))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn delete-4-digit-accounts [] @(d/transact (d/connect uri) (transduce (comp (map first) (map (fn [old-account-id] [:db/retractEntity old-account-id]))) conj [] (d/query {:query {:find ['?e] :in ['$] :where ['[?e :account/numeric-code ?z] '[(<= ?z 9999)]]} :args [(d/db (d/connect uri))]}))) ) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-conflicting-accounts [] (filter (fn [[_ v]] (> (count v) 1)) (reduce (fn [acc [e z]] (update acc z conj e)) {} (d/query {:query {:find ['?e '?z] :in ['$] :where ['[?e :account/numeric-code ?z]]} :args [(d/db (d/connect uri))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn customize-accounts [customer filename] (let [conn (d/connect uri) [_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) [client-id] (first (d/query (-> {:query {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} :args [(d/db (d/connect uri)) customer]}))) _ (println client-id) code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} :args [(d/db conn)]}))) existing-account-overrides (d/query (-> {:query {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} :args [(d/db (d/connect uri)) client-id]})) _ (when-let [bad-rows (seq (->> rows (group-by (fn [[_ account]] account)) vals (filter #(> (count %) 1)) (filter (fn [duplicates] (apply not= (map rest duplicates)))) #_(map (fn [[[_ account]]] account)) ))] (throw (Exception. (str "These accounts are duplicated:" (str bad-rows))))) rows (vec (set (map rest rows))) txes (transduce (comp (mapcat (fn parse-map [[account account-name override-name _ type]] (let [code (some-> account not-empty Integer/parseInt) existing (code->existing-account code)] (cond (not code) [] (and existing (or (#{:account-applicability/optional :account-applicability/customized} (:db/ident (:account/applicability existing))) (and (not-empty override-name) (not-empty account-name) (not= override-name account-name) ))) [{:db/id (:db/id existing) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] (not existing) [{:account/applicability :account-applicability/customized :account/name account-name :account/account-set "default" :account/numeric-code code :account/code (str code) :account/type (if (str/blank? type) :account-type/expense (keyword "account-type" (str/lower-case type))) :account/client-overrides [{:account-client-override/client client-id :account-client-override/name (or (not-empty override-name) (not-empty account-name))}]}] :else []))))) conj (mapv (fn [[x]] [:db/retractEntity x]) existing-account-overrides) rows)] txes #_@(d/transact conn txes))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn fix-transactions-without-locations [client-code location] (->> (d/query {:query {:find ['(pull ?e [*])] :in ['$ '?client-code] :where ['[?e :transaction/accounts ?ta] '[?e :transaction/matched-rule] '[?e :transaction/approval-status :transaction-approval-status/approved] '(not [?ta :transaction-account/location]) '[?e :transaction/client ?c] '[?c :client/code ?client-code] ]} :args [(d/db (d/connect uri)) client-code]}) (mapcat (fn [[{:transaction/keys [accounts]}]] (mapv (fn [a] {:db/id (:db/id a) :transaction-account/location location} ) accounts) ) ) vec)) ;; TODO uncommenting the two below this makes lein build not work, probably related to the ledger #_(defn patch-missing-ledger-entries [] @(d/transact (d/connect uri) (mapv #(l/entity-change->ledger (d/db (d/connect uri)) [:transaction %]) (concat (->> (d/query {:query {:find ['?t ] :in ['$] :where ['[?t :transaction/date] '(not [?t :transaction/approval-status :transaction-approval-status/excluded]) '(not-join [?t] [?e :journal-entry/original-entity ?t])]} :args [(d/db (d/connect uri))]}) (map first)))))) #_(defn check-for-out-of-date-ledger [code] [(d/query {:query {:find ['(count ?e)] :in ['$ '?code] :where ['[?e :transaction/accounts ?ta] '[?e :transaction/matched-rule] '[?e :transaction/approval-status :transaction-approval-status/approved] '(not [?ta :transaction-account/location]) '[?e :transaction/client ?c] '[?c :client/code ?code] ]} :args [(d/db (d/connect uri)) code]}) (d/query {:query {:find ['?t ] :in ['$] :where ['[?t :transaction/date] '[?t :transaction/client ?c] '[?c :client/code ?code] '(not [?t :transaction/approval-status :transaction-approval-status/excluded]) '(not-join [?t] [?e :journal-entry/original-entity ?t])]} :args [(d/db (d/connect uri))]}) (d/query {:query {:find ['?t ] :in ['$] :where ['[?t :transaction/date] '[?t :transaction/client ?c] '[?c :client/code ?code] '(not [?t :transaction/approval-status :transaction-approval-status/excluded]) '[?t :transaction/vendor ?v] '[?j :journal-entry/original-entity ?t] '(not [?j :journal-entry/vendor ?v]) #_'(not-join [?t] [?e :journal-entry/original-entity ?t])]} :args [(d/db (d/connect uri))]}) (d/query {:query {:find ['(count ?i) ] :in ['$] :where ['[?i :invoice/client ?c] '(not [?i :invoice/status :invoice-status/voided]) '[?c :client/code ?code] '(not-join [?i] [?e :journal-entry/original-entity ?i])]} :args [(d/db (d/connect uri))]})]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn go [] (require '[mount.core :as mount]) (require '[auto-ap.server]) (mount/start-without #'auto-ap.server/jetty)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history [i] (vec (sort-by first (d/query {:query {:find ['?tx '?z '?v ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z] '[(= ?ad true)]]} :args [i (d/history (d/db (d/connect uri)))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history-with-revert [i] (vec (sort-by first (d/query {:query {:find ['?tx '?z '?v '?ad ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z]]} :args [i (d/history (d/db (d/connect uri)))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-detail [i] (map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v) (:data (first (d/tx-range (d/log (d/connect uri)) i (inc i)))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-range-detail [i] (map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v) (mapcat :data (d/tx-range (d/log (d/connect uri)) (- i 100) (+ i 100))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn start-db [] (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn}))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn spit-csv [columns data ] (csv/write-csv *out* (into [(map name columns)] (for [r data] ((apply juxt columns) r ))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-queries [words] (let [obj (s3/list-objects-v2 :bucket-name (:data-bucket env) :prefix (str "queries/")) concurrent 30 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan (comp (map #(do [(:key %) (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env) :key (:key %)))))])) (filter #(->> words (every? (fn [w] (str/includes? (second %) w))))) (map first) (map #(str/replace % #"queries/" "")) ) (async/to-chan! (:object-summaries obj)) true (fn [e] (println "failed " e))) (async/> (per/periodic-seq (t/plus (t/today) (t/days (- days))) (t/today) (t/days 2)) (map (fn [d] [(atime/unparse (t/plus d (t/days 1)) atime/iso-date) (atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))] (square/upsert-settlements client square-location))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-sales-for-day [date] (doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...] :where ['?e :client/square-locations ]] (d/db auto-ap.datomic/conn)) square-location (:client/square-locations client) :when (:square-location/client-location square-location)] (println client) (println "orders") (lc/with-context {:source "Historical loading data"} (square/upsert client square-location (c/to-date-time date) (t/plus (c/to-date-time date) (t/days 1)))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn upsert-invoice-amounts [tsv] (let [data (with-open [reader (io/reader (char-array tsv))] (doall (csv/read-csv reader :separator \tab))) db (d/db auto-ap.datomic/conn) i->invoice-id (fn [i] (try (Long/parseLong i) (catch Exception e (:db/id (d/pull db '[:db/id] [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) invoice-totals (->> data (drop 1) (group-by first) (map (fn [[k values]] [(i->invoice-id k) (reduce + 0.0 (->> values (map (fn [[_ _ _ _ amount]] (- (Double/parseDouble amount)))))) ])) (into {}))] (->> (for [[i invoice-expense-account-id target-account target-date amount _ location] (drop 1 data) :let [ invoice-id (i->invoice-id i) invoice (d/entity db invoice-id) current-total (:invoice/total invoice) target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice)))))) invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice))) (d/tempid :db.part/user)) invoice-expense-account (when-not new-account? (or (d/entity db invoice-expense-account-id) (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) target-account-id (Long/parseLong (str/trim target-account)) target-date (clj-time.coerce/to-date (atime/parse target-date atime/normal-date)) current-date (:invoice/date invoice) current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0) target-expense-account-amount (- (Double/parseDouble amount)) current-expense-account-location (:invoice-expense-account/location invoice-expense-account) target-expense-account-location location [[_ _ invoice-payment]] (vec (d/q '[:find ?p ?a ?ip :in $ ?i :where [?ip :invoice-payment/invoice ?i] [?ip :invoice-payment/amount ?a] [?ip :invoice-payment/payment ?p] ] db invoice-id))] :when current-total] [ (when (not (auto-ap.utils/dollars= current-total target-total)) {:db/id invoice-id :invoice/total target-total}) (when new-account? {:db/id invoice-id :invoice/expense-accounts invoice-expense-account-id}) (when (and target-date (not= current-date target-date)) {:db/id invoice-id :invoice/date target-date}) (when (and (not (auto-ap.utils/dollars= current-total target-total)) invoice-payment) [:db/retractEntity invoice-payment]) (when (or new-account? (not (auto-ap.utils/dollars= current-expense-account-amount target-expense-account-amount))) {:db/id invoice-expense-account-id :invoice-expense-account/amount target-expense-account-amount}) (when (not= current-expense-account-location target-expense-account-location) {:db/id invoice-expense-account-id :invoice-expense-account/location target-expense-account-location}) (when (not= current-account-id target-account-id ) {:db/id invoice-expense-account-id :invoice-expense-account/account target-account-id})]) (mapcat identity) (filter identity) vec))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-schema [prefix] (->> (d/q '[:find ?i :in $ ?p :where [_ :db/ident ?i] [(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix) (mapcat identity) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-idents [] (->> (d/q '[:find ?i :in $ :where [_ :db/ident ?i]] (d/db auto-ap.datomic/conn) ) (mapcat identity) (map str) (sort) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn init-repl [] (set! nrepl.middleware.print/*print-fn* clojure.pprint/pprint))