(ns auto-ap.datomic (:require [auto-ap.utils :refer [default-pagination-size]] [clojure.tools.logging :as log] [config.core :refer [env]] [datomic.api :as d] [mount.core :as mount])) (def uri (:datomic-url env)) (mount/defstate conn :start (d/connect uri) :stop (d/release conn)) #_(def uri "datomic:mem://datomic-transactor:4334/invoice") #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn create-database [] (d/create-database uri)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn drop-database [] (d/delete-database uri)) (defn remove-nils [m] (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m )) {} m)] (if (seq result) result nil))) (defn replace-nils-with-retract [updated original] (let [result (reduce-kv (fn [[m & retractions] k v] (cond (and (nil? v) (not (nil? (get original k)))) (into [m] (conj retractions [:db/retract (:db/id original) k (or (:db/id (get original k)) (get original k))])) (nil? v) (into [m] retractions) :else (into [(assoc m k v)] retractions))) [{}] updated)] (if (seq result) result nil))) (def vendor-schema [{:db/ident :vendor/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id from the old system"} {:db/ident :vendor/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's human-friendly name"} {:db/ident :vendor/code :db/valueType :db.type/string :db/unique :db.unique/identity :db/cardinality :db.cardinality/one :db/doc "A vendor's computer-friendly name"} {:db/ident :vendor/print-as :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Set if you want to override how this vendor's name is printed on checks"} {:db/ident :vendor/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's email address"} {:db/ident :vendor/phone :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's phone number"} {:db/ident :vendor/invoice-reminder-schedule :db/valueType :db.type/keyword :db/cardinality :db.cardinality/one :db/doc "How often to email this vendor about invoices"} {:db/ident :vendor/default-expense-account :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The vendor's default expense account"} {:db/ident :vendor/primary-contact :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "The vendor's primary contact"} {:db/ident :vendor/secondary-contact :db/valueType :db.type/ref :db/isComponent true :db/cardinality :db.cardinality/one :db/doc "The vendor's secondary contact"} {:db/id #db/id[:db.part/db] :db/ident :vendor/address :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db.install/_attribute :db.part/db :db/doc "The vendor's address"} ]) (def client-schema [{:db/ident :client/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id from the old system"} {:db/ident :client/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A client's human-friendly name"} {:db/ident :client/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "hello@example.com"} {:db/ident :client/code :db/valueType :db.type/string :db/unique :db.unique/identity :db/cardinality :db.cardinality/one :db/doc "A client's computer-friendly name"} {:db/ident :client/signature-file :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A check signature image"} {:db/ident :client/locations :db/valueType :db.type/string :db/cardinality :db.cardinality/many :db/doc "A client's locations"} {:db/ident :client/address :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "Address of the client"} {:db/ident :client/bank-accounts :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true :db/doc "Bank accounts for the client"}]) (def address-schema [ {:db/ident :address/street1 :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "123 main st"} {:db/ident :address/street2 :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Apt A"} {:db/ident :address/city :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Campbell"} {:db/ident :address/state :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "CA"} {:db/ident :address/zip :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "95014"}]) (def contact-schema [ {:db/ident :contact/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "John Smith"} {:db/ident :contact/phone :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "4255773578"} {:db/ident :contact/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "hello@example.com"}]) (def bank-account-schema [{:db/ident :bank-account/external-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Identifier for bank account"} {:db/ident :bank-account/original-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "matching the orignal tuple of [company,bank]"} {:db/ident :bank-account/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A1127 Chase Card"} {:db/ident :bank-account/bank-name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "e.g. Bank of America"} {:db/ident :bank-account/bank-code :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The code to list under the bank's name"} {:db/ident :bank-account/routing :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The bank's routing number"} {:db/ident :bank-account/number :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The account number"} {:db/ident :bank-account/type :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "The type of account number, either :bank-account-type/check or :bank-account-type/cash"} {:db/ident :bank-account/yodlee-account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "yodlee's account identifier"} {:db/ident :bank-account/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Current check number"} {:db/ident :bank-account-type/check} {:db/ident :bank-account-type/cash}]) (def invoice-schema [{:db/ident :invoice/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id in old system"} {:db/ident :invoice/invoice-number :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor-specified number for the invoice"} {:db/ident :invoice/customer-identifier :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "An identifier found to suggest the customer"} {:db/ident :invoice/status :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Status of payment/import of the invoice [:paid, :unpaid]"} {:db/ident :invoice/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Which client this invoice is for"} {:db/ident :invoice/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Which vendor this invoice is for"} {:db/ident :invoice/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date for this invoice"} {:db/ident :invoice/total :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "Total $ for this invoice"} {:db/ident :invoice/outstanding-balance :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The unpaid balance of this invoice"} {:db/ident :invoice/default-location :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The default location that expense-accounts will be created with for this invoice"} {:db/ident :invoice/default-expense-account :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The default expense account for this invoice"} {:db/ident :invoice/expense-accounts :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true :db/doc "The expense account categories for this invoice"} {:db/ident :invoice-status/paid} {:db/ident :invoice-status/unpaid} {:db/ident :invoice-status/voided}]) (def invoice-expense-account-schema [{:db/ident :invoice-expense-account/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id in old system"} {:db/ident :invoice-expense-account/expense-account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The code for the expense account"} {:db/ident :invoice-expense-account/location :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Location for this expense account"} {:db/ident :invoice-expense-account/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that this contributes to"}]) (def payment-schema [{:db/ident :payment/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :payment/s3-uuid :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The uuid that matches the key for this check"} {:db/ident :payment/s3-key :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The s3 key with pdf of this check"} {:db/ident :payment/s3-url :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The s3 url with pdf of this check"} {:db/ident :payment/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The check number"} {:db/ident :payment/memo :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The check's memo line"} {:db/ident :payment/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "The date the payment was made"} {:db/ident :payment/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that was paid to the vendor"} {:db/ident :payment/paid-to :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Who the paid was made out to"} {:db/ident :payment/status :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The status of the payment [:pending :cleared :voided]"} {:db/ident :payment/type :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The type of the payment [:cash :check :debit]"} {:db/ident :payment/pdf-data :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "raw data used to generate check pdf"} ;; relations {:db/ident :payment/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The vendor for which this payment was for"} {:db/ident :payment/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The client for which this payment"} {:db/ident :payment/bank-account :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The bank account that was used to pay"} {:db/ident :payment/invoices :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/doc "Any invoices this payment is related to"} ;; enums {:db/ident :payment-status/pending} {:db/ident :payment-status/voided} {:db/ident :payment-status/cleared} {:db/ident :payment-type/cash} {:db/ident :payment-type/check} {:db/ident :payment-type/debit} ]) (def invoice-payment-schema [{:db/ident :invoice-payment/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :invoice-payment/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that was paid to this invoice"} ;; relations {:db/ident :invoice-payment/invoice :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The invoice for this payment"} {:db/ident :invoice-payment/payment :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The total payment for this payment"}]) (def transaction-schema [{:db/ident :transaction/original-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :transaction/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount of the transaction"} {:db/ident :transaction/description-original :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "full description of the transaction"} {:db/ident :transaction/description-simple :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "short description of the transaction"} {:db/ident :transaction/merchant-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "a yodlee id for the merchant"} {:db/ident :transaction/merchant-name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "a name for the merchant"} {:db/ident :transaction/id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "A key to match against"} {:db/ident :transaction/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date that the transaction showed up"} {:db/ident :transaction/post-date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date that the transaction posted"} {:db/ident :transaction/type :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Yodlee description of the transaction"} {:db/ident :transaction/status :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Yodlee status of the transaction"} {:db/ident :transaction/account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Yodlee account id"} {:db/ident :transaction/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The check number that was parsed from the description"} ;; relations {:db/ident :transaction/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Vendor for who we think this transaction is from"} {:db/ident :transaction/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Client for who we think this transaction is for"} {:db/ident :transaction/bank-account :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The bank accout used for this transaction"} {:db/ident :transaction/payment :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The payment that this transaction matched to"} ]) (def user-schema [{:db/ident :user/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :user/provider :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Provider for oauth for the user"} {:db/ident :user/provider-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "the id from the provider"} {:db/ident :user/role :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The role [:user :admin :none]"} {:db/ident :user/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "name of the user"} {:db/ident :user/clients :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/doc "The clients this user can view"} ;;enums {:db/ident :user-role/admin} {:db/ident :user-role/user} {:db/ident :user-role/none} ]) (def base-schema [ address-schema contact-schema vendor-schema client-schema bank-account-schema invoice-schema invoice-expense-account-schema payment-schema invoice-payment-schema transaction-schema user-schema]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-vendors [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-clients [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-invoices [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-payments [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-invoices-payments [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-invoices-expense-accounts [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-transactions [_] [[]]) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn migrate-users [_] [[]]) (defn merge-query [query-part-1 query-part-2] (-> query-part-1 (update-in [:query :find] into (get-in query-part-2 [:query :find])) (update-in [:query :in] into (get-in query-part-2 [:query :in])) (update-in [:query :where] into (get-in query-part-2 [:query :where])) (update-in [:args] into (get-in query-part-2 [:args])))) (defn add-sorter-fields [q sort-map args] (log/info "sort-map" (pr-str sort-map)) (reduce (fn [q {:keys [sort-key]}] (merge-query q {:query {:find [(symbol (str "?sort-" sort-key))] :where (sort-map sort-key (println "Warning, trying to sort by unsupported field" sort-key))}})) q (:sort args))) (defn apply-sort-3 [args results] (let [sort-bys (conj (:sort args) {:sort-key "default" :asc (if (contains? args :default-asc?) (:default-asc? args) true)}) length (count sort-bys) comparator (fn [xs ys] (reduce (fn [_ i] (let [comparison (if (:asc (nth sort-bys i)) (compare (nth xs i) (nth ys i)) (compare (nth ys i) (nth xs i)))] (if (not= 0 comparison) (reduced comparison) 0))) 0 (range length)))] (sort comparator results ))) (defn apply-pagination [args results] (log/info (take 4 results)) {:ids (->> results (drop (:start args 0)) (take (:count args (or (:per-page args) default-pagination-size))) (map last)) :count (count results)}) (defn audit-transact-batch [txes id] (let [batch-id (.toString (java.util.UUID/randomUUID))] (reduce (fn [full-tx batch] (let [batch (conj (vec batch) {:db/id "datomic.tx" :audit/user (str (:user/role id) "-" (:user/name id)) :audit/batch batch-id}) _ (log/info "transacting batch " batch-id " " (count batch)) tx-result @(d/transact conn batch) _ (Thread/sleep 1000)] (cond-> full-tx (:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result))) (not (:tx-data full-tx)) (assoc :tx-data (vec (:tx-data tx-result))) (not (:db-before full-tx)) (assoc :db-before (:db-before tx-result)) true (assoc :db-after (:db-after tx-result)) true (update :tempids merge (:tempids tx-result))))) {} (partition-all 50 txes)))) (defn audit-transact [txes id] @(d/transact conn (conj txes {:db/id "datomic.tx" :audit/user (str (:user/role id) "-" (:user/name id))})))