(ns auto-ap.datomic.migrate (:require [auto-ap.datomic :refer [uri conn]] [datomic.api :as d] [auto-ap.datomic.migrate.add-client-codes :refer [add-client-codes]] [auto-ap.datomic.migrate.add-bank-account-codes :refer [add-bank-account-codes]] [auto-ap.datomic.migrate.invoice-converter :refer [add-import-status-existing-invoices]] [auto-ap.datomic.migrate.add-general-ledger :as add-general-ledger] [auto-ap.datomic.migrate.ledger :as ledger] [auto-ap.datomic.migrate.sales :as sales] [auto-ap.datomic.migrate.clients :as clients] [auto-ap.datomic.migrate.audit :as audit] [clojure.java.io :as io] [io.rkn.conformity :as c] [mount.core :as mount]) (:import [datomic Util]) (:gen-class)) (defn read-dtm "Reads a dtm file (i.e., an edn file with datomic tags in it) from the classpath and returns a vector of all forms contained within." [filename] (-> (io/resource filename) (io/reader) (Util/readAll))) (defn functions [conn] [[{:db/ident :pay :db/doc "Data function that increments value of attribute a by amount." :db/fn (d/function '{:lang "clojure" :params [db e amount] :code (let [current-outstanding-balance (-> (d/entity db e) :invoice/outstanding-balance) new-outstanding-balance (- current-outstanding-balance amount)] [[:db/add e :invoice/outstanding-balance new-outstanding-balance] [:db/add e :invoice/status (if (> new-outstanding-balance 0) :invoice-status/unpaid :invoice-status/paid)]])})} {:db/ident :inc :db/doc "Data function that increments value of attribute a by amount." :db/fn (d/function '{:lang "clojure" :params [db e a amount] :code [[:db/add e a (-> (d/entity db e) a (+ amount))]] })}]] ) (defn fix-pay-function [conn] [[{:db/ident :pay :db/doc "Data function that increments value of attribute a by amount." :db/fn (d/function '{:lang "clojure" :params [db e amount] :code (let [current-outstanding-balance (-> (d/entity db e) :invoice/outstanding-balance) new-outstanding-balance (- current-outstanding-balance amount)] [[:db/add e :invoice/outstanding-balance new-outstanding-balance] [:db/add e :invoice/status (if (< -0.001 new-outstanding-balance 0.001) :invoice-status/paid :invoice-status/unpaid)]])})}]] ) (def add-client-identifier [[{:db/ident :invoice/client-identifier :db/doc "An identifier found in an uploaded invoice" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]]) (defn reset-function [conn] [[{:db/ident :reset :db/doc "Resets a relationship to the values specified " :db/fn (d/function '{:lang "clojure" :params [db e a vs] :code (let [ids (when-not (string? e) (->> (d/query {:query {:find ['?z] :in ['$ '?e '?a] :where [['?e '?a '?z]]} :args [db e a]}) (map first))) new-id-set (set (map :db/id vs)) retract-ids (filter (complement new-id-set) ids)] (cond-> [] true (into (map (fn [i] [:db/retract e a i ]) retract-ids)) (seq vs) (conj {:db/id e a vs})))})}]]) (defn add-include-in-reports [conn] (let [existing-accounts (->> (d/query {:query {:find ['?e] :in ['$] :where ['[?e :bank-account/name]]} :args [(d/db conn)]}) (transduce (comp (map first) (map (fn [bank-account] {:db/id bank-account :bank-account/include-in-reports true}))) conj []))] [existing-accounts])) (defn migrate [conn] (let [ norms-map (merge {:auto-ap/base-schema {:txes auto-ap.datomic/base-schema} :auto-ap/functions {:txes-fn 'auto-ap.datomic.migrate/functions :requires [:auto-ap/base-schema]} :auto-ap/fx-pay-function-2 {:txes-fn 'auto-ap.datomic.migrate/fix-pay-function :requires [:auto-ap/functions]} :auto-ap/migrate-vendors {:txes-fn 'auto-ap.datomic/migrate-vendors :requires [:auto-ap/base-schema]} :auto-ap/migrate-clients {:txes-fn 'auto-ap.datomic/migrate-clients :requires [:auto-ap/migrate-vendors]} :auto-ap/migrate-users {:txes-fn 'auto-ap.datomic/migrate-users :requires [:auto-ap/migrate-clients]} :auto-ap/migrate-invoices {:txes-fn 'auto-ap.datomic/migrate-invoices :requires [:auto-ap/migrate-vendors :auto-ap/migrate-clients]} :auto-ap/migrate-payments {:txes-fn 'auto-ap.datomic/migrate-payments :requires [:auto-ap/migrate-invoices]} :auto-ap/migrate-invoices-payments {:txes-fn 'auto-ap.datomic/migrate-invoices-payments :requires [:auto-ap/migrate-payments :auto-ap/migrate-invoices]} :auto-ap/migrate-invoices-expense-accounts {:txes-fn 'auto-ap.datomic/migrate-invoices-expense-accounts :requires [:auto-ap/migrate-invoices-payments]} :auto-ap/migrate-transactions {:txes-fn 'auto-ap.datomic/migrate-transactions :requires [:auto-ap/migrate-invoices-expense-accounts]} :auto-ap/add-client-codes {:txes-fn 'auto-ap.datomic.migrate.add-client-codes/add-client-codes :requires [:auto-ap/migrate-transactions]} :auto-ap/add-bank-account-codes-schema {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes-schema :requires [:auto-ap/add-client-codes]} :auto-ap/add-bank-account-codes {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes :requires [:auto-ap/add-bank-account-codes-schema]} :auto-ap/add-nick-the-greek {:txes [[{:client/name "Nick the Greek" :client/code "NGAK" :client/locations ["MH"] :client/bank-accounts [{:bank-account/code "NGAK-0" :bank-account/type :bank-account-type/cash :bank-account/name "Cash"}]}]] :requires [:auto-ap/add-bank-account-codes]} :auto-ap/rename-codes-1 {:txes-fn 'auto-ap.datomic.migrate.rename-codes/rename-codes-1 :requires [:auto-ap/add-nick-the-greek]} :auto-ap/invoice-converter {:txes auto-ap.datomic.migrate.invoice-converter/add-matches :requires [:auto-ap/rename-codes-1]} :auto-ap/starter {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-starter :requires [:auto-ap/invoice-converter]} :auto-ap/add-default-location {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location :requires [:auto-ap/invoice-converter]} :auto-ap/add-default-location-2 {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location-2 :requires [:auto-ap/add-default-location]} :auto-ap/add-import-status {:txes auto-ap.datomic.migrate.invoice-converter/add-import-status :requires [:auto-ap/add-default-location-2]} :auto-ap/add-import-status-existing-invoices {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-import-status-existing-invoices :requires [:auto-ap/add-import-status]} :auto-ap/fix-check-numbers {:txes-fn 'auto-ap.datomic.migrate.check-numbers/fix-check-numbers :requires [:auto-ap/add-import-status-existing-invoices]} :auto-ap/add-new-vendors {:txes-fn 'auto-ap.datomic.migrate.add-new-vendors/add-new-vendors :requires [:auto-ap/fix-check-numbers]} :auto-ap/add-account-visibility-fields {:txes-fn 'auto-ap.datomic.migrate.account-sorting/add-account-visibility-fields :requires [:auto-ap/add-new-vendors]} :auto-ap/make-every-account-visible {:txes-fn 'auto-ap.datomic.migrate.account-sorting/make-every-account-visible :requires [:auto-ap/add-account-visibility-fields]} :auto-ap/add-general-ledger6 {:txes add-general-ledger/add-general-ledger :requires [:auto-ap/make-every-account-visible]} :auto-ap/add-general-ledger-fns2 {:txes-fn 'auto-ap.datomic.migrate.add-general-ledger/add-general-ledger-fns :requires [:auto-ap/add-general-ledger6]} :auto-ap/add-accounts {:txes auto-ap.datomic.migrate.add-general-ledger/add-accounts :requires [:auto-ap/add-general-ledger-fns2]} :auto-ap/add-transaction-account {:txes auto-ap.datomic.migrate.add-general-ledger/add-transaction-account :requires [:auto-ap/add-accounts]} :auto-ap/change-expense-account-to-entity {:txes auto-ap.datomic.migrate.add-general-ledger/change-expense-account-to-entity :requires [:auto-ap/add-transaction-account]} :auto-ap/add-account-to-vendor {:txes auto-ap.datomic.migrate.add-general-ledger/add-account-to-vendor :requires [:auto-ap/change-expense-account-to-entity]} :auto-ap/add-location-to-transaction {:txes add-general-ledger/add-location-to-transaction :requires [:auto-ap/add-account-to-vendor]} :auto-ap/add-credit-bank-account {:txes add-general-ledger/add-credit-bank-account :requires [:auto-ap/add-location-to-transaction]} :auto-ap/add-hidden-to-vendor {:txes-fn `add-general-ledger/add-hidden-to-vendor :requires [:auto-ap/add-credit-bank-account]} :auto-ap/convert-vendors {:txes-fn `add-general-ledger/convert-vendors :requires [:auto-ap/add-hidden-to-vendor]} :auto-ap/convert-invoices {:txes-fn `add-general-ledger/convert-invoices :requires [:auto-ap/convert-vendors]} :auto-ap/add-yodlee-merchant2 {:txes add-general-ledger/add-yodlee-merchant :requires [:auto-ap/convert-vendors]} :auto-ap/add-external-id-to-ledger {:txes add-general-ledger/add-external-id-to-ledger :requires [:auto-ap/add-yodlee-merchant2]} :auto-ap/add-exclude-to-transaction {:txes add-general-ledger/add-exclude-to-transaction :requires [:auto-ap/add-external-id-to-ledger]} :auto-ap/add-client-identifier2 {:txes add-client-identifier :requires [:auto-ap/make-every-account-visible]} :auto-ap/add-transaction-rules {:txes add-general-ledger/add-transaction-rules :requires [:auto-ap/add-exclude-to-transaction]} :auto-ap/add-bank-account-locations {:txes add-general-ledger/add-bank-account-locations :requires [:auto-ap/add-transaction-rules]} :auto-ap/convert-transactions {:txes-fn `add-general-ledger/convert-transactions :requires [:auto-ap/add-bank-account-locations]} :auto-ap/add-exclude-to-invoice {:txes add-general-ledger/add-exclude-to-invoice :requires [:auto-ap/convert-transactions]} :auto-ap/add-terms {:txes [[{:db/ident :vendor/terms :db/doc "How many days till you pay" :db/valueType :db.type/long :db/cardinality :db.cardinality/one}]]} :auto-ap/add-due {:txes [[{:db/ident :invoice/due :db/doc "When you gotta pay" :db/valueType :db.type/instant :db/cardinality :db.cardinality/one}]]} :auto-ap/add-vendor-overrides {:txes [[{:db/ident :vendor-account-override/account :db/doc "the account for invoices" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one} {:db/ident :vendor-account-override/client :db/doc "How many days till you pay" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one} {:db/ident :vendor-terms-override/terms :db/doc "How many days till you pay" :db/valueType :db.type/long :db/cardinality :db.cardinality/one} {:db/ident :vendor-terms-override/client :db/doc "How many days till you pay" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one} {:db/ident :vendor/terms-overrides :db/doc "Overrides per-client" :db/isComponent true :db/valueType :db.type/ref :db/cardinality :db.cardinality/many} {:db/ident :vendor/account-overrides :db/isComponent true :db/doc "Overrides per-client" :db/valueType :db.type/ref :db/cardinality :db.cardinality/many}]]} :auto-ap/add-reset-rels {:txes-fn `reset-function} :auto-ap/add-account-overrides {:txes [[{:db/ident :account/applicability :db/doc ":global, :optional :customized" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one} {:db/ident :account-applicability/global :db/doc "The account applies to all cutsomers"} {:db/ident :account-applicability/optional :db/doc "This account is optional"} {:db/ident :account-applicability/customized :db/doc "This account is customized per-customer"} {:db/ident :account/client-overrides :db/doc "Customizations per customer" :db/valueType :db.type/ref :db/isComponent true :db/cardinality :db.cardinality/many} {:db/ident :account-client-override/client :db/doc "The client for the override" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one} {:db/ident :account-client-override/enabled :db/doc "Does this apply?" :db/valueType :db.type/boolean :db/cardinality :db.cardinality/one} {:db/ident :account-client-override/name :db/doc "client override" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]]} :auto-ap/add-cleared-against {:txes [[{:db/ident :transaction/cleared-against :db/doc "which entitiy it was cleared against" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]]} :auto-ap/add-cash-flow-schema {:txes [[{:db/ident :client/weekly-debits :db/doc "How much money gets debited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :client/weekly-credits :db/doc "How much money gets credited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :client/forecasted-transactions :db/doc "Regular, planned transactions" :db/valueType :db.type/ref :db/isComponent true :db/cardinality :db.cardinality/many} {:db/ident :forecasted-transaction/amount :db/doc "Amount of a forcested transaction" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :forecasted-transaction/day-of-month :db/doc "Which day the transaction occurs" :db/valueType :db.type/long :db/cardinality :db.cardinality/one} {:db/ident :forecasted-transaction/identifier :db/doc "An identifier for this forcasted transaction, e.g., 'RENT'" :db/valueType :db.type/string :db/cardinality :db.cardinality/one}]]} :auto-ap/add-manager-schema {:txes [[{:db/ident :user-role/manager}]]} :auto-ap/add-include-in-reports1 {:txes [[{:db/ident :bank-account/include-in-reports :db/doc "Whether to include this bank account in balance sheet, etc." :db/valueType :db.type/boolean :db/cardinality :db.cardinality/one}]] :depends-on [:auto-ap/add-manager-schema]} :auto-ap/migrate-include-in-reports {:txes-fn `add-include-in-reports :depends-on [:auto-ap/add-include-in-reports1] } :auto-ap/add-forecasted-transaction-match {:txes [[{:db/ident :transaction/forecast-match :db/doc "Which forecast this transaction matches, for cashflow" :db/valueType :db.type/ref :db/cardinality :db.cardinality/one}]]} :auto-ap/add-week-a-and-b {:txes [[{:db/ident :client/week-a-debits :db/doc "How much money gets debited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :client/week-a-credits :db/doc "How much money gets credited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :client/week-b-debits :db/doc "How much money gets debited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one} {:db/ident :client/week-b-credits :db/doc "How much money gets credited each week" :db/valueType :db.type/double :db/cardinality :db.cardinality/one}]]} :auto-ap/auto-pay {:txes [[{:db/ident :vendor/automatically-paid-when-due :db/doc "The clients for which invoices will automatically be paid." :db/valueType :db.type/ref :db/cardinality :db.cardinality/many} {:db/ident :invoice/automatically-paid-when-due :db/doc "Whether this invoice should be marked as paid when it's due" :db/valueType :db.type/boolean :db/cardinality :db.cardinality/one}]]} :auto-ap/fix-reset-rels {:txes-fn `reset-function}} sales/norms-map clients/norms-map ledger/norms-map audit/norms-map) ] (println "Conforming database...") (c/ensure-conforms conn norms-map) #_(when (not (seq args)) (d/release conn)) (println "Done"))) (defn -main [& args] (mount/start (mount/only #{#'conn})) (migrate conn) (mount/stop) ) (mount/defstate migrate-start :start (future (migrate conn)) :stop (deref migrate-start))