406 lines
33 KiB
Clojure
406 lines
33 KiB
Clojure
(ns auto-ap.datomic.migrate
|
|
(:require [auto-ap.datomic :refer [uri conn]]
|
|
[datomic.api :as d]
|
|
[auto-ap.datomic.migrate.add-client-codes :refer [add-client-codes]]
|
|
[auto-ap.datomic.migrate.add-bank-account-codes :refer [add-bank-account-codes]]
|
|
[auto-ap.datomic.migrate.invoice-converter :refer [add-import-status-existing-invoices]]
|
|
[auto-ap.datomic.migrate.add-general-ledger :as add-general-ledger]
|
|
[auto-ap.datomic.migrate.ledger :as ledger]
|
|
[auto-ap.datomic.migrate.sales :as sales]
|
|
[auto-ap.datomic.migrate.vendors :as vendors]
|
|
[auto-ap.datomic.migrate.clients :as clients]
|
|
[auto-ap.datomic.migrate.audit :as audit]
|
|
[auto-ap.datomic.migrate.yodlee2 :as yodlee2]
|
|
|
|
[clojure.java.io :as io]
|
|
[io.rkn.conformity :as c]
|
|
[mount.core :as mount])
|
|
(:import [datomic Util])
|
|
(:gen-class))
|
|
|
|
|
|
(defn read-dtm
|
|
"Reads a dtm file (i.e., an edn file with datomic tags in it) from the classpath
|
|
and returns a vector of all forms contained within."
|
|
[filename]
|
|
(-> (io/resource filename) (io/reader) (Util/readAll)))
|
|
|
|
(defn functions [conn]
|
|
[[{:db/ident :pay
|
|
:db/doc "Data function that increments value of attribute a by amount."
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db e amount]
|
|
:code (let [current-outstanding-balance (-> (d/entity db e) :invoice/outstanding-balance)
|
|
new-outstanding-balance (- current-outstanding-balance amount)]
|
|
[[:db/add e :invoice/outstanding-balance new-outstanding-balance]
|
|
[:db/add e :invoice/status (if (> new-outstanding-balance 0)
|
|
:invoice-status/unpaid
|
|
:invoice-status/paid)]])})}
|
|
{:db/ident :inc
|
|
:db/doc "Data function that increments value of attribute a by amount."
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db e a amount]
|
|
:code [[:db/add e a
|
|
(-> (d/entity db e) a (+ amount))]] })}]] )
|
|
|
|
(defn fix-pay-function [conn]
|
|
[[{:db/ident :pay
|
|
:db/doc "Data function that increments value of attribute a by amount."
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db e amount]
|
|
:code (let [current-outstanding-balance (-> (d/entity db e) :invoice/outstanding-balance)
|
|
new-outstanding-balance (- current-outstanding-balance amount)]
|
|
[[:db/add e :invoice/outstanding-balance new-outstanding-balance]
|
|
[:db/add e :invoice/status (if (< -0.001 new-outstanding-balance 0.001)
|
|
:invoice-status/paid
|
|
:invoice-status/unpaid)]])})}]] )
|
|
|
|
(def add-client-identifier
|
|
[[{:db/ident :invoice/client-identifier
|
|
:db/doc "An identifier found in an uploaded invoice"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one}]])
|
|
|
|
(defn reset-function [conn]
|
|
[[{:db/ident :reset
|
|
:db/doc "Resets a relationship to the values specified "
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db e a vs]
|
|
:code (let [ids (when-not (string? e)
|
|
(->> (d/query {:query {:find ['?z]
|
|
:in ['$ '?e '?a]
|
|
:where [['?e '?a '?z]]}
|
|
:args [db e a]})
|
|
(map first)))
|
|
new-id-set (set (map :db/id vs))
|
|
retract-ids (filter (complement new-id-set) ids)]
|
|
(cond-> []
|
|
true (into (map (fn [i] [:db/retract e a i ]) retract-ids))
|
|
(seq vs) (conj {:db/id e
|
|
a vs})))})}]])
|
|
|
|
|
|
(defn propose-invoice-fn [conn]
|
|
[[{:db/ident :propose-invoice
|
|
:db/doc "Adds an invoice if it's not found"
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db invoice]
|
|
:code (let [existing? (boolean (seq (d/query {:query {:find ['?i]
|
|
:in ['$ '?invoice-number '?client '?vendor]
|
|
:where ['[?i :invoice/invoice-number ?invoice-number]
|
|
'[?i :invoice/client ?client]
|
|
'[?i :invoice/vendor ?vendor]
|
|
'(not [?i :invoice/status :invoice-status/voided])
|
|
]}
|
|
:args [db (:invoice/invoice-number invoice) (:invoice/client invoice) (:invoice/vendor invoice)]})))]
|
|
(if existing?
|
|
[]
|
|
[invoice]))})}]])
|
|
|
|
(defn add-include-in-reports [conn]
|
|
(let [existing-accounts (->> (d/query {:query {:find ['?e]
|
|
:in ['$]
|
|
:where ['[?e :bank-account/name]]}
|
|
:args [(d/db conn)]})
|
|
|
|
(transduce
|
|
(comp
|
|
(map first)
|
|
(map (fn [bank-account]
|
|
{:db/id bank-account :bank-account/include-in-reports true})))
|
|
conj
|
|
[]))]
|
|
[existing-accounts]))
|
|
|
|
(defn migrate [conn]
|
|
(let [
|
|
norms-map (merge {:auto-ap/base-schema {:txes auto-ap.datomic/base-schema}
|
|
:auto-ap/functions {:txes-fn 'auto-ap.datomic.migrate/functions
|
|
:requires [:auto-ap/base-schema]}
|
|
:auto-ap/fx-pay-function-2 {:txes-fn 'auto-ap.datomic.migrate/fix-pay-function
|
|
:requires [:auto-ap/functions]}
|
|
:auto-ap/migrate-vendors {:txes-fn 'auto-ap.datomic/migrate-vendors
|
|
:requires [:auto-ap/base-schema]}
|
|
:auto-ap/migrate-clients {:txes-fn 'auto-ap.datomic/migrate-clients
|
|
:requires [:auto-ap/migrate-vendors]}
|
|
:auto-ap/migrate-users {:txes-fn 'auto-ap.datomic/migrate-users
|
|
:requires [:auto-ap/migrate-clients]}
|
|
:auto-ap/migrate-invoices {:txes-fn 'auto-ap.datomic/migrate-invoices
|
|
:requires [:auto-ap/migrate-vendors :auto-ap/migrate-clients]}
|
|
:auto-ap/migrate-payments {:txes-fn 'auto-ap.datomic/migrate-payments
|
|
:requires [:auto-ap/migrate-invoices]}
|
|
:auto-ap/migrate-invoices-payments {:txes-fn 'auto-ap.datomic/migrate-invoices-payments
|
|
:requires [:auto-ap/migrate-payments :auto-ap/migrate-invoices]}
|
|
:auto-ap/migrate-invoices-expense-accounts {:txes-fn 'auto-ap.datomic/migrate-invoices-expense-accounts
|
|
:requires [:auto-ap/migrate-invoices-payments]}
|
|
:auto-ap/migrate-transactions {:txes-fn 'auto-ap.datomic/migrate-transactions
|
|
:requires [:auto-ap/migrate-invoices-expense-accounts]}
|
|
:auto-ap/add-client-codes {:txes-fn 'auto-ap.datomic.migrate.add-client-codes/add-client-codes
|
|
:requires [:auto-ap/migrate-transactions]}
|
|
:auto-ap/add-bank-account-codes-schema {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes-schema
|
|
:requires [:auto-ap/add-client-codes]}
|
|
:auto-ap/add-bank-account-codes {:txes-fn 'auto-ap.datomic.migrate.add-bank-account-codes/add-bank-account-codes
|
|
:requires [:auto-ap/add-bank-account-codes-schema]}
|
|
:auto-ap/add-nick-the-greek {:txes [[{:client/name "Nick the Greek" :client/code "NGAK" :client/locations ["MH"] :client/bank-accounts [{:bank-account/code "NGAK-0" :bank-account/type :bank-account-type/cash :bank-account/name "Cash"}]}]]
|
|
:requires [:auto-ap/add-bank-account-codes]}
|
|
:auto-ap/rename-codes-1 {:txes-fn 'auto-ap.datomic.migrate.rename-codes/rename-codes-1
|
|
:requires [:auto-ap/add-nick-the-greek]}
|
|
:auto-ap/invoice-converter {:txes auto-ap.datomic.migrate.invoice-converter/add-matches
|
|
:requires [:auto-ap/rename-codes-1]}
|
|
:auto-ap/starter {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-starter
|
|
:requires [:auto-ap/invoice-converter]}
|
|
:auto-ap/add-default-location {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location
|
|
:requires [:auto-ap/invoice-converter]}
|
|
:auto-ap/add-default-location-2 {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-default-location-2
|
|
:requires [:auto-ap/add-default-location]}
|
|
:auto-ap/add-import-status {:txes auto-ap.datomic.migrate.invoice-converter/add-import-status
|
|
:requires [:auto-ap/add-default-location-2]}
|
|
:auto-ap/add-import-status-existing-invoices {:txes-fn 'auto-ap.datomic.migrate.invoice-converter/add-import-status-existing-invoices
|
|
:requires [:auto-ap/add-import-status]}
|
|
:auto-ap/fix-check-numbers {:txes-fn 'auto-ap.datomic.migrate.check-numbers/fix-check-numbers
|
|
:requires [:auto-ap/add-import-status-existing-invoices]}
|
|
:auto-ap/add-new-vendors {:txes-fn 'auto-ap.datomic.migrate.add-new-vendors/add-new-vendors
|
|
:requires [:auto-ap/fix-check-numbers]}
|
|
:auto-ap/add-account-visibility-fields {:txes-fn 'auto-ap.datomic.migrate.account-sorting/add-account-visibility-fields
|
|
:requires [:auto-ap/add-new-vendors]}
|
|
:auto-ap/make-every-account-visible {:txes-fn 'auto-ap.datomic.migrate.account-sorting/make-every-account-visible
|
|
:requires [:auto-ap/add-account-visibility-fields]}
|
|
:auto-ap/add-general-ledger6 {:txes add-general-ledger/add-general-ledger
|
|
:requires [:auto-ap/make-every-account-visible]}
|
|
:auto-ap/add-general-ledger-fns2 {:txes-fn 'auto-ap.datomic.migrate.add-general-ledger/add-general-ledger-fns
|
|
:requires [:auto-ap/add-general-ledger6]}
|
|
:auto-ap/add-accounts {:txes auto-ap.datomic.migrate.add-general-ledger/add-accounts
|
|
:requires [:auto-ap/add-general-ledger-fns2]}
|
|
:auto-ap/add-transaction-account {:txes auto-ap.datomic.migrate.add-general-ledger/add-transaction-account
|
|
:requires [:auto-ap/add-accounts]}
|
|
:auto-ap/change-expense-account-to-entity {:txes auto-ap.datomic.migrate.add-general-ledger/change-expense-account-to-entity
|
|
:requires [:auto-ap/add-transaction-account]}
|
|
:auto-ap/add-account-to-vendor {:txes auto-ap.datomic.migrate.add-general-ledger/add-account-to-vendor
|
|
:requires [:auto-ap/change-expense-account-to-entity]}
|
|
:auto-ap/add-location-to-transaction {:txes add-general-ledger/add-location-to-transaction
|
|
:requires [:auto-ap/add-account-to-vendor]}
|
|
:auto-ap/add-credit-bank-account {:txes add-general-ledger/add-credit-bank-account
|
|
:requires [:auto-ap/add-location-to-transaction]}
|
|
:auto-ap/add-hidden-to-vendor {:txes-fn `add-general-ledger/add-hidden-to-vendor :requires [:auto-ap/add-credit-bank-account]}
|
|
#_#_:auto-ap/convert-vendors {:txes-fn `add-general-ledger/convert-vendors
|
|
:requires [:auto-ap/add-hidden-to-vendor]}
|
|
#_#_:auto-ap/convert-invoices {:txes-fn `add-general-ledger/convert-invoices
|
|
:requires [:auto-ap/convert-vendors]}
|
|
:auto-ap/add-yodlee-merchant2 {:txes add-general-ledger/add-yodlee-merchant }
|
|
:auto-ap/add-external-id-to-ledger {:txes add-general-ledger/add-external-id-to-ledger :requires [:auto-ap/add-yodlee-merchant2]}
|
|
:auto-ap/add-exclude-to-transaction {:txes add-general-ledger/add-exclude-to-transaction :requires [:auto-ap/add-external-id-to-ledger]}
|
|
:auto-ap/add-client-identifier2 {:txes add-client-identifier :requires [:auto-ap/make-every-account-visible]}
|
|
:auto-ap/add-transaction-rules {:txes add-general-ledger/add-transaction-rules :requires [:auto-ap/add-exclude-to-transaction]}
|
|
:auto-ap/add-bank-account-locations {:txes add-general-ledger/add-bank-account-locations :requires [:auto-ap/add-transaction-rules]}
|
|
|
|
;; should not be needed.
|
|
#_#_:auto-ap/convert-transactions {:txes-fn `add-general-ledger/convert-transactions :requires [:auto-ap/add-bank-account-locations]}
|
|
:auto-ap/add-exclude-to-invoice {:txes add-general-ledger/add-exclude-to-invoice }
|
|
:auto-ap/add-terms {:txes [[{:db/ident :vendor/terms
|
|
:db/doc "How many days till you pay"
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-due {:txes [[{:db/ident :invoice/due
|
|
:db/doc "When you gotta pay"
|
|
:db/valueType :db.type/instant
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-vendor-overrides {:txes [[{:db/ident :vendor-account-override/account
|
|
:db/doc "the account for invoices"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :vendor-account-override/client
|
|
:db/doc "How many days till you pay"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :vendor-terms-override/terms
|
|
:db/doc "How many days till you pay"
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :vendor-terms-override/client
|
|
:db/doc "How many days till you pay"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :vendor/terms-overrides
|
|
:db/doc "Overrides per-client"
|
|
:db/isComponent true
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many}
|
|
{:db/ident :vendor/account-overrides
|
|
:db/isComponent true
|
|
:db/doc "Overrides per-client"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many}]]}
|
|
:auto-ap/add-reset-rels {:txes-fn `reset-function}
|
|
:auto-ap/add-account-overrides {:txes [[{:db/ident :account/applicability
|
|
:db/doc ":global, :optional :customized"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
|
|
{:db/ident :account-applicability/global
|
|
:db/doc "The account applies to all cutsomers"}
|
|
|
|
{:db/ident :account-applicability/optional
|
|
:db/doc "This account is optional"}
|
|
|
|
{:db/ident :account-applicability/customized
|
|
:db/doc "This account is customized per-customer"}
|
|
|
|
{:db/ident :account/client-overrides
|
|
:db/doc "Customizations per customer"
|
|
:db/valueType :db.type/ref
|
|
:db/isComponent true
|
|
:db/cardinality :db.cardinality/many}
|
|
|
|
|
|
{:db/ident :account-client-override/client
|
|
:db/doc "The client for the override"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
|
|
{:db/ident :account-client-override/enabled
|
|
:db/doc "Does this apply?"
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one}
|
|
|
|
{:db/ident :account-client-override/name
|
|
:db/doc "client override"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-cleared-against {:txes [[{:db/ident :transaction/cleared-against
|
|
:db/doc "which entitiy it was cleared against"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-cash-flow-schema {:txes [[{:db/ident :client/weekly-debits
|
|
:db/doc "How much money gets debited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :client/weekly-credits
|
|
:db/doc "How much money gets credited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :client/forecasted-transactions
|
|
:db/doc "Regular, planned transactions"
|
|
:db/valueType :db.type/ref
|
|
:db/isComponent true
|
|
:db/cardinality :db.cardinality/many}
|
|
{:db/ident :forecasted-transaction/amount
|
|
:db/doc "Amount of a forcested transaction"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :forecasted-transaction/day-of-month
|
|
:db/doc "Which day the transaction occurs"
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :forecasted-transaction/identifier
|
|
:db/doc "An identifier for this forcasted transaction, e.g., 'RENT'"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-manager-schema {:txes [[{:db/ident :user-role/manager}]]}
|
|
:auto-ap/add-include-in-reports1 {:txes [[{:db/ident :bank-account/include-in-reports
|
|
:db/doc "Whether to include this bank account in balance sheet, etc."
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one}]]
|
|
:depends-on [:auto-ap/add-manager-schema]}
|
|
:auto-ap/migrate-include-in-reports {:txes-fn `add-include-in-reports :depends-on [:auto-ap/add-include-in-reports1] }
|
|
:auto-ap/add-forecasted-transaction-match {:txes [[{:db/ident :transaction/forecast-match
|
|
:db/doc "Which forecast this transaction matches, for cashflow"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-week-a-and-b {:txes [[{:db/ident :client/week-a-debits
|
|
:db/doc "How much money gets debited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :client/week-a-credits
|
|
:db/doc "How much money gets credited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
|
|
{:db/ident :client/week-b-debits
|
|
:db/doc "How much money gets debited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :client/week-b-credits
|
|
:db/doc "How much money gets credited each week"
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/auto-pay {:txes [[{:db/ident :vendor/automatically-paid-when-due
|
|
:db/doc "The clients for which invoices will automatically be paid."
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many}
|
|
|
|
{:db/ident :invoice/automatically-paid-when-due
|
|
:db/doc "Whether this invoice should be marked as paid when it's due"
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
|
|
:auto-ap/fix-reset-rels {:txes-fn `reset-function}
|
|
:auto-ap/add-scheduled-payment {:txes [[{:db/ident :invoice/scheduled-payment
|
|
:db/doc "When an invoice gets marked as paid"
|
|
:db/valueType :db.type/instant
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-scheduled-payment-dom {:txes [[{:db/ident :vendor/schedule-payment-dom
|
|
:db/doc "When an invoice gets marked as paid"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many
|
|
:db/isComponent true}
|
|
{:db/ident :vendor-schedule-payment-dom/client
|
|
:db/doc "Which client the override is for"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :vendor-schedule-payment-dom/dom
|
|
:db/doc "What day of the month"
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one}]]}
|
|
:auto-ap/add-power-user-schema {:txes [[{:db/ident :user-role/power-user}]]}
|
|
:auto-ap/add-transaction-date-index {:txes [[{:db/ident :transaction/date
|
|
:db/index true}]]}
|
|
:auto-ap/add-invoice-index {:txes [[{:db/ident :invoice/invoice-number
|
|
:db/index true}]]
|
|
:depends-on [:auto-ap/base-schema]}
|
|
|
|
:auto-ap/add-propose-invoice {:txes-fn `propose-invoice-fn
|
|
:depends-on [:auto-ap/base-schema]}
|
|
:auto-ap/add-intuit-banks-4 {:txes [[{:db/ident :intuit-bank-account/external-id
|
|
:db/doc "Id of the intui bank"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/unique :db.unique/identity}
|
|
{:db/ident :intuit-bank-account/name
|
|
:db/doc "Name of intuit bank"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/unique :db.unique/identity}
|
|
{:db/ident :bank-account/intuit-bank-account
|
|
:db/doc "intuit external bank account"
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one}
|
|
{:db/ident :transaction/raw-id
|
|
:db/doc "An unhashed version of the id"
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one}]]
|
|
:depends-on [:auto-ap/base-schema]}}
|
|
|
|
|
|
|
|
sales/norms-map
|
|
clients/norms-map
|
|
ledger/norms-map
|
|
yodlee2/norms-map
|
|
audit/norms-map
|
|
vendors/norms-map)]
|
|
(println "Conforming database...")
|
|
(c/ensure-conforms conn norms-map)
|
|
#_(when (not (seq args))
|
|
(d/release conn))
|
|
(println "Done")))
|
|
|
|
(defn -main [& args]
|
|
(mount/start (mount/only #{#'conn}))
|
|
(migrate conn)
|
|
(mount/stop))
|
|
|
|
|
|
(mount/defstate migrate-start
|
|
:start (future (migrate conn))
|
|
:stop (deref migrate-start))
|