442 lines
16 KiB
Clojure
442 lines
16 KiB
Clojure
(ns auto-ap.datomic.migrate.add-general-ledger
|
|
(:require [datomic.api :as d]
|
|
[auto-ap.datomic :refer [remove-nils]]
|
|
[auto-ap.datomic.accounts :as accounts]
|
|
[auto-ap.expense-accounts :as expense-accounts]
|
|
[auto-ap.ledger :as ledger]))
|
|
|
|
(defn test-run [txs-set]
|
|
(println "processing " (count txs-set))
|
|
(doseq [[i tx] (map vector (range) txs-set)]
|
|
@(d/transact (d/connect auto-ap.datomic/uri)
|
|
tx)
|
|
(when (= 0 (mod i 1000))
|
|
(println "processed " i))))
|
|
|
|
|
|
|
|
(def add-general-ledger
|
|
[[{:db/ident :journal-entry/source
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The type of entity that created this entry"}
|
|
{:db/ident :journal-entry/client
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The client for the ledger"}
|
|
{:db/ident :journal-entry/date
|
|
:db/valueType :db.type/instant
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The time for this entry"}
|
|
{:db/ident :journal-entry/original-entity
|
|
:db/valueType :db.type/ref
|
|
:db/unique :db.unique/identity
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The thing that created this entry"}
|
|
{:db/ident :journal-entry/vendor
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The vendor for the ledger entry"}
|
|
{:db/ident :journal-entry/amount
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The amount for the entry"}
|
|
|
|
{:db/ident :journal-entry/line-items
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many
|
|
:db/isComponent true
|
|
:db/doc "Each of the line items in the general ledger"}
|
|
|
|
{:db/ident :journal-entry/cleared
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Has this cleared?"}
|
|
|
|
{:db/ident :journal-entry-line/expense-account
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The expense account being changed"}
|
|
{:db/ident :journal-entry-line/account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The other type of account being changed"}
|
|
{:db/ident :journal-entry-line/debit
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The amount to debit"}
|
|
{:db/ident :journal-entry-line/credit
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The amount to credit"}
|
|
{:db/ident :journal-entry-line/location
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Location of the entry"}
|
|
|
|
]
|
|
]
|
|
)
|
|
(def add-transaction-account
|
|
[[{:db/ident :transaction/accounts
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many
|
|
:db/isComponent true
|
|
:db/doc "The debit(s)/credit(s) for this transaction"}
|
|
|
|
{:db/ident :transaction-account/account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Which account to debit/credit for this transaction"}
|
|
|
|
{:db/ident :transaction-account/location
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Location for this expense account"}
|
|
|
|
{:db/ident :transaction-account/amount
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "How much to debit/credit - must be positive"}]])
|
|
|
|
(def add-yodlee-merchant
|
|
[[{:db/ident :yodlee-merchant/name
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The yodlee merchant name"}
|
|
|
|
{:db/ident :yodlee-merchant/yodlee-id
|
|
:db/valueType :db.type/string
|
|
:db/unique :db.unique/identity
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The yodlee merchant id"}
|
|
|
|
{:db/ident :transaction/yodlee-merchant
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The yodlee merchant"}]])
|
|
|
|
(def add-accounts
|
|
[[
|
|
{:db/ident :account/code
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The code for the expense account (e.g., A1123)"}
|
|
|
|
{:db/ident :account/numeric-code
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The numeric-only for the expense account (e.g., 5150)"}
|
|
|
|
{:db/ident :account/name
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The name of the code (e.g., \"Telephone - HQ\")"}
|
|
|
|
{:db/ident :account/location
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "A forced location for this code, e.g., HQ."}
|
|
|
|
{:db/ident :account/account-set
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The set of accounts this entry belongs to. Allows customization."}
|
|
|
|
{:db/ident :account/type
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The type of account, (e.g., :account-type/expense :account-type/liability)"}
|
|
|
|
{:db/ident :account-type/expense}
|
|
{:db/ident :account-type/liability}
|
|
{:db/ident :account-type/revenue}
|
|
{:db/ident :account-type/dividend}
|
|
{:db/ident :account-type/asset}
|
|
{:db/ident :account-type/equity}]
|
|
|
|
(->> expense-accounts/chooseable-expense-accounts
|
|
(filter (fn [[numeric]]
|
|
(not (#{0 2100 5800 9600 1300 8200 7400 8500 2400 8800 1400 2500 7200 1200 1600 1500 3300 9800 9200 2800 1100 5700 2600 9300 9500 8100 7500 7100 2700} numeric))))
|
|
|
|
(mapv
|
|
(fn [[numeric {:keys [name location]}]]
|
|
(remove-nils
|
|
{:account/type (cond (<= 1000 numeric 1999)
|
|
:account-type/asset
|
|
|
|
(<= 2000 numeric 2999)
|
|
:account-type/liability
|
|
|
|
(<= 3000 numeric 3999)
|
|
:account-type/equity
|
|
|
|
(<= 4000 numeric 4999)
|
|
:account-type/revenue
|
|
|
|
:else
|
|
:account-type/expense)
|
|
:account/numeric-code numeric
|
|
:account/code (str numeric)
|
|
:account/name name
|
|
:account/location location
|
|
:account/account-set "default"}))))])
|
|
|
|
(defn add-general-ledger-fns [conn]
|
|
[[{:db/ident :replace-general-ledger
|
|
:db/doc "Deletes the general ledger entries for an entity"
|
|
:db/fn (d/function '{:lang "clojure"
|
|
:params [db origin]
|
|
:code (let [ids (->> (d/query {:query {:find ['?e]
|
|
:in ['$ '?origin]
|
|
:where [['?e :journal-entry/original-entity '?origin]]}
|
|
:args [db origin]})
|
|
(map first))]
|
|
(into []
|
|
(map (fn [i] [:db/retractEntity i ]) ids)))})}]] )
|
|
|
|
(defn bulk-load-invoice-ledger [conn]
|
|
(let [invoice-ids (map first (d/query {:query {:find '[?e]
|
|
:in '[$]
|
|
:where ['[?e :invoice/total]]}
|
|
:args [(d/db conn)]}))
|
|
z (->> invoice-ids
|
|
(map #(ledger/entity-change->ledger (d/db conn) [:invoice %]))
|
|
(partition-all 100))]
|
|
z))
|
|
|
|
|
|
(defn bulk-load-transaction-ledger [conn]
|
|
(let [transaction-ids (map first (d/query {:query {:find '[?e]
|
|
:in '[$]
|
|
:where ['[?e :transaction/id]]}
|
|
:args [(d/db conn)]}))
|
|
_ (println (count transaction-ids))
|
|
z (->> transaction-ids
|
|
(map #(ledger/entity-change->ledger (d/db conn) [:transaction %]))
|
|
(filter identity)
|
|
(partition-all 100)
|
|
#_(mapv #(vector %)))]
|
|
z))
|
|
|
|
#_(test-run (bulk-load-transaction-ledger (d/connect auto-ap.datomic/uri)))
|
|
#_(test-run (bulk-load-invoice-ledger (d/connect auto-ap.datomic/uri)))
|
|
#_(test-run (reset-ledger (d/connect auto-ap.datomic/uri)))
|
|
|
|
#_(do (doseq [tran (convert-transactions (d/connect auto-ap.datomic/uri))]
|
|
@(d/transact (d/connect auto-ap.datomic/uri) tran))
|
|
(println "done."))
|
|
|
|
#_(do @(d/transact (d/connect auto-ap.datomic/uri)
|
|
(bulk-load-transaction-ledger (d/connect auto-ap.datomic/uri)))
|
|
(println "test"))
|
|
|
|
|
|
(def change-expense-account-to-entity
|
|
[[{:db/ident :invoice-expense-account/account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The account entity that this expense is for"}]])
|
|
|
|
|
|
|
|
(def add-account-to-vendor
|
|
[[{:db/ident :vendor/default-account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The account will be used when a bill is created."}]])
|
|
|
|
(defn convert-vendors [conn]
|
|
(let [vendors (d/query {:query {:find '[?v ?expense-account-id]
|
|
:in '[$]
|
|
:where ['[?v :vendor/default-expense-account ?expense-account-id]]}
|
|
:args [(d/db conn)]})]
|
|
[(mapv
|
|
(fn [[v expense-account-id]]
|
|
(remove-nils {:db/id v
|
|
:vendor/default-account (accounts/get-account-by-numeric-code-and-sets expense-account-id ["default"])}))
|
|
vendors)]))
|
|
|
|
(defn convert-invoices [conn]
|
|
(let [invoice-expense-accounts (d/query {:query {:find '[?iea ?expense-account-id]
|
|
:in '[$]
|
|
:where ['[?iea :invoice-expense-account/expense-account-id ?expense-account-id]]}
|
|
:args [(d/db conn)]})]
|
|
[(mapv
|
|
(fn [[iea expense-account-id]]
|
|
(remove-nils {:db/id iea
|
|
:invoice-expense-account/account (accounts/get-account-by-numeric-code-and-sets expense-account-id ["default"])}))
|
|
invoice-expense-accounts)]))
|
|
|
|
(defn convert-transactions [conn]
|
|
(let [matched-transactions (d/query {:query {:find '[?transaction ?v ?amount]
|
|
:in '[$]
|
|
:where ['[?transaction :transaction/payment ?payment]
|
|
'[?transaction :transaction/amount ?amount]
|
|
'[?payment :payment/invoices ?i]
|
|
'[?i :invoice/vendor ?v]]}
|
|
:args [(d/db (d/connect auto-ap.datomic/uri))]})]
|
|
[(mapv
|
|
(fn [[transaction-id vendor-id amount]]
|
|
(remove-nils {:db/id transaction-id
|
|
:transaction/vendor vendor-id
|
|
:transaction/location "A"
|
|
:transaction/approval-status :transaction-approval-status/excluded
|
|
:transaction/accounts [#:transaction-account {:account (:db/id (accounts/get-account-by-numeric-code-and-sets 2110 ["default"]))
|
|
:location "A"
|
|
:amount (Math/abs amount)}]
|
|
}))
|
|
matched-transactions)]))
|
|
|
|
(def add-location-to-transaction
|
|
[[{:db/ident :transaction/location
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Location of the transaction's target"}]])
|
|
|
|
(def add-external-id-to-ledger
|
|
[[{:db/ident :journal-entry/external-id
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/unique :db.unique/identity
|
|
:db/doc "For externally imported id"}]])
|
|
|
|
(def add-exclude-to-transaction
|
|
[[{:db/ident :transaction/exclude-from-ledger
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Whether to exclude from the ledger"}]])
|
|
|
|
(def add-transaction-rules
|
|
[[{:db/ident :transaction-rule/client
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The specific client this rule is for"}
|
|
|
|
{:db/ident :transaction-rule/bank-account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The specific bank account this rule is for"}
|
|
|
|
{:db/ident :transaction-rule/yodlee-merchant
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Apply this rule if the yodlee merchant matches"}
|
|
|
|
{:db/ident :transaction-rule/description
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "A description to match this rule against"}
|
|
|
|
{:db/ident :transaction-rule/note
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "A friendly description for this rule (internal)"}
|
|
|
|
{:db/ident :transaction-rule/amount-lte
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Amount has to be less than or equal to this"}
|
|
|
|
{:db/ident :transaction-rule/amount-gte
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Amount has to be greater than or equal to this"}
|
|
|
|
{:db/ident :transaction-rule/dom-lte
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Day of month has to be less than or equal to this"}
|
|
|
|
{:db/ident :transaction-rule/dom-gte
|
|
:db/valueType :db.type/long
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Day of month has to be greater than or equal to this"}
|
|
|
|
{:db/ident :transaction-rule/vendor
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The vendor to assign"}
|
|
|
|
{:db/ident :transaction-rule-account/percentage
|
|
:db/valueType :db.type/double
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "How much should go to this account"}
|
|
|
|
{:db/ident :transaction-rule-account/location
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The location this split is for"}
|
|
|
|
{:db/ident :transaction-rule-account/account
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The account of this split"}
|
|
|
|
{:db/ident :transaction-rule/accounts
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/many
|
|
:db/isComponent true
|
|
:db/doc "The outcome split"}
|
|
|
|
{:db/ident :transaction/approval-status
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Status of a transaction"}
|
|
|
|
{:db/ident :transaction/matched-rule
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "The rule that this transaction matched"}
|
|
|
|
{:db/ident :transaction-rule/transaction-approval-status
|
|
:db/valueType :db.type/ref
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Status of a transaction"}
|
|
|
|
{:db/ident :transaction-approval-status/approved}
|
|
{:db/ident :transaction-approval-status/unapproved}
|
|
{:db/ident :transaction-approval-status/requires-feedback}
|
|
{:db/ident :transaction-approval-status/excluded}
|
|
]])
|
|
|
|
(def add-bank-account-locations
|
|
[[{:db/ident :bank-account/locations
|
|
:db/valueType :db.type/string
|
|
:db/cardinality :db.cardinality/many
|
|
:db/doc "Which bank accounts this bank account is valid for"}]])
|
|
|
|
(def add-credit-bank-account
|
|
[[{:db/ident :bank-account-type/credit}]])
|
|
|
|
(defn add-hidden-to-vendor [conn]
|
|
[[{:db/ident :vendor/hidden
|
|
:db/valueType :db.type/boolean
|
|
:db/cardinality :db.cardinality/one
|
|
:db/doc "Whether or not to exclude the vendor"}]
|
|
(let [vendors (d/query {:query {:find '[?v]
|
|
:in '[$]
|
|
:where ['[?v :vendor/name]]}
|
|
:args [(d/db conn)]})]
|
|
(mapv
|
|
(fn [[v ]]
|
|
{:db/id v
|
|
:vendor/hidden false})
|
|
vendors))])
|
|
|
|
|
|
|
|
(defn reset-ledger [conn]
|
|
(let [results (->> (d/query {:query {:find '[?e]
|
|
:in '[$]
|
|
:where ['[?e :journal-entry/original-entity ]]}
|
|
:args [(d/db conn)]})
|
|
(mapv (fn [[id]]
|
|
[:db/retractEntity id]))
|
|
(partition-all 100))]
|
|
results))
|
|
|
|
#_(do (doseq [tran (convert-transactions (d/connect auto-ap.datomic/uri))]
|
|
@(d/transact (d/connect auto-ap.datomic/uri) tran))
|
|
(println "done."))
|