From 53063f47f330bea5efe512fceec70174cd942421 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 13 Apr 2019 12:26:51 -0700 Subject: [PATCH] supports bulk loading data. --- src/clj/auto_ap/datomic/migrate.clj | 1 + .../datomic/migrate/add_general_ledger.clj | 54 ++++++++++++++++++- src/clj/auto_ap/ledger.clj | 39 +++++++------- 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index 7539621d..a20e4aa6 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -87,6 +87,7 @@ :auto-ap/add-account-to-vendor {:txes auto-ap.datomic.migrate.add-general-ledger/add-account-to-vendor :requires [:auto-ap/change-expense-account-to-entity]} :auto-ap/convert-vendors {:txes-fn `add-general-ledger/convert-vendors :requires [:auto-ap/add-account-to-vendor]} :auto-ap/convert-invoices {:txes-fn `add-general-ledger/convert-invoices :requires [:auto-ap/add-account-to-vendor]} + :auto-ap/convert-transactions {:txes-fn `add-general-ledger/convert-transactions :requires [:auto-ap/convert-invoices]} #_#_:auto-ap/bulk-load-invoice-ledger2 {:txes-fn 'auto-ap.datomic.migrate.add-general-ledger/bulk-load-invoice-ledger :requires [:auto-ap/make-entity-not-unique]} }] diff --git a/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj b/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj index 27ac953c..0c5f9fbd 100644 --- a/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj +++ b/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj @@ -5,6 +5,14 @@ [auto-ap.expense-accounts :as expense-accounts] [auto-ap.ledger :as ledger])) +(defn test-run [txs-set] + (println "processing " (count txs-set)) + (doseq [[i tx] (map vector (range) txs-set)] + @(d/transact (d/connect auto-ap.datomic/uri) + tx) + (when (= 0 (mod i 1000)) + (println "processed " i)))) + (def add-general-ledger @@ -143,10 +151,34 @@ :args [(d/db conn)]})) z (->> invoice-ids (mapv #(ledger/entity-change->ledger (d/db conn) [:invoice %])))] - (clojure.pprint/pprint z) z)) -#_(bulk-load-invoice-ledger (d/connect auto-ap.datomic/uri)) +#_(do @(d/transact (d/connect auto-ap.datomic/uri) + (bulk-load-invoice-ledger (d/connect auto-ap.datomic/uri))) + (println "test")) + +(defn bulk-load-transaction-ledger [conn] + (let [transaction-ids (map first (d/query {:query {:find '[?e] + :in '[$] + :where ['[?e :transaction/id]]} + :args [(d/db conn)]})) + _ (println (count transaction-ids)) + z (->> transaction-ids + (map #(ledger/entity-change->ledger (d/db conn) [:transaction %])) + (filter identity) + (mapv identity))] + z)) + +#_(test-run (bulk-load-transaction-ledger (d/connect auto-ap.datomic/uri))) + +#_(do (doseq [tran (convert-transactions (d/connect auto-ap.datomic/uri))] + @(d/transact (d/connect auto-ap.datomic/uri) tran)) + (println "done.")) + +#_(do @(d/transact (d/connect auto-ap.datomic/uri) + (bulk-load-transaction-ledger (d/connect auto-ap.datomic/uri))) + (println "test")) + (def change-expense-account-to-entity [[{:db/ident :invoice-expense-account/account @@ -184,4 +216,22 @@ :invoice-expense-account/account (accounts/get-account-by-numeric-code-and-sets expense-account-id ["default"])})) invoice-expense-accounts)])) +(defn convert-transactions [conn] + (let [matched-transactions (d/query {:query {:find '[?transaction ?v] + :in '[$] + :where ['[?transaction :transaction/payment ?payment] + '[?payment :payment/invoices ?i] + '[?i :invoice/vendor ?v]]} + :args [(d/db (d/connect auto-ap.datomic/uri))]})] + [(mapv + (fn [[transaction-id vendor-id]] + (remove-nils {:db/id transaction-id + :transaction/vendor vendor-id + :transaction/account (:db/id (accounts/get-account-by-numeric-code-and-sets 2110 ["default"])) + })) + matched-transactions)])) + +#_(do (doseq [tran (convert-transactions (d/connect auto-ap.datomic/uri))] + @(d/transact (d/connect auto-ap.datomic/uri) tran)) + (println "done.")) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 1db68a98..8d7b3e7b 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -45,9 +45,9 @@ :journal-entry/line-items (into [{:journal-entry-line/account (a/get-account-by-numeric-code-and-sets 2110 ["default"]) :journal-entry-line/credit (:invoice/total entity)}] (map (fn [ea] - {:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)) - :journal-entry-line/location (:invoice-expense-account/location ea) - :journal-entry-line/debit (:invoice-expense-account/amount ea)}) + (remove-nils {:journal-entry-line/account (:db/id (:invoice-expense-account/account ea)) + :journal-entry-line/location (or (:invoice-expense-account/location ea) "HQ") ;; TODO? + :journal-entry-line/debit (:invoice-expense-account/amount ea)})) (:invoice/expense-accounts entity))) :journal-entry/cleared (and (< (:invoice/outstanding-balance entity) 0.01) @@ -57,24 +57,25 @@ (defmethod entity-change->ledger :transaction [db [type id]] (let [entity (d/pull db ['* {:transaction/vendor '[*] :transaction/client '[*] :transaction/account '[*]}] id)] - (remove-nils - {:journal-entry/source "transaction" - :journal-entry/client (:db/id (:transaction/client entity)) - :journal-entry/date (doto (:transaction/date entity) println) - :journal-entry/original-entity (:db/id entity) - :journal-entry/vendor (:db/id (:transaction/vendor entity)) - :journal-entry/amount (Math/abs (:transaction/amount entity)) + (when (:transaction/vendor entity) + (remove-nils + {:journal-entry/source "transaction" + :journal-entry/client (:db/id (:transaction/client entity)) + :journal-entry/date (:transaction/date entity) + :journal-entry/original-entity (:db/id entity) + :journal-entry/vendor (:db/id (:transaction/vendor entity)) + :journal-entry/amount (Math/abs (:transaction/amount entity)) - :journal-entry/line-items [{:journal-entry-line/account (:db/id (:transaction/account entity)) - :journal-entry-line/location "HQ" - :journal-entry-line/debit (Math/abs (:transaction/amount entity))} + :journal-entry/line-items [(remove-nils{:journal-entry-line/account (:db/id (:transaction/account entity)) + :journal-entry-line/location "HQ" + :journal-entry-line/debit (Math/abs (:transaction/amount entity))}) - {:journal-entry-line/account (:db/id (:transaction/bank-account entity)) - :journal-entry-line/location "HQ" - :journal-entry-line/credit (Math/abs (:transaction/amount entity))} - ] - - :journal-entry/cleared true}))) + (remove-nils {:journal-entry-line/account (:db/id (:transaction/bank-account entity)) + :journal-entry-line/location "HQ" + :journal-entry-line/credit (Math/abs (:transaction/amount entity))}) + ] + + :journal-entry/cleared true})))) (defmethod entity-change->ledger :invoice-expense-account [db [entity changes]]