From e35bc97a7da673bfbf69068cb4e2418b924ed680 Mon Sep 17 00:00:00 2001 From: BC Date: Thu, 14 Mar 2019 23:23:58 -0700 Subject: [PATCH] updating general ledger automatically. --- src/clj/auto_ap/datomic.clj | 21 ++--- src/clj/auto_ap/datomic/ledger.clj | 12 +++ src/clj/auto_ap/datomic/migrate.clj | 8 ++ .../datomic/migrate/add_general_ledger.clj | 77 +++++++++++++++++++ src/clj/auto_ap/ledger.clj | 76 +++++++++++------- src/clj/auto_ap/routes/exports.clj | 5 ++ src/clj/auto_ap/server.clj | 2 + 7 files changed, 160 insertions(+), 41 deletions(-) create mode 100644 src/clj/auto_ap/datomic/ledger.clj create mode 100644 src/clj/auto_ap/datomic/migrate/add_general_ledger.clj diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index e7d83f66..fe84be10 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -722,33 +722,28 @@ (d/db (d/connect uri)))) (defn migrate-vendors [conn] - [(let [all-vendors (v/get-all)] - (load-vendors all-vendors))]) + [[]]) (defn migrate-clients [conn] - (let [all-clients (c/get-all)] - [(load-clients all-clients)])) + [[]]) (defn migrate-invoices [conn] - (load-invoices (i/get-all))) + [[]]) (defn migrate-payments [conn] - (load-payments (checks/get-all))) + [[]]) (defn migrate-invoices-payments [conn] - (load-invoices-payments (ic/get-all))) + [[]]) (defn migrate-invoices-expense-accounts [conn] - (load-invoices-expense-accounts (iea/get-all))) + [[]]) (defn migrate-transactions [conn] - (let [trans (load-transactions (transactions/get-all))] - (if (seq trans) - trans - [[]]))) + [[]]) (defn migrate-users [conn] - [(load-users (users/get-all))]) + [[]]) (defn merge-query [query-part-1 query-part-2] (-> query-part-1 diff --git a/src/clj/auto_ap/datomic/ledger.clj b/src/clj/auto_ap/datomic/ledger.clj new file mode 100644 index 00000000..0cf20a05 --- /dev/null +++ b/src/clj/auto_ap/datomic/ledger.clj @@ -0,0 +1,12 @@ +(ns auto-ap.datomic.ledger + (:require [datomic.api :as d] + [auto-ap.datomic :refer [uri]])) + +(defn get-graphql [args] + (->> (d/q '[:find (pull ?e [* {:general-ledger/client [:client/name :client/code :db/id] + :general-ledger/vendor [:vendor/name :db/id] + }]) + :where [?e :general-ledger/original-entity]] + (d/db (d/connect uri))) + (map first))) + diff --git a/src/clj/auto_ap/datomic/migrate.clj b/src/clj/auto_ap/datomic/migrate.clj index fb0cfe97..7235faa7 100644 --- a/src/clj/auto_ap/datomic/migrate.clj +++ b/src/clj/auto_ap/datomic/migrate.clj @@ -4,6 +4,7 @@ [auto-ap.datomic.migrate.add-client-codes :refer [add-client-codes]] [auto-ap.datomic.migrate.add-bank-account-codes :refer [add-bank-account-codes]] [auto-ap.datomic.migrate.invoice-converter :refer [add-import-status-existing-invoices]] + [auto-ap.datomic.migrate.add-general-ledger :as add-general-ledger] [clojure.java.io :as io] [io.rkn.conformity :as c]) (:import [datomic Util]) @@ -65,6 +66,13 @@ :auto-ap/add-new-vendors {:txes-fn 'auto-ap.datomic.migrate.add-new-vendors/add-new-vendors :requires [:auto-ap/fix-check-numbers]} :auto-ap/add-account-visibility-fields {:txes-fn 'auto-ap.datomic.migrate.account-sorting/add-account-visibility-fields :requires [:auto-ap/add-new-vendors]} :auto-ap/make-every-account-visible {:txes-fn 'auto-ap.datomic.migrate.account-sorting/make-every-account-visible :requires [:auto-ap/add-account-visibility-fields]} + :auto-ap/add-general-ledger3 {:txes add-general-ledger/add-general-ledger :requires [:auto-ap/make-every-account-visible]} + :auto-ap/add-general-ledger-fns {:txes-fn 'auto-ap.datomic.migrate.add-general-ledger/add-general-ledger-fns :requires [:auto-ap/add-general-ledger3]} + + + :auto-ap/make-entity-not-unique {:txes auto-ap.datomic.migrate.add-general-ledger/make-entity-not-unique :requires [:auto-ap/add-general-ledger-fns]} + :auto-ap/bulk-load-invoice-ledger2 {:txes-fn 'auto-ap.datomic.migrate.add-general-ledger/bulk-load-invoice-ledger :requires [:auto-ap/make-entity-not-unique]} + }] (println "Conforming database...") diff --git a/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj b/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj new file mode 100644 index 00000000..3f29a0ab --- /dev/null +++ b/src/clj/auto_ap/datomic/migrate/add_general_ledger.clj @@ -0,0 +1,77 @@ +(ns auto-ap.datomic.migrate.add-general-ledger + (:require [datomic.api :as d] + [auto-ap.ledger :as ledger])) + +(def add-general-ledger + [[{:db/ident :general-ledger/source + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The type of entity that created this entry"} + {:db/ident :general-ledger/client + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The client for the ledger"} + {:db/ident :general-ledger/date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/doc "The time for this entry"} + {:db/ident :general-ledger/original-entity + :db/valueType :db.type/ref + :db/unique :db.unique/identity + :db/cardinality :db.cardinality/one + :db/doc "The thing that created this entry"} + {:db/ident :general-ledger/vendor + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The vendor for the ledger entry"} + {:db/ident :general-ledger/amount + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The amount for the entry"} + {:db/ident :general-ledger/location + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Location of the entry"} + {:db/ident :general-ledger/from-expense-account + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The expense account from"} + {:db/ident :general-ledger/to-expense-account + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The expense account to"} + {:db/ident :general-ledger/cleared + :db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one + :db/doc "Has this cleared?"}]]) + +(defn add-general-ledger-fns [conn] + [[{:db/ident :replace-general-ledger + :db/doc "Deletes the general ledger entries for an entity" + :db/fn (d/function '{:lang "clojure" + :params [db origin] + :code (let [ids (->> (d/query {:query {:find ['?e] + :in ['$ '?origin] + :where [['?e :general-ledger/original-entity '?origin]]} + :args [db origin]}) + (map first))] + (into [] + (map (fn [i] [:db/retractEntity i ]) ids)))})}]] ) + +(defn bulk-load-invoice-ledger [conn] + (let [invoice-ids (map first (d/query {:query {:find '[?e] + :in '[$] + :where ['[?e :invoice/total]]} + :args [(d/db conn)]})) + z (->> invoice-ids + (mapv #(ledger/entity-change->ledger (d/db conn) [:invoice %])))] + (clojure.pprint/pprint z) + z)) + +#_(bulk-load-invoice-ledger (d/connect auto-ap.datomic/uri)) +(def make-entity-not-unique + [[[:db/retract :general-ledger/original-entity :db/unique]]]) + + + + diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 5194de5a..d72157b9 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -1,6 +1,6 @@ (ns auto-ap.ledger (:require [datomic.api :as d] - [auto-ap.datomic :refer [uri]])) + [auto-ap.datomic :refer [uri remove-nils]])) (defonce report-queue (d/tx-report-queue (d/connect uri) ) ) @@ -30,23 +30,23 @@ (defmethod entity-change->ledger :invoice [db [type id]] - (println "HI") (let [entity (d/pull db ['* {:invoice/vendor '[*] :invoice/payment '[*]}] id)] (map (fn [ea] - {:general-ledger/source "invoice" - :general-ledger/client (:invoice/client entity) - :general-ledger/date (:invoice/date entity) - :general-ledger/original-entity (:db/id entity) - :general-ledger/vendor (:db/id (:invoice/vendor entity)) - :general-ledger/amount (- (:invoice-expense-account/amount ea)) - :general-ledger/location (:invoice-expense-account/location ea) - :general-ledger/from-expense-account 2110 - :general-ledger/to-expense-account (:invoice-expense-account/expense-account-id ea) - :general-ledger/cleared (and (< (:invoice/outstanding-balance entity) 0.01) - (every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity)) - )}) + (remove-nils + {:general-ledger/source "invoice" + :general-ledger/client (:db/id (:invoice/client entity)) + :general-ledger/date (:invoice/date entity) + :general-ledger/original-entity (:db/id entity) + :general-ledger/vendor (:db/id (:invoice/vendor entity)) + :general-ledger/amount (- (:invoice-expense-account/amount ea)) + :general-ledger/location (:invoice-expense-account/location ea) + :general-ledger/from-expense-account 2110 + :general-ledger/to-expense-account (:invoice-expense-account/expense-account-id ea) + :general-ledger/cleared (and (< (:invoice/outstanding-balance entity) 0.01) + (every? #(= :payment-status/cleared (:payment/status %)) (:invoice/payments entity)) + )})) (:invoice/expense-accounts entity)))) (defmethod entity-change->ledger :invoice-expense-account @@ -65,19 +65,39 @@ (into [[:replace-general-ledger (:general-ledger/original-entity (first entries))]] entries)) -(let [transaction (.peek report-queue) - db (:db-after transaction)] - (->> (:tx-data transaction) - (map (fn [^datomic.db.Datum x] - {:e (:e x) - :a (d/ident db (:a x)) - :v (:v x) - :added (:added x)})) - (group-by :e) - (mapcat #(datums->impacted-entity db %)) - (set) - (map #(entity-change->ledger db %)) - (filter seq) - (map ledger-entries->transaction))) +(defn process-one [] + (let [transaction (.take report-queue) + db (:db-after transaction) + affected-entities (->> (:tx-data transaction) + (map (fn [^datomic.db.Datum x] + {:e (:e x) + :a (d/ident db (:a x)) + :v (:v x) + :added (:added x)})) + (group-by :e) + (mapcat #(datums->impacted-entity db %)) + (set)) + d-txs (->> affected-entities + (map #(entity-change->ledger db %)) + (filter seq)) + retractions (map (fn [[_ e]] [:replace-general-ledger e]) affected-entities)] + + @(d/transact (d/connect uri) retractions) + + (doseq [d-tx d-txs] + (println "updating general-ledger " d-tx) + @(d/transact (d/connect uri) d-tx)))) + +(defn process-all [] + (while (not (Thread/interrupted)) + (process-one))) + + + +#_(process-one) + +#_(process-all) + + diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index fde51090..beb98e65 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -2,6 +2,7 @@ (:require [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.vendors :as d-vendors] + [auto-ap.datomic.ledger :as d-ledger] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.utils :refer [by]] [auto-ap.parse :as parse] @@ -53,6 +54,10 @@ (GET "/vendors/export" {:keys [query-params identity]} (assert-admin identity) (map <-graphql (d-vendors/get-graphql {}))) + (GET "/ledger/export" {:keys [query-params identity]} + (assert-admin identity) + (map <-graphql (d-ledger/get-graphql {}))) + (GET "/transactions/export" {:keys [query-params identity]} (assert-admin identity) (let [[transactions] (d-transactions/get-graphql {:client-code (query-params "client-code") diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 3b6ccf2c..77144668 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -1,6 +1,7 @@ (ns auto-ap.server (:require #_[auto-ap.background.mail :refer [always-process-sqs]] [auto-ap.handler :refer [app]] + [auto-ap.ledger :refer [process-all]] [clojure.tools.nrepl.server :refer [start-server stop-server]] [config.core :refer [env]] [ring.adapter.jetty :refer [run-jetty]]) @@ -13,5 +14,6 @@ (defn -main [& args] (start-server :port 9000 :bind "0.0.0.0" :handler (cider-nrepl-handler)) (let [port (Integer/parseInt (or (env :port) "3000"))] + (future (process-all)) #_(future (always-process-sqs)) (run-jetty app {:port port :join? false})))