From d38f8d8020c36eadb813a7702fb1d49887d15c75 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 30 Dec 2021 06:47:47 -0800 Subject: [PATCH 1/3] maybe lowering pipelining will help. --- src/clj/auto_ap/yodlee/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/auto_ap/yodlee/core.clj b/src/clj/auto_ap/yodlee/core.clj index 15548635..dad7b35b 100644 --- a/src/clj/auto_ap/yodlee/core.clj +++ b/src/clj/auto_ap/yodlee/core.clj @@ -302,7 +302,7 @@ (defn get-provider-accounts-with-details [] (let [provider-accounts (get-provider-accounts)] - (let [concurrent 20 + (let [concurrent 10 output-chan (async/chan)] (async/pipeline-blocking concurrent output-chan From a982864810e9b0f122cfb35ee33bf8964c4ab2ac Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Mon, 3 Jan 2022 19:36:02 -0800 Subject: [PATCH 2/3] fix sysco. --- src/clj/auto_ap/background/sysco.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj/auto_ap/background/sysco.clj b/src/clj/auto_ap/background/sysco.clj index 83ea08d6..b6df974b 100644 --- a/src/clj/auto_ap/background/sysco.clj +++ b/src/clj/auto_ap/background/sysco.clj @@ -70,6 +70,7 @@ (and customer-identifier (parse/best-match clients customer-identifier))) total (Double/parseDouble (summary-row "TotalExtendedPrice")) + tax (Double/parseDouble (summary-row "TotalTaxAmount")) date (t/parse (header-row "InvoiceDate") "yyMMdd") @@ -79,7 +80,7 @@ (log/infof "Importing %s for %s" (header-row "InvoiceNumber") (header-row "CustomerName")) [:propose-invoice (cond-> #:invoice {:invoice-number (header-row "InvoiceNumber") - :total total + :total (+ total tax) :outstanding-balance total :date (coerce/to-date date) :vendor (:db/id sysco-vendor ) From 9915a4c41c3a3a745001f3ce7cb7c73fd2c12101 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 4 Jan 2022 06:48:12 -0800 Subject: [PATCH 3/3] data cleanup template. --- scratch-sessions/data-cleanup.clj | 135 ++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 scratch-sessions/data-cleanup.clj diff --git a/scratch-sessions/data-cleanup.clj b/scratch-sessions/data-cleanup.clj new file mode 100644 index 00000000..c7488a9b --- /dev/null +++ b/scratch-sessions/data-cleanup.clj @@ -0,0 +1,135 @@ +;; This buffer is for Clojure experiments and evaluation. + +;; Press C-j to evaluate the last expression. + +;; You can also press C-u C-j to evaluate the expression and pretty-print its result. + +(def c auto-ap.datomic/conn) + +(defn check-fn [query checker] + (fn [db] + (assert (checker (d/q query db)) + (str "query failed!" query)))) + +(defn check-transaction [fs tx] + + (println "checking before...") + (doseq [f fs] + (f (d/db c))) + + (println "checing after...") + (let [updated (:db-after (d/with (d/db c) + tx))] + (doseq [f fs] + (f updated))) + (println "success!")) + +(let [invoices-without-clients (check-fn '[:find ?i + :in $ + :where + [?i :invoice/invoice-number] + (not [?i :invoice/client ])] + #(= (count %) 0)) + transactions-without-clients (check-fn '[:find ?t + :in $ + :where + [?t :transaction/amount] + (not [?t :transaction/client ])] + #(= (count %) 0)) + account-overide-without-clients (check-fn '[:find ?t + :in $ + :where + [?t :account-client-override/name] + (not [?t :account-client-override/client ])] + #(= (count %) 0)) + vendor-schedule-payment-dom (check-fn '[:find ?t + :in $ + :where + [?t :vendor-schedule-payment-dom/dom] + (not [?t :vendor-schedule-payment-dom/client ])] + #(= (count %) 0)) + payments-without-clients (check-fn '[:find ?t + :in $ + :where + [?t :payment/date] + (not [?t :payment/client ])] + #(= (count %) 0)) + vendor-usage-without-client (check-fn '[:find ?t + :in $ + :where + [?t :vendor-usage/vendor] + (not [?t :vendor-usage/client ])] + #(= (count %) 0)) + journal-entries-without-client (check-fn '[:find ?t + :in $ + :where + [?t :journal-entry/date] + (not [?t :journal-entry/client ])] + #(= (count %) 0)) + userless-users (check-fn '[:find ?t + :in $ + :where + [?t :user/role :user-role/user] + (not [?t :user/clients ])] + #(= (count %) 0)) + invoice-payments-without-invoices (check-fn '[:find ?t + :in $ + :where + [?t :invoice-payment/amount] + (not [?t :invoice-payment/invoice ])] + #(= (count %) 0)) + + transaction + (->> [[:db/retractEntity [:client/code "CBC"]]] + (into (map (fn [[i]] + [:db/retractEntity i]) + (d/q '[:find ?i + :in $ + :where (or [?i :invoice/client [:client/code "CBC"]] + [?i :vendor-usage/client [:client/code "CBC"]] + [?i :transaction/client [:client/code "CBC"]] + [?i :payment/client [:client/code "CBC"]] + [?i :transaction-rule/client [:client/code "CBC"]] + [?i :journal-entry/client [:client/code "CBC"]] + [?i :user/clients [:client/code "CBC"]])] + (d/db c)))) + (into (map (fn [[i]] + [:db/retractEntity i]) + (d/q '[:find ?ip + :in $ + :where [?i :invoice/client [:client/code "CBC"]] + [?ip :invoice-payment/invoice ?i]] + (d/db c)))))] + + (check-transaction [invoices-without-clients transactions-without-clients + account-overide-without-clients vendor-schedule-payment-dom + payments-without-clients vendor-usage-without-client + journal-entries-without-client userless-users + invoice-payments-without-invoices] + transaction) + (auto-ap.datomic/audit-transact-batch transaction {:user/role "admin" + :user/name "Removing CBC"})) + +(clojure.pprint/pprint (get-idents)) +(get-schema "invoice-payment") + + + +(d/q '[:find ?n + :in $ + :where [_ ?a 17592186046483] + [?a :db/ident ?n]] + (d/db c)) + +(->> (d/q '[:find ?ip + :in $ + :where [?ip :invoice-payment/amount] + (not [?ip :invoice-payment/invoice]) + #_(not [?ip :invoice-payment/payment])] + (d/db c)) + (map (fn [[ip]] + [:db/retractEntity ip])) + (into [{:db/id "datomic.tx" :db/doc "Removing payments that were disconnected from invoices"}]) + (d/transact c) + deref + )