Merge branch 'master' into staging

scratch-sessions/data-cleanup.clj

@@ -0,0 +1,135 @@
+;; This buffer is for Clojure experiments and evaluation.
+
+;; Press C-j to evaluate the last expression.
+
+;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
+
+(def c auto-ap.datomic/conn)
+
+(defn check-fn [query checker]
+  (fn [db]
+    (assert (checker (d/q query db))
+            (str "query failed!" query))))
+
+(defn check-transaction [fs tx]
+
+  (println "checking before...")
+  (doseq [f fs]
+    (f (d/db c)))
+
+  (println "checing after...")
+  (let [updated (:db-after (d/with (d/db c)
+                                   tx))]
+    (doseq [f fs]
+      (f updated)))
+  (println "success!"))
+
+(let [invoices-without-clients (check-fn '[:find ?i
+                                           :in $
+                                           :where
+                                           [?i :invoice/invoice-number]
+                                           (not [?i :invoice/client ])]
+                                         #(= (count %) 0))
+      transactions-without-clients (check-fn '[:find ?t
+                                               :in $
+                                               :where
+                                               [?t :transaction/amount]
+                                               (not [?t :transaction/client ])]
+                                             #(= (count %) 0))
+      account-overide-without-clients (check-fn '[:find ?t
+                                                  :in $
+                                                  :where
+                                                  [?t :account-client-override/name]
+                                                  (not [?t :account-client-override/client ])]
+                                                #(= (count %) 0))
+      vendor-schedule-payment-dom (check-fn '[:find ?t
+                                                  :in $
+                                                  :where
+                                                  [?t :vendor-schedule-payment-dom/dom]
+                                                  (not [?t :vendor-schedule-payment-dom/client ])]
+                                            #(= (count %) 0))
+      payments-without-clients (check-fn '[:find ?t
+                                              :in $
+                                              :where
+                                              [?t :payment/date]
+                                              (not [?t :payment/client ])]
+                                         #(= (count %) 0))
+      vendor-usage-without-client (check-fn '[:find ?t
+                                           :in $
+                                           :where
+                                           [?t :vendor-usage/vendor]
+                                           (not [?t :vendor-usage/client ])]
+                                         #(= (count %) 0))
+      journal-entries-without-client (check-fn '[:find ?t
+                                                 :in $
+                                                 :where
+                                                 [?t :journal-entry/date]
+                                                 (not [?t :journal-entry/client ])]
+                                               #(= (count %) 0))
+      userless-users (check-fn '[:find ?t
+                                 :in $
+                                 :where
+                                 [?t :user/role :user-role/user]
+                                 (not [?t :user/clients ])]
+                               #(= (count %) 0))
+      invoice-payments-without-invoices (check-fn '[:find ?t
+                                                    :in $
+                                                    :where
+                                                    [?t :invoice-payment/amount]
+                                                    (not [?t :invoice-payment/invoice ])]
+                                                  #(= (count %) 0))
+
+      transaction
+      (->> [[:db/retractEntity [:client/code "CBC"]]]
+           (into (map (fn [[i]]
+                        [:db/retractEntity i])
+                      (d/q '[:find ?i
+                             :in $
+                             :where (or [?i :invoice/client [:client/code "CBC"]]
+                                        [?i :vendor-usage/client [:client/code "CBC"]]
+                                        [?i :transaction/client [:client/code "CBC"]]
+                                        [?i :payment/client [:client/code "CBC"]]
+                                        [?i :transaction-rule/client [:client/code "CBC"]]
+                                        [?i :journal-entry/client [:client/code "CBC"]]
+                                        [?i :user/clients [:client/code "CBC"]])]
+                           (d/db c))))
+           (into (map (fn [[i]]
+                        [:db/retractEntity i])
+                      (d/q '[:find ?ip
+                             :in $
+                             :where [?i :invoice/client [:client/code "CBC"]]
+                             [?ip :invoice-payment/invoice ?i]]
+                           (d/db c)))))]
+
+  (check-transaction [invoices-without-clients transactions-without-clients
+                      account-overide-without-clients vendor-schedule-payment-dom
+                      payments-without-clients vendor-usage-without-client
+                      journal-entries-without-client userless-users
+                      invoice-payments-without-invoices]
+                     transaction)
+  (auto-ap.datomic/audit-transact-batch transaction {:user/role "admin"
+                                     :user/name "Removing CBC"}))
+
+(clojure.pprint/pprint (get-idents))
+(get-schema "invoice-payment")
+
+
+
+(d/q '[:find ?n
+       :in $
+       :where [_ ?a 17592186046483]
+       [?a :db/ident ?n]]
+     (d/db c))
+
+(->> (d/q '[:find ?ip
+            :in $
+            :where [?ip :invoice-payment/amount]
+            (not [?ip :invoice-payment/invoice])
+            #_(not [?ip :invoice-payment/payment])]
+          (d/db c))
+     (map (fn [[ip]]
+            [:db/retractEntity ip]))
+     (into [{:db/id "datomic.tx" :db/doc "Removing payments that were disconnected from invoices"}])
+     (d/transact c)
+     deref
+     )

src/clj/auto_ap/background/sysco.clj

@@ -70,6 +70,7 @@
                             (and customer-identifier
                                  (parse/best-match clients customer-identifier)))
         total (Double/parseDouble (summary-row "TotalExtendedPrice"))
+        tax (Double/parseDouble (summary-row "TotalTaxAmount"))
         date (t/parse
               (header-row "InvoiceDate")
               "yyMMdd")
@@ -79,7 +80,7 @@
     (log/infof "Importing %s for %s" (header-row "InvoiceNumber") (header-row "CustomerName"))
 
     [:propose-invoice (cond-> #:invoice {:invoice-number (header-row "InvoiceNumber")
-                                         :total total
+                                         :total (+ total tax)
                                          :outstanding-balance total
                                          :date  (coerce/to-date date)
                                          :vendor (:db/id sysco-vendor )

src/clj/auto_ap/yodlee/core.clj

@@ -302,7 +302,7 @@
 
 (defn get-provider-accounts-with-details []
   (let [provider-accounts (get-provider-accounts)]
-    (let [concurrent 20
+    (let [concurrent 10
           output-chan (async/chan)]
       (async/pipeline-blocking concurrent
                                output-chan