much more background process tracking

src/clj/auto_ap/import/intuit.clj

@@ -2,18 +2,18 @@
   (:require
    [auto-ap.datomic :refer [conn]]
    [auto-ap.import.transactions :as t]
-   [auto-ap.time :as atime]
    [auto-ap.intuit.core :as i]
-   [auto-ap.utils :refer [allow-once]]
+   [auto-ap.time :as atime]
+   [auto-ap.utils :refer [allow-once heartbeat]]
    [clj-time.coerce :as coerce]
    [clj-time.core :as time]
+   [clojure.string :as str]
+   [clojure.tools.logging :as log]
    [com.unbounce.dogstatsd.core :as statsd]
    [datomic.api :as d]
    [mount.core :as mount]
    [unilog.context :as lc]
-   [yang.scheduler :as scheduler]
-   [clojure.string :as str]
-   [clojure.tools.logging :as log]))
+   [yang.scheduler :as scheduler]))
 
 (defn get-intuit-bank-accounts [db]
   (d/q '[:find ?external-id ?ba ?c
@@ -47,34 +47,33 @@
        (t/apply-synthetic-ids)))
 
 (defn import-intuit []
-  (lc/with-context {:source "Import intuit transactions"}
-    (statsd/event {:title "Intuit import started"
-                   :text "Starting"
-                   :priority :low}
-                  nil)
-    (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
-          db (d/db conn)
-          end (auto-ap.time/local-now)
-          start (time/plus end (time/days -30))]
-      (try
-        (doseq [[external-id bank-account-id client-id] (get-intuit-bank-accounts db)
-                transaction (-> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
-                                                    (auto-ap.time/unparse end auto-ap.time/iso-date)
-                                                    external-id)
-                                (intuits->transactions bank-account-id client-id))]
-          (t/import-transaction! import-batch transaction))
-        (t/finish! import-batch)
-        (statsd/event {:title "Intuit import Finished"
-                       :text (pr-str (t/get-stats import-batch))
-                       :priority :low}
-                      nil)
-        (catch Exception e
-          (t/fail! import-batch e)
-          (statsd/event {:title "Intuit import failed"
-                         :text (str e)
-                         :alert-type :warning
-                         :priority :normal}
-                        nil))))))
+  (statsd/event {:title    "Intuit import started"
+                 :text     "Starting"
+                 :priority :low}
+                nil)
+  (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
+        db           (d/db conn)
+        end          (auto-ap.time/local-now)
+        start        (time/plus end (time/days -30))]
+    (try
+      (doseq [[external-id bank-account-id client-id] (get-intuit-bank-accounts db)
+              transaction                             (-> (i/get-transactions (auto-ap.time/unparse start auto-ap.time/iso-date)
+                                                  (auto-ap.time/unparse end auto-ap.time/iso-date)
+                                                  external-id)
+                              (intuits->transactions bank-account-id client-id))]
+        (t/import-transaction! import-batch transaction))
+      (t/finish! import-batch)
+      (statsd/event {:title    "Intuit import Finished"
+                     :text     (pr-str (t/get-stats import-batch))
+                     :priority :low}
+                    nil)
+      (catch Exception e
+        (t/fail! import-batch e)
+        (statsd/event {:title      "Intuit import failed"
+                       :text       (str e)
+                       :alert-type :warning
+                       :priority   :normal}
+                      nil)))))
 
 (def upsert-transactions (allow-once upsert-transactions))
 
@@ -88,7 +87,7 @@
                       bank-accounts))))
 
 (mount/defstate import-worker
-  :start (scheduler/every (* 1000 60 60 24) import-intuit)
+  :start (scheduler/every (* 1000 60 60 24) (heartbeat import-intuit "import-intuit"))
   :stop (scheduler/stop import-worker))
 
 (mount/defstate account-worker

src/clj/auto_ap/import/plaid.clj

@@ -1,17 +1,15 @@
 (ns auto-ap.import.plaid
   (:require
    [auto-ap.datomic :refer [conn]]
-   [auto-ap.plaid.core :as p]
-   [auto-ap.utils :refer [allow-once by]]
    [auto-ap.import.transactions :as t]
+   [auto-ap.plaid.core :as p]
+   [auto-ap.utils :refer [allow-once by heartbeat]]
+   [clj-time.coerce :as coerce]
    [clj-time.core :as time]
-   [clojure.tools.logging :as log]
    [datomic.api :as d]
    [mount.core :as mount]
    [unilog.context :as lc]
-   [yang.scheduler :as scheduler]
-   [clj-time.coerce :as coerce]))
-
+   [yang.scheduler :as scheduler]))
 
 (defn get-plaid-accounts [db]
   (-> (d/q '[:find ?ba ?c ?external-id ?t 
@@ -59,7 +57,7 @@
 (def import-plaid (allow-once import-plaid))
 
 (mount/defstate import-worker
-  :start (scheduler/every (* 1000 60 60 3) import-plaid)
+  :start (scheduler/every (* 1000 60 60 3) (heartbeat import-plaid "import-plaid"))
   :stop (scheduler/stop import-worker))
 
 

src/clj/auto_ap/import/yodlee.clj

@@ -3,7 +3,7 @@
    [auto-ap.datomic :refer [conn]]
    [auto-ap.import.transactions :as t]
    [auto-ap.time :as atime]
-   [auto-ap.utils :refer [allow-once]]
+   [auto-ap.utils :refer [allow-once heartbeat]]
    [auto-ap.yodlee.core :as client]
    [clj-time.coerce :as coerce]
    [clojure.string :as str]
@@ -45,43 +45,42 @@
      :status status}))
 
 (defn import-yodlee []
-  (lc/with-context {:source "Import yodlee transactions"}
-    (statsd/event {:title "Yodlee import started"
-                   :text "Starting"
-                   :priority :low}
-                  nil)
-    (let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")]
-      (try
-        (let [account-lookup (d/q '[:find ?ya ?ba ?c
-                                    :in $
-                                    :where [?ba :bank-account/yodlee-account-id ?ya]
-                                    [?c :client/bank-accounts ?ba]]
-                                  (d/db conn))]
-          (doseq [[yodlee-account bank-account client-id] account-lookup
-                  transaction (try
-                                (client/get-specific-transactions yodlee-account (client/get-auth-header))
-                                (catch Exception e
-                                  (log/warn e)
-                                  []))]
-            (t/import-transaction! import-batch (assoc (yodlee->transaction transaction)
+  (statsd/event {:title    "Yodlee import started"
+                 :text     "Starting"
+                 :priority :low}
+                nil)
+  (let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")]
+    (try
+      (let [account-lookup (d/q '[:find ?ya ?ba ?c
+                                  :in $
+                                  :where [?ba :bank-account/yodlee-account-id ?ya]
+                                  [?c :client/bank-accounts ?ba]]
+                                (d/db conn))]
+        (doseq [[yodlee-account bank-account client-id] account-lookup
+                transaction                             (try
+                              (client/get-specific-transactions yodlee-account (client/get-auth-header))
+                              (catch Exception e
+                                (log/warn e)
+                                []))]
+          (t/import-transaction! import-batch (assoc (yodlee->transaction transaction)
                                                      :transaction/bank-account bank-account
                                                      :transaction/client client-id)))
 
-          (t/finish! import-batch))
-        (statsd/event {:title "Yodlee import Finished"
-                       :text (pr-str (t/get-stats import-batch))
-                       :priority :low}
-                      nil)
-        (catch Exception e
-          (t/fail! import-batch e)
-          (statsd/event {:title "Yodlee import failed"
-                         :text (str e)
-                         :alert-type :warning
-                         :priority :normal}
-                        nil))))))
+        (t/finish! import-batch))
+      (statsd/event {:title    "Yodlee import Finished"
+                     :text     (pr-str (t/get-stats import-batch))
+                     :priority :low}
+                    nil)
+      (catch Exception e
+        (t/fail! import-batch e)
+        (statsd/event {:title      "Yodlee import failed"
+                       :text       (str e)
+                       :alert-type :warning
+                       :priority   :normal}
+                      nil)))))
 
 (def import-yodlee (allow-once import-yodlee))
 
 (mount/defstate import-worker
-  :start (scheduler/every (* 1000 60 60 8) import-yodlee)
+  :start (scheduler/every (* 1000 60 60 8) (heartbeat import-yodlee "import-yodlee"))
   :stop (scheduler/stop import-worker))

src/clj/auto_ap/import/yodlee2.clj

@@ -3,7 +3,7 @@
    [auto-ap.datomic :refer [conn]]
    [auto-ap.import.transactions :as t]
    [auto-ap.import.yodlee :as y]
-   [auto-ap.utils :refer [allow-once]]
+   [auto-ap.utils :refer [allow-once heartbeat]]
    [auto-ap.yodlee.core2 :as client2]
    [com.unbounce.dogstatsd.core :as statsd]
    [datomic.api :as d]
@@ -53,9 +53,9 @@
 
 
 (mount/defstate import-worker
-  :start (scheduler/every (* 1000 60 60 4) import-yodlee2)
+  :start (scheduler/every (* 1000 60 60 4) (heartbeat import-yodlee2 "import-yodlee"))
   :stop (scheduler/stop import-worker))
 
 (mount/defstate account-worker
-  :start (scheduler/every (* 5 60 1000) client2/upsert-accounts)
+  :start (scheduler/every (* 5 60 1000) (heartbeat client2/upsert-accounts "upsert-yodlee2-accounts"))
   :stop (scheduler/stop account-worker))