Updates for events.

src/clj/auto_ap/background/sysco.clj

@@ -18,7 +18,8 @@
    [unilog.context :as lc]
    [clj-time.core :as time]
    [mount.core :as mount]
-   [yang.scheduler :as scheduler]))
+   [yang.scheduler :as scheduler]
+   [com.unbounce.dogstatsd.core :as statsd]))
 
 (def bucket-name (:data-bucket env))
 
@@ -124,6 +125,11 @@
                                          :prefix "sysco/pending"})
                     :object-summaries
                     (map :key))]
+
+      (statsd/event {:title "Sysco import started"
+                     :text (format "Found %d sysco invoice to import: %s" (count keys) (pr-str keys))
+                     :priority :low}
+                    nil)
       (log/infof "Found %d sysco invoice to import: %s" (count keys) (pr-str keys))
       
       (let [result @(d/transact conn (mapv (fn [k]
@@ -134,7 +140,10 @@
                                            keys))]
         (log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
       (doseq [k keys]
-        (mark-key k)))))
+        (mark-key k))
+      (statsd/event {:title "Sysco import ended"
+                     :text "Sysco completed"
+                     :priority :low} nil))))
 
 
 (mount/defstate sysco-invoice-importer

src/clj/auto_ap/import/intuit.clj

@@ -1,18 +1,16 @@
 (ns auto-ap.import.intuit
-  (:require [amazonica.aws.s3 :as s3]
-            [auto-ap.datomic :refer [conn remove-nils]]
-            [auto-ap.intuit.core :as i]
-            [auto-ap.utils :refer [by allow-once]]
-            [auto-ap.import.transactions :as t]
-            [clj-time.coerce :as coerce]
-            [clj-time.core :as time]
-            [clj-time.format :as f]
-            [clojure.string :as str]
-            [clojure.tools.logging :as log]
-            [datomic.api :as d]
-            [mount.core :as mount]
-            [unilog.context :as lc]
-            [yang.scheduler :as scheduler]))
+  (:require
+   [auto-ap.datomic :refer [conn]]
+   [auto-ap.import.transactions :as t]
+   [auto-ap.intuit.core :as i]
+   [auto-ap.utils :refer [allow-once]]
+   [clj-time.coerce :as coerce]
+   [clj-time.core :as time]
+   [com.unbounce.dogstatsd.core :as statsd]
+   [datomic.api :as d]
+   [mount.core :as mount]
+   [unilog.context :as lc]
+   [yang.scheduler :as scheduler]))
 
 (defn get-intuit-bank-accounts [db]
   (d/q '[:find ?external-id ?ba ?c
@@ -39,6 +37,10 @@
 
 (defn import-intuit []
   (lc/with-context {:source "Import intuit transactions"}
+    (statsd/event {:title "Intuit import started"
+                   :text "Starting"
+                   :priority :low}
+                  nil)
     (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user")
           db (d/db conn)
           end (auto-ap.time/local-now)
@@ -51,8 +53,17 @@
                                 (intuits->transactions bank-account-id client-id))]
           (t/import-transaction! import-batch transaction))
         (t/finish! import-batch)
+        (statsd/event {:title "Intuit import Finished"
+                       :text (pr-str (t/get-stats import-batch))
+                       :priority :low}
+                      nil)
         (catch Exception e
-          (t/fail! import-batch e))))))
+          (t/fail! import-batch e)
+          (statsd/event {:title "Intuit import failed"
+                         :text (str e)
+                         :alert-type :warning
+                         :priority :normal}
+                        nil))))))
 
 (def upsert-transactions (allow-once upsert-transactions))
 

src/clj/auto_ap/import/yodlee.clj

@@ -1,17 +1,19 @@
 (ns auto-ap.import.yodlee
-  (:require [auto-ap.datomic :refer [conn]]
-            [auto-ap.import.transactions :as t]
-            [auto-ap.time :as atime]
-            [auto-ap.utils :refer [allow-once]]
-            [auto-ap.yodlee.core :as client]
-            [clj-time.coerce :as coerce]
-            [clojure.string :as str]
-            [datomic.api :as d]
-            [digest :refer [sha-256]]
-            [mount.core :as mount]
-            [unilog.context :as lc]
-            [yang.scheduler :as scheduler]
-            [clojure.tools.logging :as log]))
+  (:require
+   [auto-ap.datomic :refer [conn]]
+   [auto-ap.import.transactions :as t]
+   [auto-ap.time :as atime]
+   [auto-ap.utils :refer [allow-once]]
+   [auto-ap.yodlee.core :as client]
+   [clj-time.coerce :as coerce]
+   [clojure.string :as str]
+   [clojure.tools.logging :as log]
+   [com.unbounce.dogstatsd.core :as statsd]
+   [datomic.api :as d]
+   [digest :refer [sha-256]]
+   [mount.core :as mount]
+   [unilog.context :as lc]
+   [yang.scheduler :as scheduler]))
 
 (defn yodlee->transaction [transaction]
   (let [{post-date :postDate
@@ -44,6 +46,10 @@
 
 (defn import-yodlee []
   (lc/with-context {:source "Import yodlee transactions"}
+    (statsd/event {:title "Yodlee import started"
+                   :text "Starting"
+                   :priority :low}
+                  nil)
     (let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")]
       (try
         (let [account-lookup (d/q '[:find ?ya ?ba ?c
@@ -62,8 +68,17 @@
                                                      :transaction/client client-id)))
 
           (t/finish! import-batch))
+        (statsd/event {:title "Yodlee import Finished"
+                       :text (pr-str (t/get-stats import-batch))
+                       :priority :low}
+                      nil)
         (catch Exception e
-          (t/fail! import-batch e))))))
+          (t/fail! import-batch e)
+          (statsd/event {:title "Yodlee import failed"
+                         :text (str e)
+                         :alert-type :warning
+                         :priority :normal}
+                        nil))))))
 
 (def import-yodlee (allow-once import-yodlee))
 

src/clj/auto_ap/import/yodlee2.clj

@@ -1,16 +1,22 @@
 (ns auto-ap.import.yodlee2
-  (:require [auto-ap.datomic :refer [conn]]
-            [auto-ap.import.transactions :as t]
-            [auto-ap.import.yodlee :as y]
-            [auto-ap.utils :refer [allow-once]]
-            [auto-ap.yodlee.core2 :as client2]
-            [datomic.api :as d]
-            [mount.core :as mount]
-            [unilog.context :as lc]
-            [yang.scheduler :as scheduler]))
+  (:require
+   [auto-ap.datomic :refer [conn]]
+   [auto-ap.import.transactions :as t]
+   [auto-ap.import.yodlee :as y]
+   [auto-ap.utils :refer [allow-once]]
+   [auto-ap.yodlee.core2 :as client2]
+   [com.unbounce.dogstatsd.core :as statsd]
+   [datomic.api :as d]
+   [mount.core :as mount]
+   [unilog.context :as lc]
+   [yang.scheduler :as scheduler]))
 
 (defn import-yodlee2 []
   (lc/with-context {:source "Import yodlee2 transactions"}
+    (statsd/event {:title "Yodlee2 import started"
+                   :text "Starting"
+                   :priority :low}
+                  nil)
     (let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")]
       (try
         (let [account-lookup (d/q '[:find ?ya ?ba ?cd
@@ -28,8 +34,18 @@
                                                        :transaction/client [:client/code client-code])))
 
           (t/finish! import-batch))
+        (statsd/event {:title "Yodlee2 import Finished"
+                       :text (pr-str (t/get-stats import-batch))
+                       :priority :low}
+                      nil)
+
         (catch Exception e
-          (t/fail! import-batch e))))))
+          (t/fail! import-batch e)
+          (statsd/event {:title "Yodlee2 import failed"
+                         :text (str e)
+                         :alert-type :warning
+                         :priority :normal}
+                        nil))))))
 
 
 

src/clj/auto_ap/ledger.clj

@@ -1,15 +1,14 @@
 (ns auto-ap.ledger
-  (:require [datomic.api :as d]
-            [yang.scheduler :as scheduler]
-            [mount.core :as mount]
-            [auto-ap.datomic.accounts :as a]
-            [auto-ap.datomic :refer [uri remove-nils conn]]
-            [clojure.spec.alpha :as s]
-            [auto-ap.utils :refer [dollars-0? dollars=] ]
-            [clojure.tools.logging :as log]
-            [auto-ap.logging :refer [info-event]]
-            [unilog.context :as lc]))
-
+  (:require
+   [auto-ap.datomic :refer [conn remove-nils]]
+   [auto-ap.logging :refer [info-event]]
+   [auto-ap.utils :refer [dollars-0? dollars=]]
+   [clojure.tools.logging :as log]
+   [com.unbounce.dogstatsd.core :as statsd]
+   [datomic.api :as d]
+   [mount.core :as mount]
+   [unilog.context :as lc]
+   [yang.scheduler :as scheduler]))
 
 (defn datums->impacted-entity [db [e changes]]
   (let [entity (d/pull db '[* {:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e)
@@ -372,6 +371,10 @@
 
 (defn touch-broken-ledger []
   (lc/with-context {:source "touch-broken-ledger"}
+    (statsd/event {:title "Reconciling Ledger"
+                   :text "This process looks for unbalance ledger entries, or missing ledger entries"
+                   :priority :low}
+                  nil)
     (try 
       (log/info "Attempting to fix transactions that are in the ledger but are wrong")
       (let [mismatched-ts (mismatched-transactions)]
@@ -403,6 +406,10 @@
               (touch-invoice m)))))
 
       (log/info "Finished fixing invoices that are in the ledger but are wrong")
+      (statsd/event {:title "Finished Reconciling Ledger"
+                     :text "This process looks for unbalance ledger entries, or missing ledger entries"
+                     :priority :low}
+                    nil)
       (catch Exception e
         (log/error e)))))
 

sysco-poller/script.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-lftp "sftp://NICKGREEK:Nic_$&^632@sb2b.sysco.com" -e "mirror Outbound /home; exit;"
+lftp "sftp://NICKGREEK:Nic_$&^632@sb2b.sysco.com" -e "mirror --Remove-source-files Outbound /home; exit;"
 aws s3 sync /home/ s3://data.prod.app.integreatconsult.com/sysco/pending