diff --git a/src/clj/auto_ap/background/sysco.clj b/src/clj/auto_ap/background/sysco.clj index 82483909..ed5c98f2 100644 --- a/src/clj/auto_ap/background/sysco.clj +++ b/src/clj/auto_ap/background/sysco.clj @@ -18,7 +18,8 @@ [unilog.context :as lc] [clj-time.core :as time] [mount.core :as mount] - [yang.scheduler :as scheduler])) + [yang.scheduler :as scheduler] + [com.unbounce.dogstatsd.core :as statsd])) (def bucket-name (:data-bucket env)) @@ -124,6 +125,11 @@ :prefix "sysco/pending"}) :object-summaries (map :key))] + + (statsd/event {:title "Sysco import started" + :text (format "Found %d sysco invoice to import: %s" (count keys) (pr-str keys)) + :priority :low} + nil) (log/infof "Found %d sysco invoice to import: %s" (count keys) (pr-str keys)) (let [result @(d/transact conn (mapv (fn [k] @@ -134,7 +140,10 @@ keys))] (log/infof "Imported %d invoices" (/ (count (:tempids result)) 2))) (doseq [k keys] - (mark-key k))))) + (mark-key k)) + (statsd/event {:title "Sysco import ended" + :text "Sysco completed" + :priority :low} nil)))) (mount/defstate sysco-invoice-importer diff --git a/src/clj/auto_ap/import/intuit.clj b/src/clj/auto_ap/import/intuit.clj index 03cb9c05..d88e3f55 100644 --- a/src/clj/auto_ap/import/intuit.clj +++ b/src/clj/auto_ap/import/intuit.clj @@ -1,18 +1,16 @@ (ns auto-ap.import.intuit - (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [conn remove-nils]] - [auto-ap.intuit.core :as i] - [auto-ap.utils :refer [by allow-once]] - [auto-ap.import.transactions :as t] - [clj-time.coerce :as coerce] - [clj-time.core :as time] - [clj-time.format :as f] - [clojure.string :as str] - [clojure.tools.logging :as log] - [datomic.api :as d] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) + (:require + [auto-ap.datomic :refer [conn]] + [auto-ap.import.transactions :as t] + [auto-ap.intuit.core :as i] + [auto-ap.utils :refer [allow-once]] + [clj-time.coerce :as coerce] + [clj-time.core :as time] + [com.unbounce.dogstatsd.core :as statsd] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) (defn get-intuit-bank-accounts [db] (d/q '[:find ?external-id ?ba ?c @@ -39,6 +37,10 @@ (defn import-intuit [] (lc/with-context {:source "Import intuit transactions"} + (statsd/event {:title "Intuit import started" + :text "Starting" + :priority :low} + nil) (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user") db (d/db conn) end (auto-ap.time/local-now) @@ -51,8 +53,17 @@ (intuits->transactions bank-account-id client-id))] (t/import-transaction! import-batch transaction)) (t/finish! import-batch) + (statsd/event {:title "Intuit import Finished" + :text (pr-str (t/get-stats import-batch)) + :priority :low} + nil) (catch Exception e - (t/fail! import-batch e)))))) + (t/fail! import-batch e) + (statsd/event {:title "Intuit import failed" + :text (str e) + :alert-type :warning + :priority :normal} + nil)))))) (def upsert-transactions (allow-once upsert-transactions)) diff --git a/src/clj/auto_ap/import/yodlee.clj b/src/clj/auto_ap/import/yodlee.clj index c711f840..ad78f350 100644 --- a/src/clj/auto_ap/import/yodlee.clj +++ b/src/clj/auto_ap/import/yodlee.clj @@ -1,17 +1,19 @@ (ns auto-ap.import.yodlee - (:require [auto-ap.datomic :refer [conn]] - [auto-ap.import.transactions :as t] - [auto-ap.time :as atime] - [auto-ap.utils :refer [allow-once]] - [auto-ap.yodlee.core :as client] - [clj-time.coerce :as coerce] - [clojure.string :as str] - [datomic.api :as d] - [digest :refer [sha-256]] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler] - [clojure.tools.logging :as log])) + (:require + [auto-ap.datomic :refer [conn]] + [auto-ap.import.transactions :as t] + [auto-ap.time :as atime] + [auto-ap.utils :refer [allow-once]] + [auto-ap.yodlee.core :as client] + [clj-time.coerce :as coerce] + [clojure.string :as str] + [clojure.tools.logging :as log] + [com.unbounce.dogstatsd.core :as statsd] + [datomic.api :as d] + [digest :refer [sha-256]] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) (defn yodlee->transaction [transaction] (let [{post-date :postDate @@ -44,6 +46,10 @@ (defn import-yodlee [] (lc/with-context {:source "Import yodlee transactions"} + (statsd/event {:title "Yodlee import started" + :text "Starting" + :priority :low} + nil) (let [import-batch (t/start-import-batch :import-source/yodlee "Automated yodlee user")] (try (let [account-lookup (d/q '[:find ?ya ?ba ?c @@ -62,8 +68,17 @@ :transaction/client client-id))) (t/finish! import-batch)) + (statsd/event {:title "Yodlee import Finished" + :text (pr-str (t/get-stats import-batch)) + :priority :low} + nil) (catch Exception e - (t/fail! import-batch e)))))) + (t/fail! import-batch e) + (statsd/event {:title "Yodlee import failed" + :text (str e) + :alert-type :warning + :priority :normal} + nil)))))) (def import-yodlee (allow-once import-yodlee)) diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj index 5f1b9198..dd06c009 100644 --- a/src/clj/auto_ap/import/yodlee2.clj +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -1,16 +1,22 @@ (ns auto-ap.import.yodlee2 - (:require [auto-ap.datomic :refer [conn]] - [auto-ap.import.transactions :as t] - [auto-ap.import.yodlee :as y] - [auto-ap.utils :refer [allow-once]] - [auto-ap.yodlee.core2 :as client2] - [datomic.api :as d] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) + (:require + [auto-ap.datomic :refer [conn]] + [auto-ap.import.transactions :as t] + [auto-ap.import.yodlee :as y] + [auto-ap.utils :refer [allow-once]] + [auto-ap.yodlee.core2 :as client2] + [com.unbounce.dogstatsd.core :as statsd] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) (defn import-yodlee2 [] (lc/with-context {:source "Import yodlee2 transactions"} + (statsd/event {:title "Yodlee2 import started" + :text "Starting" + :priority :low} + nil) (let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")] (try (let [account-lookup (d/q '[:find ?ya ?ba ?cd @@ -28,8 +34,18 @@ :transaction/client [:client/code client-code]))) (t/finish! import-batch)) + (statsd/event {:title "Yodlee2 import Finished" + :text (pr-str (t/get-stats import-batch)) + :priority :low} + nil) + (catch Exception e - (t/fail! import-batch e)))))) + (t/fail! import-batch e) + (statsd/event {:title "Yodlee2 import failed" + :text (str e) + :alert-type :warning + :priority :normal} + nil)))))) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index dc6c3bd0..8cb0e1a5 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -1,15 +1,14 @@ (ns auto-ap.ledger - (:require [datomic.api :as d] - [yang.scheduler :as scheduler] - [mount.core :as mount] - [auto-ap.datomic.accounts :as a] - [auto-ap.datomic :refer [uri remove-nils conn]] - [clojure.spec.alpha :as s] - [auto-ap.utils :refer [dollars-0? dollars=] ] - [clojure.tools.logging :as log] - [auto-ap.logging :refer [info-event]] - [unilog.context :as lc])) - + (:require + [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.logging :refer [info-event]] + [auto-ap.utils :refer [dollars-0? dollars=]] + [clojure.tools.logging :as log] + [com.unbounce.dogstatsd.core :as statsd] + [datomic.api :as d] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) (defn datums->impacted-entity [db [e changes]] (let [entity (d/pull db '[* {:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) @@ -372,6 +371,10 @@ (defn touch-broken-ledger [] (lc/with-context {:source "touch-broken-ledger"} + (statsd/event {:title "Reconciling Ledger" + :text "This process looks for unbalance ledger entries, or missing ledger entries" + :priority :low} + nil) (try (log/info "Attempting to fix transactions that are in the ledger but are wrong") (let [mismatched-ts (mismatched-transactions)] @@ -403,6 +406,10 @@ (touch-invoice m))))) (log/info "Finished fixing invoices that are in the ledger but are wrong") + (statsd/event {:title "Finished Reconciling Ledger" + :text "This process looks for unbalance ledger entries, or missing ledger entries" + :priority :low} + nil) (catch Exception e (log/error e))))) diff --git a/sysco-poller/script.sh b/sysco-poller/script.sh index 7fbdaae1..0691664b 100755 --- a/sysco-poller/script.sh +++ b/sysco-poller/script.sh @@ -1,3 +1,3 @@ #!/bin/bash -lftp "sftp://NICKGREEK:Nic_$&^632@sb2b.sysco.com" -e "mirror Outbound /home; exit;" +lftp "sftp://NICKGREEK:Nic_$&^632@sb2b.sysco.com" -e "mirror --Remove-source-files Outbound /home; exit;" aws s3 sync /home/ s3://data.prod.app.integreatconsult.com/sysco/pending