notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More background task cleanup

Browse Source
This commit is contained in:
Bryce Covert
2022-09-23 12:09:39 -07:00
parent e201a8b03e
commit ed759bac31
19 changed files with 102 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/clj/auto_ap/graphql/clients.clj
View File

@@ -7,16 +7,14 @@
:refer [->graphql assert-admin can-see-client? is-admin?]]
[auto-ap.routes.queries :as q]
[auto-ap.square.core :as square]
[auto-ap.utils :refer [heartbeat]]
[clj-time.coerce :as coerce]
[clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.walmartlabs.lacinia.util :refer [attach-resolvers]]
[datomic.api :as d]
[mount.core :as mount]
[unilog.context :as lc]
[clojure.set :as set])
[unilog.context :as lc])
(:import
(java.util UUID)
(org.apache.commons.codec.binary Base64)))

5
src/clj/auto_ap/import/intuit.clj
View File

@@ -5,15 +5,12 @@
[auto-ap.import.transactions :as t]
[auto-ap.intuit.core :as i]
[auto-ap.time :as atime]
[auto-ap.utils :refer [heartbeat]]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[clojure.string :as str]
[clojure.tools.logging :as log]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
[datomic.api :as d]))
(defn get-intuit-bank-accounts [db]
(d/q '[:find ?external-id ?ba ?c

6
src/clj/auto_ap/import/plaid.clj
View File

@@ -5,14 +5,12 @@
[auto-ap.import.transactions :as t]
[auto-ap.plaid.core :as p]
[auto-ap.time :as atime]
[auto-ap.utils :refer [allow-once by heartbeat]]
[auto-ap.utils :refer [allow-once by]]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
[datomic.api :as d]
[digest :as di]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
[unilog.context :as lc]))
(defn get-plaid-accounts [db]
(-> (d/q '[:find ?ba ?c ?external-id ?t

12
src/clj/auto_ap/import/yodlee2.clj
View File

@@ -3,17 +3,15 @@
[auto-ap.datomic :refer [conn]]
[auto-ap.import.common :refer [wrap-integration]]
[auto-ap.import.transactions :as t]
[auto-ap.utils :refer [allow-once heartbeat]]
[auto-ap.time :as atime]
[auto-ap.utils :refer [allow-once]]
[auto-ap.yodlee.core2 :as client2]
[auto-ap.time :as atime ]
[clojure.string :as str]
[clj-time.coerce :as coerce]
[digest :as di]
[clojure.string :as str]
[com.unbounce.dogstatsd.core :as statsd]
[datomic.api :as d]
[mount.core :as mount]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
[digest :as di]
[unilog.context :as lc]))
#_{:clj-kondo/ignore [:unresolved-var]}
(defn yodlee->transaction [transaction use-date-instead-of-post-date?]

15
src/clj/auto_ap/background/invoices.clj → src/clj/auto_ap/jobs/close_auto_invoices.clj
View File

@@ -1,13 +1,12 @@
(ns auto-ap.background.invoices
(ns auto-ap.jobs.close-auto-invoices
(:gen-class)
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.time :as time]
[auto-ap.utils :refer [heartbeat]]
[clj-time.coerce :as coerce]
[clojure.tools.logging :as log]
[datomic.api :as d]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
[datomic.api :as d]))
(defn close-auto-invoices []
(let [invoices-to-close (d/query {:query {:find ['?e]
@@ -27,6 +26,6 @@
deref)
(log/info "Closed " (count invoices-to-close) "scheduled invoices")))
(mount/defstate close-auto-invoices-worker
:start (scheduler/every 60000 (heartbeat close-auto-invoices "close-auto-invoices"))
:stop (scheduler/stop close-auto-invoices-worker))
(defn -main [& _]
(execute "close-auto-invoices" close-auto-invoices))

18
src/clj/auto_ap/jobs/core.clj Normal file
View File

@@ -0,0 +1,18 @@
(ns auto-ap.jobs.core
(:require [auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]))
(defn execute [name f]
(try
(lc/with-context {:background-job name}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat f name))
(mount/stop)
(log/info "Stopping " name)
(Thread/sleep 15000))
(finally
(System/exit 0))))

22
src/clj/auto_ap/jobs/current_balance_cache.clj
View File

@@ -1,23 +1,11 @@
(ns auto-ap.jobs.current-balance-cache
(:gen-class)
(:require [auto-ap.graphql.clients :as clients]
[auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]))
(:require
[auto-ap.graphql.clients :as clients]
[auto-ap.jobs.core :refer [execute]]))
(defn -main [& _]
(try
(lc/with-context {:background-job "current-balance-cache"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat clients/refresh-current-balance "current-balance-cache"))
(mount/stop )
(log/info "Stopping current-balance-cache")
(Thread/sleep 15000)
(System/exit 0))
(finally
(System/exit 0))))
(execute "current-balance-cache" clients/refresh-current-balance))

13
src/clj/auto_ap/background/mail.clj → src/clj/auto_ap/jobs/import_uploaded_invoices.clj
View File

@@ -1,19 +1,17 @@
(ns auto-ap.background.mail
(ns auto-ap.jobs.import-uploaded-invoices
(:require
[amazonica.aws.s3 :as s3]
[amazonica.aws.simpleemail :as ses]
[amazonica.aws.sqs :as sqs]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.parse :as parse]
[auto-ap.routes.invoices :as invoices]
[auto-ap.utils :refer [heartbeat]]
[clojure-mail.message :as message]
[clojure.data.json :as json]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.tools.logging :as log]
[config.core :refer [env]]
[mount.core :as mount]
[yang.scheduler :as scheduler])
[config.core :refer [env]])
(:import
(java.util Properties UUID)
(javax.mail Session)
@@ -78,6 +76,5 @@
(io/delete-file filename)))))))
(sqs/delete-message (assoc message :queue-url (:invoice-import-queue-url env) ))))
(mount/defstate import-invoices
:start (scheduler/every (* 60 5000) (heartbeat process-sqs "import-uploaded-invoices"))
:stop (scheduler/stop import-invoices))
(defn -main [& _]
(execute "import-uploaded-invoices" process-sqs))

23
src/clj/auto_ap/jobs/intuit.clj
View File

@@ -1,21 +1,10 @@
(ns auto-ap.jobs.intuit
(:gen-class)
(:require [auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]
[auto-ap.import.intuit :as intuit]))
(:require
[auto-ap.import.intuit :as intuit]
[auto-ap.jobs.core :refer [execute]]))
(defn -main [& _]
(try
(lc/with-context {:background-job "import-intuit"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat intuit/upsert-accounts "import-intuit-accounts"))
((heartbeat intuit/import-intuit "import-intuit"))
(mount/stop)
(log/info "Stopping intuit import")
(Thread/sleep 15000))
(finally
(System/exit 0))))
(execute "import-intuit" (fn []
(intuit/upsert-accounts)
(intuit/import-intuit))))

23
src/clj/auto_ap/jobs/ledger_reconcile.clj
View File

@@ -1,22 +1,11 @@
(ns auto-ap.jobs.ledger-reconcile
(:gen-class)
(:require [auto-ap.ledger :as ledger]
[auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]))
(:require
[auto-ap.jobs.core :refer [execute]]
[auto-ap.ledger :as ledger]))
(defn -main [& _]
(try
(lc/with-context {:background-job "reconcile-ledger"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat ledger/reconcile-ledger "reconcile-ledger"))
((heartbeat ledger/touch-broken-ledger "touch-broken-ledger"))
(mount/stop)
(log/info "Stopping Ledger reconciliation")
(Thread/sleep 15000))
(finally
(System/exit 0))))
(execute "reconcile-ledger" (fn []
(ledger/reconcile-ledger)
(ledger/touch-broken-ledger))))

20
src/clj/auto_ap/jobs/plaid.clj
View File

@@ -1,20 +1,8 @@
(ns auto-ap.jobs.plaid
(:gen-class)
(:require [auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]
[auto-ap.import.plaid :as plaid]))
(:require
[auto-ap.import.plaid :as plaid]
[auto-ap.jobs.core :refer [execute]]))
(defn -main [& _]
(try
(lc/with-context {:background-job "import-plaid"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat plaid/import-plaid "import-plaid"))
(mount/stop)
(log/info "Stopping plaid import")
(Thread/sleep 15000))
(finally
(System/exit 0))))
(execute "import-plaid" plaid/import-plaid))

18
src/clj/auto_ap/jobs/square.clj
View File

@@ -1,19 +1,9 @@
(ns auto-ap.jobs.square
(:gen-class)
(:require [auto-ap.square.core :as square]
[auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]))
(:require
[auto-ap.jobs.core :refer [execute]]
[auto-ap.square.core :as square]))
(defn -main [& _]
(lc/with-context {:background-job "square-loading-new"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat square/upsert-all "square-loading"))
(mount/stop )
(log/info "Stopping Square loading")
(Thread/sleep 15000)
(System/exit 0)))
(execute "square-loading" square/upsert-all))

12
src/clj/auto_ap/background/sysco.clj → src/clj/auto_ap/jobs/sysco.clj
View File

@@ -1,4 +1,4 @@
(ns auto-ap.background.sysco
(ns auto-ap.jobs.sysco
(:require
[amazonica.aws.s3 :as s3]
[auto-ap.datomic :refer [conn]]
@@ -6,7 +6,6 @@
[auto-ap.datomic.invoices :refer [code-invoice]]
[auto-ap.parse :as parse]
[auto-ap.time :as t]
[auto-ap.utils :refer [heartbeat]]
[clj-time.coerce :as coerce]
[clojure.data.csv :as csv]
[clojure.java.io :as io]
@@ -15,11 +14,11 @@
[com.unbounce.dogstatsd.core :as statsd]
[config.core :refer [env]]
[datomic.api :as d]
[mount.core :as mount]
[yang.scheduler :as scheduler])
[auto-ap.jobs.core :refer [execute]])
(:import
(java.util UUID)))
(def bucket-name (:data-bucket env))
(def header-keys ["TransCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "AccountName" "AccountDunsNo" "InvoiceDate" "AccountDate" "CustomerPONo" "PaymentTerms" "TermsDescription" "StoreNumber" "CustomerName" "AddressLine1" "AddressLine2" "City1" "State1" "Zip1" "Phone1" "Duns1" "Hin1" "Dea1" "TIDCustomer" "ChainNumber" "BidNumber" "ContractNumber" "CompanyNumber" "BriefName" "Address" "Address2" "City2" "State2" "Zip2" "Phone2" "Duns2" "Hin2" "Dea2" "Tid_OPCO" "ObligationIndicator" "Manifest" "Route" "Stop" "TermsDiscountPercent" "TermsDiscountDueDate" "TermsNetDueDate" "TermsDiscountAmount" "TermsDiscountCode" "OrderDate" "DepartmentCode"])
@@ -158,6 +157,5 @@
:priority :low} nil)))
(mount/defstate sysco-invoice-importer
:start (scheduler/every (* 1000 60 60) (heartbeat import-sysco "sysco-importer"))
:stop (scheduler/stop sysco-invoice-importer))
(defn -main [& _]
(execute "sysco" import-sysco))

14
src/clj/auto_ap/background/vendor.clj → src/clj/auto_ap/jobs/vendor_usages.clj
View File

@@ -1,10 +1,9 @@
(ns auto-ap.background.vendor
(ns auto-ap.jobs.vendor-usages
(:gen-class)
(:require
[auto-ap.datomic :refer [conn]]
[auto-ap.utils :refer [heartbeat]]
[datomic.api :as d]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
[auto-ap.jobs.core :refer [execute]]
[datomic.api :as d]))
(defn refresh-vendor-usages []
(->> {:query {:find ['?v '?c '(count ?e)]
@@ -30,6 +29,5 @@
(d/transact conn)
deref))
(mount/defstate refresh-vendor-usages-worker
:start (scheduler/every (* 60 60 1000) (heartbeat refresh-vendor-usages "vendor-usages"))
:stop (scheduler/stop refresh-vendor-usages))
(defn -main [& _]
(execute "vendor-usages" refresh-vendor-usages))

25
src/clj/auto_ap/jobs/yodlee2.clj
View File

@@ -1,23 +1,12 @@
(ns auto-ap.jobs.yodlee2
(:gen-class)
(:require [auto-ap.utils :refer [heartbeat]]
[mount.core :as mount]
[auto-ap.datomic :refer [conn]]
[auto-ap.yodlee.core2 :as client2]
[clojure.tools.logging :as log]
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
[unilog.context :as lc]
[auto-ap.import.yodlee2 :as yodlee2]))
(:require
[auto-ap.import.yodlee2 :as yodlee2]
[auto-ap.jobs.core :refer [execute]]
[auto-ap.yodlee.core2 :as client2]))
(defn -main [& _]
(try
(lc/with-context {:background-job "import-yodlee"}
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
((heartbeat client2/upsert-accounts "upsert-yodlee2-accounts"))
((heartbeat yodlee2/import-yodlee2 "import-yodlee"))
(mount/stop)
(log/info "Stopping yodlee import")
(Thread/sleep 15000))
(finally
(System/exit 0))))
(execute "import-yodlee" (fn []
(client2/upsert-accounts)
(yodlee2/import-yodlee2))))

2
src/clj/auto_ap/ledger.clj
View File

@@ -2,7 +2,7 @@
(:require
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.logging :refer [info-event]]
[auto-ap.utils :refer [dollars-0? dollars= heartbeat]]
[auto-ap.utils :refer [dollars-0? dollars=]]
[clj-time.coerce :as c]
[clj-time.core :as t]
[clojure.tools.logging :as log]

34
src/clj/auto_ap/server.clj
View File

@@ -2,17 +2,18 @@
(:gen-class)
(:require
[auto-ap.background.invoices]
[auto-ap.background.mail :as mail]
[auto-ap.background.requests :as requests]
[auto-ap.background.sysco :as sysco]
[auto-ap.background.vendor :as vendor]
[auto-ap.datomic.migrate :as migrate]
[auto-ap.handler :refer [app]]
[auto-ap.jobs.close-auto-invoices :as job-close-auto-invoices]
[auto-ap.jobs.current-balance-cache :as job-current-balance-cache]
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
[auto-ap.jobs.intuit :as job-intuit]
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
[auto-ap.jobs.plaid :as job-plaid]
[auto-ap.jobs.square :as job-square]
[auto-ap.jobs.sysco :as job-sysco]
[auto-ap.jobs.vendor-usages :as job-vendor-usages]
[auto-ap.jobs.yodlee2 :as job-yodlee2]
[auto-ap.ledger :as ledger]
[clojure.tools.logging :as log]
@@ -79,9 +80,6 @@
:start (scheduler/every (* 1000 10) collect-jetty-stats)
:stop (scheduler/stop jetty-stats))
(defn shutdown-mount []
(mount/stop))
@@ -106,24 +104,24 @@
(= job "intuit")
(job-intuit/-main)
(= job "hello-world")
(do
(log/info "HELLO WORLD")
(Thread/sleep 10000)
(log/info "GOODBYE WORLD")
(Thread/sleep 10000)
(System/exit 0))
(= job "vendor-usages")
(job-vendor-usages/-main)
(= job "import-uploaded-invoices")
(job-import-uploaded-invoices/-main)
(= job "sysco")
(job-sysco/-main)
(= job "close-auto-invoices")
(job-close-auto-invoices/-main)
:else
(let [without (cond-> []
(not (env :run-web? )) (into [#'jetty
#'jetty-stats])
(not (env :run-background?)) (into [#'vendor/refresh-vendor-usages-worker
#'mail/import-invoices
#'ledger/process-txes-worker
(not (env :run-background?)) (into [#'ledger/process-txes-worker
#'requests/request-listener
#'sysco/sysco-invoice-importer
#'auto-ap.background.invoices/close-auto-invoices-worker
#'migrate/migrate-start]))]
(log/info "starting without " without)

11
src/clj/auto_ap/square/core.clj
View File

@@ -2,7 +2,6 @@
(:require
[auto-ap.datomic :refer [conn remove-nils]]
[auto-ap.time :as atime]
[auto-ap.utils :refer [heartbeat]]
[clj-http.client :as client]
[clj-time.coerce :as coerce]
[clj-time.core :as time]
@@ -14,10 +13,8 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[datomic.api :as d]
[mount.core :as mount]
[slingshot.slingshot :refer [try+]]
[unilog.context :as lc]
[yang.scheduler :as scheduler]))
[unilog.context :as lc]))
(defn client-base-headers [client]
{"Square-Version" "2021-08-18"
@@ -530,9 +527,3 @@
(some-> (:object &throw-context) str)
"Unknown error")}))))))
#_(mount/defstate square-loader
:start (scheduler/every (* 4 59 60 1000) (heartbeat upsert-all "square-loading"))
:stop (scheduler/stop square-loader))

2
src/cljs/auto_ap/views/pages/transactions/form.cljs
View File

@@ -13,7 +13,7 @@
[auto-ap.views.components.layouts :as layouts]
[auto-ap.views.pages.transactions.common :refer [transaction-read]]
[auto-ap.views.utils
:refer [->$ date->str date-picker dispatch-event pretty with-user]]
:refer [->$ date->str date-picker dispatch-event with-user]]
[clojure.string :as str]
[malli.core :as m]
[re-frame.core :as re-frame]