From 64720a4bd62b406b18fdcde7fb07455d81b0a5cf Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Sat, 18 Dec 2021 19:11:02 -0800 Subject: [PATCH] intuit import button and sysco importing. --- config/dev.edn | 1 + config/prod-background-worker.edn | 1 + config/prod.edn | 1 + project.clj | 3 ++ src/clj/auto_ap/background/requests.clj | 42 ++++++++++++++++ src/clj/auto_ap/background/sysco.clj | 8 +-- src/clj/auto_ap/graphql.clj | 8 ++- src/clj/auto_ap/graphql/requests.clj | 11 ++++ src/clj/auto_ap/intuit/core.clj | 5 ++ src/clj/auto_ap/intuit/import.clj | 14 +++++- src/clj/auto_ap/server.clj | 4 ++ .../views/pages/admin/import_batches.cljs | 44 ++++++++++++++-- terraform/.terraform.lock.hcl | 20 ++++++++ terraform/main.tf | 20 ++++++++ .../staging-background-worker-taskdef.json | 37 ++++++++++++++ .../staging-backgroundu-worker-taskdef.json | 37 ++++++++++++++ terraform/staging.tfvars | 1 + .../prod/terraform.tfstate | 50 ++++++++++++++++--- .../prod/terraform.tfstate.backup | 14 +++--- 19 files changed, 297 insertions(+), 24 deletions(-) create mode 100644 src/clj/auto_ap/background/requests.clj create mode 100644 src/clj/auto_ap/graphql/requests.clj create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/staging-background-worker-taskdef.json create mode 100644 terraform/staging-backgroundu-worker-taskdef.json diff --git a/config/dev.edn b/config/dev.edn index 110cae82..c26973cb 100644 --- a/config/dev.edn +++ b/config/dev.edn @@ -6,6 +6,7 @@ :aws-region "us-east-1" :datomic-url "datomic:sql://invoices?jdbc:postgresql://database:5432/datomic?user=datomic&password=datomic" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging" + :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod" :invoice-email "invoices-staging@mail.app.integreatconsult.com" :data-bucket "data.staging.app.integreatconsult.com" diff --git a/config/prod-background-worker.edn b/config/prod-background-worker.edn index e002181b..bf6ba3e9 100644 --- a/config/prod-background-worker.edn +++ b/config/prod-background-worker.edn @@ -3,6 +3,7 @@ :scheme "https" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" + :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-requests-prod" :invoice-email "invoices@mail.app.integreatconsult.com" :data-bucket "data.prod.app.integreatconsult.com" :yodlee-cobrand-name "qstartus12" diff --git a/config/prod.edn b/config/prod.edn index 7ebf8ebe..56886454 100644 --- a/config/prod.edn +++ b/config/prod.edn @@ -3,6 +3,7 @@ :scheme "https" :jwt-secret "auto ap invoices are awesome" :invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod" + :requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-requests-prod" :invoice-email "invoices@mail.app.integreatconsult.com" :data-bucket "data.prod.app.integreatconsult.com" :yodlee-cobrand-name "qstartus12" diff --git a/project.clj b/project.clj index ef6132ca..5d4ca5ed 100644 --- a/project.clj +++ b/project.clj @@ -69,6 +69,9 @@ [com.amazonaws/aws-java-sdk-s3 "1.11.926" :exclusions [commons-codec org.apache.httpcomponents/httpclient]] + [com.amazonaws/aws-java-sdk-sqs "1.11.926" + :exclusions [commons-codec + org.apache.httpcomponents/httpclient]] [com.amazonaws/aws-java-sdk-dynamodb "1.11.926" :exclusions [commons-codec org.apache.httpcomponents/httpclient]] diff --git a/src/clj/auto_ap/background/requests.clj b/src/clj/auto_ap/background/requests.clj new file mode 100644 index 00000000..8101be93 --- /dev/null +++ b/src/clj/auto_ap/background/requests.clj @@ -0,0 +1,42 @@ +(ns auto-ap.background.requests + (:require + [amazonica.aws.sqs :as sqs] + [config.core :refer [env]] + [mount.core :as mount] + [clojure.tools.logging :as log] + [auto-ap.intuit.import :as i] + [unilog.context :as lc])) + +(def stopped? (atom false)) + +(defn process-1 [] + (let [[{:keys [message-id receipt-handle body]}] (:messages (sqs/receive-message {:queue-url (:requests-queue-url env) + :count 1}))] + (when message-id + (log/infof "processing message %s with body %s" message-id body ) + (if (= ":intuit" body) + (try + (i/upsert-transactions) + (catch Exception e + (log/error e)))) + (sqs/delete-message {:queue-url (:requests-queue-url env) + :receipt-handle receipt-handle} )))) + +(defn fake-message [] + (sqs/send-message {:queue-url (:requests-queue-url env) + :message-body ":intuit"} )) + +(defn listen-sqs [] + (lc/with-context {:source "Request poller"} + (loop [] + (process-1) + (log/info "Checking SQS....") + (Thread/sleep 5000) + (when-not @stopped? + (log/info "stopping") + (recur))))) + +(mount/defstate request-listener + :start (future (listen-sqs)) + :stop (reset! stopped? true)) + diff --git a/src/clj/auto_ap/background/sysco.clj b/src/clj/auto_ap/background/sysco.clj index e890f643..83ea08d6 100644 --- a/src/clj/auto_ap/background/sysco.clj +++ b/src/clj/auto_ap/background/sysco.clj @@ -16,7 +16,9 @@ [config.core :refer [env]] [datomic.api :as d] [unilog.context :as lc] - [clj-time.core :as time])) + [clj-time.core :as time] + [mount.core :as mount] + [yang.scheduler :as scheduler])) (def bucket-name (:data-bucket env)) @@ -130,10 +132,10 @@ )) keys))] (log/infof "Imported %d invoices" (/ (count (:tempids result)) 2))) - #_(doseq [k keys] + (doseq [k keys] (mark-key k))))) (mount/defstate sysco-invoice-importer - :start (scheduler/every (* 1000 60 60) import) + :start (scheduler/every (* 1000 60 60) import-sysco) :stop (scheduler/stop sysco-invoice-importer)) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 2b05e45d..b25f605a 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -12,6 +12,7 @@ [auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts] [auto-ap.graphql.invoices :as gq-invoices] [auto-ap.graphql.ledger :as gq-ledger] + [auto-ap.graphql.requests :as gq-requests] [auto-ap.graphql.sales-orders :as gq-sales-orders] [auto-ap.graphql.transaction-rules :as gq-transaction-rules] [auto-ap.graphql.transactions :as gq-transactions] @@ -999,7 +1000,11 @@ {:enum-value :requires_feedback} {:enum-value :excluded}]}} :mutations - {:reject_invoices {:type '(list :id) + {:request_intuit_import {:type 'String + :args {} + :resolve :mutation/request-intuit-import} + + :reject_invoices {:type '(list :id) :args {:invoices {:type '(list :id)}} :resolve :mutation/reject-invoices} @@ -1408,6 +1413,7 @@ :mutation/void-payment gq-checks/void-check :mutation/edit-expense-accounts gq-invoices/edit-expense-accounts :mutation/import-ledger gq-ledger/import-ledger + :mutation/request-intuit-import gq-requests/request-intuit-import :get-vendor gq-vendors/get-graphql}) schema/compile)) diff --git a/src/clj/auto_ap/graphql/requests.clj b/src/clj/auto_ap/graphql/requests.clj new file mode 100644 index 00000000..3126acec --- /dev/null +++ b/src/clj/auto_ap/graphql/requests.clj @@ -0,0 +1,11 @@ +(ns auto-ap.graphql.requests + (:require + [amazonica.aws.sqs :as sqs] + [auto-ap.graphql.utils :refer [assert-admin]] + [config.core :refer [env]])) + +(defn request-intuit-import [context value args] + (assert-admin (:id context)) + (:message-id (sqs/send-message {:queue-url (:requests-queue-url env) + :message-body ":intuit"} ))) + diff --git a/src/clj/auto_ap/intuit/core.clj b/src/clj/auto_ap/intuit/core.clj index 93f81b70..90b349dd 100644 --- a/src/clj/auto_ap/intuit/core.clj +++ b/src/clj/auto_ap/intuit/core.clj @@ -55,6 +55,11 @@ :input-stream (io/make-input-stream (.getBytes t) {}) :metadata {:content-type "application/text"})) +(defn copy-prod-refresh-token [] + (set-refresh-token (slurp (:object-content (s3/get-object + :bucket-name "data.prod.app.integreatconsult.com" + :key "intuit/refresh-token"))))) + (defn init-tokens [access refresh] (set-access-token access) (set-refresh-token refresh)) diff --git a/src/clj/auto_ap/intuit/import.clj b/src/clj/auto_ap/intuit/import.clj index aa7e7340..2a1b51de 100644 --- a/src/clj/auto_ap/intuit/import.clj +++ b/src/clj/auto_ap/intuit/import.clj @@ -25,6 +25,18 @@ [?iab :intuit-bank-account/external-id ?external-id]] db #_client-whitelist))) + +(defn allow-once [f] + (let [in-progress? (atom false)] + (fn [] + (println in-progress?) + (when (= false @in-progress?) + (try + (reset! in-progress? true) + (f) + (finally + (reset! in-progress? false))))))) + (defn upsert-transactions [] (let [db (d/db conn) import-id (y/start-import :import-source/intuit "Automated Intuit User") @@ -56,7 +68,7 @@ :import-batch/suppressed (count (:suppressed result))})))))) (y/finish-import (assoc @stats :db/id import-id)))) - +(def upsert-transactions (allow-once upsert-transactions)) (defn dry-run-upsert-transactions [] diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 1b6a4987..0681e660 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -5,6 +5,8 @@ [auto-ap.yodlee.core2 :as yodlee2] [auto-ap.graphql.clients :as gq-clients] [auto-ap.background.invoices] + [auto-ap.background.requests :as requests] + [auto-ap.background.sysco :as sysco] [auto-ap.background.vendor :as vendor] [auto-ap.square.core :as square] [auto-ap.datomic.migrate :as migrate] @@ -40,6 +42,8 @@ #'ledger/touch-broken-ledger-worker #'ledger/process-txes-worker #'ledger/ledger-reconciliation-worker + #'requests/request-listener + #'sysco/sysco-invoice-importer #'auto-ap.background.invoices/close-auto-invoices-worker #'gq-clients/current-balance-worker #'yodlee/import-transaction-worker diff --git a/src/cljs/auto_ap/views/pages/admin/import_batches.cljs b/src/cljs/auto_ap/views/pages/admin/import_batches.cljs index 681efb3b..6181043f 100644 --- a/src/cljs/auto_ap/views/pages/admin/import_batches.cljs +++ b/src/cljs/auto_ap/views/pages/admin/import_batches.cljs @@ -14,7 +14,8 @@ [auto-ap.effects.forward :as forward] [auto-ap.views.pages.data-page :as data-page] [clojure.set :as set] - [auto-ap.views.components.buttons :as buttons])) + [auto-ap.views.components.buttons :as buttons] + [clojure.string :as str])) (def default-read [:user-name :date :source :status :id :imported :suppressed :extant]) @@ -39,6 +40,25 @@ (set/rename-keys (:result result) {:import-batches :data})])}})) +(re-frame/reg-event-db + ::success-intuit + (fn [db [_ n]] + (assoc db ::msg (str "Your job " (:request-intuit-import n) " has been scheduled." )))) + +(re-frame/reg-sub + ::msg + (fn [db] + (::msg db))) + +(re-frame/reg-event-fx ::request-intuit + [with-user ] + (fn [{:keys [db user] :as cofx} [_ params]] + {:graphql {:token user + :owns-state {:single ::intuit} + :query "mutation RequestIntuitImport{request_intuit_import}" + :on-success [::success-intuit] + }})) + (re-frame/reg-event-fx ::mounted @@ -52,18 +72,32 @@ ::unmounted (fn [{:keys [db]}] {:dispatch [::data-page/dispose ::page] - ::track/dispose {:id ::params}})) + ::track/dispose {:id ::params} + :db (dissoc db ::msg)})) ;; VIEWS (def import-batch-content (with-meta (fn [] - (let [user @(re-frame/subscribe [::subs/user])] + (let [user @(re-frame/subscribe [::subs/user]) + request-import-status @(re-frame/subscribe [::status/single ::intuit]) + message @(re-frame/subscribe [::msg])] [:div [:h1.title "Import Batches"] + (when message + [:div.notification.is-info.is-light message]) (when (= "admin" (:user/role user)) - [table/table {:id :import-batches - :data-page ::page}])])) + [:div + [:div.is-pulled-right + [:div.buttons + [:button.button.is-primary-two.is-medium {:aria-haspopup true + :type "button" + :on-click (dispatch-event [::request-intuit]) + :disabled (status/disabled-for request-import-status) + :class (status/class-for request-import-status)} + "Start Intuit Import"]]] + [table/table {:id :import-batches + :data-page ::page}]])])) {:component-did-mount (dispatch-event [::mounted ]) :component-will-unmount #(re-frame/dispatch-sync [::unmounted])})) diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 00000000..d0cbd793 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,20 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.70.0" + hashes = [ + "h1:E5IKHXzPGGSizZM5rHKzNCzpwQ7lWPXmmJnms82uzDk=", + "zh:0af710e528e21b930899f0ac295b0ceef8ad7b623dd8f38e92c8ec4bc7af0321", + "zh:4cabcd4519c0aae474d91ae67a8e3a4a8c39c3945c289a9cf7c1409f64409abe", + "zh:58da1a436facb4e4f95cd2870d211ed7bcb8cf721a4a61970aa8da191665f2aa", + "zh:6465339475c1cd3c16a5c8fee61304dcad2c4a27740687d29c6cdc90d2e6423d", + "zh:7a821ed053c355d70ebe33185590953fa5c364c1f3d66fe3f9b4aba3961646b1", + "zh:7c3656cc9cc1739dcb298e7930c9a76ccfce738d2070841d7e6c62fbdae74eef", + "zh:9d9da9e3c60a0c977e156da8590f36a219ae91994bb3df5a1208de2ab3ceeba7", + "zh:a3138817c86bf3e4dca7fd3a92e099cd1bf1d45ee7c7cc9e9773ba04fc3b315a", + "zh:a8603044e935dfb3cb9319a46d26276162c6aea75e02c4827232f9c6029a3182", + "zh:aef9482332bf43d0b73317f5909dec9e95b983c67b10d72e75eacc7c4f37d084", + "zh:fc3f3cad84f2eebe566dd0b65904c934093007323b9b85e73d9dd4535ceeb29d", + ] +} diff --git a/terraform/main.tf b/terraform/main.tf index 195907c6..e9a32693 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -112,6 +112,26 @@ resource "aws_sqs_queue" "integreat-mail" { POLICY } +resource "aws_sqs_queue" "background-request" { + name = "integreat-background-request-${var.stage}" + + policy = <