notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

intuit import button and sysco importing.

Browse Source
This commit is contained in:
Bryce Covert
2021-12-18 19:11:02 -08:00
parent fcdc00ed16
commit 64720a4bd6
19 changed files with 297 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config/dev.edn
View File

@@ -6,6 +6,7 @@
:aws-region "us-east-1"
:datomic-url "datomic:sql://invoices?jdbc:postgresql://database:5432/datomic?user=datomic&password=datomic"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-staging"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod"
:invoice-email "invoices-staging@mail.app.integreatconsult.com"
:data-bucket "data.staging.app.integreatconsult.com"

1
config/prod-background-worker.edn
View File

@@ -3,6 +3,7 @@
:scheme "https"
:jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-requests-prod"
:invoice-email "invoices@mail.app.integreatconsult.com"
:data-bucket "data.prod.app.integreatconsult.com"
:yodlee-cobrand-name "qstartus12"

1
config/prod.edn
View File

@@ -3,6 +3,7 @@
:scheme "https"
:jwt-secret "auto ap invoices are awesome"
:invoice-import-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-mail-prod"
:requests-queue-url "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-requests-prod"
:invoice-email "invoices@mail.app.integreatconsult.com"
:data-bucket "data.prod.app.integreatconsult.com"
:yodlee-cobrand-name "qstartus12"

3
project.clj
View File

@@ -69,6 +69,9 @@
[com.amazonaws/aws-java-sdk-s3 "1.11.926"
:exclusions [commons-codec
org.apache.httpcomponents/httpclient]]
[com.amazonaws/aws-java-sdk-sqs "1.11.926"
:exclusions [commons-codec
org.apache.httpcomponents/httpclient]]
[com.amazonaws/aws-java-sdk-dynamodb "1.11.926"
:exclusions [commons-codec
org.apache.httpcomponents/httpclient]]

42
src/clj/auto_ap/background/requests.clj Normal file
View File

@@ -0,0 +1,42 @@
(ns auto-ap.background.requests
(:require
[amazonica.aws.sqs :as sqs]
[config.core :refer [env]]
[mount.core :as mount]
[clojure.tools.logging :as log]
[auto-ap.intuit.import :as i]
[unilog.context :as lc]))
(def stopped? (atom false))
(defn process-1 []
(let [[{:keys [message-id receipt-handle body]}] (:messages (sqs/receive-message {:queue-url (:requests-queue-url env)
:count 1}))]
(when message-id
(log/infof "processing message %s with body %s" message-id body )
(if (= ":intuit" body)
(try
(i/upsert-transactions)
(catch Exception e
(log/error e))))
(sqs/delete-message {:queue-url (:requests-queue-url env)
:receipt-handle receipt-handle} ))))
(defn fake-message []
(sqs/send-message {:queue-url (:requests-queue-url env)
:message-body ":intuit"} ))
(defn listen-sqs []
(lc/with-context {:source "Request poller"}
(loop []
(process-1)
(log/info "Checking SQS....")
(Thread/sleep 5000)
(when-not @stopped?
(log/info "stopping")
(recur)))))
(mount/defstate request-listener
:start (future (listen-sqs))
:stop (reset! stopped? true))

8
src/clj/auto_ap/background/sysco.clj
View File

@@ -16,7 +16,9 @@
[config.core :refer [env]]
[datomic.api :as d]
[unilog.context :as lc]
[clj-time.core :as time]))
[clj-time.core :as time]
[mount.core :as mount]
[yang.scheduler :as scheduler]))
(def bucket-name (:data-bucket env))
@@ -130,10 +132,10 @@
))
keys))]
(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
#_(doseq [k keys]
(doseq [k keys]
(mark-key k)))))
(mount/defstate sysco-invoice-importer
:start (scheduler/every (* 1000 60 60) import)
:start (scheduler/every (* 1000 60 60) import-sysco)
:stop (scheduler/stop sysco-invoice-importer))

8
src/clj/auto_ap/graphql.clj
View File

@@ -12,6 +12,7 @@
[auto-ap.graphql.intuit-bank-accounts :as gq-intuit-bank-accounts]
[auto-ap.graphql.invoices :as gq-invoices]
[auto-ap.graphql.ledger :as gq-ledger]
[auto-ap.graphql.requests :as gq-requests]
[auto-ap.graphql.sales-orders :as gq-sales-orders]
[auto-ap.graphql.transaction-rules :as gq-transaction-rules]
[auto-ap.graphql.transactions :as gq-transactions]
@@ -999,7 +1000,11 @@
{:enum-value :requires_feedback}
{:enum-value :excluded}]}}
:mutations
{:reject_invoices {:type '(list :id)
{:request_intuit_import {:type 'String
:args {}
:resolve :mutation/request-intuit-import}
:reject_invoices {:type '(list :id)
:args {:invoices {:type '(list :id)}}
:resolve :mutation/reject-invoices}
@@ -1408,6 +1413,7 @@
:mutation/void-payment gq-checks/void-check
:mutation/edit-expense-accounts gq-invoices/edit-expense-accounts
:mutation/import-ledger gq-ledger/import-ledger
:mutation/request-intuit-import gq-requests/request-intuit-import
:get-vendor gq-vendors/get-graphql})
schema/compile))

11
src/clj/auto_ap/graphql/requests.clj Normal file
View File

@@ -0,0 +1,11 @@
(ns auto-ap.graphql.requests
(:require
[amazonica.aws.sqs :as sqs]
[auto-ap.graphql.utils :refer [assert-admin]]
[config.core :refer [env]]))
(defn request-intuit-import [context value args]
(assert-admin (:id context))
(:message-id (sqs/send-message {:queue-url (:requests-queue-url env)
:message-body ":intuit"} )))

5
src/clj/auto_ap/intuit/core.clj
View File

@@ -55,6 +55,11 @@
:input-stream (io/make-input-stream (.getBytes t) {})
:metadata {:content-type "application/text"}))
(defn copy-prod-refresh-token []
(set-refresh-token (slurp (:object-content (s3/get-object
:bucket-name "data.prod.app.integreatconsult.com"
:key "intuit/refresh-token")))))
(defn init-tokens [access refresh]
(set-access-token access)
(set-refresh-token refresh))

14
src/clj/auto_ap/intuit/import.clj
View File

@@ -25,6 +25,18 @@
[?iab :intuit-bank-account/external-id ?external-id]]
db #_client-whitelist)))
(defn allow-once [f]
(let [in-progress? (atom false)]
(fn []
(println in-progress?)
(when (= false @in-progress?)
(try
(reset! in-progress? true)
(f)
(finally
(reset! in-progress? false)))))))
(defn upsert-transactions []
(let [db (d/db conn)
import-id (y/start-import :import-source/intuit "Automated Intuit User")
@@ -56,7 +68,7 @@
:import-batch/suppressed (count (:suppressed result))}))))))
(y/finish-import (assoc @stats :db/id import-id))))
(def upsert-transactions (allow-once upsert-transactions))
(defn dry-run-upsert-transactions []

4
src/clj/auto_ap/server.clj
View File

@@ -5,6 +5,8 @@
[auto-ap.yodlee.core2 :as yodlee2]
[auto-ap.graphql.clients :as gq-clients]
[auto-ap.background.invoices]
[auto-ap.background.requests :as requests]
[auto-ap.background.sysco :as sysco]
[auto-ap.background.vendor :as vendor]
[auto-ap.square.core :as square]
[auto-ap.datomic.migrate :as migrate]
@@ -40,6 +42,8 @@
#'ledger/touch-broken-ledger-worker
#'ledger/process-txes-worker
#'ledger/ledger-reconciliation-worker
#'requests/request-listener
#'sysco/sysco-invoice-importer
#'auto-ap.background.invoices/close-auto-invoices-worker
#'gq-clients/current-balance-worker
#'yodlee/import-transaction-worker

44
src/cljs/auto_ap/views/pages/admin/import_batches.cljs
View File

@@ -14,7 +14,8 @@
[auto-ap.effects.forward :as forward]
[auto-ap.views.pages.data-page :as data-page]
[clojure.set :as set]
[auto-ap.views.components.buttons :as buttons]))
[auto-ap.views.components.buttons :as buttons]
[clojure.string :as str]))
(def default-read [:user-name :date :source :status :id :imported :suppressed :extant])
@@ -39,6 +40,25 @@
(set/rename-keys (:result result)
{:import-batches :data})])}}))
(re-frame/reg-event-db
::success-intuit
(fn [db [_ n]]
(assoc db ::msg (str "Your job " (:request-intuit-import n) " has been scheduled." ))))
(re-frame/reg-sub
::msg
(fn [db]
(::msg db)))
(re-frame/reg-event-fx ::request-intuit
[with-user ]
(fn [{:keys [db user] :as cofx} [_ params]]
{:graphql {:token user
:owns-state {:single ::intuit}
:query "mutation RequestIntuitImport{request_intuit_import}"
:on-success [::success-intuit]
}}))
(re-frame/reg-event-fx
::mounted
@@ -52,18 +72,32 @@
::unmounted
(fn [{:keys [db]}]
{:dispatch [::data-page/dispose ::page]
::track/dispose {:id ::params}}))
::track/dispose {:id ::params}
:db (dissoc db ::msg)}))
;; VIEWS
(def import-batch-content
(with-meta
(fn []
(let [user @(re-frame/subscribe [::subs/user])]
(let [user @(re-frame/subscribe [::subs/user])
request-import-status @(re-frame/subscribe [::status/single ::intuit])
message @(re-frame/subscribe [::msg])]
[:div
[:h1.title "Import Batches"]
(when message
[:div.notification.is-info.is-light message])
(when (= "admin" (:user/role user))
[table/table {:id :import-batches
:data-page ::page}])]))
[:div
[:div.is-pulled-right
[:div.buttons
[:button.button.is-primary-two.is-medium {:aria-haspopup true
:type "button"
:on-click (dispatch-event [::request-intuit])
:disabled (status/disabled-for request-import-status)
:class (status/class-for request-import-status)}
"Start Intuit Import"]]]
[table/table {:id :import-batches
:data-page ::page}]])]))
{:component-did-mount (dispatch-event [::mounted ])
:component-will-unmount #(re-frame/dispatch-sync [::unmounted])}))

20
terraform/.terraform.lock.hcl generated Normal file
View File

@@ -0,0 +1,20 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/aws" {
version = "3.70.0"
hashes = [
"h1:E5IKHXzPGGSizZM5rHKzNCzpwQ7lWPXmmJnms82uzDk=",
"zh:0af710e528e21b930899f0ac295b0ceef8ad7b623dd8f38e92c8ec4bc7af0321",
"zh:4cabcd4519c0aae474d91ae67a8e3a4a8c39c3945c289a9cf7c1409f64409abe",
"zh:58da1a436facb4e4f95cd2870d211ed7bcb8cf721a4a61970aa8da191665f2aa",
"zh:6465339475c1cd3c16a5c8fee61304dcad2c4a27740687d29c6cdc90d2e6423d",
"zh:7a821ed053c355d70ebe33185590953fa5c364c1f3d66fe3f9b4aba3961646b1",
"zh:7c3656cc9cc1739dcb298e7930c9a76ccfce738d2070841d7e6c62fbdae74eef",
"zh:9d9da9e3c60a0c977e156da8590f36a219ae91994bb3df5a1208de2ab3ceeba7",
"zh:a3138817c86bf3e4dca7fd3a92e099cd1bf1d45ee7c7cc9e9773ba04fc3b315a",
"zh:a8603044e935dfb3cb9319a46d26276162c6aea75e02c4827232f9c6029a3182",
"zh:aef9482332bf43d0b73317f5909dec9e95b983c67b10d72e75eacc7c4f37d084",
"zh:fc3f3cad84f2eebe566dd0b65904c934093007323b9b85e73d9dd4535ceeb29d",
]
}

20
terraform/main.tf
View File

@@ -112,6 +112,26 @@ resource "aws_sqs_queue" "integreat-mail" {
POLICY
}
resource "aws_sqs_queue" "background-request" {
name = "integreat-background-request-${var.stage}"
policy = <<POLICY
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "${var.task_role_arn}"
},
"Action": "sqs:*",
"Resource": "arn:aws:sqs:*:*:integreat-background-request-${var.stage}"
}
]
}
POLICY
}
resource "aws_s3_bucket_notification" "mail_bucket_notification" {
bucket = "${aws_s3_bucket.invoices.id}"

37
terraform/staging-background-worker-taskdef.json Normal file
View File

@@ -0,0 +1,37 @@
[
{
"name": "integreat-app",
"image": "679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat",
"portMappings": [
{
"containerPort": 9000,
"hostPort": 9000,
"protocol": "tcp"
},
{
"containerPort": 9090,
"hostPort": 9090,
"protocol": "tcp"
}
],
"essential": true,
"environment": [
{
"name": "config",
"value": "/usr/local/config/prod-background-worker.edn"
}
],
"mountPoints": [],
"volumesFrom": [],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs/integreat-background-worker-prod",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "ecs"
}
}
}
]

37
terraform/staging-backgroundu-worker-taskdef.json Normal file
View File

@@ -0,0 +1,37 @@
[
{
"name": "integreat-app",
"image": "679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat",
"portMappings": [
{
"containerPort": 9000,
"hostPort": 9000,
"protocol": "tcp"
},
{
"containerPort": 9090,
"hostPort": 9090,
"protocol": "tcp"
}
],
"essential": true,
"environment": [
{
"name": "config",
"value": "/usr/local/config/prod-background-worker.edn"
}
],
"mountPoints": [],
"volumesFrom": [],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs/integreat-background-worker-prod",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "ecs"
}
}
}
]

1
terraform/staging.tfvars
View File

@@ -8,3 +8,4 @@ task_role_arn="arn:aws:iam::679918342773:role/datomic-ddb"
execution_role_arn="arn:aws:iam::679918342773:role/ecsTaskExecutionRole"
ecs_cluster="arn:aws:ecs:us-east-1:679918342773:cluster/default"
service_registry="arn:aws:servicediscovery:us-east-1:679918342773:service/srv-6auj2wqsh55k2nuj"
local_namespace="ns-gv2z744em7myo2jp"

50
terraform/terraform.tfstate.d/prod/terraform.tfstate
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.1.0",
"serial": 90,
"serial": 92,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -162,9 +162,9 @@
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_ecs_task_definition.integreat_app",
"aws_lb_target_group.integreat_app",
"aws_service_discovery_service.service",
"aws_ecs_task_definition.integreat_app"
"aws_service_discovery_service.service"
]
}
]
@@ -241,8 +241,8 @@
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_ecs_task_definition.integreat_background_worker",
"aws_service_discovery_service.background_worker_service"
"aws_service_discovery_service.background_worker_service",
"aws_ecs_task_definition.integreat_background_worker"
]
}
]
@@ -909,9 +909,9 @@
},
"sensitive_attributes": [],
"dependencies": [
"data.aws_caller_identity.current",
"aws_s3_bucket.invoices",
"aws_sqs_queue.integreat-mail"
"aws_sqs_queue.integreat-mail",
"data.aws_caller_identity.current"
]
}
]
@@ -1060,6 +1060,42 @@
}
]
},
{
"mode": "managed",
"type": "aws_sqs_queue",
"name": "background-request",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:sqs:us-east-1:679918342773:integreat-background-request-prod",
"content_based_deduplication": false,
"deduplication_scope": "",
"delay_seconds": 0,
"fifo_queue": false,
"fifo_throughput_limit": "",
"id": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod",
"kms_data_key_reuse_period_seconds": 300,
"kms_master_key_id": "",
"max_message_size": 262144,
"message_retention_seconds": 345600,
"name": "integreat-background-request-prod",
"name_prefix": "",
"policy": "{\"Statement\":[{\"Action\":\"sqs:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:sqs:*:*:integreat-background-request-prod\"}],\"Version\":\"2012-10-17\"}",
"receive_wait_time_seconds": 0,
"redrive_policy": "",
"sqs_managed_sse_enabled": false,
"tags": null,
"tags_all": {},
"url": "https://sqs.us-east-1.amazonaws.com/679918342773/integreat-background-request-prod",
"visibility_timeout_seconds": 30
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_sqs_queue",

14
terraform/terraform.tfstate.d/prod/terraform.tfstate.backup
View File

@@ -1,7 +1,7 @@
{
"version": 4,
"terraform_version": "1.1.0",
"serial": 89,
"serial": 90,
"lineage": "9b630886-8cee-a57d-c7a2-4f19f13f9c51",
"outputs": {
"aws_access_key_id": {
@@ -162,9 +162,9 @@
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_ecs_task_definition.integreat_app",
"aws_lb_target_group.integreat_app",
"aws_service_discovery_service.service"
"aws_service_discovery_service.service",
"aws_ecs_task_definition.integreat_app"
]
}
]
@@ -649,10 +649,10 @@
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lb_target_group.integreat_app",
"aws_acm_certificate.cert",
"aws_lb.integreat_app",
"aws_lb_listener.https"
"aws_lb_listener.https",
"aws_lb_target_group.integreat_app"
]
}
]
@@ -909,9 +909,9 @@
},
"sensitive_attributes": [],
"dependencies": [
"data.aws_caller_identity.current",
"aws_s3_bucket.invoices",
"aws_sqs_queue.integreat-mail",
"data.aws_caller_identity.current"
"aws_sqs_queue.integreat-mail"
]
}
]