From b200728c6abbbd7ba66fee16ee798c2d966d3cdf Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 4 May 2023 17:07:08 -0700 Subject: [PATCH] makes cloud search possible. --- config/dev.edn | 1 + config/prod-cloud.edn | 1 + config/prod.edn | 1 + data/etc/nginx/conf.d/default.conf | 0 data/etc/nginx/conf.d/nginx.conf | 0 data/etc/nginx/vhost.d/default_location | 0 project.clj | 2 +- resources/public/index.html | 1 + scratch-sessions/test_solr.clj | 3 +- src/clj/auto_ap/datomic/checks.clj | 206 +++++++------ src/clj/auto_ap/datomic/invoices.clj | 225 +++++++------- src/clj/auto_ap/datomic/sales_orders.clj | 32 +- src/clj/auto_ap/datomic/transactions.clj | 236 +++++++-------- src/clj/auto_ap/solr.clj | 32 ++ src/clj/auto_ap/ssr/core.clj | 4 +- src/clj/auto_ap/ssr/search.clj | 125 ++++++++ src/clj/auto_ap/ssr/utils.clj | 5 +- src/clj/auto_ap/time.clj | 7 + src/clj/user.clj | 98 +++++- src/cljc/auto_ap/ssr_routes.cljc | 1 + src/cljs/auto_ap/events.cljs | 1 + .../auto_ap/views/components/layouts.cljs | 18 +- src/cljs/auto_ap/views/components/search.cljs | 30 ++ .../auto_ap/views/pages/transactions.cljs | 3 +- terraform/connect-ports-cloud.sh | 2 + terraform/deploy.tf | 9 +- terraform/prod-cloud-solr-taskdef.json | 62 ++++ terraform/solr.tf | 97 ++++++ .../prod-cloud/terraform.tfstate | 278 ++++++++++++++--- .../prod-cloud/terraform.tfstate.backup | 279 +++++++++++++++--- 30 files changed, 1308 insertions(+), 451 deletions(-) mode change 100644 => 100755 data/etc/nginx/conf.d/default.conf mode change 100644 => 100755 data/etc/nginx/conf.d/nginx.conf mode change 100644 => 100755 data/etc/nginx/vhost.d/default_location create mode 100644 src/clj/auto_ap/solr.clj create mode 100644 src/clj/auto_ap/ssr/search.clj create mode 100644 src/cljs/auto_ap/views/components/search.cljs create mode 100755 terraform/connect-ports-cloud.sh create mode 100644 terraform/prod-cloud-solr-taskdef.json create mode 100644 terraform/solr.tf diff --git a/config/dev.edn b/config/dev.edn index 77a40349..ac0f828c 100644 --- a/config/dev.edn +++ b/config/dev.edn @@ -1,5 +1,6 @@ {:db {:server "localhost"} :scheme "http" + :solr-uri "http://localhost:8983" :client-config {:server-type :dev-local :system "dev"} :db-name "prod-migration2" diff --git a/config/prod-cloud.edn b/config/prod-cloud.edn index b5bde305..f1748f6b 100644 --- a/config/prod-cloud.edn +++ b/config/prod-cloud.edn @@ -1,5 +1,6 @@ {:scheme "https" :db-name "prod-mirror2" + :solr-uri "http://solr-prod-cloud.local:8983" :datomic-url "datomic:ddb://us-east-1/iol-dev/dev" :dd-env "prod-cloud" :dd-service "integreat-app-cloud" diff --git a/config/prod.edn b/config/prod.edn index c3b331e8..00054e80 100644 --- a/config/prod.edn +++ b/config/prod.edn @@ -1,5 +1,6 @@ {:db {:server "database"} :datomic-url "datomic:ddb://us-east-1/integreat/integreat-prod" + :solr-uri "http://solr-prod.local:8983" :scheme "https" :dd-env "prod" :dd-service "integreat-app" diff --git a/data/etc/nginx/conf.d/default.conf b/data/etc/nginx/conf.d/default.conf old mode 100644 new mode 100755 diff --git a/data/etc/nginx/conf.d/nginx.conf b/data/etc/nginx/conf.d/nginx.conf old mode 100644 new mode 100755 diff --git a/data/etc/nginx/vhost.d/default_location b/data/etc/nginx/vhost.d/default_location old mode 100644 new mode 100755 diff --git a/project.clj b/project.clj index be202aee..2c371aa1 100644 --- a/project.clj +++ b/project.clj @@ -146,7 +146,7 @@ [com.bhauman/rebel-readline-cljs "0.1.4" :exclusions [org.clojure/clojurescript]] [javax.servlet/servlet-api "2.5"]] :plugins [[lein-pdo "0.1.1"]] - :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms4G" "-Xmx8G"]} + :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms4G" "-Xmx20G" "-Ddatomic.valcachePath=/mnt/data/datomic-cache" "-Ddatomic.valcacheMaxGb=50"]} :uberjar diff --git a/resources/public/index.html b/resources/public/index.html index 0db895a4..c3a0d4f5 100644 --- a/resources/public/index.html +++ b/resources/public/index.html @@ -16,6 +16,7 @@ + diff --git a/scratch-sessions/test_solr.clj b/scratch-sessions/test_solr.clj index 5eb2c71c..852a7f29 100644 --- a/scratch-sessions/test_solr.clj +++ b/scratch-sessions/test_solr.clj @@ -44,7 +44,8 @@ "http://localhost:8983/solr/gettingstarted/update?commitWithin=60000" {:headers {"Content-Type" "application/json"} :method "POST" - :body (json/write-str batch)})))) + :body (json/write-str batch) + })))) (comment diff --git a/src/clj/auto_ap/datomic/checks.clj b/src/clj/auto_ap/datomic/checks.clj index 7e301b71..cd61cb3d 100644 --- a/src/clj/auto_ap/datomic/checks.clj +++ b/src/clj/auto_ap/datomic/checks.clj @@ -39,119 +39,129 @@ ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] (let [check-number-like (try (Long/parseLong (:check-number-like args)) (catch Exception _ nil)) - query (cond-> {:query {:find [] - :in ['$] - :where []} - :args [db]} - (:sort args) (add-sorter-fields {"client" ['[?e :payment/client ?c] - '[?c :client/name ?sort-client]] - "vendor" ['[?e :payment/vendor ?v] - '[?v :vendor/name ?sort-vendor]] - "bank-account" ['[?e :payment/bank-account ?ba] - '[?ba :bank-account/name ?sort-bank-account]] - "check-number" ['[(get-else $ ?e :payment/check-number 0) ?sort-check-number]] - "date" ['[?e :payment/date ?sort-date]] - "amount" ['[?e :payment/amount ?sort-amount]] - "status" ['[?e :payment/status ?sort-status]]} - args) - (:exact-match-id args) - (merge-query {:query {:in ['?e] - :where []} - :args [(:exact-match-id args)]}) + query (if (:exact-match-id args) + (cond-> {:query {:find '[?e] + :in '[$ ?e] + :where '[[?e :payment/client ?c]]} + :args [db + (:exact-match-id args)]} + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :payment/client ?xx]]} + :args [(set (map :db/id (limited-clients (:id args))))]})) + (cond-> {:query {:find [] + :in ['$] + :where []} + :args [db]} + (:sort args) (add-sorter-fields {"client" ['[?e :payment/client ?c] + '[?c :client/name ?sort-client]] + "vendor" ['[?e :payment/vendor ?v] + '[?v :vendor/name ?sort-vendor]] + "bank-account" ['[?e :payment/bank-account ?ba] + '[?ba :bank-account/name ?sort-bank-account]] + "check-number" ['[(get-else $ ?e :payment/check-number 0) ?sort-check-number]] + "date" ['[?e :payment/date ?sort-date]] + "amount" ['[?e :payment/amount ?sort-amount]] + "status" ['[?e :payment/status ?sort-status]]} + args) + (:exact-match-id args) + (merge-query {:query {:in ['?e] + :where []} + :args [(:exact-match-id args)]}) - (limited-clients (:id args)) - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :payment/client ?xx]]} - :args [(set (map :db/id (limited-clients (:id args))))]}) - + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :payment/client ?xx]]} + :args [(set (map :db/id (limited-clients (:id args))))]}) + - (:client-id args) - (merge-query {:query {:in ['?client-id] - :where ['[?e :payment/client ?client-id]]} - :args [(:client-id args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :payment/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [(:client-code args)]}) + (:client-id args) + (merge-query {:query {:in ['?client-id] + :where ['[?e :payment/client ?client-id]]} + :args [(:client-id args)]}) + (:client-code args) + (merge-query {:query {:in ['?client-code] + :where ['[?e :payment/client ?client-id] + '[?client-id :client/code ?client-code]]} + :args [(:client-code args)]}) - - (:vendor-id args) - (merge-query {:query {:in ['?vendor-id] - :where ['[?e :payment/vendor ?vendor-id]]} - :args [(:vendor-id args)]}) + + (:vendor-id args) + (merge-query {:query {:in ['?vendor-id] + :where ['[?e :payment/vendor ?vendor-id]]} + :args [(:vendor-id args)]}) - (:original-id args) - (merge-query {:query {:in ['?original-id] - :where ['[?e :payment/client ?c] - '[?c :client/original-id ?original-id]]} - :args [(:original-id args)]}) + (:original-id args) + (merge-query {:query {:in ['?original-id] + :where ['[?e :payment/client ?c] + '[?c :client/original-id ?original-id]]} + :args [(:original-id args)]}) - (:check-number args) - (merge-query {:query {:in ['?check-number] - :where ['[?e :payment/check-number ?check-number]]} - :args [(:check-number args)]}) + (:check-number args) + (merge-query {:query {:in ['?check-number] + :where ['[?e :payment/check-number ?check-number]]} + :args [(:check-number args)]}) - (not-empty (:invoice-number args)) - (merge-query {:query {:in ['?invoice-number] - :where ['[?e :payment/invoices ?i] - '[?i :invoice/invoice-number ?invoice-number]]} - :args [(:invoice-number args)]}) + (not-empty (:invoice-number args)) + (merge-query {:query {:in ['?invoice-number] + :where ['[?e :payment/invoices ?i] + '[?i :invoice/invoice-number ?invoice-number]]} + :args [(:invoice-number args)]}) - (:bank-account-id args) - (merge-query {:query {:in ['?bank-account-id] - :where ['[?e :payment/bank-account ?bank-account-id]]} - :args [(:bank-account-id args)]}) + (:bank-account-id args) + (merge-query {:query {:in ['?bank-account-id] + :where ['[?e :payment/bank-account ?bank-account-id]]} + :args [(:bank-account-id args)]}) - (:amount-gte args) - (merge-query {:query {:in ['?amount-gte] - :where ['[?e :payment/amount ?a] - '[(>= ?a ?amount-gte)]]} - :args [(:amount-gte args)]}) + (:amount-gte args) + (merge-query {:query {:in ['?amount-gte] + :where ['[?e :payment/amount ?a] + '[(>= ?a ?amount-gte)]]} + :args [(:amount-gte args)]}) - (:amount-lte args) - (merge-query {:query {:in ['?amount-lte] - :where ['[?e :payment/amount ?a] - '[(<= ?a ?amount-lte)]]} - :args [(:amount-lte args)]}) + (:amount-lte args) + (merge-query {:query {:in ['?amount-lte] + :where ['[?e :payment/amount ?a] + '[(<= ?a ?amount-lte)]]} + :args [(:amount-lte args)]}) - (:amount args) - (merge-query {:query {:in ['?amount] - :where ['[?e :payment/amount ?transaction-amount] - '[(iol-ion.query/dollars= ?transaction-amount ?amount)]]} - :args [(:amount args)]}) + (:amount args) + (merge-query {:query {:in ['?amount] + :where ['[?e :payment/amount ?transaction-amount] + '[(iol-ion.query/dollars= ?transaction-amount ?amount)]]} + :args [(:amount args)]}) - - (:status args) - (merge-query {:query {:in ['?status] - :where ['[?e :payment/status ?status]]} - :args [(:status args)]}) - - (:start (:date-range args)) - (merge-query {:query {:in '[?start-date] - :where ['[?e :payment/date ?date] - '[(>= ?date ?start-date)]]} - :args [(c/to-date (:start (:date-range args)))]}) + + (:status args) + (merge-query {:query {:in ['?status] + :where ['[?e :payment/status ?status]]} + :args [(:status args)]}) + + (:start (:date-range args)) + (merge-query {:query {:in '[?start-date] + :where ['[?e :payment/date ?date] + '[(>= ?date ?start-date)]]} + :args [(c/to-date (:start (:date-range args)))]}) - (:end (:date-range args)) - (merge-query {:query {:in '[?end-date] - :where ['[?e :payment/date ?date] - '[(<= ?date ?end-date)]]} - :args [(c/to-date (:end (:date-range args)))]}) + (:end (:date-range args)) + (merge-query {:query {:in '[?end-date] + :where ['[?e :payment/date ?date] + '[(<= ?date ?end-date)]]} + :args [(c/to-date (:end (:date-range args)))]}) - (:payment-type args) - (merge-query {:query {:in '[?payment-type] - :where ['[?e :payment/type ?payment-type]]} - :args [(:payment-type args)]}) + (:payment-type args) + (merge-query {:query {:in '[?payment-type] + :where ['[?e :payment/type ?payment-type]]} + :args [(:payment-type args)]}) - check-number-like - (merge-query {:query {:in '[?check-number-like] - :where ['[?e :payment/check-number ?check-number-like]]} - :args [check-number-like]}) + check-number-like + (merge-query {:query {:in '[?check-number-like] + :where ['[?e :payment/check-number ?check-number-like]]} + :args [check-number-like]}) - true - (merge-query {:query {:find ['?sort-default '?e] - :where ['[?e :payment/date ?sort-default]]}}))] + true + (merge-query {:query {:find ['?sort-default '?e] + :where ['[?e :payment/date ?sort-default]]}})))] (log/info query) diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj index 119c843c..9865ac31 100644 --- a/src/clj/auto_ap/datomic/invoices.clj +++ b/src/clj/auto_ap/datomic/invoices.clj @@ -46,131 +46,138 @@ ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] - (let [query (cond-> {:query {:find [] - :in ['$] - :where ['[?e :invoice/client]]} - :args [db]} + (let [query + (if (:exact-match-id args) + (cond-> {:query {:find '[?e] + :in '[$ ?e] + :where '[[?e :invoice/client ?c]]} + :args [db + (:exact-match-id args)]} + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :invoice/client ?xx]]} + :args [(set (map :db/id (limited-clients (:id args))))]})) + (cond-> {:query {:find [] + :in ['$] + :where ['[?e :invoice/client]]} + :args [db]} - (:exact-match-id args) - (merge-query {:query {:in ['?e] - :where []} - :args [(:exact-match-id args)]}) - (limited-clients (:id args)) - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :invoice/client ?xx]]} - :args [ (set (map :db/id (limited-clients (:id args))))]}) - (:client-id args) - (merge-query {:query {:in ['?client-id] - :where ['[?e :invoice/client ?client-id]]} - :args [ (:client-id args)]}) + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :invoice/client ?xx]]} + :args [ (set (map :db/id (limited-clients (:id args))))]}) + (:client-id args) + (merge-query {:query {:in ['?client-id] + :where ['[?e :invoice/client ?client-id]]} + :args [ (:client-id args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :invoice/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [ (:client-code args)]}) + (:client-code args) + (merge-query {:query {:in ['?client-code] + :where ['[?e :invoice/client ?client-id] + '[?client-id :client/code ?client-code]]} + :args [ (:client-code args)]}) - (:original-id args) - (merge-query {:query {:in ['?original-id] - :where [ - '[?e :invoice/client ?c] - '[?c :client/original-id ?original-id]]} - :args [ (cond-> (:original-id args) - (string? (:original-id args)) Long/parseLong )]}) + (:original-id args) + (merge-query {:query {:in ['?original-id] + :where [ + '[?e :invoice/client ?c] + '[?c :client/original-id ?original-id]]} + :args [ (cond-> (:original-id args) + (string? (:original-id args)) Long/parseLong )]}) - (:start (:date-range args)) (merge-query {:query {:in '[?start-date] + (:start (:date-range args)) (merge-query {:query {:in '[?start-date] + :where ['[?e :invoice/date ?date] + '[(>= ?date ?start-date)]]} + :args [(coerce/to-date (:start (:date-range args)))]}) + + (:end (:date-range args)) (merge-query {:query {:in '[?end-date] :where ['[?e :invoice/date ?date] - '[(>= ?date ?start-date)]]} - :args [(coerce/to-date (:start (:date-range args)))]}) + '[(<= ?date ?end-date)]]} + :args [(coerce/to-date (:end (:date-range args)))]}) - (:end (:date-range args)) (merge-query {:query {:in '[?end-date] - :where ['[?e :invoice/date ?date] - '[(<= ?date ?end-date)]]} - :args [(coerce/to-date (:end (:date-range args)))]}) + (:start (:due-range args)) (merge-query {:query {:in '[?start-due] + :where ['[?e :invoice/due ?due] + '[(>= ?due ?start-due)]]} + :args [(coerce/to-date (:start (:due-range args)))]}) - (:start (:due-range args)) (merge-query {:query {:in '[?start-due] - :where ['[?e :invoice/due ?due] - '[(>= ?due ?start-due)]]} - :args [(coerce/to-date (:start (:due-range args)))]}) + (:end (:due-range args)) (merge-query {:query {:in '[?end-due] + :where ['[?e :invoice/due ?due] + '[(<= ?due ?end-due)]]} + :args [(coerce/to-date (:end (:due-range args)))]}) - (:end (:due-range args)) (merge-query {:query {:in '[?end-due] - :where ['[?e :invoice/due ?due] - '[(<= ?due ?end-due)]]} - :args [(coerce/to-date (:end (:due-range args)))]}) + (:import-status args) + (merge-query {:query {:in ['?import-status] + :where ['[?e :invoice/import-status ?import-status]]} + :args [ (keyword "import-status" (:import-status args))]}) + (:status args) + (merge-query {:query {:in ['?status] + :where ['[?e :invoice/status ?status]]} + :args [ (:status args)]}) + (:vendor-id args) + (merge-query {:query {:in ['?vendor-id] + :where ['[?e :invoice/vendor ?vendor-id]]} + :args [ (:vendor-id args)]}) - (:import-status args) - (merge-query {:query {:in ['?import-status] - :where ['[?e :invoice/import-status ?import-status]]} - :args [ (keyword "import-status" (:import-status args))]}) - (:status args) - (merge-query {:query {:in ['?status] - :where ['[?e :invoice/status ?status]]} - :args [ (:status args)]}) - (:vendor-id args) - (merge-query {:query {:in ['?vendor-id] - :where ['[?e :invoice/vendor ?vendor-id]]} - :args [ (:vendor-id args)]}) + (:account-id args) + (merge-query {:query {:in ['?account-id] + :where ['[?e :invoice/expense-accounts ?iea ?] + '[?iea :invoice-expense-account/account ?account-id]]} + :args [ (:account-id args)]}) - (:account-id args) - (merge-query {:query {:in ['?account-id] - :where ['[?e :invoice/expense-accounts ?iea ?] - '[?iea :invoice-expense-account/account ?account-id]]} - :args [ (:account-id args)]}) + (:amount-gte args) + (merge-query {:query {:in ['?amount-gte] + :where ['[?e :invoice/total ?total-filter] + '[(>= ?total-filter ?amount-gte)]]} + :args [(:amount-gte args)]}) - (:amount-gte args) - (merge-query {:query {:in ['?amount-gte] - :where ['[?e :invoice/total ?total-filter] - '[(>= ?total-filter ?amount-gte)]]} - :args [(:amount-gte args)]}) + (:amount-lte args) + (merge-query {:query {:in ['?amount-lte] + :where ['[?e :invoice/total ?total-filter] + '[(<= ?total-filter ?amount-lte)]]} + :args [(:amount-lte args)]}) - (:amount-lte args) - (merge-query {:query {:in ['?amount-lte] - :where ['[?e :invoice/total ?total-filter] - '[(<= ?total-filter ?amount-lte)]]} - :args [(:amount-lte args)]}) + (seq (:invoice-number-like args)) + (merge-query {:query {:in ['?invoice-number-like] + :where ['[?e :invoice/invoice-number ?invoice-number] + '[(.contains ^String ?invoice-number ?invoice-number-like)]]} + :args [(:invoice-number-like args)]}) - (seq (:invoice-number-like args)) - (merge-query {:query {:in ['?invoice-number-like] - :where ['[?e :invoice/invoice-number ?invoice-number] - '[(.contains ^String ?invoice-number ?invoice-number-like)]]} - :args [(:invoice-number-like args)]}) + (:scheduled-payments args) + (merge-query {:query {:in [] + :where ['[?e :invoice/scheduled-payment]]} + :args []}) - (:scheduled-payments args) - (merge-query {:query {:in [] - :where ['[?e :invoice/scheduled-payment]]} - :args []}) + (:unresolved args) + (merge-query {:query {:in [] + :where ['(or-join [?e] + (not [?e :invoice/expense-accounts ]) + (and [?e :invoice/expense-accounts ?ea] + (not [?ea :invoice-expense-account/account])))]} + :args []}) - (:unresolved args) - (merge-query {:query {:in [] - :where ['(or-join [?e] - (not [?e :invoice/expense-accounts ]) - (and [?e :invoice/expense-accounts ?ea] - (not [?ea :invoice-expense-account/account])))]} - :args []}) + (seq (:location args)) + (merge-query {:query {:in ['?location] + :where ['[?e :invoice/expense-accounts ?eas] + '[?eas :invoice-expense-account/location ?location]]} + :args [(:location args)]}) - (seq (:location args)) - (merge-query {:query {:in ['?location] - :where ['[?e :invoice/expense-accounts ?eas] - '[?eas :invoice-expense-account/location ?location]]} - :args [(:location args)]}) - - (:sort args) (add-sorter-fields {"client" ['[?e :invoice/client ?c] - '[?c :client/name ?sort-client]] - "vendor" ['[?e :invoice/vendor ?v] - '[?v :vendor/name ?sort-vendor]] - "description-original" ['[?e :transaction/description-original ?sort-description-original]] - "location" ['[?e :invoice/expense-accounts ?iea] - '[?iea :invoice-expense-account/location ?sort-location]] - "date" ['[?e :invoice/date ?sort-date]] - "due" ['[(get-else $ ?e :invoice/due #inst "2050-01-01") ?sort-due]] - "invoice-number" ['[?e :invoice/invoice-number ?sort-invoice-number]] - "total" ['[?e :invoice/total ?sort-total]] - "outstanding-balance" ['[?e :invoice/outstanding-balance ?sort-outstanding-balance]]} - args) - true - (merge-query {:query {:find ['?sort-default '?e ] - :where ['[?e :invoice/client] - '[?e :invoice/date ?sort-default]]}}) )] + (:sort args) (add-sorter-fields {"client" ['[?e :invoice/client ?c] + '[?c :client/name ?sort-client]] + "vendor" ['[?e :invoice/vendor ?v] + '[?v :vendor/name ?sort-vendor]] + "description-original" ['[?e :transaction/description-original ?sort-description-original]] + "location" ['[?e :invoice/expense-accounts ?iea] + '[?iea :invoice-expense-account/location ?sort-location]] + "date" ['[?e :invoice/date ?sort-date]] + "due" ['[(get-else $ ?e :invoice/due #inst "2050-01-01") ?sort-due]] + "invoice-number" ['[?e :invoice/invoice-number ?sort-invoice-number]] + "total" ['[?e :invoice/total ?sort-total]] + "outstanding-balance" ['[?e :invoice/outstanding-balance ?sort-outstanding-balance]]} + args) + true + (merge-query {:query {:find ['?sort-default '?e ] + :where ['[?e :invoice/client] + '[?e :invoice/date ?sort-default]]}}) ))] (->> (query2 query) (apply-sort-3 args) (apply-pagination args))))) diff --git a/src/clj/auto_ap/datomic/sales_orders.clj b/src/clj/auto_ap/datomic/sales_orders.clj index 052928fa..a7533a8c 100644 --- a/src/clj/auto_ap/datomic/sales_orders.clj +++ b/src/clj/auto_ap/datomic/sales_orders.clj @@ -28,9 +28,33 @@ (update :expected-deposit first))) cs))))) -(def default-read '[* +(def default-read '[:db/id + :sales-order/external-id, + :sales-order/location, + :sales-order/date, + :sales-order/total, + :sales-order/tax, + :sales-order/tip, + :sales-order/line-items, + :sales-order/discount, + :sales-order/returns, + :sales-order/service-charge, + :sales-order/vendor, + :sales-order/source, + :sales-order/reference-link, {:sales-order/client [:client/name :db/id :client/code] - :sales-order/charges [* {:charge/processor [:db/ident]} {:expected-deposit/_charges [:db/id]}]}]) + :sales-order/charges [ + :charge/type-name, + :charge/total, + :charge/tax, + :charge/tip, + :charge/external-id, + :charge/note, + :charge/date, + :charge/client, + :charge/location, + :charge/reference-link, + {:charge/processor [:db/ident]} {:expected-deposit/_charges [:db/id]}]}]) (defn raw-graphql-ids [db args] (let [visible-clients (visible-clients (:id args)) @@ -140,8 +164,8 @@ (defn get-graphql [args] (let [db (dc/db conn) - {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] - [(->> (graphql-results ids-to-retrieve db args)) + {ids-to-retrieve :ids matching-count :count} (mu/trace ::get-sales-order-ids [] (raw-graphql-ids db args))] + [(->> (mu/trace ::get-results [] (graphql-results ids-to-retrieve db args))) matching-count (summarize-orders ids-to-retrieve)])) diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj index e9b44fbb..3ee05eb5 100644 --- a/src/clj/auto_ap/datomic/transactions.clj +++ b/src/clj/auto_ap/datomic/transactions.clj @@ -42,137 +42,143 @@ ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] (let [potential-duplicates (potential-duplicate-ids db args) - query (cond-> {:query {:find [] - :in ['$ ] - :where []} - :args [db]} + query + (if (:exact-match-id args) + (cond-> {:query {:find '[?e] + :in '[$ ?e] + :where '[[?e :transaction/client ?c]]} + :args [db + (:exact-match-id args)]} + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :transaction/client ?xx]]} + :args [(set (map :db/id (limited-clients (:id args))))]})) + (cond-> {:query {:find [] + :in ['$ ] + :where []} + :args [db]} - (:potential-duplicates args) - (merge-query {:query {:in '[[?e ...]]} - :args [potential-duplicates]}) - - (:exact-match-id args) - (merge-query {:query {:in ['?e] - :where []} - :args [(:exact-match-id args)]}) + (:potential-duplicates args) + (merge-query {:query {:in '[[?e ...]]} + :args [potential-duplicates]}) + + (limited-clients (:id args)) + (merge-query {:query {:in ['[?xx ...]] + :where ['[?e :transaction/client ?xx]]} + :args [(set (map :db/id (limited-clients (:id args))))]}) - (limited-clients (:id args)) - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :transaction/client ?xx]]} - :args [(set (map :db/id (limited-clients (:id args))))]}) + (:bank-account-id args) + (merge-query {:query {:in ['?bank-account-id] + :where ['[?e :transaction/bank-account ?bank-account-id]]} + :args [(:bank-account-id args)]}) - (:bank-account-id args) - (merge-query {:query {:in ['?bank-account-id] - :where ['[?e :transaction/bank-account ?bank-account-id]]} - :args [(:bank-account-id args)]}) + (:import-batch-id args) + (merge-query {:query {:in ['?import-batch-id] + :where ['[?import-batch-id :import-batch/entry ?e]]} + :args [(:import-batch-id args)]}) - (:import-batch-id args) - (merge-query {:query {:in ['?import-batch-id] - :where ['[?import-batch-id :import-batch/entry ?e]]} - :args [(:import-batch-id args)]}) + (:account-id args) + (merge-query {:query {:in ['?account-id] + :where ['[?e :transaction/accounts ?accounts] + '[?accounts :transaction-account/account ?account-id]]} + :args [(:account-id args)]}) - (:account-id args) - (merge-query {:query {:in ['?account-id] - :where ['[?e :transaction/accounts ?accounts] - '[?accounts :transaction-account/account ?account-id]]} - :args [(:account-id args)]}) + (:client-id args) + (merge-query {:query {:in ['?client-id] + :where ['[?e :transaction/client ?client-id]]} + :args [(:client-id args)]}) - (:client-id args) - (merge-query {:query {:in ['?client-id] - :where ['[?e :transaction/client ?client-id]]} - :args [(:client-id args)]}) + (:vendor-id args) + (merge-query {:query {:in ['?vendor-id] + :where ['[?e :transaction/vendor ?vendor-id]]} + :args [(:vendor-id args)]}) - (:vendor-id args) - (merge-query {:query {:in ['?vendor-id] - :where ['[?e :transaction/vendor ?vendor-id]]} - :args [(:vendor-id args)]}) + + (:amount-gte args) + (merge-query {:query {:in ['?amount-gte] + :where ['[?e :transaction/amount ?a] + '[(>= ?a ?amount-gte)]]} + :args [(:amount-gte args)]}) - - (:amount-gte args) - (merge-query {:query {:in ['?amount-gte] - :where ['[?e :transaction/amount ?a] - '[(>= ?a ?amount-gte)]]} - :args [(:amount-gte args)]}) + (:amount-lte args) + (merge-query {:query {:in ['?amount-lte] + :where ['[?e :transaction/amount ?a] + '[(<= ?a ?amount-lte)]]} + :args [(:amount-lte args)]}) - (:amount-lte args) - (merge-query {:query {:in ['?amount-lte] - :where ['[?e :transaction/amount ?a] - '[(<= ?a ?amount-lte)]]} - :args [(:amount-lte args)]}) + (:start (:date-range args)) + (merge-query {:query {:in ['?start-date] + :where ['[?e :transaction/date ?date] + '[(>= ?date ?start-date)]]} + :args [(coerce/to-date (:start (:date-range args)))]}) - (:start (:date-range args)) - (merge-query {:query {:in ['?start-date] - :where ['[?e :transaction/date ?date] - '[(>= ?date ?start-date)]]} - :args [(coerce/to-date (:start (:date-range args)))]}) + (:end (:date-range args)) + (merge-query {:query {:in ['?end-date] + :where ['[?e :transaction/date ?date] + '[(<= ?date ?end-date)]]} + :args [(coerce/to-date (:end (:date-range args)))]}) - (:end (:date-range args)) - (merge-query {:query {:in ['?end-date] - :where ['[?e :transaction/date ?date] - '[(<= ?date ?end-date)]]} - :args [(coerce/to-date (:end (:date-range args)))]}) + (:approval-status args) + (merge-query {:query {:in ['?approval-status] + :where ['[?e :transaction/approval-status ?approval-status]]} + :args [(:approval-status args)]}) - (:approval-status args) - (merge-query {:query {:in ['?approval-status] - :where ['[?e :transaction/approval-status ?approval-status]]} - :args [(:approval-status args)]}) + (:client-code args) + (merge-query {:query {:in ['?client-code] + :where ['[?e :transaction/client ?client-id] + '[?client-id :client/code ?client-code]]} + :args [(:client-code args)]}) + + (:original-id args) + (merge-query {:query {:in ['?original-id] + :where ['[?e :transaction/client ?c] + '[?c :client/original-id ?original-id]]} + :args [(:original-id args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :transaction/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [(:client-code args)]}) - - (:original-id args) - (merge-query {:query {:in ['?original-id] - :where ['[?e :transaction/client ?c] - '[?c :client/original-id ?original-id]]} - :args [(:original-id args)]}) + (seq (:location args)) + (merge-query {:query {:in ['?location] + :where ['[?e :transaction/accounts ?tas] + '[?tas :transaction-account/location ?location]]} + :args [(:location args)]}) - (seq (:location args)) - (merge-query {:query {:in ['?location] - :where ['[?e :transaction/accounts ?tas] - '[?tas :transaction-account/location ?location]]} - :args [(:location args)]}) + (:unresolved args) + (merge-query {:query {:where ['[?e :transaction/date] + '(or-join [?e] + (not [?e :transaction/accounts]) + (and [?e :transaction/accounts ?tas] + (not [?tas :transaction-account/account])))]}}) - (:unresolved args) - (merge-query {:query {:where ['[?e :transaction/date] - '(or-join [?e] - (not [?e :transaction/accounts]) - (and [?e :transaction/accounts ?tas] - (not [?tas :transaction-account/account])))]}}) + (:description args) + (merge-query {:query {:in ['?description] + :where ['[?e :transaction/description-original ?do] + '[(clojure.string/lower-case ?do) ?do2] + '[(.contains ?do2 ?description)]]} + :args [(clojure.string/lower-case (:description args))]}) - (:description args) - (merge-query {:query {:in ['?description] - :where ['[?e :transaction/description-original ?do] - '[(clojure.string/lower-case ?do) ?do2] - '[(.contains ?do2 ?description)]]} - :args [(clojure.string/lower-case (:description args))]}) - - (:sort args) (add-sorter-fields {"client" ['[?e :transaction/client ?c] - '[?c :client/name ?sort-client]] - "account" ['[?e :transaction/date] - '(or-join [?e ?sort-account] - (and [?e :transaction/bank-account ?c] - [?c :bank-account/name ?sort-account]) - (and - (not [?e :transaction/bank-account]) - [(ground "") ?sort-account]))] - "description-original" ['[?e :transaction/description-original ?sort-description-original]] - "date" ['[?e :transaction/date ?sort-date]] - "vendor" ['(or-join [?e ?sort-vendor] - (and [(missing? $ ?e :transaction/vendor)] - [?e :transaction/description-original ?sort-vendor]) - (and [?e :transaction/vendor ?v] - [?v :vendor/name ?sort-vendor]))] - "amount" ['[?e :transaction/amount ?sort-amount]] - "status" ['[?e :transaction/status ?sort-status]]} - args) - true - (merge-query {:query {:find ['?sort-default '?e] - :where ['[?e :transaction/id] - '[?e :transaction/date ?sort-default] - '(not [?e :transaction/approval-status :transaction-approval-status/suppressed])]}}))] + (:sort args) (add-sorter-fields {"client" ['[?e :transaction/client ?c] + '[?c :client/name ?sort-client]] + "account" ['[?e :transaction/date] + '(or-join [?e ?sort-account] + (and [?e :transaction/bank-account ?c] + [?c :bank-account/name ?sort-account]) + (and + (not [?e :transaction/bank-account]) + [(ground "") ?sort-account]))] + "description-original" ['[?e :transaction/description-original ?sort-description-original]] + "date" ['[?e :transaction/date ?sort-date]] + "vendor" ['(or-join [?e ?sort-vendor] + (and [(missing? $ ?e :transaction/vendor)] + [?e :transaction/description-original ?sort-vendor]) + (and [?e :transaction/vendor ?v] + [?v :vendor/name ?sort-vendor]))] + "amount" ['[?e :transaction/amount ?sort-amount]] + "status" ['[?e :transaction/status ?sort-status]]} + args) + true + (merge-query {:query {:find ['?sort-default '?e] + :where ['[?e :transaction/id] + '[?e :transaction/date ?sort-default] + '(not [?e :transaction/approval-status :transaction-approval-status/suppressed])]}})))] (log/info "query is" query) (cond->> (query2 query) true (apply-sort-3 (assoc args :default-asc? false)) diff --git a/src/clj/auto_ap/solr.clj b/src/clj/auto_ap/solr.clj new file mode 100644 index 00000000..dd41d123 --- /dev/null +++ b/src/clj/auto_ap/solr.clj @@ -0,0 +1,32 @@ +(ns auto-ap.solr + (:require [clojure.data.json :as json] + [clj-http.client :as client] + [config.core :refer [env]])) + +(def solr-uri (:solr-uri env)) + +(defn index-documents [xs] + (client/post + (str solr-uri "/solr/invoices/update?commitWithin=15000") + {:headers {"Content-Type" "application/json"} + :method "POST" + :body (json/write-str xs)})) + +(defn query [q] + (-> (client/post (str solr-uri "/solr/invoices/query") + {:body (json/write-str {"query" q + "fields" "id, date, amount, type, description, number, client_code, client_id, vendor_name"}) + :headers {"Content-Type" "application/json"} + :as :json} + ) + :body + :response + :docs)) + +(defn solr-delete [] + (client/post + (str solr-uri "/solr/invoices/update?commitWithin=1000") + {:headers {"Content-Type" "application/json"} + :method "POST" + :body (json/write-str {"delete" {"query" "*:*"}})}) + ) diff --git a/src/clj/auto_ap/ssr/core.clj b/src/clj/auto_ap/ssr/core.clj index 471ad3d2..43e4f2b1 100644 --- a/src/clj/auto_ap/ssr/core.clj +++ b/src/clj/auto_ap/ssr/core.clj @@ -6,6 +6,7 @@ [auto-ap.ssr.auth :as auth] [auto-ap.ssr.transaction.insights :as insights] [auto-ap.ssr.company.company-1099 :as company-1099] + [auto-ap.ssr.search :as search] [auto-ap.ssr.company-dropdown :as company-dropdown])) ;; from auto-ap.ssr-routes, because they're shared @@ -25,5 +26,6 @@ :transaction-insight-table (wrap-client-redirect-unauthenticated (wrap-secure insights/insight-table)) :transaction-insight-rows (wrap-client-redirect-unauthenticated (wrap-secure insights/transaction-rows)) :transaction-insight-approve (wrap-client-redirect-unauthenticated (wrap-secure insights/approve)) - :transaction-insight-explain (wrap-client-redirect-unauthenticated (wrap-secure insights/explain))}) + :transaction-insight-explain (wrap-client-redirect-unauthenticated (wrap-secure insights/explain)) + :search (wrap-client-redirect-unauthenticated (wrap-secure search/dialog-contents))}) diff --git a/src/clj/auto_ap/ssr/search.clj b/src/clj/auto_ap/ssr/search.clj new file mode 100644 index 00000000..9dc8bc50 --- /dev/null +++ b/src/clj/auto_ap/ssr/search.clj @@ -0,0 +1,125 @@ +(ns auto-ap.ssr.search + (:require + [auto-ap.graphql.utils :refer [can-see-client?]] + [auto-ap.ssr.utils :refer [html-response]] + [auto-ap.time :as atime] + [clj-http.client :as client] + [clojure.data.json :as json] + [clojure.string :as str] + [auto-ap.solr :as solr])) + +(defn try-cleanse-date [d] + (try + (or + (some-> (atime/parse-utc d atime/normal-date) (atime/unparse atime/solr-date)) + (some-> (atime/parse-utc d atime/iso-date) (atime/unparse atime/solr-date)) + d) + (catch Exception _ + d))) + +(defn try-parse-number [n] + (if (re-find #"^[\-]?\d+\.\d+$" n ) + (str (with-precision 2 + (some-> n + (Double/parseDouble) + bigdec + (.setScale 2 java.math.RoundingMode/HALF_UP) + (double)))) + n)) + + +(defn q->solr-q [q] + (let [matches (re-seq #"(?:\".*?\"|\S)+" q)] + (str/join " AND " + (->> matches + (map (fn [m] + (cond (= "payment" m) + "type:payment" + + (= "invoice" m) + "type:invoice" + + (= "transaction" m) + "type:transaction" + + :else + (str "_text_:\"" (try-parse-number (try-cleanse-date m)) ""\")))))))) + + +(defn search-results [q id] + (into [] + (filter (fn [d] + (can-see-client? id (first (:client_id d))))) + (solr/query (q->solr-q q)))) + + +(defn search-results* [q id] + (let [results (search-results q id)] + [:div + (if (seq results) + (for [doc results] + [:div.block + [:div.card + [:div.card-header.has-background-info-light + (cond (= "transaction" (:type doc)) + [:div.card-header-icon.icon-task-list-text-1] + + (= "invoice" (:type doc)) + [:div.card-header-icon.icon-accounting-invoice-mail] + + (= "payment" (:type doc)) + [:div.card-header-icon.icon-check-payment-sign]) + [:div.card-header-title (clojure.string/capitalize (:type doc)) + " " + " " + [:span.tags.ml-3 + [:span.tag.is-warning "client: " (:client_code doc)] + [:span.tag.is-info "amount: $" (first (:amount doc))] + (when-let [vendor-name (first (:vendor_name doc))] + [:span.tag.is-primary "vendor: " vendor-name])]] + [:a.card-header-icon.fa.fa-external-link {:href (str "/" (cond (= "invoice" + (:type doc)) + "invoices" + + (= "transaction" + (:type doc)) + "transactions" + + :else + "payments") "/?exact-match-id=" (:id doc)) + :target "_blank"}] + ] + + + [:div.card-content + [:span + + [:strong (atime/unparse (atime/parse (:date doc) atime/solr-date) atime/normal-date)] + ": " + (str (or (first (:description doc)) + (first (:number doc))))]]]] + ) + [:div.block "No results found."])])) + +(defn dialog-contents [request] + (if-let [q (:q (:params request))] + (html-response (search-results* q (:identity request))) + (html-response + [:div#search {:style {:height "400px" :overflow "auto"}} + + [:div.block + [:input#search-input.input {:type "search" + :placeholder "5/5/2034 Magheritas" + :name "q" + :hx-post "/search" + :hx-trigger "keyup changed delay:300ms, search" + :hx-target "#search-results" + :hx-indicator "#search" + :value (:q (:params request)) + :autofocus true}]] + [:style + ".htmx-request #search-results {display: none} .htmx-request .htmx-indicator { display: block !important; }"] + [:div#search-results + ] + [:div.loader.is-loading.big.htmx-indicator {:style {:display "none"}}]]))) + diff --git a/src/clj/auto_ap/ssr/utils.clj b/src/clj/auto_ap/ssr/utils.clj index fb2812b0..189762b8 100644 --- a/src/clj/auto_ap/ssr/utils.clj +++ b/src/clj/auto_ap/ssr/utils.clj @@ -4,9 +4,10 @@ [config.core :refer [env]] [hiccup2.core :as hiccup])) -(defn html-response [hiccup & {:keys [status] :or {status 200}}] +(defn html-response [hiccup & {:keys [status headers] :or {status 200 headers {}}}] {:status status - :headers {"Content-Type" "text/html"} + :headers (into {"Content-Type" "text/html"} + headers) :body (str (hiccup/html {} diff --git a/src/clj/auto_ap/time.clj b/src/clj/auto_ap/time.clj index 996d542d..68920144 100644 --- a/src/clj/auto_ap/time.clj +++ b/src/clj/auto_ap/time.clj @@ -14,9 +14,16 @@ (def normal-date "MM/dd/yyyy") (def iso-date "yyyy-MM-dd") +(def solr-date "yyyy-MM-dd'T'HH:mm:ss'Z'") (def standard-time "MM/dd/yyyy hh:mm aa") +(defn parse-utc [v format] + (try + (f/parse (f/formatter format) v) + (catch Exception _ + nil))) + (defn parse [v format] (try (time/from-time-zone (f/parse (f/formatter format) v) diff --git a/src/clj/user.clj b/src/clj/user.clj index 1cdcab5d..0cf198ad 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -4,6 +4,7 @@ [auto-ap.datomic :refer [conn pull-attr random-tempid]] [auto-ap.ledger :as l ] [clj-http.core :as http] + [clj-http.client :as client] [auto-ap.server] [auto-ap.square.core :as square] [auto-ap.square.core2 :as square2] @@ -24,12 +25,17 @@ [unilog.context :as lc] [com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog.buffer :as rb] - [datomic.api :as d]) + [datomic.api :as d] + [clojure.data.json :as json] + [auto-ap.solr :as solr]) (:import (org.apache.commons.io.input BOMInputStream))) (defn println-event [item] - (printf "%s - %s: %s\n" (:mulog/namespace item) (:mulog/event-name item) + (printf "%s - %s:%s %s\n" (:mulog/namespace item) (:mulog/event-name item) + (if (:mulog/duration item) + (str " " (int (/ (:mulog/duration item) 1000000)) "ms") + "") (pr-str (reduce (fn [acc [k v]] (assoc acc k v)) @@ -554,14 +560,96 @@ :separator \tab)))) +(defn fmt-amount [a] + (with-precision 2 + (some-> a + bigdec + (.setScale 2 java.math.RoundingMode/HALF_UP) + (double)))) + (defn index-solr [] - ) + (doseq [batch (->> (dc/qseq {:query '[:find (pull ?i [:db/id :invoice/invoice-number + :invoice/total + {:invoice/client [:client/code :db/id] + :invoice/vendor [:vendor/name :db/id]} + :invoice/date]) + :in $ + :where [?i :invoice/invoice-number] + (not [?i :invoice/status :invoice-status/voided])] + :args [ + (dc/db conn)]}) + (map (fn [[i]] + {"id" (-> i :db/id) + "client_id" (-> i :invoice/client :db/id) + "client_code" (-> i :invoice/client :client/code) + "date" (some-> i :invoice/date c/to-date-time (atime/unparse atime/iso-date) (str "T00:00:00Z")) + "amount" (-> i :invoice/total fmt-amount) + "number" (-> i :invoice/invoice-number) + "vendor_name" (-> i :invoice/vendor :vendor/name) + "vendor_id" (-> i :invoice/vendor :db/id) + "type" "invoice" + })) + (partition-all 1000))] + (print ".") + (flush) + (solr/index-documents batch)) + + (doseq [batch (->> (dc/qseq {:query '[:find (pull ?i [:db/id :payment/check-number + :payment/amount + {:payment/client [:client/code :db/id] + :payment/vendor [:vendor/name :db/id]} + :payment/date]) + :in $ + :where [?i :payment/date] + (not [?i :payment/status :payment-status/voided])] + :args [(dc/db conn)]}) + (map (fn [[i]] + {"id" (-> i :db/id) + "client_id" (-> i :payment/client :db/id) + "client_code" (-> i :payment/client :client/code) + "date" (some-> i :payment/date c/to-date-time (atime/unparse atime/iso-date) (str "T00:00:00Z")) + "amount" (-> i :payment/amount fmt-amount) + "description" (-> i :payment/check-number) + "vendor_name" (-> i :payment/vendor :vendor/name) + "vendor_id" (-> i :payment/vendor :db/id) + "type" "payment"})) + (partition-all 1000))] + (print ".") + (flush) + (solr/index-documents batch)) + + (doseq [batch (->> (dc/qseq {:query '[:find (pull ?i [:db/id :transaction/description-original + :transaction/amount + {:transaction/client [:client/code :db/id] + :transaction/vendor [:vendor/name :db/id]} + :transaction/date]) + :in $ + :where [?i :transaction/description-original] + (not [?i :transaction/approval-status :transaction-approval-status/suppressed])] + :args [(dc/db conn)]}) + (map (fn [[i]] + {"id" (-> i :db/id) + "client_id" (-> i :transaction/client :db/id) + "client_code" (-> i :transaction/client :client/code) + "date" (some-> i :transaction/date c/to-date-time (atime/unparse atime/iso-date) (str "T00:00:00Z")) + "amount" (-> i :transaction/amount fmt-amount) + "description" (-> i :transaction/description-original) + "vendor_name" (-> i :transaction/vendor :vendor/name) + "vendor_id" (-> i :transaction/vendor :db/id) + "type" "transaction"})) + (partition-all 1000))] + (print ".") + (flush) + (solr/index-documents batch) + )) (defn setup-sales-orders [] (doseq [n (->> (dc/qseq {:query '[:find ?s ?c :where [?s :sales-order/client ?c]] :args [(dc/db auto-ap.datomic/conn)]}) (map (fn [[s c]] {:db/id s :sales-order/client c})) - (partition-all 1000) - (take 10))] + (partition-all 1000))] + (print ".") @(dc/transact auto-ap.datomic/conn n))) + + diff --git a/src/cljc/auto_ap/ssr_routes.cljc b/src/cljc/auto_ap/ssr_routes.cljc index 84abe72f..1a18a21b 100644 --- a/src/cljc/auto_ap/ssr_routes.cljc +++ b/src/cljc/auto_ap/ssr_routes.cljc @@ -1,6 +1,7 @@ (ns auto-ap.ssr-routes) (def routes {"logout" :logout + "search" :search "admin" {"/history" {"" :admin-history "/" :admin-history #"/search/?" :admin-history-search diff --git a/src/cljs/auto_ap/events.cljs b/src/cljs/auto_ap/events.cljs index 74650cef..5f906d12 100644 --- a/src/cljs/auto_ap/events.cljs +++ b/src/cljs/auto_ap/events.cljs @@ -158,6 +158,7 @@ :else {:db (-> db (assoc :active-route handler + :auto-ap.views.components.modal/state nil :page-failure nil :menu nil :query-params params diff --git a/src/cljs/auto_ap/views/components/layouts.cljs b/src/cljs/auto_ap/views/components/layouts.cljs index ecb89f70..b67e0d39 100644 --- a/src/cljs/auto_ap/views/components/layouts.cljs +++ b/src/cljs/auto_ap/views/components/layouts.cljs @@ -4,6 +4,7 @@ [auto-ap.routes :as routes] [auto-ap.subs :as subs] [auto-ap.views.components.modal :as modal] + [auto-ap.views.components.search :as search] [auto-ap.views.components.vendor-dialog :as vendor-dialog] [auto-ap.views.utils :refer [active-when @@ -33,8 +34,7 @@ (println (.-activeElement js/document)) (when-not (.contains @!child (.-activeElement js/document)) (re-frame/dispatch [::events/toggle-menu id]))) - 2)) - } + 2))} [:a {:class "navbar-link login" :on-click (fn [e] (.preventDefault e) (.stopPropagation e) @@ -127,7 +127,7 @@ clients (re-frame/subscribe [::subs/clients]) is-initial-loading @(re-frame/subscribe [::subs/is-initial-loading?])] [:nav {:class "navbar has-shadow is-fixed-top is-grey"} - + [:div {:class "container"} [:div {:class "navbar-brand"} [:a {:class "navbar-item", :href "../"} @@ -166,14 +166,13 @@ (when-not is-initial-loading [:div.navbar-end (when (> (count @clients) 1) - [client-dropdown])])] + [client-dropdown]) + ])] (when-not is-initial-loading [login-dropdown])] - - - ])))) - - + (when (= "admin" (:user/role @user)) + [:div {:style {:position "fixed" :top "4px" :right "8px"}} + [search/search-button ]])])))) (defn appearing-side-bar [{:keys [visible?]} ] @@ -188,6 +187,7 @@ [:div [modal/global-modal] [navbar ap] + [:div {:class "columns has-shadow", :style {:margin-bottom "0px" :height "calc(100vh - 46px)" } :id "mail-app" } [:aside {:class "column aside menu is-2 " } [:div.main.left-nav diff --git a/src/cljs/auto_ap/views/components/search.cljs b/src/cljs/auto_ap/views/components/search.cljs new file mode 100644 index 00000000..e276f7fa --- /dev/null +++ b/src/cljs/auto_ap/views/components/search.cljs @@ -0,0 +1,30 @@ +(ns auto-ap.views.components.search + (:require + [auto-ap.views.utils :refer [dispatch-event]] + [auto-ap.views.components.modal :as modal] + [reagent.core :as reagent])) + +(defn search-dialog-body [] + (let [a (atom nil)] + (reagent/create-class + { + :component-did-mount + (fn [this] + (.process js/htmx @a)) + :should-component-update + (fn [this] + false) + :reagent-render + (fn [] + [:div {:ref (fn [x] (reset! a x)) + "hx-get" "/search" + "hx-trigger" "load"} + ""])}))) + +(defn search-button [] + [:i.fa.fa-search.button {:on-click + (dispatch-event [::modal/modal-requested + {:title "Search" + :body [search-dialog-body] + :class "semi-wide"}])} + #_[:i.fa.fa-search]]) diff --git a/src/cljs/auto_ap/views/pages/transactions.cljs b/src/cljs/auto_ap/views/pages/transactions.cljs index fcce5d4d..07f93fbd 100644 --- a/src/cljs/auto_ap/views/pages/transactions.cljs +++ b/src/cljs/auto_ap/views/pages/transactions.cljs @@ -190,7 +190,8 @@ (defn content [] - (let [is-admin? @(re-frame/subscribe [::subs/is-admin?])] + (let [is-admin? @(re-frame/subscribe [::subs/is-admin?]) + _ @(re-frame/subscribe [::data-page/params ::page])] [:div [:h1.title "Transactions"] [status/status-notification {:statuses [[::status/single ::delete-selected] diff --git a/terraform/connect-ports-cloud.sh b/terraform/connect-ports-cloud.sh new file mode 100755 index 00000000..d10e3f57 --- /dev/null +++ b/terraform/connect-ports-cloud.sh @@ -0,0 +1,2 @@ +#!/bin/sh +ssh -L 2049:172.31.38.113:2049 3.213.115.86 -L 8983:solr-prod-cloud.local:8983 -L 4334:datomic-iol-dev.local:4334 diff --git a/terraform/deploy.tf b/terraform/deploy.tf index fe8b7056..eccaf9bd 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -55,15 +55,10 @@ resource "aws_ecs_service" "integreat_app" { } capacity_provider_strategy { - base = 0 + base = 1 capacity_provider = "FARGATE_SPOT" weight = 5 } - capacity_provider_strategy { - base = 1 - capacity_provider = "FARGATE" - weight = 1 - } deployment_circuit_breaker { enable = false @@ -457,7 +452,6 @@ module "load_historical_sales_job" { memory = 4096 cpu = 1024 } -*/ module "restore_from_backup_job" { source = "./background-job/" @@ -483,3 +477,4 @@ module "ntg_job" { memory = 4096 cpu = 1024 } +*/ diff --git a/terraform/prod-cloud-solr-taskdef.json b/terraform/prod-cloud-solr-taskdef.json new file mode 100644 index 00000000..2e98e74f --- /dev/null +++ b/terraform/prod-cloud-solr-taskdef.json @@ -0,0 +1,62 @@ +[ + { + "environment": [ + { + "name": "DD_ENV", + "value": "prod-cloud" + }, + { + "name": "DD_SERVICE", + "value": "solr" + } + ], + "essential": true, + "image": "solr", + + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/solr-prod-cloud", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } + }, + "dockerLabels": { + "com.datadoghq.tags.env": "prod-cloud", + "com.datadoghq.tags.service": "solr" + }, + "mountPoints": [], + "name": "solr", + "portMappings": [ + { + "containerPort": 8983, + "hostPort": 8983, + "protocol": "tcp" + } + ], + "volumesFrom": [], + "mountPoints": [ + { + "sourceVolume": "solr-storage", + "containerPath": "/var/solr", + "readOnly": false + } + ] + }, + { + "environment": [ + { + "name": "DD_API_KEY", + "value": "ce10d932c47b358e81081ae67bd8c112" + }, + { + "name": "ECS_FARGATE", + "value": "true" + } + ], + "essential": true, + "image": "public.ecr.aws/datadog/agent:latest", + "name": "datadog-agent" + } +] + diff --git a/terraform/solr.tf b/terraform/solr.tf new file mode 100644 index 00000000..cc3d37d2 --- /dev/null +++ b/terraform/solr.tf @@ -0,0 +1,97 @@ +resource "aws_efs_file_system" "solr_storage" { + creation_token = "solr_storage-${var.stage}" + + tags = { + Name = "solr_storage_${var.stage}" + } +} + +resource "aws_ecs_task_definition" "solr" { + family = "solr_${var.stage}" + container_definitions = file("${var.stage}-solr-taskdef.json") + memory = 4096 + cpu = 1024 + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = var.execution_role_arn + task_role_arn = var.task_role_arn + + volume { + name = "solr-storage" + + efs_volume_configuration { + file_system_id = aws_efs_file_system.solr_storage.id + root_directory = "/" + /* + authorization_config { + access_point_id = aws_efs_access_point.test.id + iam = "ENABLED" + } + */ + } + } +} + + +resource "aws_ecs_service" "solr" { + name = "solr_app_${var.stage}" + cluster = var.ecs_cluster + task_definition = aws_ecs_task_definition.solr.arn + desired_count = 1 + deployment_controller { + type = "ECS" + } + scheduling_strategy = "REPLICA" + platform_version = "LATEST" + + network_configuration { + assign_public_ip = true + security_groups = [ "sg-004e5855310c453a3", "sg-02d167406b1082698"] + subnets = [ "subnet-5e675761", "subnet-8519fde2", "subnet-89bab8d4" ] + } + + service_registries { + container_port = 0 + port = 0 + registry_arn = aws_service_discovery_service.solr.arn + } + + capacity_provider_strategy { + base = 1 + capacity_provider = "FARGATE_SPOT" + weight = 5 + } + + deployment_circuit_breaker { + enable = false + rollback = false + } + + wait_for_steady_state = true + + + timeouts {} + lifecycle { + ignore_changes = [task_definition] + } +} + +resource "aws_service_discovery_service" "solr" { + name = "solr-${var.stage}" + + dns_config { + namespace_id = var.local_namespace + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } + + health_check_custom_config { + failure_threshold = 1 + } +} + diff --git a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate index d189d7e1..ead001d0 100644 --- a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate +++ b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.3.8", - "serial": 78, + "terraform_version": "1.4.3", + "serial": 104, "lineage": "cf731bb4-8fb3-47af-6e29-22030e089d96", "outputs": { "aws_access_key_id": { @@ -94,14 +94,9 @@ "attributes": { "capacity_provider_strategy": [ { - "base": 0, + "base": 1, "capacity_provider": "FARGATE_SPOT", "weight": 5 - }, - { - "base": 1, - "capacity_provider": "FARGATE", - "weight": 1 } ], "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", @@ -164,7 +159,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:17", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:51", "timeouts": { "delete": null }, @@ -180,6 +175,90 @@ } ] }, + { + "mode": "managed", + "type": "aws_ecs_service", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "capacity_provider_strategy": [ + { + "base": 1, + "capacity_provider": "FARGATE_SPOT", + "weight": 5 + } + ], + "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "deployment_circuit_breaker": [ + { + "enable": false, + "rollback": false + } + ], + "deployment_controller": [ + { + "type": "ECS" + } + ], + "deployment_maximum_percent": 200, + "deployment_minimum_healthy_percent": 100, + "desired_count": 1, + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "force_new_deployment": null, + "health_check_grace_period_seconds": 0, + "iam_role": "aws-service-role", + "id": "arn:aws:ecs:us-east-1:679918342773:service/default/solr_app_prod-cloud", + "launch_type": "", + "load_balancer": [], + "name": "solr_app_prod-cloud", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "ordered_placement_strategy": [], + "placement_constraints": [], + "platform_version": "LATEST", + "propagate_tags": "NONE", + "scheduling_strategy": "REPLICA", + "service_registries": [ + { + "container_name": "", + "container_port": 0, + "port": 0, + "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-smnd6gtc2jtbnkvu" + } + ], + "tags": null, + "tags_all": {}, + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/solr_prod-cloud:3", + "timeouts": { + "delete": null + }, + "wait_for_steady_state": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_ecs_task_definition.solr", + "aws_service_discovery_service.solr" + ] + } + ] + }, { "mode": "managed", "type": "aws_ecs_task_definition", @@ -218,6 +297,105 @@ } ] }, + { + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/solr_prod-cloud:3", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"solr\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"solr\"}],\"essential\":true,\"image\":\"solr\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/solr-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[{\"containerPath\":\"/var/solr\",\"readOnly\":false,\"sourceVolume\":\"solr-storage\"}],\"name\":\"solr\",\"portMappings\":[{\"containerPort\":8983,\"hostPort\":8983,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "solr_prod-cloud", + "id": "solr_prod-cloud", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "4096", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 3, + "runtime_platform": [], + "tags": {}, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [ + { + "docker_volume_configuration": [], + "efs_volume_configuration": [ + { + "authorization_config": [], + "file_system_id": "fs-0a72af98fd255b75e", + "root_directory": "/", + "transit_encryption": "", + "transit_encryption_port": 0 + } + ], + "fsx_windows_file_server_volume_configuration": [], + "host_path": "", + "name": "solr-storage" + } + ] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_efs_file_system.solr_storage" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_efs_file_system", + "name": "solr_storage", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:elasticfilesystem:us-east-1:679918342773:file-system/fs-0a72af98fd255b75e", + "availability_zone_id": "", + "availability_zone_name": "", + "creation_token": "solr_storage-prod-cloud", + "dns_name": "fs-0a72af98fd255b75e.efs.us-east-1.amazonaws.com", + "encrypted": false, + "id": "fs-0a72af98fd255b75e", + "kms_key_id": "", + "lifecycle_policy": [], + "number_of_mount_targets": 6, + "owner_id": "679918342773", + "performance_mode": "generalPurpose", + "provisioned_throughput_in_mibps": 0, + "size_in_bytes": [ + { + "value": 6739968, + "value_in_ia": 0, + "value_in_standard": 6739968 + } + ], + "tags": { + "Name": "solr_storage_prod-cloud" + }, + "tags_all": { + "Name": "solr_storage_prod-cloud" + }, + "throughput_mode": "bursting" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_access_key", @@ -634,7 +812,7 @@ "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], - "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}", + "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", @@ -900,6 +1078,47 @@ } ] }, + { + "mode": "managed", + "type": "aws_service_discovery_service", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-smnd6gtc2jtbnkvu", + "description": "", + "dns_config": [ + { + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "namespace_id": "ns-gv2z744em7myo2jp", + "routing_policy": "MULTIVALUE" + } + ], + "force_destroy": false, + "health_check_config": [], + "health_check_custom_config": [ + { + "failure_threshold": 1 + } + ], + "id": "srv-smnd6gtc2jtbnkvu", + "name": "solr-prod-cloud", + "namespace_id": "ns-gv2z744em7myo2jp", + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_ses_receipt_rule", @@ -1075,45 +1294,6 @@ "private": "bnVsbA==" } ] - }, - { - "module": "module.restore_from_backup_job", - "mode": "managed", - "type": "aws_ecs_task_definition", - "name": "background_taskdef", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/restore_from_backup_prod_cloud:3", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"restore-from-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"restore-from-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"restore-from-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-cloud-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat-cloud:prod-cloud\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-app-prod\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", - "cpu": "4096", - "ephemeral_storage": [], - "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", - "family": "restore_from_backup_prod_cloud", - "id": "restore_from_backup_prod_cloud", - "inference_accelerator": [], - "ipc_mode": "", - "memory": "8192", - "network_mode": "awsvpc", - "pid_mode": "", - "placement_constraints": [], - "proxy_configuration": [], - "requires_compatibilities": [ - "FARGATE" - ], - "revision": 3, - "runtime_platform": [], - "tags": null, - "tags_all": {}, - "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", - "volume": [] - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] } ], "check_results": null diff --git a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup index b581f823..d6a67121 100644 --- a/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup +++ b/terraform/terraform.tfstate.d/prod-cloud/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.3.8", - "serial": 72, + "terraform_version": "1.4.3", + "serial": 98, "lineage": "cf731bb4-8fb3-47af-6e29-22030e089d96", "outputs": { "aws_access_key_id": { @@ -94,14 +94,9 @@ "attributes": { "capacity_provider_strategy": [ { - "base": 0, + "base": 1, "capacity_provider": "FARGATE_SPOT", "weight": 5 - }, - { - "base": 1, - "capacity_provider": "FARGATE", - "weight": 1 } ], "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", @@ -164,7 +159,7 @@ ], "tags": {}, "tags_all": {}, - "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:16", + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/integreat_app_prod-cloud:51", "timeouts": { "delete": null }, @@ -180,6 +175,91 @@ } ] }, + { + "mode": "managed", + "type": "aws_ecs_service", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "capacity_provider_strategy": [ + { + "base": 1, + "capacity_provider": "FARGATE_SPOT", + "weight": 5 + } + ], + "cluster": "arn:aws:ecs:us-east-1:679918342773:cluster/default", + "deployment_circuit_breaker": [ + { + "enable": false, + "rollback": false + } + ], + "deployment_controller": [ + { + "type": "ECS" + } + ], + "deployment_maximum_percent": 200, + "deployment_minimum_healthy_percent": 100, + "desired_count": 1, + "enable_ecs_managed_tags": false, + "enable_execute_command": false, + "force_new_deployment": null, + "health_check_grace_period_seconds": 0, + "iam_role": "aws-service-role", + "id": "arn:aws:ecs:us-east-1:679918342773:service/default/solr_app_prod-cloud", + "launch_type": "", + "load_balancer": [], + "name": "solr_app_prod-cloud", + "network_configuration": [ + { + "assign_public_ip": true, + "security_groups": [ + "sg-004e5855310c453a3", + "sg-02d167406b1082698" + ], + "subnets": [ + "subnet-5e675761", + "subnet-8519fde2", + "subnet-89bab8d4" + ] + } + ], + "ordered_placement_strategy": [], + "placement_constraints": [], + "platform_version": "LATEST", + "propagate_tags": "NONE", + "scheduling_strategy": "REPLICA", + "service_registries": [ + { + "container_name": "", + "container_port": 0, + "port": 0, + "registry_arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-cmwdohq6dvf3pbjv" + } + ], + "tags": null, + "tags_all": {}, + "task_definition": "arn:aws:ecs:us-east-1:679918342773:task-definition/solr_prod-cloud:3", + "timeouts": { + "delete": null + }, + "wait_for_steady_state": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxMjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_ecs_task_definition.solr", + "aws_efs_file_system.solr_storage", + "aws_service_discovery_service.service" + ] + } + ] + }, { "mode": "managed", "type": "aws_ecs_task_definition", @@ -218,6 +298,105 @@ } ] }, + { + "mode": "managed", + "type": "aws_ecs_task_definition", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/solr_prod-cloud:3", + "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"solr\"},\"environment\":[{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"solr\"}],\"essential\":true,\"image\":\"solr\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/solr-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[{\"containerPath\":\"/var/solr\",\"readOnly\":false,\"sourceVolume\":\"solr-storage\"}],\"name\":\"solr\",\"portMappings\":[{\"containerPort\":8983,\"hostPort\":8983,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", + "cpu": "1024", + "ephemeral_storage": [], + "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", + "family": "solr_prod-cloud", + "id": "solr_prod-cloud", + "inference_accelerator": [], + "ipc_mode": "", + "memory": "4096", + "network_mode": "awsvpc", + "pid_mode": "", + "placement_constraints": [], + "proxy_configuration": [], + "requires_compatibilities": [ + "FARGATE" + ], + "revision": 3, + "runtime_platform": [], + "tags": null, + "tags_all": {}, + "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", + "volume": [ + { + "docker_volume_configuration": [], + "efs_volume_configuration": [ + { + "authorization_config": [], + "file_system_id": "fs-0a72af98fd255b75e", + "root_directory": "/", + "transit_encryption": "", + "transit_encryption_port": null + } + ], + "fsx_windows_file_server_volume_configuration": [], + "host_path": "", + "name": "solr-storage" + } + ] + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_efs_file_system.solr_storage" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_efs_file_system", + "name": "solr_storage", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:elasticfilesystem:us-east-1:679918342773:file-system/fs-0a72af98fd255b75e", + "availability_zone_id": "", + "availability_zone_name": "", + "creation_token": "solr_storage-prod-cloud", + "dns_name": "fs-0a72af98fd255b75e.efs.us-east-1.amazonaws.com", + "encrypted": false, + "id": "fs-0a72af98fd255b75e", + "kms_key_id": "", + "lifecycle_policy": [], + "number_of_mount_targets": 1, + "owner_id": "679918342773", + "performance_mode": "generalPurpose", + "provisioned_throughput_in_mibps": 0, + "size_in_bytes": [ + { + "value": 6144, + "value_in_ia": 0, + "value_in_standard": 6144 + } + ], + "tags": { + "Name": "solr_storage_prod-cloud" + }, + "tags_all": { + "Name": "solr_storage_prod-cloud" + }, + "throughput_mode": "bursting" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_access_key", @@ -634,7 +813,7 @@ "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], - "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"}],\"Version\":\"2012-10-17\"}", + "policy": "{\"Id\":\"Policy1526084187222\",\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com/*\",\"Sid\":\"Stmt1526084185514\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/datomic-ddb\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProd\"},{\"Action\":\"s3:*\",\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::679918342773:role/http-proxy\"},\"Resource\":\"arn:aws:s3:::data.prod-cloud.app.integreatconsult.com\",\"Sid\":\"AllowReadForProdProxy\"}],\"Version\":\"2012-10-17\"}", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", @@ -900,6 +1079,47 @@ } ] }, + { + "mode": "managed", + "type": "aws_service_discovery_service", + "name": "solr", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:servicediscovery:us-east-1:679918342773:service/srv-smnd6gtc2jtbnkvu", + "description": "", + "dns_config": [ + { + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "namespace_id": "ns-gv2z744em7myo2jp", + "routing_policy": "MULTIVALUE" + } + ], + "force_destroy": false, + "health_check_config": [], + "health_check_custom_config": [ + { + "failure_threshold": 1 + } + ], + "id": "srv-smnd6gtc2jtbnkvu", + "name": "solr-prod-cloud", + "namespace_id": "ns-gv2z744em7myo2jp", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_ses_receipt_rule", @@ -1075,45 +1295,6 @@ "private": "bnVsbA==" } ] - }, - { - "module": "module.restore_from_backup_job", - "mode": "managed", - "type": "aws_ecs_task_definition", - "name": "background_taskdef", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ecs:us-east-1:679918342773:task-definition/restore_from_backup_prod_cloud:2", - "container_definitions": "[{\"cpu\":0,\"dockerLabels\":{\"com.datadoghq.tags.env\":\"prod-cloud\",\"com.datadoghq.tags.service\":\"restore-from-backup\"},\"environment\":[{\"name\":\"DD_CONTAINER_ENV_AS_TAGS\",\"value\":\"{\\\"INTEGREAT_JOB\\\":\\\"background_job\\\"}\"},{\"name\":\"DD_ENV\",\"value\":\"prod-cloud\"},{\"name\":\"DD_SERVICE\",\"value\":\"restore-from-backup\"},{\"name\":\"INTEGREAT_JOB\",\"value\":\"restore-from-backup\"},{\"name\":\"config\",\"value\":\"/usr/local/config/prod-cloud-background-worker.edn\"}],\"essential\":true,\"image\":\"679918342773.dkr.ecr.us-east-1.amazonaws.com/integreat-cloud:prod-cloud\",\"logConfiguration\":{\"logDriver\":\"awslogs\",\"options\":{\"awslogs-group\":\"/ecs/integreat-background-worker-prod-cloud\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"}},\"mountPoints\":[],\"name\":\"integreat-app\",\"portMappings\":[{\"containerPort\":9000,\"hostPort\":9000,\"protocol\":\"tcp\"},{\"containerPort\":9090,\"hostPort\":9090,\"protocol\":\"tcp\"}],\"volumesFrom\":[]},{\"cpu\":0,\"environment\":[{\"name\":\"DD_API_KEY\",\"value\":\"ce10d932c47b358e81081ae67bd8c112\"},{\"name\":\"ECS_FARGATE\",\"value\":\"true\"}],\"essential\":true,\"image\":\"public.ecr.aws/datadog/agent:latest\",\"mountPoints\":[],\"name\":\"datadog-agent\",\"portMappings\":[],\"volumesFrom\":[]}]", - "cpu": "4096", - "ephemeral_storage": [], - "execution_role_arn": "arn:aws:iam::679918342773:role/ecsTaskExecutionRole", - "family": "restore_from_backup_prod_cloud", - "id": "restore_from_backup_prod_cloud", - "inference_accelerator": [], - "ipc_mode": "", - "memory": "8192", - "network_mode": "awsvpc", - "pid_mode": "", - "placement_constraints": [], - "proxy_configuration": [], - "requires_compatibilities": [ - "FARGATE" - ], - "revision": 2, - "runtime_platform": [], - "tags": null, - "tags_all": {}, - "task_role_arn": "arn:aws:iam::679918342773:role/datomic-ddb", - "volume": [] - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] } ], "check_results": null