diff --git a/iol_ion/src/iol_ion/query.clj b/iol_ion/src/iol_ion/query.clj index cce48832..51ad7cd2 100644 --- a/iol_ion/src/iol_ion/query.clj +++ b/iol_ion/src/iol_ion/query.clj @@ -4,6 +4,10 @@ [clj-time.format :as f] [datomic.api :as dc])) +;; TODO WILL NOT WORK IN DATOMIC CLOUD +(defn entid [db i] + (dc/entid db i)) + #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn dollars-0? [amt] (< -0.001 amt 0.001)) @@ -61,3 +65,39 @@ (.toInstant) (.atZone (java.time.ZoneId/of "US/Pacific")) (.get java.time.temporal.ChronoField/DAY_OF_MONTH))) + +(defn scan-invoices [db clients start end] + (for [c clients + :let [c (entid db c)] + r (seq (dc/index-range db + :invoice/client+date + [c (or start #inst "2001-01-01T08:00:00.000-00:00") ] + [c (or end #inst "2030-03-05T08:00:00.000-00:00") ]))] + [(:e r) (first (:v r)) (second (:v r))])) + +(defn scan-transactions [db clients start end] + (for [c clients + :let [c (entid db c)] + r (seq (dc/index-range db + :transaction/client+date + [c (or start #inst "2001-01-01T08:00:00.000-00:00") ] + [c (or end #inst "2030-03-05T08:00:00.000-00:00") ]))] + [(:e r) (first (:v r)) (second (:v r))])) + +(defn scan-ledger [db clients start end] + (for [c clients + :let [c (entid db c)] + r (seq (dc/index-range db + :journal-entry/client+date + [c (or start #inst "2001-01-01T08:00:00.000-00:00") ] + [c (or end #inst "2030-03-05T08:00:00.000-00:00") ]))] + [(:e r) (first (:v r)) (second (:v r))])) + +(defn scan-payments [db clients start end] + (for [c clients + :let [c (entid db c)] + r (seq (dc/index-range db + :payment/client+date + [c (or start #inst "2001-01-01T08:00:00.000-00:00") ] + [c (or end #inst "2030-03-05T08:00:00.000-00:00") ]))] + [(:e r) (first (:v r)) (second (:v r))])) diff --git a/iol_ion/src/iol_ion/tx.clj b/iol_ion/src/iol_ion/tx.clj index 8ad133db..133c402d 100644 --- a/iol_ion/src/iol_ion/tx.clj +++ b/iol_ion/src/iol_ion/tx.clj @@ -36,9 +36,6 @@ - - - (defn regenerate-literals [] (require 'com.github.ivarref.gen-fn) (spit diff --git a/package-lock.json b/package-lock.json index 4018dafc..ae80585f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "dropzone": "^4.3.0", "flowbite": "^1.6.5", "minisearch": "^3.0.2", + "pako": "^2.1.0", "prop-types": "^15.7.2", "react": "^17.0.1", "react-dom": "^17.0.1", @@ -1674,6 +1675,11 @@ "node": ">=6" } }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4159,6 +4165,11 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, + "pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==" + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", diff --git a/package.json b/package.json index 8c4c0249..aa99e048 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dropzone": "^4.3.0", "flowbite": "^1.6.5", "minisearch": "^3.0.2", + "pako": "^2.1.0", "prop-types": "^15.7.2", "react": "^17.0.1", "react-dom": "^17.0.1", diff --git a/resources/schema.edn b/resources/schema.edn index 90ff4027..287f359d 100644 --- a/resources/schema.edn +++ b/resources/schema.edn @@ -1782,4 +1782,30 @@ :db/valueType :db.type/tuple :db/doc "The recommended vendor, account, and transaction count, and whether seen by the client," :db/tupleTypes [:db.type/ref :db.type/ref :db.type/long :db.type/boolean] - :db/cardinality :db.cardinality/many}] + :db/cardinality :db.cardinality/many} + + {:db/ident :invoice/client+date + :db/valueType :db.type/tuple + :db/tupleAttrs [ :invoice/client :invoice/date] + :db/cardinality :db.cardinality/one + :db/index true} + + {:db/ident :transaction/client+date + :db/valueType :db.type/tuple + :db/tupleAttrs [ :transaction/client :transaction/date] + :db/cardinality :db.cardinality/one + :db/index true} + + {:db/ident :journal-entry/client+date + :db/valueType :db.type/tuple + :db/tupleAttrs [ :journal-entry/client :journal-entry/date] + :db/cardinality :db.cardinality/one + :db/index true} + + {:db/ident :payment/client+date + :db/valueType :db.type/tuple + :db/tupleAttrs [ :payment/client :payment/date] + :db/cardinality :db.cardinality/one + :db/index true}] + + diff --git a/scratch-sessions/fix-page-performance.repl b/scratch-sessions/fix-page-performance.repl new file mode 100644 index 00000000..6a626bef --- /dev/null +++ b/scratch-sessions/fix-page-performance.repl @@ -0,0 +1,168 @@ +;; This buffer is for Clojure experiments and evaluation. + +;; Press C-j to evaluate the last expression. + +;; You can also press C-u C-j to evaluate the expression and pretty-print its result. + + +'{:query , + :phases [{:sched (([(ground $__in__2) ?start-date] [?e :invoice/date ?date] [(>= ?date ?start-date)] [(ground $__in__3) [?xx ...]] [?e :invoice/client ?xx] [?e :invoice/client] [?e :invoice/date ?sort-default])), :clauses [{:clause [(ground $__in__2) ?start-date], :rows-in 0, :rows-out 1, :binds-in (), :binds-out [?start-date], :expansion 1} {:clause [?e :invoice/date ?date], :rows-in 1, :rows-out 9656, :binds-in [?start-date], :binds-out [?date ?start-date ?e], :preds ([(>= ?date ?start-date)]), :expansion 9655, :warnings {:unbound-vars #{?date ?e}}} {:clause [(ground $__in__3) [?xx ...]], :rows-in 9656, :rows-out 1303560, :binds-in [?date ?start-date ?e], :binds-out [?xx ?e], :expansion 1293904} {:clause [?e :invoice/client ?xx], :rows-in 1303560, :rows-out 9656, :binds-in [?xx ?e], :binds-out [?e]} {:clause [?e :invoice/client], :rows-in 9656, :rows-out 9656, :binds-in [?e], :binds-out [?e]} {:clause [?e :invoice/date ?sort-default], :rows-in 9656, :rows-out 9656, :binds-in [?e], :binds-out [?sort-default ?e]}]}]} +;; => {:query +;; {:find [?sort-default ?e], +;; :in [$ ?start-date [?xx ...]], +;; :where +;; [[?e :invoice/date ?date] +;; [(>= ?date ?start-date)] +;; [?e :invoice/client ?xx] +;; [?e :invoice/client] +;; [?e :invoice/date ?sort-default]]}, +;; :phases +;; [{:sched +;; (([(ground $__in__2) ?start-date] +;; [?e :invoice/date ?date] +;; [(>= ?date ?start-date)] +;; [(ground $__in__3) [?xx ...]] +;; [?e :invoice/client ?xx] +;; [?e :invoice/client] +;; [?e :invoice/date ?sort-default])), +;; :clauses +;; [{:clause [(ground $__in__2) ?start-date], +;; :rows-in 0, +;; :rows-out 1, +;; :binds-in (), +;; :binds-out [?start-date], +;; :expansion 1} +;; {:clause [?e :invoice/date ?date], +;; :rows-in 1, +;; :rows-out 9656, +;; :binds-in [?start-date], +;; :binds-out [?date ?start-date ?e], +;; :preds ([(>= ?date ?start-date)]), +;; :expansion 9655, +;; :warnings {:unbound-vars #{?date ?e}}} +;; {:clause [(ground $__in__3) [?xx ...]], +;; :rows-in 9656, +;; :rows-out 1303560, +;; :binds-in [?date ?start-date ?e], +;; :binds-out [?xx ?e], +;; :expansion 1293904} +;; {:clause [?e :invoice/client ?xx], +;; :rows-in 1303560, +;; :rows-out 9656, +;; :binds-in [?xx ?e], +;; :binds-out [?e]} +;; {:clause [?e :invoice/client], +;; :rows-in 9656, +;; :rows-out 9656, +;; :binds-in [?e], +;; :binds-out [?e]} +;; {:clause [?e :invoice/date ?sort-default], +;; :rows-in 9656, +;; :rows-out 9656, +;; :binds-in [?e], +;; :binds-out [?sort-default ?e]}]}]} + + + +(time + (count + (observable-query + {:query {:find '[?sort-default ?e], + :in '[$ [?xx ...]], + :where '[[(untuple ?xx) [?target-date ?target-client]] + [(tuple #inst "2030-01-01" ?target-client) ?xx2] + [?e :invoice/date+client ?dc] + #_[(untuple)] + [(>= ?dc ?xx)] + [(<= ?dc ?xx2)] + [(untuple ?dc) [?sort-default]] + #_[?e :invoice/date ?date] + #_[(>= ?date ?start-date)] + #_[?e :invoice/client ?xx] + + #_[?e :invoice/date ?sort-default]]} + :args [(dc/db conn) g]}))) + + + + + +(defn recent-invoices [clients start end] + (for [ c clients + r (seq (dc/index-range (dc/db conn) + :invoice/client+date + [c (or start #inst "2001-01-01T08:00:00.000-00:00") ] + [c (or end #inst "2030-03-05T08:00:00.000-00:00") ]))] + (:e r))) + +(recent-invoices all-clients + #inst "2023-03-05T08:00:00.000-00:00" + #inst "2050-03-05T08:00:00.000-00:00") + +(count + (dc/query {:query '[:find ?e + :in $ [?clients ?start ?end] + :where [(auto-ap.datomic.invoices/recent-invoices ?clients ?start ?end) [?e ...]]] + :args [(dc/db conn) + [all-clients + #inst "2023-03-05T08:00:00.000-00:00" + nil]]} + )) + +(time + (count + (for [ c all-clients + r (seq (dc/index-range (dc/db conn) + :invoice/client+date + [c #inst "2023-03-05T08:00:00.000-00:00" ] + [c #inst "2030-03-05T08:00:00.000-00:00" ]))] + r))) + +(take 500 (dc/index-range (dc/db conn) + :invoice/client+date + [17592186046356 #inst "2021-03-05T08:00:00.000-00:00" ] + [17592186046356 #inst "2053-03-05T08:00:00.000-00:00" ] + )) + +(def g + (map vector (repeat #inst "2023-03-05T08:00:00.000-00:00") + )) + +(def all-clients [17592186046356 17592186046363 17592186046366 17592186046371 17592186046373 17592186046378 17592186046380 17592186046385 17592186046394 17592186046397 17592186046400 17592186046404 17592186046409 17592186046415 17592186046422 17592186046428 17592186046437 17592186046447 17592186046456 17592186046465 17592186046472 17592186046478 17592186046491 17592186046499 17592186046513 17592186046520 17592186109023 17592193806897 17592195665346 17592196580407 17592203465691 17592204394806 17592219470393 17592219675333 17592221275294 17592232545948 17592232555238 17592232577980 17592232577988 17592232808650 17592232886729 17592232886786 17592232886793 17592232886797 17592232958090 17592233076577 17592233144301 17592233431334 17592233431814 17592234230520 17592234448526 17592234700485 17592234806481 17592234851859 17592235003983 17592235068155 17592235068158 17592235074859 17592235305451 17592235810873 17592235922068 17592236113837 17592236349303 17592236461666 17592236868866 17592236868871 17592238607837 17592238708423 17592238708558 17592241193003 17592242514773 17592242874891 17592243928280 17592244691542 17592244691558 17592244761680 17592245006804 17592245184339 17592246196802 17592247419073 17592247419185 17592247425545 17592249936256 17592249973733 17592250176578 17592250210564 17592250210567 17592250777772 17592250777854 17592253249909 17592257647588 17592258011361 17592258246710 17592258866422 17592260775415 17592262632255 17592264489873 17592264560519 17592265120447 17592271573602 17592271576404 17592273679867 17592275987623 17592275995130 17592275995135 17592276037255 17592277469850 17592278756108 17592280134939 17592281881509 17592284920305 17592285467793 17592286618015 17592288587524 17592289178758 17592289631589 17592290322936 17592291273963 17592291296031 17592291325248 17592291325326 17592291325452 17592291325766 17592291325864 17592291325954 17592291326043 17592291326115 17592291620408 17592292122045 17592294315344 17592295112116 17592296909432 17592296909449 17592296909452 17592297811962]) + + + +(do + (auto-ap.datomic/audit-transact-batch (->> (dc/q '[:find ?e ?c + :in $ + :where [?e :invoice/client ?c]] + (dc/db conn) + ) + (map (fn [[i c]] + {:db/id i + :invoice/client c}))) + + {:user/name "hydrate-tuples"}) + (auto-ap.datomic/audit-transact-batch (->> (dc/q '[:find ?e ?c + :in $ + :where [?e :transaction/client ?c]] + (dc/db conn) + ) + (map (fn [[i c]] + {:db/id i + :transaction/client c}))) + + {:user/name "hydrate-tuples"}) + + (auto-ap.datomic/audit-transact-batch (->> (dc/q '[:find ?e ?c + :in $ + :where [?e :journal-entry/client ?c]] + (dc/db conn) + ) + (map (fn [[i c]] + {:db/id i + :journal-entry/client c}))) + + {:user/name "hydrate-tuples"}) + + ) diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index de418149..441d25b6 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -19,7 +19,8 @@ [mount.core :as mount] [clojure.java.io :as io] [datomic.db :refer [id-literal]] - [datomic.function :refer [construct]]) + [datomic.function :refer [construct]] + [auto-ap.logging :as alog]) (:import (java.util UUID))) @@ -579,8 +580,6 @@ (defn add-sorter-fields [q sort-map args] (reduce (fn [q {:keys [sort-key] :as z}] - (prn z) - (println (class sort-key)) (merge-query q {:query {:find [(symbol (str "?sort-" sort-key))] :where (sort-map @@ -896,3 +895,22 @@ (defn query2 [query] (apply dc/q (:query query) (:args query))) +(defn observable-q [query] + nil) + +(defn observable-query [query] + (mu/with-context {:query (:query query) + :args (:args query) + :query-stats true + :io-context ::hello} + (mu/trace ::query + [] + (let [query-results (dc/query {:query (:query query) + :args (:args query) + :query-stats true + :io-context ::hello})] + (alog/info ::query-stats + :io-stats (:io-stats query-results) + :query-stats (:query-stats query-results)) + (:ret query-results))))) + diff --git a/src/clj/auto_ap/datomic/checks.clj b/src/clj/auto_ap/datomic/checks.clj index da5fe027..332c54f3 100644 --- a/src/clj/auto_ap/datomic/checks.clj +++ b/src/clj/auto_ap/datomic/checks.clj @@ -6,13 +6,13 @@ apply-sort-3 conn merge-query - pull-many - query2]] - [auto-ap.graphql.utils :refer [limited-clients]] + observable-query + pull-many]] + [auto-ap.graphql.utils :refer [extract-client-ids]] [clj-time.coerce :as c] [clojure.set :refer [rename-keys]] - [clojure.tools.logging :as log] - [datomic.api :as dc])) + [datomic.api :as dc] + [clj-time.coerce :as coerce])) (defn <-datomic [result] (-> result @@ -38,18 +38,25 @@ (defn raw-graphql-ids ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] - (let [check-number-like (try (Long/parseLong (:check-number-like args)) (catch Exception _ nil)) + (let [valid-clients (extract-client-ids (:clients args) + (:client-id args) + (when (:client-code args) + [:client/code (:client-code args)])) + check-number-like (try (Long/parseLong (:check-number-like args)) (catch Exception _ nil)) query (if (:exact-match-id args) {:query {:find '[?e] :in '[$ ?e [?c ...]] :where '[[?e :payment/client ?c]]} :args [db (:exact-match-id args) - (map :db/id (:clients args))]} + valid-clients]} (cond-> {:query {:find [] - :in ['$] - :where []} - :args [db]} + :in '[$ [?clients ?start ?end]] + :where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]]} + :args [db + [valid-clients + (some-> (:start (:date-range args)) coerce/to-date) + (some-> (:end (:date-range args)) coerce/to-date)]]} (:sort args) (add-sorter-fields {"client" ['[?e :payment/client ?c] '[?c :client/name ?sort-client]] "vendor" ['[?e :payment/vendor ?v] @@ -66,23 +73,6 @@ :where []} :args [(:exact-match-id args)]}) - true - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :payment/client ?xx]]} - :args [(map :db/id (:clients args))]}) - - - (:client-id args) - (merge-query {:query {:in ['?client-id] - :where ['[?e :payment/client ?client-id]]} - :args [(:client-id args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :payment/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [(:client-code args)]}) - - (:vendor-id args) (merge-query {:query {:in ['?vendor-id] :where ['[?e :payment/vendor ?vendor-id]]} @@ -133,18 +123,6 @@ (merge-query {:query {:in ['?status] :where ['[?e :payment/status ?status]]} :args [(:status args)]}) - - (:start (:date-range args)) - (merge-query {:query {:in '[?start-date] - :where ['[?e :payment/date ?date] - '[(>= ?date ?start-date)]]} - :args [(c/to-date (:start (:date-range args)))]}) - - (:end (:date-range args)) - (merge-query {:query {:in '[?end-date] - :where ['[?e :payment/date ?date] - '[(<= ?date ?end-date)]]} - :args [(c/to-date (:end (:date-range args)))]}) (:payment-type args) (merge-query {:query {:in '[?payment-type] @@ -157,12 +135,10 @@ :args [check-number-like]}) true - (merge-query {:query {:find ['?sort-default '?e] - :where ['[?e :payment/date ?sort-default]]}})))] + (merge-query {:query {:find ['?sort-default '?e]}})))] - (log/info query) - (cond->> (query2 query) + (cond->> (observable-query query) true (apply-sort-3 (assoc args :default-asc? false)) true (apply-pagination args))))) diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj index 0ded6909..f585123a 100644 --- a/src/clj/auto_ap/datomic/invoices.clj +++ b/src/clj/auto_ap/datomic/invoices.clj @@ -4,13 +4,14 @@ :refer [add-sorter-fields apply-pagination query2 + observable-query apply-sort-3 conn merge-query pull-many]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.datomic.vendors :as d-vendors] - [auto-ap.graphql.utils :refer [limited-clients]] + [auto-ap.graphql.utils :refer [limited-clients extract-client-ids]] [auto-ap.time-utils :refer [next-dom]] [clj-time.coerce :as coerce] [clj-time.core :as time] @@ -48,18 +49,27 @@ ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] - (let [query + (let [valid-clients (extract-client-ids (:clients args) + (:client-id args) + (when (:client-code args) + [:client/code (:client-code args)])) + query (if (:exact-match-id args) {:query {:find '[?e] :in '[$ ?e [?c ...]] :where '[[?e :invoice/client ?c]]} :args [db (:exact-match-id args) - (map :db/id (:clients args))]} + valid-clients]} (cond-> {:query {:find [] - :in ['$] - :where []} - :args [db]} + :in '[$ [?clients ?start ?end]] + :where '[ + [(iol-ion.query/scan-invoices $ ?clients ?start ?end) [[?e _ ?sort-default] ...]] + ]} + :args [db + [valid-clients + (some-> (:start (:date-range args)) coerce/to-date) + (some-> (:end (:date-range args)) coerce/to-date)]]} (:client-id args) @@ -81,15 +91,9 @@ :args [ (cond-> (:original-id args) (string? (:original-id args)) Long/parseLong )]}) - (:start (:date-range args)) (merge-query {:query {:in '[?start-date] - :where ['[?e :invoice/date ?date] - '[(>= ?date ?start-date)]]} - :args [(coerce/to-date (:start (:date-range args)))]}) + - (:end (:date-range args)) (merge-query {:query {:in '[?end-date] - :where ['[?e :invoice/date ?date] - '[(<= ?date ?end-date)]]} - :args [(coerce/to-date (:end (:date-range args)))]}) + (:start (:due-range args)) (merge-query {:query {:in '[?start-due] :where ['[?e :invoice/due ?due] @@ -100,10 +104,7 @@ :where ['[?e :invoice/due ?due] '[(<= ?due ?end-due)]]} :args [(coerce/to-date (:end (:due-range args)))]}) - true - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :invoice/client ?xx]]} - :args [ (map :db/id (:clients args))]}) + (:import-status args) (merge-query {:query {:in ['?import-status] @@ -175,10 +176,8 @@ "outstanding-balance" ['[?e :invoice/outstanding-balance ?sort-outstanding-balance]]} args) true - (merge-query {:query {:find ['?sort-default '?e ] - :where ['[?e :invoice/client] - '[?e :invoice/date ?sort-default]]}}) ))] - (->> (query2 query) + (merge-query {:query {:find ['?sort-default '?e ]}}) ))] + (->> (observable-query query) (apply-sort-3 args) (apply-pagination args))))) diff --git a/src/clj/auto_ap/datomic/ledger.clj b/src/clj/auto_ap/datomic/ledger.clj index 3899274e..7fa71558 100644 --- a/src/clj/auto_ap/datomic/ledger.clj +++ b/src/clj/auto_ap/datomic/ledger.clj @@ -6,31 +6,33 @@ apply-sort-3 conn merge-query - pull-many - query2]] + observable-query + pull-many]] [auto-ap.datomic.accounts :as d-accounts] - [auto-ap.graphql.utils :refer [limited-clients]] - [clj-time.coerce :as c] + [auto-ap.graphql.utils :refer [extract-client-ids]] + [clj-time.coerce :as coerce] [datomic.api :as dc])) (defn raw-graphql-ids [db args] - (let [query + (let [valid-clients (extract-client-ids (:clients args) + (:client-id args) + (when (:client-code args) + [:client/code (:client-code args)])) + query (if (:exact-match-id args) {:query {:find '[?e] :in '[$ ?e [?c ...]] :where '[[?e :journal-entry/client ?c]]} :args [db (:exact-match-id args) - (map :db/id (:clients args))]} + valid-clients]} (cond-> {:query {:find [] - :in ['$ ] - :where []} - :args [db]} - - true - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :journal-entry/client ?xx]]} - :args [(set (map :db/id (:clients args)))]}) + :in ['$ '[?clients ?start ?end]] + :where '[[(iol-ion.query/scan-ledger $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]]} + :args [db + [valid-clients + (some-> (:start (:date-range args)) coerce/to-date) + (some-> (:end (:date-range args)) coerce/to-date)]]} (:only-external args) (merge-query {:query {:where ['(not [?e :journal-entry/original-entity ])]}}) @@ -51,23 +53,6 @@ :where ['[?e :journal-entry/vendor ?vendor-id]]} :args [(:vendor-id args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :journal-entry/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [(:client-code args)]}) - - (:start (:date-range args)) - (merge-query {:query {:in ['?start-date] - :where ['[?e :journal-entry/date ?date] - '[(>= ?date ?start-date)]]} - :args [(c/to-date (:start (:date-range args)))]}) - - (:end (:date-range args)) - (merge-query {:query {:in ['?end-date] - :where ['[?e :journal-entry/date ?date] - '[(<= ?date ?end-date)]]} - :args [(c/to-date (:end (:date-range args)))]}) (or (seq (:numeric-code args)) (:bank-account-id args) @@ -130,8 +115,8 @@ args) true - (merge-query {:query {:find ['?sort-default '?e] :where ['[?e :journal-entry/date ?sort-default]]}})))] - (->> (query2 query) + (merge-query {:query {:find ['?sort-default '?e]}})))] + (->> (observable-query query) (apply-sort-3 (update args :sort conj {:sort-key "default-2" :asc true})) (apply-pagination args)))) @@ -144,7 +129,7 @@ {:account-client-override/client [:db/id]}]} {:bank-account/type [*]}]}]}] ids) - (map #(update % :journal-entry/date c/from-date)) + (map #(update % :journal-entry/date coerce/from-date)) (map (fn [je] (update je :journal-entry/line-items (fn [jels] diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj index f8e27a34..26190774 100644 --- a/src/clj/auto_ap/datomic/transactions.clj +++ b/src/clj/auto_ap/datomic/transactions.clj @@ -6,13 +6,12 @@ apply-sort-3 conn merge-query - pull-many - query2]] + observable-query + pull-many]] [auto-ap.datomic.accounts :as d-accounts] - [auto-ap.graphql.utils :refer [limited-clients]] + [auto-ap.graphql.utils :refer [extract-client-ids]] [clj-time.coerce :as coerce] [clojure.string :as str] - [clojure.tools.logging :as log] [datomic.api :as dc])) (defn potential-duplicate-ids [db args] @@ -41,7 +40,11 @@ (defn raw-graphql-ids ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] - (let [potential-duplicates (potential-duplicate-ids db args) + (let [valid-clients (extract-client-ids (:clients args) + (:client-id args) + (when (:client-code args) + [:client/code (:client-code args)])) + potential-duplicates (potential-duplicate-ids db args) query (if (:exact-match-id args) (cond-> {:query {:find '[?e] @@ -49,21 +52,19 @@ :where '[[?e :transaction/client ?c]]} :args [db (:exact-match-id args) - (map :db/id (:clients args))]}) + valid-clients]}) (cond-> {:query {:find [] - :in ['$ ] - :where []} - :args [db]} + :in '[$ [?clients ?start ?end]] + :where '[[(iol-ion.query/scan-transactions $ ?clients ?start ?end) [[?e _ ?sort-default] ...]]]} + :args [db + [valid-clients + (some-> (:start (:date-range args)) coerce/to-date) + (some-> (:end (:date-range args)) coerce/to-date)]]} (:potential-duplicates args) (merge-query {:query {:in '[[?e ...]]} :args [potential-duplicates]}) - true - (merge-query {:query {:in ['[?xx ...]] - :where ['[?e :transaction/client ?xx]]} - :args [(set (map :db/id (:clients args)))]}) - (:bank-account-id args) (merge-query {:query {:in ['?bank-account-id] :where ['[?e :transaction/bank-account ?bank-account-id]]} @@ -80,11 +81,6 @@ '[?accounts :transaction-account/account ?account-id]]} :args [(:account-id args)]}) - (:client-id args) - (merge-query {:query {:in ['?client-id] - :where ['[?e :transaction/client ?client-id]]} - :args [(:client-id args)]}) - (:vendor-id args) (merge-query {:query {:in ['?vendor-id] :where ['[?e :transaction/vendor ?vendor-id]]} @@ -103,29 +99,11 @@ '[(<= ?a ?amount-lte)]]} :args [(:amount-lte args)]}) - (:start (:date-range args)) - (merge-query {:query {:in ['?start-date] - :where ['[?e :transaction/date ?date] - '[(>= ?date ?start-date)]]} - :args [(coerce/to-date (:start (:date-range args)))]}) - - (:end (:date-range args)) - (merge-query {:query {:in ['?end-date] - :where ['[?e :transaction/date ?date] - '[(<= ?date ?end-date)]]} - :args [(coerce/to-date (:end (:date-range args)))]}) - (:approval-status args) (merge-query {:query {:in ['?approval-status] :where ['[?e :transaction/approval-status ?approval-status]]} :args [(:approval-status args)]}) - (:client-code args) - (merge-query {:query {:in ['?client-code] - :where ['[?e :transaction/client ?client-id] - '[?client-id :client/code ?client-code]]} - :args [(:client-code args)]}) - (:original-id args) (merge-query {:query {:in ['?original-id] :where ['[?e :transaction/client ?c] @@ -174,10 +152,8 @@ true (merge-query {:query {:find ['?sort-default '?e] :where ['[?e :transaction/id] - '[?e :transaction/date ?sort-default] '(not [?e :transaction/approval-status :transaction-approval-status/suppressed])]}})))] - (log/info "query is" query) - (cond->> (query2 query) + (cond->> (observable-query query) true (apply-sort-3 (assoc args :default-asc? false)) true (apply-pagination args))))) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index db3ae3bd..09f3b42b 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -36,7 +36,8 @@ [com.walmartlabs.lacinia.schema :as schema] [datomic.api :as dc] [unilog.context :as lc] - [yang.time :refer [time-it]]) + [yang.time :refer [time-it]] + [auto-ap.routes.auth :as auth]) (:import (clojure.lang IPersistentMap))) @@ -243,7 +244,8 @@ :profile_image_url {:type 'String} :email {:type 'String} :role {:type :role} - :clients {:type '(list :client)}}} + :clients {:type '(list :client)} + :impersonate_jwt {:type 'String}}} :csv {:fields {:csv_content_b64 {:type 'String}}} @@ -622,7 +624,10 @@ (defn get-user [context args _] (assert-admin (:id context)) - (let [users (d-users/get-graphql args)] + (let [users (->> (d-users/get-graphql args) + (map (fn [u] + (assoc u :impersonate_jwt + (auth/user->jwt u "FAKE_TOKEN")))))] (->graphql users))) diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index 77a0db41..9992da70 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -53,7 +53,8 @@ (defn get-transaction-page [context args _] (let [args (assoc (:filters args) - :clients (:clients context)) + :clients (:clients context) + :id (:id context)) _ (assert-filtered-enough args) [transactions transactions-count] (d-transactions/get-graphql (update (<-graphql args) :approval-status enum->keyword "transaction-approval-status")) transactions (map ->graphql (map approval-status->graphql transactions))] diff --git a/src/clj/auto_ap/graphql/utils.clj b/src/clj/auto_ap/graphql/utils.clj index b31a2a31..56a4497c 100644 --- a/src/clj/auto_ap/graphql/utils.clj +++ b/src/clj/auto_ap/graphql/utils.clj @@ -5,10 +5,12 @@ [auto-ap.time :as atime] [buddy.auth :refer [throw-unauthorized]] [datomic.api :as dc] + [iol-ion.query :refer [entid]] [clojure.walk :as walk] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [clojure.tools.logging :as log] - [com.brunobonacci.mulog :as mu])) + [com.brunobonacci.mulog :as mu] + [clojure.set :as set])) (defn snake->kebab [s] @@ -163,6 +165,29 @@ resolver-key (trace-query resolver-key resolver-fn)) ) {} - m)) + m))) - ) +(defn extract-client-ids [user-clients & possible-clients] + (let [coerce-client-ids (fn coerce-client-ids [x] + (cond (and (map? x) + (:db/id x)) + [(:db/id x)] + + (nat-int? x) + [x] + + (and (vector? x) + (= :client/code (first x))) + [(entid (dc/db conn) x)] + + + (sequential? x) + (map x coerce-client-ids) + + :else + [])) + user-client-ids (set (mapcat coerce-client-ids user-clients)) + extra-client-ids (set (mapcat coerce-client-ids possible-clients))] + (if (seq extra-client-ids) + (set/intersection user-client-ids extra-client-ids) + user-client-ids))) diff --git a/src/clj/auto_ap/handler.clj b/src/clj/auto_ap/handler.clj index b01055de..48ad0f49 100644 --- a/src/clj/auto_ap/handler.clj +++ b/src/clj/auto_ap/handler.clj @@ -179,7 +179,8 @@ [handler] (fn [request] (let [x-clients (-> request :session :client-selection) - identity (-> request :session :identity) + identity (or (-> request :identity) + (-> request :session :identity)) ideal-ids (set (cond (or (= :all x-clients) (nil? x-clients)) @@ -235,6 +236,26 @@ (into new-session) (assoc :client-selection x-clients)))))))) +(defn wrap-gunzip-jwt + [handler] + (fn [{:keys [session] :as request}] + (let [request (if-let [gz-clients (some-> request :identity :gz-clients)] + (try + (assoc-in request [:identity :user/clients] + (auth/gunzip gz-clients)) + (catch Exception e + (alog/error :cant-gunzip-clients + :error e) + request)) + request)] + (handler request)))) + +#_(defn wrap-pprint-session + [handler] + (fn [request] + (clojure.pprint/pprint (:session request)) + (handler request))) + #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (def app (-> route-handler @@ -242,16 +263,19 @@ (wrap-guess-route) (wrap-hydrate-clients) (wrap-store-client-in-session) + (wrap-gunzip-jwt) (wrap-authorization auth-backend) (wrap-authentication auth-backend (session-backend {:authfn (fn [auth] (dissoc auth :exp))})) + #_(wrap-pprint-session) (wrap-idle-session-timeout) (wrap-session {:store (cookie-store {:key (byte-array [42, 52, -31, 105, -126, -33, -118, -69, -82, -59, -15, -69, -38, 103, -102, -1])} )}) + (wrap-reload) (wrap-params) (mp/wrap-multipart-params) diff --git a/src/clj/auto_ap/routes/auth.clj b/src/clj/auto_ap/routes/auth.clj index d6e4e2b9..f90b6449 100644 --- a/src/clj/auto_ap/routes/auth.clj +++ b/src/clj/auto_ap/routes/auth.clj @@ -6,7 +6,9 @@ [clj-time.core :as time] [clojure.tools.logging :as log] [config.core :refer [env]] - [com.brunobonacci.mulog :as mu])) + [com.brunobonacci.mulog :as mu] + [clojure.java.io :as io] + [clojure.edn :as edn])) (def google-client-id "264081895820-0nndcfo3pbtqf30sro82vgq5r27h8736.apps.googleusercontent.com") (def google-client-secret "OC-WemHurPXYpuIw5cT-B90g") @@ -20,6 +22,50 @@ (:jwt-secret env) {:alg :hs512})) +(defn gzip [data] + (let [data (pr-str data) + raw (java.io.ByteArrayOutputStream.)] + (with-open [output (-> raw + (io/output-stream) + (java.util.zip.GZIPOutputStream.))] + (io/copy data output)) + (.encodeToString (java.util.Base64/getEncoder) (.toByteArray raw)))) + +(defn gunzip [b64] + + (let [raw-bytes (.decode (java.util.Base64/getDecoder) b64) + raw (java.io.ByteArrayInputStream. raw-bytes) + out (java.io.ByteArrayOutputStream.)] + (with-open [compressed (-> raw + (io/input-stream) + (java.util.zip.GZIPInputStream.))] + (io/copy compressed out)) + + (edn/read-string (.toString out)))) + +(defn user->jwt [user oauth-token] + (let [auth (cond-> {:user (:user/name user) + :exp (time/plus (time/now) (time/days 30)) + :db/id (:db/id user) + :user/role (name (:user/role user)) + :user/name (:user/name user)} + (= "admin" (name (:user/role user))) + (assoc :gz-clients (->> (:user/clients user) + (map (fn [c] + (select-keys c [:client/code :db/id :client/locations]))) + + gzip)) + (not= "admin" (name (:user/role user))) + (assoc :user/clients + (->> (:user/clients user) + (map (fn [c] + (select-keys c [:client/code :db/id :client/locations]))))))] + + (when (and user oauth-token) + (jwt/sign auth + (:jwt-secret env) + {:alg :hs512})))) + (defn oauth [{{:strs [code state]} :query-params {:strs [host]} :headers :as request}] (try (let [auth (-> "https://accounts.google.com/o/oauth2/token" @@ -43,25 +89,15 @@ :user/email (:email profile) :user/profile-image-url (:picture profile) :user/name (:name profile)}) - auth {:user (:name profile) - :exp (time/plus (time/now) (time/days 30)) - :db/id (:db/id user) - :user/clients (map (fn [c] - (select-keys c [:client/code :db/id :client/locations])) - (:user/clients user)) - :user/role (name (:user/role user)) - :user/name (:name profile)} + _ (mu/log ::logged-in-as :auth auth)] ;; TODO - these namespaces are not being transmitted/deserialized properly - (if (and token user) - (let [jwt (jwt/sign auth - (:jwt-secret env) - {:alg :hs512})] - {:status 301 - :headers {"Location" (str (or (not-empty state) "/") "?jwt=" jwt)} - :session {:identity (dissoc auth :exp)}}) + (if-let [jwt (user->jwt user token)] + {:status 301 + :headers {"Location" (str (or (not-empty state) "/") "?jwt=" jwt)} + :session {:identity (dissoc auth :exp)}} {:status 401 :body "Couldn't authenticate"})) (catch Exception e diff --git a/src/cljs/auto_ap/events.cljs b/src/cljs/auto_ap/events.cljs index 708b75b4..6420f31e 100644 --- a/src/cljs/auto_ap/events.cljs +++ b/src/cljs/auto_ap/events.cljs @@ -4,7 +4,7 @@ [auto-ap.routes :as routes] [auto-ap.utils :refer [by]] [auto-ap.views.pages.data-page :as data-page] - [auto-ap.views.utils :refer [parse-jwt with-user]] + [auto-ap.views.utils :refer [parse-jwt with-user gunzip]] [bidi.bidi :as bidi] [clojure.string :as str] [clojure.edn :as edn] @@ -12,10 +12,18 @@ [re-frame.core :as re-frame] [auto-ap.ssr-routes :as ssr-routes] [cemerick.url :as url] - [auto-ap.subs :as subs])) + [auto-ap.subs :as subs] + [pako])) + + (defn jwt->data [token] - (js->clj (.parse js/JSON (b64/decodeString (second (str/split token #"\." )))))) + (let [raw (js->clj (.parse js/JSON (b64/decodeString (second (str/split token #"\." ))))) + gz-clients (or (:gz-clients raw) + (get raw "gz-clients"))] + (cond-> raw + gz-clients (assoc "user/clients" (gunzip gz-clients))))) + (defn client-query [] diff --git a/src/cljs/auto_ap/views/components/layouts.cljs b/src/cljs/auto_ap/views/components/layouts.cljs index 718495cd..e74cf2e3 100644 --- a/src/cljs/auto_ap/views/components/layouts.cljs +++ b/src/cljs/auto_ap/views/components/layouts.cljs @@ -56,6 +56,11 @@ (when (= "admin" (:user/role @user)) [:a {:class "navbar-item" :href (bidi/path-for routes/routes :admin)} "Administration"]) [:hr {:class "navbar-divider"}] + [:a.navbar-item {:on-click (fn [] + (.removeItem js/localStorage "last-client-id" nil) + (.setItem js/localStorage "last-selected-clients" ":all") + (.reload (.-location js/document ) true))} + "Full Refresh"] [:a.navbar-item {:on-click (fn [e] (.preventDefault e) (re-frame/dispatch [::events/logout]))} "Logout"]]] [:a.navbar-item {:href (login-url)} "Login"]))) diff --git a/src/cljs/auto_ap/views/pages/admin/users.cljs b/src/cljs/auto_ap/views/pages/admin/users.cljs index 655b9d32..6b811900 100644 --- a/src/cljs/auto_ap/views/pages/admin/users.cljs +++ b/src/cljs/auto_ap/views/pages/admin/users.cljs @@ -39,6 +39,7 @@ [:name :profile_image_url :email + :impersonate_jwt :id :role [:clients [:id :name]]]]]} diff --git a/src/cljs/auto_ap/views/pages/admin/users/table.cljs b/src/cljs/auto_ap/views/pages/admin/users/table.cljs index 8a2788c7..3decaa90 100644 --- a/src/cljs/auto_ap/views/pages/admin/users/table.cljs +++ b/src/cljs/auto_ap/views/pages/admin/users/table.cljs @@ -32,7 +32,7 @@ [grid/header-cell {} "Email"] [grid/header-cell {} "Role"] [grid/header-cell {} "Clients"] - [grid/header-cell {:style {:width (action-cell-width 1)}}]]] + [grid/header-cell {:style {:width (action-cell-width 5)}}]]] [grid/body (for [{:keys [id name role clients] :as c} (:data page)] ^{:key (str name "-" id)} @@ -50,6 +50,14 @@ [grid/cell {} role] [grid/cell {} (str/join ", " (map :name clients))] [grid/cell {} + [:a.button {:on-click (fn [] + (.setItem js/localStorage "jwt" (:impersonate-jwt c)) + (.removeItem js/localStorage "last-client-id" nil) + (.removeItem js/localStorage "last-selected-clients" nil) + (.reload (.-location js/document ) true))} + "Impersonate"] + + [buttons/fa-icon {:event [::form/editing c] :icon "fa-pencil"}]]])]] ])) diff --git a/src/cljs/auto_ap/views/pages/error.cljs b/src/cljs/auto_ap/views/pages/error.cljs index bae67777..b5b18891 100644 --- a/src/cljs/auto_ap/views/pages/error.cljs +++ b/src/cljs/auto_ap/views/pages/error.cljs @@ -8,10 +8,14 @@ [:div.column.is-8.is-offset-2.has-text-centered [:div.box.slideInFromBelow - [:img {:src "http://www.integreatconsult.com/wp-content/uploads/2016/11/logo.png"}] + [:img {:src "/img/logo.png"}] [:div.notification.is-danger.is-light "An unexpected error has occured. " - [:a {:on-click #(.reload (.-location js/document )) } "Click here"] - " to try again."]] + [:div [:a {:on-click (fn [] + (.removeItem js/localStorage "last-client-id" nil) + (.removeItem js/localStorage "last-selected-clients" nil) + (.reload (.-location js/document ) true)) } "Click here"] + " to try again."] + [:div "If the error continues, please try " [:a {:href "/login"} "logging in"] " again."]]] [:p.has-text-gray "Copyright Integreat 2020"]]]]]] ) diff --git a/src/cljs/auto_ap/views/utils.cljs b/src/cljs/auto_ap/views/utils.cljs index 0326e38f..8a456650 100644 --- a/src/cljs/auto_ap/views/utils.cljs +++ b/src/cljs/auto_ap/views/utils.cljs @@ -11,7 +11,8 @@ [react-transition-group :as react-transition-group] #_{:clj-kondo/ignore [:unused-namespace]} [react :as react] - [reagent.core :as r]) + [reagent.core :as r] + [pako]) (:import (goog.i18n NumberFormat) (goog.i18n.NumberFormat Format))) @@ -297,13 +298,25 @@ :else x)) +(defn gunzip [b64] + (let [raw-byte-array (->> b64 + js/atob + (map (fn [z] (.charCodeAt z 0))) + clj->js + (js/Uint8Array.))] + (or (edn/read-string (pako/inflate raw-byte-array #js {"to" "string"})) + nil))) (defn parse-jwt [jwt] (when-let [json (some-> jwt (str/split #"\.") second base64/decodeString)] - (js->clj (.parse js/JSON json) :keywordize-keys true))) + (let [raw (js->clj (.parse js/JSON json) :keywordize-keys true) + gz-clients (or (:gz-clients raw) + (get raw "gz-clients"))] + (cond-> raw + gz-clients (assoc :user/clients (gunzip gz-clients)))))) (defn coerce-float [f] (cond (str/blank? f)