diff --git a/src/clj/auto_ap/datomic/sales_orders.clj b/src/clj/auto_ap/datomic/sales_orders.clj index 144a8cf0..cc67d4f5 100644 --- a/src/clj/auto_ap/datomic/sales_orders.clj +++ b/src/clj/auto_ap/datomic/sales_orders.clj @@ -1,11 +1,9 @@ (ns auto-ap.datomic.sales-orders (:require - [auto-ap.datomic :refer [conn]] [auto-ap.storage.parquet :as pq] - [clj-time.coerce :as c] - [clojure.set :as set] - [clojure.string :as str] - [com.brunobonacci.mulog :as mu])) + [clojure.set :as set] + [clojure.string :as str] + [com.brunobonacci.mulog :as mu])) (defn <-row "Convert a flat parquet row into the shape consumers expect. @@ -29,12 +27,13 @@ (update :sales-order/date #(some-> % str)))) (defn build-where-clause [args] - (let [clauses [(when-let [c (:client-code args)] - ["external_id.client = '" c "'"]) - (when-let [v (:vendor args)] - ["external_id.vendor = '" (name v) "'"]) - (when-let [l (:location args)] - ["location = '" l "'"])] + (let [clauses (keep identity + [(when-let [c (:client-code args)] + (str "external_id.client = '" c "'")) + (when-let [v (:vendor args)] + (str "external_id.vendor = '" (name v) "'")) + (when-let [l (:location args)] + (str "location = '" l "'"))])] (when (seq clauses) (str "WHERE " (str/join " AND " clauses))))) @@ -47,18 +46,15 @@ (defn raw-graphql-ids [args] (let [start (some-> (:start (:date-range args)) .toString) - end (some-> (:end (:date-range args)) .substring 0 10) - where (build-where-clause args) - sort (build-sort-clause args) + end (some-> (:end (:date-range args)) (.substring 0 10)) limit (or (:limit args) page-size) - offset (or (:offset args) 0) - where-str (when where (str " " where))] + offset (or (:offset args) 0)] (when start (let [result (pq/get-sales-orders start end {:client (:client-code args) :vendor (:vendor args) :location (:location args) - :sort sort + :sort (or (:sort args) "date") :order "DESC" :limit limit :offset offset})] diff --git a/src/clj/auto_ap/ezcater/core.clj b/src/clj/auto_ap/ezcater/core.clj index 5b520a36..ee468234 100644 --- a/src/clj/auto_ap/ezcater/core.clj +++ b/src/clj/auto_ap/ezcater/core.clj @@ -21,42 +21,41 @@ :body (json/write-str {"query" (v/graphql-query q)}) :as :json}) :body - :data - )) + :data)) (defn get-caterers [integration] (:caterers (query integration {:venia/queries [{:query/data - [:caterers [:name :uuid [:address [:name :street]]]]}]} ))) + [:caterers [:name :uuid [:address [:name :street]]]]}]}))) (defn get-subscriptions [integration] (->> (query integration {:venia/queries [{:query/data - [:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]] ]]}]} ) + [:subscribers [:id [:subscriptions [:parentId :parentEntity :eventEntity :eventKey]]]]}]}) :subscribers first :subscriptions)) (defn get-integrations [] (map first (dc/q '[:find (pull ?i [:ezcater-integration/api-key - :ezcater-integration/subscriber-uuid - :db/id - :ezcater-integration/integration-status [:db/id]]) - :in $ - :where [?i :ezcater-integration/api-key]] - (dc/db conn)))) + :ezcater-integration/subscriber-uuid + :db/id + {:ezcater-integration/integration-status [:db/id]}]) + :in $ + :where [?i :ezcater-integration/api-key]] + (dc/db conn)))) (defn mark-integration-status [integration integration-status] @(dc/transact conn - [{:db/id (:db/id integration) - :ezcater-integration/integration-status (assoc integration-status - :db/id (or (-> integration :ezcater-integration/integration-status :db/id) - (random-tempid)))}])) + [{:db/id (:db/id integration) + :ezcater-integration/integration-status (assoc integration-status + :db/id (or (-> integration :ezcater-integration/integration-status :db/id) + (random-tempid)))}])) (defn upsert-caterers ([integration] @(dc/transact conn (for [caterer (get-caterers integration)] - {:db/id (:db/id integration) + {:db/id (:db/id integration) :ezcater-integration/caterers [{:ezcater-caterer/name (str (:name caterer) " (" (:street (:address caterer)) ")") :ezcater-caterer/search-terms (str (:name caterer) " " (:street (:address caterer))) :ezcater-caterer/uuid (:uuid caterer)}]})))) @@ -65,14 +64,14 @@ ([integration] (let [extant (get-subscriptions integration) to-ensure (set (map first (dc/q '[:find ?cu - :in $ - :where [_ :client/ezcater-locations ?el] - [?el :ezcater-location/caterer ?c] - [?c :ezcater-caterer/uuid ?cu]] - (dc/db conn)))) + :in $ + :where [_ :client/ezcater-locations ?el] + [?el :ezcater-location/caterer ?c] + [?c :ezcater-caterer/uuid ?cu]] + (dc/db conn)))) to-create (set/difference - to-ensure - (set (map :parentId extant)))] + to-ensure + (set (map :parentId extant)))] (doseq [parentId to-create] (query integration {:venia/operation {:operation/type :mutation @@ -95,7 +94,6 @@ :eventKey 'cancelled}} [[:subscription [:parentId :parentEntity :eventEntity :eventKey]]]]]}))))) - #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn upsert-ezcater ([] (upsert-ezcater (get-integrations))) @@ -116,12 +114,11 @@ (defn get-caterer [caterer-uuid] (dc/pull (dc/db conn) - '[:ezcater-caterer/name - {:ezcater-integration/_caterers [:ezcater-integration/api-key]} - {:ezcater-location/_caterer [:ezcater-location/location - {:client/_ezcater-locations [:client/code]}]}] - [:ezcater-caterer/uuid caterer-uuid])) - + '[:ezcater-caterer/name + {:ezcater-integration/_caterers [:ezcater-integration/api-key]} + {:ezcater-location/_caterer [:ezcater-location/location + {:client/_ezcater-locations [:client/code]}]}] + [:ezcater-caterer/uuid caterer-uuid])) (defn round-carry-cents [f] (with-precision 2 (double (.setScale (bigdec f) 2 java.math.RoundingMode/HALF_UP)))) @@ -137,69 +134,69 @@ :else 0.07M)] (round-carry-cents - (* commision% - 0.01M - (+ - (-> order :totals :subTotal :subunits ) - (reduce + - 0 - (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) + (* commision% + 0.01M + (+ + (-> order :totals :subTotal :subunits) + (reduce + + 0 + (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order))))))))) (defn ccp-fee [order] (round-carry-cents - (* 0.000299M - (+ - (-> order :totals :subTotal :subunits ) - (-> order :totals :salesTax :subunits ) - (reduce + - 0 - (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) + (* 0.000299M + (+ + (-> order :totals :subTotal :subunits) + (-> order :totals :salesTax :subunits) + (reduce + + 0 + (map (comp :subunits :cost) (:feesAndDiscounts (:catererCart order)))))))) (defn order->sales-order [{{:keys [timestamp]} :event {:keys [orderItems]} :catererCart :keys [client-code client-location uuid] :as order}] (let [adjustment (round-carry-cents (- (+ (-> order :totals :subTotal :subunits (* 0.01)) (-> order :totals :salesTax :subunits (* 0.01))) - (-> order :catererCart :totals :catererTotalDue ) + (-> order :catererCart :totals :catererTotalDue) (commision order) (ccp-fee order))) service-charge (+ (commision order) (ccp-fee order)) tax (-> order :totals :salesTax :subunits (* 0.01)) tip (-> order :totals :tip :subunits (* 0.01))] #:sales-order - {:date (atime/localize (coerce/to-date-time timestamp)) - :external-id (str "ezcater/order/" client-code "-" client-location "-" uuid) - :client [:client/code client-code] - :location client-location - :reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid )) - :line-items [#:order-line-item - {:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0) - :item-name "EZCater Catering" - :category "EZCater Catering" - :discount adjustment - :tax tax - :total (+ (-> order :totals :subTotal :subunits (* 0.01)) - tax - tip)}] - :charges [#:charge - {:type-name "CARD" - :date (atime/localize (coerce/to-date-time timestamp)) - :client [:client/code client-code] - :location client-location - :external-id (str "ezcater/charge/" uuid) - :processor :ccp-processor/ezcater - :total (+ (-> order :totals :subTotal :subunits (* 0.01)) - tax - tip) - :tip tip}] + {:date (atime/localize (coerce/to-date-time timestamp)) + :external-id (str "ezcater/order/" client-code "-" client-location "-" uuid) + :client [:client/code client-code] + :location client-location + :reference-link (str (url/url "https://ezmanage.ezcater.com/orders/" uuid)) + :line-items [#:order-line-item + {:external-id (str "ezcater/order/" client-code "-" client-location "-" uuid "-" 0) + :item-name "EZCater Catering" + :category "EZCater Catering" + :discount adjustment + :tax tax + :total (+ (-> order :totals :subTotal :subunits (* 0.01)) + tax + tip)}] + :charges [#:charge + {:type-name "CARD" + :date (atime/localize (coerce/to-date-time timestamp)) + :client [:client/code client-code] + :location client-location + :external-id (str "ezcater/charge/" uuid) + :processor :ccp-processor/ezcater + :total (+ (-> order :totals :subTotal :subunits (* 0.01)) + tax + tip) + :tip tip}] - :total (+ (-> order :totals :subTotal :subunits (* 0.01)) - tax - tip) - :discount adjustment - :service-charge service-charge - :tax tax - :tip tip - :returns 0.0 - :vendor :vendor/ccp-ezcater}})) + :total (+ (-> order :totals :subTotal :subunits (* 0.01)) + tax + tip) + :discount adjustment + :service-charge service-charge + :tax tax + :tip tip + :returns 0.0 + :vendor :vendor/ccp-ezcater})) (defn- flatten-order-to-parquet! [order] "Flatten a sales-order into entity-type tagged maps and buffer to parquet." @@ -222,81 +219,73 @@ (when-let [charges (:sales-order/charges order)] (doseq [chg charges] (parquet/buffer! "charge" - {:entity-type "charge" - :external-id (:charge/external-id chg) - :type-name (:charge/type-name chg) - :total (:charge/total chg) - :tax (:charge/tax chg) - :tip (:charge/tip chg) - :date so-date - :processor (some-> (:charge/processor chg) name) - :sales-order-external-id so-ext-id}))) + {:entity-type "charge" + :external-id (:charge/external-id chg) + :type-name (:charge/type-name chg) + :total (:charge/total chg) + :tax (:charge/tax chg) + :tip (:charge/tip chg) + :date so-date + :processor (some-> (:charge/processor chg) name) + :sales-order-external-id so-ext-id}))) (when-let [items (:sales-order/line-items order)] (doseq [li items] (parquet/buffer! "line-item" - {:entity-type "line-item" - :item-name (:order-line-item/item-name li) - :category (:order-line-item/category li) - :total (:order-line-item/total li) - :tax (:order-line-item/tax li) - :discount (:order-line-item/discount li) - :sales-order-external-id so-ext-id}))))) - + {:entity-type "line-item" + :item-name (:order-line-item/item-name li) + :category (:order-line-item/category li) + :total (:order-line-item/total li) + :tax (:order-line-item/tax li) + :discount (:order-line-item/discount li) + :sales-order-external-id so-ext-id}))))) (defn get-by-id [integration id] - (query - integration - {:venia/queries [[:order {:id id} - [:uuid - :orderNumber - :orderSourceType - [:caterer - [:name - :uuid - [:address [:street]]]] - [:event - [:timestamp - :catererHandoffFoodTime - :orderType]] - [:catererCart [[:orderItems - [:name - :quantity - :posItemId - [:totalInSubunits - [:currency - :subunits]]]] - [:totals - [:catererTotalDue]] - [:feesAndDiscounts - {:type 'DELIVERY_FEE} - [[:cost - [:currency - :subunits]]]]]] - [:totals [[:customerTotalDue - [ - :currency - :subunits - ]] - [:pointOfSaleIntegrationFee - [ - :currency - :subunits - ]] - [:tip - [:currency - :subunits]] - [:salesTax - [ - :currency - :subunits - ]] - [:salesTaxRemittance - [:currency - :subunits - ]] - [:subTotal - [:currency - :subunits]]]]]]]})) + (query + integration + {:venia/queries [[:order {:id id} + [:uuid + :orderNumber + :orderSourceType + [:caterer + [:name + :uuid + [:address [:street]]]] + [:event + [:timestamp + :catererHandoffFoodTime + :orderType]] + [:catererCart [[:orderItems + [:name + :quantity + :posItemId + [:totalInSubunits + [:currency + :subunits]]]] + [:totals + [:catererTotalDue]] + [:feesAndDiscounts + {:type 'DELIVERY_FEE} + [[:cost + [:currency + :subunits]]]]]] + [:totals [[:customerTotalDue + [:currency + :subunits]] + [:pointOfSaleIntegrationFee + [:currency + :subunits]] + [:tip + [:currency + :subunits]] + [:salesTax + [:currency + :subunits]] + [:salesTaxRemittance + [:currency + :subunits]] + [:subTotal + [:currency + :subunits]]]]]]]})) (defn lookup-order [json] (let [caterer (get-caterer (get json "parent_id")) @@ -304,18 +293,18 @@ client (-> caterer :ezcater-location/_caterer first :client/_ezcater-locations :client/code) location (-> caterer :ezcater-location/_caterer first :ezcater-location/location)] (if (and client location) - (doto - (-> (get-by-id integration (get json "entity_id")) - (:order) - (assoc :client-code client - :client-location location)) + (doto + (-> (get-by-id integration (get json "entity_id")) + (:order) + (assoc :client-code client + :client-location location)) (#(alog/info ::order-details :detail %))) (alog/warn ::caterer-no-longer-has-location :json json)))) (defn import-order [json] (alog/info - ::try-import-order - :json json) + ::try-import-order + :json json) (when-let [order (some-> json (lookup-order) (order->sales-order) @@ -324,7 +313,7 @@ (try (flatten-order-to-parquet! order) (alog/info ::order-buffered - :external-id (:sales-order/external-id order)) + :external-id (:sales-order/external-id order)) (catch Exception e (alog/error ::buffer-failed :exception e @@ -359,30 +348,30 @@ "key" "accepted", "occurred_at" "2022-07-21T19:21:07.549Z"} ezcater-order (lookup-order lookup-map) - extant-order (dc/pull (dc/db conn) '[:sales-order/total - :sales-order/tax - :sales-order/tip - :sales-order/discount - :sales-order/external-id - {:sales-order/charges [:charge/tax - :charge/tip - :charge/total - :charge/external-id] - :sales-order/line-items [:order-line-item/external-id - :order-line-item/total - :order-line-item/tax - :order-line-item/discount]}] - [:sales-order/external-id order]) + extant-order (dc/pull (dc/db conn '[:sales-order/total] + :sales-order/tax + :sales-order/tip + :sales-order/discount + :sales-order/external-id + {:sales-order/charges [:charge/tax + :charge/tip + :charge/total + :charge/external-id] + :sales-order/line-items [:order-line-item/external-id + :order-line-item/total + :order-line-item/tax + :order-line-item/discount]}) + [:sales-order/external-id order]) updated-order (-> (order->sales-order ezcater-order) (select-keys - #{:sales-order/total - :sales-order/tax - :sales-order/tip - :sales-order/discount - :sales-order/charges - :sales-order/external-id - :sales-order/line-items}) + #{:sales-order/total + :sales-order/tax + :sales-order/tip + :sales-order/discount + :sales-order/charges + :sales-order/external-id + :sales-order/line-items}) (update :sales-order/line-items (fn [c] (map #(select-keys % #{:order-line-item/external-id diff --git a/src/clj/auto_ap/jobs/sales_summaries.clj b/src/clj/auto_ap/jobs/sales_summaries.clj index 464d3b59..509b5813 100644 --- a/src/clj/auto_ap/jobs/sales_summaries.clj +++ b/src/clj/auto_ap/jobs/sales_summaries.clj @@ -40,17 +40,14 @@ (dc/db conn) number))) - (defn delete-all [] @(dc/transact-async conn - (->> - (dc/q '[:find ?ss - :where [?ss :sales-summary/date]] - (dc/db conn)) - (map (fn [[ ss]] - [:db/retractEntity ss]))))) - - + (->> + (dc/q '[:find ?ss + :where [?ss :sales-summary/date]] + (dc/db conn)) + (map (fn [[ss]] + [:db/retractEntity ss]))))) (defn dirty-sales-summaries [c] (let [client-id (dc/entid (dc/db conn) c)] @@ -138,9 +135,9 @@ {:db/id (str (java.util.UUID/randomUUID)) :sales-summary-item/sort-order 3 :sales-summary-item/category (cond - (= type-name "CARD") "Card Refunds" - (= type-name "CASH") "Cash Refunds" - :else "Food App Refunds") + (= type-name "CARD") "Card Refunds" + (= type-name "CASH") "Cash Refunds" + :else "Food App Refunds") :ledger-mapped/amount total :ledger-mapped/ledger-side :ledger-side/credit}) refunds)))) @@ -188,8 +185,6 @@ :ledger-mapped/ledger-side :ledger-side/credit :ledger-mapped/amount (- (+ total discount) tax)}))) - - (defn get-fees [c date] (when-let [fee (get-fee c date)] {:db/id (str (java.util.UUID/randomUUID)) @@ -299,18 +294,17 @@ (filter identity) (map (fn [z] (assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account) - :sales-summary-item/manual? false)) - )) }] + :sales-summary-item/manual? false))))}] (if (seq (:sales-summary/items result)) (do (alog/info ::upserting-summaries :category-count (count (:sales-summary/items result))) @(dc/transact conn [[:upsert-entity result]])) - @(dc/transact conn [{:db/id id :sales-summary/dirty false}])))))) + @(dc/transact conn [{:db/id id :sales-summary/dirty false}])))))) (comment ;; TODO: Move to test file or proper location - (let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL" ]) + (let [c (auto-ap.datomic/pull-attr (dc/db @conn) :db/id [:client/code "NGCL"]) date #inst "2024-04-14T00:00:00-07:00"] (get-payment-items c date))) @@ -322,16 +316,13 @@ (map (fn [[sos]] [:db/retractEntity sos]))))) - - - (comment (auto-ap.datomic/transact-schema conn) @(dc/transact conn [{:db/ident :sales-summary/total-unknown-processor-payments - :db/noHistory true, - :db/valueType :db.type/double - :db/cardinality :db.cardinality/one}]) + :db/noHistory true, + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one}]) (apply mark-dirty [:client/code "NGCL"] (last-n-days 30)) @@ -350,7 +341,7 @@ [?sos :sales-summary/date ?d] [(= ?d #inst "2024-04-10T00:00:00-07:00")]] (dc/db conn)) - + (dc/q '[:find ?n ?p2 (sum ?total) :with ?c :in $ [?clients ?start-date ?end-date] @@ -363,23 +354,18 @@ (dc/db conn) [[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGHW"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-11T00:00:00-07:00"]) - (dc/q '[:find ?n + (dc/q '[:find ?n :in $ [?clients ?start-date ?end-date] :where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]] [?e :sales-order/line-items ?li] - [?li :order-line-item/item-name ?n] ] + [?li :order-line-item/item-name ?n]] (dc/db conn) [[(auto-ap.datomic/pull-attr (dc/db conn) :db/id [:client/code "NGCL"])] #inst "2024-04-11T00:00:00-07:00" #inst "2024-04-24T00:00:00-07:00"]) - -@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy} - {:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}]) - -(auto-ap.datomic/transact-schema conn) - - ) + @(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy} + {:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}]) + (auto-ap.datomic/transact-schema conn)) (defn -main [& _] (execute "sales-summaries" sales-summaries-v2)) - \ No newline at end of file diff --git a/src/clj/auto_ap/migration/cleanup_sales.clj b/src/clj/auto_ap/migration/cleanup_sales.clj index 0e1eb4f2..4b1003be 100644 --- a/src/clj/auto_ap/migration/cleanup_sales.clj +++ b/src/clj/auto_ap/migration/cleanup_sales.clj @@ -17,8 +17,8 @@ "Return all entity IDs that have :sales-order/external-id." [db] (->> (d-api/q '[:find ?e - :where [?e :sales-order/external-id]] - db) + :where [?e :sales-order/external-id]] + db) (map first))) (defn- collect-child-ids @@ -28,19 +28,19 @@ [db order-ids] (let [order-set (set order-ids) charges (->> (d-api/q '[:find ?c - :in $ [?o ...] - :where [$ ?o :sales-order/charges ?c]] - db order-set) + :in $ [?o ...] + :where [$ ?o :sales-order/charges ?c]] + db order-set) (map second)) refunds (->> (d-api/q '[:find ?r - :in $ [?o ...] - :where [$ ?o :sales-order/refunds ?r]] - db order-set) + :in $ [?o ...] + :where [$ ?o :sales-order/refunds ?r]] + db order-set) (map second)) line-items (->> (d-api/q '[:find ?li - :in $ [?c ...] - :where [$ ?c :charge/line-items ?li]] - db charges) + :in $ [?c ...] + :where [$ ?c :charge/line-items ?li]] + db charges) (map second))] {:orders order-ids :charges (vec charges) @@ -149,7 +149,7 @@ all-ids (query-sales-order-ids db) group (group-orders-by-month db all-ids) target-keys (get group [year month] [])] - (if (zero? (count target-keys)) + (if (zero? (count target-keys)) (do (println " no orders found for" year "-" month) :skipped) (do @@ -202,18 +202,18 @@ missing (:missing result)] (cond (:ok result) - (do (println "verified" y "-" m "S3 OK, deleting...") - (delete-by-month conn$ nil y m)) + (do (println "verified" y "-" m "S3 OK, deleting...") + (delete-by-month conn$ nil y m)) (> (count missing) 0) - (do (println "ERROR" y "-" m "missing in S3:" - (str/join ", " missing)) - (throw - (ex-info - "Missing S3 data — aborting!" - {:year y :month m - :missing missing}))) + (do (println "ERROR" y "-" m "missing in S3:" + (str/join ", " missing)) + (throw + (ex-info + "Missing S3 data — aborting!" + {:year y :month m + :missing missing}))) :else - (println "SKIPPING" y "-" m "no parquet files"))))) + (println "SKIPPING" y "-" m "no parquet files"))))) (println "safe-cleanup-all complete"))) diff --git a/src/clj/auto_ap/migration/sales_to_parquet.clj b/src/clj/auto_ap/migration/sales_to_parquet.clj index 84717227..0b50b3cb 100644 --- a/src/clj/auto_ap/migration/sales_to_parquet.clj +++ b/src/clj/auto_ap/migration/sales_to_parquet.clj @@ -42,7 +42,7 @@ :charge/date :charge/processor :charge/returns - {:charge/client [:client/code]}]} + {:charge/client [:client/code]}]} {:sales-order/line-items [:order-line-item/item-name :order-line-item/category @@ -122,13 +122,12 @@ (->> (dc/q '[:find ?e :in $ ?start-ms ?end-ms :where [_ :sales-order/date ?d] - [(>= ?d ?start-ms)] - [(<= ?d ?end-ms)]] + [(>= ?d ?start-ms)] + [(<= ?d ?end-ms)]] db start end) (map first) vec))) - (defn- date-seq [start end] "Seq of YYYY-MM-DD strings between start and end inclusive." (let [sd (java.time.LocalDate/parse start) @@ -163,7 +162,7 @@ "line-item" "sales-refund"]] (p/flush-to-parquet! etype)) (println "[migration]" day "complete")) - {:status :completed :total-days (count date-range)}))) + {:status :completed :total-days (count date-range)}))) (defn- write-dead-letter ([flat] @@ -192,11 +191,11 @@ (defn- get-date-range [] "Get the earliest and latest business dates from Datomic." (let [dates (->> (dc/q '[:find ?d - :where [_ :sales-order/date ?d]] - (dc/db conn)) - (map first) - distinct - sort)] + :where [_ :sales-order/date ?d]] + (dc/db conn)) + (map first) + distinct + sort)] [(when (seq dates) (.toString (first dates))) (when (seq dates) (.toString (last dates)))])) @@ -220,7 +219,7 @@ (let [flat (volatile! [])] (flatten-order-to-pieces! o flat) (doseq [r @flat] - (p/buffer! "dead" r)))) + (p/buffer! "dead" r)))) (write-day-by-day start-date end-date {:batch-size 100}) (flush-all-types) (println "[migration] done") diff --git a/src/clj/auto_ap/routes/ezcater_xls.clj b/src/clj/auto_ap/routes/ezcater_xls.clj index 26e6e7e9..c5a76022 100644 --- a/src/clj/auto_ap/routes/ezcater_xls.clj +++ b/src/clj/auto_ap/routes/ezcater_xls.clj @@ -55,54 +55,54 @@ event-date (some-> (excel/xls-date->date event-date) coerce/to-date-time atime/as-local-time - coerce/to-date )] - (cond (and event-date client-id location ) + coerce/to-date)] + (cond (and event-date client-id location) [:order #:sales-order - {:date event-date - :external-id (str "ezcater/order/" client-id "-" location "-" order-number) - :client client-id - :location location - :reference-link (str order-number) - :line-items [#:order-line-item - {:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0) - :item-name "EZCater Catering" - :category "EZCater Catering" - :discount (fmt-amount (or adjustments 0.0)) - :tax (fmt-amount tax) - :total (fmt-amount (+ food-total - tax))}] + {:date event-date + :external-id (str "ezcater/order/" client-id "-" location "-" order-number) + :client client-id + :location location + :reference-link (str order-number) + :line-items [#:order-line-item + {:external-id (str "ezcater/order/" client-id "-" location "-" order-number "-" 0) + :item-name "EZCater Catering" + :category "EZCater Catering" + :discount (fmt-amount (or adjustments 0.0)) + :tax (fmt-amount tax) + :total (fmt-amount (+ food-total + tax))}] - :charges [#:charge - {:type-name "CARD" - :date event-date - :client client-id - :location location - :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0) - :processor :ccp-processor/ezcater - :total (fmt-amount (+ food-total - tax - tip)) - :tip (fmt-amount tip)}] - :total (fmt-amount (+ food-total - tax - (or adjustments 0.0))) - :discount (fmt-amount (or adjustments 0.0)) - :service-charge (fmt-amount (+ fee commission)) - :tax (fmt-amount tax) - :tip (fmt-amount tip) - :returns 0.0 - :vendor :vendor/ccp-ezcater}] + :charges [#:charge + {:type-name "CARD" + :date event-date + :client client-id + :location location + :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0) + :processor :ccp-processor/ezcater + :total (fmt-amount (+ food-total + tax + tip)) + :tip (fmt-amount tip)}] + :total (fmt-amount (+ food-total + tax + (or adjustments 0.0))) + :discount (fmt-amount (or adjustments 0.0)) + :service-charge (fmt-amount (+ fee commission)) + :tax (fmt-amount tax) + :tip (fmt-amount tip) + :returns 0.0 + :vendor :vendor/ccp-ezcater}] - caterer-name - (do - (alog/warn ::missing-client - :order order-number - :store-name store-name - :caterer-name caterer-name) - [:missing caterer-name]) + caterer-name + (do + (alog/warn ::missing-client + :order order-number + :store-name store-name + :caterer-name caterer-name) + [:missing caterer-name]) - :else - nil))) + :else + nil))) (defn- flatten-order-to-parquet! [order] "Flatten a sales-order into entity-type tagged maps and buffer to parquet." @@ -125,25 +125,25 @@ (when-let [charges (:sales-order/charges order)] (doseq [chg charges] (parquet/buffer! "charge" - {:entity-type "charge" - :external-id (:charge/external-id chg) - :type-name (:charge/type-name chg) - :total (:charge/total chg) - :tax (:charge/tax chg) - :tip (:charge/tip chg) - :date so-date - :processor (some-> (:charge/processor chg) name) - :sales-order-external-id so-ext-id}))) + {:entity-type "charge" + :external-id (:charge/external-id chg) + :type-name (:charge/type-name chg) + :total (:charge/total chg) + :tax (:charge/tax chg) + :tip (:charge/tip chg) + :date so-date + :processor (some-> (:charge/processor chg) name) + :sales-order-external-id so-ext-id}))) (when-let [items (:sales-order/line-items order)] (doseq [li items] (parquet/buffer! "line-item" - {:entity-type "line-item" - :item-name (:order-line-item/item-name li) - :category (:order-line-item/category li) - :total (:order-line-item/total li) - :tax (:order-line-item/tax li) - :discount (:order-line-item/discount li) - :sales-order-external-id so-ext-id}))))) + {:entity-type "line-item" + :item-name (:order-line-item/item-name li) + :category (:order-line-item/category li) + :total (:order-line-item/total li) + :tax (:order-line-item/tax li) + :discount (:order-line-item/discount li) + :sales-order-external-id so-ext-id}))))) (defn stream->sales-orders [s] (let [clients (map first (dc/q '[:find (pull ?c [:client/code @@ -158,7 +158,7 @@ object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))] (mu/log ::writing-temp-xls :location object) - (s3/put-object {:bucket-name (:data-bucket env) + (s3/put-object {:bucket-name (:data-bucket env) :key object :input-stream s}) (into [] @@ -200,13 +200,13 @@ });")]])]) (defn upload-xls [{:keys [identity] :as request}] - + (let [file (or (get (:params request) :file) (get (:params request) "file"))] (mu/log ::uploading-file :file file) (with-open [s (io/input-stream (:tempfile file))] - (try + (try (let [parse-results (stream->sales-orders s) new-orders (->> parse-results (filter (comp #{:order} first)) @@ -235,7 +235,7 @@ [:li ml])]])])) (catch Exception e (alog/error ::import-error - :error e) + :error e) (html-response [:div (.getMessage e)])))))) (defn page [{:keys [matched-route request-method] :as request}] diff --git a/src/clj/auto_ap/square/core3.clj b/src/clj/auto_ap/square/core3.clj index 59234452..f2195398 100644 --- a/src/clj/auto_ap/square/core3.clj +++ b/src/clj/auto_ap/square/core3.clj @@ -28,11 +28,9 @@ "Authorization" (str "Bearer " (:client/square-auth-token client)) "Content-Type" "application/json"})) - (defn ->square-date [d] (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") d)) - (def manifold-api-stream (let [stream (s/stream 100)] (->> stream @@ -43,10 +41,10 @@ (de/loop [attempt 0] (-> (de/chain (de/future-with (ex/execute-pool) #_(log/info ::request-started - :url (:url request) - :attempt attempt - :source "Square 3" - :background-job "Square 3") + :url (:url request) + :attempt attempt + :source "Square 3" + :background-job "Square 3") (try (client/request (assoc request :socket-timeout 10000 @@ -105,7 +103,6 @@ :exception error)) [])))) - (def item-cache (atom {})) (defn fetch-catalog [client i v] @@ -125,13 +122,11 @@ #(do (swap! item-cache assoc i %) %)))) - (defn fetch-catalog-cache [client i version] (if (get @item-cache i) (de/success-deferred (get @item-cache i)) (fetch-catalog client i version))) - (defn item->category-name-impl [client item version] (capture-context->lc (cond (:item_id (:item_variation_data item)) @@ -162,7 +157,6 @@ :item item) "Uncategorized")))) - (defn item-id->category-name [client i version] (capture-context->lc (-> [client i] @@ -227,7 +221,6 @@ (concat (:orders result) continued-results)))) (:orders result))))))) - (defn search ([client location start end] (capture-context->lc @@ -251,11 +244,9 @@ (concat (:orders result) continued-results)))) (:orders result)))))))) - (defn amount->money [amt] (* 0.01 (or (:amount amt) 0.0))) - ;; to get totals: (comment (reduce @@ -281,7 +272,7 @@ :reference-link (str (url/url "https://squareup.com/receipt/preview" (:id t))) :external-id (when (:id t) (str "square/charge/" (:id t))) - :processor (cond + :processor (cond (#{"OTHER" "THIRD_PARTY_CARD"} (:type t)) (condp = (some-> (:note t) str/lower-case) "doordash" :ccp-processor/doordash @@ -354,7 +345,7 @@ #:sales-order {:date (if (= "Invoices" (:name (:source order))) (when (:closed_at order) - (coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles")))) + (coerce/to-date (time/to-time-zone (coerce/to-date-time (:closed_at order)) (time/time-zone-for-id "America/Los_Angeles")))) (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles")))) :client (:db/id client) :location (:square-location/client-location location) @@ -416,7 +407,6 @@ :client client :location location))))))) - (defn get-payment [client p] (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/payments/" p) @@ -425,7 +415,6 @@ :body :payment)) - (defn continue-payout-entry-list [c l poi cursor] (capture-context->lc lc (de/chain @@ -610,8 +599,8 @@ so-date (some-> (:sales-order/date order) .toString) client (:sales-order/client order) client-code (when client (if (map? client) - (:client/code client) - client))] + (:client/code client) + client))] (parquet/buffer! "sales-order" {:entity-type "sales-order" :external-id so-ext-id @@ -627,32 +616,32 @@ (when-let [charges (:sales-order/charges order)] (doseq [chg charges] (parquet/buffer! "charge" - {:entity-type "charge" - :external-id (:charge/external-id chg) - :type-name (:charge/type-name chg) - :total (:charge/total chg) - :tax (:charge/tax chg) - :tip (:charge/tip chg) - :date so-date - :processor (some-> (:charge/processor chg) name) - :sales-order-external-id so-ext-id}) + {:entity-type "charge" + :external-id (:charge/external-id chg) + :type-name (:charge/type-name chg) + :total (:charge/total chg) + :tax (:charge/tax chg) + :tip (:charge/tip chg) + :date so-date + :processor (some-> (:charge/processor chg) name) + :sales-order-external-id so-ext-id}) (when-let [returns (:charge/returns chg)] (doseq [rt returns] (parquet/buffer! "sales-refund" - {:entity-type "sales-refund" - :type-name (:type-name rt) - :total (:total rt) - :sales-order-external-id so-ext-id}))))) + {:entity-type "sales-refund" + :type-name (:type-name rt) + :total (:total rt) + :sales-order-external-id so-ext-id}))))) (when-let [items (:sales-order/line-items order)] (doseq [li items] (parquet/buffer! "line-item" - {:entity-type "line-item" - :item-name (:order-line-item/item-name li) - :category (:order-line-item/category li) - :total (:order-line-item/total li) - :tax (:order-line-item/tax li) - :discount (:order-line-item/discount li) - :sales-order-external-id so-ext-id}))))) + {:entity-type "line-item" + :item-name (:order-line-item/item-name li) + :category (:order-line-item/category li) + :total (:order-line-item/total li) + :tax (:order-line-item/tax li) + :discount (:order-line-item/discount li) + :sales-order-external-id so-ext-id}))))) (defn upsert ([client] @@ -674,8 +663,7 @@ (catch Exception e (log/error ::buffer-failed :exception e - :order (:sales-order/external-id order))))))))))) - + :order (:sales-order/external-id order)))))))))))) (defn upsert-payouts ([client] @@ -725,7 +713,6 @@ (log/info ::done-loading-refunds))))))) - (defn get-cash-shift [client id] (de/chain (manifold-api-call {:url (str (url/url "https://connect.squareup.com/v2/cash-drawers/shifts" id)) :method :get @@ -884,8 +871,6 @@ d1 d2)) - - (defn remove-voided-orders ([client] (apply de/zip @@ -912,7 +897,7 @@ (:sales-order/external-id o)))))) (s/map (fn [[o]] [[:db/retractEntity [:sales-order/external-id (:sales-order/external-id o)]]])) - + (s/reduce into []))) (fn [results] @@ -920,32 +905,28 @@ (doseq [x (partition-all 100 results)] (log/info ::removing-orders :count (count x)) - @(dc/transact-async conn x))))) + @(dc/transact-async conn x) (de/catch (fn [e] (log/warn ::couldnt-remove :error e) - nil) )))))) + nil))))))))))) -#_(comment - (require 'auto-ap.time-reader) +#_(comment + (require 'auto-ap.time-reader) - @(let [[c [l]] (get-square-client-and-location "DBFS") ] - (log/peek :x [ c l]) - (search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29") + @(let [[c [l]] (get-square-client-and-location "DBFS")] + (log/peek :x [c l]) + (search c l #clj-time/date-time "2026-03-28" #clj-time/date-time "2026-03-29")) - ) + @(let [[c [l]] (get-square-client-and-location "NGAK")] + (log/peek :x [c l]) - @(let [[c [l]] (get-square-client-and-location "NGAK") ] - (log/peek :x [ c l]) - - (remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15")) - (doseq [c (get-square-clients)] - (try - @(remove-voided-orders c) - (catch Exception e - nil))) - - - ) + (remove-voided-orders c l #clj-time/date-time "2024-04-11" #clj-time/date-time "2024-04-15")) + (doseq [c (get-square-clients)] + (try + @(remove-voided-orders c) + (catch Exception e + nil))) + ) (defn upsert-all [& clients] (capture-context->lc @@ -1014,8 +995,6 @@ [:clients clients] @(apply upsert-all clients))) - - (comment (defn refunds-raw-cont ([client l cursor so-far] @@ -1045,9 +1024,8 @@ (->> @(let [[c [l]] (get-square-client-and-location "NGGG")] - (search c l (time/now) (time/plus (time/now) (time/days -1)))) - + (filter (fn [r] (str/starts-with? (:created_at r) "2024-03-14")))) @@ -1055,7 +1033,6 @@ (->> @(let [[c [l]] (get-square-client-and-location "NGGG")] - (refunds-raw-cont c l nil [])) (filter (fn [r] (str/starts-with? (:created_at r) "2024-03-14"))))) @@ -1089,13 +1066,8 @@ []))] [(:client/code c) (atime/unparse-local (clj-time.coerce/to-date-time (:sales-order/date bad-row)) atime/normal-date) (:sales-order/total bad-row) (:sales-order/tax bad-row) (:sales-order/tip bad-row) (:db/id bad-row)]) :separator \tab) - - - - - ;; => - +;; => (require 'auto-ap.time-reader) @@ -1104,27 +1076,16 @@ (clojure.pprint/pprint (let [[c [l]] (get-square-client-and-location "NGVT")] l - (def z @(search c l #clj-time/date-time "2025-02-23T00:00:00-08:00" #clj-time/date-time "2025-02-28T00:00:00-08:00")) - (take 10 (map #(first (deref (order->sales-order c l %))) z))) + (take 10 (map #(first (deref (order->sales-order c l %))) z)))) - - ) - - - - - (->> z + (->> z (filter (fn [o] (seq (filter (comp #{"OTHER"} :type) (:tenders o))))) (filter #(not (:name (:source %)))) - (count) - - ) - - - + (count)) + (doseq [[code] (seq (dc/q '[:find ?code :in $ :where [?o :sales-order/date ?d] @@ -1133,32 +1094,22 @@ [?o :sales-order/client ?c] [?c :client/code ?code]] (dc/db conn))) - :let [[c [l]] (get-square-client-and-location code) - ] + :let [[c [l]] (get-square-client-and-location code)] order @(search c l #clj-time/date-time "2026-01-01T00:00:00-08:00" (time/now)) - :when (= "Invoices" (:name (:source order) )) + :when (= "Invoices" (:name (:source order))) :let [[sales-order] @(order->sales-order c l order)]] - + (when (should-import-order? order) (println "DATE IS" (:sales-order/date sales-order)) (when (some-> (:sales-order/date sales-order) coerce/to-date-time (time/after? #clj-time/date-time "2026-2-16T00:00:00-08:00")) (println "WOULD UPDATE" sales-order) - @(dc/transact auto-ap.datomic/conn [sales-order]) - ) - #_@(dc/transact ) - (println "DONE")) - - - ) + @(dc/transact auto-ap.datomic/conn [sales-order])) + #_@(dc/transact) + (println "DONE"))) #_(filter (comp #{"OTHER"} :type) (mapcat :tenders z)) - @(let [[c [l]] (get-square-client-and-location "NGRY")] #_(search c l (clj-time.coerce/from-date #inst "2025-02-28") (clj-time.coerce/from-date #inst "2025-03-01")) - (order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY" ))) - - ) - - ) + (order->sales-order c l (:order (get-order c l "KdvwntmfMNTKBu8NOocbxatOs18YY"))))) diff --git a/src/clj/auto_ap/ssr/payments.clj b/src/clj/auto_ap/ssr/payments.clj index a7ed513c..669a1a75 100644 --- a/src/clj/auto_ap/ssr/payments.clj +++ b/src/clj/auto_ap/ssr/payments.clj @@ -104,19 +104,18 @@ :size :small})]) (com/field {:label "Payment Type"} (com/radio-card {:size :small - :name "payment-type" - :value (:payment-type (:query-params request)) - :options [{:value "" - :content "All"} - {:value "cash" - :content "Cash"} - {:value "check" - :content "Check"} - {:value "debit" - :content "Debit"}]})) + :name "payment-type" + :value (:payment-type (:query-params request)) + :options [{:value "" + :content "All"} + {:value "cash" + :content "Cash"} + {:value "check" + :content "Check"} + {:value "debit" + :content "Debit"}]})) (exact-match-id* request)]]) - (def default-read '[* [:payment/date :xform clj-time.coerce/from-date] {:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]} @@ -212,7 +211,6 @@ '[(iol-ion.query/dollars= ?transaction-amount ?amount)]]} :args [(:amount query-params)]}) - (:status route-params) (merge-query {:query {:in ['?status] :where ['[?e :payment/status ?status]]} @@ -243,30 +241,30 @@ refunds)) (defn sum-visible-pending [ids] - (->> - (dc/q {:find ['?id '?o] - :in ['$ '[?id ...]] - :where ['[?id :payment/amount ?o] - '[?id :payment/status :payment-status/pending]]} - (dc/db conn) - ids) + (->> + (dc/q {:find ['?id '?o] + :in ['$ '[?id ...]] + :where ['[?id :payment/amount ?o] + '[?id :payment/status :payment-status/pending]]} + (dc/db conn) + ids) (map last) (reduce + 0.0))) (defn sum-client-pending [clients] - (->> - (dc/q {:find '[?e ?a] - :in '[$ [?clients ?start ?end]] - :where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]] - [?e :payment/status :payment-status/pending] - [?e :payment/amount ?a]]} - (dc/db conn) - [clients - nil - nil]) - + (->> + (dc/q {:find '[?e ?a] + :in '[$ [?clients ?start ?end]] + :where '[[(iol-ion.query/scan-payments $ ?clients ?start ?end) [[?e _ ?sort-default] ...]] + [?e :payment/status :payment-status/pending] + [?e :payment/amount ?a]]} + (dc/db conn) + [clients + nil + nil]) + (map last) (reduce + @@ -277,16 +275,14 @@ {ids-to-retrieve :ids matching-count :count all-ids :all-ids} (fetch-ids db request)] - [(->> (hydrate-results ids-to-retrieve db request)) matching-count (sum-visible-pending all-ids) (sum-client-pending (extract-client-ids (:clients request) - (:client request) - (:client-id (:query-params request)) - (when (:client-code (:query-params request)) - [:client/code (:client-code (:query-params request))]))) - ])) + (:client request) + (:client-id (:query-params request)) + (when (:client-code (:query-params request)) + [:client/code (:client-code (:query-params request))])))])) (def query-schema (mc/schema [:maybe [:map {:date-range [:date-range :start-date :end-date]} @@ -327,7 +323,7 @@ (assoc-in (exact-match-id* request) [1 :hx-swap-oob] true)]) :query-schema query-schema :action-buttons (fn [request] - (let [[_ _ visible-in-float total-in-float ] (:page-results request)] + (let [[_ _ visible-in-float total-in-float] (:page-results request)] [(com/pill {:color :primary} " Visible in float " (format "$%,.2f" visible-in-float)) (com/pill {:color :secondary} " Total in float " @@ -354,7 +350,7 @@ (= (-> request :query-params :sort first :name) "Bank account") (-> entity :payment/bank-account :bank-account/name) - + :else nil)) :title (fn [r] (str @@ -409,7 +405,7 @@ :render (fn [{:payment/keys [date]}] (some-> date (atime/unparse-local atime/normal-date)))} {:key "amount" - :sort-key "amount" + :sort-key "amount" :name "Amount" :render (fn [{:payment/keys [amount]}] (some->> amount (format "$%.2f")))} @@ -421,10 +417,10 @@ (map :invoice-payment/invoice) (filter identity) (map (fn [invoice] - {:link (hu/url (bidi/path-for ssr-routes/only-routes - ::invoice-route/all-page) - {:exact-match-id (:db/id invoice)}) - :content (str "Inv. " (:invoice/invoice-number invoice))}))) + {:link (hu/url (bidi/path-for ssr-routes/only-routes + ::invoice-route/all-page) + {:exact-match-id (:db/id invoice)}) + :content (str "Inv. " (:invoice/invoice-number invoice))}))) (some-> p :transaction/_payment ((fn [t] [{:link (hu/url (bidi/path-for client-routes/routes :transactions) @@ -434,8 +430,6 @@ (def row* (partial helper/row* grid-page)) - - (comment (mc/decode query-schema {"exact-match-id" "123"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) @@ -445,7 +439,6 @@ (mc/decode query-schema {"payment-type" "food"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) (mc/decode query-schema {"vendor" "87"} (mt/transformer main-transformer mt/strip-extra-keys-transformer)) - (mc/decode query-schema {"start-date" #inst "2023-12-21T08:00:00.000-00:00"} (mt/transformer main-transformer mt/strip-extra-keys-transformer))) (defn delete [{check :entity :as request identity :identity}] @@ -459,7 +452,7 @@ #(assert-can-see-client identity (:db/id (:payment/client check)))) (notify-if-locked (:db/id (:payment/client check)) (:payment/date check)) - (let [ removing-payments (mapcat (fn [x] + (let [removing-payments (mapcat (fn [x] (let [invoice (:invoice-payment/invoice x) new-balance (+ (:invoice/outstanding-balance invoice) (:invoice-payment/amount x))] @@ -475,9 +468,9 @@ :payment/status :payment-status/voided}] (audit-transact (cond-> removing-payments true (conj updated-payment) - (:transaction/_payment check) (conj [:db/retract (:db/id (first (:transaction/_payment check))) + (:transaction/_payment check) (conj [:db/retract (:db/id (first (:transaction/_payment check))) :transaction/payment - (:db/id check)])) + (:db/id check)])) identity) (html-response (row* (:identity request) updated-payment {:delete-after-settle? true :class "live-removed" @@ -578,7 +571,6 @@ (assoc-in [:query-params :start] 0) (assoc-in [:query-params :per-page] 250)))) - :else selected) updated-count (void-payments-internal ids (:identity request))] @@ -591,7 +583,7 @@ (defn wrap-status-from-source [handler] (fn [{:keys [matched-current-page-route] :as request}] - (let [ request (cond-> request + (let [request (cond-> request (= ::route/cleared-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/cleared) (= ::route/pending-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/pending) (= ::route/voided-page matched-current-page-route) (assoc-in [:route-params :status] :payment-status/voided) @@ -605,7 +597,7 @@ ::route/pending-page (-> (helper/page-route grid-page) (wrap-implied-route-param :status :payment-status/pending)) ::route/voided-page (-> (helper/page-route grid-page) - (wrap-implied-route-param :status :payment-status/voided)) + (wrap-implied-route-param :status :payment-status/voided)) ::route/all-page (-> (helper/page-route grid-page) (wrap-implied-route-param :status nil)) @@ -618,7 +610,6 @@ ::route/bulk-delete (-> bulk-delete-dialog (wrap-admin)) - ::route/table (helper/table-route grid-page)} (fn [h] (-> h diff --git a/src/clj/auto_ap/storage/parquet.clj b/src/clj/auto_ap/storage/parquet.clj index 54b4548a..7191f2bf 100644 --- a/src/clj/auto_ap/storage/parquet.clj +++ b/src/clj/auto_ap/storage/parquet.clj @@ -22,6 +22,10 @@ (let [conn (DriverManager/getConnection "jdbc:duckdb:") stmt (.createStatement conn)] (.execute stmt "INSTALL httpfs; LOAD httpfs;") + (when-let [key (:aws-access-key-id env)] + (.execute stmt (str "SET s3_access_key_id='" key "'")) + (.execute stmt (str "SET s3_secret_access_key='" (:aws-secret-access-key env) "'")) + (.execute stmt (str "SET s3_region='" (or (:aws-region env) "us-east-1") "'"))) (.close stmt) (.addShutdownHook (Runtime/getRuntime) (Thread. #(fn []))) @@ -107,9 +111,9 @@ (str entity-type ".jsonl"))] (io/make-parents wal-file) (with-open [w (io/writer wal-file :append true)] - (.write w ^String (json/write-str {:seq-no seq-no - :record record})) - (.write w (int \newline)))) + (.write w ^String (json/write-str {:seq-no seq-no + :record record})) + (.write w (int \newline)))) (catch Exception e (println "[parquet/wal]" (.getMessage e)))) entry)) @@ -138,8 +142,8 @@ (try (with-open [w (io/writer jsonl-file :append true)] (doseq [r records] - (.write w ^String (json/write-str (dissoc r :_seq-no))) - (.write w (int \newline)))) + (.write w ^String (json/write-str (dissoc r :_seq-no))) + (.write w (int \newline)))) (execute-to-parquet! (format "SELECT * FROM read_json_auto('%s')" (.getAbsolutePath jsonl-file)) @@ -150,8 +154,9 @@ (.delete ^java.io.File parquet-file) {:key s3-key :status :ok} (catch Exception e - (throw (ex-info "Flush failed" {:entity-type entity-type} - :error (.getMessage e))))))))) + (throw (ex-info "Flush failed" + {:entity-type entity-type + :error (.getMessage e)})))))))) (defn flush-by-date! [] "Flush all entity types for today." @@ -218,8 +223,9 @@ Returns map with :sql and :count-sql keys." (let [date-strs (date-seq start-date end-date) urls (vec - (map #(format "'s3://%%s/sales-details/%%s/%%s.parquet'" - *bucket* entity-type %) + (map (fn [d] + (format "'s3://%s/sales-details/%s/%s.parquet'" + *bucket* entity-type d)) date-strs)) sql (str "SELECT * FROM read_parquet([" (str/join ", " urls) @@ -244,35 +250,34 @@ (get-sales-orders start-date end-date {})) ([start-date end-date opts] (let [q (parquet-query "sales-order" - start-date end-date) - base-sql (:sql q) - count-sql (:count-sql q) - sort (get opts :sort "date") - order (get opts :order "DESC") - limit (get opts :limit) - offset (get opts :offset) + start-date end-date) + base-sql (:sql q) + count-sql (:count-sql q) + sort (get opts :sort "date") + order (get opts :order "DESC") + limit (get opts :limit) + offset (get opts :offset) where-str (build-where-clause - opts - [[:client "external_id.client"] - [:vendor "external_id.vendor"] + [[:client "client-code"] + [:vendor "vendor"] [:location "location"]]) - full-sql (if where-str - (str base-sql where-str) - base-sql) - result (cond-> full-sql - sort (str " ORDER BY " sort - " " (name order)) - limit (str " LIMIT " limit) - offset (str " OFFSET " offset)) - full-count (if where-str - (str count-sql where-str) - count-sql)] - {:rows (query-rows result) - :count (or - (int - (query-scalar - full-count)) 0)}))) + full-sql (if where-str + (str base-sql where-str) + base-sql) + result (cond-> full-sql + sort (str " ORDER BY " sort + " " (name order)) + limit (str " LIMIT " limit) + offset (str " OFFSET " offset)) + full-count (if where-str + (str count-sql where-str) + count-sql)] + {:rows (query-rows result) + :count (or + (int + (query-scalar + full-count)) 0)}))) (defn query-deduped [entity-type start-date end-date] "Query records deduplicated by external-id (latest _seq_no wins)." diff --git a/src/clj/auto_ap/storage/sales_summaries.clj b/src/clj/auto_ap/storage/sales_summaries.clj index 12e8068c..23d45a20 100644 --- a/src/clj/auto_ap/storage/sales_summaries.clj +++ b/src/clj/auto_ap/storage/sales_summaries.clj @@ -55,7 +55,7 @@ tname (+ (get b tname 0.0) total))))))) {} - rows))) + rows))) (catch Exception e (println "[sales-summaries]" (.getMessage e)) {}))))