From 05f1f009fdf9b590a89c9491bf0b6daace865c98 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 9 May 2023 16:32:00 -0700 Subject: [PATCH] Adds cloud approach to reading excel --- project.clj | 3 + src/clj/auto_ap/routes/ezcater_xls.clj | 64 ++++++++++++------- .../integration/routes/ezcater_xls.clj | 2 +- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/project.clj b/project.clj index 2c371aa1..1a5039ef 100644 --- a/project.clj +++ b/project.clj @@ -69,6 +69,9 @@ [com.amazonaws/aws-java-sdk-sqs "1.11.926" :exclusions [commons-codec org.apache.httpcomponents/httpclient]] + [com.amazonaws/aws-java-sdk-lambda "1.11.926" + :exclusions [commons-codec + org.apache.httpcomponents/httpclient]] [com.amazonaws/aws-java-sdk-ecs "1.11.926" :exclusions [commons-codec diff --git a/src/clj/auto_ap/routes/ezcater_xls.clj b/src/clj/auto_ap/routes/ezcater_xls.clj index a78b01c0..8bd53bc4 100644 --- a/src/clj/auto_ap/routes/ezcater_xls.clj +++ b/src/clj/auto_ap/routes/ezcater_xls.clj @@ -2,7 +2,10 @@ (:require [auto-ap.datomic :refer [audit-transact conn]] [auto-ap.logging :as alog] + [clojure.data.json :as json] [auto-ap.parse :as parse] + [amazonica.aws.lambda :as lambda] + [config.core :refer [env]] [auto-ap.shared-views.admin.side-bar :refer [admin-side-bar]] [auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr.ui :refer [base-page]] @@ -14,7 +17,8 @@ [com.brunobonacci.mulog :as mu] [datomic.api :as dc] [dk.ative.docjure.spreadsheet :as doc] - [hiccup2.core :as hiccup])) + [hiccup2.core :as hiccup] + [amazonica.aws.s3 :as s3])) (defn fmt-amount [a] (with-precision 2 @@ -23,15 +27,15 @@ (.setScale 2 java.math.RoundingMode/HALF_UP) (double)))) -(defn extract-sheet-details [f] - (into [] - (for [row (->> (doc/load-workbook f) - (doc/sheet-seq) - first - (doc/row-seq) - )] - (mapv doc/read-cell (doc/cell-seq row)) - ))) +(defn extract-sheet-details [bucket object] + (-> (lambda/invoke {:function-name "xls-extractor" :payload + (json/write-str + {"s3_url" object "s3_bucket" bucket})} + ) + :payload + slurp + json/read-str)) + (defn rows->maps [rows] (let [[headers & rows] rows] @@ -39,26 +43,37 @@ (into {} (map vector headers r))))) +(defn xls-date->date [f] + (when (not-empty f) + (let [f (Double/parseDouble f) + unix-days (- f 25569.0) + unix-secs (* unix-days 86400.0)] + (java.util.Date. (long (Math/round (* 1000.0 unix-secs))))))) + (defn map->sales-order [r clients] (let [order-number (get r "Order Number") event-date (get r "Event Date") store-name (get r "Store Name") - adjustments (get r "Adjustments") - tax (get r "Sales Tax") - food-total (get r "Food Total") - commission (get r "Commission") - fee (get r "Payment Transaction Fee") - tip (get r "Tip") + adjustments (some-> (get r "Adjustments") not-empty (Double/parseDouble)) + tax (some-> (get r "Sales Tax") not-empty (Double/parseDouble)) + food-total (some-> (get r "Food Total") not-empty (Double/parseDouble)) + commission (some-> (get r "Commission") not-empty (Double/parseDouble)) + fee (some-> (get r "Payment Transaction Fee") not-empty (Double/parseDouble)) + tip (some-> (get r "Tip") not-empty (Double/parseDouble)) caterer-name (get r "Caterer Name") client (some->> caterer-name not-empty (parse/exact-match clients)) client-id (:db/id client) - location (first (:client/locations client))] + location (first (:client/locations client)) + event-date (some-> (xls-date->date event-date) + coerce/to-date-time + atime/as-local-time + coerce/to-date )] (cond (and event-date client-id location ) [:order #:sales-order - {:date (coerce/to-date (atime/localize (coerce/to-date-time event-date))) + {:date event-date :external-id (str "ezcater/order/" client-id "-" location "-" order-number) :client client-id :location location @@ -75,7 +90,7 @@ :charges [#:charge {:type-name "CARD" - :date (coerce/to-date (atime/localize (coerce/to-date-time event-date))) + :date event-date :client client-id :location location :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0) @@ -114,10 +129,15 @@ :client/matches :client/locations]) :where [?c :client/code]] - (dc/db conn)))] + (dc/db conn))) + object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))] + (mu/log ::writing-temp-xls + :location object) + (s3/put-object {:bucket-name (:data-bucket env) + :key object + :input-stream s}) (into [] - (->> s - extract-sheet-details + (->> (extract-sheet-details (:data-bucket env) object) rows->maps (map #(map->sales-order % clients)) (filter identity))))) diff --git a/test/clj/auto_ap/integration/routes/ezcater_xls.clj b/test/clj/auto_ap/integration/routes/ezcater_xls.clj index 0e4eecbf..4559a217 100644 --- a/test/clj/auto_ap/integration/routes/ezcater_xls.clj +++ b/test/clj/auto_ap/integration/routes/ezcater_xls.clj @@ -8,7 +8,7 @@ (use-fixtures :each wrap-setup) -(deftest stream->sales-ordersx +(deftest stream->sales-orders (testing "Should import nothing when there are no clients" (with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))] (is (= [:missing "Nick The Greek (Santa Cruz)"] (first (sut/stream->sales-orders s))))))