Adds cloud approach to reading excel

project.clj

@@ -69,6 +69,9 @@
                  [com.amazonaws/aws-java-sdk-sqs "1.11.926"
                   :exclusions [commons-codec
                                org.apache.httpcomponents/httpclient]]
+                 [com.amazonaws/aws-java-sdk-lambda "1.11.926"
+                  :exclusions [commons-codec
+                               org.apache.httpcomponents/httpclient]]
 
                  [com.amazonaws/aws-java-sdk-ecs "1.11.926"
                   :exclusions [commons-codec

src/clj/auto_ap/routes/ezcater_xls.clj

@@ -2,7 +2,10 @@
   (:require
    [auto-ap.datomic :refer [audit-transact conn]]
    [auto-ap.logging :as alog]
+   [clojure.data.json :as json]
    [auto-ap.parse :as parse]
+   [amazonica.aws.lambda :as lambda]
+   [config.core :refer [env]]
    [auto-ap.shared-views.admin.side-bar :refer [admin-side-bar]]
    [auto-ap.ssr-routes :as ssr-routes]
    [auto-ap.ssr.ui :refer [base-page]]
@@ -14,7 +17,8 @@
    [com.brunobonacci.mulog :as mu]
    [datomic.api :as dc]
    [dk.ative.docjure.spreadsheet :as doc]
-   [hiccup2.core :as hiccup]))
+   [hiccup2.core :as hiccup]
+   [amazonica.aws.s3 :as s3]))
 
 (defn fmt-amount [a]
   (with-precision 2
@@ -23,15 +27,15 @@
             (.setScale 2 java.math.RoundingMode/HALF_UP)
             (double))))
 
-(defn extract-sheet-details [f]
+(defn extract-sheet-details [bucket object]
-  (into []
+  (-> (lambda/invoke {:function-name "xls-extractor" :payload
-        (for [row (->> (doc/load-workbook f)
+                      (json/write-str
-                       (doc/sheet-seq)
+                        {"s3_url" object "s3_bucket" bucket})}
-                       first
+                     )
-                       (doc/row-seq)
+      :payload
-                       )]
+      slurp
-          (mapv doc/read-cell (doc/cell-seq row))
+      json/read-str))
-          )))
+
 
 (defn rows->maps [rows]
   (let [[headers & rows] rows]
@@ -39,26 +43,37 @@
       (into {}
             (map vector headers r)))))
 
+(defn xls-date->date [f]
+  (when (not-empty f)
+    (let [f         (Double/parseDouble f)
+          unix-days (- f 25569.0)
+          unix-secs (* unix-days 86400.0)]
+      (java.util.Date. (long (Math/round (* 1000.0 unix-secs)))))))
+
 
 (defn map->sales-order [r clients]
   (let [order-number (get r "Order Number")
         event-date   (get r "Event Date")
         store-name   (get r "Store Name")
-        adjustments (get r "Adjustments")
+        adjustments (some-> (get r "Adjustments") not-empty (Double/parseDouble))
-        tax (get r "Sales Tax")
+        tax (some-> (get r "Sales Tax") not-empty (Double/parseDouble))
-        food-total (get r "Food Total")
+        food-total (some-> (get r "Food Total") not-empty (Double/parseDouble))
-        commission (get r "Commission")
+        commission (some-> (get r "Commission") not-empty (Double/parseDouble))
-        fee (get r "Payment Transaction Fee")
+        fee (some-> (get r "Payment Transaction Fee") not-empty (Double/parseDouble))
-        tip (get r "Tip")
+        tip (some-> (get r "Tip") not-empty (Double/parseDouble))
         caterer-name (get r "Caterer Name")
         client (some->> caterer-name
                         not-empty
                         (parse/exact-match clients))
         client-id (:db/id client)
-        location (first (:client/locations client))]
+        location (first (:client/locations client))
+        event-date (some-> (xls-date->date event-date)
+                           coerce/to-date-time
+                           atime/as-local-time
+                           coerce/to-date )]
     (cond (and event-date client-id location ) 
           [:order #:sales-order
-           {:date           (coerce/to-date (atime/localize (coerce/to-date-time event-date)))
+           {:date           event-date
             :external-id    (str "ezcater/order/" client-id "-" location "-" order-number)
             :client         client-id
             :location       location
@@ -75,7 +90,7 @@
 
             :charges        [#:charge
                              {:type-name   "CARD"
-                              :date        (coerce/to-date (atime/localize (coerce/to-date-time event-date)))
+                              :date        event-date
                               :client      client-id
                               :location    location
                               :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
@@ -114,10 +129,15 @@
                                                    :client/matches
                                                    :client/locations])
                                    :where [?c :client/code]]
-                                 (dc/db conn)))]
+                                 (dc/db conn)))
+        object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))]
+    (mu/log ::writing-temp-xls
+            :location object)
+    (s3/put-object {:bucket-name (:data-bucket env) 
+                    :key object
+                    :input-stream s})
     (into []
-          (->> s
+          (->> (extract-sheet-details (:data-bucket env) object)
-               extract-sheet-details
                rows->maps
                (map #(map->sales-order % clients))
                (filter identity)))))

test/clj/auto_ap/integration/routes/ezcater_xls.clj

@@ -8,7 +8,7 @@
 
 (use-fixtures :each wrap-setup)
 
-(deftest stream->sales-ordersx
+(deftest stream->sales-orders
   (testing "Should import nothing when there are no clients"
     (with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))]
       (is (= [:missing "Nick The Greek (Santa Cruz)"] (first (sut/stream->sales-orders s))))))