notid/integreat
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Merge branch 'port-datomic-cloud' into try-tailwind

Browse Source
This commit is contained in:
Bryce Covert
2023-05-10 10:40:51 -07:00
parent 96316bfda4 8bd73b8a98
commit 4b5227e8cb
8 changed files with 133 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
project.clj
View File

@@ -29,8 +29,6 @@
[ring/ring-jetty-adapter "1.9.6" :exclusions [ring/ring-jetty-adapter "1.9.6" :exclusions
[org.eclipse.jetty/jetty-server]] [org.eclipse.jetty/jetty-server]]
[yogthos/config "1.1.7"] [yogthos/config "1.1.7"]
[dk.ative/docjure "1.14.0"]
[clj-fuzzy "0.4.1"] [clj-fuzzy "0.4.1"]
[com.walmartlabs/lacinia "0.37.0"] [com.walmartlabs/lacinia "0.37.0"]
[vincit/venia "0.2.5"] [vincit/venia "0.2.5"]
@@ -69,6 +67,9 @@
[com.amazonaws/aws-java-sdk-sqs "1.11.926" [com.amazonaws/aws-java-sdk-sqs "1.11.926"
:exclusions [commons-codec :exclusions [commons-codec
org.apache.httpcomponents/httpclient]] org.apache.httpcomponents/httpclient]]
[com.amazonaws/aws-java-sdk-lambda "1.11.926"
:exclusions [commons-codec
org.apache.httpcomponents/httpclient]]
[com.amazonaws/aws-java-sdk-ecs "1.11.926" [com.amazonaws/aws-java-sdk-ecs "1.11.926"
:exclusions [commons-codec :exclusions [commons-codec

4
src/clj/auto_ap/graphql/checks.clj
View File

@@ -164,7 +164,7 @@
[[:cell {:colspan 12} [:spacer]]] [[:cell {:colspan 12} [:spacer]]]
[[:cell {:colspan 12} [:spacer]]] [[:cell {:colspan 12} [:spacer]]]
[[:cell {:colspan 5}] [[:cell {:colspan 3}]
[:cell {:align :right :colspan 2} [:cell {:align :right :colspan 2}
"Check:\n" "Check:\n"
"Vendor:\n" "Vendor:\n"
@@ -174,7 +174,7 @@
"Amount:\n" "Amount:\n"
"Date:\n"] "Date:\n"]
[:cell {:colspan 5} [:cell {:colspan 7}
[:paragraph check] [:paragraph check]
[:paragraph vendor-name] [:paragraph vendor-name]
[:paragraph (:client/name client)] [:paragraph (:client/name client)]

72
src/clj/auto_ap/jobs/ntg.clj
View File

@@ -65,40 +65,43 @@
(defmethod extract-invoice-details :general-produce (defmethod extract-invoice-details :general-produce
[k input-stream clients] [k input-stream clients]
(log/info ::parsing-general-produce :key k) (log/info ::parsing-general-produce :key k)
(try (let [missing-client-hints (atom #{})]
(->> (read-csv input-stream) (try
(drop 1) (->> (read-csv input-stream)
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]] (drop 1)
(= "Y" break-flag))) #_(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
(map (fn [[_ location-hint invoice-number ship-date invoice-total ]] (= "Y" break-flag)))
(let [matching-client (and location-hint (map (fn [[_ location-hint invoice-number ship-date invoice-total ]]
(parse/exact-match clients location-hint)) (let [matching-client (and location-hint
location (parse/best-location-match matching-client location-hint location-hint ) (parse/exact-match clients location-hint))
vendor (d/pull (d/db conn) '[:vendor/default-account] :vendor/general-produce)] location (parse/best-location-match matching-client location-hint location-hint )
(when-not matching-client vendor (d/pull (d/db conn) '[:vendor/default-account] :vendor/general-produce)]
(log/warn ::missing-client (when-not (and matching-client
:client-hint location-hint)) (not (@missing-client-hints location-hint)))
{:invoice/location location (log/warn ::missing-client
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date)) :client-hint location-hint)
:invoice/invoice-number invoice-number (swap! missing-client-hints conj location-hint))
:invoice/total (Double/parseDouble invoice-total) {:invoice/location location
:invoice/vendor :vendor/general-produce :invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
:invoice/outstanding-balance (Double/parseDouble invoice-total) :invoice/invoice-number invoice-number
:invoice/client (:db/id matching-client) :invoice/total (Double/parseDouble invoice-total)
:invoice/import-status :import-status/imported :invoice/vendor :vendor/general-produce
:invoice/status :invoice-status/unpaid :invoice/outstanding-balance (Double/parseDouble invoice-total)
:invoice/client-identifier location-hint :invoice/client (:db/id matching-client)
:invoice/expense-accounts [{:invoice-expense-account/account :invoice/import-status :import-status/imported
(-> vendor :vendor/default-account :db/id) :invoice/status :invoice-status/unpaid
:invoice-expense-account/location location :invoice/client-identifier location-hint
:invoice-expense-account/amount (Math/abs (Double/parseDouble invoice-total)) :invoice/expense-accounts [{:invoice-expense-account/account
}]}))) (-> vendor :vendor/default-account :db/id)
(filter :invoice/client) :invoice-expense-account/location location
(into [])) :invoice-expense-account/amount (Math/abs (Double/parseDouble invoice-total))
(catch Exception e }]})))
(log/error ::cant-import-general-produce (filter :invoice/client)
:error e) (into []))
[]))) (catch Exception e
(log/error ::cant-import-general-produce
:error e)
[]))))
(defmethod extract-invoice-details :unknown (defmethod extract-invoice-details :unknown
[k input-stream clients] [k input-stream clients]
@@ -247,6 +250,7 @@
(->> (extract-invoice-details k (->> (extract-invoice-details k
is is
clients) clients)
(set)
(map (fn [i] (map (fn [i]
(log/info ::importing-invoice (log/info ::importing-invoice
:invoice i) :invoice i)

46
src/clj/auto_ap/parse/excel.clj
View File

@@ -2,18 +2,23 @@
(:require [auto-ap.parse.templates :as t] (:require [auto-ap.parse.templates :as t]
[auto-ap.parse.util :as u] [auto-ap.parse.util :as u]
[clojure.string :as str] [clojure.string :as str]
[dk.ative.docjure.spreadsheet :as d]) [amazonica.aws.lambda :as lambda]
(:import (org.apache.poi.ss.util CellAddress))) [clojure.data.json :as json]
[config.core :refer [env]]
[clojure.java.io :as io]
[amazonica.aws.s3 :as s3])
)
(defn template-applies? [text {:keys [keywords]}] (defn template-applies? [text {:keys [keywords]}]
(every? #(re-find % text) keywords)) (every? #(re-find % text) keywords))
(defn extract [wb {:keys [extract vendor parser]}] (defn extract [wb {:keys [extract vendor parser]}]
(if (fn? extract) (if (fn? extract)
(extract wb vendor) (extract wb vendor)
[(reduce-kv #_[(reduce-kv
(fn [invoice k [regex offset-row offset-column extract-regex]] (fn [invoice k [regex offset-row offset-column extract-regex]]
(assoc invoice k (assoc invoice k
(->> wb (->> wb
@@ -38,17 +43,34 @@
{:vendor-code vendor} {:vendor-code vendor}
extract)])) extract)]))
(defn extract-sheet-details [bucket object]
(-> (lambda/invoke {:function-name "xls-extractor" :payload
(json/write-str
{"s3_url" object "s3_bucket" bucket})})
:payload
slurp
json/read-str))
(defn parse-file (defn parse-file
[file _] [file _]
(let [wb (d/load-workbook file) (let [tmp-key (str "xls-invoice/import/" (java.util.UUID/randomUUID))
text (->> wb _ (with-open [f (io/input-stream file)]
(d/sheet-seq) (s3/put-object {:bucket-name (:data-bucket env)
first :key tmp-key
(d/cell-seq) :input-stream f}))
(map d/read-cell) sheet (extract-sheet-details (:data-bucket env) tmp-key)
(str/join " "))] text (str/join " " (mapcat seq sheet))]
(->> t/excel-templates (->> t/excel-templates
(filter (partial template-applies? text)) (filter (partial template-applies? text))
first first
(extract wb) (extract sheet))))
)))
(defn xls-date->date [f]
(when (not-empty f)
(let [f (Double/parseDouble f)
unix-days (- f 25569.0)
unix-secs (* unix-days 86400.0)]
(java.util.Date. (long (Math/round (* 1000.0 unix-secs)))))))

112
src/clj/auto_ap/parse/templates.clj
View File

@@ -1,8 +1,6 @@
(ns auto-ap.parse.templates (ns auto-ap.parse.templates
(:require [dk.ative.docjure.spreadsheet :as d] (:require [auto-ap.parse.util :as u]
[auto-ap.parse.util :as u] [clojure.string :as str]))
[clojure.string :as str])
(:import (org.apache.poi.ss.util CellAddress)))
(def pdf-templates (def pdf-templates
@@ -614,97 +612,27 @@
:parser {:date [:clj-time "MM/dd/yy"] :parser {:date [:clj-time "MM/dd/yy"]
:total [:trim-commas-and-negate nil]}}]) :total [:trim-commas-and-negate nil]}}])
(defn offset [c x y]
(.toString (CellAddress. (+ y (.getRow (.getAddress c))) (+ x (.getColumn (.getAddress c))) )))
(def excel-templates (def excel-templates
[{:vendor "Isp Productions" [{:vendor "Mama Lu's Foods"
:keywords [#"ISP PRODUCTIONS"]
:extract {:customer-identifier [#"SERVICES PROVIDED TO" 1 0]
:total [#"PAY THIS" -1 0]
:date [#"INVOICE DATE" 0 1]
:invoice-number [#"INVOICE NUMBER" 0 1]}}
{:vendor "Southern Glazers"
:keywords [#"Please note that the total invoice amount may"]
:extract {:customer-identifier [#"Customer #" 1 0]
:total [#"Subtotal" 0 16 ]
:date [#"Date" 0 0 #"Date: (.*)"]
:invoice-number [#"Invoice #" 0 0 #"Invoice #: (.*)"]
:account-number [#"Customer #" 0 0 #"Customer #: (.*)"]}
:parser { :total [:trim-commas-and-remove-dollars-and-invert-parentheses nil]
:date [:clj-time "MM/dd/yyyy"]}}
{:vendor "Mama Lu's Foods"
:keywords [#"Mama Lu's Foods"] :keywords [#"Mama Lu's Foods"]
:extract (fn [wb vendor] :extract (fn [sheet vendor]
(let [[sheet] (d/sheet-seq wb)] (transduce (comp
(transduce (comp (drop 5)
(drop 5) (filter
(filter
(fn [r] (fn [r]
(and (and
r (seq r)
(->> r d/cell-seq second d/read-cell)))) (->> r second not-empty))))
(map (map
(fn [r] (fn [r]
(let [[_ customer-order-number num date name amount] (map d/read-cell (d/cell-seq r))] (let [[_ customer-order-number num date name amount] r]
{:customer-identifier (second (re-find #"([^:]*):" name)) {:customer-identifier (second (re-find #"([^:]*):" name))
:text name :text name
:full-text name :full-text name
:date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim date)) :date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim date))
:invoice-number (str customer-order-number "-" (int num)) :invoice-number (str customer-order-number "-" (Integer/parseInt num))
:total (str amount) :total (str amount)
:vendor-code vendor})))) :vendor-code vendor}))))
conj conj
[] []
(d/row-seq sheet))))} sheet))}])
{:vendor "DVW Commercial"
:keywords [#"Total for" #"Num"]
:extract (fn [wb vendor]
(let [[sheet] (d/sheet-seq wb)]
(transduce (comp (filter (fn [c]
(re-find #"Invoice" (str (d/read-cell c)))))
(map (fn [c]
(let [customer-identifier (d/read-cell (->> (d/select-cell (offset c -3 0) sheet)
(iterate (fn [c]
(d/select-cell (offset c 0 -1) sheet)))
(filter (fn [c]
(not (str/blank? (d/read-cell c)))))
first))]
{:customer-identifier customer-identifier
:text customer-identifier
:full-text customer-identifier
:date (d/read-cell (d/select-cell (offset c 2 0) sheet))
:invoice-number (d/read-cell (d/select-cell (offset c 4 0) sheet))
:total (str (d/read-cell (d/select-cell (offset c 8 0) sheet)))
:vendor-code vendor}))))
conj
[]
(d/cell-seq sheet))))}
{:vendor "Chef's Choice Produce Co"
:keywords [#"Alt_invoice_number"]
:extract (fn [wb vendor]
(let [[sheet] (d/sheet-seq wb)]
(transduce (comp
(drop-while (fn [c]
(not (re-find #"Customer_id" (str (d/read-cell c))))))
(drop 9)
(filter (fn [c]
(= 0 (.getColumnIndex c))))
(filter (fn [c]
(not (str/blank? (str/trim (or (d/read-cell (d/select-cell (offset c 1 0) sheet)) ""))))))
(map (fn [c]
{:customer-identifier (str/trim (d/read-cell (d/select-cell (offset c 1 0) sheet)))
:text (d/read-cell (d/select-cell (offset c 1 0) sheet))
:full-text (d/read-cell (d/select-cell (offset c 1 0) sheet))
:date (u/parse-value :clj-time "MM/dd/yyyy" (str/trim (d/read-cell (d/select-cell (offset c 5 0) sheet))))
:invoice-number (->>
(re-find #"^(?:0+([A-Z0-9]+))|([A-Z]+[A-Z0-9]+)" (str/trim (d/read-cell (d/select-cell (offset c 2 0) sheet))))
(drop 1 )
(filter identity)
first)
:total (str (d/read-cell (d/select-cell (offset c 7 0) sheet)))
:vendor-code vendor}))
(filter :customer-identifier))
conj
[]
(d/cell-seq sheet))))}])

54
src/clj/auto_ap/routes/ezcater_xls.clj
View File

@@ -2,7 +2,11 @@
(:require (:require
[auto-ap.datomic :refer [audit-transact conn]] [auto-ap.datomic :refer [audit-transact conn]]
[auto-ap.logging :as alog] [auto-ap.logging :as alog]
[clojure.data.json :as json]
[auto-ap.parse.excel :as excel]
[auto-ap.parse :as parse] [auto-ap.parse :as parse]
[amazonica.aws.lambda :as lambda]
[config.core :refer [env]]
[auto-ap.shared-views.admin.side-bar :refer [admin-side-bar]] [auto-ap.shared-views.admin.side-bar :refer [admin-side-bar]]
[auto-ap.ssr-routes :as ssr-routes] [auto-ap.ssr-routes :as ssr-routes]
[auto-ap.ssr.ui :refer [base-page]] [auto-ap.ssr.ui :refer [base-page]]
@@ -13,8 +17,8 @@
[clojure.java.io :as io] [clojure.java.io :as io]
[com.brunobonacci.mulog :as mu] [com.brunobonacci.mulog :as mu]
[datomic.api :as dc] [datomic.api :as dc]
[dk.ative.docjure.spreadsheet :as doc] [hiccup2.core :as hiccup]
[hiccup2.core :as hiccup])) [amazonica.aws.s3 :as s3]))
(defn fmt-amount [a] (defn fmt-amount [a]
(with-precision 2 (with-precision 2
@@ -23,15 +27,8 @@
(.setScale 2 java.math.RoundingMode/HALF_UP) (.setScale 2 java.math.RoundingMode/HALF_UP)
(double)))) (double))))
(defn extract-sheet-details [f]
(into []
(for [row (->> (doc/load-workbook f)
(doc/sheet-seq)
first
(doc/row-seq)
)]
(mapv doc/read-cell (doc/cell-seq row))
)))
(defn rows->maps [rows] (defn rows->maps [rows]
(let [[headers & rows] rows] (let [[headers & rows] rows]
@@ -40,25 +37,31 @@
(map vector headers r))))) (map vector headers r)))))
(defn map->sales-order [r clients] (defn map->sales-order [r clients]
(let [order-number (get r "Order Number") (let [order-number (get r "Order Number")
event-date (get r "Event Date") event-date (get r "Event Date")
store-name (get r "Store Name") store-name (get r "Store Name")
adjustments (get r "Adjustments") adjustments (some-> (get r "Adjustments") not-empty (Double/parseDouble))
tax (get r "Sales Tax") tax (some-> (get r "Sales Tax") not-empty (Double/parseDouble))
food-total (get r "Food Total") food-total (some-> (get r "Food Total") not-empty (Double/parseDouble))
commission (get r "Commission") commission (some-> (get r "Commission") not-empty (Double/parseDouble))
fee (get r "Payment Transaction Fee") fee (some-> (get r "Payment Transaction Fee") not-empty (Double/parseDouble))
tip (get r "Tip") tip (some-> (get r "Tip") not-empty (Double/parseDouble))
caterer-name (get r "Caterer Name") caterer-name (get r "Caterer Name")
client (some->> caterer-name client (some->> caterer-name
not-empty not-empty
(parse/exact-match clients)) (parse/exact-match clients))
client-id (:db/id client) client-id (:db/id client)
location (first (:client/locations client))] location (first (:client/locations client))
event-date (some-> (excel/xls-date->date event-date)
coerce/to-date-time
atime/as-local-time
coerce/to-date )]
(cond (and event-date client-id location ) (cond (and event-date client-id location )
[:order #:sales-order [:order #:sales-order
{:date (coerce/to-date (atime/localize (coerce/to-date-time event-date))) {:date event-date
:external-id (str "ezcater/order/" client-id "-" location "-" order-number) :external-id (str "ezcater/order/" client-id "-" location "-" order-number)
:client client-id :client client-id
:location location :location location
@@ -75,7 +78,7 @@
:charges [#:charge :charges [#:charge
{:type-name "CARD" {:type-name "CARD"
:date (coerce/to-date (atime/localize (coerce/to-date-time event-date))) :date event-date
:client client-id :client client-id
:location location :location location
:external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0) :external-id (str "ezcater/charge/" client-id "-" location "-" order-number "-" 0)
@@ -114,10 +117,15 @@
:client/matches :client/matches
:client/locations]) :client/locations])
:where [?c :client/code]] :where [?c :client/code]]
(dc/db conn)))] (dc/db conn)))
object (str "/ezcater-xls/" (str (java.util.UUID/randomUUID)))]
(mu/log ::writing-temp-xls
:location object)
(s3/put-object {:bucket-name (:data-bucket env)
:key object
:input-stream s})
(into [] (into []
(->> s (->> (excel/extract-sheet-details (:data-bucket env) object)
extract-sheet-details
rows->maps rows->maps
(map #(map->sales-order % clients)) (map #(map->sales-order % clients))
(filter identity))))) (filter identity)))))

5
src/clj/auto_ap/square/core3.clj
View File

@@ -631,7 +631,7 @@
(defn cash-drawer-shifts (defn cash-drawer-shifts
([client l] ([client l]
(cash-drawer-shifts client l (time/plus (time/now) (time/days -14)) (time/now))) (cash-drawer-shifts client l (time/plus (time/now) (time/days -75)) (time/now)))
([client l start end] ([client l start end]
(de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/cash-drawers/shifts" (de/chain (manifold-api-call {:url (str "https://connect.squareup.com/v2/cash-drawers/shifts"
"?" "?"
@@ -639,7 +639,8 @@
(url/map->query (url/map->query
{:location_id (:square-location/square-id l) {:location_id (:square-location/square-id l)
:begin_time (->square-date start) :begin_time (->square-date start)
:end_time (->square-date end)})) :end_time (->square-date end)
:limit 1000}))
:method :get :method :get
:headers (client-base-headers client "2023-04-19") :headers (client-base-headers client "2023-04-19")

5
test/clj/auto_ap/integration/routes/ezcater_xls.clj
View File

@@ -8,7 +8,7 @@
(use-fixtures :each wrap-setup) (use-fixtures :each wrap-setup)
(deftest stream->sales-ordersx (deftest stream->sales-orders
(testing "Should import nothing when there are no clients" (testing "Should import nothing when there are no clients"
(with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))] (with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))]
(is (= [:missing "Nick The Greek (Santa Cruz)"] (first (sut/stream->sales-orders s)))))) (is (= [:missing "Nick The Greek (Santa Cruz)"] (first (sut/stream->sales-orders s))))))
@@ -20,8 +20,7 @@
:client/name "The client" :client/name "The client"
:client/matches ["Nick the Greek (Elk Grove)"])])] :client/matches ["Nick the Greek (Elk Grove)"])])]
(with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))] (with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))]
(is (seq (sut/stream->sales-orders s))) (is (seq (sut/stream->sales-orders s))))
)
(with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))] (with-open [s (io/input-stream (io/resource "sample-ezcater.xlsx"))]
(is (= #:sales-order (is (= #:sales-order
{:vendor :vendor/ccp-ezcater {:vendor :vendor/ccp-ezcater