From 179c2d13c6460258bb6358208cf60298e551d35f Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 8 Sep 2020 18:00:07 -0700 Subject: [PATCH] lots of fixes. --- docker-compose.dev.yml | 2 + src/clj/auto_ap/datomic/migrate/sales.clj | 16 +- src/clj/auto_ap/graphql.clj | 5 +- src/clj/auto_ap/graphql/transaction_rules.clj | 3 +- src/clj/auto_ap/routes/exports.clj | 42 ++++- src/clj/auto_ap/square/core.clj | 155 ++++++++++++++---- 6 files changed, 182 insertions(+), 41 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index de624feb..20b0d93b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -16,5 +16,7 @@ services: datomic-transactor: environment: - DATOMIC_PORT=4334 + - XMS=-Xms4g + - XMX=-Xmx8g ports: - 4334:4334 diff --git a/src/clj/auto_ap/datomic/migrate/sales.clj b/src/clj/auto_ap/datomic/migrate/sales.clj index 1a2e4096..a6cc1a56 100644 --- a/src/clj/auto_ap/datomic/migrate/sales.clj +++ b/src/clj/auto_ap/datomic/migrate/sales.clj @@ -121,7 +121,21 @@ {:db/ident :expected-deposit/fee :db/doc "The total fee on the deposit" :db/valueType :db.type/double - :db/cardinality :db.cardinality/one}]]} }) + :db/cardinality :db.cardinality/one}]]} + + :add-discounts {:txes [[{:db/ident :sales-order/discount + :db/doc "The total amount on the sale" + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one} + + {:db/ident :order-line-item/discount + :db/doc "The total cost on the item" + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one}]]} + :add-category {:txes [[{:db/ident :order-line-item/category + :db/doc "The item category" + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one}]]}}) diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 69eb0eca..bd66643f 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -194,7 +194,9 @@ {:fields {:id {:type :id} :item_name {:type 'String} :total {:type :money} - :tax {:type :money}}} + :tax {:type :money} + :category {:type 'String} + :discount {:type :money}}} :charge {:fields {:id {:type :id} :type_name {:type 'String} @@ -208,6 +210,7 @@ :total {:type :money} :tip {:type :money} :tax {:type :money} + :discount {:type :money} :client {:type :client} :date {:type 'String} :charges {:type '(list :charge)} diff --git a/src/clj/auto_ap/graphql/transaction_rules.clj b/src/clj/auto_ap/graphql/transaction_rules.clj index 790110e3..7831ee50 100644 --- a/src/clj/auto_ap/graphql/transaction_rules.clj +++ b/src/clj/auto_ap/graphql/transaction_rules.clj @@ -19,7 +19,8 @@ [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] - [datomic.api :as d]) + [datomic.api :as d] + [clj-time.coerce :as c]) (:import java.time.temporal.ChronoField)) (defn get-transaction-rule-page [context args value] diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index d3f794ca..8d41d215 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -14,7 +14,9 @@ [compojure.core :refer [GET POST context defroutes wrap-routes]] [clojure.string :as str] [venia.core :as venia] - [clojure.tools.logging :as log])) + [clojure.tools.logging :as log] + [clj-time.core :as time] + [clj-time.coerce :as coerce])) (defroutes routes (wrap-routes @@ -61,12 +63,37 @@ :total :tip :tax + :discount :date [:charges [:type_name :total :tip]] - [:line_items [:item_name :total :tax]] + [:line_items [:item_name :total :tax :discount :category]] [:client [:id :name :code]]]]] - payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] - (seq (:all-sales-orders (:data payments))))) + payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)})) + parsedouble #(some-> % Double/parseDouble) ] + (seq (map + (fn [s] + (-> s + (assoc :utc_date (:date s)) + (update :date (fn [d] + (coerce/to-string (coerce/to-local-date-time (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles")))))) + (update :total parsedouble) + (update :tax parsedouble) + (update :tip parsedouble) + (update :line-items (fn [lis] + (map + (fn [li] + (-> li + (update :tax parsedouble) + (update :total parsedouble))) + lis))) + (update :charges (fn [charges] + (map + (fn [charge] + (-> charge + (update :tip parsedouble) + (update :total parsedouble))) + charges))))) + (:all-sales-orders (:data payments)))))) (GET "/expected-deposit/export" {:keys [query-params identity]} (assert-admin identity) @@ -82,7 +109,12 @@ :fee :date]]] payments (graphql/query identity (venia/graphql-query {:venia/queries (->graphql query)}))] - (seq (:all-expected-deposits (:data payments))))) + (seq (map + (fn [d] + (-> d + (update :fee #(some-> % Double/parseDouble)) + (update :total #(some-> % Double/parseDouble)))) + (:all-expected-deposits (:data payments)))))) (GET "/clients/export" {:keys [query-params identity]} (assert-admin identity) diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index a95b4551..106c0545 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -1,13 +1,15 @@ (ns auto-ap.square.core (:require [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.utils :refer [by]] [clj-http.client :as client] [clj-time.coerce :as coerce] [clj-time.core :as time] + [clj-time.format :as f] [clojure.data.json :as json] - [datomic.api :as d] - [unilog.context :as lc] [clojure.tools.logging :as log] + [datomic.api :as d] [mount.core :as mount] + [unilog.context :as lc] [yang.scheduler :as scheduler])) (defn locations [] @@ -19,16 +21,97 @@ :body :locations)) -(defn search [l] - (log/info "Searching for" l) - (->> (client/post "https://connect.squareup.com/v2/orders/search" - {:headers {"Square-Version" "2020-08-12" - "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" - "Content-Type" "application/json"} - :body (json/write-str {"location_ids" [l] "limit" 4000}) - :as :json}) - :body - :orders)) +(defn fetch-catalog [i] + (if i + (do + (log/info "looking up catalog for" (str "https://connect.squareup.com/v2/catalog/object/" i)) + (->> (client/get (str "https://connect.squareup.com/v2/catalog/object/" i) + {:headers {"Square-Version" "2020-08-12" + "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" + "Content-Type" "application/json"} + :query-params {"include_related_items" "true"} + :as :json}) + :body + :object)) + (log/warn "Trying to look up non existant "))) + +(def fetch-catalog-fast (memoize fetch-catalog)) + +(defn item-id->category-name [i] + (let [item (fetch-catalog-fast i)] + (cond (:item_variation_data item) + (item-id->category-name (:item_id (:item_variation_data item))) + + (:category_id (:item_data item)) + (:name (:category_data (fetch-catalog-fast (:category_id (:item_data item))))) + + (:item_data item) + "Uncategorized" + + :else + (do + (log/error "couldn't look up" i) + "Uncategorized")))) + +(defn categories [] + (by :id (comp :name :category_data) )) + +(def potential-query + {"query" {"filter" {"date_time_filter" + { + "created_at" { + "start_at" "2020-08-28T00:00:00-07:00", + "end_at" "2020-08-28T23:59:59-07:00" + } + + + + } + "state_filter" {"states" ["COMPLETED"]}} + + "sort" { + "sort_field" "CREATED_AT" + "sort_order" "DESC" + }}}) + + +(defn pc [d] + {"query" {"filter" {"date_time_filter" + { + "created_at" { + "start_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (coerce/to-date-time d) (time/time-zone-for-id "America/Los_Angeles"))) + "end_at" (f/unparse (f/formatter "YYYY-MM-dd'T'HH:mm:ssZZ") (time/to-time-zone (time/plus (coerce/to-date-time d) (time/days 1)) (time/time-zone-for-id "America/Los_Angeles"))) + } + + + + } + "state_filter" {"states" ["COMPLETED"]}} + + "sort" { + "sort_field" "CREATED_AT" + "sort_order" "DESC" + }}}) + + +(defn search + ([l] + (search l nil)) + ([l d] + (log/info "Searching for" l) + (let [result (->> (client/post "https://connect.squareup.com/v2/orders/search" + {:headers {"Square-Version" "2020-08-12" + "Authorization" "Bearer EAAAEO2xSqesDutZz71hz3eulKmrlKTiEqG3uZ4j25x5GYlOluQ2cj2JxNUXqXD7" + "Content-Type" "application/json"} + :body (json/write-str (cond-> {"location_ids" [l] "limit" 4000} + d (merge (pc d)))) + :as :json}) + :body + :orders)] + (println (cond-> {"location_ids" [l] "limit" 4000} + d (merge (pc d)))) + (log/info "found " (count result)) + result))) @@ -44,11 +127,11 @@ "4X8T65741AEPS" ["NGVZ" "NP"] "KMVFQ9CRCXJ10" ["NGZO" "VT"]} location)) -(defn daily-results [] +(defn daily-results [d] (->> (locations) (map :id) (filter location_id->client-location) - (mapcat search) + (mapcat #(search % d)) (map (fn [order] (let [[client loc] (location_id->client-location (:location_id order))] (remove-nils @@ -56,10 +139,11 @@ {:date (coerce/to-date (time/to-time-zone (coerce/to-date-time (:created_at order)) (time/time-zone-for-id "America/Los_Angeles"))) :client [:client/code client] :location loc - :external-id (str "square-" client "-" loc "-" (:id order)) + :external-id (str "square/order/" client "-" loc "-" (:id order)) :total (-> order :net_amounts :total_money amount->money) :tax (-> order :net_amounts :tax_money amount->money) :tip (-> order :net_amounts :tip_money amount->money) + :discount (-> order :net_amounts :discount_money amount->money) :charges (->> (:tenders order) (map (fn [t] (remove-nils @@ -72,8 +156,10 @@ (remove-nils #:order-line-item {:item-name (:name li) + :category (item-id->category-name (:catalog_object_id li)) :total (amount->money (:total_money li)) - :tax (amount->money (:total_tax_money li))}))))})))))) + :tax (amount->money (:total_tax_money li)) + :discount (amount->money (:total_discount_money li))}))))})))))) #_(daily-results) @@ -126,23 +212,26 @@ :date (-> (:initiated_at settlement) (coerce/to-date))}))))) -(defn upsert [] - (lc/with-context {:source "Square loading"} - (try - (let [existing (->> (d/query {:query {:find ['?external-id] - :in ['$] - :where ['[_ :sales-order/external-id ?external-id]]} - :args [(d/db conn)]}) - (map first) - set) - _ (log/info (count existing) "Sales orders already exist") - to-create (filter #(not (existing (:sales-order/external-id %))) - (daily-results))] - (doseq [x (partition-all 20 to-create)] - (log/info "Loading " (count x)) - @(d/transact conn x))) - (catch Exception e - (log/error e))))) +(defn upsert + ([] + (upsert nil)) + ([d] + (lc/with-context {:source "Square loading"} + (try + (let [existing (->> (d/query {:query {:find ['?external-id] + :in ['$] + :where ['[_ :sales-order/external-id ?external-id]]} + :args [(d/db conn)]}) + (map first) + set) + _ (log/info (count existing) "Sales orders already exist") + to-create (filter #(not (existing (:sales-order/external-id %))) + (daily-results d))] + (doseq [x (partition-all 20 to-create)] + (log/info "Loading " (count x)) + @(d/transact conn x))) + (catch Exception e + (log/error e)))))) (defn upsert-settlements [] (lc/with-context {:source "Square settlements loading "}