Added the ability to load historcial sales yourself.
This commit is contained in:
99
src/clj/auto_ap/jobs/load_historical_sales.clj
Normal file
99
src/clj/auto_ap/jobs/load_historical_sales.clj
Normal file
@@ -0,0 +1,99 @@
|
||||
(ns auto-ap.jobs.load-historical-sales
|
||||
(:gen-class)
|
||||
(:require
|
||||
[unilog.context :as lc]
|
||||
[auto-ap.jobs.core :refer [execute]]
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[auto-ap.square.core :as square]
|
||||
[auto-ap.square.core2 :as square2]
|
||||
[clj-time.periodic :as per]
|
||||
[clj-time.core :as time]
|
||||
[datomic.api :as d]
|
||||
[auto-ap.utils :refer [dollars=]]
|
||||
[clojure.string :as str]
|
||||
[clj-time.coerce :as coerce]
|
||||
[auto-ap.time :as atime]
|
||||
[auto-ap.datomic :refer [conn audit-transact]]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.java.io :as io]
|
||||
[config.core :refer [env]]
|
||||
[clojure.tools.logging :as log]))
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn historical-load-sales [client days]
|
||||
(log/info "loading old approach")
|
||||
(let [client (d/pull (d/db auto-ap.datomic/conn)
|
||||
square/square-read
|
||||
client)]
|
||||
(doseq [square-location (:client/square-locations client)
|
||||
:when (:square-location/client-location square-location)]
|
||||
|
||||
(println "orders")
|
||||
(lc/with-context {:source "Historical loading data"}
|
||||
(doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
|
||||
(time/today)
|
||||
(time/days 1))]
|
||||
(println d)
|
||||
(square/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1))))))
|
||||
|
||||
(println "refunds")
|
||||
(square/upsert-refunds client square-location)
|
||||
|
||||
|
||||
(println "settlements")
|
||||
(with-redefs [square/lookup-dates (fn lookup-dates []
|
||||
(->> (per/periodic-seq (time/plus (time/today) (time/days (- days)))
|
||||
(time/today)
|
||||
(time/days 2))
|
||||
(map (fn [d]
|
||||
[(atime/unparse (time/plus d (time/days 1)) atime/iso-date)
|
||||
|
||||
(atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))]
|
||||
|
||||
(square/upsert-settlements client square-location)))))
|
||||
|
||||
|
||||
(defn historical-load-sales2 [client days]
|
||||
(log/info "loading new approach")
|
||||
(let [client (d/pull (d/db auto-ap.datomic/conn)
|
||||
square/square-read
|
||||
client)]
|
||||
(doseq [square-location (:client/square-locations client)
|
||||
:when (:square-location/client-location square-location)]
|
||||
|
||||
(println "orders")
|
||||
(lc/with-context {:source "Historical loading data"}
|
||||
(doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
|
||||
(time/today)
|
||||
(time/days 1))]
|
||||
(println d)
|
||||
(square2/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1))))))
|
||||
|
||||
(println "refunds")
|
||||
(square2/upsert-refunds client square-location)
|
||||
|
||||
|
||||
(println "settlements")
|
||||
(with-redefs [square2/lookup-dates (fn lookup-dates []
|
||||
(->> (per/periodic-seq (time/plus (time/today) (time/days (- days)))
|
||||
(time/today)
|
||||
(time/days 2))
|
||||
(map (fn [d]
|
||||
[(atime/unparse (time/plus d (time/days 1)) atime/iso-date)
|
||||
|
||||
(atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))]
|
||||
|
||||
(square2/upsert-settlements client square-location)))))
|
||||
|
||||
|
||||
(defn load-historical-sales [args]
|
||||
(let [{:keys [days client]} args
|
||||
client (Long/parseLong client)
|
||||
feature-flags (-> (d/pull (d/db conn) '[:client/feature-flags] client)
|
||||
:client/feature-flags
|
||||
set)]
|
||||
(if (get feature-flags "new-square")
|
||||
(historical-load-sales2 client days)
|
||||
(historical-load-sales client days))))
|
||||
|
||||
(defn -main [& _]
|
||||
(execute "load-historical-sales" #(load-historical-sales (:args env))))
|
||||
@@ -124,8 +124,8 @@
|
||||
vec)))
|
||||
|
||||
(defn register-invoice-import [args]
|
||||
(let [{:keys [ledger-url]} args
|
||||
data (s3->csv ledger-url)]
|
||||
(let [{:keys [invoice-url]} args
|
||||
data (s3->csv invoice-url)]
|
||||
(log/info "contains " (count data) " rows")
|
||||
(doseq [n (partition-all 50 (register-invoice-import* data))]
|
||||
(log/info "transacting" n)
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
|
||||
[auto-ap.jobs.intuit :as job-intuit]
|
||||
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
|
||||
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
|
||||
[auto-ap.jobs.plaid :as job-plaid]
|
||||
[auto-ap.jobs.register-invoice-import :as job-register-invoice-import]
|
||||
[auto-ap.jobs.square :as job-square]
|
||||
@@ -130,6 +131,9 @@
|
||||
(= job "register-invoice-import")
|
||||
(job-register-invoice-import/-main)
|
||||
|
||||
(= job "load-historical-sales")
|
||||
(job-load-historical-sales/-main)
|
||||
|
||||
(= job "bulk-journal-import")
|
||||
(job-bulk-journal-import/-main)
|
||||
|
||||
|
||||
@@ -435,70 +435,6 @@
|
||||
(async/<!! (async/into [] output-chan))))
|
||||
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn historical-load-sales [client-code days]
|
||||
(let [client (d/pull (d/db auto-ap.datomic/conn)
|
||||
square/square-read
|
||||
[:client/code client-code])]
|
||||
(doseq [square-location (:client/square-locations client)
|
||||
:when (:square-location/client-location square-location)]
|
||||
|
||||
(println "orders")
|
||||
(lc/with-context {:source "Historical loading data"}
|
||||
(doseq [d (per/periodic-seq (t/plus (t/today) (t/days (- days)))
|
||||
(t/today)
|
||||
(t/days 1))]
|
||||
(println d)
|
||||
(square/upsert client square-location (c/to-date-time d) (c/to-date-time (t/plus d (t/days 1))))))
|
||||
|
||||
(println "refunds")
|
||||
(square/upsert-refunds client square-location)
|
||||
|
||||
|
||||
(println "settlements")
|
||||
(with-redefs [square/lookup-dates (fn lookup-dates []
|
||||
(->> (per/periodic-seq (t/plus (t/today) (t/days (- days)))
|
||||
(t/today)
|
||||
(t/days 2))
|
||||
(map (fn [d]
|
||||
[(atime/unparse (t/plus d (t/days 1)) atime/iso-date)
|
||||
|
||||
(atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))]
|
||||
|
||||
(square/upsert-settlements client square-location)))))
|
||||
|
||||
|
||||
(defn historical-load-sales2 [client-code days]
|
||||
(let [client (d/pull (d/db auto-ap.datomic/conn)
|
||||
square/square-read
|
||||
[:client/code client-code])]
|
||||
(doseq [square-location (:client/square-locations client)
|
||||
:when (:square-location/client-location square-location)]
|
||||
|
||||
(println "orders")
|
||||
(lc/with-context {:source "Historical loading data"}
|
||||
(doseq [d (per/periodic-seq (t/plus (t/today) (t/days (- days)))
|
||||
(t/today)
|
||||
(t/days 1))]
|
||||
(println d)
|
||||
(square2/upsert client square-location (c/to-date-time d) (c/to-date-time (t/plus d (t/days 1))))))
|
||||
|
||||
(println "refunds")
|
||||
(square2/upsert-refunds client square-location)
|
||||
|
||||
|
||||
(println "settlements")
|
||||
(with-redefs [square2/lookup-dates (fn lookup-dates []
|
||||
(->> (per/periodic-seq (t/plus (t/today) (t/days (- days)))
|
||||
(t/today)
|
||||
(t/days 2))
|
||||
(map (fn [d]
|
||||
[(atime/unparse (t/plus d (t/days 1)) atime/iso-date)
|
||||
|
||||
(atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))]
|
||||
|
||||
(square2/upsert-settlements client square-location)))))
|
||||
|
||||
#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
|
||||
(defn load-sales-for-day [date]
|
||||
(doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...]
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
[auto-ap.views.components.admin.side-bar :refer [admin-side-bar]]
|
||||
[auto-ap.views.components.layouts :refer [side-bar-layout]]
|
||||
[auto-ap.views.pages.admin.jobs.table :as table]
|
||||
[auto-ap.views.components.modal :as modal]
|
||||
[auto-ap.views.components :as com]
|
||||
[auto-ap.views.pages.data-page :as data-page]
|
||||
[auto-ap.views.utils :refer [dispatch-event with-user]]
|
||||
[clojure.set :as set]
|
||||
@@ -15,9 +17,10 @@
|
||||
[auto-ap.forms :as forms]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(def a 1)
|
||||
|
||||
|
||||
(def default-read [:name :start-date :end-date :status])
|
||||
(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :register-invoice-import :ezcater-upsert])
|
||||
|
||||
(re-frame/reg-event-fx
|
||||
::params-change
|
||||
@@ -51,23 +54,38 @@
|
||||
:dispatch [::params-change]}))
|
||||
|
||||
(re-frame/reg-event-fx
|
||||
::request
|
||||
::saving
|
||||
[with-user
|
||||
(re-frame/inject-cofx ::inject/sub [::forms/form ::bulk-journal-form])
|
||||
(re-frame/inject-cofx ::inject/sub [::forms/form ::form])
|
||||
]
|
||||
(fn [{:keys [user] ::forms/keys [form]} [_ which]]
|
||||
{:graphql {:token user
|
||||
:owns-state {:single which}
|
||||
:query-obj
|
||||
{:venia/operation {:operation/type :mutation
|
||||
:operation/name "RequestJob"}
|
||||
:venia/queries [{:query/data
|
||||
[:request-job
|
||||
{:which (name which)
|
||||
:args (when (#{:bulk-journal-import :register-invoice-import} which)
|
||||
(str/escape (pr-str (:data form)) {\" "\\\""}))}
|
||||
[:message]]}]}
|
||||
:on-success [::success]}}))
|
||||
(let [which (:job (:data form))]
|
||||
|
||||
{:graphql {:token user
|
||||
:owns-state {:single which}
|
||||
:query-obj
|
||||
{:venia/operation {:operation/type :mutation
|
||||
:operation/name "RequestJob"}
|
||||
:venia/queries [{:query/data
|
||||
[:request-job
|
||||
{:which (str which)
|
||||
:args (-> (cond (= which :bulk-journal-import)
|
||||
(select-keys (:data form) [:ledger-url])
|
||||
|
||||
(= which :register-invoice-import)
|
||||
(select-keys (:data form) [:invoice-url])
|
||||
|
||||
(= which :load-historical-sales)
|
||||
{:days (-> form :data :days)
|
||||
:client (-> form :data :client :id)}
|
||||
|
||||
:else
|
||||
nil
|
||||
)
|
||||
pr-str
|
||||
(str/escape {\" "\\\""}))}
|
||||
[:message]]}]}
|
||||
:on-success [::success]}})))
|
||||
|
||||
(re-frame/reg-event-fx
|
||||
::mounted
|
||||
@@ -80,11 +98,7 @@
|
||||
(re-frame/reg-event-fx
|
||||
::unmounted
|
||||
(fn [{:keys [db]}]
|
||||
{:dispatch-n (into [[::data-page/dispose ::page]]
|
||||
(map (fn [jt]
|
||||
[::status/dispose-single jt]
|
||||
)
|
||||
job-types))
|
||||
{:dispatch-n [[::data-page/dispose ::page]]
|
||||
::track/dispose {:id ::params}
|
||||
:db (dissoc db ::msg)}))
|
||||
|
||||
@@ -98,42 +112,77 @@
|
||||
content]))
|
||||
|
||||
(defn notification []
|
||||
[:<>
|
||||
(when-let [msg @(re-frame/subscribe [::msg])]
|
||||
[:div.notification.is-info.is-light msg])
|
||||
(doall (for [status job-types
|
||||
:let [state @(re-frame/subscribe [::status/single status])]]
|
||||
^{:key (str status) }
|
||||
[:div
|
||||
(cond (:info state)
|
||||
[:div.notification.is-info.is-light (:info state)]
|
||||
(let [state @(re-frame/subscribe [::status/single ::form])]
|
||||
[:<>
|
||||
(when-let [msg @(re-frame/subscribe [::msg])]
|
||||
[:div.notification.is-info.is-light msg])
|
||||
[:div
|
||||
(cond (:info state)
|
||||
[:div.notification.is-info.is-light (:info state)]
|
||||
|
||||
(seq (:error state))
|
||||
[:div.notification.is-info.is-warning.is-light (:message (first (:error state)))]
|
||||
(seq (:error state))
|
||||
[:div.notification.is-info.is-warning.is-light (:message (first (:error state)))]
|
||||
|
||||
:else
|
||||
nil)]))])
|
||||
:else
|
||||
nil)]]))
|
||||
|
||||
(defn bulk-journal-import-button []
|
||||
[form-builder/builder {:submit-event [::request :bulk-journal-import]
|
||||
:id ::bulk-journal-form}
|
||||
[:p.field.has-addons
|
||||
[:p.control
|
||||
[:a.button.is-static.is-small
|
||||
"s3://data.prod.app.integreatconsult.com/bulk-import/"]]
|
||||
[:p.control
|
||||
[form-builder/raw-field-v2 {:field :ledger-url}
|
||||
[:input.input.is-small {:placeholder "ledger-data.csv"}]]]
|
||||
[:p.control
|
||||
[form-builder/submit-button {:class "is-small"} "Ledger Import"]
|
||||
]]])
|
||||
(defn new-job-form []
|
||||
(let [{:keys [data]} @(re-frame/subscribe [::forms/form ::form])
|
||||
clients @(re-frame/subscribe [::subs/clients])]
|
||||
[:<>
|
||||
[form-builder/builder {:submit-event [::saving]
|
||||
:id ::form}
|
||||
[form-builder/field-v2 {:required? true
|
||||
:field :job}
|
||||
"Job Type"
|
||||
[com/select-field {:options [[:yodlee2 "Yodlee Import"]
|
||||
[:yodlee2-accounts "Yodlee Account Import"]
|
||||
[:intuit "Intuit import"]
|
||||
[:plaid "Plaid import"]
|
||||
[:bulk-journal-import "Bulk Journal Import"]
|
||||
[:register-invoice-import "Register Invoice Import "]
|
||||
[:ezcater-upsert "Upsert recent ezcater orders"]
|
||||
[:load-historical-sales "Load Historical Square Sales"]]
|
||||
:allow-nil? true
|
||||
:keywordize? true}]]
|
||||
(cond (= :bulk-journal-import (:job data))
|
||||
[:<>
|
||||
[:label.label "Url"]
|
||||
[:p.field.has-addons
|
||||
[:p.control
|
||||
[:a.button.is-static
|
||||
"s3://data.prod.app.integreatconsult.com/bulk-import/"]]
|
||||
[:p.control
|
||||
[form-builder/raw-field-v2 {:field :ledger-url}
|
||||
[:input.input {:placeholder "ledger-data.csv"}]]]]]
|
||||
|
||||
(defn register-invoice-import-button []
|
||||
[form-builder/builder {:submit-event [::request :register-invoice-import]
|
||||
:id ::register-invoice-import-form}
|
||||
[:p.control
|
||||
[form-builder/submit-button {:class "is-small"} "Register Invoice Import"]
|
||||
]])
|
||||
(= :register-invoice-import (:job data))
|
||||
[:<>
|
||||
[:label.label "Url"]
|
||||
[:p.field.has-addons
|
||||
[:p.control
|
||||
[:a.button.is-static
|
||||
"s3://data.prod.app.integreatconsult.com/bulk-import/"]]
|
||||
[:p.control
|
||||
[form-builder/raw-field-v2 {:field :invoice-url}
|
||||
[:input.input {:placeholder "invoice-data.csv"}]]]]]
|
||||
|
||||
(= :load-historical-sales (:job data))
|
||||
[:<>
|
||||
[form-builder/field-v2 {:required? true
|
||||
:field [:client]}
|
||||
"Client"
|
||||
[com/entity-typeahead {:entities @(re-frame/subscribe [::subs/clients])
|
||||
:entity->text :name}]]
|
||||
|
||||
[form-builder/field-v2 {:field :days}
|
||||
"Days to load (recommended 60)"
|
||||
[com/number-input {:placeholder "60"
|
||||
:style {:width "7em"}}]]]
|
||||
:else
|
||||
nil)
|
||||
|
||||
[form-builder/hidden-submit-button]]]))
|
||||
|
||||
|
||||
|
||||
@@ -149,15 +198,18 @@
|
||||
(when (= "admin" (:user/role user))
|
||||
[:div
|
||||
[:div.is-pulled-right
|
||||
[:div.buttons
|
||||
|
||||
[job-button {:which :yodlee2} "Start Yodlee Import"]
|
||||
[job-button {:which :yodlee2-accounts} "Start Yodlee Account Import"]
|
||||
[job-button {:which :intuit} "Start Intuit"]
|
||||
[job-button {:which :plaid} "Start Plaid"]
|
||||
[job-button {:which :ezcater-upsert} "EZCater Sync"]
|
||||
[bulk-journal-import-button]
|
||||
[register-invoice-import-button]]]
|
||||
[:button.button.is-primary.is-small {:type "button"
|
||||
:on-click (dispatch-event [::modal/modal-requested
|
||||
{:title "Start a new job"
|
||||
:body [new-job-form]
|
||||
:cancel? false
|
||||
:confirm {:value "Submit"
|
||||
:status-from [::status/single ::form]
|
||||
:class "is-primary"
|
||||
:on-click (dispatch-event [::saving])
|
||||
:close-event [::status/completed ::form]}}])}
|
||||
"Run new job"]
|
||||
]
|
||||
[table/table {:id :jobs
|
||||
:data-page ::page}]])]))
|
||||
{:component-did-mount (dispatch-event [::mounted ])
|
||||
|
||||
Reference in New Issue
Block a user