diff --git a/src/clj/auto_ap/jobs/load_historical_sales.clj b/src/clj/auto_ap/jobs/load_historical_sales.clj new file mode 100644 index 00000000..69c31ad5 --- /dev/null +++ b/src/clj/auto_ap/jobs/load_historical_sales.clj @@ -0,0 +1,99 @@ +(ns auto-ap.jobs.load-historical-sales + (:gen-class) + (:require + [unilog.context :as lc] + [auto-ap.jobs.core :refer [execute]] + [amazonica.aws.s3 :as s3] + [auto-ap.square.core :as square] + [auto-ap.square.core2 :as square2] + [clj-time.periodic :as per] + [clj-time.core :as time] + [datomic.api :as d] + [auto-ap.utils :refer [dollars=]] + [clojure.string :as str] + [clj-time.coerce :as coerce] + [auto-ap.time :as atime] + [auto-ap.datomic :refer [conn audit-transact]] + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [config.core :refer [env]] + [clojure.tools.logging :as log])) +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(defn historical-load-sales [client days] + (log/info "loading old approach") + (let [client (d/pull (d/db auto-ap.datomic/conn) + square/square-read + client)] + (doseq [square-location (:client/square-locations client) + :when (:square-location/client-location square-location)] + + (println "orders") + (lc/with-context {:source "Historical loading data"} + (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days))) + (time/today) + (time/days 1))] + (println d) + (square/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1)))))) + + (println "refunds") + (square/upsert-refunds client square-location) + + + (println "settlements") + (with-redefs [square/lookup-dates (fn lookup-dates [] + (->> (per/periodic-seq (time/plus (time/today) (time/days (- days))) + (time/today) + (time/days 2)) + (map (fn [d] + [(atime/unparse (time/plus d (time/days 1)) atime/iso-date) + + (atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))] + + (square/upsert-settlements client square-location))))) + + +(defn historical-load-sales2 [client days] + (log/info "loading new approach") + (let [client (d/pull (d/db auto-ap.datomic/conn) + square/square-read + client)] + (doseq [square-location (:client/square-locations client) + :when (:square-location/client-location square-location)] + + (println "orders") + (lc/with-context {:source "Historical loading data"} + (doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days))) + (time/today) + (time/days 1))] + (println d) + (square2/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1)))))) + + (println "refunds") + (square2/upsert-refunds client square-location) + + + (println "settlements") + (with-redefs [square2/lookup-dates (fn lookup-dates [] + (->> (per/periodic-seq (time/plus (time/today) (time/days (- days))) + (time/today) + (time/days 2)) + (map (fn [d] + [(atime/unparse (time/plus d (time/days 1)) atime/iso-date) + + (atime/unparse (time/plus d (time/days 2)) atime/iso-date)]))))] + + (square2/upsert-settlements client square-location))))) + + +(defn load-historical-sales [args] + (let [{:keys [days client]} args + client (Long/parseLong client) + feature-flags (-> (d/pull (d/db conn) '[:client/feature-flags] client) + :client/feature-flags + set)] + (if (get feature-flags "new-square") + (historical-load-sales2 client days) + (historical-load-sales client days)))) + +(defn -main [& _] + (execute "load-historical-sales" #(load-historical-sales (:args env)))) diff --git a/src/clj/auto_ap/jobs/register_invoice_import.clj b/src/clj/auto_ap/jobs/register_invoice_import.clj index 5073adf5..29723aa1 100644 --- a/src/clj/auto_ap/jobs/register_invoice_import.clj +++ b/src/clj/auto_ap/jobs/register_invoice_import.clj @@ -124,8 +124,8 @@ vec))) (defn register-invoice-import [args] - (let [{:keys [ledger-url]} args - data (s3->csv ledger-url)] + (let [{:keys [invoice-url]} args + data (s3->csv invoice-url)] (log/info "contains " (count data) " rows") (doseq [n (partition-all 50 (register-invoice-import* data))] (log/info "transacting" n) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 80619072..84f79fff 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -9,6 +9,7 @@ [auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices] [auto-ap.jobs.intuit :as job-intuit] [auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger] + [auto-ap.jobs.load-historical-sales :as job-load-historical-sales] [auto-ap.jobs.plaid :as job-plaid] [auto-ap.jobs.register-invoice-import :as job-register-invoice-import] [auto-ap.jobs.square :as job-square] @@ -130,6 +131,9 @@ (= job "register-invoice-import") (job-register-invoice-import/-main) + (= job "load-historical-sales") + (job-load-historical-sales/-main) + (= job "bulk-journal-import") (job-bulk-journal-import/-main) diff --git a/src/clj/user.clj b/src/clj/user.clj index 302b131c..aee27369 100644 --- a/src/clj/user.clj +++ b/src/clj/user.clj @@ -435,70 +435,6 @@ (async/> (per/periodic-seq (t/plus (t/today) (t/days (- days))) - (t/today) - (t/days 2)) - (map (fn [d] - [(atime/unparse (t/plus d (t/days 1)) atime/iso-date) - - (atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))] - - (square/upsert-settlements client square-location))))) - - -(defn historical-load-sales2 [client-code days] - (let [client (d/pull (d/db auto-ap.datomic/conn) - square/square-read - [:client/code client-code])] - (doseq [square-location (:client/square-locations client) - :when (:square-location/client-location square-location)] - - (println "orders") - (lc/with-context {:source "Historical loading data"} - (doseq [d (per/periodic-seq (t/plus (t/today) (t/days (- days))) - (t/today) - (t/days 1))] - (println d) - (square2/upsert client square-location (c/to-date-time d) (c/to-date-time (t/plus d (t/days 1)))))) - - (println "refunds") - (square2/upsert-refunds client square-location) - - - (println "settlements") - (with-redefs [square2/lookup-dates (fn lookup-dates [] - (->> (per/periodic-seq (t/plus (t/today) (t/days (- days))) - (t/today) - (t/days 2)) - (map (fn [d] - [(atime/unparse (t/plus d (t/days 1)) atime/iso-date) - - (atime/unparse (t/plus d (t/days 2)) atime/iso-date)]))))] - - (square2/upsert-settlements client square-location))))) - #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-sales-for-day [date] (doseq [client (d/q [:find [(list 'pull '?e square/square-read ) '...] diff --git a/src/cljs/auto_ap/views/pages/admin/jobs.cljs b/src/cljs/auto_ap/views/pages/admin/jobs.cljs index 57c5e953..ac497b29 100644 --- a/src/cljs/auto_ap/views/pages/admin/jobs.cljs +++ b/src/cljs/auto_ap/views/pages/admin/jobs.cljs @@ -5,6 +5,8 @@ [auto-ap.views.components.admin.side-bar :refer [admin-side-bar]] [auto-ap.views.components.layouts :refer [side-bar-layout]] [auto-ap.views.pages.admin.jobs.table :as table] + [auto-ap.views.components.modal :as modal] + [auto-ap.views.components :as com] [auto-ap.views.pages.data-page :as data-page] [auto-ap.views.utils :refer [dispatch-event with-user]] [clojure.set :as set] @@ -15,9 +17,10 @@ [auto-ap.forms :as forms] [clojure.string :as str])) +(def a 1) + (def default-read [:name :start-date :end-date :status]) -(def job-types [:yodlee2 :yodlee2-accounts :intuit :plaid :bulk-journal-import :register-invoice-import :ezcater-upsert]) (re-frame/reg-event-fx ::params-change @@ -51,23 +54,38 @@ :dispatch [::params-change]})) (re-frame/reg-event-fx - ::request + ::saving [with-user - (re-frame/inject-cofx ::inject/sub [::forms/form ::bulk-journal-form]) + (re-frame/inject-cofx ::inject/sub [::forms/form ::form]) ] (fn [{:keys [user] ::forms/keys [form]} [_ which]] - {:graphql {:token user - :owns-state {:single which} - :query-obj - {:venia/operation {:operation/type :mutation - :operation/name "RequestJob"} - :venia/queries [{:query/data - [:request-job - {:which (name which) - :args (when (#{:bulk-journal-import :register-invoice-import} which) - (str/escape (pr-str (:data form)) {\" "\\\""}))} - [:message]]}]} - :on-success [::success]}})) + (let [which (:job (:data form))] + + {:graphql {:token user + :owns-state {:single which} + :query-obj + {:venia/operation {:operation/type :mutation + :operation/name "RequestJob"} + :venia/queries [{:query/data + [:request-job + {:which (str which) + :args (-> (cond (= which :bulk-journal-import) + (select-keys (:data form) [:ledger-url]) + + (= which :register-invoice-import) + (select-keys (:data form) [:invoice-url]) + + (= which :load-historical-sales) + {:days (-> form :data :days) + :client (-> form :data :client :id)} + + :else + nil + ) + pr-str + (str/escape {\" "\\\""}))} + [:message]]}]} + :on-success [::success]}}))) (re-frame/reg-event-fx ::mounted @@ -80,11 +98,7 @@ (re-frame/reg-event-fx ::unmounted (fn [{:keys [db]}] - {:dispatch-n (into [[::data-page/dispose ::page]] - (map (fn [jt] - [::status/dispose-single jt] - ) - job-types)) + {:dispatch-n [[::data-page/dispose ::page]] ::track/dispose {:id ::params} :db (dissoc db ::msg)})) @@ -98,42 +112,77 @@ content])) (defn notification [] - [:<> - (when-let [msg @(re-frame/subscribe [::msg])] - [:div.notification.is-info.is-light msg]) - (doall (for [status job-types - :let [state @(re-frame/subscribe [::status/single status])]] - ^{:key (str status) } - [:div - (cond (:info state) - [:div.notification.is-info.is-light (:info state)] + (let [state @(re-frame/subscribe [::status/single ::form])] + [:<> + (when-let [msg @(re-frame/subscribe [::msg])] + [:div.notification.is-info.is-light msg]) + [:div + (cond (:info state) + [:div.notification.is-info.is-light (:info state)] - (seq (:error state)) - [:div.notification.is-info.is-warning.is-light (:message (first (:error state)))] + (seq (:error state)) + [:div.notification.is-info.is-warning.is-light (:message (first (:error state)))] - :else - nil)]))]) + :else + nil)]])) -(defn bulk-journal-import-button [] - [form-builder/builder {:submit-event [::request :bulk-journal-import] - :id ::bulk-journal-form} - [:p.field.has-addons - [:p.control - [:a.button.is-static.is-small - "s3://data.prod.app.integreatconsult.com/bulk-import/"]] - [:p.control - [form-builder/raw-field-v2 {:field :ledger-url} - [:input.input.is-small {:placeholder "ledger-data.csv"}]]] - [:p.control - [form-builder/submit-button {:class "is-small"} "Ledger Import"] - ]]]) +(defn new-job-form [] + (let [{:keys [data]} @(re-frame/subscribe [::forms/form ::form]) + clients @(re-frame/subscribe [::subs/clients])] + [:<> + [form-builder/builder {:submit-event [::saving] + :id ::form} + [form-builder/field-v2 {:required? true + :field :job} + "Job Type" + [com/select-field {:options [[:yodlee2 "Yodlee Import"] + [:yodlee2-accounts "Yodlee Account Import"] + [:intuit "Intuit import"] + [:plaid "Plaid import"] + [:bulk-journal-import "Bulk Journal Import"] + [:register-invoice-import "Register Invoice Import "] + [:ezcater-upsert "Upsert recent ezcater orders"] + [:load-historical-sales "Load Historical Square Sales"]] + :allow-nil? true + :keywordize? true}]] + (cond (= :bulk-journal-import (:job data)) + [:<> + [:label.label "Url"] + [:p.field.has-addons + [:p.control + [:a.button.is-static + "s3://data.prod.app.integreatconsult.com/bulk-import/"]] + [:p.control + [form-builder/raw-field-v2 {:field :ledger-url} + [:input.input {:placeholder "ledger-data.csv"}]]]]] -(defn register-invoice-import-button [] - [form-builder/builder {:submit-event [::request :register-invoice-import] - :id ::register-invoice-import-form} - [:p.control - [form-builder/submit-button {:class "is-small"} "Register Invoice Import"] - ]]) + (= :register-invoice-import (:job data)) + [:<> + [:label.label "Url"] + [:p.field.has-addons + [:p.control + [:a.button.is-static + "s3://data.prod.app.integreatconsult.com/bulk-import/"]] + [:p.control + [form-builder/raw-field-v2 {:field :invoice-url} + [:input.input {:placeholder "invoice-data.csv"}]]]]] + + (= :load-historical-sales (:job data)) + [:<> + [form-builder/field-v2 {:required? true + :field [:client]} + "Client" + [com/entity-typeahead {:entities @(re-frame/subscribe [::subs/clients]) + :entity->text :name}]] + + [form-builder/field-v2 {:field :days} + "Days to load (recommended 60)" + [com/number-input {:placeholder "60" + :style {:width "7em"}}]]] + :else + nil) + + [form-builder/hidden-submit-button]]])) @@ -149,15 +198,18 @@ (when (= "admin" (:user/role user)) [:div [:div.is-pulled-right - [:div.buttons - - [job-button {:which :yodlee2} "Start Yodlee Import"] - [job-button {:which :yodlee2-accounts} "Start Yodlee Account Import"] - [job-button {:which :intuit} "Start Intuit"] - [job-button {:which :plaid} "Start Plaid"] - [job-button {:which :ezcater-upsert} "EZCater Sync"] - [bulk-journal-import-button] - [register-invoice-import-button]]] + [:button.button.is-primary.is-small {:type "button" + :on-click (dispatch-event [::modal/modal-requested + {:title "Start a new job" + :body [new-job-form] + :cancel? false + :confirm {:value "Submit" + :status-from [::status/single ::form] + :class "is-primary" + :on-click (dispatch-event [::saving]) + :close-event [::status/completed ::form]}}])} + "Run new job"] + ] [table/table {:id :jobs :data-page ::page}]])])) {:component-did-mount (dispatch-event [::mounted ]) diff --git a/terraform/deploy.tf b/terraform/deploy.tf index ec67e636..7c8dd398 100644 --- a/terraform/deploy.tf +++ b/terraform/deploy.tf @@ -456,4 +456,16 @@ module "register_invoice_import_job" { use_schedule = false memory = 8192 cpu = 2048 +} + +module "load_historical_sales_job" { + source = "./background-job/" + ecs_cluster = var.ecs_cluster + task_role_arn = var.task_role_arn + stage = var.stage + job_name = "load-historical-sales" + execution_role_arn = var.execution_role_arn + use_schedule = false + memory = 4096 + cpu = 1024 } \ No newline at end of file