Add vendor pre-population for bulk code and individual edit forms
- Add vendor-changed HTMX handlers for both bulk code and individual edit - Pre-populate default account at 100% when vendor is selected and no accounts exist - Fix render-accounts-section to render from step-params correctly - Change bulk code vendor-changed from hx-get to hx-post to include form data - Add routes for vendor-changed endpoints - Update e2e tests to cover vendor pre-population - Run lein cljfmt fix across codebase
This commit is contained in:
@@ -12,12 +12,11 @@
|
||||
(defn line->id [{:keys [source id client-code]}]
|
||||
(str client-code "-" source "-" id))
|
||||
|
||||
|
||||
(defn csv->graphql-rows [lines]
|
||||
(for [lines (partition-by line->id (drop 1 lines))
|
||||
:let [{:keys [source client-code date vendor-name note cleared-against] :as line} (first lines)]]
|
||||
:let [{:keys [source client-code date vendor-name note cleared-against] :as line} (first lines)]]
|
||||
{:source source
|
||||
:external_id (line->id line)
|
||||
:external_id (line->id line)
|
||||
:client_code client-code
|
||||
:date date
|
||||
:note note
|
||||
@@ -33,8 +32,8 @@
|
||||
{:account_identifier account-identifier
|
||||
:location (some-> location str/trim)
|
||||
:debit (if (str/blank? debit)
|
||||
0.0
|
||||
(Double/parseDouble debit))
|
||||
0.0
|
||||
(Double/parseDouble debit))
|
||||
:credit (if (str/blank? credit)
|
||||
0.0
|
||||
(Double/parseDouble credit))})
|
||||
|
||||
@@ -20,8 +20,7 @@
|
||||
|
||||
(mapv (fn [[i]] {:db/id i
|
||||
:invoice/outstanding-balance 0.0
|
||||
:invoice/status :invoice-status/paid}))
|
||||
))
|
||||
:invoice/status :invoice-status/paid}))))
|
||||
|
||||
(alog/info ::closed :count (count invoices-to-close))))
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
(ns auto-ap.jobs.core
|
||||
(:require [auto-ap.utils :refer [heartbeat]]
|
||||
[mount.core :as mount]
|
||||
[auto-ap.datomic :refer [conn ]]
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.logging :as alog]
|
||||
[nrepl.server :refer [start-server]]
|
||||
[auto-ap.background.metrics :refer [metrics-setup container-tags container-data logging-context]]
|
||||
@@ -13,8 +13,8 @@
|
||||
:service name}
|
||||
(mu/trace ::execute-background-job
|
||||
[]
|
||||
(try
|
||||
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data }))
|
||||
(try
|
||||
(mount/start (mount/only #{#'conn #'metrics-setup #'container-tags #'logging-context #'container-data}))
|
||||
(start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler))
|
||||
((heartbeat f name))
|
||||
(alog/info ::stopping :job name)
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
(javax.mail Session)
|
||||
(javax.mail.internet MimeMessage)))
|
||||
|
||||
(defn send-email-about-failed-message [mail-bucket mail-key message]
|
||||
(defn send-email-about-failed-message [mail-bucket mail-key message]
|
||||
(let [target-key (str "failed-emails/" mail-key ".eml")
|
||||
target-url (str "https://" (:data-bucket env) "/" target-key)]
|
||||
(alog/info ::sending-failure-email :who (:import-failure-destination-emails env))
|
||||
@@ -29,7 +29,6 @@
|
||||
:body {:html (str "<div>You can download the original email <a href=\"" target-url "\">here</a>.<p><pre>" message "</pre></p></div>")
|
||||
:text (str "<div>You can download the original email here: " target-url)}}})))
|
||||
|
||||
|
||||
(defn process-sqs []
|
||||
(alog/info ::fetching-sqs)
|
||||
(doseq [message (:messages (sqs/receive-message {:queue-url (:invoice-import-queue-url env)
|
||||
@@ -79,27 +78,20 @@
|
||||
(defn -main [& _]
|
||||
(execute "import-uploaded-invoices" process-sqs))
|
||||
|
||||
|
||||
(comment
|
||||
(comment
|
||||
(with-open [i (io/output-stream "/tmp/bryce.pdf")]
|
||||
(clojure.java.io/copy
|
||||
(clojure.java.io/copy
|
||||
(-> (s3/get-object :bucket-name (:data-bucket env)
|
||||
:key "invoice-files/f0e73dcb-e5e5-4c81-b82b-319b7caedacf.pdf"
|
||||
|
||||
)
|
||||
:key "invoice-files/f0e73dcb-e5e5-4c81-b82b-319b7caedacf.pdf")
|
||||
|
||||
:input-stream)
|
||||
i))
|
||||
|
||||
(parse/parse-file "/tmp/bryce.pdf" "/tmp/bryce.pdf")
|
||||
|
||||
|
||||
(-> (clojure.java.shell/sh "pdftotext" "-layout" "/tmp/bryce.pdf" "-")
|
||||
:out
|
||||
)
|
||||
(-> (clojure.java.shell/sh "pdftotext" "-layout" "/tmp/bryce.pdf" "-")
|
||||
:out)
|
||||
|
||||
|
||||
1
|
||||
|
||||
(user/init-repl)
|
||||
|
||||
)
|
||||
(user/init-repl))
|
||||
@@ -23,39 +23,39 @@
|
||||
[?t :transaction/client ?c]])))
|
||||
|
||||
(defn get-pinecone [transaction-id]
|
||||
(->
|
||||
(http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
|
||||
(->
|
||||
(http2/get (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/vectors/fetch"
|
||||
url/url
|
||||
(assoc :query {:ids transaction-id})
|
||||
str)
|
||||
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
|
||||
:as :json
|
||||
:keywordize? false})
|
||||
:body
|
||||
:vectors
|
||||
((keyword (str transaction-id)))
|
||||
:values))
|
||||
:body
|
||||
:vectors
|
||||
((keyword (str transaction-id)))
|
||||
:values))
|
||||
|
||||
(defn get-pinecone-similarities [transaction-id]
|
||||
(if-let [vector (get-pinecone transaction-id)]
|
||||
(filter
|
||||
(fn [{:keys [score]}]
|
||||
(> score 0.95))
|
||||
(->
|
||||
(http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
|
||||
url/url
|
||||
str)
|
||||
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
|
||||
:form-params {"vector" vector
|
||||
"topK" 200,
|
||||
"includeMetadata" true
|
||||
"namespace" ""}
|
||||
:content-type :json
|
||||
:as :json})
|
||||
:body
|
||||
:matches
|
||||
(doto (#(alog/info ::similarities-found :transaction transaction-id :count (count %) :sample (take 3 %))))))
|
||||
|
||||
(filter
|
||||
(fn [{:keys [score]}]
|
||||
(> score 0.95))
|
||||
(->
|
||||
(http2/post (-> "https://transactions-a8257ba.svc.us-west4-gcp-free.pinecone.io/query"
|
||||
url/url
|
||||
str)
|
||||
{:headers {"Api-Key" "f2d3a78e-bcea-4fcd-88b6-2527b8423607"}
|
||||
:form-params {"vector" vector
|
||||
"topK" 200,
|
||||
"includeMetadata" true
|
||||
"namespace" ""}
|
||||
:content-type :json
|
||||
:as :json})
|
||||
:body
|
||||
:matches
|
||||
(doto (#(alog/info ::similarities-found :transaction transaction-id :count (count %) :sample (take 3 %))))))
|
||||
|
||||
(do (alog/info ::no-matches-for :transaction transaction-id)
|
||||
[])))
|
||||
|
||||
|
||||
@@ -10,16 +10,15 @@
|
||||
[config.core :refer [env]]
|
||||
[datomic.api :as dc]))
|
||||
|
||||
|
||||
(defn historical-load-sales [client days]
|
||||
(alog/info ::new-sales-loading :client (:client/code client) :days days)
|
||||
(let [client (dc/pull (dc/db auto-ap.datomic/conn)
|
||||
square3/square-read
|
||||
client)
|
||||
days (cond-> days (string? days) ( #(Long/parseLong %)))]
|
||||
square3/square-read
|
||||
client)
|
||||
days (cond-> days (string? days) (#(Long/parseLong %)))]
|
||||
(doseq [square-location (:client/square-locations client)
|
||||
:when (:square-location/client-location square-location)]
|
||||
|
||||
|
||||
(println "orders")
|
||||
(doseq [d (per/periodic-seq (time/plus (time/today) (time/days (- days)))
|
||||
(time/plus (time/today) (time/days 2))
|
||||
@@ -28,14 +27,13 @@
|
||||
@(square3/upsert client square-location (coerce/to-date-time d) (coerce/to-date-time (time/plus d (time/days 1)))))
|
||||
|
||||
(println "refunds")
|
||||
@(square3/upsert-refunds client square-location)
|
||||
@(square3/upsert-payouts client square-location (time/plus (time/now) (time/days (- days))) (time/now)))))
|
||||
|
||||
@(square3/upsert-refunds client square-location)
|
||||
@(square3/upsert-payouts client square-location (time/plus (time/now) (time/days (- days))) (time/now)))))
|
||||
|
||||
(defn load-historical-sales [args]
|
||||
(let [{:keys [days client]} args
|
||||
client (cond-> client
|
||||
( string? client) ( #( Long/parseLong %)))]
|
||||
client (cond-> client
|
||||
(string? client) (#(Long/parseLong %)))]
|
||||
(historical-load-sales client days)))
|
||||
|
||||
(defn -main [& _]
|
||||
|
||||
@@ -28,19 +28,17 @@
|
||||
(defn read-xml [stream]
|
||||
(-> (slurp stream)
|
||||
(.getBytes)
|
||||
(java.io.ByteArrayInputStream. )
|
||||
(java.io.ByteArrayInputStream.)
|
||||
xml/parse
|
||||
zip/xml-zip))
|
||||
|
||||
|
||||
|
||||
(defn mark-key [k]
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:destination-key (str/replace-first k "pending" "imported")
|
||||
:source-key k})
|
||||
#_(s3/delete-object {:bucket-name bucket-name
|
||||
:key k}))
|
||||
:key k}))
|
||||
|
||||
(defn is-csv-file? [x]
|
||||
(= "dat" (last (str/split x #"[\\.]"))))
|
||||
@@ -54,7 +52,7 @@
|
||||
(and (str/includes? k "GeneralProduce")
|
||||
(str/includes? k "FRANCHISEE")
|
||||
(is-csv-file? k))
|
||||
:general-produce
|
||||
:general-produce
|
||||
|
||||
:else
|
||||
:unknown))
|
||||
@@ -66,15 +64,15 @@
|
||||
[k input-stream clients]
|
||||
(log/info ::parsing-general-produce :key k)
|
||||
(let [missing-client-hints (atom #{})]
|
||||
(try
|
||||
(try
|
||||
(->> (read-csv input-stream)
|
||||
(drop 1)
|
||||
#_(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
|
||||
(= "Y" break-flag)))
|
||||
(map (fn [[_ location-hint invoice-number ship-date invoice-total ]]
|
||||
(map (fn [[_ location-hint invoice-number ship-date invoice-total]]
|
||||
(let [matching-client (and location-hint
|
||||
(parse/exact-match clients location-hint))
|
||||
location (parse/best-location-match matching-client location-hint location-hint )
|
||||
location (parse/best-location-match matching-client location-hint location-hint)
|
||||
vendor (d/pull (d/db conn) '[:vendor/default-account] :vendor/general-produce)]
|
||||
(when-not (and matching-client
|
||||
(not (@missing-client-hints location-hint))
|
||||
@@ -99,8 +97,7 @@
|
||||
(-> vendor :vendor/default-account :db/id)
|
||||
:invoice-expense-account/location location
|
||||
:invoice-expense-account/amount (Math/abs (Double/parseDouble invoice-total))
|
||||
:db/id (random-tempid)
|
||||
}]})))
|
||||
:db/id (random-tempid)}]})))
|
||||
(filter :invoice/client)
|
||||
(reduce (fn [[seen-so-far list] i]
|
||||
(let [k [(:invoice/invoice-number i) (:invoice/client i)]]
|
||||
@@ -108,8 +105,7 @@
|
||||
[seen-so-far list]
|
||||
[(conj seen-so-far k) (conj list i)])))
|
||||
[#{} []])
|
||||
(second)
|
||||
)
|
||||
(second))
|
||||
(catch Exception e
|
||||
(log/error ::cant-import-general-produce
|
||||
:error e)
|
||||
@@ -123,8 +119,8 @@
|
||||
|
||||
(defn zip-seq [zipper]
|
||||
(->> (zip/xml-zip (zip/node zipper))
|
||||
(iterate zip/next )
|
||||
(take-while (complement zip/end?))))
|
||||
(iterate zip/next)
|
||||
(take-while (complement zip/end?))))
|
||||
|
||||
(defmethod extract-invoice-details :cintas
|
||||
[k input-stream clients]
|
||||
@@ -160,10 +156,10 @@
|
||||
atime/localize
|
||||
(atime/unparse atime/iso-date)
|
||||
(atime/parse atime/iso-date))))
|
||||
location (parse/best-location-match matching-client location-hint location-hint )
|
||||
location (parse/best-location-match matching-client location-hint location-hint)
|
||||
due (-> invoice-date
|
||||
(time/plus (time/days 30))
|
||||
(coerce/to-date))
|
||||
(time/plus (time/days 30))
|
||||
(coerce/to-date))
|
||||
total (->> node-seq
|
||||
(filter (fn [zipper]
|
||||
(= (:tag (zip/node zipper))
|
||||
@@ -178,7 +174,7 @@
|
||||
:content
|
||||
first
|
||||
Double/parseDouble)
|
||||
invoice {:db/id (random-tempid )
|
||||
invoice {:db/id (random-tempid)
|
||||
:invoice/vendor :vendor/cintas
|
||||
:invoice/import-status :import-status/imported
|
||||
:invoice/status :invoice-status/unpaid
|
||||
@@ -188,37 +184,36 @@
|
||||
:invoice/total total
|
||||
:invoice/outstanding-balance total
|
||||
:invoice/invoice-number (->> node-seq
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= (:tag node)
|
||||
:InvoiceDetailRequestHeader)))
|
||||
first
|
||||
(#(-> % :attrs :invoiceID)))
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= (:tag node)
|
||||
:InvoiceDetailRequestHeader)))
|
||||
first
|
||||
(#(-> % :attrs :invoiceID)))
|
||||
:invoice/due due
|
||||
|
||||
:invoice/scheduled-payment (when-not ((into #{} (->> matching-client
|
||||
:client/feature-flags))
|
||||
"manually-pay-cintas")
|
||||
due)
|
||||
due)
|
||||
|
||||
:invoice/date (coerce/to-date invoice-date)
|
||||
:invoice/expense-accounts [{:invoice-expense-account/account
|
||||
(-> vendor :vendor/default-account :db/id)
|
||||
:invoice-expense-account/location location
|
||||
:invoice-expense-account/amount (Math/abs total)
|
||||
:db/id (random-tempid)
|
||||
}]}]
|
||||
:db/id (random-tempid)}]}]
|
||||
(log/info ::cintas-invoice-importing
|
||||
:invoice invoice)
|
||||
[invoice])
|
||||
(do
|
||||
(do
|
||||
;; disabling logging for cintas
|
||||
#_(log/warn ::missing-client
|
||||
:client-hint location-hint)
|
||||
:client-hint location-hint)
|
||||
[]))))
|
||||
|
||||
(defn mark-error [k]
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key (str "ntg-invoices/error/"
|
||||
@@ -232,17 +227,17 @@
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key invoice-key })
|
||||
:destination-key invoice-key})
|
||||
invoice-key))
|
||||
|
||||
(defn get-all-keys
|
||||
([]
|
||||
(let [first-page-result (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
(let [first-page-result (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
:prefix "ntg-invoices/pending"})]
|
||||
(lazy-seq (concat (:object-summaries first-page-result) (get-all-keys (:next-continuation-token first-page-result))))))
|
||||
([next-token ]
|
||||
(when next-token
|
||||
(let [page-result (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
([next-token]
|
||||
(when next-token
|
||||
(let [page-result (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
:prefix "ntg-invoices/pending"
|
||||
:continuation-token next-token})]
|
||||
(println "getting next page " next-token)
|
||||
@@ -250,60 +245,58 @@
|
||||
(lazy-seq (concat (:object-summaries page-result) (get-all-keys (:next-continuation-token page-result)))))))))
|
||||
|
||||
(defn recent? [k]
|
||||
(time/after? (:last-modified k) (time/plus (time/now) (time/days -15)))
|
||||
)
|
||||
(time/after? (:last-modified k) (time/plus (time/now) (time/days -15))))
|
||||
|
||||
(defn import-ntg-invoices
|
||||
([] (import-ntg-invoices (->> (get-all-keys)
|
||||
(filter recent?)
|
||||
(map :key))))
|
||||
([keys]
|
||||
(let [clients (map first (d/q '[:find (pull ?c [:client/code
|
||||
:db/id
|
||||
:client/feature-flags
|
||||
{:client/location-matches [:location-match/matches :location-match/location]}
|
||||
:client/name
|
||||
:client/matches
|
||||
:client/locations])
|
||||
:where [?c :client/code]]
|
||||
(d/db conn)))]
|
||||
(log/info ::found-invoice-keys
|
||||
:keys keys )
|
||||
(let [transaction (->> keys
|
||||
(mapcat (fn [k]
|
||||
(try
|
||||
(let [invoice-key (copy-readable-version k)
|
||||
invoice-url (str "https://" bucket-name "/" invoice-key)]
|
||||
(with-open [is (-> (s3/get-object {:bucket-name bucket-name
|
||||
:key k})
|
||||
:input-stream)]
|
||||
(->> (extract-invoice-details k
|
||||
is
|
||||
clients)
|
||||
(set)
|
||||
(map (fn [i]
|
||||
(log/info ::importing-invoice
|
||||
:invoice i)
|
||||
i))
|
||||
(mapv (fn [i]
|
||||
(if (= :vendor/cintas (:invoice/vendor i))
|
||||
[:propose-invoice (assoc i :invoice/source-url invoice-url)]
|
||||
[:propose-invoice i]))))))
|
||||
(catch Exception e
|
||||
(log/error ::cant-load-file
|
||||
:key k
|
||||
:exception e)
|
||||
(mark-error k)
|
||||
[]))))
|
||||
(into []))]
|
||||
(doseq [t transaction]
|
||||
(audit-transact [t] {:user/name "sysco importer" :user/role "admin"}))
|
||||
(log/info ::success
|
||||
:count (count transaction)
|
||||
:sample (take 3 transaction)))
|
||||
(doseq [k keys]
|
||||
(mark-key k)))))
|
||||
|
||||
(let [clients (map first (d/q '[:find (pull ?c [:client/code
|
||||
:db/id
|
||||
:client/feature-flags
|
||||
{:client/location-matches [:location-match/matches :location-match/location]}
|
||||
:client/name
|
||||
:client/matches
|
||||
:client/locations])
|
||||
:where [?c :client/code]]
|
||||
(d/db conn)))]
|
||||
(log/info ::found-invoice-keys
|
||||
:keys keys)
|
||||
(let [transaction (->> keys
|
||||
(mapcat (fn [k]
|
||||
(try
|
||||
(let [invoice-key (copy-readable-version k)
|
||||
invoice-url (str "https://" bucket-name "/" invoice-key)]
|
||||
(with-open [is (-> (s3/get-object {:bucket-name bucket-name
|
||||
:key k})
|
||||
:input-stream)]
|
||||
(->> (extract-invoice-details k
|
||||
is
|
||||
clients)
|
||||
(set)
|
||||
(map (fn [i]
|
||||
(log/info ::importing-invoice
|
||||
:invoice i)
|
||||
i))
|
||||
(mapv (fn [i]
|
||||
(if (= :vendor/cintas (:invoice/vendor i))
|
||||
[:propose-invoice (assoc i :invoice/source-url invoice-url)]
|
||||
[:propose-invoice i]))))))
|
||||
(catch Exception e
|
||||
(log/error ::cant-load-file
|
||||
:key k
|
||||
:exception e)
|
||||
(mark-error k)
|
||||
[]))))
|
||||
(into []))]
|
||||
(doseq [t transaction]
|
||||
(audit-transact [t] {:user/name "sysco importer" :user/role "admin"}))
|
||||
(log/info ::success
|
||||
:count (count transaction)
|
||||
:sample (take 3 transaction)))
|
||||
(doseq [k keys]
|
||||
(mark-key k)))))
|
||||
|
||||
(defn -main [& _]
|
||||
(execute "ntg" import-ntg-invoices))
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
(def bucket (:data-bucket env))
|
||||
|
||||
(defn s3->csv [url]
|
||||
(try
|
||||
(try
|
||||
(->> (-> (s3/get-object {:bucket-name bucket
|
||||
:key (str "bulk-import/" url)})
|
||||
:input-stream
|
||||
@@ -26,9 +26,9 @@
|
||||
csv/read-csv))
|
||||
(catch Exception e
|
||||
(alog/error
|
||||
:file-not-found
|
||||
:error e
|
||||
:url url)
|
||||
:file-not-found
|
||||
:error e
|
||||
:url url)
|
||||
(throw e))))
|
||||
|
||||
(defn register-invoice-import* [data]
|
||||
@@ -45,106 +45,100 @@
|
||||
(reduce + 0.0
|
||||
(->> values
|
||||
(map (fn [[_ _ _ _ amount]]
|
||||
(- (Double/parseDouble amount))))))
|
||||
]))
|
||||
(- (Double/parseDouble amount))))))]))
|
||||
(into {}))]
|
||||
(->>
|
||||
(for [[i
|
||||
invoice-expense-account-id
|
||||
target-account
|
||||
target-date
|
||||
amount
|
||||
_
|
||||
location] (drop 1 data)
|
||||
:let [invoice-id (i->invoice-id i)
|
||||
(->>
|
||||
(for [[i
|
||||
invoice-expense-account-id
|
||||
target-account
|
||||
target-date
|
||||
amount
|
||||
_
|
||||
location] (drop 1 data)
|
||||
:let [invoice-id (i->invoice-id i)
|
||||
|
||||
invoice (dc/pull db '[*] invoice-id)
|
||||
current-total (:invoice/total invoice)
|
||||
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
|
||||
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
||||
(:db/id (first (:invoice/expense-accounts invoice))))))
|
||||
invoice (dc/pull db '[*] invoice-id)
|
||||
current-total (:invoice/total invoice)
|
||||
target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible
|
||||
new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
||||
(:db/id (first (:invoice/expense-accounts invoice))))))
|
||||
|
||||
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
||||
(:db/id (first (:invoice/expense-accounts invoice)))
|
||||
(str (UUID/randomUUID)))
|
||||
invoice-expense-account (when-not new-account?
|
||||
(or (dc/pull db '[*] invoice-expense-account-id)
|
||||
(dc/pull db '[*] [:invoice-expense-account/original-id invoice-expense-account-id])))
|
||||
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
|
||||
target-account-id (Long/parseLong (str/trim target-account))
|
||||
invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong)
|
||||
(:db/id (first (:invoice/expense-accounts invoice)))
|
||||
(str (UUID/randomUUID)))
|
||||
invoice-expense-account (when-not new-account?
|
||||
(or (dc/pull db '[*] invoice-expense-account-id)
|
||||
(dc/pull db '[*] [:invoice-expense-account/original-id invoice-expense-account-id])))
|
||||
current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account))
|
||||
target-account-id (Long/parseLong (str/trim target-account))
|
||||
|
||||
target-date (coerce/to-date (atime/parse target-date atime/normal-date))
|
||||
current-date (:invoice/date invoice)
|
||||
|
||||
target-date (coerce/to-date (atime/parse target-date atime/normal-date))
|
||||
current-date (:invoice/date invoice)
|
||||
|
||||
current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0)
|
||||
target-expense-account-amount (- (Double/parseDouble amount))
|
||||
current-expense-account-amount (:invoice-expense-account/amount invoice-expense-account 0.0)
|
||||
target-expense-account-amount (- (Double/parseDouble amount))
|
||||
|
||||
current-expense-account-location (:invoice-expense-account/location invoice-expense-account)
|
||||
target-expense-account-location location
|
||||
|
||||
current-expense-account-location (:invoice-expense-account/location invoice-expense-account)
|
||||
target-expense-account-location location
|
||||
[[_ _ invoice-payment]] (vec (dc/q
|
||||
'[:find ?p ?a ?ip
|
||||
:in $ ?i
|
||||
:where [?ip :invoice-payment/invoice ?i]
|
||||
[?ip :invoice-payment/amount ?a]
|
||||
[?ip :invoice-payment/payment ?p]]
|
||||
db invoice-id))]
|
||||
:when current-total]
|
||||
|
||||
[(when (not (dollars= current-total target-total))
|
||||
{:db/id invoice-id
|
||||
:invoice/total target-total})
|
||||
|
||||
[[_ _ invoice-payment]] (vec (dc/q
|
||||
'[:find ?p ?a ?ip
|
||||
:in $ ?i
|
||||
:where [?ip :invoice-payment/invoice ?i]
|
||||
[?ip :invoice-payment/amount ?a]
|
||||
[?ip :invoice-payment/payment ?p]
|
||||
]
|
||||
db invoice-id))]
|
||||
:when current-total]
|
||||
(when (and (not (dollars= 0.0 target-total))
|
||||
(= :invoice-status/voided (:db/ident (:invoice/status invoice))))
|
||||
{:db/id invoice-id
|
||||
:invoice/total target-total
|
||||
:invoice/status :invoice-status/paid})
|
||||
|
||||
[
|
||||
(when (not (dollars= current-total target-total))
|
||||
{:db/id invoice-id
|
||||
:invoice/total target-total})
|
||||
(when new-account?
|
||||
{:db/id invoice-id
|
||||
:invoice/expense-accounts invoice-expense-account-id})
|
||||
|
||||
(when (and (not (dollars= 0.0 target-total))
|
||||
(= :invoice-status/voided (:db/ident (:invoice/status invoice))))
|
||||
{:db/id invoice-id
|
||||
:invoice/total target-total
|
||||
:invoice/status :invoice-status/paid})
|
||||
(when (and target-date (not= current-date target-date))
|
||||
{:db/id invoice-id
|
||||
:invoice/date target-date})
|
||||
|
||||
(when new-account?
|
||||
{:db/id invoice-id
|
||||
:invoice/expense-accounts invoice-expense-account-id})
|
||||
(when (and
|
||||
(not (dollars= current-total target-total))
|
||||
invoice-payment)
|
||||
[:db/retractEntity invoice-payment])
|
||||
|
||||
(when (and target-date (not= current-date target-date))
|
||||
{:db/id invoice-id
|
||||
:invoice/date target-date})
|
||||
(when (or new-account?
|
||||
(not (dollars= current-expense-account-amount target-expense-account-amount)))
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/amount target-expense-account-amount})
|
||||
|
||||
(when (and
|
||||
(not (dollars= current-total target-total))
|
||||
invoice-payment)
|
||||
[:db/retractEntity invoice-payment])
|
||||
(when (not= current-expense-account-location
|
||||
target-expense-account-location)
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/location target-expense-account-location})
|
||||
|
||||
(when (or new-account?
|
||||
(not (dollars= current-expense-account-amount target-expense-account-amount)))
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/amount target-expense-account-amount})
|
||||
|
||||
(when (not= current-expense-account-location
|
||||
target-expense-account-location)
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/location target-expense-account-location})
|
||||
|
||||
(when (not= current-account-id target-account-id )
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/account target-account-id})])
|
||||
(mapcat identity)
|
||||
(filter identity)
|
||||
vec)))
|
||||
(when (not= current-account-id target-account-id)
|
||||
{:db/id invoice-expense-account-id
|
||||
:invoice-expense-account/account target-account-id})])
|
||||
(mapcat identity)
|
||||
(filter identity)
|
||||
vec)))
|
||||
|
||||
(defn register-invoice-import [args]
|
||||
(let [{:keys [invoice-url]} args
|
||||
data (s3->csv invoice-url)]
|
||||
(alog/info ::rows
|
||||
:count (count data))
|
||||
:count (count data))
|
||||
(doseq [n (partition-all 50 (register-invoice-import* data))]
|
||||
(alog/info ::transacting
|
||||
:count (count n)
|
||||
:sample (take 2 n))
|
||||
:count (count n)
|
||||
:sample (take 2 n))
|
||||
(audit-transact n {:user/name "register-invoice-import"
|
||||
:user/role "admin"}))))
|
||||
|
||||
|
||||
@@ -20,32 +20,29 @@
|
||||
(def buffered (ex/fixed-thread-executor 100))
|
||||
|
||||
(defn order-of-insert [entity-dependencies]
|
||||
(loop [entity-dependencies entity-dependencies
|
||||
(loop [entity-dependencies entity-dependencies
|
||||
order []]
|
||||
(let [next-order (for [[entity deps] entity-dependencies
|
||||
:when (not (seq deps))]
|
||||
entity)
|
||||
next-deps (reduce
|
||||
(fn [entity-dependencies next-entity]
|
||||
(into {}
|
||||
(map
|
||||
(fn [[k v]]
|
||||
[k (disj v next-entity)])
|
||||
entity-dependencies)))
|
||||
(apply dissoc entity-dependencies next-order)
|
||||
next-order)]
|
||||
(fn [entity-dependencies next-entity]
|
||||
(into {}
|
||||
(map
|
||||
(fn [[k v]]
|
||||
[k (disj v next-entity)])
|
||||
entity-dependencies)))
|
||||
(apply dissoc entity-dependencies next-order)
|
||||
next-order)]
|
||||
(if (seq next-deps)
|
||||
(recur next-deps (into order next-order))
|
||||
(into order next-order)))))
|
||||
|
||||
|
||||
|
||||
|
||||
(def loaded (atom #{}))
|
||||
|
||||
(defn upsert-batch
|
||||
[batch context]
|
||||
(de/future-with request-pool
|
||||
(de/future-with request-pool
|
||||
(mu/with-context context
|
||||
(transact-with-backoff batch))
|
||||
batch))
|
||||
@@ -68,39 +65,39 @@
|
||||
(mu/with-context {:entity entity}
|
||||
(mu/log ::starting)
|
||||
@(s/consume (fn [batch]
|
||||
(mu/with-context {:entity entity}
|
||||
(try
|
||||
(swap! so-far #(+ % (count batch)))
|
||||
(mu/log ::loaded :count (count batch)
|
||||
:so-far @so-far)
|
||||
(catch Exception e
|
||||
(mu/log ::error
|
||||
:exception e)
|
||||
(throw e)))))
|
||||
(->> (partition-all 1000 entities)
|
||||
(s/->source)
|
||||
(s/onto buffered)
|
||||
(s/map (fn [entities]
|
||||
(when @die?
|
||||
(reset! die? false)
|
||||
(throw (Exception. "dead")))
|
||||
(upsert-batch entities {:entity entity
|
||||
:service "restore-from-backup"
|
||||
:background-job "restore-from-backup"})))
|
||||
(s/buffer 20)
|
||||
(s/realize-each)))
|
||||
(mu/with-context {:entity entity}
|
||||
(try
|
||||
(swap! so-far #(+ % (count batch)))
|
||||
(mu/log ::loaded :count (count batch)
|
||||
:so-far @so-far)
|
||||
(catch Exception e
|
||||
(mu/log ::error
|
||||
:exception e)
|
||||
(throw e)))))
|
||||
(->> (partition-all 1000 entities)
|
||||
(s/->source)
|
||||
(s/onto buffered)
|
||||
(s/map (fn [entities]
|
||||
(when @die?
|
||||
(reset! die? false)
|
||||
(throw (Exception. "dead")))
|
||||
(upsert-batch entities {:entity entity
|
||||
:service "restore-from-backup"
|
||||
:background-job "restore-from-backup"})))
|
||||
(s/buffer 20)
|
||||
(s/realize-each)))
|
||||
(swap! loaded conj entity))))
|
||||
|
||||
(defn load-from-backup
|
||||
(defn load-from-backup
|
||||
([backup-id connection] (load-from-backup backup-id connection nil))
|
||||
([backup-id connection starting-at]
|
||||
(let [schema (edn/read-string (slurp (pull-file backup-id "schema.edn")))
|
||||
full-dependencies (edn/read-string (slurp (pull-file backup-id "full-dependencies.edn")))
|
||||
entity-dependencies (edn/read-string (slurp (pull-file backup-id "entity-dependencies.edn")))]
|
||||
@(dc/transact connection [{:db/ident :entity/migration-key
|
||||
:db/unique :db.unique/identity
|
||||
:db/cardinality :db.cardinality/one
|
||||
:db/valueType :db.type/long}])
|
||||
:db/unique :db.unique/identity
|
||||
:db/cardinality :db.cardinality/one
|
||||
:db/valueType :db.type/long}])
|
||||
@(dc/transact connection (map
|
||||
(fn [s]
|
||||
(set/rename-keys s {:db/id :entity/migration-key}))
|
||||
@@ -108,14 +105,13 @@
|
||||
|
||||
;; TEMP - this has been fixed in current export (ezcater-olaciotn)
|
||||
@(dc/transact connection [{:entity/migration-key 17592257603901 :vendor/name "unknown"}
|
||||
{:entity/migration-key 17592232621701}
|
||||
{:entity/migration-key 17592263907739}
|
||||
{:entity/migration-key 17592271516922}])
|
||||
|
||||
{:entity/migration-key 17592232621701}
|
||||
{:entity/migration-key 17592263907739}
|
||||
{:entity/migration-key 17592271516922}])
|
||||
|
||||
(doseq [entity (cond->> (order-of-insert entity-dependencies)
|
||||
true (filter #(not= "audit" %))
|
||||
starting-at (drop-while #(not= starting-at %)))
|
||||
starting-at (drop-while #(not= starting-at %)))
|
||||
:let [_ (reset! so-far 0)
|
||||
_ (mu/log ::querying :entity entity)
|
||||
entities (mu/trace ::file-pulled
|
||||
@@ -136,9 +132,8 @@
|
||||
(mu/log ::refresh-running-balance-cache-complete)
|
||||
(mu/log ::done))
|
||||
|
||||
|
||||
(defn -main [& _]
|
||||
(try
|
||||
(try
|
||||
(println "restore")
|
||||
(execute "restore-from-backup" #(restore-fresh-from-backup (:args env)))
|
||||
(catch Exception e
|
||||
@@ -151,13 +146,11 @@
|
||||
(throw e))))
|
||||
|
||||
;; cloud load
|
||||
#_(comment
|
||||
#_(comment
|
||||
|
||||
;; /datomic-backup/079df203-eae0-4acf-94d5-8608ba8b8a9a
|
||||
(load-from-backup "079df203-eae0-4acf-94d5-8608ba8b8a9a" auto-ap.datomic/conn ["charge"])
|
||||
(load-from-backup "079df203-eae0-4acf-94d5-8608ba8b8a9a" auto-ap.datomic/conn ["charge"])
|
||||
|
||||
(load-entity "charge" (ednl/slurp "/tmp/tmp-edn"))
|
||||
(load-entity "charge" (ednl/slurp "/tmp/tmp-edn")))
|
||||
|
||||
|
||||
)
|
||||
;; => nil
|
||||
|
||||
@@ -39,17 +39,14 @@
|
||||
(dc/db conn)
|
||||
number)))
|
||||
|
||||
|
||||
(defn delete-all []
|
||||
@(dc/transact-async conn
|
||||
(->>
|
||||
(dc/q '[:find ?ss
|
||||
:where [?ss :sales-summary/date]]
|
||||
(dc/db conn))
|
||||
(map (fn [[ ss]]
|
||||
[:db/retractEntity ss])))))
|
||||
|
||||
|
||||
(->>
|
||||
(dc/q '[:find ?ss
|
||||
:where [?ss :sales-summary/date]]
|
||||
(dc/db conn))
|
||||
(map (fn [[ss]]
|
||||
[:db/retractEntity ss])))))
|
||||
|
||||
(defn dirty-sales-summaries [c]
|
||||
(let [client-id (dc/entid (dc/db conn) c)]
|
||||
@@ -99,53 +96,53 @@
|
||||
"food app refunds" 41400})
|
||||
|
||||
(defn get-payment-items [c date]
|
||||
(->>
|
||||
(dc/q '[:find ?processor ?type-name (sum ?total)
|
||||
:with ?c
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-order/charges ?c]
|
||||
[?c :charge/type-name ?type-name]
|
||||
(or-join [?c ?processor]
|
||||
(and [?c :charge/processor ?p]
|
||||
[?p :db/ident ?processor])
|
||||
(and
|
||||
(not [?c :charge/processor])
|
||||
[(ground :ccp-processor/na) ?processor]))
|
||||
[?c :charge/total ?total]]
|
||||
(dc/db conn)
|
||||
[[c] date date])
|
||||
(reduce
|
||||
(fn [acc [processor type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Payments"
|
||||
(= type-name "CASH")
|
||||
"Cash Payments"
|
||||
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
|
||||
"Gift Card Payments"
|
||||
(#{:ccp-processor/toast
|
||||
#_:ccp-processor/ezcater
|
||||
#_:ccp-processor/koala
|
||||
:ccp-processor/doordash
|
||||
:ccp-processor/grubhub
|
||||
:ccp-processor/uber-eats} processor)
|
||||
"Food App Payments"
|
||||
:else
|
||||
"Unknown")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(map (fn [[k v]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 0
|
||||
:sales-summary-item/category k
|
||||
|
||||
:ledger-mapped/amount (if (= "Card Payments" k)
|
||||
(- v (get-fee c date))
|
||||
v)
|
||||
:ledger-mapped/ledger-side :ledger-side/debit}))))
|
||||
(->>
|
||||
(dc/q '[:find ?processor ?type-name (sum ?total)
|
||||
:with ?c
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
:where [(iol-ion.query/scan-sales-orders $ ?clients ?start-date ?end-date) [[?e _ ?sort-default] ...]]
|
||||
[?e :sales-order/charges ?c]
|
||||
[?c :charge/type-name ?type-name]
|
||||
(or-join [?c ?processor]
|
||||
(and [?c :charge/processor ?p]
|
||||
[?p :db/ident ?processor])
|
||||
(and
|
||||
(not [?c :charge/processor])
|
||||
[(ground :ccp-processor/na) ?processor]))
|
||||
[?c :charge/total ?total]]
|
||||
(dc/db conn)
|
||||
[[c] date date])
|
||||
(reduce
|
||||
(fn [acc [processor type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Payments"
|
||||
(= type-name "CASH")
|
||||
"Cash Payments"
|
||||
(#{"SQUARE_GIFT_CARD" "WALLET" "GIFT_CARD"} type-name)
|
||||
"Gift Card Payments"
|
||||
(#{:ccp-processor/toast
|
||||
#_:ccp-processor/ezcater
|
||||
#_:ccp-processor/koala
|
||||
:ccp-processor/doordash
|
||||
:ccp-processor/grubhub
|
||||
:ccp-processor/uber-eats} processor)
|
||||
"Food App Payments"
|
||||
:else
|
||||
"Unknown")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(map (fn [[k v]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 0
|
||||
:sales-summary-item/category k
|
||||
|
||||
:ledger-mapped/amount (if (= "Card Payments" k)
|
||||
(- v (get-fee c date))
|
||||
v)
|
||||
:ledger-mapped/ledger-side :ledger-side/debit}))))
|
||||
|
||||
(defn get-discounts [c date]
|
||||
(when-let [discount (ffirst (dc/q '[:find (sum ?discount)
|
||||
@@ -162,7 +159,7 @@
|
||||
:ledger-mapped/ledger-side :ledger-side/debit}))
|
||||
|
||||
(defn get-refund-items [c date]
|
||||
(->>
|
||||
(->>
|
||||
(dc/q '[:find ?type-name (sum ?t)
|
||||
:with ?e
|
||||
:in $ [?clients ?start-date ?end-date]
|
||||
@@ -173,26 +170,24 @@
|
||||
(dc/db conn)
|
||||
[[c] date date])
|
||||
(reduce
|
||||
(fn [acc [type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Refunds"
|
||||
(= type-name "CASH")
|
||||
"Cash Refunds"
|
||||
:else
|
||||
"Food App Refunds")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(fn [acc [type-name total]]
|
||||
(update
|
||||
acc
|
||||
(cond (= type-name "CARD")
|
||||
"Card Refunds"
|
||||
(= type-name "CASH")
|
||||
"Cash Refunds"
|
||||
:else
|
||||
"Food App Refunds")
|
||||
(fnil + 0.0)
|
||||
total))
|
||||
{})
|
||||
(map (fn [[k v]]
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 3
|
||||
:sales-summary-item/category k
|
||||
:ledger-mapped/amount v
|
||||
:ledger-mapped/ledger-side :ledger-side/credit}))))
|
||||
|
||||
|
||||
{:db/id (str (java.util.UUID/randomUUID))
|
||||
:sales-summary-item/sort-order 3
|
||||
:sales-summary-item/category k
|
||||
:ledger-mapped/amount v
|
||||
:ledger-mapped/ledger-side :ledger-side/credit}))))
|
||||
|
||||
(defn get-fees [c date]
|
||||
(when-let [fee (get-fee c date)]
|
||||
@@ -278,17 +273,17 @@
|
||||
|
||||
(defn sales-summaries-v2 []
|
||||
(doseq [[c client-code] (dc/q '[:find ?c ?client-code
|
||||
:in $
|
||||
:where [?c :client/code ?client-code]]
|
||||
(dc/db conn))
|
||||
{:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)]
|
||||
:in $
|
||||
:where [?c :client/code ?client-code]]
|
||||
(dc/db conn))
|
||||
{:sales-summary/keys [date] :db/keys [id] :as existing-summary} (dirty-sales-summaries c)]
|
||||
(mu/with-context {:client-code client-code
|
||||
:date date}
|
||||
(alog/info ::updating)
|
||||
(let [manual-items (->> existing-summary
|
||||
:sales-summary/items
|
||||
(filter :sales-summary-item/manual?))
|
||||
calculated-items (->>
|
||||
:date date}
|
||||
(alog/info ::updating)
|
||||
(let [manual-items (->> existing-summary
|
||||
:sales-summary/items
|
||||
(filter :sales-summary-item/manual?))
|
||||
calculated-items (->>
|
||||
(get-sales c date)
|
||||
(concat (get-payment-items c date))
|
||||
(concat (get-refund-items c date))
|
||||
@@ -301,20 +296,19 @@
|
||||
(map (fn [z]
|
||||
(assoc z :ledger-mapped/account (some-> z :sales-summary-item/category str/lower-case name->number lookup-account)
|
||||
:sales-summary-item/manual? false))))
|
||||
all-items (concat calculated-items manual-items)
|
||||
result {:db/id id
|
||||
:sales-summary/client c
|
||||
:sales-summary/date date
|
||||
:sales-summary/dirty false
|
||||
:sales-summary/client+date [c date]
|
||||
:sales-summary/items all-items}]
|
||||
(if (seq (:sales-summary/items result))
|
||||
(do
|
||||
(alog/info ::upserting-summaries
|
||||
:category-count (count (:sales-summary/items result)))
|
||||
@(dc/transact conn [[:upsert-sales-summary result]]))
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
|
||||
all-items (concat calculated-items manual-items)
|
||||
result {:db/id id
|
||||
:sales-summary/client c
|
||||
:sales-summary/date date
|
||||
:sales-summary/dirty false
|
||||
:sales-summary/client+date [c date]
|
||||
:sales-summary/items all-items}]
|
||||
(if (seq (:sales-summary/items result))
|
||||
(do
|
||||
(alog/info ::upserting-summaries
|
||||
:category-count (count (:sales-summary/items result)))
|
||||
@(dc/transact conn [[:upsert-sales-summary result]]))
|
||||
@(dc/transact conn [{:db/id id :sales-summary/dirty false}]))))))
|
||||
|
||||
(defn reset-summaries []
|
||||
@(dc/transact conn (->> (dc/q '[:find ?sos
|
||||
@@ -324,9 +318,6 @@
|
||||
(map (fn [[sos]]
|
||||
[:db/retractEntity sos])))))
|
||||
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(auto-ap.datomic/transact-schema conn)
|
||||
|
||||
@@ -336,26 +327,19 @@
|
||||
|
||||
(dirty-sales-summaries [:client/code "NGWH"])
|
||||
|
||||
|
||||
(apply mark-dirty [:client/code "NGWH"] (last-n-days 5))
|
||||
|
||||
(iol-ion.tx.upsert-sales-summary-ledger/summary->journal-entry (dc/db conn) 17592314245819)
|
||||
|
||||
|
||||
(iol-ion.tx.upsert-sales-summary-ledger/upsert-sales-summary (dc/db conn) {:db/id 17592314241429})
|
||||
|
||||
|
||||
|
||||
(mark-all-dirty 5)
|
||||
(delete-all)
|
||||
|
||||
|
||||
(sales-summaries-v2)
|
||||
|
||||
|
||||
1
|
||||
|
||||
|
||||
(dc/q '[:find (pull ?sos [* {:sales-summary/sales-items [*]}])
|
||||
:in $
|
||||
:where [?sos :sales-summary/client [:client/code "NGHW"]]
|
||||
@@ -386,15 +370,7 @@
|
||||
@(dc/transact conn [{:db/id :sales-summary/total-tax :db/ident :sales-summary/total-tax-legacy}
|
||||
{:db/id :sales-summary/total-tip :db/ident :sales-summary/total-tip-legacy}])
|
||||
|
||||
(auto-ap.datomic/transact-schema conn)
|
||||
|
||||
|
||||
|
||||
|
||||
)
|
||||
|
||||
|
||||
(auto-ap.datomic/transact-schema conn))
|
||||
|
||||
(defn -main [& _]
|
||||
(execute "sales-summaries" sales-summaries-v2))
|
||||
|
||||
@@ -44,7 +44,6 @@
|
||||
(dc/db conn)
|
||||
50000))))
|
||||
|
||||
|
||||
(def ^:dynamic bucket-name (:data-bucket env))
|
||||
|
||||
(def header-keys ["TransCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "AccountName" "AccountDunsNo" "InvoiceDate" "AccountDate" "CustomerPONo" "PaymentTerms" "TermsDescription" "StoreNumber" "CustomerName" "AddressLine1" "AddressLine2" "City1" "State1" "Zip1" "Phone1" "Duns1" "Hin1" "Dea1" "TIDCustomer" "ChainNumber" "BidNumber" "ContractNumber" "CompanyNumber" "BriefName" "Address" "Address2" "City2" "State2" "Zip2" "Phone2" "Duns2" "Hin2" "Dea2" "Tid_OPCO" "ObligationIndicator" "Manifest" "Route" "Stop" "TermsDiscountPercent" "TermsDiscountDueDate" "TermsNetDueDate" "TermsDiscountAmount" "TermsDiscountCode" "OrderDate" "DepartmentCode"])
|
||||
@@ -56,14 +55,13 @@
|
||||
(defn get-sysco-vendor []
|
||||
(let [db (dc/db conn)]
|
||||
(->
|
||||
(dc/q '[:find (pull ?v r)
|
||||
:in $ r
|
||||
:where [?v :vendor/name "Sysco"]]
|
||||
db
|
||||
d-vendors/default-read)
|
||||
first
|
||||
first)))
|
||||
|
||||
(dc/q '[:find (pull ?v r)
|
||||
:in $ r
|
||||
:where [?v :vendor/name "Sysco"]]
|
||||
db
|
||||
d-vendors/default-read)
|
||||
first
|
||||
first)))
|
||||
|
||||
(defn read-sysco-csv [k]
|
||||
(-> (s3/get-object {:bucket-name bucket-name
|
||||
@@ -73,34 +71,32 @@
|
||||
csv/read-csv))
|
||||
|
||||
(defn check-okay-amount? [i]
|
||||
(dollars=
|
||||
(dollars=
|
||||
(:invoice/total i)
|
||||
(reduce + 0.0 (map :invoice-expense-account/amount (:invoice/expense-accounts i)))))
|
||||
|
||||
(defn code-individual-items [invoice csv-rows tax]
|
||||
(let [items (->> csv-rows
|
||||
butlast
|
||||
(reduce
|
||||
(fn [acc row]
|
||||
(update acc (get-line-account (nth row item-name-index))
|
||||
(fnil + 0.0)
|
||||
(Double/parseDouble (nth row item-price-index))
|
||||
)
|
||||
)
|
||||
{})
|
||||
)
|
||||
(reduce
|
||||
(fn [acc row]
|
||||
(update acc (get-line-account (nth row item-name-index))
|
||||
(fnil + 0.0)
|
||||
(Double/parseDouble (nth row item-price-index))))
|
||||
|
||||
{}))
|
||||
items-with-tax (update items (get-line-account "TAX")
|
||||
(fnil + 0.0)
|
||||
(fnil + 0.0)
|
||||
tax)
|
||||
updated-invoice (assoc invoice :invoice/expense-accounts
|
||||
(for [[account amount] items-with-tax]
|
||||
#:invoice-expense-account {:db/id (random-tempid)
|
||||
:account account
|
||||
:location (:invoice/location invoice)
|
||||
:amount amount}))]
|
||||
updated-invoice (assoc invoice :invoice/expense-accounts
|
||||
(for [[account amount] items-with-tax]
|
||||
#:invoice-expense-account {:db/id (random-tempid)
|
||||
:account account
|
||||
:location (:invoice/location invoice)
|
||||
:amount amount}))]
|
||||
(if (check-okay-amount? updated-invoice)
|
||||
updated-invoice
|
||||
(do (alog/warn ::itemized-expenses-not-adding-up
|
||||
(do (alog/warn ::itemized-expenses-not-adding-up
|
||||
:invoice updated-invoice)
|
||||
invoice))))
|
||||
|
||||
@@ -122,11 +118,11 @@
|
||||
(header-row "AddressLine2")
|
||||
(header-row "City1")
|
||||
(header-row "City2")])
|
||||
|
||||
|
||||
account-number (some-> account-number Long/parseLong str)
|
||||
matching-client (and account-number
|
||||
(d-clients/exact-match account-number))
|
||||
|
||||
|
||||
_ (when-not matching-client
|
||||
(throw (ex-info "cannot find matching client"
|
||||
{:account-number account-number
|
||||
@@ -153,9 +149,9 @@
|
||||
:client/locations]
|
||||
(:db/id matching-client))
|
||||
location-hint
|
||||
location-hint )
|
||||
location-hint)
|
||||
:date (coerce/to-date date)
|
||||
:vendor (:db/id sysco-vendor )
|
||||
:vendor (:db/id sysco-vendor)
|
||||
:client (:db/id matching-client)
|
||||
:import-status :import-status/imported
|
||||
:status :invoice-status/unpaid
|
||||
@@ -180,64 +176,54 @@
|
||||
(s3/delete-object {:bucket-name bucket-name
|
||||
:key k}))
|
||||
|
||||
(defn get-test-invoice-file
|
||||
(defn get-test-invoice-file
|
||||
([] (get-test-invoice-file 999))
|
||||
( [i]
|
||||
([i]
|
||||
(nth (->> (s3/list-objects-v2 {:bucket-name "data.prod.app.integreatconsult.com"
|
||||
:prefix "sysco/imported"})
|
||||
:object-summaries
|
||||
(map :key)
|
||||
)
|
||||
(map :key))
|
||||
i)))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
(with-bindings { #'bucket-name "data.prod.app.integreatconsult.com"}
|
||||
(doall
|
||||
(for [n (range 930 940 )
|
||||
:let [result (-> (get-test-invoice-file n)
|
||||
read-sysco-csv
|
||||
(extract-invoice-details (get-sysco-vendor))
|
||||
)]
|
||||
#_#_:when (not (check-okay-amount? result))]
|
||||
(comment
|
||||
(with-bindings {#'bucket-name "data.prod.app.integreatconsult.com"}
|
||||
(doall
|
||||
(for [n (range 930 940)
|
||||
:let [result (-> (get-test-invoice-file n)
|
||||
read-sysco-csv
|
||||
(extract-invoice-details (get-sysco-vendor)))]
|
||||
#_#_:when (not (check-okay-amount? result))]
|
||||
|
||||
result)))
|
||||
|
||||
(with-bindings { #'bucket-name "data.prod.app.integreatconsult.com"}
|
||||
(let [result (-> "sysco/error/SYSCO050_00175962_20241010122639019.csv"
|
||||
(with-bindings {#'bucket-name "data.prod.app.integreatconsult.com"}
|
||||
(let [result (-> "sysco/error/SYSCO050_00175962_20241010122639019.csv"
|
||||
read-sysco-csv
|
||||
(extract-invoice-details (get-sysco-vendor))
|
||||
)]
|
||||
(extract-invoice-details (get-sysco-vendor)))]
|
||||
|
||||
result))
|
||||
|
||||
)
|
||||
result)))
|
||||
|
||||
(defn import-sysco []
|
||||
(let [sysco-vendor (get-sysco-vendor)
|
||||
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
:prefix "sysco/pending"})
|
||||
:object-summaries
|
||||
(map :key))]
|
||||
|
||||
:object-summaries
|
||||
(map :key))]
|
||||
|
||||
(alog/info ::importing-sysco
|
||||
:count (count keys)
|
||||
:keys (pr-str keys))
|
||||
|
||||
|
||||
(let [transaction (->> keys
|
||||
(mapcat (fn [k]
|
||||
(try
|
||||
(try
|
||||
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
|
||||
invoice-url (str "https://" (:data-bucket env) "/" invoice-key)]
|
||||
(s3/copy-object {:source-bucket-name (:data-bucket env)
|
||||
:destination-bucket-name (:data-bucket env)
|
||||
:source-key k
|
||||
:destination-key invoice-key})
|
||||
[[:propose-invoice
|
||||
[[:propose-invoice
|
||||
(-> k
|
||||
read-sysco-csv
|
||||
(extract-invoice-details sysco-vendor)
|
||||
@@ -246,7 +232,7 @@
|
||||
(alog/error ::cant-load-file
|
||||
:file k
|
||||
:error e e)
|
||||
(s3/copy-object {:source-bucket-name (:data-bucket env)
|
||||
(s3/copy-object {:source-bucket-name (:data-bucket env)
|
||||
:destination-bucket-name (:data-bucket env)
|
||||
:source-key k
|
||||
:destination-key (str "sysco/error/"
|
||||
@@ -256,6 +242,5 @@
|
||||
(doseq [k keys]
|
||||
(mark-key k))))
|
||||
|
||||
|
||||
(defn -main [& _]
|
||||
(execute "sysco" import-sysco))
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
:where
|
||||
[?v :vendor/name]
|
||||
(or-join [?v ?c ?e]
|
||||
(and
|
||||
(and
|
||||
[?e :invoice/vendor ?v]
|
||||
[?e :invoice/client ?c])
|
||||
(and
|
||||
|
||||
Reference in New Issue
Block a user