merged.
This commit is contained in:
528
src/clj/auto_ap/backup.clj
Normal file
528
src/clj/auto_ap/backup.clj
Normal file
@@ -0,0 +1,528 @@
|
||||
;; This buffer is for Clojure experiments and evaluation.
|
||||
|
||||
;; Press C-j to evaluate the last expression.
|
||||
|
||||
;; You can also press C-u C-j to evaluate the expression and pretty-print its result.
|
||||
|
||||
(ns auto-ap.backup
|
||||
(:require [datomic.api :as d]
|
||||
[manifold.deferred :as de]
|
||||
[manifold.executor :as ex]
|
||||
[manifold.stream :as s]
|
||||
[manifold.time :as mt]
|
||||
[auto-ap.jobs.core :refer [execute]]
|
||||
[clojure.java.io :as io]
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[config.core :refer [env]]
|
||||
[clojure.core.async :as a]
|
||||
[lambdaisland.edn-lines :as ednl]
|
||||
[clojure.set :as set]
|
||||
[com.brunobonacci.mulog :as mu]))
|
||||
|
||||
|
||||
|
||||
|
||||
(def request-pool (ex/fixed-thread-executor 30))
|
||||
|
||||
(def buffered (ex/fixed-thread-executor 30))
|
||||
|
||||
(defn get-schema [remote-db]
|
||||
(let [everything (->> (d/q '[:find [(pull ?e [:db/ident
|
||||
{:db/valueType [:db/ident]}
|
||||
{:db/cardinality [:db/ident]}
|
||||
:db.attr/preds
|
||||
{:db/unique [:db/ident]}
|
||||
:db/isComponent
|
||||
:db/id
|
||||
:db/noHistory
|
||||
:db/tupleAttrs
|
||||
:db.entity/attrs
|
||||
:db.entity/preds
|
||||
:db/doc]) ...]
|
||||
:where [?e :db/ident]]
|
||||
remote-db))
|
||||
schema-attrs (->> everything
|
||||
(filter :db/ident)
|
||||
(filter (fn [{:db/keys [ident]}]
|
||||
(if (namespace ident)
|
||||
(re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident))
|
||||
true
|
||||
))))
|
||||
meta-schema-schema (filter #(-> % :db/ident not) everything)]
|
||||
schema-attrs))
|
||||
|
||||
|
||||
(def entity->best-key
|
||||
{"transaction-rule"
|
||||
[:transaction-rule/description, :transaction-rule/note :transaction-rule/vendor]
|
||||
"square-location"
|
||||
:square-location/square-id,
|
||||
"expected-deposit"
|
||||
:expected-deposit/date,
|
||||
"journal-entry-line"
|
||||
[:journal-entry-line/account, :journal-entry-line/debit :journal-entry-line/credit]
|
||||
"vendor"
|
||||
[:vendor/name, :vendor/default-account, :vendor/hidden]
|
||||
"transaction"
|
||||
:transaction/amount,
|
||||
"yodlee-provider-account"
|
||||
:yodlee-provider-account/id,
|
||||
"journal-entry"
|
||||
:journal-entry/source,
|
||||
"yodlee-merchant" :yodlee-merchant/yodlee-id,
|
||||
"invoice"
|
||||
:invoice/invoice-number,
|
||||
"vendor-terms-override"
|
||||
:vendor-terms-override/client,
|
||||
"integration-status"
|
||||
:integration-status/state,
|
||||
"conformity" :conformity/conformed-norms-index,
|
||||
"user"
|
||||
:user/provider-id,
|
||||
"sales-refund"
|
||||
:sales-refund/total,
|
||||
"plaid-account"
|
||||
:plaid-account/name,
|
||||
"charge"
|
||||
[:charge/total, :charge/external-id]
|
||||
"location-match" :location-match/location,
|
||||
"vendor-schedule-payment-dom"
|
||||
:vendor-schedule-payment-dom/dom,
|
||||
"account-client-override"
|
||||
:account-client-override/client,
|
||||
"plaid-item"
|
||||
:plaid-item/client,
|
||||
"transaction-account"
|
||||
:transaction-account/account,
|
||||
"address"
|
||||
[:address/street1, :address/city :address/state :address/zip]
|
||||
"order-line-item"
|
||||
:order-line-item/total,
|
||||
"ezcater-location" [:ezcater-location/location, :ezcater-location/caterer]
|
||||
"account"
|
||||
[:account/numeric-code, :account/code :account/name :account/type]
|
||||
"intuit-bank-account"
|
||||
:intuit-bank-account/name,
|
||||
"saved-query"
|
||||
:saved-query/guid,
|
||||
"ezcater-caterer"
|
||||
:ezcater-caterer/uuid,
|
||||
"forecasted-transaction"
|
||||
:forecasted-transaction/day-of-month,
|
||||
"audit" :audit/user,
|
||||
"yodlee-account"
|
||||
:yodlee-account/id,
|
||||
"transaction-rule-account"
|
||||
[:transaction-rule-account/account, :transaction-rule-account/location]
|
||||
"ezcater-integration"
|
||||
:ezcater-integration/subscriber-uuid,
|
||||
"report"
|
||||
:report/created,
|
||||
"bank-account"
|
||||
:bank-account/code,
|
||||
"vendor-usage"
|
||||
:vendor-usage/key,
|
||||
"invoice-expense-account"
|
||||
[:invoice-expense-account/expense-account-id, :invoice-expense-account/account :invoice-expense-account/location :invoice-expense-account/amount]
|
||||
"sales-order"
|
||||
:sales-order/date,
|
||||
"client"
|
||||
:client/code,
|
||||
"email-contact" :email-contact/email,
|
||||
"invoice-payment"
|
||||
:invoice-payment/amount,
|
||||
"contact"
|
||||
[:contact/name, :contact/phone :contact/email]
|
||||
"import-batch"
|
||||
:import-batch/date,
|
||||
"payment"
|
||||
[:payment/date, :payment/bank-account]
|
||||
"vendor-account-override"
|
||||
:vendor-account-override/client})
|
||||
|
||||
#_(defn references [schema]
|
||||
(filter (comp #{:db.type/ref} :db/ident :db/valueType) schema ))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#_(defn reference->entity [remote-db]
|
||||
(->> (d/q '[:find ?a ?v3
|
||||
:in $ $$ [?a ...]
|
||||
:where [$$ _ ?a ?e]
|
||||
[$ ?e ?v _ _]
|
||||
[$ ?v :db/ident ?v2 _ _]
|
||||
[(namespace ?v2) ?v3]
|
||||
[(namespace ?v2) ?v3]]
|
||||
remote-db
|
||||
(d/since remote-db #inst "2022-06-01")
|
||||
(map :db/ident references)
|
||||
)
|
||||
(group-by first)
|
||||
(map (fn [[k v]]
|
||||
[k (disj (set (map second v)) "db")]))
|
||||
(into {})))
|
||||
|
||||
#_(def manual-dependencies
|
||||
{:client/location-matches #{"location-match"}
|
||||
:transaction/yodlee-merchant #{"yodlee-merchant"}
|
||||
:vendor-account-override/account #{"account"}
|
||||
:vendor-account-override/client #{"client"}
|
||||
:vendor/secondary-contact #{"contact"}
|
||||
:vendor/account-overrides #{"vendor-account-override"}
|
||||
:client/bank-accounts #{"bank-account"}
|
||||
:transaction-rule/yodlee-merchant #{"yodlee-merchant"}
|
||||
:client/forecasted-transactions #{"forecasted-transaction"}
|
||||
:transaction/forecast-match #{"forecasted-transaction"}
|
||||
:vendor/automatically-paid-when-due #{"client"}
|
||||
:vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"}
|
||||
:vendor/terms-overrides #{"vendor-terms-override"}
|
||||
:vendor-schedule-payment-dom/client #{"client"}})
|
||||
|
||||
#_(defn full-dependencies [remote-db]
|
||||
(update (merge-with into (reference->entity remote-db) manual-dependencies)
|
||||
:journal-entry/original-entity
|
||||
#(disj % "journal-entry")))
|
||||
|
||||
#_(defn entity-dependencies [schema]
|
||||
(let [base-dependencies
|
||||
(into
|
||||
{}
|
||||
(map (fn [i]
|
||||
[i #{}])
|
||||
(set (map (comp namespace :db/ident)
|
||||
(filter :db/valueType
|
||||
schema))))
|
||||
)
|
||||
]
|
||||
(into base-dependencies (reduce
|
||||
(fn [acc [ref deps]]
|
||||
(update acc (namespace ref) (fnil #(into % deps) #{})))
|
||||
{}
|
||||
(full-dependencies remote-db)))))
|
||||
|
||||
(def full-dependencies
|
||||
{:invoice/client #{"client"},
|
||||
:sales-order/client #{"client"},
|
||||
:transaction-rule/transaction-approval-status #{},
|
||||
:transaction/forecast-match #{"forecasted-transaction"},
|
||||
:user/role #{},
|
||||
:vendor-schedule-payment-dom/client #{"client"},
|
||||
:invoice-payment/payment #{"payment"},
|
||||
:transaction-rule/client #{"client"},
|
||||
:invoice/status #{},
|
||||
:payment/type #{},
|
||||
:expected-deposit/client #{"client"},
|
||||
:transaction/bank-account #{"bank-account"},
|
||||
:transaction-rule-account/account #{"account"},
|
||||
:import-batch/status #{},
|
||||
:user/clients #{"client"},
|
||||
:payment/client #{"client"},
|
||||
:expected-deposit/charges #{"charge"},
|
||||
:vendor/automatically-paid-when-due #{"client"},
|
||||
:payment/invoices #{"invoice"},
|
||||
:client/forecasted-transactions #{"forecasted-transaction"},
|
||||
:transaction/matched-rule #{"transaction-rule"},
|
||||
:invoice/import-status #{},
|
||||
:charge/processor #{},
|
||||
:expected-deposit/vendor #{"vendor"},
|
||||
:client/square-locations #{"square-location"},
|
||||
:payment/status #{},
|
||||
:client/location-matches #{"location-match"},
|
||||
:saved-query/client #{"client"},
|
||||
:transaction/payment #{"payment"},
|
||||
:transaction-rule/vendor #{"vendor"},
|
||||
:plaid-item/client #{"client"},
|
||||
:account/applicability #{},
|
||||
:journal-entry-line/account #{"account" "bank-account"},
|
||||
:client/bank-accounts #{"bank-account"},
|
||||
:yodlee-provider-account/client #{"client"},
|
||||
:account/vendor-allowance #{},
|
||||
:payment/bank-account #{"bank-account"},
|
||||
:account/default-allowance #{},
|
||||
:transaction-rule/yodlee-merchant #{"yodlee-merchant"},
|
||||
:vendor/account-overrides #{"vendor-account-override"},
|
||||
:transaction/client #{"client"},
|
||||
:invoice/vendor #{"vendor"},
|
||||
:sales-order/vendor #{"vendor"},
|
||||
:expected-deposit/status #{},
|
||||
:journal-entry/original-entity #{"transaction" "invoice"},
|
||||
:vendor-usage/client #{"client"},
|
||||
:transaction/expected-deposit #{"expected-deposit"},
|
||||
:client/ezcater-locations #{"ezcater-location"},
|
||||
:journal-entry/client #{"client"},
|
||||
:vendor/secondary-contact #{"contact"},
|
||||
:journal-entry/line-items #{"journal-entry-line"},
|
||||
:vendor/legal-entity-1099-type #{},
|
||||
:transaction-rule/bank-account #{"bank-account"},
|
||||
:transaction-account/account #{"account"},
|
||||
:vendor/terms-overrides #{"vendor-terms-override"},
|
||||
:vendor/default-account #{"account"},
|
||||
:transaction/yodlee-merchant #{"yodlee-merchant"},
|
||||
:sales-refund/client #{"client"},
|
||||
:client/emails #{"email-contact"},
|
||||
:payment/vendor #{"vendor"},
|
||||
:invoice-payment/invoice #{"invoice"},
|
||||
:report/client #{"client"},
|
||||
:transaction-rule/accounts #{"transaction-rule-account"},
|
||||
:charge/client #{"client"},
|
||||
:bank-account/type #{},
|
||||
:invoice-expense-account/account #{"account"},
|
||||
:vendor/legal-entity-tin-type #{},
|
||||
:transaction/approval-status #{},
|
||||
:import-batch/entry #{"transaction"},
|
||||
:bank-account/intuit-bank-account #{"intuit-bank-account"},
|
||||
:account/type #{},
|
||||
:sales-refund/vendor #{"vendor"},
|
||||
:bank-account/yodlee-account #{"yodlee-account"},
|
||||
:vendor/address #{"address"},
|
||||
:integration-status/state #{},
|
||||
:transaction/accounts #{"transaction-account"},
|
||||
:sales-order/charges #{"charge"},
|
||||
:client/address #{"address"},
|
||||
:ezcater-location/caterer #{"ezcater-caterer"},
|
||||
:vendor-account-override/client #{"client"},
|
||||
:bank-account/integration-status #{"integration-status"},
|
||||
:yodlee-provider-account/accounts #{"yodlee-account"},
|
||||
:account/invoice-allowance #{},
|
||||
:journal-entry/vendor #{"vendor"},
|
||||
:plaid-item/accounts #{"plaid-account"},
|
||||
:vendor-usage/vendor #{"vendor"},
|
||||
:sales-order/line-items #{"order-line-item"},
|
||||
:invoice/expense-accounts #{"invoice-expense-account"},
|
||||
:account-client-override/client #{"client"},
|
||||
:vendor/primary-contact #{"contact"},
|
||||
:vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"},
|
||||
:account/client-overrides #{"account-client-override"},
|
||||
:transaction/vendor #{"vendor"},
|
||||
:client/square-integration-status #{"integration-status"},
|
||||
:ezcater-integration/caterers #{"ezcater-caterer"},
|
||||
:ezcater-integration/integration-status #{"integration-status"}
|
||||
:vendor-account-override/account #{"account"},
|
||||
:import-batch/source #{}})
|
||||
|
||||
(def entity-dependencies
|
||||
{"transaction-rule"
|
||||
#{"vendor" "yodlee-merchant" "transaction-rule-account" "bank-account"
|
||||
"client"},
|
||||
"square-location" #{},
|
||||
"expected-deposit" #{"vendor" "charge" "client"},
|
||||
"journal-entry-line" #{"account" "bank-account"},
|
||||
"vendor"
|
||||
#{"vendor-schedule-payment-dom" "address" "account" "client" "contact"
|
||||
"vendor-account-override"},
|
||||
"transaction"
|
||||
#{"transaction-rule" "expected-deposit" "vendor" "yodlee-merchant"
|
||||
"transaction-account" "forecasted-transaction" "bank-account" "client"
|
||||
"payment"},
|
||||
"yodlee-provider-account" #{"yodlee-account" "client"},
|
||||
"journal-entry"
|
||||
#{"journal-entry-line" "vendor" "transaction" "invoice" "client"},
|
||||
"yodlee-merchant" #{},
|
||||
"invoice" #{"vendor" "invoice-expense-account" "client"},
|
||||
"vendor-terms-override" #{},
|
||||
"integration-status" #{},
|
||||
"conformity" #{},
|
||||
"user" #{"client"},
|
||||
"sales-refund" #{"vendor" "client"},
|
||||
"plaid-account" #{},
|
||||
"charge" #{"client"},
|
||||
"location-match" #{},
|
||||
"vendor-schedule-payment-dom" #{"client"},
|
||||
"account-client-override" #{"client"},
|
||||
"plaid-item" #{"plaid-account" "client"},
|
||||
"transaction-account" #{"account"},
|
||||
"address" #{},
|
||||
"order-line-item" #{},
|
||||
"ezcater-location" #{"ezcater-caterer"},
|
||||
"account" #{"account-client-override"},
|
||||
"intuit-bank-account" #{},
|
||||
"saved-query" #{"client"},
|
||||
"ezcater-caterer" #{},
|
||||
"forecasted-transaction" #{},
|
||||
"audit" #{},
|
||||
"yodlee-account" #{},
|
||||
"transaction-rule-account" #{"account"},
|
||||
"ezcater-integration" #{"ezcater-caterer" "integration-status"},
|
||||
"report" #{"client"},
|
||||
"bank-account" #{"integration-status" "intuit-bank-account" "yodlee-account"},
|
||||
"vendor-usage" #{"vendor" "client"},
|
||||
"invoice-expense-account" #{"account"},
|
||||
"sales-order" #{"vendor" "charge" "order-line-item" "client"},
|
||||
"client"
|
||||
#{"square-location" "integration-status" "location-match" "address"
|
||||
"ezcater-location" "forecasted-transaction" "bank-account" "email-contact"},
|
||||
"email-contact" #{},
|
||||
"invoice-payment" #{"invoice" "payment"},
|
||||
"contact" #{},
|
||||
"import-batch" #{"transaction"},
|
||||
"payment" #{"vendor" "invoice" "bank-account" "client"},
|
||||
"vendor-account-override" #{"account" "client"}})
|
||||
|
||||
(defn order-of-insert [entity-dependencies]
|
||||
(loop [entity-dependencies entity-dependencies
|
||||
order []]
|
||||
(let [next-order (for [[entity deps] entity-dependencies
|
||||
:when (not (seq deps))]
|
||||
entity)
|
||||
next-deps (reduce
|
||||
(fn [entity-dependencies next-entity]
|
||||
(into {}
|
||||
(map
|
||||
(fn [[k v]]
|
||||
[k (disj v next-entity)])
|
||||
entity-dependencies)))
|
||||
(apply dissoc entity-dependencies next-order)
|
||||
next-order)]
|
||||
(if (seq next-deps)
|
||||
(recur next-deps (into order next-order))
|
||||
(into order next-order)))))
|
||||
|
||||
|
||||
(def loaded (atom #{}))
|
||||
|
||||
|
||||
(def dumped (atom #{}))
|
||||
|
||||
(defn write-s3 [data location]
|
||||
(spit (io/file "/tmp/temp-edn")
|
||||
(with-out-str (clojure.pprint/pprint data)))
|
||||
(s3/put-object :bucket-name (:data-bucket env)
|
||||
:key location
|
||||
:input-stream (io/make-input-stream (io/file "/tmp/temp-edn") {})))
|
||||
|
||||
(defn dump-schema [schema backup]
|
||||
(write-s3 (map
|
||||
(fn [s]
|
||||
(set/rename-keys s {:db/id :entity/migration-key}))
|
||||
schema)
|
||||
(str backup "/schema.edn"))
|
||||
(write-s3 full-dependencies
|
||||
(str backup "/full-dependencies.edn"))
|
||||
(write-s3 entity-dependencies
|
||||
(str backup "/entity-dependencies.edn")))
|
||||
|
||||
(defn pull-batch [remote-db schema entity entities]
|
||||
(de/future-with request-pool
|
||||
(mu/with-context {:entity entity}
|
||||
(try
|
||||
(when (= 0 (rand-int 100))
|
||||
(mu/log ::pulling :count (count entities)))
|
||||
(->> (d/pull-many remote-db
|
||||
(->> schema
|
||||
(filter :db/valueType)
|
||||
(mapv :db/ident)
|
||||
(filter #(= entity (namespace %)))
|
||||
(into [:db/id]))
|
||||
entities)
|
||||
(mapv (fn [m ]
|
||||
(reduce
|
||||
(fn [m [k v]]
|
||||
(cond
|
||||
(= k :db/id)
|
||||
(-> m
|
||||
(assoc :entity/migration-key v)
|
||||
(dissoc :db/id))
|
||||
(full-dependencies k)
|
||||
(if (vector? v)
|
||||
(assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v))
|
||||
(assoc m k [:entity/migration-key (:db/id v)]))
|
||||
:else
|
||||
(dissoc m :payment/pdf-data
|
||||
:payment/memo
|
||||
:vendor/invoice-reminder-schedule)))
|
||||
m
|
||||
m))))
|
||||
(catch Throwable e
|
||||
(mu/log ::pull-error
|
||||
:exception e)
|
||||
|
||||
(throw e))))))
|
||||
|
||||
(def in-flight (atom 0))
|
||||
|
||||
(def so-far (atom 0))
|
||||
(def total (atom 0))
|
||||
|
||||
(defn dump-all
|
||||
([] (dump-all nil))
|
||||
([item-list]
|
||||
(let [backup-id (str "/datomic-backup/" (java.util.UUID/randomUUID))
|
||||
_ (mu/log ::starting-backup :backup backup-id)
|
||||
remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod"))
|
||||
_ (mu/log ::fetching-schema)
|
||||
schema (get-schema remote-db)
|
||||
]
|
||||
(mu/log ::dumping-schema)
|
||||
(dump-schema schema backup-id)
|
||||
(mu/log ::schema-dumped)
|
||||
(doseq [entity (or item-list (filter (complement (conj @loaded "audit")) (order-of-insert entity-dependencies)))
|
||||
:let [_ (swap! dumped conj entity)
|
||||
_ (reset! so-far 0)
|
||||
_ (mu/log ::querying :entity entity)
|
||||
entities (d/q '[:find [?e ...]
|
||||
:in $ [?a ...]
|
||||
:where [?e ?a]]
|
||||
remote-db
|
||||
(cond-> (entity->best-key entity)
|
||||
(not (vector? (entity->best-key entity))) vector))
|
||||
|
||||
_ (reset! total (count entities))
|
||||
_ (mu/log ::entity-total-found :count (count entities) :entity entity)]]
|
||||
(mu/trace ::single-entity
|
||||
[:entity entity]
|
||||
(mu/with-context {:entity entity :total @total}
|
||||
(mu/log ::starting)
|
||||
(mu/log ::deleting)
|
||||
(io/delete-file (io/file "/tmp/tmp-ednl") true)
|
||||
(mu/log ::pulling)
|
||||
(ednl/with-append [append "/tmp/tmp-ednl" ]
|
||||
@(s/consume (fn [batch]
|
||||
(mu/with-context {:entity entity :total @total}
|
||||
(doseq [a batch]
|
||||
(try
|
||||
(append a)
|
||||
(catch Exception e
|
||||
(mu/log ::error
|
||||
:exception e)
|
||||
(throw e)))
|
||||
)
|
||||
(swap! so-far #(+ % (count batch)))
|
||||
(when (= 0 (rand-int 100))
|
||||
(mu/log ::appended :count (count batch)
|
||||
:so-far @so-far))))
|
||||
(->> (partition-all 100 entities)
|
||||
(into [])
|
||||
(s/->source)
|
||||
(s/onto buffered)
|
||||
(s/buffer 20)
|
||||
(s/map (fn [entities]
|
||||
(pull-batch remote-db schema entity entities)))
|
||||
(s/buffer 20)
|
||||
(s/realize-each)))
|
||||
)
|
||||
(try
|
||||
(mu/log ::copying)
|
||||
(let [f (io/file "/tmp/tmp-ednl")]
|
||||
(s3/put-object :bucket-name (:data-bucket env)
|
||||
:key (str backup-id "/" entity ".ednl")
|
||||
:input-stream (io/make-input-stream f {})
|
||||
:metadata {:content-length (.length f)}))
|
||||
(mu/log ::copied)
|
||||
(catch Exception e
|
||||
(mu/log ::upload-error
|
||||
:exception e)
|
||||
(throw e)))))))))
|
||||
|
||||
(defn -main [& _]
|
||||
(try
|
||||
(execute "export-backup" #(dump-all))
|
||||
(catch Exception e
|
||||
(println e)
|
||||
(mu/log ::quit-error
|
||||
:exception e
|
||||
:background-job "export-backup"
|
||||
:service "export-backup")
|
||||
(Thread/sleep 5000)
|
||||
(throw e))))
|
||||
@@ -1,53 +1,38 @@
|
||||
(ns auto-ap.jobs.ntg
|
||||
(:require
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic.clients :as d-clients]
|
||||
[iol-ion.tx :refer [propose-invoice]]
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.jobs.core :refer [execute]]
|
||||
[iol-ion.tx :refer [propose-invoice]]
|
||||
[auto-ap.logging :as log]
|
||||
[auto-ap.parse :as parse]
|
||||
[auto-ap.time :as atime]
|
||||
[clj-time.coerce :as coerce]
|
||||
[clj-time.core :as time]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str])
|
||||
[clojure.string :as str]
|
||||
[clojure.xml :as xml]
|
||||
[clojure.zip :as zip]
|
||||
[datomic.client.api :as d])
|
||||
(:import
|
||||
(java.util UUID)))
|
||||
|
||||
(def bucket-name "data.prod.app.integreatconsult.com" #_(:data-bucket env))
|
||||
|
||||
(defn read-csv [k]
|
||||
(log/info ::reading-csv :key k)
|
||||
(-> (s3/get-object {:bucket-name bucket-name
|
||||
:key k})
|
||||
:input-stream
|
||||
(defn read-csv [stream]
|
||||
(-> stream
|
||||
io/reader
|
||||
csv/read-csv))
|
||||
|
||||
(defn read-xml [stream]
|
||||
(-> (slurp stream)
|
||||
(.getBytes)
|
||||
(java.io.ByteArrayInputStream. )
|
||||
xml/parse
|
||||
zip/xml-zip))
|
||||
|
||||
|
||||
(defn extract-invoice-details [csv-rows clients]
|
||||
(clojure.pprint/pprint (take 4 csv-rows))
|
||||
(->> csv-rows
|
||||
(drop 1)
|
||||
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
|
||||
|
||||
(= "Y" break-flag)))
|
||||
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
|
||||
|
||||
(let [[matching-client similarity] (and location-hint
|
||||
(parse/best-match clients location-hint 0.0))]
|
||||
(clojure.pprint/pprint {:invoice/vendor vendor
|
||||
:invoice/location (parse/best-location-match matching-client location-hint location-hint )
|
||||
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
|
||||
:invoice/invoice-number invoice-number
|
||||
:invoice/total (Double/parseDouble invoice-total)
|
||||
:invoice/outstanding-balance (Double/parseDouble invoice-total)
|
||||
:invoice/client matching-client
|
||||
:invoice/import-status :import-status/completed
|
||||
:invoice/status :invoice-status/unpaid
|
||||
:invoice/client-identifier location-hint
|
||||
}))))
|
||||
(filter :invoice/client)))
|
||||
|
||||
(defn mark-key [k]
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
@@ -60,46 +45,209 @@
|
||||
(defn is-csv-file? [x]
|
||||
(= "csv" (last (str/split x #"[\\.]"))))
|
||||
|
||||
(defn decipher-source [k]
|
||||
(cond
|
||||
(str/includes? k "Cintas")
|
||||
:cintas
|
||||
|
||||
(and (str/includes? k "GeneralProduce")
|
||||
(is-csv-file? k))
|
||||
:general-produce
|
||||
|
||||
:else
|
||||
:unknown))
|
||||
|
||||
(defmulti extract-invoice-details (fn [key input-stream clients]
|
||||
(decipher-source key)))
|
||||
|
||||
(defmethod extract-invoice-details :general-produce
|
||||
[k input-stream clients]
|
||||
(log/info ::parsing-general-produce :key k)
|
||||
(->> (read-csv input-stream)
|
||||
(drop 1)
|
||||
(filter (fn [[_ _ _ _ _ _ _ _ _ _ _ break-flag]]
|
||||
(= "Y" break-flag)))
|
||||
(map (fn [[vendor location-hint invoice-number ship-date invoice-total ]]
|
||||
(let [[matching-client similarity] (and location-hint
|
||||
(parse/best-match clients location-hint 0.0))]
|
||||
{:invoice/vendor vendor
|
||||
:invoice/location (parse/best-location-match matching-client location-hint location-hint )
|
||||
:invoice/date (coerce/to-date (atime/parse ship-date atime/normal-date))
|
||||
:invoice/invoice-number invoice-number
|
||||
:invoice/total (Double/parseDouble invoice-total)
|
||||
:invoice/outstanding-balance (Double/parseDouble invoice-total)
|
||||
:invoice/client matching-client
|
||||
:invoice/import-status :import-status/imported
|
||||
:invoice/status :invoice-status/unpaid
|
||||
:invoice/client-identifier location-hint
|
||||
})))
|
||||
(filter :invoice/client)))
|
||||
|
||||
(defmethod extract-invoice-details :unknown
|
||||
[k input-stream clients]
|
||||
(log/warn ::unknown-invoice-format
|
||||
:key k)
|
||||
[])
|
||||
|
||||
(defn zip-seq [zipper]
|
||||
(->> (zip/xml-zip (zip/node zipper))
|
||||
(iterate zip/next )
|
||||
(take-while (complement zip/end?))))
|
||||
|
||||
(defmethod extract-invoice-details :cintas
|
||||
[k input-stream clients]
|
||||
(log/info ::parsing-cintas :key k)
|
||||
(let [vendor (d/pull (d/db conn) '[:vendor/default-account] :vendor/cintas)
|
||||
top (read-xml input-stream)
|
||||
node-seq (->> top
|
||||
(iterate zip/next)
|
||||
(take-while (complement zip/end?)))
|
||||
location-hint (->> node-seq
|
||||
(filter (fn [z]
|
||||
(= (:tag (zip/node z))
|
||||
:InvoiceDetailShipping)))
|
||||
first
|
||||
zip-seq
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= :Street
|
||||
(:tag node))))
|
||||
first
|
||||
:content
|
||||
first)
|
||||
[matching-client similarity] (and location-hint (parse/best-match clients location-hint 0.0))
|
||||
|
||||
]
|
||||
(if matching-client
|
||||
(let [invoice-date (->> node-seq
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= (:tag node)
|
||||
:InvoiceDetailRequestHeader)))
|
||||
first
|
||||
(#(-> % :attrs :invoiceDate
|
||||
coerce/to-date-time
|
||||
atime/localize
|
||||
(atime/unparse atime/iso-date)
|
||||
(atime/parse atime/iso-date))))
|
||||
location (parse/best-location-match matching-client location-hint location-hint )
|
||||
due (-> invoice-date
|
||||
(time/plus (time/days 30))
|
||||
(coerce/to-date))
|
||||
total (->> node-seq
|
||||
(filter (fn [zipper]
|
||||
(= (:tag (zip/node zipper))
|
||||
:NetAmount)))
|
||||
first
|
||||
zip-seq
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= :Money
|
||||
(:tag node))))
|
||||
first
|
||||
:content
|
||||
first
|
||||
Double/parseDouble)
|
||||
invoice {:invoice/vendor :vendor/cintas
|
||||
:invoice/import-status :import-status/imported
|
||||
:invoice/status :invoice-status/unpaid
|
||||
:invoice/location location
|
||||
:invoice/client-identifier location-hint
|
||||
:invoice/client (:db/id matching-client)
|
||||
:invoice/total total
|
||||
:invoice/outstanding-balance total
|
||||
:invoice/invoice-number (->> node-seq
|
||||
(map zip/node)
|
||||
(filter (fn [node]
|
||||
(= (:tag node)
|
||||
:InvoiceDetailRequestHeader)))
|
||||
first
|
||||
(#(-> % :attrs :invoiceID)))
|
||||
:invoice/due due
|
||||
|
||||
:invoice/scheduled-payment (when-not ((into #{} (->> matching-client
|
||||
:client/feature-flags))
|
||||
"manually-pay-cintas")
|
||||
due)
|
||||
|
||||
:invoice/date (coerce/to-date invoice-date)
|
||||
:invoice/expense-accounts [{:invoice-expense-account/account
|
||||
(-> vendor :vendor/default-account :db/id)
|
||||
:invoice-expense-account/location location
|
||||
:invoice-expense-account/amount (Math/abs total)
|
||||
}]}]
|
||||
(log/info ::cintas-invoice-importing
|
||||
:invoice invoice)
|
||||
[invoice])
|
||||
(do
|
||||
(log/info ::missing-client
|
||||
:client-hint location-hint)
|
||||
[]))))
|
||||
|
||||
(defn mark-error [k]
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key (str "ntg-invoices/error/"
|
||||
(.getName (io/file k)))}))
|
||||
|
||||
(defn copy-readable-version [k]
|
||||
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) "." (last (str/split k #"[\\.]")))]
|
||||
(log/info ::assigned-random-key
|
||||
:key k
|
||||
:invoice-key invoice-key)
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key invoice-key })
|
||||
invoice-key))
|
||||
|
||||
(defn import-ntg-invoices []
|
||||
(let [clients (d-clients/get-all)
|
||||
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
:prefix "ntg-invoices/pending"})
|
||||
:object-summaries
|
||||
(map :key))]
|
||||
(let [clients (map first (d/q '[:find (pull ?c [:client/code
|
||||
:db/id
|
||||
:client/feature-flags
|
||||
{:client/location-matches [:location-match/matches :location-match/location]}
|
||||
:client/name
|
||||
:client/matches
|
||||
:client/locations])
|
||||
:where [?c :client/code]]
|
||||
(d/db conn)))
|
||||
keys (->> (s3/list-objects-v2 {:bucket-name bucket-name
|
||||
:prefix "ntg-invoices/pending"})
|
||||
:object-summaries
|
||||
(map :key))]
|
||||
|
||||
|
||||
(log/info ::found-invoice-keys
|
||||
:keys keys )
|
||||
|
||||
(let [transaction (->> keys
|
||||
(filter is-csv-file?)
|
||||
(mapcat (fn [k]
|
||||
(try
|
||||
(log/info ::trying-csv :key k)
|
||||
(let [invoice-key (str "invoice-files/" (UUID/randomUUID) ".csv") ;
|
||||
(let [invoice-key (copy-readable-version k)
|
||||
invoice-url (str "http://" bucket-name ".s3-website-us-east-1.amazonaws.com/" invoice-key)]
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key invoice-key})
|
||||
(->> (extract-invoice-details (read-csv k) clients)
|
||||
(map (fn [i]
|
||||
[`(propose-invoice ~(assoc i :invoice/source-url invoice-url))]))
|
||||
))
|
||||
(catch Exception e
|
||||
(with-open [is (-> (s3/get-object {:bucket-name bucket-name
|
||||
:key k})
|
||||
:input-stream)]
|
||||
(->> (extract-invoice-details k
|
||||
is
|
||||
clients)
|
||||
(map (fn [i]
|
||||
(log/info ::importing-invoice
|
||||
:invoice i)
|
||||
i))
|
||||
(mapv (fn [i]
|
||||
`(propose-invoice ~(assoc i :invoice/source-url invoice-url))))))) (catch Exception e
|
||||
(log/error ::cant-load-file
|
||||
:key k
|
||||
:exception e)
|
||||
(log/info
|
||||
(s3/copy-object {:source-bucket-name bucket-name
|
||||
:destination-bucket-name bucket-name
|
||||
:source-key k
|
||||
:destination-key (str "ntg-invoices/error/"
|
||||
(.getName (io/file k)))}))
|
||||
[])))))
|
||||
#_result #_(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]
|
||||
(clojure.pprint/pprint transaction)
|
||||
#_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2)))
|
||||
(mark-error k)
|
||||
[]))))
|
||||
(into []))]
|
||||
(transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})
|
||||
(log/info ::success
|
||||
:count (count transaction)
|
||||
:sample (take 3 transaction)))
|
||||
(doseq [k keys]
|
||||
(mark-key k))))
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
[auto-ap.jobs.ezcater-upsert :as job-ezcater-upsert]
|
||||
[auto-ap.jobs.import-uploaded-invoices :as job-import-uploaded-invoices]
|
||||
[auto-ap.jobs.intuit :as job-intuit]
|
||||
#_[auto-ap.backup :as backup]
|
||||
[auto-ap.jobs.ledger-reconcile :as job-reconcile-ledger]
|
||||
[auto-ap.jobs.load-historical-sales :as job-load-historical-sales]
|
||||
[auto-ap.jobs.plaid :as job-plaid]
|
||||
@@ -141,6 +142,10 @@
|
||||
(= job "restore-from-backup")
|
||||
(job-restore-from-backup/-main)
|
||||
|
||||
(comment
|
||||
(= job "export-backup")
|
||||
(backup/-main))
|
||||
|
||||
:else
|
||||
(do
|
||||
(add-shutdown-hook! shutdown-mount)
|
||||
|
||||
@@ -75,6 +75,7 @@
|
||||
|
||||
:operating-activities [
|
||||
;; BEN EDIT STARTING HERE
|
||||
["20100-20199 Credit Card Balances" 20100 20199 :add]
|
||||
["21000-24000 Accounts Payable" 21000 23999 :add]
|
||||
["25000-28000 Accounts Payable" 25000 27999 :add]
|
||||
["24000-25000 Accrual Liabilities" 24000 24999 :add]
|
||||
@@ -678,15 +679,17 @@
|
||||
:financing-activities
|
||||
(str prefix " Financing Activities")))
|
||||
|
||||
(conj (cashflow-subtotal-by-column-row (map #(-> %
|
||||
(filter-categories [:operating-activities :investment-activities :financing-activities :sales :cogs :payroll :controllable :fixed-overhead :ownership-controllable])
|
||||
(negate #{:cogs :payroll :controllable :fixed-overhead :ownership-controllable}))
|
||||
pnl-datas)
|
||||
"Change in Cash and Cash Equivalents"))
|
||||
|
||||
(into (cash-flow-detail-rows pnl-datas
|
||||
:cash
|
||||
(str prefix " Bank Accounts / Cash")))
|
||||
|
||||
(conj (cashflow-subtotal-by-column-row (map #(-> %
|
||||
(filter-categories [:operating-activities :investment-activities :financing-activities :cash])
|
||||
#_(negate #{:cogs :payroll :controllable :fixed-overhead :ownership-controllable}))
|
||||
pnl-datas)
|
||||
"Change in Cash and Cash Equivalents")))]
|
||||
)]
|
||||
{:header (cash-flow-headers pnl-datas "Cash Flow")
|
||||
:rows table}))
|
||||
|
||||
|
||||
@@ -545,7 +545,8 @@
|
||||
[com/multi-field-v2 {:allow-change? true
|
||||
:template [[form-builder/raw-field-v2 {:field :feature-flag}
|
||||
[com/select-field {:options [[nil nil]
|
||||
["new-square" "New Square+Ezcater"]]
|
||||
["new-square" "New Square+Ezcater"]
|
||||
["manually-pay-cintas" "Manually Pay Cintas"]]
|
||||
:allow-nil? false
|
||||
:style {:width "18em"}}]]]
|
||||
:key-fn :id
|
||||
|
||||
@@ -144,7 +144,8 @@
|
||||
[:square2-import-job "Square2 Import"]
|
||||
[:register-invoice-import "Register Invoice Import "]
|
||||
[:ezcater-upsert "Upsert recent ezcater orders"]
|
||||
[:load-historical-sales "Load Historical Square Sales"]]
|
||||
[:load-historical-sales "Load Historical Square Sales"]
|
||||
[:export-backup "Export Backup"]]
|
||||
:allow-nil? true
|
||||
:keywordize? true}]]
|
||||
(cond (= :bulk-journal-import (:job data))
|
||||
|
||||
@@ -416,7 +416,7 @@ NOTE: Please review the transactions we may have question for you here: https://
|
||||
report (l-reports/summarize-cash-flows pnl-data)
|
||||
table (rtable/concat-tables (:details report))]
|
||||
[:div
|
||||
[:h1.title "Profit and Loss - " (str/join ", " (map (comp :name :client) (:clients args)))]
|
||||
[:h1.title "Statement of Cash Flows - " (str/join ", " (map (comp :name :client) (:clients args)))]
|
||||
(when (:warning report)
|
||||
[:div.notification.is-warning.is-light
|
||||
(:warning report)])
|
||||
|
||||
Reference in New Issue
Block a user