diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index 26c26fab..fdc26fb2 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -2,7 +2,15 @@ (:require [datomic.api :as d] [auto-ap.db.vendors :as v] [auto-ap.db.companies :as c] - [clojure.string :as str])) + [auto-ap.db.invoices :as i] + [auto-ap.db.checks :as checks] + [auto-ap.db.users :as users] + [auto-ap.db.invoices-expense-accounts :as iea] + [auto-ap.db.invoices-checks :as ic] + [auto-ap.db.transactions :as transactions] + [clojure.string :as str] + [clj-time.core :as time] + [clj-time.coerce :as coerce])) (def uri "datomic:sql://invoices?jdbc:postgresql://database:5432/datomic?user=datomic&password=datomic") @@ -114,6 +122,7 @@ {:db/ident :client/bank-accounts :db/valueType :db.type/ref :db/cardinality :db.cardinality/many + :db/isComponent true :db/doc "Bank accounts for the client"}]) (def address-schema @@ -154,9 +163,352 @@ :db/cardinality :db.cardinality/one :db/doc "hello@example.com"}]) + + +(def bank-account-schema + [{:db/ident :bank-account/id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "Identifier for bank account"} + {:db/ident :bank-account/original-id + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "matching the orignal tuple of [company,bank]"} + {:db/ident :bank-account/name + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "A1127 Chase Card"} + {:db/ident :bank-account/bank-name + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "e.g. Bank of America"} + {:db/ident :bank-account/bank-code + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The code to list under the bank's name"} + {:db/ident :bank-account/routing + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The bank's routing number"} + {:db/ident :bank-account/number + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The account number"} + {:db/ident :bank-account/type + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/isComponent true + :db/doc "The type of account number, either :bank-account-type/check or :bank-account-type/cash"} + {:db/ident :bank-account/yodlee-account-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "yodlee's account identifier"} + {:db/ident :bank-account/check-number + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "Current check number"} + + {:db/ident :bank-account-type/check} + {:db/ident :bank-account-type/cash} + ]) + +(def invoice-schema + [{:db/ident :invoice/original-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "Original id in old system"} + {:db/ident :invoice/invoice-number + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "A vendor-specified number for the invoice"} + {:db/ident :invoice/customer-identifier + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "An identifier found to suggest the customer"} + {:db/ident :invoice/status + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Status of payment/import of the invoice [:paid, :unpaid]"} + + {:db/ident :invoice/client + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Which client this invoice is for"} + + {:db/ident :invoice/vendor + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Which vendor this invoice is for"} + + {:db/ident :invoice/date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/doc "Date for this invoice"} + + {:db/ident :invoice/total + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "Total $ for this invoice"} + + {:db/ident :invoice/outstanding-balance + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The unpaid balance of this invoice"} + + {:db/ident :invoice/default-location + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The default location that expense-accounts will be created with for this invoice"} + + {:db/ident :invoice/default-expense-account + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The default expense account for this invoice"} + + {:db/ident :invoice/expense-accounts + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/many + :db/isComponent true + :db/doc "The expense account categories for this invoice"} + + + {:db/ident :invoice-status/paid} + {:db/ident :invoice-status/unpaid} + {:db/ident :invoice-status/voided}]) + +(def invoice-expense-account-schema + [{:db/ident :invoice-expense-account/original-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "Original id in old system"} + {:db/ident :invoice-expense-account/expense-account-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The code for the expense account"} + {:db/ident :invoice-expense-account/location + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Location for this expense account"} + {:db/ident :invoice-expense-account/amount + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The amount that this contributes to"}]) + + + +(def payment-schema + [{:db/ident :payment/original-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "The id in the old system"} + {:db/ident :payment/s3-uuid + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The uuid that matches the key for this check"} + {:db/ident :payment/s3-key + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The s3 key with pdf of this check"} + {:db/ident :payment/s3-url + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The s3 url with pdf of this check"} + {:db/ident :payment/check-number + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The check number"} + {:db/ident :payment/memo + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "The check's memo line"} + {:db/ident :payment/date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/doc "The date the payment was made"} + {:db/ident :payment/amount + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The amount that was paid to the vendor"} + {:db/ident :payment/paid-to + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Who the paid was made out to"} + {:db/ident :payment/status + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The status of the payment [:pending :cleared :voided]"} + {:db/ident :payment/type + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The type of the payment [:cash :check :debit]"} + {:db/ident :payment/pdf-data + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "raw data used to generate check pdf"} + + + ;; relations + {:db/ident :payment/vendor + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The vendor for which this payment was for"} + {:db/ident :payment/client + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The client for which this payment"} + {:db/ident :payment/bank-account + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The bank account that was used to pay"} + + ;; enums + {:db/ident :payment-status/pending} + {:db/ident :payment-status/voided} + {:db/ident :payment-status/cleared} + + {:db/ident :payment-type/cash} + {:db/ident :payment-type/check} + {:db/ident :payment-type/debit} + ]) + +(def invoice-payment-schema + [{:db/ident :invoice-payment/original-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "The id in the old system"} + + {:db/ident :invoice-payment/amount + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The amount that was paid to this invoice"} + + ;; relations + {:db/ident :invoice-payment/invoice + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The invoice for this payment"} + {:db/ident :invoice-payment/payment + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The total payment for this payment"}]) + +(def transaction-schema + [{:db/ident :transaction/original-id + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "The id in the old system"} + {:db/ident :transaction/amount + :db/valueType :db.type/double + :db/cardinality :db.cardinality/one + :db/doc "The amount of the transaction"} + {:db/ident :transaction/description-original + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "full description of the transaction"} + {:db/ident :transaction/description-simple + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "short description of the transaction"} + {:db/ident :transaction/merchant-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "a yodlee id for the merchant"} + {:db/ident :transaction/merchant-name + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "a name for the merchant"} + {:db/ident :transaction/id + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "A key to match against"} + {:db/ident :transaction/date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/doc "Date that the transaction showed up"} + {:db/ident :transaction/post-date + :db/valueType :db.type/instant + :db/cardinality :db.cardinality/one + :db/doc "Date that the transaction posted"} + {:db/ident :transaction/type + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Yodlee description of the transaction"} + {:db/ident :transaction/status + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Yodlee status of the transaction"} + {:db/ident :transaction/account-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "Yodlee account id"} + {:db/ident :transaction/check-number + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/doc "The check number that was parsed from the description"} + + + ;; relations + {:db/ident :transaction/vendor + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Vendor for who we think this transaction is from"} + {:db/ident :transaction/client + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "Client for who we think this transaction is for"} + {:db/ident :transaction/bank-account + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The bank accout used for this transaction"} + {:db/ident :transaction/payment + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The payment that this transaction matched to"} + ]) + +(def user-schema + [{:db/ident :user/original-id + :db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity + :db/doc "The id in the old system"} + {:db/ident :user/provider + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "Provider for oauth for the user"} + {:db/ident :user/provider-id + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "the id from the provider"} + {:db/ident :user/role + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/one + :db/doc "The role [:user :admin :none]"} + {:db/ident :user/name + :db/valueType :db.type/string + :db/cardinality :db.cardinality/one + :db/doc "name of the user"} + {:db/ident :user/companies + :db/valueType :db.type/ref + :db/cardinality :db.cardinality/many + :db/doc "The companies this user can view"} + + ;;enums + {:db/ident :user-role/admin} + {:db/ident :user-role/user} + {:db/ident :user-role/none} + ]) + (defn create-schema [] (->> - (concat address-schema contact-schema vendor-schema client-schema) + (concat address-schema contact-schema vendor-schema client-schema bank-account-schema invoice-schema invoice-expense-account-schema payment-schema invoice-payment-schema transaction-schema user-schema) (d/transact (d/connect uri)))) @@ -201,42 +553,187 @@ (defn load-clients [clients] (->> clients (map - (fn [{:keys [name address id code locations email bank-accounts signature-file]}] - (doto (remove-nils #:client {:original-id id - :name (str name "-test") - :code nil - :email email - :signature-file signature-file - :locations locations - :address (remove-nils #:address {:street1 (:street1 address) - :street2 (:street2 address) - :city (:city address) - :state (:state address) - :zip (:zip address)}) - }) - println))) + (fn [{:keys [name address id code locations email bank-accounts signature-file] client-id :id}] + (remove-nils #:client {:original-id id + :name (str name "-test") + :code nil + :email email + :signature-file signature-file + :locations locations + :address (remove-nils #:address {:street1 (:street1 address) + :street2 (:street2 address) + :city (:city address) + :state (:state address) + :zip (:zip address)}) + :bank-accounts (conj (map + (fn [{:keys [number id check-number bank-name bank-code routing name yodlee-account-id type] }] + (remove-nils #:bank-account {:number number + :original-id (str client-id "-" id) + :id id + :check-number check-number + :bank-name bank-name + :bank-code bank-code + :routing routing + :name name + :yodlee-account-id yodlee-account-id + :type (if type + (keyword "bank-account-type" type ) + :bank-account-type/check)})) + bank-accounts) + #:bank-account {:original-id (str client-id "-" 0) + :id 0 + :type :bank-account-type/cash})} ))) (d/transact (d/connect uri)))) +(defn load-invoices [invoices] + (->> invoices + (map + (fn [{:keys [id status total outstanding-balance invoice-number date customer-identifier company-id vendor-id default-location default-expense-account] invoice-id :id}] + (remove-nils #:invoice {:original-id id + :invoice-number invoice-number + :date (coerce/to-date date) + :customer-identifier customer-identifier + :client [:client/original-id company-id] + :vendor [:vendor/original-id vendor-id] + :default-location default-location + :default-expense-account default-expense-account + :total (double total) + :outstanding-balance (double outstanding-balance) + :status (keyword "invoice-status" status)}))) + (d/transact (d/connect uri)))) + +(defn load-invoices-expense-accounts [invoices-expense-accounts] + (->> invoices-expense-accounts + (map + (fn [{:keys [id expense-account-id location amount invoice-id]}] + (remove-nils #:invoice {:original-id invoice-id + :expense-accounts [#:invoice-expense-account {:original-id id + :expense-account-id expense-account-id + :location location + :amount (double amount)}]}))) + (d/transact (d/connect uri)))) + + +(defn load-payments [checks] + (->> checks + (map + (fn [{:keys [id s3-uuid s3-key s3-url vendor-id company-id check-number memo date amount paid-to data bank-account-id status type] invoice-id :id}] + (remove-nils #:payment {:original-id id + :s3-uuid s3-uuid + :s3-key s3-key + :s3-url s3-url + :vendor [:vendor/original-id vendor-id] + :client [:client/original-id company-id] + :bank-account (when bank-account-id [:bank-account/original-id (str company-id "-" bank-account-id)]) + :check-number check-number + :memo memo + :date (coerce/to-date date) + :amount (double amount) + :paid-to paid-to + :pdf-data data + :status (keyword "payment-status" status) + :type (if type + (keyword "payment-type" type) + :payment-type/check)}))) + (d/transact (d/connect uri)))) + +(defn load-invoices-payments [invoices-checks] + (->> invoices-checks + (map + (fn [{:keys [id invoice-id check-id amount]}] + (remove-nils #:invoice-payment {:original-id id + :payment [:payment/original-id check-id] + :invoice [:invoice/original-id invoice-id] + :amount (double amount)}))) + (d/transact (d/connect uri)))) + +(defn load-transactions [transactions] + (->> transactions + (map + (fn [{:keys [id amount description-original description-simple merchant-id merchant-name + date post-date type account-id status vendor-id company-id check-id check-number + bank-account-id]}] + (remove-nils #:transaction {:original-id id + :description-original description-original + :description-simple description-simple + :merchant-id merchant-id + :merchant-name merchant-name + :date (coerce/to-date date) + :post-date (coerce/to-date post-date) + :type type + :status status + :amount (double amount) + :account-id account-id + :check-number check-number + :vendor (when vendor-id [:vendor/original-id vendor-id]) + :client (when company-id [:client/original-id company-id]) + :payment (when check-id [:payment/original-id check-id]) + :bank-account (when bank-account-id + [:bank-account/original-id (str company-id "-" bank-account-id)])}))) + (d/transact (d/connect uri))) + ) + +(defn load-users [users] + (->> users + (map + (fn [{:keys [id role provider-id provider companies]}] + (remove-nils #:user {:original-id id + :role (keyword "user-role" role) + :provider-id provider-id + :provider provider + :companies (map (fn [c] [:client/original-id c]) companies)}))) + (d/transact (d/connect uri)))) (defn query-vendors [] - (d/q '[:find (pull ?e [:db/id :vendor/invoice-reminder-schedule ]) - :where [?e :vendor/address]] + (d/q '[:find (pull ?e [*]) + :where [?e :vendor/original-id]] (d/db (d/connect uri)))) (defn query-clients [] - (d/q '[:find (pull ?e [*]) (min ?tx-time) (max ?tx-time2) - :where [?e :client/original-id] - [?e _ _ ?tx] - [?tx :db/txInstant ?tx-time] - [?tx :db/txInstant ?tx-time2]] + (d/q '[:find (pull ?e [*]) + :where [?e :client/original-id]] + (d/db (d/connect uri)))) + +(defn query-invoices [] + (d/q '[:find (pull ?e [* {:invoice/vendor [*] + :invoice/expense-accounts [*]}]) + :where [?e :invoice/original-id]] + (d/db (d/connect uri)))) + +(defn query-payments [] + (d/q '[:find (pull ?e [* {:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}]) + :where [?e :payment/original-id]] + (d/db (d/connect uri)))) + +(defn query-check-payments [] + (d/q '[:find (pull ?e [*]) + :where [?e :invoice-payment/original-id]] + (d/db (d/connect uri)))) + +(defn query-transactions [] + (d/q '[:find (pull ?e [*]) + :where [?e :transaction/original-id]] + (d/db (d/connect uri)))) + +(defn query-users [] + (d/q '[:find (pull ?e [*]) + :where [?e :user/original-id]] (d/db (d/connect uri)))) -(defn do [] +(defn do-it [] (create-database ) @(create-schema ) (let [all-vendors (v/get-all)] @(load-vendors all-vendors)) -(let [all-clients (c/get-all)] + (let [all-clients (c/get-all)] @(load-clients all-clients)) - (count (query-clients))) + @(load-invoices (i/get-all)) + @(load-payments (checks/get-all)) + @(load-invoices-payments (ic/get-all)) + @(load-invoices-expense-accounts (iea/get-all)) + @(load-transactions (transactions/get-all)) + @(load-users (users/get-all)) + + (count (clojure.pprint/pprint (query-payments)))) + diff --git a/src/clj/auto_ap/db/invoices_expense_accounts.clj b/src/clj/auto_ap/db/invoices_expense_accounts.clj index 41d4d770..ad588edf 100644 --- a/src/clj/auto_ap/db/invoices_expense_accounts.clj +++ b/src/clj/auto_ap/db/invoices_expense_accounts.clj @@ -3,6 +3,10 @@ [clojure.java.jdbc :as j] [auto-ap.db.utils :refer [clj->db kebab->snake db->clj get-conn query] :as utils] [honeysql.helpers :as helpers])) +(defn get-all [] + (query {:select [:*] + :from [:invoices-expense-accounts]})) + (defn get-missing [] (query {:select [:i.id :v.default-expense-account :i.total] :from [[:invoices :i]] diff --git a/src/clj/auto_ap/db/transactions.clj b/src/clj/auto_ap/db/transactions.clj index e28163e7..0edf4779 100644 --- a/src/clj/auto_ap/db/transactions.clj +++ b/src/clj/auto_ap/db/transactions.clj @@ -19,6 +19,8 @@ (def base-query (sql/build :select :* :from :transactions)) +(defn get-all [] + (query base-query)) (defn base-graphql [{:keys [company-id id]}] (println "ID" id) (cond-> base-query