(ns auto-ap.datomic (:require [datomic.api :as d] [auto-ap.db.vendors :as v] [auto-ap.db.companies :as c] [auto-ap.db.invoices :as i] [auto-ap.db.checks :as checks] [auto-ap.db.users :as users] [auto-ap.db.invoices-expense-accounts :as iea] [auto-ap.db.invoices-checks :as ic] [auto-ap.db.transactions :as transactions] [clojure.string :as str] [clj-time.core :as time] [clj-time.coerce :as coerce])) (def uri "datomic:sql://invoices?jdbc:postgresql://database:5432/datomic?user=datomic&password=datomic") (defn create-database [] (d/create-database uri)) (defn drop-database [] (d/delete-database uri)) (defn remove-nils [m] (let [result (reduce-kv (fn [m k v] (if (not (nil? v)) (assoc m k v) m )) {} m)] (if (seq result) result nil))) (def vendor-schema [{:db/ident :vendor/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id from the old system"} {:db/ident :vendor/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's human-friendly name"} {:db/ident :vendor/code :db/valueType :db.type/string :db/unique :db.unique/identity :db/cardinality :db.cardinality/one :db/doc "A vendor's computer-friendly name"} {:db/ident :vendor/print-as :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Set if you want to override how this vendor's name is printed on checks"} {:db/ident :vendor/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's email address"} {:db/ident :vendor/phone :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor's phone number"} {:db/ident :vendor/invoice-reminder-schedule :db/valueType :db.type/keyword :db/cardinality :db.cardinality/one :db/doc "How often to email this vendor about invoices"} {:db/ident :vendor/default-expense-account :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The vendor's default expense account"} {:db/ident :vendor/primary-contact :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "The vendor's primary contact"} {:db/ident :vendor/secondary-contact :db/valueType :db.type/ref :db/isComponent true :db/cardinality :db.cardinality/one :db/doc "The vendor's secondary contact"} {:db/id #db/id[:db.part/db] :db/ident :vendor/address :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db.install/_attribute :db.part/db :db/doc "The vendor's address"} ]) (def client-schema [{:db/ident :client/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id from the old system"} {:db/ident :client/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A client's human-friendly name"} {:db/ident :client/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "hello@example.com"} {:db/ident :client/code :db/valueType :db.type/string :db/unique :db.unique/identity :db/cardinality :db.cardinality/one :db/doc "A client's computer-friendly name"} {:db/ident :client/signature-file :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A check signature image"} {:db/ident :client/locations :db/valueType :db.type/string :db/cardinality :db.cardinality/many :db/doc "A client's locations"} {:db/ident :client/address :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "Address of the client"} {:db/ident :client/bank-accounts :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true :db/doc "Bank accounts for the client"}]) (def address-schema [ {:db/ident :address/street1 :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "123 main st"} {:db/ident :address/street2 :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Apt A"} {:db/ident :address/city :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Campbell"} {:db/ident :address/state :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "CA"} {:db/ident :address/zip :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "95014"}]) (def contact-schema [ {:db/ident :contact/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "John Smith"} {:db/ident :contact/phone :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "4255773578"} {:db/ident :contact/email :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "hello@example.com"}]) (def bank-account-schema [{:db/ident :bank-account/external-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Identifier for bank account"} {:db/ident :bank-account/original-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "matching the orignal tuple of [company,bank]"} {:db/ident :bank-account/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A1127 Chase Card"} {:db/ident :bank-account/bank-name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "e.g. Bank of America"} {:db/ident :bank-account/bank-code :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The code to list under the bank's name"} {:db/ident :bank-account/routing :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The bank's routing number"} {:db/ident :bank-account/number :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The account number"} {:db/ident :bank-account/type :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true :db/doc "The type of account number, either :bank-account-type/check or :bank-account-type/cash"} {:db/ident :bank-account/yodlee-account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "yodlee's account identifier"} {:db/ident :bank-account/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Current check number"} {:db/ident :bank-account-type/check} {:db/ident :bank-account-type/cash}]) (def invoice-schema [{:db/ident :invoice/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id in old system"} {:db/ident :invoice/invoice-number :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "A vendor-specified number for the invoice"} {:db/ident :invoice/customer-identifier :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "An identifier found to suggest the customer"} {:db/ident :invoice/status :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Status of payment/import of the invoice [:paid, :unpaid]"} {:db/ident :invoice/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Which client this invoice is for"} {:db/ident :invoice/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Which vendor this invoice is for"} {:db/ident :invoice/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date for this invoice"} {:db/ident :invoice/total :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "Total $ for this invoice"} {:db/ident :invoice/outstanding-balance :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The unpaid balance of this invoice"} {:db/ident :invoice/default-location :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The default location that expense-accounts will be created with for this invoice"} {:db/ident :invoice/default-expense-account :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The default expense account for this invoice"} {:db/ident :invoice/expense-accounts :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/isComponent true :db/doc "The expense account categories for this invoice"} {:db/ident :invoice-status/paid} {:db/ident :invoice-status/unpaid} {:db/ident :invoice-status/voided}]) (def invoice-expense-account-schema [{:db/ident :invoice-expense-account/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "Original id in old system"} {:db/ident :invoice-expense-account/expense-account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The code for the expense account"} {:db/ident :invoice-expense-account/location :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Location for this expense account"} {:db/ident :invoice-expense-account/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that this contributes to"}]) (def payment-schema [{:db/ident :payment/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :payment/s3-uuid :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The uuid that matches the key for this check"} {:db/ident :payment/s3-key :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The s3 key with pdf of this check"} {:db/ident :payment/s3-url :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The s3 url with pdf of this check"} {:db/ident :payment/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The check number"} {:db/ident :payment/memo :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "The check's memo line"} {:db/ident :payment/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "The date the payment was made"} {:db/ident :payment/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that was paid to the vendor"} {:db/ident :payment/paid-to :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Who the paid was made out to"} {:db/ident :payment/status :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The status of the payment [:pending :cleared :voided]"} {:db/ident :payment/type :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The type of the payment [:cash :check :debit]"} {:db/ident :payment/pdf-data :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "raw data used to generate check pdf"} ;; relations {:db/ident :payment/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The vendor for which this payment was for"} {:db/ident :payment/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The client for which this payment"} {:db/ident :payment/bank-account :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The bank account that was used to pay"} {:db/ident :payment/invoices :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/doc "Any invoices this payment is related to"} ;; enums {:db/ident :payment-status/pending} {:db/ident :payment-status/voided} {:db/ident :payment-status/cleared} {:db/ident :payment-type/cash} {:db/ident :payment-type/check} {:db/ident :payment-type/debit} ]) (def invoice-payment-schema [{:db/ident :invoice-payment/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :invoice-payment/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount that was paid to this invoice"} ;; relations {:db/ident :invoice-payment/invoice :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The invoice for this payment"} {:db/ident :invoice-payment/payment :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The total payment for this payment"}]) (def transaction-schema [{:db/ident :transaction/original-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :transaction/amount :db/valueType :db.type/double :db/cardinality :db.cardinality/one :db/doc "The amount of the transaction"} {:db/ident :transaction/description-original :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "full description of the transaction"} {:db/ident :transaction/description-simple :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "short description of the transaction"} {:db/ident :transaction/merchant-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "a yodlee id for the merchant"} {:db/ident :transaction/merchant-name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "a name for the merchant"} {:db/ident :transaction/id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "A key to match against"} {:db/ident :transaction/date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date that the transaction showed up"} {:db/ident :transaction/post-date :db/valueType :db.type/instant :db/cardinality :db.cardinality/one :db/doc "Date that the transaction posted"} {:db/ident :transaction/type :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Yodlee description of the transaction"} {:db/ident :transaction/status :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Yodlee status of the transaction"} {:db/ident :transaction/account-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "Yodlee account id"} {:db/ident :transaction/check-number :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/doc "The check number that was parsed from the description"} ;; relations {:db/ident :transaction/vendor :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Vendor for who we think this transaction is from"} {:db/ident :transaction/client :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "Client for who we think this transaction is for"} {:db/ident :transaction/bank-account :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The bank accout used for this transaction"} {:db/ident :transaction/payment :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The payment that this transaction matched to"} ]) (def user-schema [{:db/ident :user/original-id :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/doc "The id in the old system"} {:db/ident :user/provider :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "Provider for oauth for the user"} {:db/ident :user/provider-id :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "the id from the provider"} {:db/ident :user/role :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/doc "The role [:user :admin :none]"} {:db/ident :user/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/doc "name of the user"} {:db/ident :user/clients :db/valueType :db.type/ref :db/cardinality :db.cardinality/many :db/doc "The clients this user can view"} ;;enums {:db/ident :user-role/admin} {:db/ident :user-role/user} {:db/ident :user-role/none} ]) (def base-schema [ address-schema contact-schema vendor-schema client-schema bank-account-schema invoice-schema invoice-expense-account-schema payment-schema invoice-payment-schema transaction-schema user-schema]) (defn query-entities [] (d/q '[:find (pull ?e [:vendor/name]) :where [?e :vendor/name]] (d/db (d/connect uri)))) (defn load-vendors [vendors] (->> vendors (map (fn [{:keys [primary-phone address email primary-contact secondary-email secondary-contact primary-email name default-expense-account id code secondary-phone invoice-reminder-schedule print-as]}] (let [vendor-id (d/tempid :db.part/user) address-id (d/tempid :db.part/user)] (remove-nils #:vendor {:original-id id :name name :code (if (seq code) code nil) :email email :default-expense-account default-expense-account :invoice-reminder-schedule nil :address (remove-nils #:address {:street1 (:street1 address) :street2 (:street2 address) :city (:city address) :state (:state address) :zip (:zip address)}) :primary-contact (remove-nils #:contact {:name primary-contact :phone primary-phone :email primary-email}) :secondary-contact (remove-nils #:contact {:name secondary-contact :phone secondary-phone :email secondary-email})})))) ) ) (defn load-clients [clients] (->> clients (map (fn [{:keys [name address id code locations email bank-accounts signature-file] client-id :id}] (remove-nils #:client {:original-id id :name (str name) :code nil :email email :signature-file signature-file :locations locations :address (remove-nils #:address {:street1 (:street1 address) :street2 (:street2 address) :city (:city address) :state (:state address) :zip (:zip address)}) :bank-accounts (conj (map (fn [{:keys [number id check-number bank-name bank-code routing name yodlee-account-id type] }] (remove-nils #:bank-account {:number number :original-id (str client-id "-" id) :external-id id :check-number check-number :bank-name bank-name :bank-code bank-code :routing routing :name name :yodlee-account-id yodlee-account-id :type (if type (keyword "bank-account-type" type ) :bank-account-type/check)})) bank-accounts) #:bank-account {:original-id (str client-id "-" 0) :external-id 0 :type :bank-account-type/cash})} ))))) (defn load-invoices [invoices] (->> invoices (map (fn [{:keys [id status total outstanding-balance invoice-number date customer-identifier company-id vendor-id default-location default-expense-account] invoice-id :id}] [(remove-nils #:invoice {:original-id id :invoice-number invoice-number :date (coerce/to-date date) :customer-identifier customer-identifier :client [:client/original-id company-id] :vendor (when vendor-id [:vendor/original-id vendor-id]) :default-location default-location :default-expense-account default-expense-account :total (double total) :outstanding-balance (double outstanding-balance) :status (keyword "invoice-status" status)})])))) (defn load-invoices-expense-accounts [invoices-expense-accounts] (->> invoices-expense-accounts (map (fn [{:keys [id expense-account-id location amount invoice-id]}] [(remove-nils #:invoice {:original-id invoice-id :expense-accounts [(remove-nils #:invoice-expense-account {:original-id id :expense-account-id expense-account-id :location location :amount (double amount)})]})])))) (defn load-payments [checks] (->> checks (map (fn [{:keys [id s3-uuid s3-key s3-url vendor-id company-id check-number memo date amount paid-to data bank-account-id status type] invoice-id :id}] [(remove-nils #:payment {:original-id id :s3-uuid s3-uuid :s3-key s3-key :s3-url s3-url :vendor (when vendor-id [:vendor/original-id vendor-id]) :client [:client/original-id company-id] :bank-account (when (and bank-account-id (not= "38-3" (str company-id "-" bank-account-id))) ;; TODO - 38-3 got removed at some ponitn [:bank-account/original-id (str company-id "-" bank-account-id)]) :check-number check-number :memo memo :date (coerce/to-date date) :amount (double amount) :paid-to paid-to :pdf-data data :status (keyword "payment-status" status) :type (if type (keyword "payment-type" type) :payment-type/check)})])))) (defn load-invoices-payments [invoices-checks] (->> invoices-checks (map (fn [{:keys [id invoice-id check-id amount]}] [(remove-nils #:invoice-payment {:original-id id :payment [:payment/original-id check-id] :invoice [:invoice/original-id invoice-id] :amount (double amount)})])))) (defn load-transactions [transactions] (let [transactions (->> transactions (map (fn [{:keys [id amount description-original description-simple merchant-id merchant-name date post-date type account-id status vendor-id company-id check-id check-number bank-account-id]}] (remove-nils #:transaction {:original-id id :id id :description-original description-original :description-simple description-simple :merchant-id merchant-id :merchant-name merchant-name :date (coerce/to-date date) :post-date (coerce/to-date post-date) :type type :status status :amount (double amount) :account-id account-id :check-number check-number :vendor (when vendor-id [:vendor/original-id vendor-id]) :client (when company-id [:client/original-id company-id]) :payment (when check-id [:payment/original-id check-id]) :bank-account (when (and bank-account-id (not= "38-3" (str company-id "-" bank-account-id))) [:bank-account/original-id (str company-id "-" bank-account-id)])}))) (partition-all 10))] (println "transactions: " (count transactions) "batches of 50") transactions)) (defn load-users [users] (->> users (map (fn [{:keys [id role provider-id provider companies name]}] (remove-nils #:user {:original-id id :name name :role (keyword "user-role" role) :provider-id provider-id :provider provider :clients (map (fn [c] [:client/original-id c]) companies)}))) )) (defn query-vendors [] (d/q '[:find (pull ?e [*]) :where [?e :vendor/original-id]] (d/db (d/connect uri)))) (defn query-clients [] (d/q '[:find (pull ?e [*]) :where [?e :client/original-id]] (d/db (d/connect uri)))) (defn query-invoices [] (d/q '[:find (pull ?e [* {:invoice/vendor [*] :invoice/expense-accounts [*]}]) :where [?e :invoice/original-id]] (d/db (d/connect uri)))) (defn query-payments [] (d/q '[:find (pull ?e [* {:invoice-payment/_payment [* {:invoice-payment/invoice [*]}]}]) :where [?e :payment/original-id]] (d/db (d/connect uri)))) (defn query-check-payments [] (d/q '[:find (pull ?e [* {:invoice-payment/payment [*]}]) :where [?e :invoice-payment/original-id]] (d/db (d/connect uri)))) (defn query-transactions [] (d/q '[:find (pull ?e [*]) :where [?e :transaction/original-id]] (d/db (d/connect uri)))) (defn query-users [] (d/q '[:find (pull ?e [*]) :where [?e :user/original-id]] (d/db (d/connect uri)))) (defn migrate-vendors [conn] [(let [all-vendors (v/get-all)] (load-vendors all-vendors))]) (defn migrate-clients [conn] (let [all-clients (c/get-all)] [(load-clients all-clients)])) (defn migrate-invoices [conn] (load-invoices (i/get-all))) (defn migrate-payments [conn] (load-payments (checks/get-all))) (defn migrate-invoices-payments [conn] (load-invoices-payments (ic/get-all))) (defn migrate-invoices-expense-accounts [conn] (load-invoices-expense-accounts (iea/get-all))) (defn migrate-transactions [conn] (let [trans (load-transactions (transactions/get-all))] (if (seq trans) trans [[]]))) (defn migrate-users [conn] [(load-users (users/get-all))]) (defn merge-query [query-part-1 query-part-2] (-> query-part-1 (update-in [:query :find] into (get-in query-part-2 [:query :find])) (update-in [:query :in] into (get-in query-part-2 [:query :in])) (update-in [:query :where] into (get-in query-part-2 [:query :where])) (update-in [:args] into (get-in query-part-2 [:args]))))