From d02fba2b44a8f8ecb81605b2fe2d3af72508d411 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Tue, 23 Aug 2022 12:13:12 -0700 Subject: [PATCH] Makes integreat run on datomic cloud --- project.clj | 6 +- resources/datomic/ion-config.edn | 1 + scratch-sessions/dump-edn.clj | 429 ++++++++++++++++++ src/clj/auto_ap/datomic.clj | 314 +++++++++++-- src/clj/auto_ap/datomic/accounts.clj | 60 +-- src/clj/auto_ap/datomic/bank_accounts.clj | 8 +- src/clj/auto_ap/datomic/checks.clj | 43 +- src/clj/auto_ap/datomic/clients.clj | 36 +- src/clj/auto_ap/datomic/expected_deposit.clj | 14 +- src/clj/auto_ap/datomic/invoices.clj | 110 +++-- src/clj/auto_ap/datomic/ledger.clj | 27 +- src/clj/auto_ap/datomic/reports.clj | 15 +- src/clj/auto_ap/datomic/sales_orders.clj | 120 ++++- src/clj/auto_ap/datomic/transaction_rules.clj | 25 +- src/clj/auto_ap/datomic/transactions.clj | 54 +-- src/clj/auto_ap/datomic/users.clj | 26 +- src/clj/auto_ap/datomic/vendors.clj | 20 +- src/clj/auto_ap/datomic/yodlee2.clj | 10 +- src/clj/auto_ap/datomic/yodlee_merchants.clj | 8 +- src/clj/auto_ap/ezcater/core.clj | 26 +- src/clj/auto_ap/graphql.clj | 37 +- src/clj/auto_ap/graphql/accounts.clj | 106 +++-- src/clj/auto_ap/graphql/checks.clj | 18 +- src/clj/auto_ap/graphql/clients.clj | 242 +++++----- src/clj/auto_ap/graphql/import_batch.clj | 15 +- .../auto_ap/graphql/intuit_bank_accounts.clj | 6 +- src/clj/auto_ap/graphql/invoices.clj | 130 +++--- src/clj/auto_ap/graphql/ledger.clj | 49 +- src/clj/auto_ap/graphql/plaid.clj | 47 +- src/clj/auto_ap/graphql/reports.clj | 8 +- src/clj/auto_ap/graphql/transaction_rules.clj | 64 ++- src/clj/auto_ap/graphql/transactions.clj | 141 +++--- src/clj/auto_ap/graphql/utils.clj | 4 +- src/clj/auto_ap/graphql/vendors.clj | 36 +- src/clj/auto_ap/import/common.clj | 31 +- src/clj/auto_ap/import/intuit.clj | 18 +- src/clj/auto_ap/import/manual.clj | 10 +- src/clj/auto_ap/import/plaid.clj | 6 +- src/clj/auto_ap/import/transactions.clj | 83 ++-- src/clj/auto_ap/import/yodlee2.clj | 11 +- src/clj/auto_ap/jobs/close_auto_invoices.clj | 19 +- src/clj/auto_ap/jobs/sysco.clj | 51 ++- src/clj/auto_ap/jobs/vendor_usages.clj | 44 +- src/clj/auto_ap/ledger.clj | 193 ++++---- src/clj/auto_ap/pdf/ledger.clj | 45 +- src/clj/auto_ap/routes/exports.clj | 5 +- src/clj/auto_ap/routes/invoices.clj | 190 ++++---- src/clj/auto_ap/routes/queries.clj | 24 +- src/clj/auto_ap/routes/yodlee2.clj | 6 +- src/clj/auto_ap/search.clj | 72 +++ src/clj/auto_ap/server.clj | 4 - src/clj/auto_ap/square/core.clj | 78 ++-- src/clj/auto_ap/yodlee/core2.clj | 10 +- src/clj/user.clj | 251 +++++----- src/cljs/auto_ap/events.cljs | 2 +- .../auto_ap/views/pages/admin/accounts.cljs | 2 +- .../views/pages/admin/clients/form.cljs | 3 +- things-to-search-for.txt | 7 + 58 files changed, 2163 insertions(+), 1257 deletions(-) create mode 100644 resources/datomic/ion-config.edn create mode 100644 scratch-sessions/dump-edn.clj create mode 100644 src/clj/auto_ap/search.clj create mode 100644 things-to-search-for.txt diff --git a/project.clj b/project.clj index 2fea659e..6a71ed37 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,9 @@ :repositories {"my.datomic.com" {:url "https://my.datomic.com/repo" :username "datomic@brycecovertoperations.com" :password "9a382afc-d119-44db-83c2-98d8057d7666"}} + :dependencies [[org.clojure/clojure "1.10.1"] + [com.datomic/dev-local "1.0.243"] [com.datomic/datomic-pro "0.9.5783" :exclusions [com.google.guava/guava org.apache.httpcomponents/httpclient @@ -17,6 +19,8 @@ [bidi "2.1.6"] [ring/ring-defaults "0.3.2" :exclusions [ring ring/ring-core]] [mount "0.1.16"] + [org.apache.lucene/lucene-core "9.3.0"] + [org.apache.lucene/lucene-queryparser "9.3.0"] [metosin/malli "0.8.9"] [tolitius/yang "0.1.23"] [ring "1.8.2" :exclusions [commons-codec @@ -146,7 +150,7 @@ [com.bhauman/rebel-readline-cljs "0.1.4" :exclusions [org.clojure/clojurescript]] [javax.servlet/servlet-api "2.5"]] :plugins [[lein-pdo "0.1.1"]] - :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml"]} + :jvm-opts ["-Dconfig=config/dev.edn" "-Dlogback.configurationFile=logback.xml" "-Xms8G" "-Xmx12G"]} :uberjar diff --git a/resources/datomic/ion-config.edn b/resources/datomic/ion-config.edn new file mode 100644 index 00000000..579d4a2e --- /dev/null +++ b/resources/datomic/ion-config.edn @@ -0,0 +1 @@ +{:xforms [dump-edn/codify]} diff --git a/scratch-sessions/dump-edn.clj b/scratch-sessions/dump-edn.clj new file mode 100644 index 00000000..4143dea0 --- /dev/null +++ b/scratch-sessions/dump-edn.clj @@ -0,0 +1,429 @@ +;; This buffer is for Clojure experiments and evaluation. + +;; Press C-j to evaluate the last expression. + +;; You can also press C-u C-j to evaluate the expression and pretty-print its result. + +(ns dump-edn + (:require [datomic.api :as d] + + [clojure.java.io :as io] + [amazonica.aws.s3 :as s3] + [config.core :refer [env]] + [datomic.client.api :as dc] + [datomic.dev-local :as dl] + [clojure.set :as set])) + + +(def remote-db (d/db (datomic.api/connect "datomic:ddb://us-east-1/integreat/integreat-prod"))) +(def local-client (dc/client {:server-type :dev-local + :system "dev"})) + +(dc/list-databases local-client {}) + +(def schema (let [everything (->> (d/q '[:find [(pull ?e [:db/ident + {:db/valueType [:db/ident]} + {:db/cardinality [:db/ident]} + :db.attr/preds + {:db/unique [:db/ident]} + :db/isComponent + :db/id + :db/noHistory + :db/tupleAttrs + :db.entity/attrs + :db.entity/preds + :db/doc]) ...] + :where [?e :db/ident]] + remote-db)) + schema-attrs (->> everything + (filter :db/ident) + (filter (fn [{:db/keys [ident]}] + (if (namespace ident) + (re-matches #"^(?!cartographer)(?!db)(?!fressian).+" (namespace ident)) + true + )))) + meta-schema-schema (filter #(-> % :db/ident not) everything)] + schema-attrs)) + +(def best-key-helper + (->> schema + (filter :db/valueType) + (map :db/ident) + (group-by namespace) + #_(map (fn [[k v]] + [k ]))) + ) + +(def entity->best-key + {"transaction-rule" + [:transaction-rule/description, :transaction-rule/note :transaction-rule/vendor] + "square-location" + :square-location/square-id, + "expected-deposit" + :expected-deposit/date, + "journal-entry-line" + [:journal-entry-line/account, :journal-entry-line/debit :journal-entry-line/credit] + "vendor" + [:vendor/name,] + "transaction" + :transaction/amount, + "yodlee-provider-account" + :yodlee-provider-account/id, + "journal-entry" + :journal-entry/source, + "yodlee-merchant" :yodlee-merchant/yodlee-id, + "invoice" + :invoice/invoice-number, + "vendor-terms-override" + :vendor-terms-override/client, + "integration-status" + :integration-status/state, + "conformity" :conformity/conformed-norms-index, + "user" + :user/provider-id, + "sales-refund" + :sales-refund/total, + "plaid-account" + :plaid-account/name, + "charge" + [:charge/total, :charge/external-id] + "location-match" :location-match/location, + "vendor-schedule-payment-dom" + :vendor-schedule-payment-dom/dom, + "account-client-override" + :account-client-override/client, + "plaid-item" + :plaid-item/client, + "transaction-account" + :transaction-account/account, + "address" + [:address/street1, :address/city :address/state :address/zip] + "order-line-item" + :order-line-item/total, + "ezcater-location" :ezcater-location/location, + "account" + [:account/numeric-code, :account/code :account/name :account/type] + "intuit-bank-account" + :intuit-bank-account/name, + "saved-query" + :saved-query/guid, + "ezcater-caterer" + :ezcater-caterer/uuid, + "forecasted-transaction" + :forecasted-transaction/day-of-month, + "audit" :audit/user, + "yodlee-account" + :yodlee-account/id, + "transaction-rule-account" + [:transaction-rule-account/account, :transaction-rule-account/location] + "ezcater-integration" + :ezcater-integration/subscriber-uuid, + "report" + :report/created, + "bank-account" + :bank-account/code, + "vendor-usage" + :vendor-usage/key, + "invoice-expense-account" + [:invoice-expense-account/expense-account-id, :invoice-expense-account/account :invoice-expense-account/location :invoice-expense-account/amount] + "sales-order" + :sales-order/date, + "client" + :client/code, + "email-contact" :email-contact/email, + "invoice-payment" + :invoice-payment/amount, + "contact" + [:contact/name, :contact/phone :contact/email] + "import-batch" + :import-batch/date, + "payment" + [:payment/date, :payment/bank-account] + "vendor-account-override" + :vendor-account-override/client}) + +(def references (filter (comp #{:db.type/ref} :db/ident :db/valueType) schema )) + + + + + +(def reference->entity + (->> (d/q '[:find ?a ?v3 + :in $ $$ [?a ...] + :where [$$ _ ?a ?e] + [$ ?e ?v _ _] + [$ ?v :db/ident ?v2 _ _] + [(namespace ?v2) ?v3] + [(namespace ?v2) ?v3]] + remote-db + (d/since remote-db #inst "2022-06-01") + (map :db/ident references) + ) + (group-by first) + (map (fn [[k v]] + [k (disj (set (map second v)) "db")])) + (into {}))) + +(def entities-that-need-manual + (set (map namespace (filter (complement reference->entity) (map :db/ident references))))) + +(def manual-dependencies + {:client/location-matches #{"location-match"} + :transaction/yodlee-merchant #{"yodlee-merchant"} + :vendor-account-override/account #{"account"} + :vendor-account-override/client #{"client"} + :vendor/account-overrides #{"vendor-account-override"} + :transaction-rule/yodlee-merchant #{"yodlee-merchant"} + :client/forecasted-transactions #{"forecasted-transaction"} + :transaction/forecast-match #{"forecasted-transaction"} + :vendor/automatically-paid-when-due #{"client"} + :vendor/schedule-payment-dom #{"vendor-schedule-payment-dom"} + :vendor-schedule-payment-dom/client #{"client"}}) + +(def full-dependencies + (merge-with into reference->entity manual-dependencies)) + +(def entity-dependencies + (let [base-dependencies + (into + {} + (map (fn [i] + [i #{}]) + (set (map (comp namespace :db/ident) + (filter :db/valueType + schema)))) + ) + ] + (into base-dependencies (reduce + (fn [acc [ref deps]] + (update acc (namespace ref) (fnil #(into % deps) #{}))) + {} + full-dependencies)))) + +(def order-of-insert + (loop [entity-dependencies entity-dependencies + order [] + ] + (let [next-order (for [[entity deps] entity-dependencies + :when (not (seq deps))] + entity) + next-deps (reduce + (fn [entity-dependencies next-entity] + (into {} + (map + (fn [[k v]] + [k (disj v next-entity)]) + entity-dependencies))) + (apply dissoc entity-dependencies next-order) + next-order)] + (if (seq next-deps) + (recur next-deps (into order next-order)) + (into order next-order))))) + +#_(def best-attributes + (->> (map :db/ident (filter :db/valueType schema)) + (d/q + '[:find ?a2 (count ?e) + :in $ [?a2 ...] + :where + [?a :db/ident ?a2] + [?e ?a]] + remote-db) + (map (fn [[a count]] + [(namespace a) [a count]])) + (reduce (fn [acc [namespace [attr count]]] + (update acc namespace + (fnil + (fn [[curr-attr curr-count]] + (if (> count curr-count) + [attr count] + [curr-attr curr-count])) + ["" 0]))) + {}) + (map (fn [[k v]] + [k (first v)])))) + + + + +(def loaded (atom #{"charge" "order-line-item" "journal-entry-line"})) + +(defn migrate [] + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:db/ident :entity/migration-key + :db/unique :db.unique/identity + :db/cardinality :db.cardinality/one + :db/valueType :db.type/long}]}) + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data (map + (fn [s] + (set/rename-keys s {:db/id :entity/migration-key})) + schema)}) + + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) {:tx-data [{:entity/migration-key 17592257603901 :vendor/name "unknown"} + {:entity/migration-key 17592232621701} + {:entity/migration-key 17592263907739} + {:entity/migration-key 17592271516922}] + + }) + + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) + {:tx-data (->> (d/q '[:find ?e + :in $ $$ + :where [$$ ?e :transaction-rule/note _ _ true] + (not [$ ?e :transaction-rule/note] )] + remote-db + (d/history remote-db)) + (map (fn [[old-rule]] + {:entity/migration-key old-rule + :db/doc "A transaction rule that was deleted"})))}) + + #_(dc/transact (dc/connect local-client {:db-name "prod-migration"}) + {:tx-data (->> (d/q '[:find (count ?e) + :in $ + :where + #_[$$ ?e :transaction-account/account _ _ true] + [_ :transaction/accounts ?e] + (not [?e :transaction-account/account _] ) + ] + remote-db + #_(d/history remote-db)) + (map (fn [[old-rule]] + {:entity/migration-key old-rule + :db/doc "A transaction account that was deleted"})))}) + + + (doseq [entity (drop-while #(not= % "journal-entry") (filter (complement #{"audit"}) order-of-insert)) + :let [_ (swap! loaded conj entity) + _ (println "querying for " entity) + entities (d/q '[:find [?e ...] + :in $ [?a ...] + :where [?e ?a]] + remote-db + (cond-> (entity->best-key entity) + (not (vector? (entity->best-key entity))) vector))]] + (println "Inserting " entity ": " (count entities)) + (doseq [batch (partition-all 2000 entities)] + (print ".") + (let [transaction {:tx-data (->> (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= entity (namespace %))) + (into [:db/id])) + batch) + (mapv (fn [m ] + (reduce + (fn [m [k v]] + (cond + (= k :db/id) + (-> m + (assoc :entity/migration-key v) + (dissoc :db/id)) + (full-dependencies k) + (if (vector? v) + (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) + (assoc m k [:entity/migration-key (:db/id v)])) + :else + (dissoc m :payment/pdf-data + :payment/memo))) + m + m))))}] + (try + (dc/transact (dc/connect local-client {:db-name "prod-migration"}) transaction) + (Thread/sleep 50) + (catch Exception e + (clojure.pprint/pprint transaction) + (println e) + (throw e) + ))) + (flush)) + (println) + (println "Done"))) + +(defn reset-migrate [] + (dc/delete-database local-client {:db-name "prod-migration"}) + (dc/create-database local-client {:db-name "prod-migration"}) + (migrate)) + + + +(comment + (reset-migrate) + + (migrate) + + (datomic.dev-local/release-db {:db-name "prod-migration" + :system "dev"}) + + + (d/pull remote-db '[*] 17592271182081) + ;; => {:db/id 17592271182081, + ;; :charge/type-name "NO_SALE", + ;; :charge/total 0.0, + ;; :charge/tip 0.0, + ;; :charge/processor #:db{:id 17592237965415}, + ;; :charge/external-id "square/charge/ndHN8rnam4dsfnDtK78LL1KiuaB"} + (def local-charges + (into #{} + (map first) + (dc/q '[:find ?m + :where (or [?e :charge/total] + [?e :charge/external-id]) + [?e :entity/migration-key ?m]] + (dc/db (dc/connect local-client {:db-name "prod-migration"}))))) + + + (count local-charges) + + + ;; => [[2815787]] + (def remote-charges + (into #{} + (map first) + (d/q '[:find ?e + :where (or [?e :charge/total] + [?e :charge/external-id])] + remote-db))) + + ;; => [[2815955]] + + (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= "charge" (namespace %))) + (into [:db/id])) + (vec (clojure.set/difference remote-charges local-charges))) + + (dc/transact + (dc/connect local-client {:db-name "prod-migration"}) + {:tx-data (->> (d/pull-many remote-db + (->> schema + (filter :db/valueType) + (mapv :db/ident) + (filter #(= "charge" (namespace %))) + (into [:db/id])) + (vec (clojure.set/difference remote-charges local-charges))) + (mapv (fn [m ] + (reduce + (fn [m [k v]] + (cond + (= k :db/id) + (-> m + (assoc :entity/migration-key v) + (dissoc :db/id)) + (full-dependencies k) + (if (vector? v) + (assoc m k (mapv (fn [r] [:entity/migration-key (:db/id r)]) v)) + (assoc m k [:entity/migration-key (:db/id v)])) + :else + (dissoc m :payment/pdf-data + :payment/memo))) + m + m))))}) + + (dc/pull (dc/db (dc/connect local-client {:db-name "prod-migration"})) '[*] [:entity/migration-key 17592271182081]) + (d/pull remote-db '[*] 17592232186542) + + ) +;; => nil diff --git a/src/clj/auto_ap/datomic.clj b/src/clj/auto_ap/datomic.clj index e5baf03c..31131b6d 100644 --- a/src/clj/auto_ap/datomic.clj +++ b/src/clj/auto_ap/datomic.clj @@ -1,25 +1,31 @@ (ns auto-ap.datomic (:require - [auto-ap.utils :refer [default-pagination-size]] + [auto-ap.utils :refer [default-pagination-size by]] [clojure.tools.logging :as log] [config.core :refer [env]] - [datomic.api :as d] - [mount.core :as mount])) + [datomic.client.api :as dc] + [mount.core :as mount]) + (:import + (java.util UUID))) (def uri (:datomic-url env)) +(mount/defstate client + :start (dc/client {:server-type :dev-local + :system "dev"}) + :stop nil) (mount/defstate conn - :start (d/connect uri) - :stop (d/release conn)) + :start (dc/connect client {:db-name "prod-migration"}) + :stop nil) #_(def uri "datomic:mem://datomic-transactor:4334/invoice") #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(defn create-database [] +#_(defn create-database [] (d/create-database uri)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} -(defn drop-database [] +#_(defn drop-database [] (d/delete-database uri)) (defn remove-nils [m] @@ -35,26 +41,6 @@ result nil))) -(defn replace-nils-with-retract [updated original] - (let [result (reduce-kv - (fn [[m & retractions] k v] - (cond (and (nil? v) - (not (nil? (get original k)))) - (into [m] (conj retractions [:db/retract (:db/id original) k (or (:db/id (get original k)) - (get original k))])) - - (nil? v) - (into [m] retractions) - - :else - (into [(assoc m k v)] retractions))) - [{}] - updated)] - (if (seq result) - result - nil))) - - (def vendor-schema [{:db/ident :vendor/original-id :db/valueType :db.type/long @@ -101,8 +87,7 @@ :db/isComponent true :db/cardinality :db.cardinality/one :db/doc "The vendor's secondary contact"} - {:db/id #db/id[:db.part/db] - :db/ident :vendor/address + {:db/ident :vendor/address :db/valueType :db.type/ref :db/cardinality :db.cardinality/one :db/isComponent true @@ -588,6 +573,20 @@ q (:sort args))) +(defn add-sorter-fields-2 [q sort-map args] + (log/info "sort-map" (pr-str sort-map)) + (reduce + (fn [q {:keys [sort-key]}] + (merge-query q + {:query {:find [(last (last (sort-map + sort-key + (println "Warning, trying to sort by unsupported field" sort-key))))] + :where (sort-map + sort-key + (println "Warning, trying to sort by unsupported field" sort-key))}})) + q + (:sort args))) + (defn apply-sort-3 [args results] (let [sort-bys (conj (:sort args) {:sort-key "default" :asc (if (contains? args :default-asc?) @@ -625,8 +624,7 @@ :audit/user (str (:user/role id) "-" (:user/name id)) :audit/batch batch-id}) _ (log/info "transacting batch " batch-id " " (count batch)) - tx-result @(d/transact conn batch) - _ (Thread/sleep 100)] + tx-result (dc/transact conn {:tx-data batch})] (cond-> full-tx (:tx-data full-tx) (update :tx-data #(into % (:tx-data tx-result))) @@ -639,5 +637,255 @@ (partition-all 50 txes)))) (defn audit-transact [txes id] - @(d/transact conn (conj txes {:db/id "datomic.tx" - :audit/user (str (:user/role id) "-" (:user/name id))}))) + (dc/transact conn {:tx-data (conj txes {:db/id "datomic.tx" + :audit/user (str (:user/role id) "-" (:user/name id))})})) + +(defn pull-many [db read ids ] + (->> (dc/q '[:find (pull ?e r) + :in $ [?e ...] r] + db + ids + read) + (map first))) + +(defn pull-many-by-id [db read ids ] + (into {} + (map (fn [[e]] + [(:db/id e) e])) + (dc/q '[:find (pull ?e r) + :in $ [?e ...] r] + db + ids + read))) + +(defn random-tempid [] + (str (UUID/randomUUID))) + +(defn pull-attr [db k id] + (get (dc/pull db [k] id) k)) + +(defn pull-ref [db k id] + (:db/id (pull-attr db k id))) + +(declare upsert-entity) + +(defn reset-rels [db e a vs] + (assert (every? :db/id vs) (format "In order to reset attribute %s, every value must have :db/id" a)) + (let [ids (when-not (string? e) + (->> (dc/q '[:find ?z + :in $ ?e ?a + :where [?e ?a ?z]] + db e a) + (map first))) + new-id-set (set (map :db/id vs)) + retract-ids (filter (complement new-id-set) ids) + {is-component? :db/isComponent} (dc/pull db [:db/isComponent] a) + new-rels (filter (complement (set ids)) (map :db/id vs))] + (-> [] + (into (map (fn [i] (if is-component? + [:db/retractEntity i] + [:db/retract e a i ])) retract-ids)) + (into (map (fn [i] [:db/add e a (:db/id i)]) new-rels)) + (into (mapcat (fn [i] (upsert-entity db i)) vs))))) + +(defn reset-scalars [db e a vs] + + (let [extant (when-not (string? e) + (->> (dc/q '[:find ?z + :in $ ?e ?a + :where [?e ?a ?z]] + db e a) + (map first))) + retracts (filter (complement (set vs)) extant) + new (filter (complement (set extant)) vs)] + (-> [] + (into (map (fn [i] [:db/retract e a i ]) retracts)) + (into (map (fn [i] [:db/add e a i]) new))))) + + +(defn upsert-entity [db entity] + (assert (:db/id entity) "Cannot upsert without :db/id") + (let [e (:db/id entity) + is-new? (string? e) + extant-entity (when-not is-new? + (dc/pull db (keys entity) (:db/id entity))) + ident->value-type (by :db/ident (comp :db/ident + :db/valueType) + (pull-many + db + [:db/valueType :db/ident] + (keys entity))) + ops (->> entity + (reduce + (fn [ops [a v]] + (cond + (= :db/id a) + ops + + (or (= v (a extant-entity)) + (= v (:db/ident (a extant-entity) :nope)) + (= v (:db/id (a extant-entity)) :nope)) + ops + + (and (nil? v) + (not (nil? (a extant-entity)))) + (conj ops [:db/retract e a (cond-> (a extant-entity) + (:db/id (a extant-entity)) :db/id)]) + + (nil? v) + ops + + ;; reset relationships if it's refs, and not a lookup (i.e., seq of maps, or empty seq) + (and (sequential? v) (= :db.type/ref (ident->value-type a)) (every? map? v)) + (into ops (reset-rels db e a v)) + + (and (sequential? v) (not= :db.type/ref (ident->value-type a))) + (into ops (reset-scalars db e a v)) + + (and (map? v) + (= :db.type/ref (ident->value-type a))) + (let [id (or (:db/id v) (random-tempid))] + (-> ops + (conj [:db/add e a id]) + (into (upsert-entity db (assoc v :db/id id))))) + + :else + (conj ops [:db/add e a v]) + )) + []))] + ops)) + +(defn plus [db e a amount] + [[:db/add e a (-> (dc/pull db [a] e) a (+ amount))]]) + +(comment + (dc/pull (dc/db conn) '[*] 175921860633685) + + (upsert-entity (dc/db conn) {:db/id 175921860633685 :invoice/invoice-number nil :invoice/date #inst "2021-01-01" :invoice/expense-accounts [:reset-rels [{:db/id "new" :invoice-expense-account/amount 1}]]}) + + (upsert-entity (dc/db conn) {:invoice/client #:db{:id 79164837221949}, + :invoice/status #:db{:id 101155069755470, :ident :invoice-status/paid}, + :invoice/due #inst "2020-12-23T08:00:00.000-00:00", + :invoice/invoice-number "12648", + :invoice/import-status + :import-status/imported, + :invoice/vendor nil, + :invoice/date #inst "2020-12-16T08:00:00.000-00:00", + :entity/migration-key 17592234924273, + :db/id 175921860633685, + :invoice/outstanding-balance 0.0, + :invoice/expense-accounts + [{:entity/migration-key 17592234924274, + :invoice-expense-account/location nil + :invoice-expense-account/amount 360.0, + :invoice-expense-account/account #:db{:id 92358976759248}}],}) + + + + #_(dc/pull (dc/db conn) auto-ap.datomic.clients 79164837221904) + (upsert-entity (dc/db conn) {:client/name "20Twenty - WG Development LLC", + :client/square-locations + [{:db/id 83562883711605, + :entity/migration-key 17592258901782, + :square-location/square-id "L2579ATQ0X1ET", + :square-location/name "20Twenty", + :square-location/client-location "WG"}], + :client/square-auth-token + "EAAAEEr749Ea6AdPTdngsmUPwIM3ETbPwcx3QQl_NS0KWuIL-JNzAg4f3W9DGQhb", + :client/bank-accounts + [{:bank-account/sort-order 2, + :bank-account/include-in-reports true, + :bank-account/number "3467", + :bank-account/code "20TY-WFCC3467", + :bank-account/locations ["WG"], + :entity/migration-key 17592245102834, + :bank-account/current-balance 11160.289999999979, + :bank-account/name "Wells Fargo CC - 3467", + :db/id 83562883732805, + :bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00", + :bank-account/visible true, + :bank-account/type + #:db{:id 101155069755504, :ident :bank-account-type/credit}, + :bank-account/intuit-bank-account #:db{:id 105553116286744}, + :bank-account/integration-status + {:db/id 74766790691480, + :entity/migration-key 17592267080690, + :integration-status/last-updated #inst "2022-08-23T03:47:44.892-00:00", + :integration-status/last-attempt #inst "2022-08-23T03:47:44.892-00:00", + :integration-status/state + #:db{:id 101155069755529, :ident :integration-state/success}}, + :bank-account/bank-name "Wells Fargo"} + {:bank-account/sort-order 0, + :bank-account/include-in-reports true, + :bank-account/numeric-code 11301, + :bank-account/check-number 301, + :bank-account/number "1734742859", + :bank-account/code "20TY-WF2882", + :bank-account/locations ["WG"], + :bank-account/bank-code "11-4288/1210 4285", + :entity/migration-key 17592241193004, + :bank-account/current-balance -47342.54000000085, + :bank-account/name "Wells Fargo Main - 2859", + :db/id 83562883732846, + :bank-account/start-date #inst "2021-12-01T08:00:00.000-00:00", + :bank-account/visible true, + :bank-account/type + #:db{:id 101155069755468, :ident :bank-account-type/check}, + :bank-account/intuit-bank-account #:db{:id 105553116286745}, + :bank-account/routing "121042882", + :bank-account/integration-status + {:db/id 74766790691458, + :entity/migration-key 17592267080255, + :integration-status/last-updated #inst "2022-08-23T03:46:45.879-00:00", + :integration-status/last-attempt #inst "2022-08-23T03:46:45.879-00:00", + :integration-status/state + #:db{:id 101155069755529, :ident :integration-state/success}}, + :bank-account/bank-name "Wells Fargo"} + {:bank-account/sort-order 1, + :bank-account/include-in-reports true, + :bank-account/numeric-code 20101, + :bank-account/yodlee-account-id 27345526, + :bank-account/number "41006", + :bank-account/code "20TY-Amex41006", + :bank-account/locations ["WG"], + :entity/migration-key 17592241193006, + :bank-account/current-balance 9674.069999999963, + :bank-account/name "Amex - 41006", + :db/id 83562883732847, + :bank-account/visible true, + :bank-account/type + #:db{:id 101155069755504, :ident :bank-account-type/credit}, + :bank-account/bank-name "American Express"} + {:bank-account/sort-order 3, + :bank-account/include-in-reports true, + :bank-account/numeric-code 11101, + :bank-account/code "20TY-0", + :bank-account/locations ["WG"], + :entity/migration-key 17592241193005, + :bank-account/current-balance 0.0, + :bank-account/name "CASH", + :db/id 83562883732848, + :bank-account/visible true, + :bank-account/type + #:db{:id 101155069755469, :ident :bank-account-type/cash}}], + :entity/migration-key 17592241193003, + :db/id 79164837221904, + :client/address + {:db/id 105553116285906, + :entity/migration-key 17592250661126, + :address/street1 "1389 Lincoln Ave", + :address/city "San Jose", + :address/state "CA", + :address/zip "95125"}, + :client/code "NY", + :client/locations ["WE" "NG"], + :client/square-integration-status + {:db/id 74766790691447, + :entity/migration-key 17592267072653, + :integration-status/last-updated #inst "2022-08-23T13:09:16.082-00:00", + :integration-status/last-attempt #inst "2022-08-23T13:08:47.018-00:00", + :integration-status/state + #:db{:id 101155069755529, :ident :integration-state/success}}}) + + ) + diff --git a/src/clj/auto_ap/datomic/accounts.clj b/src/clj/auto_ap/datomic/accounts.clj index 3182578f..04d77335 100644 --- a/src/clj/auto_ap/datomic/accounts.clj +++ b/src/clj/auto_ap/datomic/accounts.clj @@ -1,9 +1,9 @@ (ns auto-ap.datomic.accounts (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]] + :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query conn pull-many]] [clojure.string :as str] - [datomic.api :as d] + [datomic.client.api :as dc] [clojure.tools.logging :as log])) (defn <-datomic [a] @@ -37,21 +37,21 @@ (let [query (cond-> {:query {:find [(list 'pull '?e default-read)] :in ['$] :where [['?e :account/name]]} - :args [(d/db (d/connect uri))]} + :args [(dc/db conn)]} (:account-set args) (merge-query {:query {:in ['?account-set] :where [['?e :account/account-set '?account-set]]} :args [(:account-set args)]}))] (->> - (d/query query) + (dc/q query) (map first) (map <-datomic))))) (defn get-by-id [id] (let [query {:query {:find [(list 'pull '?e default-read)] :in ['$ '?e]} - :args [(d/db (d/connect uri) ) id]}] + :args [(dc/db conn ) id]}] (->> - (d/query query) + (dc/q query) (map first) (map <-datomic) first))) @@ -59,38 +59,40 @@ (defn get-for-vendor [vendor-id client-id] (if client-id (->> - (d/q [:find (list 'pull '?e default-read) - :in '$ '?v '?c - :where '(or-join [?v ?c ?e] - (and [?v :vendor/account-overrides ?ao] - [?ao :vendor-account-override/client ?c] - [?ao :vendor-account-override/account ?e]) - (and [?v :vendor/account-overrides ?ao] - (not [?ao vendor-account-override/client ?c]) - [?v :vendor/default-account ?e]) - (and (not [?v :vendor/account-overrides]) - [?v :vendor/default-account ?e]))] + (dc/q '[:find (pull ?e r) + :in $ ?v ?c r + :where (or-join [?v ?c ?e] + (and [?v :vendor/account-overrides ?ao] + [?ao :vendor-account-override/client ?c] + [?ao :vendor-account-override/account ?e]) + (and [?v :vendor/account-overrides ?ao] + (not [?ao :vendor-account-override/client ?c]) + [?v :vendor/default-account ?e]) + (and (not [?v :vendor/account-overrides]) + [?v :vendor/default-account ?e]))] - (d/db conn ) + (dc/db conn ) vendor-id - client-id) + client-id + default-read) (map first) (map <-datomic) first) - (d/q [:find (list 'pull '?e default-read) - :in '$ '?v - :where '[?v :vendor/default-account ?e]] + (<-datomic (dc/q '[:find (pull ?e r) + :in $ ?v r + :where [?v :vendor/default-account ?e]] - (d/db conn ) - vendor-id))) + (dc/db conn ) + vendor-id + default-read)))) (defn get-account-by-numeric-code-and-sets [numeric-code _] (let [query (cond-> {:query {:find ['(pull ?e [* {:account/type [:db/ident :db/id]}])] :in ['$ '?numeric-code] :where ['[?e :account/numeric-code ?numeric-code]]} - :args [(d/db (d/connect uri)) numeric-code]})] + :args [(dc/db conn) numeric-code]})] (->> - (d/query query) + (dc/q query) (map first) (map <-datomic) (first)))) @@ -121,13 +123,13 @@ (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) accounts (->> ids (map results) @@ -137,7 +139,7 @@ (defn get-graphql [args] (log/info "ARGS" args) - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count])) diff --git a/src/clj/auto_ap/datomic/bank_accounts.clj b/src/clj/auto_ap/datomic/bank_accounts.clj index 69aa2cab..f9998784 100644 --- a/src/clj/auto_ap/datomic/bank_accounts.clj +++ b/src/clj/auto_ap/datomic/bank_accounts.clj @@ -1,7 +1,7 @@ (ns auto-ap.datomic.bank-accounts (:require - [auto-ap.datomic :refer [uri]] - [datomic.api :as d])) + [auto-ap.datomic :refer [conn]] + [datomic.client.api :as dc])) (defn add-arg [query name value where & rest] (let [query (-> query @@ -21,10 +21,10 @@ (defn get-by-id [id] (->> - (d/query (-> {:query {:find [default-read] + (dc/q (-> {:query {:find [default-read] :in ['$] :where []} - :args [(d/db (d/connect uri))]} + :args [(dc/db conn)]} (add-arg '?e id ['?e]))) (map first) (<-datomic) diff --git a/src/clj/auto_ap/datomic/checks.clj b/src/clj/auto_ap/datomic/checks.clj index ac86fcba..eb115db4 100644 --- a/src/clj/auto_ap/datomic/checks.clj +++ b/src/clj/auto_ap/datomic/checks.clj @@ -1,10 +1,17 @@ (ns auto-ap.datomic.checks - (:require [datomic.api :as d] - [auto-ap.datomic :refer [merge-query apply-sort-3 apply-pagination add-sorter-fields conn]] - [auto-ap.graphql.utils :refer [limited-clients]] - [clojure.set :refer [rename-keys]] - [clj-time.coerce :as c] - [clojure.tools.logging :as log])) + (:require + [auto-ap.datomic + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-many]] + [auto-ap.graphql.utils :refer [limited-clients]] + [clj-time.coerce :as c] + [datomic.client.api :as dc] + [clojure.set :refer [rename-keys]] + [clojure.tools.logging :as log])) (defn <-datomic [result] (-> result @@ -28,7 +35,7 @@ {:transaction/_payment [:db/id :transaction/date]}]) (defn raw-graphql-ids - ([args] (raw-graphql-ids (d/db conn) args)) + ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] (let [check-number-like (try (Long/parseLong (:check-number-like args)) (catch Exception _ nil)) query (cond-> {:query {:find [] @@ -148,12 +155,12 @@ (log/info query) (cond->> query - true (d/query) - true (apply-sort-3 args) + true (dc/q) + true (apply-sort-3 (assoc args :default-asc? false)) true (apply-pagination args))))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) payments (->> ids (map results) @@ -163,7 +170,7 @@ (defn get-graphql [args] (log/info (:id args)) - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) @@ -174,13 +181,21 @@ (->> {:query {:find ['?e] :in ['$ '[?e ...]] :where ['[?e :payment/date]]} - :args [(d/db conn) ids]} - (d/query) + :args [(dc/db conn) ids]} + (dc/q) (map first) vec) [])) (defn get-by-id [id] (->> - (d/pull (d/db conn) default-read id) + (dc/pull (dc/db conn) default-read id) (<-datomic))) + +(defn pay [db e amount] + (let [current-outstanding-balance (-> (dc/pull db [:invoice/outstanding-balance] e) :invoice/outstanding-balance) + new-outstanding-balance (- current-outstanding-balance amount)] + [[:db/add e :invoice/outstanding-balance new-outstanding-balance] + [:db/add e :invoice/status (if (> new-outstanding-balance 0) + :invoice-status/unpaid + :invoice-status/paid)]])) diff --git a/src/clj/auto_ap/datomic/clients.clj b/src/clj/auto_ap/datomic/clients.clj index b899f28c..c181c437 100644 --- a/src/clj/auto_ap/datomic/clients.clj +++ b/src/clj/auto_ap/datomic/clients.clj @@ -1,8 +1,10 @@ (ns auto-ap.datomic.clients (:require - [auto-ap.datomic :refer [conn uri]] + [auto-ap.datomic :refer [conn]] + [auto-ap.search :as search] [clj-time.coerce :as coerce] - [datomic.api :as d])) + [clojure.string :as str] + [datomic.client.api :as dc])) (defn cleanse [e] (-> e @@ -27,7 +29,7 @@ (update :bank-account/sort-order (fn [so] (or so i))))) (range) bas))))) (defn get-all [] - (->> (d/q '[:find (pull ?e [* + (->> (dc/q '[:find (pull ?e [* {:client/square-integration-status [:integration-status/message :integration-status/last-attempt :integration-status/last-updated @@ -56,14 +58,13 @@ {:plaid-item/_client [*]} {:client/emails [:db/id :email-contact/email :email-contact/description]}]) :where [?e :client/name]] - (d/db (d/connect uri))) + (dc/db conn)) (map first) (map cleanse))) (defn get-by-id [id] - (->> - (d/pull (d/db conn ) + (dc/pull (dc/db conn ) '[* {:client/bank-accounts [* {:bank-account/type [*] :bank-account/yodlee-account [:yodlee-account/name :yodlee-account/id :yodlee-account/number] :bank-account/intuit-bank-account [:intuit-bank-account/name :intuit-bank-account/external-id :db/id] @@ -79,9 +80,28 @@ (defn code->id [code] (->> - (d/query (-> {:query {:find ['?e] + (dc/q (-> {:query {:find ['?e] :in ['$ '?code] :where [['?e :client/code '?code ]]} - :args [(d/db (d/connect uri)) code]})) + :args [(dc/db conn) code]})) (first) (first))) + +(defn best-match [identifier] + (first (search/search-ids {:q (str/replace identifier #"[\(\)\-/\*\]\[\#:\&]" " ")} "client"))) + +(defn exact-match [identifier] + (first (search/search-ids {:exact-match (str/upper-case identifier)} "client"))) + +(defn rebuild-search-index [] + (search/full-index-query + (for [result (map first (dc/qseq '[:find (pull ?v [:client/name :client/matches :db/id]) + :in $ + :where [?v :client/code]] + (dc/db conn))) + match (conj (or (:client/matches result) []) + (:client/name result))] + {:id (:db/id result) + :text match + :exact-match (str/upper-case match)}) + "client")) diff --git a/src/clj/auto_ap/datomic/expected_deposit.clj b/src/clj/auto_ap/datomic/expected_deposit.clj index 8518060b..fc57fb10 100644 --- a/src/clj/auto_ap/datomic/expected_deposit.clj +++ b/src/clj/auto_ap/datomic/expected_deposit.clj @@ -1,10 +1,10 @@ (ns auto-ap.datomic.expected-deposit (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]] + :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many]] [auto-ap.graphql.utils :refer [limited-clients]] [clj-time.coerce :as c] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn <-datomic [result] (let [transaction (first (:transaction/_expected-deposit result)) @@ -88,12 +88,12 @@ :where ['[?e :expected-deposit/date ?sort-default]]}}))] (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) payments (->> ids (map results) @@ -101,7 +101,7 @@ (mapv <-datomic) (map (fn get-totals [ed] (assoc ed :totals - (->> (d/q '[:find ?d4 (count ?c) (sum ?a) + (->> (dc/q '[:find ?d4 (count ?c) (sum ?a) :in $ ?ed :where [?ed :expected-deposit/charges ?c] [?c :charge/total ?a] @@ -110,7 +110,7 @@ [(clj-time.coerce/from-date ?d) ?d2] [(auto-ap.time/localize ?d2) ?d3] [(clj-time.coerce/to-local-date ?d3) ?d4]] - (d/db conn) + (dc/db conn) (:db/id ed)) (map (fn [[date count amount]] {:date (c/to-date-time date) @@ -119,7 +119,7 @@ payments)) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) diff --git a/src/clj/auto_ap/datomic/invoices.clj b/src/clj/auto_ap/datomic/invoices.clj index 2e967fd9..be0f46d5 100644 --- a/src/clj/auto_ap/datomic/invoices.clj +++ b/src/clj/auto_ap/datomic/invoices.clj @@ -1,7 +1,13 @@ (ns auto-ap.datomic.invoices (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]] + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-many + remove-nils]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [limited-clients]] @@ -9,7 +15,7 @@ [clj-time.coerce :as coerce] [clj-time.core :as time] [clojure.set :refer [rename-keys]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (def default-read '[* {:invoice/client [:client/name :db/id :client/locations :client/code]} @@ -37,7 +43,7 @@ (defn raw-graphql-ids ([args] - (raw-graphql-ids (d/db conn) args)) + (raw-graphql-ids (dc/db conn) args)) ([db args] (->> (cond-> {:query {:find [] :in ['$] @@ -164,13 +170,13 @@ (merge-query {:query {:find ['?sort-default '?e ] :where ['[?e :invoice/client] '[?e :invoice/date ?sort-default]]}}) ) - (d/query) + (dc/q) (apply-sort-3 args) (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) invoices (->> ids @@ -182,11 +188,11 @@ (defn sum-outstanding [ids] (->> - (d/query {:query {:find ['?id '?o] + (dc/q {:query {:find ['?id '?o] :in ['$ '[?id ...]] :where ['[?id :invoice/outstanding-balance ?o]] } - :args [(d/db conn) + :args [(dc/db conn) ids]}) (map last) (reduce @@ -196,11 +202,11 @@ (defn sum-total-amount [ids] (->> - (d/query {:query {:find ['?id '?o] + (dc/q {:query {:find ['?id '?o] :in ['$ '[?id ...]] :where ['[?id :invoice/total ?o]] } - :args [(d/db conn) + :args [(dc/db conn) ids]}) (map last) (reduce @@ -209,7 +215,7 @@ (defn get-graphql [args] - (let [db (d/db (d/connect uri)) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args) outstanding (sum-outstanding ids-to-retrieve) total-amount (sum-total-amount ids-to-retrieve)] @@ -221,34 +227,34 @@ total-amount])) (defn get-by-id [id] - (-> (d/db (d/connect uri)) - (d/pull default-read id) + (-> (dc/db conn) + (dc/pull default-read id) (<-datomic))) (defn get-multi [ids] (map <-datomic - (-> (d/db (d/connect uri)) - (d/pull-many default-read ids)))) + (pull-many (dc/db conn) default-read ids ))) (defn find-conflicting [{:keys [:invoice/invoice-number :invoice/vendor :invoice/client :db/id]}] - - (->> (d/query - (cond-> {:query {:find [(list 'pull '?e default-read)] - :in ['$ '?invoice-number '?vendor '?client '?invoice-id] - :where '[[?e :invoice/invoice-number ?invoice-number] - [?e :invoice/vendor ?vendor] - [?e :invoice/client ?client] - (not [?e :invoice/status :invoice-status/voided]) - [(not= ?e ?invoice-id)]]} - :args [(d/db (d/connect uri)) invoice-number vendor client (or id 0)]})) + + (->> (dc/q + (cond-> {:query {:find [(list 'pull '?e default-read)] + :in ['$ '?invoice-number '?vendor '?client '?invoice-id] + :where '[[?e :invoice/invoice-number ?invoice-number] + [?e :invoice/vendor ?vendor] + [?e :invoice/client ?client] + (not [?e :invoice/status :invoice-status/voided]) + [(not= ?e ?invoice-id)]]} + + :args [(dc/db conn) invoice-number vendor client (or id 0)]})) (map first) (map <-datomic))) (defn get-existing-set [] - (let [vendored-results (set (d/query {:query {:find ['?vendor '?client '?invoice-number] + (let [vendored-results (set (dc/q {:query {:find ['?vendor '?client '?invoice-number] :in ['$] :where '[[?e :invoice/invoice-number ?invoice-number] [?e :invoice/vendor ?vendor] @@ -256,8 +262,8 @@ (not [?e :invoice/status :invoice-status/voided]) ]} - :args [(d/db (d/connect uri))]})) - vendorless-results (->> (d/query {:query {:find ['?client '?invoice-number] + :args [(dc/db conn)]})) + vendorless-results (->> (dc/q {:query {:find ['?client '?invoice-number] :in ['$] :where '[[?e :invoice/invoice-number ?invoice-number] (not [?e :invoice/vendor]) @@ -265,7 +271,7 @@ (not [?e :invoice/status :invoice-status/voided]) ]} - :args [(d/db (d/connect uri))]}) + :args [(dc/db conn)]}) (mapv (fn [[client invoice-number]] [nil client invoice-number]) ) set)] @@ -276,37 +282,37 @@ (->> {:query {:find ['?e] :in ['$ '[?e ...]] :where ['[?e :invoice/date]]} - :args [(d/db conn) ids]} - (d/query) + :args [(dc/db conn) ids]} + (dc/q) (map first) vec) [])) (defn code-invoice [invoice] - (let [db (d/db auto-ap.datomic/conn) + (let [db (dc/db auto-ap.datomic/conn) client-id (:invoice/client invoice) vendor-id (:invoice/vendor invoice) date (:invoice/date invoice) - vendor (d/pull db '[*] vendor-id) + vendor (dc/pull db '[*] vendor-id) due (when (:vendor/terms vendor) (-> date (coerce/to-date-time) (time/plus (time/days (d-vendors/terms-for-client-id vendor client-id))) coerce/to-date)) - automatically-paid? (boolean (seq (d/q '[:find [?c ...] - :in $ ?v ?c - :where [?v :vendor/automatically-paid-when-due ?c]] + automatically-paid? (boolean (seq (map first (dc/q '[:find ?c + :in $ ?v ?c + :where [?v :vendor/automatically-paid-when-due ?c]] + db + vendor-id + client-id)))) + [schedule-payment-dom] (map first (dc/q '[:find ?dom + :in $ ?v ?c + :where [?v :vendor/schedule-payment-dom ?sp ] + [?sp :vendor-schedule-payment-dom/client ?c] + [?sp :vendor-schedule-payment-dom/dom ?dom]] db vendor-id - client-id))) - [schedule-payment-dom] (d/q '[:find [?dom ...] - :in $ ?v ?c - :where [?v :vendor/schedule-payment-dom ?sp ] - [?sp :vendor-schedule-payment-dom/client ?c] - [?sp :vendor-schedule-payment-dom/dom ?dom]] - db - vendor-id - client-id) + client-id)) scheduled-payment (cond automatically-paid? due @@ -324,3 +330,19 @@ true (assoc :invoice/expense-accounts [default-expense-account]) due (assoc :invoice/due due) scheduled-payment (assoc :invoice/scheduled-payment scheduled-payment)))) + +(defn propose-invoice [db invoice] + (let [existing? (boolean (seq (dc/q '[:find ?i + :in $ ?invoice-number ?client ?vendor + :where + [?i :invoice/invoice-number ?invoice-number] + [?i :invoice/client ?client] + [?i :invoice/vendor ?vendor] + (not [?i :invoice/status :invoice-status/voided])] + db + (:invoice/invoice-number invoice) + (:invoice/client invoice) + (:invoice/vendor invoice))))] + (if existing? + [] + [(doto (remove-nils invoice) println)]))) diff --git a/src/clj/auto_ap/datomic/ledger.clj b/src/clj/auto_ap/datomic/ledger.clj index 5965e56d..1e60f97c 100644 --- a/src/clj/auto_ap/datomic/ledger.clj +++ b/src/clj/auto_ap/datomic/ledger.clj @@ -1,9 +1,16 @@ (ns auto-ap.datomic.ledger - (:require [datomic.api :as d] - [auto-ap.graphql.utils :refer [limited-clients]] - [auto-ap.datomic :refer [merge-query apply-sort-3 apply-pagination add-sorter-fields conn]] - [clj-time.coerce :as c] - [auto-ap.datomic.accounts :as d-accounts])) + (:require + [auto-ap.datomic + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-many]] + [auto-ap.datomic.accounts :as d-accounts] + [auto-ap.graphql.utils :refer [limited-clients]] + [clj-time.coerce :as c] + [datomic.client.api :as dc])) (defn raw-graphql-ids [db args] (let [query (cond-> {:query {:find [] @@ -120,12 +127,12 @@ true (merge-query {:query {:find ['?sort-default '?e] :where ['[?e :journal-entry/date ?sort-default]]}}))] (->> query - (d/query) + (dc/q) (apply-sort-3 (update args :sort conj {:sort-key "default-2" :asc true})) (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db '[* {:journal-entry/client [:client/name :client/code :db/id] + (let [results (->> (pull-many db '[* {:journal-entry/client [:client/name :client/code :db/id] :journal-entry/vendor [:vendor/name :db/id] :journal-entry/line-items [* {:journal-entry-line/account [* {:account/type [*]} @@ -154,7 +161,7 @@ (map first)))) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count])) @@ -164,8 +171,8 @@ (->> {:query {:find ['?e] :in ['$ '[?e ...]] :where ['[?e :journal-entry/date]]} - :args [(d/db conn) ids]} - (d/query) + :args [(dc/db conn) ids]} + (dc/q) (map first) vec) [])) diff --git a/src/clj/auto_ap/datomic/reports.clj b/src/clj/auto_ap/datomic/reports.clj index e4e383dd..4bc55309 100644 --- a/src/clj/auto_ap/datomic/reports.clj +++ b/src/clj/auto_ap/datomic/reports.clj @@ -1,10 +1,15 @@ (ns auto-ap.datomic.reports (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]] + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-many]] [auto-ap.graphql.utils :refer [can-see-client? limited-clients]] [clj-time.coerce :as c] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn raw-graphql-ids [db args] (let [query (cond-> {:query {:find [] @@ -33,12 +38,12 @@ true (merge-query {:query {:find ['?sort-default '?e] :where ['[?e :report/created ?sort-default]]}}))] (->> query - (d/query) + (dc/q) (apply-sort-3 (update args :sort conj {:sort-key "default-2" :asc true})) (apply-pagination args)))) (defn graphql-results [ids db args] - (let [results (->> (d/pull-many db '[:db/id :report/client :report/created :report/url :report/name :report/creator] + (let [results (->> (pull-many db '[:db/id :report/client :report/created :report/url :report/name :report/creator] ids) (map #(update % :report/created c/from-date)) (group-by :db/id))] @@ -53,7 +58,7 @@ (map :db/id (:report/client r)))))))) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) diff --git a/src/clj/auto_ap/datomic/sales_orders.clj b/src/clj/auto_ap/datomic/sales_orders.clj index 522867d1..adc3125c 100644 --- a/src/clj/auto_ap/datomic/sales_orders.clj +++ b/src/clj/auto_ap/datomic/sales_orders.clj @@ -1,10 +1,18 @@ (ns auto-ap.datomic.sales-orders - (:require [auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]] - [auto-ap.graphql.utils :refer [limited-clients]] - [clj-time.coerce :as c] - [datomic.api :as d] - [clojure.tools.logging :as log] - [clojure.set :as set])) + (:require + [auto-ap.datomic + :refer [add-sorter-fields-2 + apply-pagination + apply-sort-3 + merge-query + pull-many + conn]] + [auto-ap.graphql.utils :refer [limited-clients]] + [clj-time.coerce :as c] + [clojure.set :as set] + [clojure.tools.logging :as log] + [datomic.client.api :as dc] + )) (defn <-datomic [result] (-> result @@ -21,12 +29,96 @@ {:sales-order/client [:client/name :db/id :client/code] :sales-order/charges [* {:charge/processor [:db/ident]} {:expected-deposit/_charges [:db/id]}]}]) +#_(defn client-index-search [db attr client-ids [start end]] + (for [c client-ids + d + (dc/q '[:find ?e + :in $ ?start ?end + :where [?e :sales-order/client+date ?g] + [(>= ?g ?start)] + [(<= ?g ?end)]] + db [c start] [c end]) + + #_(dc/index-range db {:attrid attr + :start [c start] + :end [c end]})] + (first d))) + +#_(comment + (do + (println "starting") + (doseq [n (partition-all 500 (dc/q '[:find ?s ?d + :where [?s :sales-order/date ?d]] + (dc/db conn)))] + (print ".") + (dc/transact conn {:tx-data (map (fn [[n d]] {:db/id n :sales-order/date d}) n)})) + (println "done")) + + + (time (count (client-index-search (dc/db conn) + :sales-order/client+date + [3575611821801781] + [#inst "2021-05-21T18:06:02.000-00:00" + #inst "2022-09-25T18:06:02.000-00:00"] + ))) + + (time (count (dc/q '[:find ?e + :in $ ?client ?start ?end + :where [?e :sales-order/client ?client] + [?e :sales-order/date ?date] + [(>= ?date ?start)] + [(<= ?date ?end)]] + (dc/db conn) + 3575611821801781 + + #inst "2021-05-21T18:06:02.000-00:00" + #inst "2022-09-25T18:06:02.000-00:00" + ))) + + (time (do (count (dc/q '[:find ?e, + :in $ [[?s1 ?e1]] + :where + #_[(untuple ?period) [?s1 ?e1]] + [(q '[:find ?e + :in $ ?s1 ?e1 + :where [?e :sales-order/client+date ?g] + [(>= ?g ?s1)] + [(<= ?g ?e1)]] + $ ?s1 ?e1) [?e ...]] + ] + (dc/db conn) [[[3575611821801781 #inst "2022-08-21T18:06:02.000-00:00" ] [3575611821801781 #inst "2022-09-25T18:06:02.000-00:00"]] + [[3575611821801766 #inst "2022-08-21T18:06:02.000-00:00" ] [3575611821801766 #inst "2022-09-25T18:06:02.000-00:00"]]])) + (println "done"))) + + (dc/index-range (dc/db conn) {:attrid :sales-order/date+client3 + :start [#inst "2022-08-25T18:06:02.000-00:00" ] + :end [#inst "2022-08-25T20:06:02.000-00:00" ]}) + (dc/pull (dc/db conn) '[*] :sales-order/date+client3) + (dc/pull (dc/db conn) '[*] 5040161313201309) + + #_(time (do (dc/q {:query '{:find [(count ?x)], + :in [$ ?client-id], + :where [[?e :sales-order/date+client3 ?x]]}, + :args [(dc/db conn) 3575611821801781]}) + (println "done"))) + + #_(datomic.dev-local.async/return-1) + + (clojure.core.async/ {:query {:find [] :in ['$] :where []} :args [db]} - (:sort args) (add-sorter-fields {"client" ['[?e :sales-order/client ?c] + (:sort args) (add-sorter-fields-2 {"client" ['[?e :sales-order/client ?c] '[?c :client/name ?sort-client]] "location" ['[?e :sales-order/location ?sort-location]] "source" ['[?e :sales-order/source ?sort-source]] @@ -100,17 +192,17 @@ :args [(:total args)]}) true - (merge-query {:query {:find ['?sort-default '?e] - :where ['[?e :sales-order/date ?sort-default]]}}))] + (merge-query {:query {:find ['?date '?e] + :where ['[?e :sales-order/date ?date]]}}))] (log/info "Sales query" query) (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 (assoc args :default-asc? false)) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) payments (->> ids (map results) @@ -121,19 +213,19 @@ (defn summarize-orders [ids] (let [[total tax] (->> - (d/query {:query {:find ['(sum ?t) '(sum ?tax)] + (dc/q {:query {:find ['(sum ?t) '(sum ?tax)] :with ['?id] :in ['$ '[?id ...]] :where ['[?id :sales-order/total ?t] '[?id :sales-order/tax ?tax]]} - :args [(d/db (d/connect uri)) + :args [(dc/db conn) ids]}) first)] {:total total :tax tax})) (defn get-graphql [args] - (let [db (d/db (d/connect uri)) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count diff --git a/src/clj/auto_ap/datomic/transaction_rules.clj b/src/clj/auto_ap/datomic/transaction_rules.clj index 074fa7e3..dae7d040 100644 --- a/src/clj/auto_ap/datomic/transaction_rules.clj +++ b/src/clj/auto_ap/datomic/transaction_rules.clj @@ -1,10 +1,15 @@ (ns auto-ap.datomic.transaction-rules (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]] + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-many]] [auto-ap.graphql.utils :refer [limited-clients]] [clojure.string :as str] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn <-datomic [result] result) @@ -72,12 +77,12 @@ (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) transaction-rules (->> ids (map results) @@ -86,19 +91,19 @@ transaction-rules)) (defn get-graphql [args] - (let [db (d/db (d/connect uri)) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count])) (defn get-by-id [id] (->> - (d/pull (d/db (d/connect uri)) default-read id) + (dc/pull (dc/db conn) default-read id) (<-datomic))) (defn get-all [] (mapv first - (d/query {:query {:find [(list 'pull '?e default-read )] - :in ['$] - :where ['[?e :transaction-rule/transaction-approval-status]]} - :args [(d/db (d/connect uri))]}))) + (dc/q {:query {:find [(list 'pull '?e default-read )] + :in ['$] + :where ['[?e :transaction-rule/transaction-approval-status]]} + :args [(dc/db conn)]}))) diff --git a/src/clj/auto_ap/datomic/transactions.clj b/src/clj/auto_ap/datomic/transactions.clj index 424edbe8..edfe452b 100644 --- a/src/clj/auto_ap/datomic/transactions.clj +++ b/src/clj/auto_ap/datomic/transactions.clj @@ -1,18 +1,18 @@ (ns auto-ap.datomic.transactions (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query uri]] + :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query conn pull-many]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.graphql.utils :refer [limited-clients]] [clj-time.coerce :as coerce] [clojure.string :as str] [clojure.tools.logging :as log] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn potential-duplicate-ids [db args] (when (and (:potential-duplicates args) (:bank-account-id args)) - (->> (d/q '[:find ?tx ?amount ?date + (->> (dc/q '[:find ?tx ?amount ?date :in $ ?ba :where [?tx :transaction/bank-account ?ba] @@ -33,7 +33,7 @@ (defn raw-graphql-ids - ([args] (raw-graphql-ids (d/db conn) args)) + ([args] (raw-graphql-ids (dc/db conn) args)) ([db args] (let [potential-duplicates (potential-duplicate-ids db args) query (cond-> {:query {:find [] @@ -169,28 +169,28 @@ '(not [?e :transaction/approval-status :transaction-approval-status/suppressed])]}}))] (log/info "query is" query) (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 (assoc args :default-asc? false)) true (apply-pagination args))))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db '[* {:transaction/client [:client/name :db/id :client/code] - :transaction/approval-status [:db/ident :db/id] - :transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance] - :transaction/forecast-match [:db/id :forecasted-transaction/identifier] - :transaction/vendor [:db/id :vendor/name] - :transaction/matched-rule [:db/id :transaction-rule/note] - :transaction/payment [:db/id :payment/date] - :transaction/expected-deposit [:db/id :expected-deposit/date] - :transaction/accounts [:transaction-account/amount - :db/id - :transaction-account/location - {:transaction-account/account [:account/name :db/id - :account/location - {:account/client-overrides [:account-client-override/name - {:account-client-override/client [:db/id]}]}]}] - :transaction/yodlee-merchant [:db/id :yodlee-merchant/yodlee-id :yodlee-merchant/name]}] - ids) + (let [results (->> (pull-many db '[* {:transaction/client [:client/name :db/id :client/code] + :transaction/approval-status [:db/ident :db/id] + :transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance] + :transaction/forecast-match [:db/id :forecasted-transaction/identifier] + :transaction/vendor [:db/id :vendor/name] + :transaction/matched-rule [:db/id :transaction-rule/note] + :transaction/payment [:db/id :payment/date] + :transaction/expected-deposit [:db/id :expected-deposit/date] + :transaction/accounts [:transaction-account/amount + :db/id + :transaction-account/location + {:transaction-account/account [:account/name :db/id + :account/location + {:account/client-overrides [:account-client-override/name + {:account-client-override/client [:db/id]}]}]}] + :transaction/yodlee-merchant [:db/id :yodlee-merchant/yodlee-id :yodlee-merchant/name]}] + ids) (map #(update % :transaction/date coerce/from-date)) (map #(update % :transaction/post-date coerce/from-date)) (map #(update % :transaction/accounts @@ -203,7 +203,7 @@ (cond-> transaction (:transaction/payment transaction) (update-in [:transaction/payment :payment/date] coerce/from-date) (:transaction/expected-deposit transaction) (update-in [:transaction/expected-deposit :expected-deposit/date] coerce/from-date)) - )) + )) (map #(dissoc % :transaction/id)) (group-by :db/id))] @@ -212,7 +212,7 @@ (map first)))) (defn get-graphql [args] - (let [db (d/db (d/connect uri)) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) @@ -223,15 +223,15 @@ (->> {:query {:find ['?e] :in ['$ '[?e ...]] :where ['[?e :transaction/date]]} - :args [(d/db conn) ids]} - (d/query) + :args [(dc/db conn) ids]} + (dc/q) (map first) vec) [])) (defn get-by-id [id] (-> - (d/pull (d/db (d/connect uri)) + (dc/pull (dc/db conn) '[* {:transaction/client [:client/name :db/id :client/code :client/locations] :transaction/approval-status [:db/ident :db/id] :transaction/bank-account [:bank-account/name :bank-account/code :bank-account/yodlee-account-id :db/id :bank-account/locations :bank-account/current-balance] diff --git a/src/clj/auto_ap/datomic/users.clj b/src/clj/auto_ap/datomic/users.clj index c2ac8c4b..cade604f 100644 --- a/src/clj/auto_ap/datomic/users.clj +++ b/src/clj/auto_ap/datomic/users.clj @@ -1,7 +1,7 @@ (ns auto-ap.datomic.users (:require - [auto-ap.datomic :refer [uri]] - [datomic.api :as d])) + [auto-ap.datomic :refer [conn]] + [datomic.client.api :as dc])) (defn add-arg [query name value where & rest] (let [query (-> query @@ -16,35 +16,35 @@ {:user/role [:db/ident]}])] :in ['$] :where []} - :args [(d/db (d/connect uri))]} + :args [(dc/db conn)]} (add-arg '?e id ['?e]))] - (->> (d/query query) + (->> (dc/q query) (map first) (map #(update % :user/role :db/ident)) first))) (defn find-or-insert! [{:keys [:user/provider :user/provider-id] :as new-user}] - (let [is-first-user? (not (seq (d/query {:query [:find '?e + (let [is-first-user? (not (seq (dc/q {:query [:find '?e :in '$ :where '[?e :user/provider]] - :args [(d/db (d/connect uri))]}))) + :args [(dc/db conn)]}))) user (some-> {:query [:find '(pull ?e [* {:user/clients [*]} {:user/role [:db/ident]}]) :in '$ '?provider '?provider-id :where '[?e :user/provider ?provider] '[?e :user/provider-id ?provider-id]] - :args [(d/db (d/connect uri)) provider provider-id]} - (d/query) + :args [(dc/db conn) provider provider-id]} + (dc/q) first first (update :user/role :db/ident))] (if user user - (let [new-user-trans @(d/transact (d/connect uri) [(cond-> new-user - true (assoc :db/id "user") - is-first-user? (assoc :user/role :user-role/admin))])] + (let [new-user-trans (dc/transact conn {:tx-data [(cond-> new-user + true (assoc :db/id "user") + is-first-user? (assoc :user/role :user-role/admin))]})] (get-by-id (-> new-user-trans :tempids (get "user"))))))) (defn raw-graphql [_] @@ -53,9 +53,9 @@ {:user/role [:db/ident]}])] :in ['$] :where ['[?e :user/role]]} - :args [(d/db (d/connect uri))]})] + :args [(dc/db conn)]})] - (->> (d/query query) + (->> (dc/q query) (map first) (map #(update % :user/role :db/ident)) ))) diff --git a/src/clj/auto_ap/datomic/vendors.clj b/src/clj/auto_ap/datomic/vendors.clj index c72998d9..7a267de5 100644 --- a/src/clj/auto_ap/datomic/vendors.clj +++ b/src/clj/auto_ap/datomic/vendors.clj @@ -1,9 +1,9 @@ (ns auto-ap.datomic.vendors (:require - [auto-ap.datomic :refer [conn merge-query uri add-sorter-fields apply-pagination merge-query apply-sort-3]] + [auto-ap.datomic :refer [conn merge-query add-sorter-fields apply-pagination merge-query apply-sort-3 pull-many]] [auto-ap.graphql.utils :refer [limited-clients]] [clojure.string :as str] - [datomic.api :as d] + [datomic.client.api :as dc] [auto-ap.datomic.accounts :as d-accounts])) (defn <-datomic [a] @@ -63,7 +63,7 @@ (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) @@ -83,7 +83,7 @@ )) (defn graphql-results [ids db args] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id)) vendors (->> ids (map results) @@ -93,7 +93,7 @@ vendors)) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count]) @@ -104,8 +104,8 @@ (->> (cond-> {:query {:find [(list 'pull '?e default-read)] :in ['$ '?e] :where ['[?e :vendor/name]]} - :args [(d/db (d/connect uri)) id]}) - (d/query) + :args [(dc/db conn) id]}) + (dc/q) (map first) (map #(cleanse (:id args) %)) (map <-datomic) @@ -114,7 +114,7 @@ (defn get-by-id [id] - (->> (d/q '[:find (pull ?e [* + (->> (dc/q '[:find (pull ?e [* {:vendor/default-account [:account/name :db/id :account/location] :vendor/legal-entity-tin-type [:db/ident :db/id] :vendor/legal-entity-1099-type [:db/ident :db/id] @@ -125,7 +125,7 @@ :vendor/automatically-paid-when-due [:db/id :client/name]}]) :in $ ?e :where [?e]] - (d/db (d/connect uri)) + (dc/db conn) id) (map first) (map <-datomic) @@ -165,3 +165,5 @@ client-id))) first boolean)) + + diff --git a/src/clj/auto_ap/datomic/yodlee2.clj b/src/clj/auto_ap/datomic/yodlee2.clj index 93249860..e98ecf55 100644 --- a/src/clj/auto_ap/datomic/yodlee2.clj +++ b/src/clj/auto_ap/datomic/yodlee2.clj @@ -1,10 +1,10 @@ (ns auto-ap.datomic.yodlee2 (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query uri]] + :refer [add-sorter-fields apply-pagination apply-sort-3 merge-query conn pull-many]] [auto-ap.graphql.utils :refer [limited-clients]] [clj-time.coerce :as c] - [datomic.api :as d])) + [datomic.client.api :as dc])) (def default-read '[*]) @@ -39,13 +39,13 @@ (merge-query {:query {:find ['?e ] :where ['[?e :yodlee-provider-account/id]]}}) ) - (d/query) + (dc/q) (apply-sort-3 args) (apply-pagination args))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) + (let [results (->> (pull-many db default-read ids) (group-by :db/id))] (->> ids (map results) @@ -54,7 +54,7 @@ (defn get-graphql [args] - (let [db (d/db (d/connect uri)) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(->> (graphql-results ids-to-retrieve db args)) matching-count])) diff --git a/src/clj/auto_ap/datomic/yodlee_merchants.clj b/src/clj/auto_ap/datomic/yodlee_merchants.clj index 5497a1e3..db1f56e3 100644 --- a/src/clj/auto_ap/datomic/yodlee_merchants.clj +++ b/src/clj/auto_ap/datomic/yodlee_merchants.clj @@ -1,14 +1,14 @@ (ns auto-ap.datomic.yodlee-merchants (:require - [auto-ap.datomic :refer [uri]] - [datomic.api :as d])) + [auto-ap.datomic :refer [conn]] + [datomic.client.api :as dc])) (defn get-merchants [_] ;; TODO admin? (let [query {:query {:find ['(pull ?e [:yodlee-merchant/name :yodlee-merchant/yodlee-id :db/id])] :in ['$] :where [['?e :yodlee-merchant/name]]} - :args [(d/db (d/connect uri))]}] + :args [(dc/db conn)]}] (->> - (d/query query) + (dc/q query) (mapv first)))) diff --git a/src/clj/auto_ap/ezcater/core.clj b/src/clj/auto_ap/ezcater/core.clj index d16f4116..0b35d7e2 100644 --- a/src/clj/auto_ap/ezcater/core.clj +++ b/src/clj/auto_ap/ezcater/core.clj @@ -35,13 +35,13 @@ :subscriptions)) (defn get-integrations [] - (d/q '[:find [(pull ?i [:ezcater-integration/api-key - :ezcater-integration/subscriber-uuid - :db/id - :ezcater-integration/integration-status [:db/id]]) ...] - :in $ - :where [?i :ezcater-integration/api-key]] - (d/db conn))) + (map first (d/q '[:find (pull ?i [:ezcater-integration/api-key + :ezcater-integration/subscriber-uuid + :db/id + :ezcater-integration/integration-status [:db/id]]) + :in $ + :where [?i :ezcater-integration/api-key]] + (d/db conn)))) (defn mark-integration-status [integration integration-status] @(d/transact conn @@ -63,12 +63,12 @@ (defn upsert-used-subscriptions ([integration] (let [extant (get-subscriptions integration) - to-ensure (set (d/q '[:find [?cu ...] - :in $ - :where [_ :client/ezcater-locations ?el] - [?el :ezcater-location/caterer ?c] - [?c :ezcater-caterer/uuid ?cu]] - (d/db conn))) + to-ensure (set (map first (d/q '[:find ?cu + :in $ + :where [_ :client/ezcater-locations ?el] + [?el :ezcater-location/caterer ?c] + [?c :ezcater-caterer/uuid ?cu]] + (d/db conn)))) to-create (set/difference to-ensure (set (map :parentId extant)))] diff --git a/src/clj/auto_ap/graphql.clj b/src/clj/auto_ap/graphql.clj index 720b9650..6f139b6c 100644 --- a/src/clj/auto_ap/graphql.clj +++ b/src/clj/auto_ap/graphql.clj @@ -1,6 +1,6 @@ (ns auto-ap.graphql (:require - [auto-ap.datomic :refer [merge-query uri]] + [auto-ap.datomic :refer [merge-query conn]] [auto-ap.datomic.users :as d-users] [auto-ap.graphql.accounts :as gq-accounts] [auto-ap.graphql.checks :as gq-checks] @@ -34,7 +34,8 @@ [com.walmartlabs.lacinia :refer [execute]] [com.walmartlabs.lacinia.parser :as p] [com.walmartlabs.lacinia.schema :as schema] - [datomic.api :as d] + [com.walmartlabs.lacinia.util :refer [attach-resolvers]] + [datomic.client.api :as dc] [unilog.context :as lc] [yang.time :refer [time-it]]) (:import @@ -137,7 +138,8 @@ :address - {:fields {:street1 {:type 'String} + {:fields {:id {:type :id} + :street1 {:type 'String} :street2 {:type 'String} :city {:type 'String} :state {:type 'String} @@ -432,7 +434,8 @@ :email {:type 'String} :phone {:type 'String}}} :add_address - {:fields {:street1 {:type 'String} + {:fields {:id {:type :id} + :street1 {:type 'String} :street2 {:type 'String} :city {:type 'String} :state {:type 'String} @@ -629,7 +632,7 @@ (let [result (cond-> {:query {:find ['?account '?account-name '(sum ?amount)] :in ['$] :where []} - :args [(d/db (d/connect uri)) client_id]} + :args [(dc/db conn) client_id]} client_id (merge-query {:query {:in ['?c]} :args [client_id]}) @@ -641,7 +644,7 @@ '[?account :account/name ?account-name] '[?expense-account :invoice-expense-account/amount ?amount]]}}) - true (d/query))] + true (dc/q))] (for [[account-id account-name total] result] {:account {:id account-id :name account-name} :total total}))) @@ -649,7 +652,7 @@ (let [result (cond-> {:query {:find ['?name '(sum ?outstanding-balance) '(sum ?total)] :in ['$] :where []} - :args [(d/db (d/connect uri)) client_id]} + :args [(dc/db conn) client_id]} client_id (merge-query {:query {:in ['?c]} :args [client_id]}) (not client_id) (merge-query {:query {:where ['[?c :client/name]]}}) @@ -662,7 +665,7 @@ '[(.between java.time.temporal.ChronoUnit/DAYS (java.time.Instant/now) ?d2 ) ?d3] '[(auto-ap.graphql/categorize ?d3) ?name]]}}) - true (d/query)) + true (dc/q)) result (group-by first result)] (for [[id name] [[:due "Due"] [:due-30 "0-30 days"] [:due-60 "31-60 days"] [:due-later ">60 days"]] @@ -689,13 +692,13 @@ (defn get-cash-flow [_ {:keys [client_id]} _] (when client_id - (let [{:client/keys [week-a-credits week-a-debits week-b-credits week-b-debits forecasted-transactions ]} (d/pull (d/db (d/connect uri)) '[*] client_id) + (let [{:client/keys [week-a-credits week-a-debits week-b-credits week-b-debits forecasted-transactions ]} (dc/pull (dc/db conn) '[*] client_id) total-cash (reduce (fn [total [credit debit]] (- (+ total credit) debit)) 0.0 - (d/query {:query {:find '[?debit ?credit] + (dc/q {:query {:find '[?debit ?credit] :in '[$ ?client] :where ['[?j :journal-entry/client ?client] '[?j :journal-entry/line-items ?je] @@ -703,8 +706,8 @@ '[?ba :bank-account/type :bank-account-type/check] '[(get-else $ ?je :journal-entry-line/debit 0.0) ?debit] '[(get-else $ ?je :journal-entry-line/credit 0.0) ?credit]]} - :args [(d/db (d/connect uri)) client_id]})) - bills-due-soon (d/query {:query {:find '[?due ?outstanding ?invoice-number ?vendor-id ?vendor-name] + :args [(dc/db conn) client_id]})) + bills-due-soon (dc/q {:query {:find '[?due ?outstanding ?invoice-number ?vendor-id ?vendor-name] :in '[$ ?client ?due-before] :where ['[?i :invoice/client ?client] '[?i :invoice/status :invoice-status/unpaid] @@ -714,11 +717,11 @@ '[?i :invoice/invoice-number ?invoice-number] '[?i :invoice/vendor ?vendor-id] '[?vendor-id :vendor/name ?vendor-name]]} - :args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]}) + :args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]}) outstanding-checks (reduce + 0.0 - (map first (d/query {:query {:find '[?amount] + (map first (dc/q {:query {:find '[?amount] :in '[$ ?client ?due-before] :where ['[?p :payment/client ?client] '[?p :payment/status :payment-status/pending] @@ -726,14 +729,14 @@ '(or [?p :payment/type :payment-type/debit] [?p :payment/type :payment-type/check])]} - :args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]}))) - recent-fulfillments (d/query {:query {:find '[?f ?d] + :args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/days 180)))]}))) + recent-fulfillments (dc/q {:query {:find '[?f ?d] :in '[$ ?client ?min-date] :where ['[?t :transaction/forecast-match ?f] '[?t :transaction/date ?d] '[?t :transaction/client ?client] '[(>= ?d ?min-date)]]} - :args [(d/db (d/connect uri)) client_id (coerce/to-date (t/plus (time/local-now) (t/months -2)))]}) + :args [(dc/db conn) client_id (coerce/to-date (t/plus (time/local-now) (t/months -2)))]}) forecasted-transactions (for [{:forecasted-transaction/keys [amount identifier day-of-month] :db/keys [id]} forecasted-transactions month (range -1 7) diff --git a/src/clj/auto_ap/graphql/accounts.clj b/src/clj/auto_ap/graphql/accounts.clj index a4af1c93..3820dd34 100644 --- a/src/clj/auto_ap/graphql/accounts.clj +++ b/src/clj/auto_ap/graphql/accounts.clj @@ -1,6 +1,6 @@ (ns auto-ap.graphql.accounts (:require - [auto-ap.datomic :refer [audit-transact conn remove-nils]] + [auto-ap.datomic :refer [audit-transact conn upsert-entity]] [auto-ap.datomic.accounts :as d-accounts] [auto-ap.graphql.utils :refer [->graphql @@ -11,7 +11,12 @@ enum->keyword is-admin? result->page]] - [datomic.api :as d])) + [auto-ap.search :as search] + [datomic.client.api :as dc] + [clojure.string :as str]) + (:import + (org.apache.lucene.search TermInSetQuery) + (org.apache.lucene.util BytesRef))) (defn get-graphql [context args _] (assert-admin (:id context)) @@ -33,48 +38,38 @@ (defn upsert-account [context args _] (let [{{:keys [id client-overrides numeric-code location applicability account-set name invoice-allowance vendor-allowance type]} :account} (<-graphql args)] (when-not id - (when (seq (d/query {:query {:find ['?e] - :in '[$ ?account-set ?numeric-code] - :where ['[?e :account/account-set ?account-set] - '[?e :account/numeric-code ?numeric-code]]} - :args [(d/db conn) account-set numeric-code]})) + (when (seq (dc/q {:query {:find ['?e] + :in '[$ ?account-set ?numeric-code] + :where ['[?e :account/account-set ?account-set] + '[?e :account/numeric-code ?numeric-code]]} + :args [(dc/db conn) account-set numeric-code]})) (throw (ex-info (str "Account set " account-set " already has an account for code " numeric-code) {} )))) - - (let [original (when id - (d/entity (d/db conn) id)) - result (audit-transact (cond-> - [(remove-nils - {:db/id (or id "new-account") - :account/name name - :account/search-terms name - :account/type (keyword "account-type" (clojure.core/name type)) - :account/applicability (or (enum->keyword applicability "account-applicability") - :account-applicability/global) - - :account/invoice-allowance (some-> invoice-allowance (enum->keyword "allowance")) - :account/vendor-allowance (some-> vendor-allowance (enum->keyword "allowance")) - :account/default-allowance :allowance/allowed - :account/account-set account-set - :account/location location - :account/numeric-code (when-not id - numeric-code) - :account/code (when-not id - (str numeric-code))}) - [:reset (or id "new-account") :account/client-overrides - (mapv - (fn [client-override] - (remove-nils - {:db/id (:id client-override) - :account-client-override/client (:client-id client-override) - :account-client-override/name (:name client-override) - :account-client-override/search-terms (:name client-override)})) - client-overrides)]] - (and (not location) (:account/location original)) (conj [:db/retract (or id "new-account") :account/location (:account/location original)])) + (let [result (audit-transact [`(upsert-entity + ~{:db/id (or id "new-account") + :account/name name + :account/search-terms name + :account/type (keyword "account-type" (clojure.core/name type)) + :account/applicability (or (enum->keyword applicability "account-applicability") + :account-applicability/global) + :account/invoice-allowance (some-> invoice-allowance (enum->keyword "allowance")) + :account/vendor-allowance (some-> vendor-allowance (enum->keyword "allowance")) + :account/default-allowance :allowance/allowed + :account/account-set account-set + :account/location location + :account/numeric-code numeric-code + :account/code (str numeric-code) + :account/client-overrides (mapv + (fn [client-override] + {:db/id (:id client-override) + :account-client-override/client (:client-id client-override) + :account-client-override/name (:name client-override) + :account-client-override/search-terms (:name client-override)}) + client-overrides)})] (:id context))] (->graphql - (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"]))))))) + (d-accounts/get-by-id (or id (get-in result [:tempids "new-account"]))))))) (def search-pattern [:db/id :account/numeric-code @@ -86,6 +81,7 @@ (when client (assert-can-see-client (:id context) client)) (let [query (cleanse-query query) + _ (println query) num (some-> (re-find #"([0-9]+)" query) second (not-empty ) @@ -160,3 +156,35 @@ (sequence xform))) []))) +(defn rebuild-search-index [] + (search/full-index-query + (for [result (map first (dc/qseq '[:find (pull ?aco [:account-client-override/search-terms :account-client-override/client :db/id {:account/_client-overrides [:account/numeric-code :account/location :db/id]}]) + :in $ + :where [?aco :account-client-override/client ] + [?aco :account-client-override/search-terms ]] + (dc/db conn))) + :when (:account/numeric-code (:account/_client-overrides result))] + {:id (:db/id (:account/_client-overrides result)) + :text (:account-client-override/search-terms result) + :client (str (:db/id (:account-client-override/client result))) + :numeric-code (:account/numeric-code (:account/_client-overrides result)) + :location (:account/location (:account/_client-overrides result))}) + "account-client-override") + + (search/full-index-query + (for [result (map first (dc/qseq '[:find (pull ?a [:account/numeric-code + :account/search-terms + {:account/applicability [:db/ident]} + :db/id + :account/location]) + :in $ + :where [?a :account/search-terms ]] + (dc/db conn))) + :when (:account/search-terms result) + ] + {:id (:db/id result) + :text (:account/search-terms result) + :numeric-code (:account/numeric-code result) + :location (:account/location result) + :applicability (name (:db/ident (:account/applicability result)))}) + "account")) diff --git a/src/clj/auto_ap/graphql/checks.clj b/src/clj/auto_ap/graphql/checks.clj index e22ce198..bff642ed 100644 --- a/src/clj/auto_ap/graphql/checks.clj +++ b/src/clj/auto_ap/graphql/checks.clj @@ -1,7 +1,7 @@ (ns auto-ap.graphql.checks (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.datomic :refer [conn remove-nils plus]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.bank-accounts :as d-bank-accounts] [auto-ap.datomic.checks :as d-checks] @@ -34,9 +34,9 @@ [com.brunobonacci.mulog :as mu] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [config.core :refer [env]] - [datomic.api :as d] - [digest] - [clj-time.coerce :as coerce]) + [clj-time.coerce :as coerce] + [datomic.client.api :as dc] + [digest]) (:import (java.io ByteArrayOutputStream) (java.text DecimalFormat) @@ -230,7 +230,7 @@ [{:invoice-payment/payment (-> invoice :invoice/vendor :db/id str) :invoice-payment/amount invoice-amount :invoice-payment/invoice (:db/id invoice)} - [:pay (:db/id invoice) invoice-amount]]) + `(d-checks/pay ~(:db/id invoice) ~invoice-amount)]) (reduce into []))) (defn base-payment [invoices vendor client bank-account _ _ invoice-amounts] @@ -391,7 +391,6 @@ (let [type (keyword "payment-type" (name type)) invoices (d-invoices/get-multi (map :invoice-id invoice-payments)) client (d-clients/get-by-id client-id) - invoice-amounts (by :invoice-id :amount invoice-payments) invoices-grouped-by-vendor (group-by (comp :db/id :invoice/vendor) invoices) vendors (->> (d/pull-many (d/db conn) d-vendors/default-read (keys invoices-grouped-by-vendor)) @@ -409,14 +408,16 @@ (reduce into []) doall)) checks (if (= type :payment-type/check) - (conj checks [:inc (:db/id bank-account) :bank-account/check-number (count invoices-grouped-by-vendor)]) + (conj checks `(plus ~(:db/id bank-account) ~:bank-account/check-number ~(count invoices-grouped-by-vendor))) checks)] (when (= type :payment-type/check) (mu/trace ::making-pdfs [:checks checks] (make-pdfs (filter #(and (= :payment-type/check (:payment/type %)) (> (:payment/amount %) 0.0)) checks)))) - (transact-with-ledger checks id) + (transact-with-ledger (map #(if (map? %) + (dissoc % :payment/pdf-data) + %) checks ) id) {:invoices (d-invoices/get-multi (map :invoice-id invoice-payments)) @@ -565,7 +566,6 @@ (log/info "Voiding " (count all-ids) args) (void-payments-internal all-ids (:id context)) - {:message (str "Succesfully voided " (count all-ids))})) (defn get-all-payments [context args _] diff --git a/src/clj/auto_ap/graphql/clients.clj b/src/clj/auto_ap/graphql/clients.clj index c9f1ec6d..4f0aba9a 100644 --- a/src/clj/auto_ap/graphql/clients.clj +++ b/src/clj/auto_ap/graphql/clients.clj @@ -1,19 +1,19 @@ (ns auto-ap.graphql.clients (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [audit-transact conn remove-nils]] + [auto-ap.datomic :refer [audit-transact conn upsert-entity random-tempid]] [auto-ap.datomic.clients :as d-clients] + [auto-ap.square.core :as square] [auto-ap.graphql.utils :refer [->graphql assert-admin can-see-client? is-admin?]] [auto-ap.routes.queries :as q] - [auto-ap.square.core :as square] [clj-time.coerce :as coerce] [clojure.java.io :as io] [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d] + [datomic.client.api :as dc] [unilog.context :as lc] [auto-ap.graphql.utils :refer [attach-tracing-resolvers]] [com.brunobonacci.mulog :as mu]) @@ -22,10 +22,10 @@ (org.apache.commons.codec.binary Base64))) (defn assert-client-code-is-unique [code] - (when (seq (d/query {:query {:find '[?id] - :in ['$ '?code] - :where ['[?id :client/code ?code]]} - :args [(d/db conn) code]})) + (when (seq (dc/q {:query {:find '[?id] + :in ['$ '?code] + :where ['[?id :client/code ?code]]} + :args [(dc/db conn) code]})) (throw (ex-info "Client is not unique" {:validation-error (str "Client code '" code "' is not unique.")})))) (defn upload-signature-data [signature-data] @@ -44,9 +44,9 @@ (str "https://integreat-signature-images.s3.amazonaws.com/" signature-id ".jpg"))))) (defn assert-no-shared-transaction-sources [client-code txes] - (let [new-db (:db-after (d/with (d/db conn) - txes))] - (when (seq (->> (d/q '[:find ?src (count ?ba) + (let [new-db (:db-after (dc/with (dc/with-db conn) + {:tx-data txes}))] + (when (seq (->> (dc/q '[:find ?src (count ?ba) :in $ ?c :where [?c :client/bank-accounts ?ba] (or @@ -66,116 +66,84 @@ (let [client (when (:id edit_client) (d-clients/get-by-id (:id edit_client))) id (or (:db/id client) "new-client") signature-file (upload-signature-data (:signature_data edit_client)) - _ (when client - (audit-transact (-> [] - (into (mapv (fn [lm] [:db/retractEntity (:db/id lm)]) (:client/location-matches client))) - (into (mapv (fn [m] [:db/retract (:db/id client) :client/matches m]) (:client/matches client))) - (into (mapv (fn [m] [:db/retract (:db/id client) :client/feature-flags m]) (:client/feature-flags client)))) - (:id context))) - reverts (-> [] - (into (->> (:bank_accounts edit_client) - (filter #(nil? (:yodlee_account_id %))) - (filter #(:bank-account/yodlee-account-id (d/entity (d/db conn) (:id %)))) - (map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account-id (:bank-account/yodlee-account-id (d/entity (d/db conn) (:id ba)))])))) - (into (->> (:bank_accounts edit_client) - (filter #(nil? (:yodlee_account %))) - (filter #(:bank-account/yodlee-account (d/entity (d/db conn) (:id %)))) - (map (fn [ba] [:db/retract (:id ba) :bank-account/yodlee-account (:db/id (:bank-account/yodlee-account (d/entity (d/db conn) (:id ba))))])))) - (into (->> (:bank_accounts edit_client) - (filter #(nil? (:intuit_bank_account %))) - (filter #(:bank-account/intuit-bank-account (d/entity (d/db conn) (:id %)))) - (map (fn [ba] [:db/retract (:id ba) :bank-account/intuit-bank-account (:db/id (:bank-account/intuit-bank-account (d/entity (d/db conn) (:id ba))))])))) - (into (->> (:bank_accounts edit_client) - (filter #(nil? (:plaid_account %))) - (filter #(:bank-account/plaid-account (d/entity (d/db conn) (:id %)))) - (map (fn [ba] [:db/retract (:id ba) :bank-account/plaid-account (:db/id (:bank-account/plaid-account (d/entity (d/db conn) (:id ba))))]))))) - - client-code (if (str/blank? (:client/code client)) (:code edit_client) (:client/code client)) - transactions (into [(remove-nils {:db/id id - :client/code client-code - :client/name (:name edit_client) - :client/matches (:matches edit_client) - :client/signature-file signature-file - :client/email (:email edit_client) - :client/locked-until (some-> (:locked_until edit_client) (coerce/to-date)) - :client/locations (filter identity (:locations edit_client)) - :client/week-a-debits (:week_a_debits edit_client) - :client/week-a-credits (:week_a_credits edit_client) - :client/week-b-debits (:week_b_debits edit_client) - :client/square-auth-token (:square_auth_token edit_client) - :client/square-locations (map - (fn [sl] - (remove-nils - {:db/id (:id sl) - :square-location/client-location (:client_location sl)})) - (:square_locations edit_client)) + updated-entity {:db/id id + :client/code client-code + :client/name (:name edit_client) + :client/matches (:matches edit_client) + :client/signature-file signature-file + :client/email (:email edit_client) + :client/locked-until (some-> (:locked_until edit_client) (coerce/to-date)) + :client/locations (filter identity (:locations edit_client)) + :client/week-a-debits (:week_a_debits edit_client) + :client/week-a-credits (:week_a_credits edit_client) + :client/week-b-debits (:week_b_debits edit_client) + :client/square-auth-token (:square_auth_token edit_client) + :client/square-locations (map + (fn [sl] + {:db/id (or (:id sl) (random-tempid)) + :square-location/client-location (:client_location sl)}) + (:square_locations edit_client)) - :client/ezcater-locations (map - (fn [el] - (remove-nils - {:db/id (:id el) - :ezcater-location/location (:location el) - :ezcater-location/caterer (:caterer el)})) - (:ezcater_locations edit_client)) - :client/week-b-credits (:week_b_credits edit_client) - :client/location-matches (->> (:location_matches edit_client) - (filter (fn [lm] (and (:location lm) (:match lm)))) - (map (fn [lm] {:location-match/location (:location lm) - :location-match/matches [(:match lm)]}))) - :client/address (remove-nils { - :address/street1 (:street1 (:address edit_client)) - :address/street2 (:street2 (:address edit_client)) - :address/city (:city (:address edit_client)) - :address/state (:state (:address edit_client)) - :address/zip (:zip (:address edit_client))}) - :client/feature-flags (:feature_flags edit_client) - :client/bank-accounts (map #(remove-nils - (cond-> {:db/id (:id %) - :bank-account/code (:code %) - :bank-account/bank-name (:bank_name %) - :bank-account/bank-code (:bank_code %) - :bank-account/start-date (-> (:start_date %) (coerce/to-date)) - :bank-account/routing (:routing %) - :bank-account/include-in-reports (:include_in_reports %) + :client/emails (map (fn [e] + {:db/id (or (:id e) + (random-tempid)) + :email-contact/email (:email e) + :email-contact/description (:description e)}) + (:emails edit_client)) - :bank-account/name (:name %) - :bank-account/visible (:visible %) - :bank-account/number (:number %) - :bank-account/check-number (:check_number %) - :bank-account/numeric-code (:numeric_code %) - :bank-account/sort-order (:sort_order %) - :bank-account/locations (:locations %) - :bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date %)) + :client/feature-flags (:feature_flags edit_client) + :client/ezcater-locations (map + (fn [el] + {:db/id (or (:id el) (random-tempid)) + :ezcater-location/location (:location el) + :ezcater-location/caterer (:caterer el)}) + (:ezcater_locations edit_client)) + :client/week-b-credits (:week_b_credits edit_client) + :client/location-matches (->> (:location_matches edit_client) + (filter (fn [lm] (and (:location lm) (:match lm)))) + (map (fn [lm] {:db/id (or (:id lm ) (random-tempid)) + :location-match/location (:location lm) + :location-match/matches [(:match lm)]}))) + :client/address {:db/id (or (:id (:address edit_client)) (random-tempid)) + :address/street1 (:street1 (:address edit_client)) + :address/street2 (:street2 (:address edit_client)) + :address/city (:city (:address edit_client)) + :address/state (:state (:address edit_client)) + :address/zip (:zip (:address edit_client))} + :client/bank-accounts (map (fn [ba] + {:db/id (:id ba) + :bank-account/code (:code ba) + :bank-account/bank-name (:bank_name ba) + :bank-account/bank-code (:bank_code ba) + :bank-account/start-date (-> (:start_date ba) (coerce/to-date)) + :bank-account/routing (:routing ba) + :bank-account/include-in-reports (:include_in_reports ba) - :bank-account/yodlee-account-id (:yodlee_account_id %) - :bank-account/type (keyword "bank-account-type" (name (:type %)))} - (:yodlee_account %) (assoc :bank-account/yodlee-account [:yodlee-account/id (:yodlee_account %)]) - (:plaid_account %) (assoc :bank-account/plaid-account (:plaid_account %)) - (:intuit_bank_account %) (assoc :bank-account/intuit-bank-account (:intuit_bank_account %)))) - (:bank_accounts edit_client)) + :bank-account/name (:name ba) + :bank-account/visible (:visible ba) + :bank-account/number (:number ba) + :bank-account/check-number (:check_number ba) + :bank-account/numeric-code (:numeric_code ba) + :bank-account/sort-order (:sort_order ba) + :bank-account/locations (:locations ba) + :bank-account/use-date-instead-of-post-date? (boolean (:use_date_instead_of_post_date ba)) - }) - - [:reset id :client/emails (map #(remove-nils - {:db/id (or (:id %) - (str (UUID/randomUUID))) - :email-contact/email (:email %) - :email-contact/description (:description %)}) - (:emails edit_client))] - [:reset id :client/forecasted-transactions (map #(remove-nils - {:db/id (:id %) - :forecasted-transaction/day-of-month (:day_of_month %) - :forecasted-transaction/identifier (:identifier %) - :forecasted-transaction/amount (:amount %)} - ) - (:forecasted_transactions edit_client))]] - reverts) - _ (assert-no-shared-transaction-sources client-code transactions) - _ (log/info "upserting client" transactions) - result (audit-transact transactions (:id context))] + :bank-account/yodlee-account-id (:yodlee_account_id ba) + :bank-account/type (keyword "bank-account-type" (name (:type ba))) + :bank-account/yodlee-account (when (:yodlee_account ba) + [:yodlee-account/id (:yodlee_account ba)]) + :bank-account/plaid-account (:plaid_account ba) + :bank-account/intuit-bank-account (:intuit_bank_account ba)}) + (:bank_accounts edit_client)) + + } + _ (assert-no-shared-transaction-sources client-code [`(upsert-entity ~updated-entity)]) + _ (log/info "upserting client" updated-entity) + + result (audit-transact [`(upsert-entity ~updated-entity)] (:id context))] (when (:square_auth_token edit_client) (square/upsert-locations (-> result :tempids (get id) (or id) d-clients/get-by-id))) (-> (-> result :tempids (get id) (or id) d-clients/get-by-id) @@ -194,13 +162,13 @@ ->graphql))) (defn refresh-bank-account-current-balance [bank-account-id] - (let [all-transactions (d/query + (let [all-transactions (dc/q {:query {:find ['?e '?debit '?credit] :in ['$ '?bank-account-id] :where '[[?e :journal-entry-line/account ?bank-account-id] [(get-else $ ?e :journal-entry-line/debit 0.0) ?debit] [(get-else $ ?e :journal-entry-line/credit 0.0) ?credit]]} - :args [(d/db conn) bank-account-id]}) + :args [(dc/db conn) bank-account-id]}) debits (->> all-transactions (map (fn [[_ debit _]] debit)) @@ -209,46 +177,45 @@ (map (fn [[_ _ credit]] credit)) (reduce + 0.0)) - current-balance (if (= :bank-account-type/check (:bank-account/type (d/entity (d/db conn) bank-account-id))) + current-balance (if (= :bank-account-type/check (:db/ident (:bank-account/type (dc/pull (dc/db conn) [:bank-account/type] bank-account-id)))) (- debits credits) (- credits debits))] - @(d/transact conn [{:db/id bank-account-id - :bank-account/current-balance current-balance}]))) + (dc/transact conn {:tx-data [{:db/id bank-account-id + :bank-account/current-balance current-balance}]}))) (defn bank-accounts-needing-refresh [] - (let [last-refreshed (->> (d/query - {:query {:find ['?ba '?tx] + (let [last-refreshed (->> (dc/q {:query {:find ['?ba '?tx] :in ['$ ] :where ['[?ba :bank-account/current-balance _ ?tx true]]} - :args [(d/history (d/db conn))]}) + :args [(dc/history (dc/db conn))]}) (group-by first) (map (fn [[ba balance-txes]] [ba (->> balance-txes (map second) (sort-by #(- %)) first)]))) - has-newer-transaction (->> (d/query + has-newer-transaction (->> (dc/q {:query {:find ['?ba] :in '[$ [[?ba ?last-refreshed] ...] ] :where ['[_ :journal-entry-line/account ?ba ?tx] '[(>= ?tx ?last-refreshed)]]} - :args [(d/history (d/db conn)) + :args [(dc/history (dc/db conn)) last-refreshed]}) (map first) (set)) - no-current-balance (->> (d/query {:query {:find ['?ba] + no-current-balance (->> (dc/q {:query {:find ['?ba] :in '[$] :where ['[?ba :bank-account/code] '(not [?ba :bank-account/current-balance]) ]} - :args [(d/db conn)]}) + :args [(dc/db conn)]}) (map first))] (into has-newer-transaction no-current-balance))) (defn build-current-balance [bank-accounts] (doseq [bank-account bank-accounts] - (log/info "Refreshing bank account" (-> (d/db conn) (d/entity bank-account) :bank-account/code)) + (log/info "Refreshing bank account" (-> (dc/db conn) (dc/pull [:bank-account/code] bank-account))) (try (refresh-bank-account-current-balance bank-account) (catch Exception e @@ -257,10 +224,10 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn refresh-all-current-balance [] (lc/with-context {:source "current-balance-cache"} - (build-current-balance (->> (d/query {:query {:find ['?ba] - :in '[$] - :where ['[?ba :bank-account/code]]} - :args [(d/db conn)]}) + (build-current-balance (->> (dc/q {:query {:find ['?ba] + :in '[$] + :where ['[?ba :bank-account/code]]} + :args [(dc/db conn)]}) (map first))))) (defn refresh-current-balance [] @@ -410,8 +377,8 @@ (defn setup-sales-queries [context args _] (assert-admin (:id context)) - (let [{client-code :client/code feature-flags :client/feature-flags} (d/pull (d/db conn) '[:client/code :client/feature-flags] (:client_id args)) - is-new-square? ((set feature-flags) "new-square")] +(let [{client-code :client/code feature-flags :client/feature-flags} (dc/pull (dc/db conn) '[:client/code :client/feature-flags] (:client_id args)) + is-new-square? ((set feature-flags) "new-square")] (q/put-query (str (UUID/randomUUID)) (format sales-summary-query client-code) (str "sales query for " client-code) @@ -442,12 +409,11 @@ (str "refunds query for " client-code) (str client-code "-refund") [:client/code client-code]) - - (let [sales-summary-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-summary")])) - sales-category-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-category")])) - expected-deposit-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-expected-deposit")])) - tender-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-tender")])) - refund-id (:saved-query/guid (d/pull (d/db auto-ap.datomic/conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-refund")]))] + (let [sales-summary-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-summary")])) + sales-category-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-sales-category")])) + expected-deposit-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-expected-deposit")])) + tender-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-tender")])) + refund-id (:saved-query/guid (dc/pull (dc/db conn) [:saved-query/guid] [:saved-query/lookup-key (str client-code "-refund")]))] {:message (str/join "\n" [ (str "For " client-code ":") diff --git a/src/clj/auto_ap/graphql/import_batch.clj b/src/clj/auto_ap/graphql/import_batch.clj index d5f05ac6..e1213387 100644 --- a/src/clj/auto_ap/graphql/import_batch.clj +++ b/src/clj/auto_ap/graphql/import_batch.clj @@ -1,14 +1,13 @@ (ns auto-ap.graphql.import-batch (:require - [auto-ap.datomic - :refer - [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]] + [auto-ap.datomic :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query pull-many-by-id]] [auto-ap.graphql.utils :refer [<-graphql assert-admin ident->enum-f result->page]] [clj-time.coerce :as coerce] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.api :as d] + [datomic.client.api :as dc] [auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) (def default-read '[:db/id @@ -36,19 +35,17 @@ (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) - (group-by :db/id))] + (let [results (pull-many-by-id db default-read ids)] (->> ids - (map results) - (map first)))) + (map results)))) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(graphql-results ids-to-retrieve db args) matching-count])) diff --git a/src/clj/auto_ap/graphql/intuit_bank_accounts.clj b/src/clj/auto_ap/graphql/intuit_bank_accounts.clj index d89f055f..4a137c3e 100644 --- a/src/clj/auto_ap/graphql/intuit_bank_accounts.clj +++ b/src/clj/auto_ap/graphql/intuit_bank_accounts.clj @@ -2,11 +2,11 @@ (:require [auto-ap.datomic :refer [conn]] [auto-ap.graphql.utils :refer [->graphql assert-admin]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn get-intuit-bank-accounts [context _ _] (assert-admin (:id context)) - (->graphql (map first (d/q '[:find (pull ?e [*]) + (->graphql (map first (dc/q '[:find (pull ?e [*]) :in $ :where [?e :intuit-bank-account/external-id]] - (d/db conn))))) + (dc/db conn))))) diff --git a/src/clj/auto_ap/graphql/invoices.clj b/src/clj/auto_ap/graphql/invoices.clj index bd909491..978424fb 100644 --- a/src/clj/auto_ap/graphql/invoices.clj +++ b/src/clj/auto_ap/graphql/invoices.clj @@ -1,6 +1,8 @@ (ns auto-ap.graphql.invoices (:require - [auto-ap.datomic :refer [conn remove-nils uri]] + [auto-ap.datomic + :refer [conn remove-nils upsert-entity]] + [auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.invoices :as d-invoices] [auto-ap.datomic.vendors :as d-vendors] @@ -21,11 +23,12 @@ [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as coerce] [clj-time.core :as time] - [clojure.set :as set] [clojure.tools.logging :as log] [datomic.api :as d] [manifold.deferred :as de] - [com.brunobonacci.mulog :as mu])) + [com.brunobonacci.mulog :as mu] + [com.walmartlabs.lacinia.util :refer [attach-resolvers]] + [datomic.client.api :as dc])) (defn ->graphql [invoice user ] (if (= "admin" (:user/role user)) @@ -61,7 +64,7 @@ (defn reject-invoices [context {:keys [invoices]} _] (assert-power-user (:id context)) (doseq [i invoices] - (assert-can-see-client (:id context) (:db/id (:invoice/client (d/entity (d/db conn) i))))) + (assert-can-see-client (:id context) (:db/id (:invoice/client (dc/pull (dc/db conn) [{:invoice/client [:db/id]}] i))))) (let [transactions (map (fn [i] [:db/retractEntity i]) invoices)] (transact-with-ledger transactions (:id context)) invoices)) @@ -69,7 +72,9 @@ (defn approve-invoices [context {:keys [invoices]} _] (assert-power-user (:id context)) (doseq [i invoices - :let [invoice (d/entity (d/db conn) i)]] + :let [invoice (dc/pull (dc/db conn) [{:invoice/client [:db/id]} + :invoice/date] + i)]] (assert-can-see-client (:id context) (-> invoice :invoice/client :db/id)) (assert-not-locked (-> invoice :invoice/client :db/id) (-> invoice :invoice/date))) (let [transactions (map (fn [i] {:db/id i :invoice/import-status :import-status/imported}) invoices)] @@ -104,34 +109,25 @@ due)) _ (when-not (:db/id account) (throw (ex-info (str "Vendor '" (:vendor/name vendor) "' does not have a default expense acount.") {:vendor-id vendor_id})))] - (cond-> - {:db/id "invoice" - :invoice/invoice-number invoice_number - :invoice/client client_id - :invoice/vendor vendor_id - :invoice/import-status :import-status/imported - :invoice/total total - :invoice/outstanding-balance total - :invoice/status :invoice-status/unpaid - :invoice/date (coerce/to-date date) - :invoice/expense-accounts (map expense-account->entity - expense_accounts)} - due (assoc :invoice/due (coerce/to-date due)) - scheduled_payment (assoc :invoice/scheduled-payment (coerce/to-date scheduled_payment))))) - -(defn deleted-expense-accounts [invoice expense-accounts] - (let [current-expense-accounts (:invoice/expense-accounts invoice) - specified-ids (->> expense-accounts - (map :id) - set) - existing-ids (->> current-expense-accounts - (map :db/id) - set)] - (set/difference existing-ids specified-ids))) + `(upsert-entity ~{:db/id "invoice" + :invoice/invoice-number invoice_number + :invoice/client client_id + :invoice/vendor vendor_id + :invoice/import-status :import-status/imported + :invoice/total total + :invoice/outstanding-balance total + :invoice/status :invoice-status/unpaid + :invoice/date (coerce/to-date date) + :invoice/expense-accounts (map expense-account->entity + expense_accounts) + :invoice/due (coerce/to-date due) + :invoice/scheduled-payment (coerce/to-date scheduled_payment)}))) (defn assert-valid-expense-accounts [expense_accounts vendor_id] (doseq [expense-account expense_accounts - :let [account (d/entity (d/db conn) (:account_id expense-account))]] + :let [account (dc/pull (dc/db conn) + [:account/location] + (:account_id expense-account))]] (when (empty? (:location expense-account)) (throw (ex-info "Expense account is missing location" {:validation-error "Expense account is missing location"}))) @@ -178,7 +174,7 @@ (assert-valid-expense-accounts expense_accounts vendor_id) (assert-invoice-amounts-add-up in) - (let [transaction-result (transact-with-ledger [(add-invoice-transaction in)] (:id context))] + (let [transaction-result (transact-with-ledger (add-invoice-transaction in) (:id context))] (-> (d-invoices/get-by-id (get-in transaction-result [:tempids "invoice"])) (->graphql (:id context))))) @@ -218,23 +214,21 @@ paid-amount (- (:invoice/total invoice) (:invoice/outstanding-balance invoice)) _ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice))) - deleted (deleted-expense-accounts invoice expense_accounts) _ (assert-not-locked (:db/id (:invoice/client invoice)) (:date in)) _ (assert-valid-expense-accounts expense_accounts vendor_id) _ (assert-invoice-amounts-add-up in) - updated-invoice (cond-> {:db/id id - :invoice/invoice-number invoice_number - :invoice/date (coerce/to-date date) + updated-invoice {:db/id id + :invoice/invoice-number invoice_number + :invoice/date (coerce/to-date date) - :invoice/total total - :invoice/outstanding-balance (- total paid-amount) - :invoice/expense-accounts (map expense-account->entity - expense_accounts)} - due (assoc :invoice/due (coerce/to-date due)) - scheduled_payment (assoc :invoice/scheduled-payment (coerce/to-date scheduled_payment)))] - (transact-with-ledger (concat [updated-invoice] - (map (fn [d] [:db/retract id :invoice/expense-accounts d]) deleted)) + :invoice/total total + :invoice/outstanding-balance (- total paid-amount) + :invoice/expense-accounts (map expense-account->entity + expense_accounts) + :invoice/due (coerce/to-date due) + :invoice/scheduled-payment (coerce/to-date scheduled_payment)}] + (transact-with-ledger [`(upsert-entity ~updated-invoice)] (:id context)) (-> (d-invoices/get-by-id id) (->graphql (:id context))))) @@ -300,15 +294,15 @@ (log/info "Voiding " (count all-ids) args) (transact-with-ledger (->> all-ids - (d/q '[:find [(pull ?i [:db/id :invoice/date {:invoice/expense-accounts [:db/id]} - ]) ...] + (dc/q '[:find (pull ?i [:db/id :invoice/date {:invoice/expense-accounts [:db/id]}]) :in $ [?i ...] :where (not [_ :invoice-payment/invoice ?i]) [?i :invoice/client ?c] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [?i :invoice/date ?d] [(>= ?d ?lu)]] - (d/db conn)) + (dc/db conn)) + (map first) (mapcat (fn [i] (into @@ -328,16 +322,15 @@ (let [invoice (d-invoices/get-by-id id) _ (assert-can-see-client (:id context) (:db/id (:invoice/client invoice))) _ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice)) - conn (d/connect uri) - history (d/history (d/db conn)) - txs (d/query {:query {:find ['?tx '?e '?original-status '?original-outstanding '?total '?ea '?ea-amount] - :where ['[?e :invoice/status :invoice-status/voided ?tx true] - '[?e :invoice/status ?original-status ?tx false] - '[?e :invoice/outstanding-balance ?original-outstanding ?tx false] - '[?e :invoice/total ?total ?tx false] - '[?ea :invoice-expense-account/amount ?ea-amount ?tx false]] - :in ['$ '?e]} - :args [history id]}) + history (dc/history (dc/db conn)) + txs (dc/q {:query {:find ['?tx '?e '?original-status '?original-outstanding '?total '?ea '?ea-amount] + :where ['[?e :invoice/status :invoice-status/voided ?tx true] + '[?e :invoice/status ?original-status ?tx false] + '[?e :invoice/outstanding-balance ?original-outstanding ?tx false] + '[?e :invoice/total ?total ?tx false] + '[?ea :invoice-expense-account/amount ?ea-amount ?tx false]] + :in ['$ '?e]} + :args [history id]}) [last-transaction] (->> txs (sort-by first) (last))] (transact-with-ledger [(->> txs (filter (fn [[tx]] (= tx last-transaction))) @@ -354,7 +347,11 @@ (->graphql (:id context))))) (defn unautopay-invoice [context {id :invoice_id} _] - (let [invoice (d/entity (d/db conn) id)] + (let [invoice (dc/pull (dc/db conn) [{:invoice/client [:db/id]} + :invoice-payment/_invoice + :invoice/total + :invoice/scheduled-payment + :invoice/date] id)] (assert (:invoice/client invoice)) (assert-can-see-client (:id context) (:db/id (:invoice/client invoice))) (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice)) @@ -370,18 +367,15 @@ (defn edit-expense-accounts [context args _] (assert-can-see-client (:id context) (:db/id (:invoice/client (d-invoices/get-by-id (:invoice_id args))))) (let [invoice-id (:invoice_id args) - invoice (d-invoices/get-by-id invoice-id) - _ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice)) - _ (assert-valid-expense-accounts (:expense_accounts args) (:db/id (:invoice/vendor invoice ))) - deleted (deleted-expense-accounts invoice (:expense_accounts args)) - updated {:db/id invoice-id - :invoice/expense-accounts (map - expense-account->entity - (:expense_accounts args))}] + invoice (d-invoices/get-by-id invoice-id) + _ (assert-not-locked (:db/id (:invoice/client invoice)) (:invoice/date invoice)) + _ (assert-valid-expense-accounts (:expense_accounts args) (:db/id (:invoice/vendor invoice )))] - (transact-with-ledger (concat [updated] - (map (fn [d] [:db/retract invoice-id :invoice/expense-accounts d]) deleted)) - (:id context)) + (transact-with-ledger [`(upsert-entity ~{:db/id invoice-id + :invoice/expense-accounts (map + expense-account->entity + (:expense_accounts args))})] + (:id context)) (->graphql (d-invoices/get-by-id (:invoice_id args)) (:id context)))) diff --git a/src/clj/auto_ap/graphql/ledger.clj b/src/clj/auto_ap/graphql/ledger.clj index d9377de5..cb8e5207 100644 --- a/src/clj/auto_ap/graphql/ledger.clj +++ b/src/clj/auto_ap/graphql/ledger.clj @@ -1,6 +1,7 @@ (ns auto-ap.graphql.ledger (:require - [auto-ap.datomic :refer [audit-transact-batch conn remove-nils uri]] + [auto-ap.datomic + :refer [audit-transact-batch conn pull-attr pull-many remove-nils]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.clients :as d-clients] [auto-ap.datomic.ledger :as l] @@ -17,7 +18,7 @@ [clojure.tools.logging :as log] [clojure.data.csv :as csv] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] - [datomic.api :as d] + [datomic.client.api :as dc] [mount.core :as mount] [com.brunobonacci.mulog :as mu] [unilog.context :as lc] @@ -135,19 +136,19 @@ [])))) (defn build-account-lookup [client-id] - (let [accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :account/name + (let [accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :account/name :account/numeric-code {:account/type [:db/ident] :account/client-overrides [:account-client-override/client :account-client-override/name]} ])] :in ['$] :where ['[?e :account/name]]} - :args [(d/db (d/connect uri) )]}))) + :args [(dc/db conn )]}))) - bank-accounts (by :db/id (map first (d/query {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])] + bank-accounts (by :db/id (map first (dc/q {:query {:find ['(pull ?e [:db/id :bank-account/name :bank-account/numeric-code {:bank-account/type [:db/ident]}])] :in ['$] :where ['[?e :bank-account/name]]} - :args [(d/db (d/connect uri))]}))) + :args [(dc/db conn)]}))) overrides-by-client (->> accounts vals (mapcat (fn [a] @@ -171,7 +172,7 @@ :client_id client-id}))) (defn full-ledger-for-client [client-id] - (->> (d/query + (->> (dc/q {:query {:find ['?d '?jel '?account '?location '?debit '?credit] :in ['$ '?client-id] :where '[[?e :journal-entry/client ?client-id] @@ -191,7 +192,7 @@ [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] [(get-else $ ?jel :journal-entry-line/location "") ?location]] } - :args [(d/db (d/connect uri)) client-id]}) + :args [(dc/db conn) client-id]}) (sort-by first))) (defn get-balance-sheet [context args _] @@ -259,14 +260,15 @@ (defn all-ids-not-locked [all-ids] (->> all-ids - (d/q '[:find [?t ...] + (dc/q '[:find ?t :in $ [?t ...] :where [?t :journal-entry/client ?c] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [?t :journal-entry/date ?d] [(>= ?d ?lu)]] - (d/db conn)))) + (dc/db conn)) + (map first))) (defn delete-external-ledger [context args _] (let [_ (assert-admin (:id context)) @@ -275,7 +277,7 @@ (assoc :only-external true) (<-graphql) (assoc :per-page Integer/MAX_VALUE) - (#(l/raw-graphql-ids (d/db conn) %)) + (#(l/raw-graphql-ids (dc/db conn) %)) :ids) specific-ids (l/filter-ids (:ids args)) all-ids (all-ids-not-locked (into (set ids) specific-ids))] @@ -426,7 +428,6 @@ (:location ea) "'") {:status :error}))) - (when (and matching-account (not (:account/location matching-account)) (= "A" (:location ea))) @@ -513,7 +514,7 @@ (defn running-balance-for [client-id] (let [lookup-account (build-account-lookup client-id)] - (->> (d/query + (->> (dc/q {:query {:find ['?d '?e '?jel '?account '?location '?debit '?credit] :in ['$ '?client-id] :where '[[?e :journal-entry/client ?client-id] @@ -533,7 +534,7 @@ [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] [(get-else $ ?jel :journal-entry-line/location "") ?location]] } - :args [(d/db conn) client-id]}) + :args [(dc/db conn) client-id]}) (sort-by (juxt first second)) (build-running-balance lookup-account)))) @@ -541,25 +542,23 @@ (defn build-running-balance-cache [] (let [clients-needing-refresh (if-let [last-run @last-run-running-balance] - (->> (d/query + (->> (dc/q {:query {:find ['?v] - :in ['$ '?log '?since '?till] - :where ['[(tx-ids ?log ?since ?till) [?tx ...]] - '[$ _ :journal-entry/client ?v ?tx]]} - :args [(d/history (d/db conn)) - (d/log conn) - last-run - (java.util.Date.)]}) + :in ['$ '[?tx ...]] + :where ['[$ _ :journal-entry/client ?v ?tx]]} + :args [(dc/history (dc/db conn)) + (map :t (mapcat :data (dc/tx-range conn {:start last-run + :end (java.util.Date.)})))]}) (map first) - (into #{})) - (into #{} (map :db/id (d-clients/get-all)))) + (into #{})) + (into #{} (map :db/id (d-clients/get-all)))) starting (java.util.Date.)] (log/info (count clients-needing-refresh) "Clients need their balance cache refreshed.") (swap! running-balance-cache merge (reduce (fn [acc client] - (log/info "Computing running balance cache for " (:client/code (d/entity (d/db conn) client))) + (log/info "Computing running balance cache for " (pull-attr (dc/db conn) :client/code client )) (assoc acc client (running-balance-for client))) {} clients-needing-refresh)) diff --git a/src/clj/auto_ap/graphql/plaid.clj b/src/clj/auto_ap/graphql/plaid.clj index 558d065b..9e887b23 100644 --- a/src/clj/auto_ap/graphql/plaid.clj +++ b/src/clj/auto_ap/graphql/plaid.clj @@ -1,7 +1,13 @@ (ns auto-ap.graphql.plaid (:require [auto-ap.datomic - :refer [add-sorter-fields apply-pagination apply-sort-3 conn merge-query]] + :refer [add-sorter-fields + apply-pagination + apply-sort-3 + conn + merge-query + pull-attr + pull-many-by-id]] [auto-ap.graphql.utils :refer [->graphql <-graphql @@ -14,13 +20,15 @@ [clj-time.coerce :as coerce] [clj-time.core :as time] [clojure.tools.logging :as log] + [datomic.client.api :as dc] + [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [datomic.api :as d])) (defn plaid-link-token [context value _] (when-not (:client_id value) (throw (ex-info "Client ID is required" {:validation-error "Client ID is required"}))) (assert-can-see-client (:id context) (:client_id value)) - (let [client-code (:client/code (d/pull (d/db conn) [:client/code] (:client_id value)))] + (let [client-code (pull-attr (dc/db conn) :client/code (:client_id value))] {:token (p/get-link-token client-code)})) (defn link-plaid [context value _] @@ -28,8 +36,8 @@ (throw (ex-info "Client not provided" {:validation-error "Client not provided."}))) (when-not (:public_token value) (throw (ex-info "Public token not provided" {:validation-error "public token not provided"}))) - (log/info (:id context) (:db/id (d/pull (d/db conn) [:db/id] [:client/code (:client_code value)]))) - (assert-can-see-client (:id context) (:db/id (d/pull (d/db conn) [:db/id] [:client/code (:client_code value)]))) + (log/info (:id context) (pull-attr (dc/db conn) :db/id [:client/code (:client_code value)])) + (assert-can-see-client (:id context) (pull-attr (dc/db conn) :db/id [:client/code (:client_code value)])) (let [access-token (:access_token (p/exchange-public-token (:public_token value) (:client_code value))) account-result (p/get-accounts access-token ) item {:plaid-item/client [:client/code (:client_code value)] @@ -40,15 +48,16 @@ :plaid-item/last-updated (coerce/to-date (time/now)) :db/id "plaid-item"}] - @(d/transact conn (->> (:accounts account-result) - (map (fn [a] - (let [balance (some-> a :balances :current (* 0.01))] - (cond-> {:plaid-account/external-id (:account_id a) - :plaid-account/number (:mask a) - :plaid-account/name (str (:name a) " " (:mask a)) - :plaid-item/_accounts "plaid-item"} - balance (assoc :plaid-account/balance balance))))) - (into [item]))) + (dc/transact conn {:tx-data + (->> (:accounts account-result) + (map (fn [a] + (let [balance (some-> a :balances :current (* 0.01))] + (cond-> {:plaid-account/external-id (:account_id a) + :plaid-account/number (:mask a) + :plaid-account/name (str (:name a) " " (:mask a)) + :plaid-item/_accounts "plaid-item"} + balance (assoc :plaid-account/balance balance))))) + (into [item]))}) (log/info "Access token was " access-token) {:message (str "Plaid linked successfully.")})) @@ -87,19 +96,17 @@ :where ['[?e :plaid-item/external-id]]}}))] (cond->> query - true (d/query) + true (dc/q) true (apply-sort-3 args) true (apply-pagination args)))) (defn graphql-results [ids db _] - (let [results (->> (d/pull-many db default-read ids) - (group-by :db/id))] + (let [results (pull-many-by-id db default-read ids)] (->> ids - (map results) - (map first)))) + (map results)))) (defn get-graphql [args] - (let [db (d/db conn) + (let [db (dc/db conn) {ids-to-retrieve :ids matching-count :count} (raw-graphql-ids db args)] [(graphql-results ids-to-retrieve db args) matching-count])) @@ -121,7 +128,7 @@ (defn delete-plaid-item [context args _] (assert-admin (:id context)) (assert-present args :id) - @(d/transact conn [[:db/retractEntity (:id args)]]) + (dc/transact conn {:tx-data [[:db/retractEntity (:id args)]]}) {:message "Item deleted."}) (defn attach [schema] diff --git a/src/clj/auto_ap/graphql/reports.clj b/src/clj/auto_ap/graphql/reports.clj index 0f070840..f356f7c6 100644 --- a/src/clj/auto_ap/graphql/reports.clj +++ b/src/clj/auto_ap/graphql/reports.clj @@ -6,7 +6,7 @@ [auto-ap.graphql.utils :refer [<-graphql assert-admin attach-tracing-resolvers result->page]] [config.core :refer [env]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn get-report-page [context args _] (let [args (assoc args :id (:id context)) @@ -17,15 +17,15 @@ (defn delete-report [context args _] (assert-admin (:id context)) - (let [[id-to-delete key] (first (d/q '[:find ?i ?k + (let [[id-to-delete key] (first (dc/q '[:find ?i ?k :in $ ?i :where [?i :report/key ?k]] - (d/db conn) + (dc/db conn) (:id args)))] (when id-to-delete (s3/delete-object :bucket-name (:data-bucket env) :key key) - @(d/transact conn [[:db/retractEntity id-to-delete]])) + (dc/transact conn {:tx-data [[:db/retractEntity id-to-delete]]})) {:message (format "deleted %s successfully" key)})) diff --git a/src/clj/auto_ap/graphql/transaction_rules.clj b/src/clj/auto_ap/graphql/transaction_rules.clj index 4c3064f3..08e236a5 100644 --- a/src/clj/auto_ap/graphql/transaction_rules.clj +++ b/src/clj/auto_ap/graphql/transaction_rules.clj @@ -1,11 +1,6 @@ (ns auto-ap.graphql.transaction-rules (:require - [auto-ap.datomic - :refer [audit-transact - conn - merge-query - remove-nils - replace-nils-with-retract]] + [auto-ap.datomic :refer [audit-transact conn merge-query upsert-entity]] [auto-ap.datomic.transaction-rules :as tr] [auto-ap.datomic.transactions :as d-transactions] [auto-ap.graphql.utils @@ -20,7 +15,7 @@ [auto-ap.utils :refer [dollars=]] [clj-time.coerce :as c] [clojure.string :as str] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn get-transaction-rule-page [context args _] (let [args (assoc args :id (:id context)) @@ -37,10 +32,10 @@ nil)) (defn transaction-rule-account->entity [{:keys [id account_id percentage location]}] - (remove-nils #:transaction-rule-account {:percentage percentage - :db/id id - :account account_id - :location location})) + #:transaction-rule-account {:percentage percentage + :db/id id + :account account_id + :location location}) (defn delete-transaction-rule [context {:keys [transaction_rule_id ]} _] (assert-admin (:id context)) @@ -53,8 +48,7 @@ (defn upsert-transaction-rule [context {{:keys [id description yodlee_merchant_id note client_id bank_account_id amount_lte amount_gte vendor_id accounts transaction_approval_status dom_gte dom_lte]} :transaction_rule} _] (assert-admin (:id context)) - (let [existing-transaction (tr/get-by-id id) - account-total (reduce + 0 (map (fn [x] (:percentage x)) accounts)) + (let [account-total (reduce + 0 (map (fn [x] (:percentage x)) accounts)) _ (try (. java.util.regex.Pattern (compile description java.util.regex.Pattern/CASE_INSENSITIVE)) (catch Exception e @@ -67,8 +61,8 @@ (let [error (str "You must provide a description or a yodlee merchant")] (throw (ex-info error {:validation-error error})))) _ (doseq [a accounts - :let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a)) - client (d/entity (d/db conn) client_id) + :let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn) [:account/location :account/name] (:account_id a)) + client (dc/pull (dc/db conn) [:client/locations] client_id) ]] (when (and location (not= location (:location a))) (let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)] @@ -82,27 +76,25 @@ rule-id (if id id "transaction-rule") - transaction (replace-nils-with-retract #:transaction-rule {:db/id rule-id - :description description - :note note - :client client_id - :bank-account bank_account_id - :yodlee-merchant yodlee_merchant_id - :dom-lte dom_lte - :dom-gte dom_gte - :amount-lte amount_lte - :amount-gte amount_gte - :vendor vendor_id - :transaction-approval-status - (some->> transaction_approval_status - name - snake->kebab - (keyword "transaction-approval-status"))} - existing-transaction) + transaction [`(upsert-entity ~#:transaction-rule {:db/id rule-id + :description description + :note note + :client client_id + :bank-account bank_account_id + :yodlee-merchant yodlee_merchant_id + :dom-lte dom_lte + :dom-gte dom_gte + :amount-lte amount_lte + :amount-gte amount_gte + :vendor vendor_id + :transaction-approval-status + (some->> transaction_approval_status + name + snake->kebab + (keyword "transaction-approval-status")) + :transaction-rule/accounts (map transaction-rule-account->entity accounts)})] - transaction (conj transaction - [:reset rule-id :transaction-rule/accounts (map transaction-rule-account->entity accounts)]) transaction-result (audit-transact transaction (:id context))] (-> (tr/get-by-id (or (-> transaction-result :tempids (get "transaction-rule")) id)) @@ -111,14 +103,14 @@ (defn -test-transaction-rule [id {:keys [:transaction-rule/description :transaction-rule/client :transaction-rule/bank-account :transaction-rule/amount-lte :transaction-rule/amount-gte :transaction-rule/dom-lte :transaction-rule/dom-gte :transaction-rule/yodlee-merchant]} include-coded? count] (->> - (d/query + (dc/q (cond-> {:query {:find ['(pull ?e [* {:transaction/client [:client/name] :transaction/bank-account [:bank-account/name] :transaction/payment [:db/id]} ])] :in ['$ ] :where []} - :args [(d/db conn)]} + :args [(dc/db conn)]} (limited-clients id) (merge-query {:query {:in ['[?xx ...]] diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index 3512637d..a7955966 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -1,6 +1,14 @@ (ns auto-ap.graphql.transactions (:require - [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.datomic + :refer [audit-transact + audit-transact-batch + conn + pull-attr + pull-many + pull-ref + remove-nils + upsert-entity]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.invoices :as d-invoices] @@ -27,6 +35,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] + [datomic.client.api :as dc] [datomic.api :as d] [auto-ap.graphql.utils :refer [attach-tracing-resolvers]])) @@ -74,14 +83,15 @@ (defn all-ids-not-locked [all-ids] (->> all-ids - (d/q '[:find [?t ...] + (dc/q '[:find ?t :in $ [?t ...] :where [?t :transaction/client ?c] [(get-else $ ?c :client/locked-until #inst "2000-01-01") ?lu] [?t :transaction/date ?d] [(>= ?d ?lu)]] - (d/db conn)))) + (dc/db conn)) + (map first))) (defn bulk-change-status [context args _] (let [_ (assert-admin (:id context)) args (assoc args :id (:id context)) @@ -140,11 +150,11 @@ (throw (ex-info "Client is required" {:validation-error "client is required"}))) (let [args (assoc args :id (:id context)) - locations (:client/locations (d/pull (d/db conn) - [:client/locations] - (:client_id args))) + locations (pull-attr (dc/db conn) + :client/locations + (:client_id args)) all-ids (all-ids-not-locked (get-ids-matching-filters args)) - transactions (d/pull-many (d/db conn) '[:db/id :transaction/amount] (vec all-ids)) + transactions (pull-many (dc/db conn) [:db/id :transaction/amount] (vec all-ids)) account-total (reduce + 0 (map (fn [x] (:percentage x)) (:accounts args)))] (log/info "client is" locations) @@ -156,7 +166,9 @@ (throw (ex-info error {:validation-error error})))) (doseq [a (:accounts args) - :let [{:keys [:account/location :account/name]} (d/entity (d/db conn) (:account_id a))]] + :let [{:keys [:account/location :account/name]} (dc/pull (dc/db conn) + [:account/location :account/name] + (:account_id a))]] (when (and location (not= location (:location a))) (let [err (str "Account " name " uses location " (:location a) ", but is supposed to be " location)] (throw (ex-info err {:validation-error err}) ))) @@ -184,12 +196,14 @@ (let [_ (assert-admin (:id context)) args (assoc args :id (:id context)) all-ids (all-ids-not-locked (get-ids-matching-filters args)) - db (d/db conn)] + db (dc/db conn)] (log/info "Deleting " (count all-ids) args) (transact-batch-with-ledger (mapcat (fn [i] - (let [transaction (d/entity db i) + (let [transaction (dc/pull db [:transaction/payment + :transaction/expected-deposit + :db/id] i) payment-id (-> transaction :transaction/payment :db/id) expected-deposit-id (-> transaction :transaction/expected-deposit :db/id) transaction-tx (if (:suppress args) @@ -233,7 +247,7 @@ (let [_ (assert-power-user (:id context)) args (assoc args :id (:id context)) transaction-id (:transaction_id args) - transaction (d/pull (d/db conn) + transaction (dc/pull (dc/db conn) [:transaction/approval-status :transaction/status :transaction/date @@ -249,12 +263,12 @@ (assert-not-locked (:db/id (:transaction/client transaction)) (-> transaction :transaction/payment :payment/date))) _ (log/info "Unlinking" transaction) payment (-> transaction :transaction/payment ) - is-autopay-payment? (some->> (doto (d/query {:query {:find ['?sp] + is-autopay-payment? (some->> (doto (dc/q {:query {:find ['?sp] :in ['$ '?payment] :where ['[?ip :invoice-payment/payment ?payment] '[?ip :invoice-payment/invoice ?i] '[(get-else $ ?i :invoice/scheduled-payment "N/A") ?sp]]} - :args [(d/db conn) (:db/id payment)]}) + :args [(dc/db conn) (:db/id payment)]}) log/info) seq (map first) @@ -288,10 +302,10 @@ true (into (map (fn [[invoice-payment]] [:db/retractEntity invoice-payment]) - (d/query {:query {:find ['?ip] - :in ['$ '?p] - :where ['[?ip :invoice-payment/payment ?p]]} - :args [(d/db conn) (:db/id payment)]} )))) + (dc/q {:query {:find ['?ip] + :in ['$ '?p] + :where ['[?ip :invoice-payment/payment ?p]]} + :args [(dc/db conn) (:db/id payment)]} )))) (:id context)) (transact-with-ledger (into (cond-> [{:db/id (:db/id payment) @@ -314,25 +328,17 @@ (defn transaction-account->entity [{:keys [id account_id amount location]}] - (remove-nils #:transaction-account {:amount amount - :db/id id - :account account_id - :location location})) + #:transaction-account {:amount amount + :db/id id + :account account_id + :location location}) -(defn deleted-accounts [transaction accounts] - (let [current-accounts (:transaction/accounts transaction) - specified-ids (->> accounts - (map :id) - set) - existing-ids (->> current-accounts - (map :db/id) - set)] - (set/difference existing-ids specified-ids))) - (defn assert-valid-expense-accounts [accounts] (doseq [trans-account accounts - :let [account (d/entity (d/db conn) (:account_id trans-account))]] + :let [account (dc/pull (dc/db conn) + [:account/location] + (:account_id trans-account))]] (when (empty? (:location trans-account)) (throw (ex-info "Account is missing location" {:validation-error "Account is missing location"}))) @@ -358,7 +364,6 @@ _ (assert-can-see-client (:id context) (:transaction/client existing-transaction) ) _ (assert-valid-expense-accounts accounts) _ (assert-not-locked (:db/id (:transaction/client existing-transaction)) (:transaction/date existing-transaction)) - deleted (deleted-accounts existing-transaction accounts) account-total (reduce + 0 (map (fn [x] (:amount x)) accounts)) missing-locations (seq (set/difference (->> accounts @@ -376,27 +381,14 @@ (when missing-locations (throw (ex-info (str "Location '" (str/join ", " missing-locations) "' not found on client.") {})) ) - (transact-with-ledger (concat [(remove-nils {:db/id id - :transaction/vendor vendor_id - :transaction/approval-status (some->> approval_status - name - snake->kebab - (keyword "transaction-approval-status")) - :transaction/accounts (map transaction-account->entity accounts) - }) - ] - (cond forecast_match - [[:db/add id :transaction/forecast-match forecast_match]] - - (:db/id (:transaction/forecast-match existing-transaction)) - [[:db/retract id :transaction/forecast-match (:db/id (:transaction/forecast-match existing-transaction))]] - - :else - []) - - (map (fn [d] - [:db/retract id :transaction/accounts d]) - deleted)) + (transact-with-ledger [`(upsert-entity ~{:db/id id + :transaction/vendor vendor_id + :transaction/approval-status (some->> approval_status + name + snake->kebab + (keyword "transaction-approval-status")) + :transaction/accounts (map transaction-account->entity accounts) + :transaction/forecast-match forecast_match})] (:id context)) (-> (d-transactions/get-by-id id) approval-status->graphql @@ -440,9 +432,9 @@ (let [_ (assert-power-user (:id context)) transaction (d-transactions/get-by-id transaction_id) _ (assert-can-see-client (:id context) (:transaction/client transaction) ) - db (d/db conn) - invoice-clients (set (map (comp :db/id :invoice/client #(d/entity db %)) autopay_invoice_ids)) - invoice-amount (reduce + 0.0 (map (comp :invoice/total #(d/entity db %)) autopay_invoice_ids)) + db (dc/db conn) + invoice-clients (set (map #(pull-ref db :invoice/client %) autopay_invoice_ids)) + invoice-amount (reduce + 0.0 (map #(pull-attr db :invoice/total %) autopay_invoice_ids)) _ (assert-not-locked (:db/id (:transaction/client transaction)) (:transaction/date transaction))] (when (:transaction/payment transaction) (throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"}))) @@ -454,22 +446,15 @@ (when-not (dollars= (- (:transaction/amount transaction)) invoice-amount) (throw (ex-info "Amounts don't match" {:validation-error "Amounts don't match"}))) - #_(log/info [#_(select-keys (d/entity db transaction_id) #{:transaction/amount :db/id})] - (->> autopay_invoice_ids - (map (fn [id] - (let [entity (d/entity db id)] - [(-> entity :invoice/vendor :db/id) - (-> entity :db/id) - (-> entity :invoice/total)]))))) - (let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] - (map (fn [id] - (let [entity (d/entity db id)] - [(-> entity :invoice/vendor :db/id) - (-> entity :db/id) - (-> entity :invoice/total)])) - autopay_invoice_ids) - (:db/id (:transaction/bank-account transaction)) - (:db/id (:transaction/client transaction)))] + (let [payment-tx (i-transactions/add-new-payment [(dc/pull db [:transaction/amount :transaction/date :db/id] transaction_id)] + (map (fn [id] + (let [entity (dc/pull db [:invoice/vendor :db/id :invoice/total] id)] + [(-> entity :invoice/vendor :db/id) + (-> entity :db/id) + (-> entity :invoice/total)])) + autopay_invoice_ids) + (:db/id (:transaction/bank-account transaction)) + (:db/id (:transaction/client transaction)))] (log/info "Adding a new payment" payment-tx) (transact-with-ledger payment-tx (:id context))) @@ -483,9 +468,9 @@ _ (assert-can-see-client (:id context) (:transaction/client transaction) ) _ (assert-not-locked (:db/id (:transaction/client transaction)) (:transaction/date transaction)) - db (d/db conn) - invoice-clients (set (map (comp :db/id :invoice/client #(d/entity db %)) unpaid_invoice_ids)) - invoice-amount (reduce + 0.0 (map (comp :invoice/outstanding-balance #(d/entity db %)) unpaid_invoice_ids))] + db (dc/db conn) + invoice-clients (set (map #(pull-ref db :invoice/client %) unpaid_invoice_ids)) + invoice-amount (reduce + 0.0 (map #(pull-attr db :invoice/outstanding-balance %) unpaid_invoice_ids))] (when (or (> (count invoice-clients) 1) (not= (:db/id (:transaction/client transaction)) (first invoice-clients))) @@ -498,9 +483,9 @@ (when (:transaction/payment transaction) (throw (ex-info "Transaction already linked" {:validation-error "Transaction already linked"}))) - (let [payment-tx (i-transactions/add-new-payment [(select-keys (d/entity db transaction_id) #{:transaction/amount :transaction/date :db/id})] + (let [payment-tx (i-transactions/add-new-payment [(dc/pull db [:transaction/amount :transaction/date :db/id] transaction_id)] (map (fn [id] - (let [entity (d/entity db id)] + (let [entity (dc/pull db [:invoice/vendor :db/id :invoice/total] id)] [(-> entity :invoice/vendor :db/id) (-> entity :db/id) (-> entity :invoice/total)])) diff --git a/src/clj/auto_ap/graphql/utils.clj b/src/clj/auto_ap/graphql/utils.clj index 8482bff5..e04ab46a 100644 --- a/src/clj/auto_ap/graphql/utils.clj +++ b/src/clj/auto_ap/graphql/utils.clj @@ -4,7 +4,7 @@ [clj-time.coerce :as coerce] [auto-ap.time :as atime] [buddy.auth :refer [throw-unauthorized]] - [datomic.api :as d] + [datomic.client.api :as dc] [clojure.walk :as walk] [com.walmartlabs.lacinia.util :refer [attach-resolvers]] [clojure.tools.logging :as log] @@ -113,7 +113,7 @@ (some->> e name snake->kebab (keyword namespace))) (defn get-locked-until [client-id] - (:client/locked-until (d/pull (d/db conn) [:client/locked-until] client-id))) + (:client/locked-until (dc/pull (dc/db conn) [:client/locked-until] client-id))) (defn assert-not-locked [client-id date] (let [locked-until (get-locked-until client-id)] diff --git a/src/clj/auto_ap/graphql/vendors.clj b/src/clj/auto_ap/graphql/vendors.clj index db43d1d8..30790a8b 100644 --- a/src/clj/auto_ap/graphql/vendors.clj +++ b/src/clj/auto_ap/graphql/vendors.clj @@ -1,6 +1,6 @@ (ns auto-ap.graphql.vendors (:require - [auto-ap.datomic :refer [audit-transact conn remove-nils]] + [auto-ap.datomic :refer [audit-transact conn pull-attr remove-nils]] [auto-ap.datomic.vendors :as d-vendors] [auto-ap.graphql.utils :refer [->graphql @@ -14,20 +14,22 @@ [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] - [datomic.api :as d])) + [datomic.client.api :as dc] + [auto-ap.search :as search])) (defn can-user-edit-vendor? [vendor-id id] (if (is-admin? id) true (empty? - (set/difference (set (d/q '[:find [?c ...] - :in $ ?v - :where [?vu :vendor-usage/vendor ?v] - [?vu :vendor-usage/client ?c] - [?vu :vendor-usage/count ?d] - [(>= ?d 0)]] - (d/db conn) - vendor-id)) + (set/difference (set (->> (dc/q '[:find ?c + :in $ ?v + :where [?vu :vendor-usage/vendor ?v] + [?vu :vendor-usage/client ?c] + [?vu :vendor-usage/count ?d] + [(>= ?d 0)]] + (dc/db conn) + vendor-id) + (map first))) (set (map :db/id (:user/clients id))))))) (defn upsert-vendor [context {{:keys [id name hidden terms code print_as primary_contact secondary_contact address default_account_id invoice_reminder_schedule schedule_payment_dom terms_overrides account_overrides] :as in} :vendor} _] @@ -59,8 +61,7 @@ hidden (if (is-admin? (:id context)) hidden false) - existing (when id - (d/pull (d/db conn) '[:vendor/name] id)) + existing (pull-attr (dc/db conn) :vendor/name id) terms-overrides (mapv (fn [to] (cond-> @@ -145,11 +146,11 @@ (->graphql)))) (defn merge-vendors [context {:keys [from to]} _] - (let [transaction (->> (d/query {:query {:find '[?x ?a2] + (let [transaction (->> (dc/q {:query {:find '[?x ?a2] :in '[$ ?vendor-from ] :where ['[?x ?a ?vendor-from] '[?a :db/ident ?a2]]} - :args [(d/db conn) from]}) + :args [(dc/db conn) from]}) (mapcat (fn [[src attr]] [[:db/retract src attr from] @@ -182,6 +183,13 @@ matches)) (defn search [context args _] + (comment (let [search-query (cleanse-query (:query args))] + (for [[id name] (search/search (cond-> {:q search-query} + (not (is-admin? (:id context))) (assoc :hidden false)) + "vendors")] + {:name name + :id (Long/parseLong id)}) + )) (if-let [search-query (cleanse-query (:query args))] (let [data (if (is-admin? (:id context)) (d/q '[:find ?n ?i ?s diff --git a/src/clj/auto_ap/import/common.clj b/src/clj/auto_ap/import/common.clj index 1307bc8c..65fdd80b 100644 --- a/src/clj/auto_ap/import/common.clj +++ b/src/clj/auto_ap/import/common.clj @@ -1,28 +1,29 @@ (ns auto-ap.import.common (:require - [auto-ap.datomic :refer [conn]] + [auto-ap.datomic :refer [conn pull-ref random-tempid]] [clojure.tools.logging :as log] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn bank-account->integration-id [bank-account] - (or (->> bank-account - (d/pull (d/db conn) [:bank-account/integration-status]) - :bank-account/integration-status - :db/id) - #db/id[:db.part/user])) + (or (pull-ref (dc/db conn) :bank-account/integration-status bank-account) + (random-tempid))) (defn wrap-integration [f bank-account] (try (let [result (f)] - @(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account) - :integration-status/state :integration-state/success - :integration-status/last-attempt (java.util.Date.) - :integration-status/last-updated (java.util.Date.)}}]) + (dc/transact conn {:tx-data [{:db/id bank-account + :bank-account/integration-status + {:db/id (bank-account->integration-id bank-account) + :integration-status/state :integration-state/success + :integration-status/last-attempt (java.util.Date.) + :integration-status/last-updated (java.util.Date.)}}]}) result) (catch Exception e - @(d/transact conn [{:db/id bank-account :bank-account/integration-status {:db/id (bank-account->integration-id bank-account) - :integration-status/state :integration-state/failed - :integration-status/last-attempt (java.util.Date.) - :integration-status/message (.getMessage e)}}]) + (dc/transact conn {:tx-data [{:db/id bank-account + :bank-account/integration-status + {:db/id (bank-account->integration-id bank-account) + :integration-status/state :integration-state/failed + :integration-status/last-attempt (java.util.Date.) + :integration-status/message (.getMessage e)}}]}) (log/warn e) nil))) diff --git a/src/clj/auto_ap/import/intuit.clj b/src/clj/auto_ap/import/intuit.clj index f4c02166..a0869201 100644 --- a/src/clj/auto_ap/import/intuit.clj +++ b/src/clj/auto_ap/import/intuit.clj @@ -10,10 +10,12 @@ [clojure.string :as str] [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] - [datomic.api :as d])) + [datomic.client.api :as dc] + [mount.core :as mount] + [yang.scheduler :as scheduler])) (defn get-intuit-bank-accounts [db] - (d/q '[:find ?external-id ?ba ?c + (dc/q '[:find ?external-id ?ba ?c :in $ :where [?c :client/bank-accounts ?ba] @@ -49,7 +51,7 @@ :priority :low} nil) (let [import-batch (t/start-import-batch :import-source/intuit "Automated intuit user") - db (d/db conn) + db (dc/db conn) end (auto-ap.time/local-now) start (time/plus end (time/days -30))] (try @@ -76,8 +78,8 @@ (defn upsert-accounts [] (let [token (i/get-fresh-access-token) bank-accounts (i/get-bank-accounts token)] - @(d/transact conn (mapv - (fn [ba] - {:intuit-bank-account/external-id (:name ba) - :intuit-bank-account/name (:name ba)}) - bank-accounts)))) + (dc/transact conn {:tx-data (mapv + (fn [ba] + {:intuit-bank-account/external-id (:name ba) + :intuit-bank-account/name (:name ba)}) + bank-accounts)}))) diff --git a/src/clj/auto_ap/import/manual.clj b/src/clj/auto_ap/import/manual.clj index f32534d3..90d82844 100644 --- a/src/clj/auto_ap/import/manual.clj +++ b/src/clj/auto_ap/import/manual.clj @@ -5,7 +5,7 @@ [auto-ap.import.transactions :as t] [clj-time.coerce :as coerce] [clojure.data.csv :as csv] - [datomic.api :as d] + [datomic.client.api :as dc] [unilog.context :as lc])) @@ -35,17 +35,17 @@ (defn import-batch [transactions user] (lc/with-context {:source "Manual import transactions"} (let [bank-account-code->client (into {} - (d/q '[:find ?bac ?c + (dc/q '[:find ?bac ?c :in $ :where [?c :client/bank-accounts ?ba] [?ba :bank-account/code ?bac]] - (d/db conn))) + (dc/db conn))) bank-account-code->bank-account (into {} - (d/q '[:find ?bac ?ba + (dc/q '[:find ?bac ?ba :in $ :where [?ba :bank-account/code ?bac]] - (d/db conn))) + (dc/db conn))) import-batch (t/start-import-batch :import-source/manual user) transactions (->> transactions (map (fn [t] diff --git a/src/clj/auto_ap/import/plaid.clj b/src/clj/auto_ap/import/plaid.clj index 1e173da7..39bf95b7 100644 --- a/src/clj/auto_ap/import/plaid.clj +++ b/src/clj/auto_ap/import/plaid.clj @@ -8,12 +8,12 @@ [auto-ap.utils :refer [allow-once by]] [clj-time.coerce :as coerce] [clj-time.core :as time] - [datomic.api :as d] + [datomic.client.api :as dc] [digest :as di] [unilog.context :as lc])) (defn get-plaid-accounts [db] - (-> (d/q '[:find ?ba ?c ?external-id ?t + (-> (dc/q '[:find ?ba ?c ?external-id ?t :in $ :where [?c :client/bank-accounts ?ba] @@ -42,7 +42,7 @@ end (atime/local-now) start (time/plus end (time/days -30))] (try - (doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (d/db conn)) + (doseq [[bank-account-id client-id external-id access-token] (get-plaid-accounts (dc/db conn)) :let [transaction-result (wrap-integration #(p/get-transactions access-token external-id start end) bank-account-id) accounts-by-id (by :account_id (:accounts transaction-result))] diff --git a/src/clj/auto_ap/import/transactions.clj b/src/clj/auto_ap/import/transactions.clj index 06b3521c..94d8699a 100644 --- a/src/clj/auto_ap/import/transactions.clj +++ b/src/clj/auto_ap/import/transactions.clj @@ -1,6 +1,6 @@ (ns auto-ap.import.transactions (:require - [auto-ap.datomic :refer [conn remove-nils uri]] + [auto-ap.datomic :refer [audit-transact conn random-tempid remove-nils]] [auto-ap.datomic.accounts :as a] [auto-ap.datomic.checks :as d-checks] [auto-ap.datomic.transaction-rules :as tr] @@ -13,7 +13,7 @@ [clj-time.core :as t] [clojure.core.cache :as cache] [clojure.tools.logging :as log] - [datomic.api :as d] + [datomic.client.api :as dc] [digest :as di])) (defn rough-match [client-id bank-account-id amount] @@ -56,19 +56,19 @@ (defn match-transaction-to-unfulfilled-autopayments [amount client-id] (log/info "trying to find uncleared autopay invoices") - (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?total '?sd] - :in ['$ '?client-id] - :where ['[?e :invoice/client ?client-id] - '[?e :invoice/scheduled-payment ?sd] - '[?e :invoice/status :invoice-status/paid] - '(not [_ :invoice-payment/invoice ?e]) - '[?e :invoice/vendor ?vendor-id] - '[?e :invoice/total ?total]]} - :args [(d/db conn) client-id]}) + (let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?total '?sd] + :in ['$ '?client-id] + :where ['[?e :invoice/client ?client-id] + '[?e :invoice/scheduled-payment ?sd] + '[?e :invoice/status :invoice-status/paid] + '(not [_ :invoice-payment/invoice ?e]) + '[?e :invoice/vendor ?vendor-id] + '[?e :invoice/total ?total]]} + :args [(dc/db conn) client-id]}) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) - considerations (for [candidate-invoices candidate-invoices-vendor-groups + considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] @@ -81,19 +81,19 @@ (defn match-transaction-to-unpaid-invoices [amount client-id] (log/info "trying to find unpaid invoices for " client-id amount) - (let [candidate-invoices-vendor-groups (->> (d/query {:query {:find ['?vendor-id '?e '?outstanding-balance '?d] - :in ['$ '?client-id] - :where ['[?e :invoice/client ?client-id] - '[?e :invoice/status :invoice-status/unpaid] - '(not [_ :invoice-payment/invoice ?e]) - '[?e :invoice/vendor ?vendor-id] - '[?e :invoice/outstanding-balance ?outstanding-balance] - '[?e :invoice/date ?d]]} - :args [(d/db conn) client-id]}) + (let [candidate-invoices-vendor-groups (->> (dc/q {:query {:find ['?vendor-id '?e '?outstanding-balance '?d] + :in ['$ '?client-id] + :where ['[?e :invoice/client ?client-id] + '[?e :invoice/status :invoice-status/unpaid] + '(not [_ :invoice-payment/invoice ?e]) + '[?e :invoice/vendor ?vendor-id] + '[?e :invoice/outstanding-balance ?outstanding-balance] + '[?e :invoice/date ?d]]} + :args [(dc/db conn) client-id]}) (sort-by last) ;; sort by scheduled payment date (group-by first) ;; group by vendors vals) - considerations (for [candidate-invoices candidate-invoices-vendor-groups + considerations (for [candidate-invoices candidate-invoices-vendor-groups invoice-count (range 1 32) consideration (partition invoice-count 1 candidate-invoices) :when (dollars= (reduce (fn [acc [_ _ amount]] @@ -111,7 +111,7 @@ (defn add-new-payment [[transaction :as tx] [[vendor] :as invoice-payments] bank-account-id client-id] (log/info "Adding a new payment for transaction " (:transaction/id transaction) " and invoices " invoice-payments) - (let [payment-id (d/tempid :db.part/user)] + (let [payment-id (random-tempid)] (-> tx (conj {:payment/bank-account bank-account-id @@ -142,7 +142,6 @@ :amount (Math/abs (:transaction/amount transaction))}]])))) (defn extract-check-number [{:transaction/keys [description-original]}] - (if-let [[_ _ check-number] (re-find #"(?i)check(card|[^0-9]+([0-9]*))" description-original)] (try (Integer/parseInt check-number) @@ -152,8 +151,8 @@ (defn find-expected-deposit [client-id amount date] (when date - (-> (d/q - '[:find [(pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}]) ...] + (-> (dc/q + '[:find (pull ?ed [:db/id {:expected-deposit/vendor [:db/id]}]) :in $ ?c ?a ?d-start :where [?ed :expected-deposit/client ?c] @@ -163,8 +162,8 @@ [?ed :expected-deposit/total ?a2] [(auto-ap.utils/dollars= ?a2 ?a)] ] - (d/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) - first))) + (dc/db conn) client-id amount (coerce/to-date (t/plus date (t/days -10)))) + ffirst))) (defn categorize-transaction [transaction bank-account existing] @@ -242,7 +241,7 @@ )))) (defn maybe-code [{:transaction/keys [client amount] :as transaction} apply-rules valid-locations] - (when-not (seq (match-transaction-to-unpaid-invoices amount client)) + (when (seq (match-transaction-to-unpaid-invoices amount client)) (apply-rules transaction valid-locations))) (defn transaction->txs [transaction bank-account apply-rules] @@ -267,13 +266,13 @@ (defn get-existing [bank-account] (log/info "looking up bank account data for" bank-account) (into {} - (d/query {:query {:find ['?tid '?as2] + (dc/q {:query {:find ['?tid '?as2] :in ['$ '?ba] :where ['[?e :transaction/bank-account ?ba] '[?e :transaction/id ?tid] '[?e :transaction/approval-status ?as] '[?as :db/ident ?as2]]} - :args [(d/db (d/connect uri)) bank-account]}))) + :args [(dc/db conn) bank-account]}))) (defprotocol ImportBatch (import-transaction! [this transaction]) @@ -288,17 +287,17 @@ :import-batch/not-ready 0 :import-batch/extant 0}) extant-cache (atom (cache/ttl-cache-factory {} :ttl 60000 )) - import-id (get (:tempids @(d/transact conn [{:db/id "import-batch" - :import-batch/date (coerce/to-date (t/now)) - :import-batch/source source - :import-batch/status :import-status/started - :import-batch/user-name user}])) "import-batch") + import-id (get (:tempids (dc/transact conn {:tx-data [{:db/id "import-batch" + :import-batch/date (coerce/to-date (t/now)) + :import-batch/source source + :import-batch/status :import-status/started + :import-batch/user-name user}]})) "import-batch") rule-applying-function (rm/rule-applying-fn (tr/get-all))] (log/info "Importing transactions from " source) (reify ImportBatch (import-transaction! [_ transaction] - (let [bank-account (d/pull (d/db conn) + (let [bank-account (dc/pull (dc/db conn) [:bank-account/code :db/id :bank-account/locations @@ -327,14 +326,14 @@ (fail! [_ error] (log/errorf "Couldn't complete import %d with error." import-id) (log/error error) - @(d/transact conn [(merge {:db/id import-id - :import-batch/status :import-status/completed - :import-batch/error-message (str error)} - @stats)])) + (dc/transact conn {:tx-data [(merge {:db/id import-id + :import-batch/status :import-status/completed + :import-batch/error-message (str error)} + @stats)]})) (finish! [_] (log/infof "Finishing import batch %d for %s with stats %s " import-id (name source) (pr-str @stats)) - @(d/transact conn [(merge {:db/id import-id + (dc/transact conn [(merge {:db/id import-id :import-batch/status :import-status/completed} @stats)]))))) diff --git a/src/clj/auto_ap/import/yodlee2.clj b/src/clj/auto_ap/import/yodlee2.clj index 5dc0712b..103e6b23 100644 --- a/src/clj/auto_ap/import/yodlee2.clj +++ b/src/clj/auto_ap/import/yodlee2.clj @@ -9,9 +9,10 @@ [clj-time.coerce :as coerce] [clojure.string :as str] [com.unbounce.dogstatsd.core :as statsd] - [datomic.api :as d] - [digest :as di] - [unilog.context :as lc])) + [datomic.client.api :as dc] + [mount.core :as mount] + [unilog.context :as lc] + [yang.scheduler :as scheduler])) #_{:clj-kondo/ignore [:unresolved-var]} (defn yodlee->transaction [transaction use-date-instead-of-post-date?] @@ -51,7 +52,7 @@ nil) (let [import-batch (t/start-import-batch :import-source/yodlee2 "Automated yodlee2 user")] (try - (let [account-lookup (d/q '[:find ?ya ?ba ?cd ?ud + (let [account-lookup (dc/q '[:find ?ya ?ba ?cd ?ud :in $ :where [?ba :bank-account/yodlee-account ?y] @@ -60,7 +61,7 @@ [?c :client/code ?cd] [?y :yodlee-account/id ?ya] ] - (d/db conn))] + (dc/db conn))] (doseq [[yodlee-account bank-account client-code use-date-instead-of-post-date?] account-lookup transaction (wrap-integration #(client2/get-specific-transactions client-code yodlee-account) bank-account)] diff --git a/src/clj/auto_ap/jobs/close_auto_invoices.clj b/src/clj/auto_ap/jobs/close_auto_invoices.clj index e34f5482..09000832 100644 --- a/src/clj/auto_ap/jobs/close_auto_invoices.clj +++ b/src/clj/auto_ap/jobs/close_auto_invoices.clj @@ -6,24 +6,23 @@ [auto-ap.time :as time] [clj-time.coerce :as coerce] [clojure.tools.logging :as log] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn close-auto-invoices [] - (let [invoices-to-close (d/query {:query {:find ['?e] + (let [invoices-to-close (dc/q {:query {:find ['?e] :in ['$ '?today] :where ['[?e :invoice/scheduled-payment ?d] '[?e :invoice/status :invoice-status/unpaid] '[(<= ?d ?today)]]} - :args [(d/db conn) (coerce/to-date (time/local-now))]})] + :args [(dc/db conn) (coerce/to-date (time/local-now))]})] (log/info "Closing " (count invoices-to-close) "scheduled invoices") - (some->> invoices-to-close - seq + (dc/transact conn {:tx-data (some->> invoices-to-close + seq - (mapv (fn [[i]] {:db/id i - :invoice/outstanding-balance 0.0 - :invoice/status :invoice-status/paid})) - (d/transact conn) - deref) + (mapv (fn [[i]] {:db/id i + :invoice/outstanding-balance 0.0 + :invoice/status :invoice-status/paid})) + )}) (log/info "Closed " (count invoices-to-close) "scheduled invoices"))) diff --git a/src/clj/auto_ap/jobs/sysco.clj b/src/clj/auto_ap/jobs/sysco.clj index cef4156e..3f2ca360 100644 --- a/src/clj/auto_ap/jobs/sysco.clj +++ b/src/clj/auto_ap/jobs/sysco.clj @@ -2,9 +2,9 @@ (:require [amazonica.aws.s3 :as s3] [auto-ap.datomic :refer [conn]] - [auto-ap.datomic.clients :as d-clients] - [auto-ap.datomic.invoices :refer [code-invoice]] [auto-ap.jobs.core :refer [execute]] + [auto-ap.datomic.clients :as d-clients] + [auto-ap.datomic.invoices :refer [code-invoice propose-invoice]] [auto-ap.ledger :refer [transact-with-ledger]] [auto-ap.parse :as parse] [auto-ap.time :as t] @@ -15,7 +15,8 @@ [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] [config.core :refer [env]] - [datomic.api :as d]) + [datomic.client.api :as dc] + [auto-ap.datomic.vendors :as d-vendors]) (:import (java.util UUID))) @@ -26,16 +27,15 @@ (def summary-keys ["TranCode" "GroupID" "Company" "CustomerNumber" "InvoiceNumber" "RecordType" "Item" "InvoiceDocument" "TotalLines" "TotalQtyInvoice" "TotalQty" "TotalQtySplit" "TotalQtyPounds" "TotalExtendedPrice" "TotalTaxAmount" "TotalInvoiceAmount" "AccountDate"]) (defn get-sysco-vendor [] - (let [db (d/db conn)] - (d/entity - db - (-> - (d/q '[:find ?v - :in $ - :where [?v :vendor/name "Sysco"]] - db) + (let [db (dc/db conn)] + (-> + (dc/q '[:find (pull ?v r) + :in $ r + :where [?v :vendor/name "Sysco"]] + db + d-vendors/default-read) first - first)))) + first))) (defn read-sysco-csv [k] @@ -45,7 +45,7 @@ io/reader csv/read-csv)) -(defn extract-invoice-details [csv-rows clients sysco-vendor] +(defn extract-invoice-details [csv-rows sysco-vendor] (let [[header-row & csv-rows] csv-rows header-row (into {} (map vector header-keys header-row)) summary-row (->> csv-rows @@ -65,8 +65,8 @@ (header-row "City2")]) account-number (some-> account-number Long/parseLong str) - [matching-client similarity] (and account-number - (parse/best-match clients account-number 0.0)) + matching-client (and account-number + (d-clients/exact-match account-number)) _ (when-not matching-client (throw (ex-info "cannot find matching client" {:account-number account-number @@ -81,14 +81,19 @@ (cond-> #:invoice {:invoice-number (header-row "InvoiceNumber") :total (+ total tax) :outstanding-balance (+ total tax) - :location (parse/best-location-match matching-client location-hint location-hint ) + :location (parse/best-location-match (dc/pull (dc/db conn) + [{:client/location-matches [:location-match/location :location-match/matches]} + :client/default-location + :client/locations] + matching-client) + location-hint + location-hint ) :date (coerce/to-date date) :vendor (:db/id sysco-vendor ) :client (:db/id matching-client) :import-status :import-status/completed :status :invoice-status/unpaid :client-identifier customer-identifier} - similarity (assoc :invoice/similarity (- 1.0 (double similarity))) true (code-invoice)))) (defn mark-key [k] @@ -111,7 +116,6 @@ (defn import-sysco [] (let [sysco-vendor (get-sysco-vendor) - clients (d-clients/get-all) keys (->> (s3/list-objects-v2 {:bucket-name bucket-name :prefix "sysco/pending"}) :object-summaries @@ -129,11 +133,11 @@ :destination-bucket-name (:data-bucket env) :source-key k :destination-key invoice-key}) - [[:propose-invoice - (-> k + [`(propose-invoice + ~(-> k read-sysco-csv - (extract-invoice-details clients sysco-vendor) - (assoc :invoice/source-url invoice-url))]]) + (extract-invoice-details sysco-vendor) + (assoc :invoice/source-url invoice-url)))]) (catch Exception e (log/error (str "Cannot load file " k) e) (log/info @@ -144,8 +148,7 @@ (.getName (io/file k))) println)})) []))))) - result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})] - #_(log/infof "Imported %d invoices" (/ (count (:tempids result)) 2))) +result (transact-with-ledger transaction {:user/name "sysco importer" :user/role "admin"})]) (doseq [k keys] (mark-key k)))) diff --git a/src/clj/auto_ap/jobs/vendor_usages.clj b/src/clj/auto_ap/jobs/vendor_usages.clj index 1cb2bdd7..e58edf2c 100644 --- a/src/clj/auto_ap/jobs/vendor_usages.clj +++ b/src/clj/auto_ap/jobs/vendor_usages.clj @@ -3,31 +3,29 @@ (:require [auto-ap.datomic :refer [conn]] [auto-ap.jobs.core :refer [execute]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn refresh-vendor-usages [] - (->> {:query {:find ['?v '?c '(count ?e)] - :in ['$] - :where ['[?v :vendor/name] - '(or-join [?v ?c ?e] - (and - [?e :invoice/vendor ?v] - [?e :invoice/client ?c]) - (and - [?e :transaction/vendor ?v] - [?e :transaction/client ?c]) - (and - [?e :journal-entry/vendor ?v] - [?e :journal-entry/client ?c]))]} - :args [(d/db conn)]} - (d/query) - (map (fn [[v c cnt]] - #:vendor-usage {:vendor v - :client c - :key (str v "-" c) - :count cnt})) - (d/transact conn) - deref)) + (dc/transact conn {:tx-data (->> (dc/q '[:find ?v ?c (count ?e) + :in $ + :where + [?v :vendor/name] + (or-join [?v ?c ?e] + (and + [?e :invoice/vendor ?v] + [?e :invoice/client ?c]) + (and + [?e :transaction/vendor ?v] + [?e :transaction/client ?c]) + (and + [?e :journal-entry/vendor ?v] + [?e :journal-entry/client ?c]))] + (dc/db conn)) + (map (fn [[v c cnt]] + #:vendor-usage {:vendor v + :client c + :key (str v "-" c) + :count cnt})))})) (defn -main [& _] (execute "vendor-usages" refresh-vendor-usages)) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 7295bb0d..0fd6d327 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -1,21 +1,17 @@ (ns auto-ap.ledger (:require - [auto-ap.datomic :refer [audit-transact conn remove-nils]] - [auto-ap.logging :refer [info-event]] + [auto-ap.datomic :refer [conn remove-nils pull-ref audit-transact]] [auto-ap.utils :refer [dollars-0? dollars=]] [clj-time.coerce :as c] [clj-time.core :as t] [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] - [datomic.api :as d] - [mount.core :as mount] - [unilog.context :as lc] - [yang.scheduler :as scheduler])) + [datomic.client.api :as dc])) (defn datums->impacted-entity [db [e changes]] - (let [entity (d/pull db '[* {:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) + (let [entity (dc/pull db '[{:invoice/_expense-accounts [:db/id] :transaction/_accounts [:db/id]}] e) namespaces (->> changes - (map :a) + (map #(:db/ident (dc/pull db '[:db/ident] (:a %)))) (map namespace) set)] (cond (namespaces "invoice" ) [[:invoice e]] @@ -31,7 +27,7 @@ (defmethod entity-change->ledger :invoice [db [_ id]] (when id - (let [entity (d/pull db ['* {:invoice/vendor '[*] + (let [entity (dc/pull db ['* {:invoice/vendor '[*] :invoice/payment '[*] :invoice/status '[:db/ident] :invoice/import-status '[:db/ident]}] id) @@ -71,7 +67,7 @@ (defmethod entity-change->ledger :transaction [db [_ id]] (when id - (let [entity (d/pull db ['* {:transaction/vendor '[*] + (let [entity (dc/pull db ['* {:transaction/vendor '[*] :transaction/client '[*] :transaction/approval-status '[*] :transaction/bank-account '[* {:bank-account/type [:db/ident]}] @@ -144,13 +140,12 @@ [_ _] nil) - (defn reconcile-ledger ([] (reconcile-ledger (-> (t/now) (t/plus (t/months -6)) (c/to-date)))) ([start-date] - (let [txes-missing-ledger-entries (->> (d/query {:query {:find ['?t ] + (let [txes-missing-ledger-entries (->> (dc/q {:query {:find ['?t ] :in ['$ '?sd] :where [ '[?t :transaction/date ?d] @@ -160,12 +155,12 @@ '(not [?t :transaction/approval-status :transaction-approval-status/excluded]) '(not [?t :transaction/approval-status :transaction-approval-status/suppressed]) ]} - :args [(d/db conn) start-date]}) + :args [(dc/db conn) start-date]}) (map first) - (mapv #(entity-change->ledger (d/db conn) [:transaction %]))) + (mapv #(entity-change->ledger (dc/db conn) [:transaction %]))) - invoices-missing-ledger-entries (->> (d/query {:query {:find ['?t ] + invoices-missing-ledger-entries (->> (dc/q {:query {:find ['?t ] :in ['$ '?sd] :where ['[?t :invoice/date ?d] '[(>= ?d ?sd)] @@ -176,45 +171,71 @@ '(not [?t :invoice/import-status :import-status/pending]) '(not [?t :invoice/exclude-from-ledger true]) ]} - :args [(d/db conn) start-date]}) + :args [(dc/db conn) start-date]}) (map first) - (mapv #(entity-change->ledger (d/db conn) [:invoice %]))) + (mapv #(entity-change->ledger (dc/db conn) [:invoice %]))) repairs (vec (concat txes-missing-ledger-entries invoices-missing-ledger-entries))] (when (seq repairs) (log/info (take 3 repairs)) (log/warn "repairing " (count txes-missing-ledger-entries) " missing transactions, " (count invoices-missing-ledger-entries) " missing invoices that were missing ledger entries") - @(d/transact conn repairs))))) + (dc/transact conn {:tx-data repairs}))))) (defn touch-transaction [e] - @(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]]) - (when-let [change (entity-change->ledger (d/db conn) - [:transaction e])] - @(d/transact conn [{:db/id "datomic.tx" - :db/doc "touching transaction to update ledger"} - change]))) + (dc/transact conn {:tx-data [[:db/retractEntity [:journal-entry/original-entity e]]]}) +(when-let [change (entity-change->ledger (d/db conn) + [:transaction e])] + (dc/transact conn {:tx-data [{:db/id "datomic.tx" + :db/doc "touching transaction to update ledger"} + (entity-change->ledger (dc/db conn) + [:transaction e])]})) + ) (defn touch-invoice [e] - @(d/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]]) - (when-let [change (entity-change->ledger (d/db conn) - [:invoice e])] - @(d/transact conn [{:db/id "datomic.tx" - :db/doc "touching invoice to update ledger"} - change]))) + (dc/transact conn [[:db/retractEntity [:journal-entry/original-entity e]]]) +(when-let [change (entity-change->ledger (d/db conn) + [:invoice e])] + (dc/transact conn [{:db/id "datomic.tx" + :db/doc "touching invoice to update ledger"} + (entity-change->ledger (dc/db conn) + [:invoice e])])) + ) + +(defn lazy-tx-range + ([start end xf] (lazy-tx-range start end xf 0)) + ([start end xf o] + (let [next-results (dc/tx-range conn {:start start + :end end + :offset o + :limit 200})] + (lazy-seq + (if (seq next-results) + (concat (sequence (comp (mapcat :data) + xf) next-results) (lazy-tx-range start + end + xf + (+ (count next-results) + o))) + next-results))))) + + + +(defn recently-changed-entities [start end] + (set (map (fn [d] + (:e d)) + (mapcat :data (dc/tx-range conn {:start start + :end end}))))) + +(defn entities-since-last-ledger-entry [] + (count (dc/tx-range conn {:start (coerce/to-date (time/plus (time/now) (time/days -5))) + :end (coerce/to-date (time/now))}))) (defn mismatched-transactions ([] (mismatched-transactions (c/to-date (t/minus (t/now) (t/days 7))) (c/to-date (t/minus (t/now) (t/hours 1)))) ) ([changed-between-start changed-between-end] - (let [entities-to-consider (d/q '[:find [?t ...] - :in $ ?log ?start ?end - :where - [(tx-ids ?log ?start ?end) [?tx ...]] - [(tx-data ?log ?tx) [[?t]]] - [?t :transaction/date]] - (d/db auto-ap.datomic/conn) - (d/log auto-ap.datomic/conn) + (let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end) _ (log/info "checking" (count entities-to-consider) "transactions looking for mismatches between" changed-between-start changed-between-end) @@ -222,20 +243,21 @@ (fn [acc [e lia]] (update acc e (fnil conj #{} ) lia)) {} - (d/q '[:find ?e ?lia + (dc/q '[:find ?e ?lia :in $ [?e ...] :where [?je :journal-entry/original-entity ?e] + [?e :transaction/date] [?je :journal-entry/line-items ?li] [?li :journal-entry-line/account ?lia] [?lia :account/name]] - (d/db auto-ap.datomic/conn) + (dc/db conn) entities-to-consider)) transaction-accounts (reduce (fn [acc [e lia]] (update acc e (fnil conj #{} ) lia)) {} - (d/q '[:find ?e ?lia + (dc/q '[:find ?e ?lia :in $ [?e ...] :where [?e :transaction/date ?d] @@ -246,7 +268,7 @@ [?lia :account/name] [?e :transaction/amount ?amt] [(not= ?amt 0.0)]] - (d/db auto-ap.datomic/conn) + (dc/db conn) entities-to-consider))] (->> transaction-accounts (filter @@ -256,20 +278,9 @@ ([] (unbalanced-transactions (c/to-date (t/minus (t/now) (t/days 7))) (c/to-date (t/minus (t/now) (t/hours 1))))) ([changed-between-start changed-between-end] - (let [entities-to-consider (d/q '[:find [?je ...] - :in $ ?log ?start ?end - :where - [(tx-ids ?log ?start ?end) [?tx ...]] - [(tx-data ?log ?tx) [[?je]]] - [?je :journal-entry/amount] - [?je :journal-entry/original-entity ?i] - [?i :transaction/date]] - (d/db auto-ap.datomic/conn) - (d/log auto-ap.datomic/conn) - changed-between-start - changed-between-end)] + (let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end)] (log/info "checking" (count entities-to-consider) "transaction journal entries looking for mismatches between" changed-between-start changed-between-end) - (->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit) + (->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit) :with ?jel :in $ [?je ...] :where [?je :journal-entry/amount ?a] @@ -277,52 +288,42 @@ [(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] ] - (d/db auto-ap.datomic/conn) + (dc/db conn) entities-to-consider) (filter (fn [[_ a d c]] (or (not (dollars= a d)) (not (dollars= a c))))) (map first) (map (fn [je] - (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) - je)))) - (map :db/id))))) + (pull-ref (dc/db conn) :journal-entry/original-entity je))))))) + + (defn unbalanced-invoices ([] (unbalanced-invoices (c/to-date (t/minus (t/now) (t/days 7))) (c/to-date (t/minus (t/now) (t/hours 1))))) ([changed-between-start changed-between-end] - (let [entities-to-consider (d/q '[:find [?je ...] - :in $ ?log ?start ?end - :where - [(tx-ids ?log ?start ?end) [?tx ...]] - [(tx-data ?log ?tx) [[?je]]] - [?je :journal-entry/amount] - [?je :journal-entry/original-entity ?i] - [?i :invoice/date]] - (d/db auto-ap.datomic/conn) - (d/log auto-ap.datomic/conn) - changed-between-start - changed-between-end)] + (let [entities-to-consider (recently-changed-entities + changed-between-start + changed-between-end)] (log/info "checking" (count entities-to-consider) "invoice journal entries looking for mismatches between" changed-between-start changed-between-end) - (->> (d/q '[:find ?je ?a (sum ?debit) (sum ?credit) - :with ?jel - :in $ [?je ...] - :where [?je :journal-entry/amount ?a] - [?je :journal-entry/line-items ?jel] - [(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] - [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit] - ] - (d/db auto-ap.datomic/conn) - entities-to-consider) + (->> (dc/q '[:find ?je ?a (sum ?debit) (sum ?credit) + :with ?jel + :in $ [?je ...] + :where [?je :journal-entry/amount ?a] + [?je :journal-entry/original-entity ?i] + [?i :invoice/date] + [?je :journal-entry/line-items ?jel] + [(get-else $ ?jel :journal-entry-line/debit 0.0) ?debit] + [(get-else $ ?jel :journal-entry-line/credit 0.0) ?credit]] + (dc/db conn) + entities-to-consider) (filter (fn [[_ a d c]] (or (not (dollars= a d)) (not (dollars= a c))))) (map first) (map (fn [je] - (:journal-entry/original-entity (d/entity (d/db auto-ap.datomic/conn) - je)))) - (map :db/id))))) + (pull-ref (dc/db conn) :journal-entry/original-entity je))))))) (defn mismatched-invoices ([] @@ -330,36 +331,28 @@ (c/to-date (t/minus (t/now) (t/hours 1)))) ) ([changed-between-start changed-between-end] - (let [entities-to-consider (d/q '[:find [?i ...] - :in $ ?log ?start ?end - :where - [(tx-ids ?log ?start ?end) [?tx ...]] - [(tx-data ?log ?tx) [[?i]]] - [?i :invoice/date]] - (d/db auto-ap.datomic/conn) - (d/log auto-ap.datomic/conn) - changed-between-start - changed-between-end) + (let [entities-to-consider (recently-changed-entities changed-between-start changed-between-end) _ (log/info (count entities-to-consider) "invoices have changed between" changed-between-start "and" changed-between-end) jel-accounts (reduce (fn [acc [e lia]] (update acc e (fnil conj #{} ) lia)) {} - (d/q '[:find ?e ?lia + (dc/q '[:find ?e ?lia :in $ [?e ...] :where [?je :journal-entry/original-entity ?e] + [?e :invoice/date] [?je :journal-entry/line-items ?li] [?li :journal-entry-line/account ?lia] (not [?lia :account/numeric-code 21000]) [?lia :account/name]] - (d/db auto-ap.datomic/conn) + (dc/db conn) entities-to-consider)) invoice-accounts (reduce (fn [acc [e lia]] (update acc e (fnil conj #{} ) lia)) {} - (d/q '[:find ?e ?lia + (dc/q '[:find ?e ?lia :in $ [?e ...] :where [?e :invoice/expense-accounts ?li] @@ -370,7 +363,7 @@ (not [?e :invoice/status :invoice-status/voided]) (not [?e :invoice/exclude-from-ledger true]) [?e :invoice/import-status :import-status/imported]] - (d/db auto-ap.datomic/conn) + (dc/db conn) entities-to-consider)) ] (filter @@ -427,12 +420,12 @@ nil)) (defn transact-with-ledger [transaction id] - (let [db (d/db conn) + (let [db (dc/db conn) tx (audit-transact transaction id) affected-entities (->> (:tx-data tx) (map (fn [^datomic.db.Datum x] {:e (:e x) - :a (d/ident db (:a x)) + :a (:a x) :v (:v x) :added (:added x)})) (group-by :e) diff --git a/src/clj/auto_ap/pdf/ledger.clj b/src/clj/auto_ap/pdf/ledger.clj index 89104c02..d4d648bb 100644 --- a/src/clj/auto_ap/pdf/ledger.clj +++ b/src/clj/auto_ap/pdf/ledger.clj @@ -1,17 +1,17 @@ (ns auto-ap.pdf.ledger (:require [amazonica.aws.s3 :as s3] - [auto-ap.ledger.reports :as l-reports] - [auto-ap.datomic :refer [conn]] + [auto-ap.datomic :refer [conn pull-attr pull-many]] [auto-ap.graphql.utils :refer [<-graphql]] + [auto-ap.ledger.reports :as l-reports] [auto-ap.time :as atime] [auto-ap.utils :refer [by dollars-0?]] [clj-pdf.core :as pdf] [clojure.java.io :as io] [clojure.string :as str] + [clojure.tools.logging :as log] [config.core :refer [env]] - [datomic.api :as d] - [clojure.tools.logging :as log]) + [datomic.client.api :as dc]) (:import (java.io ByteArrayOutputStream) (java.text DecimalFormat) @@ -152,7 +152,7 @@ args (assoc args :periods (filter identity (cond-> [(:date args)] (:include-comparison args) (conj (:comparison-date args))))) - clients (d/pull-many (d/db conn) '[:client/code :client/name :db/id] [(:client-id args)]) + clients (pull-many (dc/db conn) [:client/code :client/name :db/id] [(:client-id args)]) data (concat (->> (:balance-sheet-accounts data) (map (fn [b] (assoc b @@ -183,7 +183,7 @@ (let [data (<-graphql data) args (<-graphql args) - clients (d/pull-many (d/db conn) '[:client/code :client/name :db/id] (:client-ids args)) + clients (pull-many (dc/db conn) [:client/code :client/name :db/id] (:client-ids args)) data (->> data :periods (mapcat (fn [p1 p2] @@ -255,7 +255,7 @@ max-date (atime/unparse-local (->> args :periods (map :end) last) atime/iso-date) - names (str/replace (->> args :client_ids (d/pull-many (d/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )] + names (str/replace (->> args :client_ids (pull-many (dc/db conn) [:client/name]) (map :client/name) (str/join "-")) #" " "_" )] (format "Profit-and-loss-%s-to-%s-for-%s" min-date max-date names))) (defn journal-detail-args->name [args] @@ -273,7 +273,7 @@ (let [date (atime/unparse-local (:date args) atime/iso-date) - name (str/replace (->> args :client_id (d/pull (d/db conn) [:client/name]) :client/name ) #" " "_" )] + name (str/replace (->> args :client_id (pull-attr (dc/db conn) :client/name)) #" " "_" )] (format "Balance-sheet-%s-for-%s" date name))) (defn print-pnl [user args data] @@ -287,13 +287,13 @@ :input-stream (io/make-input-stream pdf-data {}) :metadata {:content-length (count pdf-data) :content-type "application/pdf"}) - @(d/transact conn - [{:report/name name - :report/client (:client_ids args) - :report/key key - :report/url url - :report/creator (:user user) - :report/created (java.util.Date.)}]) + (dc/transact conn + {:tx-data [{:report/name name + :report/client (:client_ids args) + :report/key key + :report/url url + :report/creator (:user user) + :report/created (java.util.Date.)}]}) {:report/name name :report/url url })) @@ -308,13 +308,14 @@ :input-stream (io/make-input-stream pdf-data {}) :metadata {:content-length (count pdf-data) :content-type "application/pdf"}) - @(d/transact conn - [{:report/name name - :report/client [(:client_id args)] - :report/key key - :report/url url - :report/creator (:user user) - :report/created (java.util.Date.)}]) + (dc/transact conn + {:tx-data + [{:report/name name + :report/client [(:client_id args)] + :report/key key + :report/url url + :report/creator (:user user) + :report/created (java.util.Date.)}]}) {:report/name name :report/url url })) diff --git a/src/clj/auto_ap/routes/exports.clj b/src/clj/auto_ap/routes/exports.clj index bd92a79b..aa0f2bae 100644 --- a/src/clj/auto_ap/routes/exports.clj +++ b/src/clj/auto_ap/routes/exports.clj @@ -20,6 +20,9 @@ [com.unbounce.dogstatsd.core :as statsd] [config.core :refer [env]] [datomic.api :as d] + [compojure.core :refer [context defroutes GET routes wrap-routes]] + [config.core :refer [env]] + [datomic.client.api :as dc] [ring.middleware.json :refer [wrap-json-response]] [venia.core :as venia])) @@ -48,8 +51,6 @@ [(auto-ap.time/unparse ?d3 auto-ap.time/normal-date) ?d4]]} :args [(d/db conn) client-id]})))) - - (defn client-tag [params] (when-let [code (or (params "client-code") (:client-code params))] diff --git a/src/clj/auto_ap/routes/invoices.clj b/src/clj/auto_ap/routes/invoices.clj index c844b30b..8fce9d9c 100644 --- a/src/clj/auto_ap/routes/invoices.clj +++ b/src/clj/auto_ap/routes/invoices.clj @@ -54,8 +54,8 @@ (defn parse-invoice-number [{:keys [invoice-number]}] (or invoice-number "")) -(defn parse-vendor [{:keys [vendor-name check]} vendors] - (let [v (vendors vendor-name)] +(defn parse-vendor [{:keys [vendor-name check]} vendor-name->vendor] + (let [v (vendor-name->vendor vendor-name)] (cond v v @@ -65,9 +65,6 @@ :else (throw (Exception. (str "Vendor '" vendor-name "' not found.")))))) -(defn parse-vendor-id [{:keys [vendor]}] - (:db/id vendor)) - (defn parse-automatically-paid-when-due [{:keys [vendor client-id]}] (boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id))) @@ -77,22 +74,31 @@ (defn parse-invoice-rows [excel-rows] (let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code] - all-vendors (->> (d/q '[:find [?e ...] - :in $ - :where [?e :vendor/name]] - (d/db conn)) - (d/pull-many (d/db conn) d-vendors/default-read) - (by :vendor/name)) - all-clients (d-clients/get-all) - all-clients (merge (by :client/code all-clients) (by :client/name all-clients)) - rows (->> (str/split excel-rows #"\n") - (map #(str/split % #"\t")) - (map #(into {} (map (fn [c k] [k c]) % columns))) + tabulated (->> (str/split excel-rows #"\n") + (map #(str/split % #"\t")) + (map #(into {} (map (fn [c k] [k c]) % columns)))) + vendor-name->vendor (->> + (set (map :vendor-name tabulated)) + (dc/q '[:find ?n ?v + :in $ [?n ...] + :where [?v :vendor/name ?n]] + (dc/db conn) + ) + (into {})) + all-clients (merge (into {}(dc/q '[:find ?n (pull ?v [:db/id :client/locations]) + :in $ + :where [?v :client/name ?n]] + (dc/db conn))) + (into {} + (dc/q '[:find ?n (pull ?v [:db/id :client/locations]) + :in $ + :where [?v :client/code ?n]] + (dc/db conn)))) + rows (->> tabulated (map reset-id) (map assoc-client-code) (map (c/parse-or-error :client-id #(parse-client % all-clients))) - (map (c/parse-or-error :vendor #(parse-vendor % all-vendors))) - (map (c/parse-or-error :vendor-id #(parse-vendor-id %))) + (map (c/parse-or-error :vendor-id #(parse-vendor % vendor-name->vendor))) (map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %))) (map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %))) (map (c/parse-or-error :account-id c/parse-account-numeric-code)) @@ -107,51 +113,54 @@ (throw (ex-info (str "No vendor found. Please supply an forced vendor.") {:vendor-code vendor-code}))) (let [vendor-id (or forced-vendor - (->> (d/query + (->> (dc/q {:query {:find ['?vendor] :in ['$ '?vendor-name] :where ['[?vendor :vendor/name ?vendor-name]]} - :args [(d/db (d/connect uri)) vendor-code]}) + :args [(dc/db conn) vendor-code]}) first first))] (when-not vendor-id (throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.") {:vendor-code vendor-code}))) - (if-let [matching-vendor (->> (d/query + (if-let [matching-vendor (->> (dc/q {:query {:find [(list 'pull '?vendor-id d-vendors/default-read)] :in ['$ '?vendor-id]} - :args [(d/db (d/connect uri)) vendor-id]}) + :args [(dc/db conn) vendor-id]}) first first)] matching-vendor (throw (ex-info (str "No vendor with the name " vendor-code " was found.") {:vendor-code vendor-code}))))) -(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]} clients] - (let [[matching-client similarity] (cond - account-number (parse/best-match clients account-number 0.0) - customer-identifier (parse/best-match clients customer-identifier) - client-override [(first (filter (fn [c] - (= (:db/id c) (Long/parseLong client-override))) - clients)) - 1.0]) - matching-vendor (match-vendor vendor-code vendor-override) - matching-location (or (when-not (str/blank? location-override) +(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}] + (let [matching-client (cond + account-number (d-clients/exact-match account-number) + customer-identifier (d-clients/best-match customer-identifier) + client-override (Long/parseLong client-override)) + + matching-vendor (match-vendor vendor-code vendor-override) + matching-location (or (when-not (str/blank? location-override) location-override) - (parse/best-location-match matching-client text full-text))] - (remove-nils #:invoice {:invoice/client (:db/id matching-client) - :invoice/client-identifier (or account-number customer-identifier) - :invoice/vendor (:db/id matching-vendor) - :invoice/similarity (some-> similarity double (#(- 1.0 %))) - :invoice/source-url source-url - :invoice/invoice-number invoice-number - :invoice/total (Double/parseDouble total) - :invoice/date (to-date date) - :invoice/location matching-location - :invoice/import-status (or import-status :import-status/pending) - :invoice/outstanding-balance (Double/parseDouble total) - :invoice/status :invoice-status/unpaid}))) + (parse/best-location-match (dc/pull (dc/db conn) + [{:client/location-matches [:location-match/location :location-match/matches]} + :client/default-location + :client/locations] + matching-client) + text + full-text))] + #:invoice {:invoice/client matching-client + :invoice/client-identifier (or account-number customer-identifier) + :invoice/vendor (:db/id matching-vendor) + :invoice/source-url source-url + :invoice/invoice-number invoice-number + :invoice/total (Double/parseDouble total) + :invoice/date (to-date date) + :invoice/location matching-location + :invoice/import-status (or import-status :import-status/pending) + :invoice/outstanding-balance (Double/parseDouble total) + :invoice/status :invoice-status/unpaid})) (defn validate-invoice [invoice user] (when-not (:invoice/client invoice) @@ -168,7 +177,7 @@ (defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}] (try - (->> (d/query + (->> (dc/q (cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2] :in ['$ '?invoice-number '?vendor '?client] :where '[[?e :invoice/invoice-number ?invoice-number] @@ -178,7 +187,7 @@ [?e :invoice/status ?status] [?e :invoice/import-status ?import-status] [?import-status :db/ident ?import-status2]]} - :args [(d/db (d/connect uri)) invoice-number vendor client]})) + :args [(dc/db conn) invoice-number vendor client]})) first boolean) (catch Exception e @@ -222,13 +231,13 @@ :status :payment-status/cleared :date (:invoice/date invoice)})}) transaction (when (= :invoice-status/paid (:invoice/status invoice)) - (let [[bank-account] (d/q '[:find [?ba ...] - :in $ ?c - :where [?c :client/bank-accounts ?ba] - [?ba :bank-account/type :bank-account-type/cash] - ] - (d/db conn) - client-id)] + (let [[[bank-account]] (dc/q '[:find ?ba + :in $ ?c + :where [?c :client/bank-accounts ?ba] + [?ba :bank-account/type :bank-account-type/cash] + ] + (dc/db conn) + client-id)] #:transaction {:amount (- (:invoice/total invoice)) :payment payment-id :client (:invoice/client invoice) @@ -244,8 +253,8 @@ :transaction-account/location "A" :transaction-account/amount (Math/abs (:invoice/total invoice))}]})) ] - [[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice) - user))] + [`(d-invoices/propose-invoice ~(d-invoices/code-invoice (validate-invoice (remove-nils invoice) + user))) (some-> payment remove-nils) transaction]))) (filter identity))) @@ -260,19 +269,23 @@ (defn import-uploaded-invoice [user imports] (lc/with-context {:area "upload-invoice"} (log/info "Number of invoices to import is" (count imports)) - (let [clients (d-clients/get-all) - potential-invoices (->> imports - (mapv #(import->invoice % clients)) - (mapv #(validate-invoice % user)) + (let [potential-invoices (->> imports + (map import->invoice) + (map #(validate-invoice % user)) admin-only-if-multiple-clients - (filter #(not (extant-invoice? %))) (mapv d-invoices/code-invoice) - (mapv (fn [i] [:propose-invoice i])))] - (when-not (seq potential-invoices) - (throw (ex-info "No new invoices found." - {}))) + (mapv (fn [i] `(d-invoices/propose-invoice ~i))))] + (log/info "creating invoice" potential-invoices) - (transact-with-ledger potential-invoices user)))) + (let [tx (transact-with-ledger potential-invoices user)] + (when-not (seq (dc/q '[:find ?i + :in $ [?i ...] + :where [?i :invoice/invoice-number]] + (:db-after tx) + (map :e (:tx-data tx)))) + (throw (ex-info "No new invoices found." + {}))) + tx)))) (defn validate-account-rows [rows code->existing-account] (when-let [bad-types (seq (->> rows @@ -296,23 +309,22 @@ {:rows duplicate-rows})))) (defn import-account-overrides [customer filename] - (let [conn (d/connect uri) - [_ & rows] (-> filename (io/reader) csv/read-csv) - [client-id] (first (d/query (-> {:query {:find ['?e] - :in ['$ '?z] - :where [['?e :client/code '?z]]} - :args [(d/db (d/connect uri)) customer]}))) - code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code + (let [[_ & rows] (-> filename (io/reader) csv/read-csv) + [client-id] (first (dc/q (-> {:query {:find ['?e] + :in ['$ '?z] + :where [['?e :client/code '?z]]} + :args [(dc/db conn) customer]}))) + code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} - :args [(d/db conn)]}))) + :args [(dc/db conn)]}))) - existing-account-overrides (d/query (-> {:query {:find ['?e] - :in ['$ '?client-id] - :where [['?e :account-client-override/client '?client-id]]} - :args [(d/db (d/connect uri)) client-id]})) + existing-account-overrides (dc/q (-> {:query {:find ['?e] + :in ['$ '?client-id] + :where [['?e :account-client-override/client '?client-id]]} + :args [(dc/db conn) client-id]})) rows (transduce (comp (map (fn [[_ account account-name override-name _ type]] [account account-name override-name type])) @@ -367,21 +379,21 @@ existing-account-overrides) rows)] - @(d/transact conn txes) + (dc/transact conn {:tx-data txes}) txes)) (defn import-transactions-cleared-against [file] (let [[_ & rows] (-> file (io/reader) csv/read-csv) - txes (transduce - (comp - (filter (fn [[transaction-id _]] - (d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id)))) - (map (fn [[transaction-id cleared-against]] - {:db/id (Long/parseLong transaction-id) - :transaction/cleared-against cleared-against}))) - conj - [] - rows)] + txes (transduce + (comp + (filter (fn [[transaction-id _]] + (dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id)))) + (map (fn [[transaction-id cleared-against]] + {:db/id (Long/parseLong transaction-id) + :transaction/cleared-against cleared-against}))) + conj + [] + rows)] (transact-with-ledger txes nil))) (defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}] diff --git a/src/clj/auto_ap/routes/queries.clj b/src/clj/auto_ap/routes/queries.clj index 1affc898..cca07e4d 100644 --- a/src/clj/auto_ap/routes/queries.clj +++ b/src/clj/auto_ap/routes/queries.clj @@ -1,7 +1,7 @@ (ns auto-ap.routes.queries (:require [amazonica.aws.s3 :as s3] - [auto-ap.datomic :refer [conn]] + [auto-ap.datomic :refer [conn pull-attr upsert-entity]] [auto-ap.graphql.utils :refer [assert-admin]] [clojure.data.csv :as csv] [clojure.edn :as edn] @@ -9,8 +9,11 @@ [clojure.string :as str] [clojure.tools.logging :as log] [com.unbounce.dogstatsd.core :as statsd] - [config.core :refer [env]] +[config.core :refer [env]] +[datomic.client.api :as dc] [datomic.api :as d] + [compojure.core + :refer [context defroutes GET POST PUT routes wrap-routes]] [ring.middleware.json :refer [wrap-json-response]] [ring.util.request :refer [body-string]] [unilog.context :as lc]) @@ -34,21 +37,20 @@ (let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env) :key (str "queries/" (:query-id params))))))] (log/info "Executing query " query-string) - (into (list) (apply d/q (edn/read-string query-string) - (into [(d/db conn)] (edn/read-string (get query-params "args" "[]"))))))))) + (into (list) (apply dc/q (edn/read-string query-string) + (into [(dc/db conn)] (edn/read-string (get query-params "args" "[]"))))))))) (defn put-query [guid body note & [lookup-key client]] (let [guid (if lookup-key - (or (:saved-query/guid (d/pull (d/db conn) [:saved-query/guid] [:saved-query/lookup-key lookup-key])) + (or (pull-attr (dc/db conn) :saved-query/guid [:saved-query/lookup-key lookup-key]) guid) guid)] - @(d/transact conn [(cond-> - {:saved-query/guid guid - :saved-query/description note - :saved-query/key (str "queries/" guid)} - client (assoc :saved-query/client client) - lookup-key (assoc :saved-query/lookup-key lookup-key))]) + (dc/transact conn [`(upsert-entity ~{:saved-query/guid guid + :saved-query/description note + :saved-query/key (str "queries/" guid) + :saved-query/client client + :saved-query/lookup-key lookup-key})]) (s3/put-object :bucket-name (:data-bucket env) :key (str "queries/" guid) :input-stream (io/make-input-stream (.getBytes body) {}) diff --git a/src/clj/auto_ap/routes/yodlee2.clj b/src/clj/auto_ap/routes/yodlee2.clj index ba8ebfea..261c8519 100644 --- a/src/clj/auto_ap/routes/yodlee2.clj +++ b/src/clj/auto_ap/routes/yodlee2.clj @@ -1,16 +1,16 @@ (ns auto-ap.routes.yodlee2 (:require - [auto-ap.datomic :refer [conn]] + [auto-ap.datomic :refer [conn pull-attr]] [auto-ap.datomic.clients :as d-clients] [auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]] [auto-ap.routes.utils :refer [wrap-secure]] [auto-ap.yodlee.core2 :as yodlee] [clojure.tools.logging :as log] [config.core :refer [env]] - [datomic.api :as d])) + [datomic.client.api :as dc])) (defn fastlink [{:keys [query-params identity]}] - (assert-can-see-client identity (d/pull (d/db conn) [:db/id] [:client/code (get query-params "client")])) + (assert-can-see-client identity (pull-attr (dc/db conn) :db/id [:client/code (get query-params "client")])) (let [token (if-let [client-id (get query-params "client-id")] (-> client-id diff --git a/src/clj/auto_ap/search.clj b/src/clj/auto_ap/search.clj new file mode 100644 index 00000000..417e6147 --- /dev/null +++ b/src/clj/auto_ap/search.clj @@ -0,0 +1,72 @@ +(ns auto-ap.search + (:import + (java.nio.file Paths) + (org.apache.lucene.analysis.standard StandardAnalyzer) + (org.apache.lucene.document Document Field$Store StoredField StringField TextField) + (org.apache.lucene.index DirectoryReader IndexWriter IndexWriterConfig Term) + (org.apache.lucene.queryparser.classic QueryParser) + (org.apache.lucene.search BooleanClause$Occur BooleanQuery$Builder IndexSearcher PhraseQuery$Builder Query TermQuery) + (org.apache.lucene.store FSDirectory))) + +(defn full-index-query [results index-name] + (let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name)))) + analyzer (StandardAnalyzer.) + index-writer-config (IndexWriterConfig. analyzer) + index-writer (IndexWriter. directory index-writer-config)] + (.deleteAll index-writer) + (try + (doseq [{:keys [text id] :as x} results + :let [doc (doto + (Document.) + (.add (TextField. "name" text Field$Store/YES)) + (.add (StoredField. "id" (long id))))]] + (doseq [k (filter (complement #{:text :id}) (keys x))] + (println "K" (name k) (get x k)) + (.add doc (StringField. (name k) (str (get x k)) Field$Store/YES))) + (println "adding" text) + (flush) + (.addDocument index-writer doc)) + (finally + (.close index-writer))))) + +(defn make-query [n] + (let [ + text-query (when (:q n) + (.parse (QueryParser. "name" (StandardAnalyzer.)) (:q n))) + text-query-exact (when (:q-exact n) + (.build (doto (PhraseQuery$Builder. ) + (.add (Term. "name" (:q-exact n)) 0)))) + full-query (BooleanQuery$Builder.) + ] + (when text-query + (.add full-query text-query BooleanClause$Occur/MUST)) + (when text-query-exact + (.add full-query text-query-exact BooleanClause$Occur/MUST)) + (doseq [[k v] (dissoc n :q :q-exact)] + (if (instance? Query v) + (.add full-query v BooleanClause$Occur/MUST) + (.add full-query (TermQuery. (Term. (name k) (str v))) BooleanClause$Occur/MUST))) + (.build full-query))) + +(defn search + ([n index-name] + (search n index-name [])) + ([n index-name other-keys] + (let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name)))) + index-reader (DirectoryReader/open directory) + index-searcher (IndexSearcher. index-reader)] + (for [x (seq (.scoreDocs (.search index-searcher (make-query n) 10)))] + (into + [(.get (.doc index-searcher (.-doc x)) "id") + (.get (.doc index-searcher (.-doc x)) "name")] + (map (fn [o] + (.get (.doc index-searcher (.-doc x)) o)) + other-keys))))) + ) + +(defn search-ids [n index-name] + (let [directory (FSDirectory/open (Paths/get (java.net.URI. (str "file:///tmp/" index-name)))) + index-reader (DirectoryReader/open directory) + index-searcher (IndexSearcher. index-reader)] + (for [x (seq (.scoreDocs (.search index-searcher (make-query n) 100)))] + (Long/parseLong (.get (.doc index-searcher (.-doc x)) "id"))))) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 84f79fff..0f6d74d6 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -86,7 +86,6 @@ (mount/stop)) (defn -main [& _] - (let [job (System/getenv "INTEGREAT_JOB")] (cond (= job "square-import-job") (job-square/-main) @@ -138,12 +137,9 @@ (job-bulk-journal-import/-main) :else - (do (add-shutdown-hook! shutdown-mount) (start-server :port 9000 :bind "0.0.0.0" #_#_:handler (cider-nrepl-handler)) (mount/start) #_(alter-var-root #'nrepl.middleware.print/*print-fn* (constantly clojure.pprint/pprint)))))) -(comment - ) diff --git a/src/clj/auto_ap/square/core.clj b/src/clj/auto_ap/square/core.clj index 9590b9e5..0c891b7e 100644 --- a/src/clj/auto_ap/square/core.clj +++ b/src/clj/auto_ap/square/core.clj @@ -1,6 +1,6 @@ (ns auto-ap.square.core (:require - [auto-ap.datomic :refer [conn remove-nils]] + [auto-ap.datomic :refer [conn random-tempid remove-nils]] [auto-ap.time :as atime] [clj-http.client :as client] [clj-time.coerce :as coerce] @@ -12,7 +12,7 @@ [clojure.set :as set] [clojure.string :as str] [clojure.tools.logging :as log] - [datomic.api :as d] + [datomic.client.api :as dc] [slingshot.slingshot :refer [try+]] [unilog.context :as lc])) @@ -395,11 +395,11 @@ (upsert client square-location (time/plus (time/now) (time/days -3)) (time/now)))) ([client location start end] (lc/with-context {:source "Square loading"} - (let [existing (->> (d/query {:query {:find ['?external-id] + (let [existing (->> (dc/q {:query {:find ['?external-id] :in ['$ '?client] :where ['[?o :sales-order/client ?client] '[?o :sales-order/external-id ?external-id]]} - :args [(d/db conn) (:db/id client)]}) + :args [(dc/db conn) (:db/id client)]}) (map first) set) _ (log/info (count existing) "Sales orders already exist") @@ -407,7 +407,7 @@ (daily-results client location start end))] (doseq [x (partition-all 20 to-create)] (log/info "Loading " (count x)) - @(d/transact conn x)))))) + (dc/transact conn {:tx-data x})))))) (defn upsert-settlements ([client] @@ -419,7 +419,7 @@ :client (:client/code client)} (doseq [x (partition-all 20 (daily-settlements client location))] (log/info "Loading expected deposit" (count x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (log/info "Done loading settlements")))) (defn upsert-refunds @@ -433,7 +433,7 @@ :location (:square-location/client-location client)} (doseq [x (partition-all 20 (refunds client location))] (log/info "Loading refund" (count x)) - @(d/transact conn x)) + (dc/transact conn {:tx-data x})) (log/info "Done loading refunds")))) (def square-read [:db/id @@ -443,26 +443,26 @@ (defn get-square-clients ([] - (d/q '[:find [(pull ?c [:db/id - :client/square-integration-status - :client/code - :client/square-auth-token - {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] - :in $ - :where [?c :client/square-auth-token] - (not [?c :client/feature-flags "new-square"])] - (d/db conn))) + (map first (dc/q '[:find (pull ?c [:db/id + :client/square-integration-status + :client/code + :client/square-auth-token + {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) + :in $ + :where [?c :client/square-auth-token] + (not [?c :client/feature-flags "new-square"])] + (dc/db conn)))) ([ & codes] - (d/q '[:find [(pull ?c [:db/id - :client/code - :client/square-auth-token - {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) ...] - :in $ [?code ...] - :where [?c :client/square-auth-token] - (not [?c :client/feature-flags "new-square"]) - [?c :client/code ?code]] - (d/db conn) - codes))) + (map first (dc/q '[:find (pull ?c [:db/id + :client/code + :client/square-auth-token + {:client/square-locations [:db/id :square-location/name :square-location/square-id :square-location/client-location]}]) + :in $ [?code ...] + :where [?c :client/square-auth-token] + (not [?c :client/feature-flags "new-square"]) + [?c :client/code ?code]] + (dc/db conn) + codes)))) (defn upsert-locations ([] (doseq [client (get-square-clients)] @@ -474,31 +474,29 @@ [(:square-location/square-id sl) (:db/id sl)]) (:client/square-locations client)))] - (->> (for [square-location (client-locations client)] - {:db/id (or (square-id->id (:id square-location)) (d/tempid :db.part/user)) - :client/_square-locations (:db/id client) - :square-location/name (:name square-location) - :square-location/square-id (:id square-location)}) - (d/transact conn) - deref)))) + (dc/transact conn {:tx-data (for [square-location (client-locations client)] + {:db/id (or (square-id->id (:id square-location)) (random-tempid)) + :client/_square-locations (:db/id client) + :square-location/name (:name square-location) + :square-location/square-id (:id square-location)})})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn reset [] (->> - (d/query {:query {:find ['?e] + (dc/q {:query {:find ['?e] :in ['$] :where ['(or [?e :sales-order/date] [?e :expected-deposit/date])]} - :args [(d/db conn)]}) + :args [(dc/db conn)]}) (map first) (map (fn [x] [:db/retractEntity x])))) (defn mark-integration-status [client integration-status] - @(d/transact conn - [{:db/id (:db/id client) - :client/square-integration-status (assoc integration-status - :db/id (or (-> client :client/square-integration-status :db/id) - #db/id [:db.part/user]))}])) + (dc/transact conn + {:tx-data [{:db/id (:db/id client) + :client/square-integration-status (assoc integration-status + :db/id (or (-> client :client/square-integration-status :db/id) + #db/id [:db.part/user]))}]})) (defn upsert-all [ & clients] (doseq [client (apply get-square-clients clients) diff --git a/src/clj/auto_ap/yodlee/core2.clj b/src/clj/auto_ap/yodlee/core2.clj index cb64ab98..e51e410b 100644 --- a/src/clj/auto_ap/yodlee/core2.clj +++ b/src/clj/auto_ap/yodlee/core2.clj @@ -11,7 +11,7 @@ #_{:clj-kondo/ignore [:unused-namespace]} [yang.scheduler :as scheduler] [clj-time.coerce :as coerce] - [datomic.api :as d] + [datomic.client.api :as dc] [auto-ap.datomic :refer [conn]] [auto-ap.datomic.clients :as d-clients])) ;; switch all of this to use tokens instead of passing around client codes, particularly because the codes @@ -293,7 +293,7 @@ :body :providerAccount first)) - @(d/transact conn [[:db/retractEntity (:db/id (d/entity (d/db conn) [:yodlee-provider-account/id id]))]])) + (dc/transact conn {:tx-data [[:db/retractEntity [:yodlee-provider-account/id id]]]})) (defn upsert-accounts-tx ([client-code] @@ -322,8 +322,8 @@ (defn refresh-provider-account [client-code id] (log/info "refreshing yodlee provider account id" id) - @(d/transact conn (upsert-accounts-tx client-code - [(get-provider-account client-code id)]))) + (dc/transact conn {:tx-data (upsert-accounts-tx client-code + [(get-provider-account client-code id)])})) (defn upsert-accounts [] (let [concurrent 20 @@ -341,7 +341,7 @@ (async/to-chan! (d-clients/get-all))) (let [result (async/> - (d/query {:query {:find '[?e] - :in '[$ ?client ?end ] - :where [ - '[?e :invoice/client ?c] - '[?c :client/code ?client] - '[?e :invoice/date ?d ] - '[(<= ?d ?end) ]]} - :args [(d/db conn) - client - (c/to-date end)]}) - (mapv first) - (mapv (fn [i] - {:db/id i - :invoice/exclude-from-ledger true})) - (partition-all 100))] + (doseq [p (->> + (dc/q {:query {:find '[?e] + :in '[$ ?client ?end ] + :where [ + '[?e :invoice/client ?c] + '[?c :client/code ?client] + '[?e :invoice/date ?d ] + '[(<= ?d ?end) ]]} + :args [(dc/db conn) + client + (c/to-date end)]}) + (mapv first) + (mapv (fn [i] + {:db/id i + :invoice/exclude-from-ledger true})) + (partition-all 100))] - (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) - (println "process 100")) + (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) + (println "process 100")) - (doseq [p (->> - (d/query {:query {:find '[?e] - :in '[$ ?client ?end ] - :where [ - '[?e :transaction/client ?c] - '[?c :client/code ?client] - '[?e :transaction/date ?d ] - '[(<= ?d ?end) ]]} - :args [(d/db conn) - client - (c/to-date end)]}) - (mapv first) - (mapv (fn [i] - {:db/id i - :transaction/approval-status :transaction-approval-status/excluded})) - (partition-all 100))] + (doseq [p (->> + (dc/q {:query {:find '[?e] + :in '[$ ?client ?end ] + :where [ + '[?e :transaction/client ?c] + '[?c :client/code ?client] + '[?e :transaction/date ?d ] + '[(<= ?d ?end) ]]} + :args [(dc/db conn) + client + (c/to-date end)]}) + (mapv first) + (mapv (fn [i] + {:db/id i + :transaction/approval-status :transaction-approval-status/excluded})) + (partition-all 100))] - (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) (println "process 100")))) + (transact-with-ledger p {:user/name "mark-until-date" :user/role "admin"}) + (println "process 100"))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn load-accounts [conn] (let [[header & rows] (-> "master-account-list.csv" (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) - code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code + code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code :db/id])] :in ['$] :where ['[?e :account/name]]} - :args [(d/db conn)]}))) + :args [(dc/db conn)]}))) also-merge-txes (fn [also-merge old-account-id] (if old-account-id (let [[sunset-account] - (first (d/query {:query {:find ['?a ] - :in ['$ '?ac ] - :where ['[?a :account/numeric-code ?ac]]} - :args [(d/db conn) also-merge ]}))] + (first (dc/q {:query {:find ['?a ] + :in ['$ '?ac ] + :where ['[?a :account/numeric-code ?ac]]} + :args [(dc/db conn) also-merge ]}))] (into (mapv (fn [[entity id _]] [:db/add entity id old-account-id]) - (d/query {:query {:find ['?e '?id '?a ] + (dc/q {:query {:find ['?e '?id '?a ] :in ['$ '?ac ] :where ['[?a :account/numeric-code ?ac] '[?e ?at ?a] '[?at :db/ident ?id]]} - :args [(d/db conn) also-merge ]})) + :args [(dc/db conn) also-merge ]})) [[:db/retractEntity sunset-account]])) [])) @@ -186,32 +186,33 @@ conj [] rows)] - @(d/transact conn txes))) + (dc/transact conn {:tx-data txes}))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn find-bad-accounts [] - (set (map second (d/query {:query {:find ['(pull ?x [*]) '?z] + (set (map second (dc/q {:query {:find ['(pull ?x [*]) '?z] :in ['$] :where ['[?e :account/numeric-code ?z] '[(<= ?z 9999)] '[?x ?a ?e]]} - :args [(d/db (d/connect uri))]})))) + :args [(dc/db conn)]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn delete-4-digit-accounts [] - @(d/transact (d/connect uri) - (transduce - (comp - (map first) - (map (fn [old-account-id] - [:db/retractEntity old-account-id]))) - conj - [] - (d/query {:query {:find ['?e] - :in ['$] - :where ['[?e :account/numeric-code ?z] - '[(<= ?z 9999)]]} - :args [(d/db (d/connect uri))]}))) + (dc/transact conn + {:tx-data + (transduce + (comp + (map first) + (map (fn [old-account-id] + [:db/retractEntity old-account-id]))) + conj + [] + (dc/q {:query {:find ['?e] + :in ['$] + :where ['[?e :account/numeric-code ?z] + '[(<= ?z 9999)]]} + :args [(dc/db conn)]}))}) ) @@ -224,31 +225,30 @@ (fn [acc [e z]] (update acc z conj e)) {} - (d/query {:query {:find ['?e '?z] - :in ['$] - :where ['[?e :account/numeric-code ?z]]} - :args [(d/db (d/connect uri))]})))) + (dc/q {:query {:find ['?e '?z] + :in ['$] + :where ['[?e :account/numeric-code ?z]]} + :args [(dc/db conn)]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn customize-accounts [customer filename] - (let [conn (d/connect uri) - [_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) - [client-id] (first (d/query (-> {:query {:find ['?e] + (let [[_ & rows] (-> filename (io/resource) io/input-stream (BOMInputStream.) (io/reader) csv/read-csv) + [client-id] (first (dc/q (-> {:query {:find ['?e] :in ['$ '?z] :where [['?e :client/code '?z]]} - :args [(d/db (d/connect uri)) customer]}))) + :args [(dc/db conn) customer]}))) _ (println client-id) - code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code + code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code {:account/applicability [:db/ident]} :db/id])] :in ['$] :where ['[?e :account/name]]} - :args [(d/db conn)]}))) + :args [(dc/db conn)]}))) - existing-account-overrides (d/query (-> {:query {:find ['?e] + existing-account-overrides (dc/q (-> {:query {:find ['?e] :in ['$ '?client-id] :where [['?e :account-client-override/client '?client-id]]} - :args [(d/db (d/connect uri)) client-id]})) + :args [(dc/db conn) client-id]})) @@ -315,16 +315,16 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn fix-transactions-without-locations [client-code location] (->> - (d/query {:query {:find ['(pull ?e [*])] - :in ['$ '?client-code] - :where ['[?e :transaction/accounts ?ta] - '[?e :transaction/matched-rule] - '[?e :transaction/approval-status :transaction-approval-status/approved] - '(not [?ta :transaction-account/location]) - '[?e :transaction/client ?c] - '[?c :client/code ?client-code] - ]} - :args [(d/db (d/connect uri)) client-code]}) + (dc/q {:query {:find ['(pull ?e [*])] + :in ['$ '?client-code] + :where ['[?e :transaction/accounts ?ta] + '[?e :transaction/matched-rule] + '[?e :transaction/approval-status :transaction-approval-status/approved] + '(not [?ta :transaction-account/location]) + '[?e :transaction/client ?c] + '[?c :client/code ?client-code] + ]} + :args [(dc/db conn) client-code]}) (mapcat (fn [[{:transaction/keys [accounts]}]] (mapv @@ -405,44 +405,48 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history [i] - (vec (sort-by first (d/query + (vec (sort-by first (dc/q {:query {:find ['?tx '?z '?v ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z] '[(= ?ad true)]]} - :args [i (d/history (d/db (d/connect uri)))]})))) + :args [i (dc/history (dc/db conn))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn entity-history-with-revert [i] - (vec (sort-by first (d/query + (vec (sort-by first (dc/q {:query {:find ['?tx '?z '?v '?ad ] :in ['?i '$] :where ['[?i ?a ?v ?tx ?ad] '[?a :db/ident ?z]]} - :args [i (d/history (d/db (d/connect uri)))]})))) + :args [i (dc/history (dc/db conn))]})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-detail [i] - (map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v :added) + (map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v) (:data (first - (d/tx-range (d/log (d/connect uri)) - i - (inc i)))))) - + (dc/tx-range conn + {:start i + :end (inc i)}))))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn tx-range-detail [i] - (map (juxt :e #(d/ident (d/db (d/connect uri)) (:a %)) :v) + (map (juxt :e #(pull-attr (dc/db conn) :db/ident (:a %)) :v) - (mapcat :data (d/tx-range (d/log (d/connect uri)) - (- i 100) - (+ i 100))))) + (mapcat :data (dc/tx-range conn + {:start (- i 100) + :end (+ i 100)})))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn start-db [] (mu/start-publisher! {:type :dev}) - (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn}))) + (mount.core/start (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client}))) + +(defn restart-db [] + (datomic.dev-local/release-db {:system "dev" :db-name "prod-migration"}) + (mount.core/stop (mount.core/only #{#'auto-ap.datomic/conn #'auto-ap.datomic/client})) + (start-db)) @@ -481,29 +485,15 @@ (async/invoice-id (fn [i] (try (Long/parseLong i) (catch Exception e - (:db/id (d/pull db '[:db/id] + (:db/id (dc/pull db '[:db/id] [:invoice/original-id (Long/parseLong (first (str/split i #"-")))]))))) invoice-totals (->> data (drop 1) @@ -521,7 +511,7 @@ :let [ invoice-id (i->invoice-id i) - invoice (d/entity db invoice-id) + invoice (dc/pull db '[FILL_IN] invoice-id) current-total (:invoice/total invoice) target-total (invoice-totals invoice-id) ;; TODO should include expense accounts not visible new-account? (not (boolean (or (some-> invoice-expense-account-id not-empty Long/parseLong) @@ -529,10 +519,9 @@ invoice-expense-account-id (or (some-> invoice-expense-account-id not-empty Long/parseLong) (:db/id (first (:invoice/expense-accounts invoice))) - (d/tempid :db.part/user)) + (random-tempid)) invoice-expense-account (when-not new-account? - (or (d/entity db invoice-expense-account-id) - (d/entity db [:invoice-expense-account/original-id invoice-expense-account-id]))) + (dc/pull db '[FILL_IN]invoice-expense-account-id)) current-account-id (:db/id (:invoice-expense-account/account invoice-expense-account)) target-account-id (Long/parseLong (str/trim target-account)) @@ -548,19 +537,19 @@ target-expense-account-location location - [[_ _ invoice-payment]] (vec (d/q - '[:find ?p ?a ?ip - :in $ ?i - :where [?ip :invoice-payment/invoice ?i] - [?ip :invoice-payment/amount ?a] - [?ip :invoice-payment/payment ?p] - ] - db invoice-id))] + [[_ _ invoice-payment]] (vec (dc/q + '[:find ?p ?a ?ip + :in $ ?i + :where [?ip :invoice-payment/invoice ?i] + [?ip :invoice-payment/amount ?a] + [?ip :invoice-payment/payment ?p] + ] + db invoice-id))] :when current-total] [ (when (not (auto-ap.utils/dollars= current-total target-total)) - {:db/id invoice-id + {:db/id invoice-id :invoice/total target-total}) (when new-account? @@ -596,19 +585,19 @@ #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-schema [prefix] - (->> (d/q '[:find ?i + (->> (dc/q '[:find ?i :in $ ?p :where [_ :db/ident ?i] - [(namespace ?i) ?p]] (d/db auto-ap.datomic/conn) prefix) + [(namespace ?i) ?p]] (dc/db auto-ap.datomic/conn) prefix) (mapcat identity) vec)) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defn get-idents [] - (->> (d/q '[:find ?i + (->> (dc/q '[:find ?i :in $ :where [_ :db/ident ?i]] - (d/db auto-ap.datomic/conn) ) + (dc/db conn) ) (mapcat identity) (map str) (sort) diff --git a/src/cljs/auto_ap/events.cljs b/src/cljs/auto_ap/events.cljs index 5679baff..094147bf 100644 --- a/src/cljs/auto_ap/events.cljs +++ b/src/cljs/auto_ap/events.cljs @@ -30,7 +30,7 @@ [:intuit-bank-account [:name :id :external-id]] :use-date-instead-of-post-date :locations :include-in-reports :current-balance :yodlee-balance-old] ] - [:address [:street1 :street2 :city :state :zip]] + [:address [:id :street1 :street2 :city :state :zip]] [:forecasted-transactions [:id :amount :identifier :day-of-month]]] (= "admin" (or (get (jwt->data token) "role") (get (jwt->data token) "user/role")) ) (into [[:yodlee-provider-accounts [:id [:accounts [:id :name :number :available-balance]]]] [:plaid-items [:id [:accounts [:id :name :number :balance]]]]]))) diff --git a/src/cljs/auto_ap/views/pages/admin/accounts.cljs b/src/cljs/auto_ap/views/pages/admin/accounts.cljs index 02a9d21c..b4c0f7dc 100644 --- a/src/cljs/auto_ap/views/pages/admin/accounts.cljs +++ b/src/cljs/auto_ap/views/pages/admin/accounts.cljs @@ -16,7 +16,7 @@ [reagent.core :as reagent] [vimsical.re-frame.fx.track :as track])) -(def default-read [:numeric-code :name :location :type :account_set :applicability :invoice-allowance :vendor-allowance :id [:client-overrides [:name [:client [:name :id]]]]]) +(def default-read [:numeric-code :name :location :type :account_set :applicability :invoice-allowance :vendor-allowance :id [:client-overrides [:id :name [:client [:name :id]]]]]) diff --git a/src/cljs/auto_ap/views/pages/admin/clients/form.cljs b/src/cljs/auto_ap/views/pages/admin/clients/form.cljs index 3f868a0b..b361a52c 100644 --- a/src/cljs/auto_ap/views/pages/admin/clients/form.cljs +++ b/src/cljs/auto_ap/views/pages/admin/clients/form.cljs @@ -169,7 +169,8 @@ :week-a-debits (:week-a-debits new-client-data) :week-b-credits (:week-b-credits new-client-data) :week-b-debits (:week-b-debits new-client-data) - :address {:street1 (:street1 (:address new-client-data)) + :address {:id (:id (:address new-client-data)) + :street1 (:street1 (:address new-client-data)) :street2 (:street2 (:address new-client-data)), :city (:city (:address new-client-data)) :state (:state (:address new-client-data)) diff --git a/things-to-search-for.txt b/things-to-search-for.txt new file mode 100644 index 00000000..a045c14a --- /dev/null +++ b/things-to-search-for.txt @@ -0,0 +1,7 @@ +Look for getting all vendors (too slow) or clients +or-join syntax changed? +helper to lookup rel db-id +New way of automating ledger entries +Make sure no history on ledger +it looks like there are a bbunch of orrphaned customizations for accounts, breaking indexes +automatically rebuild search indexes