Makes integreat run on datomic cloud
This commit is contained in:
@@ -20,6 +20,9 @@
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[config.core :refer [env]]
|
||||
[datomic.api :as d]
|
||||
[compojure.core :refer [context defroutes GET routes wrap-routes]]
|
||||
[config.core :refer [env]]
|
||||
[datomic.client.api :as dc]
|
||||
[ring.middleware.json :refer [wrap-json-response]]
|
||||
[venia.core :as venia]))
|
||||
|
||||
@@ -48,8 +51,6 @@
|
||||
[(auto-ap.time/unparse ?d3 auto-ap.time/normal-date) ?d4]]}
|
||||
:args [(d/db conn) client-id]}))))
|
||||
|
||||
|
||||
|
||||
(defn client-tag [params]
|
||||
(when-let [code (or (params "client-code")
|
||||
(:client-code params))]
|
||||
|
||||
@@ -54,8 +54,8 @@
|
||||
(defn parse-invoice-number [{:keys [invoice-number]}]
|
||||
(or invoice-number ""))
|
||||
|
||||
(defn parse-vendor [{:keys [vendor-name check]} vendors]
|
||||
(let [v (vendors vendor-name)]
|
||||
(defn parse-vendor [{:keys [vendor-name check]} vendor-name->vendor]
|
||||
(let [v (vendor-name->vendor vendor-name)]
|
||||
(cond v
|
||||
v
|
||||
|
||||
@@ -65,9 +65,6 @@
|
||||
:else
|
||||
(throw (Exception. (str "Vendor '" vendor-name "' not found."))))))
|
||||
|
||||
(defn parse-vendor-id [{:keys [vendor]}]
|
||||
(:db/id vendor))
|
||||
|
||||
(defn parse-automatically-paid-when-due [{:keys [vendor client-id]}]
|
||||
(boolean ((set (map :db/id (:vendor/automatically-paid-when-due vendor))) client-id)))
|
||||
|
||||
@@ -77,22 +74,31 @@
|
||||
|
||||
(defn parse-invoice-rows [excel-rows]
|
||||
(let [columns [:raw-date :vendor-name :check :location :invoice-number :amount :client-name :bill-entered :bill-rejected :added-on :exported-on :account-numeric-code]
|
||||
all-vendors (->> (d/q '[:find [?e ...]
|
||||
:in $
|
||||
:where [?e :vendor/name]]
|
||||
(d/db conn))
|
||||
(d/pull-many (d/db conn) d-vendors/default-read)
|
||||
(by :vendor/name))
|
||||
all-clients (d-clients/get-all)
|
||||
all-clients (merge (by :client/code all-clients) (by :client/name all-clients))
|
||||
rows (->> (str/split excel-rows #"\n")
|
||||
(map #(str/split % #"\t"))
|
||||
(map #(into {} (map (fn [c k] [k c]) % columns)))
|
||||
tabulated (->> (str/split excel-rows #"\n")
|
||||
(map #(str/split % #"\t"))
|
||||
(map #(into {} (map (fn [c k] [k c]) % columns))))
|
||||
vendor-name->vendor (->>
|
||||
(set (map :vendor-name tabulated))
|
||||
(dc/q '[:find ?n ?v
|
||||
:in $ [?n ...]
|
||||
:where [?v :vendor/name ?n]]
|
||||
(dc/db conn)
|
||||
)
|
||||
(into {}))
|
||||
all-clients (merge (into {}(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
|
||||
:in $
|
||||
:where [?v :client/name ?n]]
|
||||
(dc/db conn)))
|
||||
(into {}
|
||||
(dc/q '[:find ?n (pull ?v [:db/id :client/locations])
|
||||
:in $
|
||||
:where [?v :client/code ?n]]
|
||||
(dc/db conn))))
|
||||
rows (->> tabulated
|
||||
(map reset-id)
|
||||
(map assoc-client-code)
|
||||
(map (c/parse-or-error :client-id #(parse-client % all-clients)))
|
||||
(map (c/parse-or-error :vendor #(parse-vendor % all-vendors)))
|
||||
(map (c/parse-or-error :vendor-id #(parse-vendor-id %)))
|
||||
(map (c/parse-or-error :vendor-id #(parse-vendor % vendor-name->vendor)))
|
||||
(map (c/parse-or-error :automatically-paid-when-due #(parse-automatically-paid-when-due %)))
|
||||
(map (c/parse-or-error :schedule-payment-dom #(parse-schedule-payment-dom %)))
|
||||
(map (c/parse-or-error :account-id c/parse-account-numeric-code))
|
||||
@@ -107,51 +113,54 @@
|
||||
(throw (ex-info (str "No vendor found. Please supply an forced vendor.")
|
||||
{:vendor-code vendor-code})))
|
||||
(let [vendor-id (or forced-vendor
|
||||
(->> (d/query
|
||||
(->> (dc/q
|
||||
{:query {:find ['?vendor]
|
||||
:in ['$ '?vendor-name]
|
||||
:where ['[?vendor :vendor/name ?vendor-name]]}
|
||||
:args [(d/db (d/connect uri)) vendor-code]})
|
||||
:args [(dc/db conn) vendor-code]})
|
||||
first
|
||||
first))]
|
||||
(when-not vendor-id
|
||||
(throw (ex-info (str "Vendor matching name \"" vendor-code "\" not found.")
|
||||
{:vendor-code vendor-code})))
|
||||
|
||||
(if-let [matching-vendor (->> (d/query
|
||||
(if-let [matching-vendor (->> (dc/q
|
||||
{:query {:find [(list 'pull '?vendor-id d-vendors/default-read)]
|
||||
:in ['$ '?vendor-id]}
|
||||
:args [(d/db (d/connect uri)) vendor-id]})
|
||||
:args [(dc/db conn) vendor-id]})
|
||||
first
|
||||
first)]
|
||||
matching-vendor
|
||||
(throw (ex-info (str "No vendor with the name " vendor-code " was found.")
|
||||
{:vendor-code vendor-code})))))
|
||||
|
||||
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]} clients]
|
||||
(let [[matching-client similarity] (cond
|
||||
account-number (parse/best-match clients account-number 0.0)
|
||||
customer-identifier (parse/best-match clients customer-identifier)
|
||||
client-override [(first (filter (fn [c]
|
||||
(= (:db/id c) (Long/parseLong client-override)))
|
||||
clients))
|
||||
1.0])
|
||||
matching-vendor (match-vendor vendor-code vendor-override)
|
||||
matching-location (or (when-not (str/blank? location-override)
|
||||
(defn import->invoice [{:keys [invoice-number source-url customer-identifier account-number total date vendor-code text full-text client-override vendor-override location-override import-status]}]
|
||||
(let [matching-client (cond
|
||||
account-number (d-clients/exact-match account-number)
|
||||
customer-identifier (d-clients/best-match customer-identifier)
|
||||
client-override (Long/parseLong client-override))
|
||||
|
||||
matching-vendor (match-vendor vendor-code vendor-override)
|
||||
matching-location (or (when-not (str/blank? location-override)
|
||||
location-override)
|
||||
(parse/best-location-match matching-client text full-text))]
|
||||
(remove-nils #:invoice {:invoice/client (:db/id matching-client)
|
||||
:invoice/client-identifier (or account-number customer-identifier)
|
||||
:invoice/vendor (:db/id matching-vendor)
|
||||
:invoice/similarity (some-> similarity double (#(- 1.0 %)))
|
||||
:invoice/source-url source-url
|
||||
:invoice/invoice-number invoice-number
|
||||
:invoice/total (Double/parseDouble total)
|
||||
:invoice/date (to-date date)
|
||||
:invoice/location matching-location
|
||||
:invoice/import-status (or import-status :import-status/pending)
|
||||
:invoice/outstanding-balance (Double/parseDouble total)
|
||||
:invoice/status :invoice-status/unpaid})))
|
||||
(parse/best-location-match (dc/pull (dc/db conn)
|
||||
[{:client/location-matches [:location-match/location :location-match/matches]}
|
||||
:client/default-location
|
||||
:client/locations]
|
||||
matching-client)
|
||||
text
|
||||
full-text))]
|
||||
#:invoice {:invoice/client matching-client
|
||||
:invoice/client-identifier (or account-number customer-identifier)
|
||||
:invoice/vendor (:db/id matching-vendor)
|
||||
:invoice/source-url source-url
|
||||
:invoice/invoice-number invoice-number
|
||||
:invoice/total (Double/parseDouble total)
|
||||
:invoice/date (to-date date)
|
||||
:invoice/location matching-location
|
||||
:invoice/import-status (or import-status :import-status/pending)
|
||||
:invoice/outstanding-balance (Double/parseDouble total)
|
||||
:invoice/status :invoice-status/unpaid}))
|
||||
|
||||
(defn validate-invoice [invoice user]
|
||||
(when-not (:invoice/client invoice)
|
||||
@@ -168,7 +177,7 @@
|
||||
|
||||
(defn extant-invoice? [{:invoice/keys [invoice-number vendor client]}]
|
||||
(try
|
||||
(->> (d/query
|
||||
(->> (dc/q
|
||||
(cond-> {:query {:find ['?e '?outstanding-balance '?status '?import-status2]
|
||||
:in ['$ '?invoice-number '?vendor '?client]
|
||||
:where '[[?e :invoice/invoice-number ?invoice-number]
|
||||
@@ -178,7 +187,7 @@
|
||||
[?e :invoice/status ?status]
|
||||
[?e :invoice/import-status ?import-status]
|
||||
[?import-status :db/ident ?import-status2]]}
|
||||
:args [(d/db (d/connect uri)) invoice-number vendor client]}))
|
||||
:args [(dc/db conn) invoice-number vendor client]}))
|
||||
first
|
||||
boolean)
|
||||
(catch Exception e
|
||||
@@ -222,13 +231,13 @@
|
||||
:status :payment-status/cleared
|
||||
:date (:invoice/date invoice)})})
|
||||
transaction (when (= :invoice-status/paid (:invoice/status invoice))
|
||||
(let [[bank-account] (d/q '[:find [?ba ...]
|
||||
:in $ ?c
|
||||
:where [?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/type :bank-account-type/cash]
|
||||
]
|
||||
(d/db conn)
|
||||
client-id)]
|
||||
(let [[[bank-account]] (dc/q '[:find ?ba
|
||||
:in $ ?c
|
||||
:where [?c :client/bank-accounts ?ba]
|
||||
[?ba :bank-account/type :bank-account-type/cash]
|
||||
]
|
||||
(dc/db conn)
|
||||
client-id)]
|
||||
#:transaction {:amount (- (:invoice/total invoice))
|
||||
:payment payment-id
|
||||
:client (:invoice/client invoice)
|
||||
@@ -244,8 +253,8 @@
|
||||
:transaction-account/location "A"
|
||||
:transaction-account/amount (Math/abs (:invoice/total invoice))}]}))
|
||||
]
|
||||
[[:propose-invoice (d-invoices/code-invoice (validate-invoice (remove-nils invoice)
|
||||
user))]
|
||||
[`(d-invoices/propose-invoice ~(d-invoices/code-invoice (validate-invoice (remove-nils invoice)
|
||||
user)))
|
||||
(some-> payment remove-nils)
|
||||
transaction])))
|
||||
(filter identity)))
|
||||
@@ -260,19 +269,23 @@
|
||||
(defn import-uploaded-invoice [user imports]
|
||||
(lc/with-context {:area "upload-invoice"}
|
||||
(log/info "Number of invoices to import is" (count imports))
|
||||
(let [clients (d-clients/get-all)
|
||||
potential-invoices (->> imports
|
||||
(mapv #(import->invoice % clients))
|
||||
(mapv #(validate-invoice % user))
|
||||
(let [potential-invoices (->> imports
|
||||
(map import->invoice)
|
||||
(map #(validate-invoice % user))
|
||||
admin-only-if-multiple-clients
|
||||
(filter #(not (extant-invoice? %)))
|
||||
(mapv d-invoices/code-invoice)
|
||||
(mapv (fn [i] [:propose-invoice i])))]
|
||||
(when-not (seq potential-invoices)
|
||||
(throw (ex-info "No new invoices found."
|
||||
{})))
|
||||
(mapv (fn [i] `(d-invoices/propose-invoice ~i))))]
|
||||
|
||||
(log/info "creating invoice" potential-invoices)
|
||||
(transact-with-ledger potential-invoices user))))
|
||||
(let [tx (transact-with-ledger potential-invoices user)]
|
||||
(when-not (seq (dc/q '[:find ?i
|
||||
:in $ [?i ...]
|
||||
:where [?i :invoice/invoice-number]]
|
||||
(:db-after tx)
|
||||
(map :e (:tx-data tx))))
|
||||
(throw (ex-info "No new invoices found."
|
||||
{})))
|
||||
tx))))
|
||||
|
||||
(defn validate-account-rows [rows code->existing-account]
|
||||
(when-let [bad-types (seq (->> rows
|
||||
@@ -296,23 +309,22 @@
|
||||
{:rows duplicate-rows}))))
|
||||
|
||||
(defn import-account-overrides [customer filename]
|
||||
(let [conn (d/connect uri)
|
||||
[_ & rows] (-> filename (io/reader) csv/read-csv)
|
||||
[client-id] (first (d/query (-> {:query {:find ['?e]
|
||||
:in ['$ '?z]
|
||||
:where [['?e :client/code '?z]]}
|
||||
:args [(d/db (d/connect uri)) customer]})))
|
||||
code->existing-account (by :account/numeric-code (map first (d/query {:query {:find ['(pull ?e [:account/numeric-code
|
||||
(let [[_ & rows] (-> filename (io/reader) csv/read-csv)
|
||||
[client-id] (first (dc/q (-> {:query {:find ['?e]
|
||||
:in ['$ '?z]
|
||||
:where [['?e :client/code '?z]]}
|
||||
:args [(dc/db conn) customer]})))
|
||||
code->existing-account (by :account/numeric-code (map first (dc/q {:query {:find ['(pull ?e [:account/numeric-code
|
||||
{:account/applicability [:db/ident]}
|
||||
:db/id])]
|
||||
:in ['$]
|
||||
:where ['[?e :account/name]]}
|
||||
:args [(d/db conn)]})))
|
||||
:args [(dc/db conn)]})))
|
||||
|
||||
existing-account-overrides (d/query (-> {:query {:find ['?e]
|
||||
:in ['$ '?client-id]
|
||||
:where [['?e :account-client-override/client '?client-id]]}
|
||||
:args [(d/db (d/connect uri)) client-id]}))
|
||||
existing-account-overrides (dc/q (-> {:query {:find ['?e]
|
||||
:in ['$ '?client-id]
|
||||
:where [['?e :account-client-override/client '?client-id]]}
|
||||
:args [(dc/db conn) client-id]}))
|
||||
rows (transduce (comp
|
||||
(map (fn [[_ account account-name override-name _ type]]
|
||||
[account account-name override-name type]))
|
||||
@@ -367,21 +379,21 @@
|
||||
existing-account-overrides)
|
||||
rows)]
|
||||
|
||||
@(d/transact conn txes)
|
||||
(dc/transact conn {:tx-data txes})
|
||||
txes))
|
||||
|
||||
(defn import-transactions-cleared-against [file]
|
||||
(let [[_ & rows] (-> file (io/reader) csv/read-csv)
|
||||
txes (transduce
|
||||
(comp
|
||||
(filter (fn [[transaction-id _]]
|
||||
(d/pull (d/db (d/connect uri)) '[:transaction/amount] (Long/parseLong transaction-id))))
|
||||
(map (fn [[transaction-id cleared-against]]
|
||||
{:db/id (Long/parseLong transaction-id)
|
||||
:transaction/cleared-against cleared-against})))
|
||||
conj
|
||||
[]
|
||||
rows)]
|
||||
txes (transduce
|
||||
(comp
|
||||
(filter (fn [[transaction-id _]]
|
||||
(dc/pull (dc/db conn) '[:transaction/amount] (Long/parseLong transaction-id))))
|
||||
(map (fn [[transaction-id cleared-against]]
|
||||
{:db/id (Long/parseLong transaction-id)
|
||||
:transaction/cleared-against cleared-against})))
|
||||
conj
|
||||
[]
|
||||
rows)]
|
||||
(transact-with-ledger txes nil)))
|
||||
|
||||
(defn batch-upload-transactions [{{:keys [data]} :edn-params user :identity}]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
(ns auto-ap.routes.queries
|
||||
(:require
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.datomic :refer [conn pull-attr upsert-entity]]
|
||||
[auto-ap.graphql.utils :refer [assert-admin]]
|
||||
[clojure.data.csv :as csv]
|
||||
[clojure.edn :as edn]
|
||||
@@ -9,8 +9,11 @@
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.logging :as log]
|
||||
[com.unbounce.dogstatsd.core :as statsd]
|
||||
[config.core :refer [env]]
|
||||
[config.core :refer [env]]
|
||||
[datomic.client.api :as dc]
|
||||
[datomic.api :as d]
|
||||
[compojure.core
|
||||
:refer [context defroutes GET POST PUT routes wrap-routes]]
|
||||
[ring.middleware.json :refer [wrap-json-response]]
|
||||
[ring.util.request :refer [body-string]]
|
||||
[unilog.context :as lc])
|
||||
@@ -34,21 +37,20 @@
|
||||
(let [query-string (str (slurp (:object-content (s3/get-object :bucket-name (:data-bucket env)
|
||||
:key (str "queries/" (:query-id params))))))]
|
||||
(log/info "Executing query " query-string)
|
||||
(into (list) (apply d/q (edn/read-string query-string)
|
||||
(into [(d/db conn)] (edn/read-string (get query-params "args" "[]")))))))))
|
||||
(into (list) (apply dc/q (edn/read-string query-string)
|
||||
(into [(dc/db conn)] (edn/read-string (get query-params "args" "[]")))))))))
|
||||
|
||||
|
||||
(defn put-query [guid body note & [lookup-key client]]
|
||||
(let [guid (if lookup-key
|
||||
(or (:saved-query/guid (d/pull (d/db conn) [:saved-query/guid] [:saved-query/lookup-key lookup-key]))
|
||||
(or (pull-attr (dc/db conn) :saved-query/guid [:saved-query/lookup-key lookup-key])
|
||||
guid)
|
||||
guid)]
|
||||
@(d/transact conn [(cond->
|
||||
{:saved-query/guid guid
|
||||
:saved-query/description note
|
||||
:saved-query/key (str "queries/" guid)}
|
||||
client (assoc :saved-query/client client)
|
||||
lookup-key (assoc :saved-query/lookup-key lookup-key))])
|
||||
(dc/transact conn [`(upsert-entity ~{:saved-query/guid guid
|
||||
:saved-query/description note
|
||||
:saved-query/key (str "queries/" guid)
|
||||
:saved-query/client client
|
||||
:saved-query/lookup-key lookup-key})])
|
||||
(s3/put-object :bucket-name (:data-bucket env)
|
||||
:key (str "queries/" guid)
|
||||
:input-stream (io/make-input-stream (.getBytes body) {})
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
(ns auto-ap.routes.yodlee2
|
||||
(:require
|
||||
[auto-ap.datomic :refer [conn]]
|
||||
[auto-ap.datomic :refer [conn pull-attr]]
|
||||
[auto-ap.datomic.clients :as d-clients]
|
||||
[auto-ap.graphql.utils :refer [assert-admin assert-can-see-client]]
|
||||
[auto-ap.routes.utils :refer [wrap-secure]]
|
||||
[auto-ap.yodlee.core2 :as yodlee]
|
||||
[clojure.tools.logging :as log]
|
||||
[config.core :refer [env]]
|
||||
[datomic.api :as d]))
|
||||
[datomic.client.api :as dc]))
|
||||
|
||||
(defn fastlink [{:keys [query-params identity]}]
|
||||
(assert-can-see-client identity (d/pull (d/db conn) [:db/id] [:client/code (get query-params "client")]))
|
||||
(assert-can-see-client identity (pull-attr (dc/db conn) :db/id [:client/code (get query-params "client")]))
|
||||
|
||||
(let [token (if-let [client-id (get query-params "client-id")]
|
||||
(-> client-id
|
||||
|
||||
Reference in New Issue
Block a user