From e3eeb4e80b055924a8a3279c2f917ca65535af17 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 16 Dec 2021 21:20:51 -0800 Subject: [PATCH] Adds intuit updater --- src/clj/auto_ap/graphql/transactions.clj | 18 +++++++++++--- src/clj/auto_ap/intuit/core.clj | 3 +-- src/clj/auto_ap/intuit/import.clj | 24 +++++++++++++++--- src/clj/auto_ap/ledger.clj | 1 + src/clj/auto_ap/server.clj | 3 +++ src/clj/auto_ap/yodlee/import.clj | 31 +++++++++++++++++++++--- 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/src/clj/auto_ap/graphql/transactions.clj b/src/clj/auto_ap/graphql/transactions.clj index f10ffd6f..3f871c68 100644 --- a/src/clj/auto_ap/graphql/transactions.clj +++ b/src/clj/auto_ap/graphql/transactions.clj @@ -73,14 +73,24 @@ (d-transactions/raw-graphql-ids ) :ids) specific-ids (d-transactions/filter-ids (:ids args)) - all-ids (into (set ids) specific-ids)] + all-ids (into (set ids) specific-ids) + db (d/db conn)] (log/info "Deleting " (count all-ids) args) (audit-transact-batch (mapcat (fn [i] - [{:db/id i - :transaction/approval-status :transaction-approval-status/suppressed} - [:db/retractEntity [:journal-entry/original-entity i]]]) + (let [transaction (d/entity db i) + payment-id (-> transaction :transaction/payment :db/id) + expected-deposit-id (-> transaction :transaction/expected-deposit :db/id)] + (cond->> [{:db/id i + :transaction/approval-status :transaction-approval-status/suppressed} + [:db/retractEntity [:journal-entry/original-entity i]]] + payment-id (into [{:db/id payment-id + :payment/status :payment-status/pending} + [:db/retract (:db/id transaction) :transaction/payment payment-id]]) + expected-deposit-id (into [{:db/id expected-deposit-id + :expected-deposit/status :expected-deposit-status/pending} + [:db/retract (:db/id transaction) :transaction/expected-deposit expected-deposit-id]])))) all-ids) (:id context)) {:message (str "Succesfully deleted " (count all-ids) " transactions.")})) diff --git a/src/clj/auto_ap/intuit/core.clj b/src/clj/auto_ap/intuit/core.clj index 8a55a4f3..b2a4c39f 100644 --- a/src/clj/auto_ap/intuit/core.clj +++ b/src/clj/auto_ap/intuit/core.clj @@ -22,7 +22,7 @@ (def authorization-code "AB11638463964I0tYPR3A1inog2HL407u2bZBXHg6LEqCbILRO") (def realm-id "4620816365202617680") -(def access-token "eyJlbmMiOiJBMTI4Q0JDLUhTMjU2IiwiYWxnIjoiZGlyIn0..PUPVn2EYxMCGTIqIjaAm5Q.jV43shT64akeaLZ0JDV_B9kW-rpDH6G-UHqOQgK6Tsupju9noyW6ct1XZxOq6m866kksisdTALzRtojdHhXGSArpk9aql6eSdzr0tq8KvxFNDfURw8GYY7cwbElkD6F0DAfotssFMXYTpGma6EfQz2u4q7UshHDnLe3Fk8EfRn4wob1FrcjB4KcuVNw86F959TMcMQ78N7joYs9PP5OwEnvQ5_Eg9HRMlKcFjs5YQJJW3t6i3j10gn0P7iXnZbIKHF_h_67cd2XJMS0bfyW-X0TfPVuHH0THNEeDM7al86L98fINtUeOKijDY7iJSDWOz0Kma41PM2fvWQbg0JVkpHqaK6mABRzMfK9XuX2pyskKc2AaODtPcB-Uv6WHcgGfND9BHlULweNeI1CsX0NZFFoH8P59XfdmYy92Ul5kHH_ND3D60e4v6mgy3TSSyPkCx6rNZdPCzxMwAUT86k-VXW1tapPmPLdDOiOfnS9UI_tc51YeExWRpDMnHRbIa2TRlaAchG9h41ZrgUiuKl9QoLRhPXP1yg8O5MCQbPzB23sQWN9OngO62opwvnqQg2kbn1S5wHxmpI0a3QPmegFCF9idy8lQVlUXp2myw_WuCZI6SZa5gbbKD33yo2crQX5-gg7ImyGT7tYGe-C440lQC8BWcL2gPr1gS3vHt9knja4EtWDfRCW278IQ2jibH3mr-XjQkWz9Rto7lPhYNLcyrUrlNmS800ZyQN5f01pRr-TrSMqiT5H0bTs7hGDGRdRp.jSDjETBLUrBxmvIiT3sOqg") + (def company-id "4620816365202617680") (def base-url "https://sandbox-quickbooks.api.intuit.com/v3") @@ -38,7 +38,6 @@ ;; "refreshToken": "AB11647191065B0olWYQ61wfq8uszBusfe6Jpn7Au7qY5exkLL", ;; "accessToken":, ;; -(def prod-access-token "eyJlbmMiOiJBMTI4Q0JDLUhTMjU2IiwiYWxnIjoiZGlyIn0..zbZ9E2iJQfn8cFNV9ROo-A.Md9HAuz1Vv08pANm-4tJSrZVGwhuBuElF8lrOFH_5mO2pBXUeYLra1ag6AdUBP7fdBgJvV53aXSWx2m7tnDygnaEHm95Wxr091HsawIAslzg59QkmAyC1UdbHCE7RX8q2pBeQAt2EBg8G-kmtFxtIIWlFuvt49vHPb1DpTG5z_Mf369OhjEBT0DrKfiCa9Jo1hcppkub6_aHt_aEFKLAshlvpGGV_peYImBiQH-Z2rM5ya7j1leuBdkmVzJH1MeH3mYJDKWoL1v7BTVPwE-4KTo72qCp-6dPvQPnVsHk1hpbvI2qjR9m5CvM3SmuwZqirpmOPGp9IL6gOJnEjGRf1mrk7MJqQ5DEr2T9nhCp9TWDoOle26hoE1dlNi83qFx0I5VwKcGHNmJz7BJxHhyidOcmi4V2RZLQ3CmA032Kc6AL8cgRKRuuhiyFcPBinlgFKzM-a82hp0lwpw6KeaaRSlR3i57W5SBmDoUaelyv9iVaIScs5tgjKAIPFv1Wu-6OnyUIAFwmhi_2dSNSj7NzXlEzDQ9ByoHJYG_DCN_2H-MwOpavNSz-rrGnqocBcTQYNH9P9D61tthS4NCxLLVtiLYHtT0ioJoS5esKuk8wM_jSz5oDNoR364CjEw0Ij9vIZ6eANXVf8Qu_IE9S8O4_G4__Wc4pi4nLKk5q3_kUj414m0ASE8Cm4Qph2b8i7wv1WsuBbQrdMOVQxuE2xFvuHYvT7lNuxO2SbV10iqKlepI.Sa9o2pE3xsVnuEMuuhXl2Q") diff --git a/src/clj/auto_ap/intuit/import.clj b/src/clj/auto_ap/intuit/import.clj index d341357e..aa7e7340 100644 --- a/src/clj/auto_ap/intuit/import.clj +++ b/src/clj/auto_ap/intuit/import.clj @@ -14,12 +14,16 @@ [unilog.context :as lc] [yang.scheduler :as scheduler])) +(def client-whitelist #{"PPFB"}) + (defn get-intuit-bank-accounts [db] (-> (d/q '[:find ?ba ?external-id - :in $ - :where [?ba :bank-account/intuit-bank-account ?iab] + :in $ + :where + [?c :client/bank-accounts ?ba] + [?ba :bank-account/intuit-bank-account ?iab] [?iab :intuit-bank-account/external-id ?external-id]] - db))) + db #_client-whitelist))) (defn upsert-transactions [] (let [db (d/db conn) @@ -53,6 +57,8 @@ (y/finish-import (assoc @stats :db/id import-id)))) + + (defn dry-run-upsert-transactions [] (let [db (d/db conn)] (clojure.data.csv/write-csv @@ -75,7 +81,9 @@ :date (auto-ap.time/parse (:Date r) auto-ap.time/iso-date) :status "posted"}) (y/grouped-new) - (map (juxt :client-code :bank-account-code :external-id :description-original :amount #(auto-ap.time/unparse-local (:date %) auto-ap.time/normal-date) :new?))) + :import + (mapcat identity) + (map (juxt #(:bank-account/code (d/entity db(:transaction/bank-account %))) :transaction/id :transaction/raw-id :transaction/amount :transaction/description-original #(auto-ap.time/unparse-local (coerce/to-date-time (:transaction/date %)) auto-ap.time/normal-date) ))) :separator \tab))) @@ -87,3 +95,11 @@ {:intuit-bank-account/external-id (:name ba) :intuit-bank-account/name (:name ba)}) bank-accounts)))) + +(mount/defstate upsert-transaction-worker + :start (scheduler/every (* 1000 60 60 24) upsert-transactions) + :stop (scheduler/stop upsert-transaction-worker)) + +(mount/defstate upsert-account-worker + :start (scheduler/every (* 1000 60 60 24) upsert-accounts) + :stop (scheduler/stop upsert-account-worker)) diff --git a/src/clj/auto_ap/ledger.clj b/src/clj/auto_ap/ledger.clj index 90a2f6b2..21db0000 100644 --- a/src/clj/auto_ap/ledger.clj +++ b/src/clj/auto_ap/ledger.clj @@ -185,6 +185,7 @@ '[?t :transaction/amount ?amt] '[(not= 0.0 ?amt)] '(not [?t :transaction/approval-status :transaction-approval-status/excluded]) + '(not [?t :transaction/approval-status :transaction-approval-status/suppressed]) '(not-join [?t] [?e :journal-entry/original-entity ?t])]} :args [(d/db conn)]}) (map first) diff --git a/src/clj/auto_ap/server.clj b/src/clj/auto_ap/server.clj index 09bb4e15..1b6a4987 100644 --- a/src/clj/auto_ap/server.clj +++ b/src/clj/auto_ap/server.clj @@ -9,6 +9,7 @@ [auto-ap.square.core :as square] [auto-ap.datomic.migrate :as migrate] [auto-ap.yodlee.import :as yodlee] + [auto-ap.intuit.import :as intuit] [nrepl.server :refer [start-server stop-server]] [config.core :refer [env]] [ring.adapter.jetty :refer [run-jetty]] @@ -34,6 +35,8 @@ (not (env :run-web? )) (into [#'jetty]) (not (env :run-background?)) (into [#'square/square-loader #'vendor/refresh-vendor-usages-worker + #'intuit/upsert-transaction-worker + #'intuit/upsert-account-worker #'ledger/touch-broken-ledger-worker #'ledger/process-txes-worker #'ledger/ledger-reconciliation-worker diff --git a/src/clj/auto_ap/yodlee/import.clj b/src/clj/auto_ap/yodlee/import.clj index cc557a7a..ffc282ab 100644 --- a/src/clj/auto_ap/yodlee/import.clj +++ b/src/clj/auto_ap/yodlee/import.clj @@ -200,8 +200,8 @@ (not= "POSTED" status) :not-posted - (and (:start-date bank-account) - (not (t/after? date (:start-date bank-account)))) + (and (:bank-account/start-date bank-account) + (not (t/after? date (:bank-account/start-date bank-account)))) :not-ready :else @@ -356,6 +356,29 @@ (log/info "Imported grouped transactions") transactions))) +(defn grouped-new [manual-transactions] + (let [transformed-transactions (->> manual-transactions + (filter #(= "posted" (:status %))) + (group-by #(select-keys % [:date :description-original :amount :client-id :bank-account-id])) + (vals) + (mapcat (fn [transaction-group] + (map + (fn [index {:keys [date description-original high-level-category amount bank-account-id client-id] :as transaction}] + {:id (str date "-" bank-account-id "-" description-original "-" amount "-" index "-" client-id) + :bank-account-id bank-account-id + :date (time/unparse date "YYYY-MM-dd") + :amount {:amount amount} + :description {:original description-original + :simple high-level-category} + :status "POSTED"}) + (range) + transaction-group)))) + all-rules (tr/get-all) + all-bank-accounts (by :db/id (get-all-bank-accounts)) + transaction->bank-account (comp all-bank-accounts :bank-account-id) + transactions (transactions->txs transformed-transactions nil transaction->bank-account (rm/rule-applying-fn all-rules) (get-existing))] + transactions)) + (defn do-import ([] (do-import (client/get-transactions))) @@ -378,13 +401,13 @@ (defn do-import2 ([] - (do-import2 (client2/get-transactions "AFH"))) + (do-import2 (client2/get-transactions "NGGL"))) ([transactions] (lc/with-context {:source "Import yodlee transactions"} (do (log/info "importing from yodlee2") - (let [import-id (start-import :import-source/yodlee "Automated Yodlee User") + (let [import-id (start-import :import-source/yodlee2 "Automated Yodlee User") all-bank-accounts (get-all-bank-accounts) transaction->bank-account (comp (by (comp :yodlee-account/id :bank-account/yodlee-account) all-bank-accounts) :accountId) all-rules (tr/get-all)