notid/integreat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disallows chatgpt for emailed invoices.

Browse Source
This commit is contained in:
Bryce
2025-03-25 21:26:40 -07:00
parent 581033001b
commit 01329bbdf5
5 changed files with 30 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
resources/public/output.css
View File

File diff suppressed because one or more lines are too long

2
src/clj/auto_ap/jobs/import_uploaded_invoices.clj
View File

@@ -62,7 +62,7 @@
:input-stream (io/input-stream filename)
:metadata {:content-type "application/pdf"
:content-length (.length (io/file filename))})
(let [imports (->> (parse/parse-file filename filename)
(let [imports (->> (parse/parse-file filename filename :allow-glimpse? false)
(map #(assoc %
:source-url (str "https://" (:data-bucket env)
"/"

44
src/clj/auto_ap/parse.clj
View File

@@ -34,14 +34,14 @@
([text full-text template]
(when (and template
(or (not (:multi-match? template))
(re-find (:multi-match? template) text )))
(re-find (:multi-match? template) text)))
[(->> template
:extract
(reduce-kv
(fn [result k v]
(let [value (some-> (or (first (map second (re-seq v text)))
(first (map second (re-seq v full-text))))
str/trim )
str/trim)
[value-parser parser-params] (-> template :parser k)]
(assoc result k (try
(u/parse-value value-parser parser-params value)
@@ -61,14 +61,20 @@
(extract-template text)))
(defmulti parse-file (fn [_ filename] (.toLowerCase (last (str/split filename #"\." )))))
(defmulti parse-file
"Parses a file based on its extension. Accepts options as additional arguments.
Options:
- :allow-glimpse? (default false) - If true, allows parsing a glimpse of the file."
(fn [_ filename & opts]
(.toLowerCase (last (str/split filename #"\.")))))
(defn invoke-glimpse2 [f]
(let [result (slurp (:payload (lambda/invoke {:client-config {:request-timeout 120000
:socket-timeout 120000}} {:function-name "glimpse2" :payload
(json/write-str
(alog/peek ::x {"url" (str "https://" "data.prod.app.integreatconsult.com" "/" f ) }))
})))]
(alog/peek ::x {"url" (str "https://" "data.prod.app.integreatconsult.com" "/" f)}))})))]
(alog/info ::glimpse2-payload :payload result)
(-> result
@@ -81,46 +87,46 @@
(s3/put-object {:bucket-name "data.prod.app.integreatconsult.com"
:key tmp-key
:input-stream f}))
is (invoke-glimpse2 tmp-key) ]
is (invoke-glimpse2 tmp-key)]
(alog/peek ::glimpse2-result is)
(for [i is]
{:date (u/parse-value :clj-time "yyyy-MM-dd" (str/trim (get i "date")))
:customer-identifier (get i "customer_identifier")
:account-number (not-empty (get i "account_number"))
:vendor-search (get i "vendor_identifier")
:vendor-code (-> (vendors/best-match (get i "vendor_identifier") )
:vendor-code (-> (vendors/best-match (get i "vendor_identifier"))
(get "label"))
:total (get i "total")
:invoice-number (get i "invoice_number")
:template "None found - defaulting to ChatGPT"})
)
:template "None found - defaulting to ChatGPT"}))
(catch Exception e
(alog/warn ::glimpse2-not-work :error e)
nil)))
(defmethod parse-file
"pdf"
[file _]
[file _ & {:keys [allow-glimpse?] :or {allow-glimpse? false}}]
(or
(-> (sh/sh "pdftotext" "-layout" file "-")
:out
parse)
(alog/peek ::glimpse2-result (glimpse2 file))))
(and allow-glimpse? (alog/peek ::glimpse2-result (glimpse2 file)))))
(defmethod parse-file
"csv"
[file filename]
[file filename & _]
(csv/parse-file file filename))
(defmethod parse-file
"xls"
[file filename]
[file filename & _]
(excel/parse-file file filename))
(defmethod parse-file
"xlsx"
[file filename]
[file filename & _]
(excel/parse-file file filename))
(defn best-match
@@ -147,17 +153,17 @@
(sort-by second)
first)
word-set (set (filter (complement str/blank?) (str/split (.toLowerCase invoice-client-name) #"[\s:\-]" )))
word-set (set (filter (complement str/blank?) (str/split (.toLowerCase invoice-client-name) #"[\s:\-]")))
client-word-match (->> clients
(map
(fn [{:keys [:client/matches :client/name] :as client :or {matches []}}]
(let [client-words (-> #{}
(into
(mapcat
(fn [match] (str/split (.toLowerCase match) #"\s" ))
(fn [match] (str/split (.toLowerCase match) #"\s"))
matches))
(into
(str/split (.toLowerCase name) #"\s" )))]
(str/split (.toLowerCase name) #"\s")))]
[client (count (set/intersection client-words word-set))])))
(filter (fn [[_ c]] (> c 0)))
(sort-by (fn [[_ c]] c))
@@ -184,14 +190,14 @@
(mapcat (fn [{:keys [:location-match/location :location-match/matches]}]
(map (fn [match] [location match]) matches)))
(filter (fn [[_ match]]
(re-find (re-pattern (str "(?i)" match)) text)) )
(re-find (re-pattern (str "(?i)" match)) text)))
first
first)
(->> client
:client/location-matches
(mapcat (fn [{:keys [:location-match/location :location-match/matches]}]
(map (fn [match] [location match]) matches)))
(filter (fn [[_ match]] (re-find (re-pattern (str "(?i)" match)) full-text)) )
(filter (fn [[_ match]] (re-find (re-pattern (str "(?i)" match)) full-text)))
first
first)
(:client/default-location client)

2
src/clj/auto_ap/routes/invoices.clj
View File

@@ -278,7 +278,7 @@
"text/csv"
"application/pdf")
:content-length (.length tempfile)})
imports (->> (parse/parse-file (.getPath tempfile) filename)
imports (->> (parse/parse-file (.getPath tempfile) filename :allow-glimpse? true)
(map #(assoc %
:client-override client
:location-override location

2
src/clj/auto_ap/ssr/invoice/import.clj
View File

@@ -716,7 +716,7 @@
:content-length (.length tempfile)})
imports (->> (if force-chatgpt
(parse/glimpse2 (.getPath tempfile))
(parse/parse-file (.getPath tempfile) filename))
(parse/parse-file (.getPath tempfile) filename :allow-glimpse? true))
(map #(assoc %
:client-override force-client
:location-override force-location