- Added ProjectModel class in models/project_model.py to define structure for Filevine project data with proper type hints and conversion methods (to_dict/from_dict) - Implemented get_firestore_document() helper function in app.py for retrieving specific Firestore documents - Enhanced dashboard pagination in app.py with improved error handling and debugging output for property contacts and project IDs - Overhauled sync.py with: * Parallel processing using ThreadPoolExecutor for efficient project synchronization * Comprehensive extraction of project data from Filevine forms (newFileReview, datesAndDeadlines, propertyInfo, etc.) * Improved error handling and logging throughout the sync process * Proper handling of date conversions and field mappings from Filevine to Firestore * Added property contacts email extraction and viewing_emails array population * Added support for filtering projects by specific ProjectId (15914808) for targeted sync - Added proper initialization of Filevine client in worker threads using thread-local storage - Improved handling of optional fields and default values in ProjectModel - Added detailed logging for progress tracking during synchronization This implementation enables reliable synchronization of Filevine project data to Firestore with proper data modeling and error handling, supporting the dashboard's data requirements.
301 lines
10 KiB
Python
301 lines
10 KiB
Python
import json
|
|
import os
|
|
from functools import wraps
|
|
from datetime import datetime, timedelta
|
|
import pytz
|
|
|
|
from flask import Flask, render_template, request, redirect, url_for, session, abort, jsonify
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
import firebase_admin
|
|
from firebase_admin import credentials, auth as fb_auth, firestore
|
|
import requests
|
|
from filevine_client import FilevineClient
|
|
|
|
|
|
app = Flask(__name__)
|
|
app.secret_key = os.environ.get("FLASK_SECRET_KEY", os.urandom(32))
|
|
|
|
# --- Firebase Admin init ---
|
|
_creds = None
|
|
json_inline = os.environ.get("FIREBASE_SERVICE_ACCOUNT_JSON")
|
|
file_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
|
if json_inline:
|
|
_creds = credentials.Certificate(json.loads(json_inline))
|
|
elif file_path and os.path.exists(file_path):
|
|
_creds = credentials.Certificate(file_path)
|
|
else:
|
|
raise RuntimeError("Firebase credentials not configured. Set GOOGLE_APPLICATION_CREDENTIALS or FIREBASE_SERVICE_ACCOUNT_JSON.")
|
|
|
|
firebase_admin.initialize_app(_creds)
|
|
db = firestore.client()
|
|
|
|
# --- Filevine env ---
|
|
FV_CLIENT_ID = os.environ.get("FILEVINE_CLIENT_ID")
|
|
FV_CLIENT_SECRET = os.environ.get("FILEVINE_CLIENT_SECRET")
|
|
FV_PAT = os.environ.get("FILEVINE_PERSONAL_ACCESS_TOKEN")
|
|
FV_ORG_ID = os.environ.get("FILEVINE_ORG_ID")
|
|
FV_USER_ID = os.environ.get("FILEVINE_USER_ID")
|
|
|
|
if not all([FV_CLIENT_ID, FV_CLIENT_SECRET, FV_PAT, FV_ORG_ID, FV_USER_ID]):
|
|
print("[WARN] Missing one or more Filevine env vars — dashboard will fail until set.")
|
|
|
|
# --- Cache ---
|
|
# No longer using cache - projects are stored in Firestore
|
|
|
|
PHASES = {
|
|
209436: "Nonpayment File Review",
|
|
209437: "Attorney File Review",
|
|
209438: "Notice Preparation",
|
|
209439: "Notice Pending",
|
|
209440: "Notice Expired",
|
|
209442: "Preparing and Filing UD",
|
|
209443: "Waiting for Answer",
|
|
209444: "Archived",
|
|
210761: "Service of Process",
|
|
211435: "Default",
|
|
211436: "Pre-Answer Motion",
|
|
211437: "Request for Trial",
|
|
211438: "Trial Prep and Trial",
|
|
211439: "Writ and Sheriff",
|
|
211440: "Lockout Pending",
|
|
211441: "Stipulation Preparation",
|
|
211442: "Stipulation Pending",
|
|
211443: "Stipulation Expired",
|
|
211446: "On Hold",
|
|
211466: "Request for Monetary Judgment",
|
|
211467: "Appeals and Post-Poss. Motions",
|
|
211957: "Migrated",
|
|
213691: "Close Out/ Invoicing",
|
|
213774: "Judgment After Stip & Order",
|
|
}
|
|
|
|
# --- Helpers ---
|
|
|
|
def login_required(view):
|
|
@wraps(view)
|
|
def wrapped(*args, **kwargs):
|
|
uid = session.get("uid")
|
|
if not uid:
|
|
return redirect(url_for("login"))
|
|
return view(*args, **kwargs)
|
|
return wrapped
|
|
|
|
|
|
def get_user_profile(uid: str):
|
|
"""Fetch user's Firestore profile: users/{uid} => { enabled, caseEmail, is_admin }"""
|
|
doc_ref = db.collection("users").document(uid)
|
|
snap = doc_ref.get()
|
|
if not snap.exists:
|
|
# bootstrap a placeholder doc so admins can fill it in
|
|
doc_ref.set({"enabled": False}, merge=True)
|
|
return {"enabled": False, "caseEmail": None, "is_admin": False}
|
|
data = snap.to_dict() or {}
|
|
return {
|
|
"enabled": bool(data.get("enabled", False)),
|
|
"caseEmail": data.get("caseEmail"),
|
|
"is_admin": bool(data.get("is_admin", False))
|
|
}
|
|
def get_firestore_document(collection_name: str, document_id: str):
|
|
"""
|
|
Retrieve a specific document from Firestore.
|
|
|
|
Args:
|
|
collection_name (str): Name of the Firestore collection
|
|
document_id (str): ID of the document to retrieve
|
|
|
|
Returns:
|
|
dict: Document data as dictionary, or None if document doesn't exist
|
|
"""
|
|
doc_ref = db.collection(collection_name).document(document_id)
|
|
doc = doc_ref.get()
|
|
if doc.exists:
|
|
return doc.to_dict()
|
|
else:
|
|
return None
|
|
|
|
|
|
|
|
def convert_to_pacific_time(date_str):
|
|
"""Convert UTC date string to Pacific Time and format as YYYY-MM-DD.
|
|
|
|
Args:
|
|
date_str (str): UTC date string in ISO 8601 format (e.g., "2025-10-24T19:20:22.377Z")
|
|
|
|
Returns:
|
|
str: Date formatted as YYYY-MM-DD in Pacific Time, or empty string if input is empty
|
|
"""
|
|
if not date_str:
|
|
return ''
|
|
|
|
try:
|
|
# Parse the UTC datetime
|
|
utc_time = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
|
|
|
|
# Set timezone to UTC
|
|
utc_time = utc_time.replace(tzinfo=pytz.UTC)
|
|
|
|
# Convert to Pacific Time
|
|
pacific_time = utc_time.astimezone(pytz.timezone('America/Los_Angeles'))
|
|
|
|
# Format as YYYY-MM-DD
|
|
return pacific_time.strftime('%Y-%m-%d')
|
|
except (ValueError, AttributeError) as e:
|
|
print(f"[WARN] Date conversion failed for '{date_str}': {e}")
|
|
return ''
|
|
|
|
|
|
def fetch_all_projects():
|
|
"""Fetch all projects for a user and store them in Firestore"""
|
|
# This function is now only used by sync.py
|
|
# In production, this should be removed or marked as deprecated
|
|
print("Fetching projects....")
|
|
# Initialize Filevine client
|
|
client = FilevineClient()
|
|
bearer = client.get_bearer_token()
|
|
|
|
# List projects (all pages)
|
|
projects = client.list_all_projects()
|
|
projects = projects[:]
|
|
|
|
# Fetch details for each
|
|
detailed_rows = []
|
|
|
|
# This functionality has been moved to sync.py
|
|
# The worker_pool module has been removed
|
|
# This function is kept for backward compatibility but should not be used in production
|
|
print("[DEPRECATED] fetch_all_projects() is deprecated. Use sync.py instead.")
|
|
return []
|
|
|
|
@app.route("/")
|
|
def index():
|
|
uid = session.get("uid")
|
|
if not uid:
|
|
return redirect(url_for("login"))
|
|
profile = get_user_profile(uid)
|
|
if profile.get("enabled") and profile.get("caseEmail"):
|
|
return redirect(url_for("dashboard"))
|
|
return redirect(url_for("welcome"))
|
|
|
|
|
|
@app.route("/login")
|
|
def login():
|
|
# Pass public Firebase config to template
|
|
fb_public = {
|
|
"apiKey": os.environ.get("FIREBASE_API_KEY", ""),
|
|
"authDomain": os.environ.get("FIREBASE_AUTH_DOMAIN", ""),
|
|
"projectId": os.environ.get("FIREBASE_PROJECT_ID", ""),
|
|
"appId": os.environ.get("FIREBASE_APP_ID", ""),
|
|
}
|
|
return render_template("login.html", firebase_config=fb_public)
|
|
|
|
|
|
@app.route("/session_login", methods=["POST"])
|
|
def session_login():
|
|
"""Exchanges Firebase ID token for a server session."""
|
|
id_token = request.json.get("idToken") if request.is_json else request.form.get("idToken")
|
|
if not id_token:
|
|
abort(400, "Missing idToken")
|
|
try:
|
|
decoded = fb_auth.verify_id_token(id_token)
|
|
uid = decoded["uid"]
|
|
session.clear()
|
|
session["uid"] = uid
|
|
# Optional: short session
|
|
session["expires_at"] = (datetime.utcnow() + timedelta(hours=8)).isoformat()
|
|
|
|
return jsonify({"ok": True})
|
|
except Exception as e:
|
|
print("[ERR] session_login:", e)
|
|
abort(401, "Invalid ID token")
|
|
|
|
|
|
@app.route("/logout")
|
|
def logout():
|
|
session.clear()
|
|
return redirect(url_for("login"))
|
|
|
|
|
|
@app.route("/welcome")
|
|
@login_required
|
|
def welcome():
|
|
uid = session.get("uid")
|
|
profile = get_user_profile(uid)
|
|
return render_template("welcome.html", profile=profile)
|
|
|
|
|
|
# --- Filevine API ---
|
|
# Filevine client is now in filevine_client.py
|
|
|
|
@app.route("/dashboard")
|
|
@app.route("/dashboard/<int:page>")
|
|
@login_required
|
|
def dashboard(page=1):
|
|
uid = session.get("uid")
|
|
profile = get_user_profile(uid)
|
|
if not profile.get("enabled"):
|
|
return redirect(url_for("welcome"))
|
|
|
|
# If user is admin and caseEmail query parameter is provided, use that instead
|
|
case_email = profile.get("caseEmail")
|
|
if profile.get("is_admin") and request.args.get('case_email'):
|
|
case_email = request.args.get('case_email').lower()
|
|
# Validate email format
|
|
if '@' not in case_email:
|
|
return abort(400, "Invalid email format")
|
|
|
|
if not case_email:
|
|
return redirect(url_for("welcome"))
|
|
|
|
# Pagination settings
|
|
per_page = 25
|
|
offset = (page - 1) * per_page
|
|
|
|
# Get total count efficiently using a count aggregation query
|
|
try:
|
|
# Firestore doesn't have a direct count() method, so we need to count documents
|
|
import time
|
|
start_time = time.time()
|
|
projects_ref = db.collection("projects")
|
|
# Filter projects where case_email is in viewing_emails array
|
|
query = projects_ref.where("viewing_emails", "array_contains", case_email.lower())
|
|
total_projects = int(query.count().get()[0][0].value)
|
|
end_time = time.time()
|
|
print(f"Filtered projects count: {total_projects} (took {end_time - start_time:.2f}s)")
|
|
except Exception as e:
|
|
print(f"[WARN] Failed to get filtered count: {e}")
|
|
total_projects = 0
|
|
|
|
# Calculate pagination
|
|
total_pages = (total_projects + per_page - 1) // per_page # Ceiling division
|
|
|
|
# Read only the current page from Firestore using limit() and offset()
|
|
import time
|
|
start_time = time.time()
|
|
# Filter projects where case_email is in viewing_emails array
|
|
projects_ref = db.collection("projects").where("viewing_emails", "array_contains", case_email.lower()).order_by("matter_description").limit(per_page).offset(offset)
|
|
docs = projects_ref.stream()
|
|
paginated_rows = []
|
|
|
|
for doc in docs:
|
|
paginated_rows.append(doc.to_dict())
|
|
|
|
end_time = time.time()
|
|
print(f"Retrieved {len(paginated_rows)} projects from Firestore (page {page} of {total_pages}) in {end_time - start_time:.2f}s")
|
|
from pprint import pprint
|
|
pprint([p['property_contacts'] for p in paginated_rows if p['property_contacts'].get('propertyManager1', None)])
|
|
pprint([p['ProjectId'] for p in paginated_rows ])
|
|
# Render table with pagination data
|
|
return render_template("dashboard.html",
|
|
rows=paginated_rows,
|
|
case_email=case_email,
|
|
current_page=page,
|
|
total_pages=total_pages,
|
|
total_projects=total_projects,
|
|
per_page=per_page)
|
|
|
|
|
|
# GAE compatibility
|
|
if __name__ == "__main__":
|
|
app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", "5004")))
|