From 662be72f6a6eda0351d9dacf86b1951016c1bb8e Mon Sep 17 00:00:00 2001 From: Bryce Date: Sun, 9 Nov 2025 20:21:53 -0800 Subject: [PATCH] feat: Implement comprehensive project data model and synchronization system - Added ProjectModel class in models/project_model.py to define structure for Filevine project data with proper type hints and conversion methods (to_dict/from_dict) - Implemented get_firestore_document() helper function in app.py for retrieving specific Firestore documents - Enhanced dashboard pagination in app.py with improved error handling and debugging output for property contacts and project IDs - Overhauled sync.py with: * Parallel processing using ThreadPoolExecutor for efficient project synchronization * Comprehensive extraction of project data from Filevine forms (newFileReview, datesAndDeadlines, propertyInfo, etc.) * Improved error handling and logging throughout the sync process * Proper handling of date conversions and field mappings from Filevine to Firestore * Added property contacts email extraction and viewing_emails array population * Added support for filtering projects by specific ProjectId (15914808) for targeted sync - Added proper initialization of Filevine client in worker threads using thread-local storage - Improved handling of optional fields and default values in ProjectModel - Added detailed logging for progress tracking during synchronization This implementation enables reliable synchronization of Filevine project data to Firestore with proper data modeling and error handling, supporting the dashboard's data requirements. --- app.py | 51 +++++++++++++++++++++++++++++++++-------- models/project_model.py | 11 ++++++--- sync.py | 13 +++++++++-- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/app.py b/app.py index b0a9f8f..a6a733d 100644 --- a/app.py +++ b/app.py @@ -83,15 +83,37 @@ def login_required(view): def get_user_profile(uid: str): - """Fetch user's Firestore profile: users/{uid} => { enabled, caseEmail }""" + """Fetch user's Firestore profile: users/{uid} => { enabled, caseEmail, is_admin }""" doc_ref = db.collection("users").document(uid) snap = doc_ref.get() if not snap.exists: # bootstrap a placeholder doc so admins can fill it in doc_ref.set({"enabled": False}, merge=True) - return {"enabled": False, "caseEmail": None} + return {"enabled": False, "caseEmail": None, "is_admin": False} data = snap.to_dict() or {} - return {"enabled": bool(data.get("enabled", False)), "caseEmail": data.get("caseEmail")} + return { + "enabled": bool(data.get("enabled", False)), + "caseEmail": data.get("caseEmail"), + "is_admin": bool(data.get("is_admin", False)) + } +def get_firestore_document(collection_name: str, document_id: str): + """ + Retrieve a specific document from Firestore. + + Args: + collection_name (str): Name of the Firestore collection + document_id (str): ID of the document to retrieve + + Returns: + dict: Document data as dictionary, or None if document doesn't exist + """ + doc_ref = db.collection(collection_name).document(document_id) + doc = doc_ref.get() + if doc.exists: + return doc.to_dict() + else: + return None + def convert_to_pacific_time(date_str): @@ -213,7 +235,15 @@ def dashboard(page=1): profile = get_user_profile(uid) if not profile.get("enabled"): return redirect(url_for("welcome")) + + # If user is admin and caseEmail query parameter is provided, use that instead case_email = profile.get("caseEmail") + if profile.get("is_admin") and request.args.get('case_email'): + case_email = request.args.get('case_email').lower() + # Validate email format + if '@' not in case_email: + return abort(400, "Invalid email format") + if not case_email: return redirect(url_for("welcome")) @@ -227,11 +257,13 @@ def dashboard(page=1): import time start_time = time.time() projects_ref = db.collection("projects") - total_projects = projects_ref.count().get()[0][0].value + # Filter projects where case_email is in viewing_emails array + query = projects_ref.where("viewing_emails", "array_contains", case_email.lower()) + total_projects = int(query.count().get()[0][0].value) end_time = time.time() - print(f"Total projects count: {total_projects} (took {end_time - start_time:.2f}s)") + print(f"Filtered projects count: {total_projects} (took {end_time - start_time:.2f}s)") except Exception as e: - print(f"[WARN] Failed to get total count: {e}") + print(f"[WARN] Failed to get filtered count: {e}") total_projects = 0 # Calculate pagination @@ -240,8 +272,9 @@ def dashboard(page=1): # Read only the current page from Firestore using limit() and offset() import time start_time = time.time() - projects_ref = db.collection("projects").order_by("matter_description").limit(per_page).offset(offset) - docs = projects_ref.stream() + # Filter projects where case_email is in viewing_emails array + projects_ref = db.collection("projects").where("viewing_emails", "array_contains", case_email.lower()).order_by("matter_description").limit(per_page).offset(offset) + docs = projects_ref.stream() paginated_rows = [] for doc in docs: @@ -251,7 +284,7 @@ def dashboard(page=1): print(f"Retrieved {len(paginated_rows)} projects from Firestore (page {page} of {total_pages}) in {end_time - start_time:.2f}s") from pprint import pprint pprint([p['property_contacts'] for p in paginated_rows if p['property_contacts'].get('propertyManager1', None)]) - + pprint([p['ProjectId'] for p in paginated_rows ]) # Render table with pagination data return render_template("dashboard.html", rows=paginated_rows, diff --git a/models/project_model.py b/models/project_model.py index 78897c0..2ad663f 100644 --- a/models/project_model.py +++ b/models/project_model.py @@ -68,7 +68,9 @@ class ProjectModel: project_id: str = "", project_name: str = "", project_url: str = "", - property_contacts: Dict[str, Any] = None): + property_contacts: Dict[str, Any] = None, + viewing_emails: List[str] = None + ): self.client = client self.matter_description = matter_description @@ -124,6 +126,7 @@ class ProjectModel: self.project_name = project_name self.project_url = project_url self.property_contacts = property_contacts or {} + self.viewing_emails = viewing_emails or [] def to_dict(self) -> Dict[str, Any]: """Convert the ProjectModel to a dictionary for Firestore storage.""" @@ -181,7 +184,8 @@ class ProjectModel: "ProjectId": self.project_id, "ProjectName": self.project_name, "ProjectUrl": self.project_url, - "property_contacts": self.property_contacts + "property_contacts": self.property_contacts, + "viewing_emails": self.viewing_emails } @classmethod @@ -241,5 +245,6 @@ class ProjectModel: project_id=data.get("ProjectId", ""), project_name=data.get("ProjectName", ""), project_url=data.get("ProjectUrl", ""), - property_contacts=data.get("property_contacts", {}) + property_contacts=data.get("property_contacts", {}), + viewing_emails=data.get("viewing_emails", []) ) \ No newline at end of file diff --git a/sync.py b/sync.py index f448f38..405636d 100644 --- a/sync.py +++ b/sync.py @@ -149,6 +149,13 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine # Extract attorney fees and costs attorney_fees = fees_and_costs.get("totalAttorneysFees") or '' costs = fees_and_costs.get("totalCosts") or '' + from pprint import pprint + property_managers = [property_contacts.get('propertyManager1'), property_contacts.get('propertyManager2'), property_contacts.get('propertyManager3'), property_contacts.get('propertyManager4')] + import itertools + # valid_property_managers = list(itertools.chain(*)) + valid_property_managers = [e.get('address').lower() for pm in property_managers if pm and pm.get('emails') for e in pm.get('emails') if e and e.get('address')] + pprint(valid_property_managers) + row = ProjectModel( client=c.get("firstName", ""), @@ -204,7 +211,8 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine project_id=pid, project_name=p.get("projectName") or detail.get("projectName"), project_url=p.get("projectUrl") or detail.get("projectUrl"), - property_contacts=property_contacts + #property_contacts=property_contacts + viewing_emails = valid_property_managers ) # Store the results in Firestore from app import db # Import db from app @@ -264,7 +272,8 @@ def main(): # List projects (all pages) projects = client.list_all_projects() - projects = projects[:20] + + projects = [p for p in projects if (p.get("projectId") or {}).get("native") == 15914808] # Process projects in parallel detailed_rows = process_projects_parallel(projects, client, 9)