feat: Implement comprehensive project data model and synchronization system
- Added ProjectModel class in models/project_model.py to define structure for Filevine project data with proper type hints and conversion methods (to_dict/from_dict) - Implemented get_firestore_document() helper function in app.py for retrieving specific Firestore documents - Enhanced dashboard pagination in app.py with improved error handling and debugging output for property contacts and project IDs - Overhauled sync.py with: * Parallel processing using ThreadPoolExecutor for efficient project synchronization * Comprehensive extraction of project data from Filevine forms (newFileReview, datesAndDeadlines, propertyInfo, etc.) * Improved error handling and logging throughout the sync process * Proper handling of date conversions and field mappings from Filevine to Firestore * Added property contacts email extraction and viewing_emails array population * Added support for filtering projects by specific ProjectId (15914808) for targeted sync - Added proper initialization of Filevine client in worker threads using thread-local storage - Improved handling of optional fields and default values in ProjectModel - Added detailed logging for progress tracking during synchronization This implementation enables reliable synchronization of Filevine project data to Firestore with proper data modeling and error handling, supporting the dashboard's data requirements.
This commit is contained in:
13
sync.py
13
sync.py
@@ -149,6 +149,13 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
|
||||
# Extract attorney fees and costs
|
||||
attorney_fees = fees_and_costs.get("totalAttorneysFees") or ''
|
||||
costs = fees_and_costs.get("totalCosts") or ''
|
||||
from pprint import pprint
|
||||
property_managers = [property_contacts.get('propertyManager1'), property_contacts.get('propertyManager2'), property_contacts.get('propertyManager3'), property_contacts.get('propertyManager4')]
|
||||
import itertools
|
||||
# valid_property_managers = list(itertools.chain(*))
|
||||
valid_property_managers = [e.get('address').lower() for pm in property_managers if pm and pm.get('emails') for e in pm.get('emails') if e and e.get('address')]
|
||||
pprint(valid_property_managers)
|
||||
|
||||
|
||||
row = ProjectModel(
|
||||
client=c.get("firstName", ""),
|
||||
@@ -204,7 +211,8 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
|
||||
project_id=pid,
|
||||
project_name=p.get("projectName") or detail.get("projectName"),
|
||||
project_url=p.get("projectUrl") or detail.get("projectUrl"),
|
||||
property_contacts=property_contacts
|
||||
#property_contacts=property_contacts
|
||||
viewing_emails = valid_property_managers
|
||||
)
|
||||
# Store the results in Firestore
|
||||
from app import db # Import db from app
|
||||
@@ -264,7 +272,8 @@ def main():
|
||||
|
||||
# List projects (all pages)
|
||||
projects = client.list_all_projects()
|
||||
projects = projects[:20]
|
||||
|
||||
projects = [p for p in projects if (p.get("projectId") or {}).get("native") == 15914808]
|
||||
|
||||
# Process projects in parallel
|
||||
detailed_rows = process_projects_parallel(projects, client, 9)
|
||||
|
||||
Reference in New Issue
Block a user