notid/rothbard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

only load 7 days

Browse Source
This commit is contained in:
Bryce
2025-11-10 10:52:57 -08:00
parent e04346a0f2
commit 5524d7308c
4 changed files with 81 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
sync.py
View File

@@ -19,26 +19,26 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
def convert_to_pacific_time(date_str):
"""Convert UTC date string to Pacific Time and format as YYYY-MM-DD.
Args:
date_str (str): UTC date string in ISO 8601 format (e.g., "2025-10-24T19:20:22.377Z")
Returns:
str: Date formatted as YYYY-MM-DD in Pacific Time, or empty string if input is empty
"""
if not date_str:
return ''
try:
# Parse the UTC datetime
utc_time = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
# Set timezone to UTC
utc_time = utc_time.replace(tzinfo=pytz.UTC)
# Convert to Pacific Time
pacific_time = utc_time.astimezone(pytz.timezone('America/Los_Angeles'))
# Format as YYYY-MM-DD
return pacific_time.strftime('%Y-%m-%d')
except (ValueError, AttributeError) as e:
@@ -70,15 +70,15 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
"""
# Set the FilevineClient for this thread
set_filevine_client(client)
p = project_data
pid = (p.get("projectId") or {}).get("native")
print(f"Working on {pid} ({index}/{total})")
client = get_filevine_client()
if pid is None:
return {}
try:
c = client.fetch_client((p.get("clientId") or {}).get("native"))
cs = client.fetch_contacts(pid)
@@ -86,9 +86,9 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
except Exception as e:
print(f"[WARN] Failed to fetch essential data for {pid}: {e}")
return {}
defendant_one = next((c.get('orgContact', {}) for c in cs if "Defendant" in c.get('orgContact', {}).get('personTypes', [])), {})
try:
new_file_review = client.fetch_form(pid, "newFileReview") or {}
dates_and_deadlines = client.fetch_form(pid, "datesAndDeadlines") or {}
@@ -98,7 +98,7 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
fees_and_costs = client.fetch_form(pid, "feesAndCosts") or {}
property_contacts = client.fetch_form(pid, "propertyContacts") or {}
lease_info_np = client.fetch_form(pid, "leaseInfoNP") or {}
tasks_result = client.fetch_project_tasks(pid)
completed_tasks = [{"description": x.get("body"),
"completed": convert_to_pacific_time(x.get("completedDate"))}
@@ -108,73 +108,73 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
"completed": convert_to_pacific_time(x.get("completedDate"))}
for x in tasks_result.get("items", [])
if not x.get("isCompleted")]
team = client.fetch_project_team(pid)
assigned_attorney = next((m.get('fullname')
for m in team
assigned_attorney = next((m.get('fullname')
for m in team
if ('Assigned Attorney' in [r.get('name') for r in m.get('teamOrgRoles')])
), '')
primary_contact = next((m.get('fullname')
for m in team
primary_contact = next((m.get('fullname')
for m in team
if ('Primary' in [r.get('name') for r in m.get('teamOrgRoles')])
), '')
secondary_paralegal = next((m.get('fullname')
for m in team
secondary_paralegal = next((m.get('fullname')
for m in team
if ('Secondary Paralegal' in [r.get('name') for r in m.get('teamOrgRoles')])
), '')
# Extract notice service and expiration dates
notice_service_date = convert_to_pacific_time(new_file_review.get("noticeServiceDate")) or ''
notice_expiration_date = convert_to_pacific_time(new_file_review.get("noticeExpirationDate")) or ''
# Extract daily rent damages
daily_rent_damages = lease_info_np.get("dailyRentDamages") or dates_and_deadlines.get("dailyRentDamages") or ''
# Extract default date
default_date = convert_to_pacific_time(dates_and_deadlines.get("defaultDate")) or ''
case_filed_date = convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled")) or ''
# Extract motion hearing dates
demurrer_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("demurrerHearingDate")) or ''
motion_to_strike_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTSHearingDate")) or ''
motion_to_quash_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTQHearingDate")) or ''
other_motion_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("otherMotion1HearingDate")) or ''
# Extract MSC details
msc_date = convert_to_pacific_time(dates_and_deadlines.get("mSCDate")) or ''
msc_time = dates_and_deadlines.get("mSCTime") or '' # Time field, not converting
msc_address = dates_and_deadlines.get("mSCAddress") or ''
msc_div_dept_room = dates_and_deadlines.get("mSCDeptDiv") or ''
# Extract trial details
trial_date = convert_to_pacific_time(dates_and_deadlines.get("trialDate")) or ''
trial_time = dates_and_deadlines.get("trialTime") or '' # Time field, not converting
trial_address = dates_and_deadlines.get("trialAddress") or ''
trial_div_dept_room = dates_and_deadlines.get("trialDeptDivRoom") or ''
# Extract final result of trial/MSC
final_result = dates_and_deadlines.get("finalResultOfTrialMSCCa") or ''
# Extract settlement details
date_of_settlement = convert_to_pacific_time(dates_and_deadlines.get("dateOfStipulation")) or ''
final_obligation = dates_and_deadlines.get("finalObligationUnderTheStip") or ''
def_comply_stip = dates_and_deadlines.get("defendantsComplyWithStip") or ''
# Extract judgment and writ details
judgment_date = convert_to_pacific_time(dates_and_deadlines.get("dateOfJudgment")) or ''
writ_issued_date = convert_to_pacific_time(dates_and_deadlines.get("writIssuedDate")) or ''
# Extract lockout and stay details
scheduled_lockout = convert_to_pacific_time(dates_and_deadlines.get("sheriffScheduledDate")) or ''
oppose_stays = dates_and_deadlines.get("opposeStays") or ''
# Extract premises safety and entry code
premises_safety = new_file_review.get("lockoutSafetyIssuesOrSpecialCareIssues") or ''
matter_gate_code = property_info.get("propertyEntryCodeOrInstructions") or ''
# Extract possession recovered date
date_possession_recovered = convert_to_pacific_time(dates_and_deadlines.get("datePossessionRecovered")) or ''
# Extract attorney fees and costs
attorney_fees = fees_and_costs.get("totalAttorneysFees") or ''
costs = fees_and_costs.get("totalCosts") or ''
@@ -185,7 +185,7 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
valid_property_managers = [e.get('address').lower() for pm in property_managers if pm and pm.get('emails') for e in pm.get('emails') if e and e.get('address')]
pprint(valid_property_managers)
row = ProjectModel(
client=c.get("firstName", ""),
matter_description=p.get("projectName", ""),
@@ -245,17 +245,19 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
)
# Store the results in Firestore
from app import db # Import db from app
projects_ref = db.collection("projects")
from pprint import pprint
# pprint([p.get("number"), property_info, new_file_review])
# Add new projects
project_id = row.project_id
if project_id:
projects_ref.document(str(project_id)).set(row.to_dict())
print(f"Finished on {pid} ({index}/{total})")
print(f"Finished on {pid} Matter {row.number} ({index}/{total})")
return row.to_dict()
except Exception as e:
print(f"[ERROR] Processing failed for {pid}: {e}")
return {}
@@ -263,12 +265,12 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
def process_projects_parallel(projects: List[dict], client: FilevineClient, max_workers: int = 9) -> List[Dict[str, Any]]:
"""
Process projects in parallel using a worker pool.
Args:
projects: List of project data dictionaries
client: FilevineClient instance
max_workers: Number of concurrent workers (default 9)
Returns:
List of processed project dictionaries
"""
@@ -277,7 +279,7 @@ def process_projects_parallel(projects: List[dict], client: FilevineClient, max_
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(client,)) as executor:
# Submit all tasks to the executor
future_to_project = {executor.submit(process_project, indx, total, project, client): project for indx, project in enumerate(projects)}
# Collect results as they complete
results = []
for future in concurrent.futures.as_completed(future_to_project):
@@ -288,7 +290,7 @@ def process_projects_parallel(projects: List[dict], client: FilevineClient, max_
print(f"[ERROR] Processing failed: {e}")
# Add empty dict or handle error appropriately
results.append({})
return results
def main():
@@ -298,18 +300,21 @@ def main():
# Initialize Filevine client
client = FilevineClient()
bearer = client.get_bearer_token()
# List projects (all pages)
projects = client.list_all_projects()
# List projects (all pages) with filter for projects updated in the last 7 days
from datetime import datetime, timedelta
seven_days_ago = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
projects = client.list_all_projects(latest_activity_since=seven_days_ago)
#projects = [p for p in projects if (p.get("projectId") or {}).get("native") == 15914808]
#projects = projects[:10]
# Process projects in parallel
detailed_rows = process_projects_parallel(projects, client, 9)
print(f"Successfully synced {len(detailed_rows)} projects to Firestore")
except Exception as e:
print(f"Error during sync: {e}")
import traceback
@@ -317,4 +322,4 @@ def main():
sys.exit(1)
if __name__ == "__main__":
main()
main()