From 903ffbbf4224b6ba9a87bdd444a28ce80b3971c6 Mon Sep 17 00:00:00 2001 From: Bryce Date: Sun, 9 Nov 2025 19:41:37 -0800 Subject: [PATCH] progress --- app.py | 27 +- .../__pycache__/project_model.cpython-310.pyc | Bin 0 -> 6494 bytes models/project_model.py | 245 ++++++++++++++++ sync.py | 264 +++++++++++++++++- worker_pool.py | 226 --------------- 5 files changed, 510 insertions(+), 252 deletions(-) create mode 100644 models/__pycache__/project_model.cpython-310.pyc create mode 100644 models/project_model.py delete mode 100644 worker_pool.py diff --git a/app.py b/app.py index c5fc565..29ddd8e 100644 --- a/app.py +++ b/app.py @@ -125,7 +125,8 @@ def convert_to_pacific_time(date_str): def fetch_all_projects(): """Fetch all projects for a user and store them in Firestore""" - + # This function is now only used by sync.py + # In production, this should be removed or marked as deprecated print("Fetching projects....") # Initialize Filevine client client = FilevineClient() @@ -138,19 +139,11 @@ def fetch_all_projects(): # Fetch details for each detailed_rows = [] - import worker_pool - detailed_rows = worker_pool.process_projects_parallel(projects, client, 9) - # Store the results in Firestore - projects_ref = db.collection("projects") - - # Add new projects - for row in detailed_rows: - project_id = str(row.get("ProjectId")) - if project_id: - projects_ref.document(project_id).set(row) - - print(f"Stored {len(detailed_rows)} projects in Firestore") - return detailed_rows + # This functionality has been moved to sync.py + # The worker_pool module has been removed + # This function is kept for backward compatibility but should not be used in production + print("[DEPRECATED] fetch_all_projects() is deprecated. Use sync.py instead.") + return [] @app.route("/") def index(): @@ -212,12 +205,6 @@ def welcome(): # --- Filevine API --- # Filevine client is now in filevine_client.py - - - - - - @app.route("/dashboard") @app.route("/dashboard/") @login_required diff --git a/models/__pycache__/project_model.cpython-310.pyc b/models/__pycache__/project_model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21c302e6db0060f84552d67465a4b16f61afb3af GIT binary patch literal 6494 zcmeHLOK{{y8J_t^(;mG zV3!V5WzSSmxpII*3OI4&!iDofcoRTCc!XDYg?C7Rgs*$rUVGUss-P;DFjN2c_t$Dk zt=6Yct@iTyYyv-XvD2v)6Ny(OIr&S0Tt);B1MI|h!cN*$P1b0eYN_p1&Db_->Fsn< z)~Bkh#e(8zzp|WsfwG}*(}#>?CZ%6IdxR~La}1@Ra|hQ zV8q6gP7ub%BTglZQ&;K-aptLp>_fR)NX8kQL4;0CY*l3QlRppSG9vgwGLaC8?IcEz z?BsT;mG1LTwcdZ*{r0)L{`3)2_QSRz3_Fcrh!#W!(Td0-atMydBiaz{hz^k!Xe(NF zXfQFDusiSKU1-yd=t1-%`Vjqy0mL9;2r-NpL5w2C5aS3FF@cywOd+NbGl*Hl9AX|( zKrA2@5le_=L=jO!tRPkqYj^eM4C>Yq8;DKBS;RTS19z@sr)ocFpTCuc1}~hbCHtcN z&@E%TRb=f;ck1m6c4Eh{FWXmc8P6rRb9M^4y&7{Z(DzBj9)31!;`hjD377B4>PUgl zI^mvKu2xMa49s#6coheVfe0N@wS%HMiqdlPLAbY~m4csjaFiRO`B=yY=2)vrzvW$_ z4kq<{<@z-fI&OINKL4x_0U?4bGm`o z5cN1$_d=&4tni>A;#{R12&=wdyCM9zqv4C1gHu>#+s65VIDcGIal(VRM|Z-xQ56=> z>G^eW5c41`@9bC&;RjwlZac}WwK&_@Q>Q4`MBHBSYK^MEcv)dNct#!{eRS%(ns)21 z1;W26yC@&VxKCG#4>ufNxka-xoTlto5bLzO6T2N`8 zL6xMf;8vjSXm~qS1Lr30TqCr6&#T2*bf8>q?xo9WzkIX)XwPol?9{#-?umVIlW@f` zh9eY~w_^n&46C9hmD15{;oYb@yJ{%ob_|fEZp#5K-f6^nci-NXjhlVwc-D79%L#&g z+%z>>oq=@S{VI6Xt32cFhcWjW4G%iPtv?9j_G8fmWq9;K+@a-%Qwp5R7V)6!8}@B`y>xPj@47laUs?N#<=uL5h|uf}~x3X#IEVeHiLR^v>?tB2(Z zcIed)tR-qt#qo$UE%m%puQ>McRc4P`J9f;E%5^E(+!2$f?F50$!vkDJ^E{QaCteY< zOo*%UE%8CTC8A`Muo4jglTivtkpWCaX`m4?U^;36vM2*=iCTe~C<|iFfw?H( znA=UD#G|&xk}Bn+cJQ{S1K1vQ0z0BEU}w|~?23AT-BB;FC+Y+CM*YCPXaLwB4FU(E zA>d#%3>=C^fWy%!a3mT7jz;6avB(6DM-xCZngmWnQ^3h+8aNfr0H>o_;7l|JoQ>vz zb5Q{}A1we2(IRjmS^_Rc%fO|m2waXzjgP4-DMl;crDzqn60HGOqcgy@XdQSa+5oOc zn~hI9as)Q)1l&8h^_fk-6~pYy*^Zx+z$N4*v`J`}&>^8yLYIVY2|W^eCG<(?moOk< zP{NReVF@D=MkS0%7?)s5n2<0jVM@ZZgc%9566Pe#ODITKkgzCWNy4%O%*zsfNy3VR zRS9bn&PZ67upwbnf)tg1PQn8c9+Yri!UYKzB|HQuoQ;h@RCoSADkn$fGYS6+2ItL{ z(wdEY>P{b_EqT~`y*cAmlau06Ut;D9Oo{6ic+2rmmeUC3I-dh`Fn5mN*f$MZ?_1fJw}q6V(yb|J%;+-_tzliPy~Z*qH);ZANJGW^NyM}|YW z1IX|wcMusaOVE7&83I9Y=;^xh66^%bj>W`GV{VoXed= z9lXn(LWX;})5!2IcLo^_=FTF+!`wM!xR^VS3?Fj~$Z#@u0U2KAE+WIt+$CiAnY)Z! zLM|f1&)gC+9L-%phNro!$Z$1xP4)#@mf>&ihQ0pr)C);?+Q#9ft6m*1!BBnwoD_f= zdS=y)Q~U$zma<7PLoq|Mv|?$^m|{$`7R6dL%P5x7tW~jA&9aJRHOncM(~K*|HOniO*Q`yk zw%vs4S-WEGnsq4Fp<8w;)~Q*SVqKbbE7q-9k77NV^(xk@S)XEkn)NHzui1cN1DXvg zHmKQkgzh+-p}jVd;(*_dKunvE+qu9>Nrso8{L6PisbHmTW^VpEz;D>kj! zjAApI%_=sl*_>i?n$0UVuUSE{f@TYfEoio=*rH}jiY;ljtk|+Vo z95&8j;~X~5VdES&&SB#mHqK$=95&8j;~X~5VdES&&SB#mHqK$=ypvwzuyGC>=df`O z8|Scb4jbpNaSj{juyGC>=df`O8|Scb4jbpNaSj{jlk_r&jdR#IhmCXCIERgM*f@uc zbJ#eCjdR#IhmCXCIERgM*f@ucbJ#dPLof1m`V!xuFY-;}_F>;i9Ttx-y|C#kZT)w! zwe`R3{Z8!rA3C^I;Bs~^zwRw7&RSTyk*gUfbIZa4PgQSeu`Db)pgM&`Az!}k{FDSk zf_&@w?*(85t%_y9nh5ti+kZ+{-_e{ZJuK^50P<5cxQ+>WCSfG0g_7^o-?)u4y-kof zLE;376C_TMIC}90LE>0-ew`q3g2V|DZ-d0!AaR1k2@)qroFH+6#0e57NSq*Xg2V|D zCrF$iae~AN5+_KUAaR1k2@)qroFH+6#0e57NSq*Xg2V|DCrF$iae~AN5+_KUAn`Ux zoFH+6#0e57NSq*Xg2c0%+1q^{oi~$ebPKKi(-`2GVO&*}!NVgokRE cd)=uxbD^n+hj613fxKr%Vj#_vnPeCJ3j@d~xc~qF literal 0 HcmV?d00001 diff --git a/models/project_model.py b/models/project_model.py new file mode 100644 index 0000000..78897c0 --- /dev/null +++ b/models/project_model.py @@ -0,0 +1,245 @@ +""" +Shared data model for Project entities used across the application. +This defines the structure of project data that is fetched from Filevine and stored in Firestore. +""" + +from typing import List, Dict, Any, Optional +from datetime import datetime +import pytz + +class ProjectModel: + """ + Data model for a Filevine project with all its associated fields. + This model defines the structure that will be used for Firestore storage + and API responses. + """ + + def __init__(self, + client: str = "", + matter_description: str = "", + defendant_1: str = "", + matter_open: str = "", + notice_type: str = "", + case_number: str = "", + premises_address: str = "", + premises_city: str = "", + responsible_attorney: str = "", + staff_person: str = "", + staff_person_2: str = "", + phase_name: str = "", + completed_tasks: List[Dict[str, Any]] = None, + pending_tasks: List[Dict[str, Any]] = None, + notice_service_date: str = "", + notice_expiration_date: str = "", + case_field_date: str = "", + daily_rent_damages: str = "", + default_date: str = "", + demurrer_hearing_date: str = "", + motion_to_strike_hearing_date: str = "", + motion_to_quash_hearing_date: str = "", + other_motion_hearing_date: str = "", + msc_date: str = "", + msc_time: str = "", + msc_address: str = "", + msc_div_dept_room: str = "", + trial_date: str = "", + trial_time: str = "", + trial_address: str = "", + trial_div_dept_room: str = "", + final_result: str = "", + date_of_settlement: str = "", + final_obligation: str = "", + def_comply_stip: str = "", + judgment_date: str = "", + writ_issued_date: str = "", + scheduled_lockout: str = "", + oppose_stays: str = "", + premises_safety: str = "", + matter_gate_code: str = "", + date_possession_recovered: str = "", + attorney_fees: str = "", + costs: str = "", + documents_url: str = "", + service_attempt_date_1: str = "", + contacts: List[Dict[str, Any]] = None, + project_email_address: str = "", + number: str = "", + incident_date: str = "", + project_id: str = "", + project_name: str = "", + project_url: str = "", + property_contacts: Dict[str, Any] = None): + + self.client = client + self.matter_description = matter_description + self.defendant_1 = defendant_1 + self.matter_open = matter_open + self.notice_type = notice_type + self.case_number = case_number + self.premises_address = premises_address + self.premises_city = premises_city + self.responsible_attorney = responsible_attorney + self.staff_person = staff_person + self.staff_person_2 = staff_person_2 + self.phase_name = phase_name + self.completed_tasks = completed_tasks or [] + self.pending_tasks = pending_tasks or [] + self.notice_service_date = notice_service_date + self.notice_expiration_date = notice_expiration_date + self.case_field_date = case_field_date + self.daily_rent_damages = daily_rent_damages + self.default_date = default_date + self.demurrer_hearing_date = demurrer_hearing_date + self.motion_to_strike_hearing_date = motion_to_strike_hearing_date + self.motion_to_quash_hearing_date = motion_to_quash_hearing_date + self.other_motion_hearing_date = other_motion_hearing_date + self.msc_date = msc_date + self.msc_time = msc_time + self.msc_address = msc_address + self.msc_div_dept_room = msc_div_dept_room + self.trial_date = trial_date + self.trial_time = trial_time + self.trial_address = trial_address + self.trial_div_dept_room = trial_div_dept_room + self.final_result = final_result + self.date_of_settlement = date_of_settlement + self.final_obligation = final_obligation + self.def_comply_stip = def_comply_stip + self.judgment_date = judgment_date + self.writ_issued_date = writ_issued_date + self.scheduled_lockout = scheduled_lockout + self.oppose_stays = oppose_stays + self.premises_safety = premises_safety + self.matter_gate_code = matter_gate_code + self.date_possession_recovered = date_possession_recovered + self.attorney_fees = attorney_fees + self.costs = costs + self.documents_url = documents_url + self.service_attempt_date_1 = service_attempt_date_1 + self.contacts = contacts or [] + self.project_email_address = project_email_address + self.number = number + self.incident_date = incident_date + self.project_id = project_id + self.project_name = project_name + self.project_url = project_url + self.property_contacts = property_contacts or {} + + def to_dict(self) -> Dict[str, Any]: + """Convert the ProjectModel to a dictionary for Firestore storage.""" + return { + "client": self.client, + "matter_description": self.matter_description, + "defendant_1": self.defendant_1, + "matter_open": self.matter_open, + "notice_type": self.notice_type, + "case_number": self.case_number, + "premises_address": self.premises_address, + "premises_city": self.premises_city, + "responsible_attorney": self.responsible_attorney, + "staff_person": self.staff_person, + "staff_person_2": self.staff_person_2, + "phase_name": self.phase_name, + "completed_tasks": self.completed_tasks, + "pending_tasks": self.pending_tasks, + "notice_service_date": self.notice_service_date, + "notice_expiration_date": self.notice_expiration_date, + "case_field_date": self.case_field_date, + "daily_rent_damages": self.daily_rent_damages, + "default_date": self.default_date, + "demurrer_hearing_date": self.demurrer_hearing_date, + "motion_to_strike_hearing_date": self.motion_to_strike_hearing_date, + "motion_to_quash_hearing_date": self.motion_to_quash_hearing_date, + "other_motion_hearing_date": self.other_motion_hearing_date, + "msc_date": self.msc_date, + "msc_time": self.msc_time, + "msc_address": self.msc_address, + "msc_div_dept_room": self.msc_div_dept_room, + "trial_date": self.trial_date, + "trial_time": self.trial_time, + "trial_address": self.trial_address, + "trial_div_dept_room": self.trial_div_dept_room, + "final_result": self.final_result, + "date_of_settlement": self.date_of_settlement, + "final_obligation": self.final_obligation, + "def_comply_stip": self.def_comply_stip, + "judgment_date": self.judgment_date, + "writ_issued_date": self.writ_issued_date, + "scheduled_lockout": self.scheduled_lockout, + "oppose_stays": self.oppose_stays, + "premises_safety": self.premises_safety, + "matter_gate_code": self.matter_gate_code, + "date_possession_recovered": self.date_possession_recovered, + "attorney_fees": self.attorney_fees, + "costs": self.costs, + "documents_url": self.documents_url, + "service_attempt_date_1": self.service_attempt_date_1, + "contacts": self.contacts, + "ProjectEmailAddress": self.project_email_address, + "Number": self.number, + "IncidentDate": self.incident_date, + "ProjectId": self.project_id, + "ProjectName": self.project_name, + "ProjectUrl": self.project_url, + "property_contacts": self.property_contacts + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'ProjectModel': + """Create a ProjectModel instance from a dictionary (e.g., from Firestore).""" + return cls( + client=data.get("client", ""), + matter_description=data.get("matter_description", ""), + defendant_1=data.get("defendant_1", ""), + matter_open=data.get("matter_open", ""), + notice_type=data.get("notice_type", ""), + case_number=data.get("case_number", ""), + premises_address=data.get("premises_address", ""), + premises_city=data.get("premises_city", ""), + responsible_attorney=data.get("responsible_attorney", ""), + staff_person=data.get("staff_person", ""), + staff_person_2=data.get("staff_person_2", ""), + phase_name=data.get("phase_name", ""), + completed_tasks=data.get("completed_tasks", []), + pending_tasks=data.get("pending_tasks", []), + notice_service_date=data.get("notice_service_date", ""), + notice_expiration_date=data.get("notice_expiration_date", ""), + case_field_date=data.get("case_field_date", ""), + daily_rent_damages=data.get("daily_rent_damages", ""), + default_date=data.get("default_date", ""), + demurrer_hearing_date=data.get("demurrer_hearing_date", ""), + motion_to_strike_hearing_date=data.get("motion_to_strike_hearing_date", ""), + motion_to_quash_hearing_date=data.get("motion_to_quash_hearing_date", ""), + other_motion_hearing_date=data.get("other_motion_hearing_date", ""), + msc_date=data.get("msc_date", ""), + msc_time=data.get("msc_time", ""), + msc_address=data.get("msc_address", ""), + msc_div_dept_room=data.get("msc_div_dept_room", ""), + trial_date=data.get("trial_date", ""), + trial_time=data.get("trial_time", ""), + trial_address=data.get("trial_address", ""), + trial_div_dept_room=data.get("trial_div_dept_room", ""), + final_result=data.get("final_result", ""), + date_of_settlement=data.get("date_of_settlement", ""), + final_obligation=data.get("final_obligation", ""), + def_comply_stip=data.get("def_comply_stip", ""), + judgment_date=data.get("judgment_date", ""), + writ_issued_date=data.get("writ_issued_date", ""), + scheduled_lockout=data.get("scheduled_lockout", ""), + oppose_stays=data.get("oppose_stays", ""), + premises_safety=data.get("premises_safety", ""), + matter_gate_code=data.get("matter_gate_code", ""), + date_possession_recovered=data.get("date_possession_recovered", ""), + attorney_fees=data.get("attorney_fees", ""), + costs=data.get("costs", ""), + documents_url=data.get("documents_url", ""), + service_attempt_date_1=data.get("service_attempt_date_1", ""), + contacts=data.get("contacts", []), + project_email_address=data.get("ProjectEmailAddress", ""), + number=data.get("Number", ""), + incident_date=data.get("IncidentDate", ""), + project_id=data.get("ProjectId", ""), + project_name=data.get("ProjectName", ""), + project_url=data.get("ProjectUrl", ""), + property_contacts=data.get("property_contacts", {}) + ) \ No newline at end of file diff --git a/sync.py b/sync.py index 46ef9ef..f448f38 100644 --- a/sync.py +++ b/sync.py @@ -6,24 +6,276 @@ This can be run manually from the command line to update the projects collection import sys import os +import concurrent.futures +import threading +from typing import List, Dict, Any, Optional +from datetime import datetime +import pytz -# Add the current directory to the Python path so we can import app +# Add the current directory to the Python path so we can import app and models sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from app import fetch_all_projects +from app import fetch_all_projects, convert_to_pacific_time +from models.project_model import ProjectModel +from filevine_client import FilevineClient + +# Global thread-local storage for FilevineClient to avoid passing it around +_thread_local = threading.local() + +def get_filevine_client(): + """Get FilevineClient from thread local storage""" + return getattr(_thread_local, 'client', None) + +def set_filevine_client(client): + """Set FilevineClient in thread local storage""" + _thread_local.client = client + +def worker_init(client: FilevineClient): + """Initialize worker with FilevineClient""" + set_filevine_client(client) + +def process_project(index: int, total: int, project_data: dict, client: FilevineClient) -> Dict[str, Any]: + """ + Process a single project with all its API calls. + This is the function that will be executed by workers in parallel. + """ + # Set the FilevineClient for this thread + set_filevine_client(client) + + p = project_data + pid = (p.get("projectId") or {}).get("native") + print(f"Working on {pid} ({index}/{total})") + client = get_filevine_client() + + if pid is None: + return {} + + try: + c = client.fetch_client((p.get("clientId") or {}).get("native")) + cs = client.fetch_contacts(pid) + detail = client.fetch_project_detail(pid) + except Exception as e: + print(f"[WARN] Failed to fetch essential data for {pid}: {e}") + return {} + + defendant_one = next((c.get('orgContact', {}) for c in cs if "Defendant" in c.get('orgContact', {}).get('personTypes', [])), {}) + + try: + new_file_review = client.fetch_form(pid, "newFileReview") or {} + dates_and_deadlines = client.fetch_form(pid, "datesAndDeadlines") or {} + service_info = client.fetch_collection(pid, "serviceInfo") or [] + property_info = client.fetch_form(pid, "propertyInfo") or {} + matter_overview = client.fetch_form(pid, "matterOverview") or {} + fees_and_costs = client.fetch_form(pid, "feesAndCosts") or {} + property_contacts = client.fetch_form(pid, "propertyContacts") or {} + lease_info_np = client.fetch_form(pid, "leaseInfoNP") or {} + + tasks_result = client.fetch_project_tasks(pid) + completed_tasks = [{"description": x.get("body"), + "completed": convert_to_pacific_time(x.get("completedDate"))} + for x in tasks_result.get("items", []) + if x.get("isCompleted")] + pending_tasks = [{"description": x.get("body"), + "completed": convert_to_pacific_time(x.get("completedDate"))} + for x in tasks_result.get("items", []) + if not x.get("isCompleted")] + + team = client.fetch_project_team(pid) + assigned_attorney = next((m.get('fullname') + for m in team + if ('Assigned Attorney' in [r.get('name') for r in m.get('teamOrgRoles')]) + ), '') + primary_contact = next((m.get('fullname') + for m in team + if ('Primary' in [r.get('name') for r in m.get('teamOrgRoles')]) + ), '') + secondary_paralegal = next((m.get('fullname') + for m in team + if ('Secondary Paralegal' in [r.get('name') for r in m.get('teamOrgRoles')]) + ), '') + + # Extract notice service and expiration dates + notice_service_date = convert_to_pacific_time(new_file_review.get("noticeServiceDate")) or '' + notice_expiration_date = convert_to_pacific_time(new_file_review.get("noticeExpirationDate")) or '' + + # Extract daily rent damages + daily_rent_damages = lease_info_np.get("dailyRentDamages") or dates_and_deadlines.get("dailyRentDamages") or '' + + # Extract default date + default_date = convert_to_pacific_time(dates_and_deadlines.get("defaultDate")) or '' + case_filed_date = convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled")) or '' + + # Extract motion hearing dates + demurrer_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("demurrerHearingDate")) or '' + motion_to_strike_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTSHearingDate")) or '' + motion_to_quash_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTQHearingDate")) or '' + other_motion_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("otherMotion1HearingDate")) or '' + + # Extract MSC details + msc_date = convert_to_pacific_time(dates_and_deadlines.get("mSCDate")) or '' + msc_time = dates_and_deadlines.get("mSCTime") or '' # Time field, not converting + msc_address = dates_and_deadlines.get("mSCAddress") or '' + msc_div_dept_room = dates_and_deadlines.get("mSCDeptDiv") or '' + + # Extract trial details + trial_date = convert_to_pacific_time(dates_and_deadlines.get("trialDate")) or '' + trial_time = dates_and_deadlines.get("trialTime") or '' # Time field, not converting + trial_address = dates_and_deadlines.get("trialAddress") or '' + trial_div_dept_room = dates_and_deadlines.get("trialDeptDivRoom") or '' + + # Extract final result of trial/MSC + final_result = dates_and_deadlines.get("finalResultOfTrialMSCCa") or '' + + # Extract settlement details + date_of_settlement = convert_to_pacific_time(dates_and_deadlines.get("dateOfStipulation")) or '' + final_obligation = dates_and_deadlines.get("finalObligationUnderTheStip") or '' + def_comply_stip = dates_and_deadlines.get("defendantsComplyWithStip") or '' + + # Extract judgment and writ details + judgment_date = convert_to_pacific_time(dates_and_deadlines.get("dateOfJudgment")) or '' + writ_issued_date = convert_to_pacific_time(dates_and_deadlines.get("writIssuedDate")) or '' + + # Extract lockout and stay details + scheduled_lockout = convert_to_pacific_time(dates_and_deadlines.get("sheriffScheduledDate")) or '' + oppose_stays = dates_and_deadlines.get("opposeStays") or '' + + # Extract premises safety and entry code + premises_safety = new_file_review.get("lockoutSafetyIssuesOrSpecialCareIssues") or '' + matter_gate_code = property_info.get("propertyEntryCodeOrInstructions") or '' + + # Extract possession recovered date + date_possession_recovered = convert_to_pacific_time(dates_and_deadlines.get("datePossessionRecovered")) or '' + + # Extract attorney fees and costs + attorney_fees = fees_and_costs.get("totalAttorneysFees") or '' + costs = fees_and_costs.get("totalCosts") or '' + + row = ProjectModel( + client=c.get("firstName", ""), + matter_description=p.get("projectName", ""), + defendant_1=defendant_one.get('fullName', 'Unknown'), + matter_open=convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled") or p.get("createdDate")), + notice_type=new_file_review.get("noticeType", '') or '', + case_number=dates_and_deadlines.get('caseNumber', '') or '', + premises_address=property_info.get("premisesAddressWithUnit", "") or '', + premises_city=property_info.get("premisesCity", "") or '', + responsible_attorney=assigned_attorney, + staff_person=primary_contact, + staff_person_2=secondary_paralegal, + phase_name=p.get("phaseName", ""), + completed_tasks=completed_tasks, + pending_tasks=pending_tasks, + notice_service_date=notice_service_date, + notice_expiration_date=notice_expiration_date, + case_field_date=case_filed_date, + daily_rent_damages=daily_rent_damages, + default_date=default_date, + demurrer_hearing_date=demurrer_hearing_date, + motion_to_strike_hearing_date=motion_to_strike_hearing_date, + motion_to_quash_hearing_date=motion_to_quash_hearing_date, + other_motion_hearing_date=other_motion_hearing_date, + msc_date=msc_date, + msc_time=msc_time, + msc_address=msc_address, + msc_div_dept_room=msc_div_dept_room, + trial_date=trial_date, + trial_time=trial_time, + trial_address=trial_address, + trial_div_dept_room=trial_div_dept_room, + final_result=final_result, + date_of_settlement=date_of_settlement, + final_obligation=final_obligation, + def_comply_stip=def_comply_stip, + judgment_date=judgment_date, + writ_issued_date=writ_issued_date, + scheduled_lockout=scheduled_lockout, + oppose_stays=oppose_stays, + premises_safety=premises_safety, + matter_gate_code=matter_gate_code, + date_possession_recovered=date_possession_recovered, + attorney_fees=attorney_fees, + costs=costs, + documents_url=matter_overview.get('documentShareFolderURL', '') or '', + service_attempt_date_1=convert_to_pacific_time(next(iter(service_info), {}).get('serviceDate')), + contacts=cs, + project_email_address=p.get("projectEmailAddress", ""), + number=p.get("number", ""), + incident_date=convert_to_pacific_time(p.get("incidentDate") or detail.get("incidentDate")), + project_id=pid, + project_name=p.get("projectName") or detail.get("projectName"), + project_url=p.get("projectUrl") or detail.get("projectUrl"), + property_contacts=property_contacts + ) + # Store the results in Firestore + from app import db # Import db from app + + projects_ref = db.collection("projects") + + # Add new projects + project_id = row.project_id + if project_id: + projects_ref.document(str(project_id)).set(row.to_dict()) + + print(f"Finished on {pid} ({index}/{total})") + return row.to_dict() + + except Exception as e: + print(f"[ERROR] Processing failed for {pid}: {e}") + return {} + +def process_projects_parallel(projects: List[dict], client: FilevineClient, max_workers: int = 9) -> List[Dict[str, Any]]: + """ + Process projects in parallel using a worker pool. + + Args: + projects: List of project data dictionaries + client: FilevineClient instance + max_workers: Number of concurrent workers (default 9) + + Returns: + List of processed project dictionaries + """ + # Create a thread pool with specified number of workers + total = len(projects) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(client,)) as executor: + # Submit all tasks to the executor + future_to_project = {executor.submit(process_project, indx, total, project, client): project for indx, project in enumerate(projects)} + + # Collect results as they complete + results = [] + for future in concurrent.futures.as_completed(future_to_project): + try: + result = future.result() + results.append(result) + except Exception as e: + print(f"[ERROR] Processing failed: {e}") + # Add empty dict or handle error appropriately + results.append({}) + + return results def main(): """Main function to fetch and sync projects""" print("Starting project sync...") try: - # Fetch all projects and store them in Firestore - projects = fetch_all_projects() - print(f"Successfully synced {len(projects)} projects to Firestore") + # Initialize Filevine client + client = FilevineClient() + bearer = client.get_bearer_token() + + # List projects (all pages) + projects = client.list_all_projects() + projects = projects[:20] + + # Process projects in parallel + detailed_rows = process_projects_parallel(projects, client, 9) + + + print(f"Successfully synced {len(detailed_rows)} projects to Firestore") + except Exception as e: print(f"Error during sync: {e}") import traceback traceback.print_exc() - sys.exit(1) if __name__ == "__main__": diff --git a/worker_pool.py b/worker_pool.py deleted file mode 100644 index a124843..0000000 --- a/worker_pool.py +++ /dev/null @@ -1,226 +0,0 @@ -import concurrent.futures -import threading -from typing import List, Any, Callable, Tuple -import time - -# Global thread-local storage for FilevineClient to avoid passing it around -_thread_local = threading.local() - -def get_filevine_client(): - """Get FilevineClient from thread local storage""" - return getattr(_thread_local, 'client', None) - -def set_filevine_client(client): - """Set FilevineClient in thread local storage""" - _thread_local.client = client - -def worker_init(client: 'FilevineClient'): - """Initialize worker with FilevineClient""" - set_filevine_client(client) - -def process_project(index: int, total: int, project_data: dict, client: 'FilevineClient') -> dict: - """ - Process a single project with all its API calls. - This is the function that will be executed by workers in parallel. - """ - # Set the FilevineClient for this thread - set_filevine_client(client) - - from app import convert_to_pacific_time - - p = project_data - pid = (p.get("projectId") or {}).get("native") - print(f"Working on {pid} ({index}/{total})") - client = get_filevine_client() - c = client.fetch_client((p.get("clientId") or {}).get("native")) - cs = client.fetch_contacts(pid) - - if pid is None: - return {} - - try: - detail = client.fetch_project_detail(pid) - except Exception as e: - print(f"[WARN] detail fetch failed for {pid}: {e}") - detail = {} - - defendant_one = next((c.get('orgContact', {}) for c in cs if "Defendant" in c.get('orgContact', {}).get('personTypes', [])), {}) - - new_file_review = client.fetch_form(pid, "newFileReview") or {} - dates_and_deadlines = client.fetch_form(pid, "datesAndDeadlines") or {} - service_info = client.fetch_collection(pid, "serviceInfo") or [] - property_info = client.fetch_form(pid, "propertyInfo") - matter_overview = client.fetch_form(pid, "matterOverview") - fees_and_costs = client.fetch_form(pid, "feesAndCosts") or {} - property_contacts = client.fetch_form(pid, "propertyContacts") or {} - lease_info_np = client.fetch_form(pid, "leaseInfoNP") or {} - - completed_tasks = [{"description": x.get("body"), - "completed": convert_to_pacific_time(x.get("completedDate"))} - for x in client.fetch_project_tasks(pid).get("items") - if x.get("isCompleted")] - pending_tasks = [{"description": x.get("body"), - "completed": convert_to_pacific_time(x.get("completedDate"))} - for x in client.fetch_project_tasks(pid).get("items") - if not x.get("isCompleted")] - - team = client.fetch_project_team(pid) - assigned_attorney = next((m.get('fullname') - for m in team - if ('Assigned Attorney' in [r.get('name') for r in m.get('teamOrgRoles')]) - ), '') - primary_contact = next((m.get('fullname') - for m in team - if ('Primary' in [r.get('name') for r in m.get('teamOrgRoles')]) - ), '') - secondary_paralegal = next((m.get('fullname') - for m in team - if ('Secondary Paralegal' in [r.get('name') for r in m.get('teamOrgRoles')]) - ), '') - - # Extract notice service and expiration dates - notice_service_date = convert_to_pacific_time(new_file_review.get("noticeServiceDate")) or '' - notice_expiration_date = convert_to_pacific_time(new_file_review.get("noticeExpirationDate")) or '' - - # Extract daily rent damages - daily_rent_damages = lease_info_np.get("dailyRentDamages") or dates_and_deadlines.get("dailyRentDamages") or '' - - # Extract default date - default_date = convert_to_pacific_time(dates_and_deadlines.get("defaultDate")) or '' - case_filed_date = convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled")) or '' - - # Extract motion hearing dates - demurrer_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("demurrerHearingDate")) or '' - motion_to_strike_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTSHearingDate")) or '' - motion_to_quash_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTQHearingDate")) or '' - other_motion_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("otherMotion1HearingDate")) or '' - - # Extract MSC details - msc_date = convert_to_pacific_time(dates_and_deadlines.get("mSCDate")) or '' - msc_time = dates_and_deadlines.get("mSCTime") or '' # Time field, not converting - msc_address = dates_and_deadlines.get("mSCAddress") or '' - msc_div_dept_room = dates_and_deadlines.get("mSCDeptDiv") or '' - - # Extract trial details - trial_date = convert_to_pacific_time(dates_and_deadlines.get("trialDate")) or '' - trial_time = dates_and_deadlines.get("trialTime") or '' # Time field, not converting - trial_address = dates_and_deadlines.get("trialAddress") or '' - trial_div_dept_room = dates_and_deadlines.get("trialDeptDivRoom") or '' - - # Extract final result of trial/MSC - final_result = dates_and_deadlines.get("finalResultOfTrialMSCCa") or '' - - # Extract settlement details - date_of_settlement = convert_to_pacific_time(dates_and_deadlines.get("dateOfStipulation")) or '' - final_obligation = dates_and_deadlines.get("finalObligationUnderTheStip") or '' - def_comply_stip = dates_and_deadlines.get("defendantsComplyWithStip") or '' - - # Extract judgment and writ details - judgment_date = convert_to_pacific_time(dates_and_deadlines.get("dateOfJudgment")) or '' - writ_issued_date = convert_to_pacific_time(dates_and_deadlines.get("writIssuedDate")) or '' - - # Extract lockout and stay details - scheduled_lockout = convert_to_pacific_time(dates_and_deadlines.get("sheriffScheduledDate")) or '' - oppose_stays = dates_and_deadlines.get("opposeStays") or '' - - # Extract premises safety and entry code - premises_safety = new_file_review.get("lockoutSafetyIssuesOrSpecialCareIssues") or '' - matter_gate_code = property_info.get("propertyEntryCodeOrInstructions") or '' - - # Extract possession recovered date - date_possession_recovered = convert_to_pacific_time(dates_and_deadlines.get("datePossessionRecovered")) or '' - - # Extract attorney fees and costs - attorney_fees = fees_and_costs.get("totalAttorneysFees") or '' - costs = fees_and_costs.get("totalCosts") or '' - - row = { - "client": c.get("firstName"), - "matter_description": p.get("projectName"), - "defendant_1": defendant_one.get('fullName', 'Unknown'), - "matter_open": convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled") or p.get("createdDate")), - "notice_type": new_file_review.get("noticeType", '') or '', - "case_number": dates_and_deadlines.get('caseNumber', '') or '', - "premises_address": property_info.get("premisesAddressWithUnit") or '', - "premises_city": property_info.get("premisesCity") or '', - "responsible_attorney": assigned_attorney, - "staff_person": primary_contact, - "staff_person_2": secondary_paralegal, - "phase_name": p.get("phaseName"), - "completed_tasks": completed_tasks, - "pending_tasks": pending_tasks, - "notice_service_date": notice_service_date, - "notice_expiration_date": notice_expiration_date, - "case_field_date": case_filed_date, - "daily_rent_damages": daily_rent_damages, - "default_date": default_date, - "demurrer_hearing_date": demurrer_hearing_date, - "motion_to_strike_hearing_date": motion_to_strike_hearing_date, - "motion_to_quash_hearing_date": motion_to_quash_hearing_date, - "other_motion_hearing_date": other_motion_hearing_date, - "msc_date": msc_date, - "msc_time": msc_time, - "msc_address": msc_address, - "msc_div_dept_room": msc_div_dept_room, - "trial_date": trial_date, - "trial_time": trial_time, - "trial_address": trial_address, - "trial_div_dept_room": trial_div_dept_room, - "final_result": final_result, - "date_of_settlement": date_of_settlement, - "final_obligation": final_obligation, - "def_comply_stip": def_comply_stip, - "judgment_date": judgment_date, - "writ_issued_date": writ_issued_date, - "scheduled_lockout": scheduled_lockout, - "oppose_stays": oppose_stays, - "premises_safety": premises_safety, - "matter_gate_code": matter_gate_code, - "date_possession_recovered": date_possession_recovered, - "attorney_fees": attorney_fees, - "costs": costs, - "documents_url": matter_overview.get('documentShareFolderURL') or '', - "service_attempt_date_1": convert_to_pacific_time(next(iter(service_info), {}).get('serviceDate')), - "contacts": cs, - "ProjectEmailAddress": p.get("projectEmailAddress"), - "Number": p.get("number"), - "IncidentDate": convert_to_pacific_time(p.get("incidentDate") or detail.get("incidentDate")), - "ProjectId": pid, - "ProjectName": p.get("projectName") or detail.get("projectName"), - "ProjectUrl": p.get("projectUrl") or detail.get("projectUrl"), - "property_contacts": property_contacts - } - print(f"Finished on {pid} ({index}/{total})") - - return row - -def process_projects_parallel(projects: List[dict], client: 'FilevineClient', max_workers: int = 9) -> List[dict]: - """ - Process projects in parallel using a worker pool. - - Args: - projects: List of project data dictionaries - client: FilevineClient instance - max_workers: Number of concurrent workers (default 20) - - Returns: - List of processed project dictionaries - """ - # Create a thread pool with specified number of workers - total = len(projects) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(client,)) as executor: - # Submit all tasks to the executor - future_to_project = {executor.submit(process_project, indx, total, project, client): project for indx, project in enumerate(projects)} - - # Collect results as they complete - results = [] - for future in concurrent.futures.as_completed(future_to_project): - try: - result = future.result() - results.append(result) - except Exception as e: - print(f"[ERROR] Processing failed: {e}") - # Add empty dict or handle error appropriately - results.append({}) - - return results