notid/rothbard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi

Browse Source
This commit is contained in:
Bryce
2025-11-07 15:06:25 -08:00
parent cfd7020e1b
commit e25ace802e
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@@ -297,7 +297,7 @@ def fetch_all_projects():
import worker_pool
detailed_rows = worker_pool.process_projects_parallel(projects, bearer, 5)
detailed_rows = worker_pool.process_projects_parallel(projects, bearer, 9)
# Store the results in Firestore
projects_ref = db.collection("projects")
# Clear existing projects

9
worker_pool.py
View File

@@ -18,7 +18,7 @@ def worker_init(bearer_token: str):
"""Initialize worker with bearer token"""
set_bearer_token(bearer_token)
def process_project(project_data: dict, bearer_token: str) -> dict:
def process_project(index: int, total: int, project_data: dict, bearer_token: str) -> dict:
"""
Process a single project with all its API calls.
This is the function that will be executed by workers in parallel.
@@ -39,7 +39,7 @@ def process_project(project_data: dict, bearer_token: str) -> dict:
p = project_data
pid = (p.get("projectId") or {}).get("native")
print(f"Working on {pid}")
print(f"Working on {pid} ({index}/{total})")
c = fetch_client(bearer_token, (p.get("clientId") or {}).get("native"))
cs = fetch_contacts(bearer_token, pid)
@@ -197,7 +197,7 @@ def process_project(project_data: dict, bearer_token: str) -> dict:
"ProjectName": p.get("projectName") or detail.get("projectName"),
"ProjectUrl": p.get("projectUrl") or detail.get("projectUrl"),
}
print(f"finished {pid}")
print(f"Finished on {pid} ({index}/{total})")
return row
@@ -214,9 +214,10 @@ def process_projects_parallel(projects: List[dict], bearer_token: str, max_worke
List of processed project dictionaries
"""
# Create a thread pool with specified number of workers
total = len(projects)
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(bearer_token,)) as executor:
# Submit all tasks to the executor
future_to_project = {executor.submit(process_project, project, bearer_token): project for project in projects}
future_to_project = {executor.submit(process_project, indx, total, project, bearer_token): project for indx, project in enumerate(projects)}
# Collect results as they complete
results = []