notid/rothbard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi

Browse Source
This commit is contained in:
Bryce
2025-11-07 15:06:25 -08:00
parent cfd7020e1b
commit e25ace802e
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@@ -297,7 +297,7 @@ def fetch_all_projects():
import worker_pool import worker_pool
detailed_rows = worker_pool.process_projects_parallel(projects, bearer, 5) detailed_rows = worker_pool.process_projects_parallel(projects, bearer, 9)
# Store the results in Firestore # Store the results in Firestore
projects_ref = db.collection("projects") projects_ref = db.collection("projects")
# Clear existing projects # Clear existing projects

9
worker_pool.py
View File

@@ -18,7 +18,7 @@ def worker_init(bearer_token: str):
"""Initialize worker with bearer token""" """Initialize worker with bearer token"""
set_bearer_token(bearer_token) set_bearer_token(bearer_token)
def process_project(project_data: dict, bearer_token: str) -> dict: def process_project(index: int, total: int, project_data: dict, bearer_token: str) -> dict:
""" """
Process a single project with all its API calls. Process a single project with all its API calls.
This is the function that will be executed by workers in parallel. This is the function that will be executed by workers in parallel.
@@ -39,7 +39,7 @@ def process_project(project_data: dict, bearer_token: str) -> dict:
p = project_data p = project_data
pid = (p.get("projectId") or {}).get("native") pid = (p.get("projectId") or {}).get("native")
print(f"Working on {pid}") print(f"Working on {pid} ({index}/{total})")
c = fetch_client(bearer_token, (p.get("clientId") or {}).get("native")) c = fetch_client(bearer_token, (p.get("clientId") or {}).get("native"))
cs = fetch_contacts(bearer_token, pid) cs = fetch_contacts(bearer_token, pid)
@@ -197,7 +197,7 @@ def process_project(project_data: dict, bearer_token: str) -> dict:
"ProjectName": p.get("projectName") or detail.get("projectName"), "ProjectName": p.get("projectName") or detail.get("projectName"),
"ProjectUrl": p.get("projectUrl") or detail.get("projectUrl"), "ProjectUrl": p.get("projectUrl") or detail.get("projectUrl"),
} }
print(f"finished {pid}") print(f"Finished on {pid} ({index}/{total})")
return row return row
@@ -214,9 +214,10 @@ def process_projects_parallel(projects: List[dict], bearer_token: str, max_worke
List of processed project dictionaries List of processed project dictionaries
""" """
# Create a thread pool with specified number of workers # Create a thread pool with specified number of workers
total = len(projects)
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(bearer_token,)) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, initializer=worker_init, initargs=(bearer_token,)) as executor:
# Submit all tasks to the executor # Submit all tasks to the executor
future_to_project = {executor.submit(process_project, project, bearer_token): project for project in projects} future_to_project = {executor.submit(process_project, indx, total, project, bearer_token): project for indx, project in enumerate(projects)}
# Collect results as they complete # Collect results as they complete
results = [] results = []