include archived
This commit is contained in:
48
sync.py
48
sync.py
@@ -441,16 +441,47 @@ def get_oldest_unsynced_projects(db, fraction: float = 0.2) -> List[int]:
|
||||
return []
|
||||
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def dump_cases_to_json(projects: List[dict], client: FilevineClient, output_dir: str = "sample-cases", limit: int = 10) -> None:
|
||||
"""Fetch project details and dump them as individual JSON files instead of Firestore.
|
||||
|
||||
Args:
|
||||
projects: List of project data dictionaries from Filevine
|
||||
client: FilevineClient instance
|
||||
output_dir: Directory to write JSON files to
|
||||
limit: Maximum number of cases to dump
|
||||
"""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
projects = projects[:limit]
|
||||
print(f"[DUMP] Processing {len(projects)} projects to {output_dir}/")
|
||||
|
||||
detailed_rows = process_projects_parallel(projects, client, max_workers=10)
|
||||
written = 0
|
||||
for row in detailed_rows:
|
||||
if row.get('ProjectId'):
|
||||
row['is_archived'] = (row.get('phase_name') == 'Archived')
|
||||
filepath = os.path.join(output_dir, f"{row['ProjectId']}.json")
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(row, f, indent=2, default=str)
|
||||
written += 1
|
||||
print(f"[DUMP] Wrote {filepath}")
|
||||
|
||||
print(f"[DUMP] Complete - {written} cases written to {output_dir}/")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to fetch and sync projects"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Sync Filevine projects to Firestore')
|
||||
parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single'],
|
||||
default='hybrid', help='Sync mode: full=all projects, last_n=recently active, oldest_percent=oldest by last_synced_at, hybrid=last_n+oldest_percent, single=one project')
|
||||
parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single', 'dump_json'],
|
||||
default='hybrid', help='Sync mode: full=all projects, last_n=recently active, oldest_percent=oldest by last_synced_at, hybrid=last_n+oldest_percent, single=one project, dump_json=export to JSON files')
|
||||
parser.add_argument('--days', type=int, default=14, help='Number of days for last_n mode (default: 14)')
|
||||
parser.add_argument('--percent', type=float, default=20.0, help='Percentage for oldest_percent mode (default: 20)')
|
||||
parser.add_argument('--project-id', type=int, help='Project ID for single mode (required when mode=single)')
|
||||
parser.add_argument('--limit', type=int, default=10, help='Number of cases to dump (default: 10)')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.mode == 'single' and not args.project_id:
|
||||
@@ -460,11 +491,14 @@ def main():
|
||||
try:
|
||||
client = FilevineClient()
|
||||
client.get_bearer_token()
|
||||
from app import db
|
||||
|
||||
recent_successes = 0
|
||||
oldest_successes = 0
|
||||
total_failures = 0
|
||||
documents = []
|
||||
|
||||
if args.mode != 'dump_json':
|
||||
from app import db
|
||||
|
||||
if args.mode == 'full':
|
||||
print("[MODE] Full sync - fetching all projects")
|
||||
@@ -561,7 +595,13 @@ def main():
|
||||
total_failures = len(detailed_rows) - len(project_ids_synced)
|
||||
record_sync_stats(db, recent_successes, oldest_successes, total_failures)
|
||||
|
||||
print(f"[SYNC] Complete - {len(documents)} projects saved to Firestore")
|
||||
elif args.mode == 'dump_json':
|
||||
print(f"[MODE] Dump JSON - fetching {args.limit} cases")
|
||||
projects = client.list_all_projects()
|
||||
dump_cases_to_json(projects, client, limit=args.limit)
|
||||
|
||||
if args.mode != 'dump_json':
|
||||
print(f"[SYNC] Complete - {len(documents)} projects saved to Firestore")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during sync: {e}")
|
||||
|
||||
Reference in New Issue
Block a user