feat: adds Sync statistics.

2026-05-13 07:39:10 -07:00
parent 8dd7ae8c95
commit 275820b502
4 changed files with 248 additions and 26 deletions
--- a/app.py
+++ b/app.py
@@ -195,12 +195,37 @@ def require_password_reset():
    return render_template("require_password_reset.html")


-@app.route("/reset-password-submit", methods=["POST"])
+@app.route("/sync-stats")
@login_required
-def reset_password_submit():
-    """Handle password reset form submission"""
-    uid = session.get("uid")
-    profile = get_user_profile(uid)
+def sync_stats():
+    """Display sync statistics as a stacked bar chart"""
+    from datetime import datetime, timedelta
+    import pytz
+
+    profile = get_user_profile(session['uid'])
+    if not profile.get("is_admin"):
+        abort(403, "Access denied. Admin privileges required.")
+
+    pacific = pytz.timezone('America/Los_Angeles')
+    today = datetime.now(pacific)
+    dates = [(today - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(13, -1, -1)]
+
+    stats = []
+    for date in dates:
+        doc_id = f"sync_{date}"
+        doc = db.collection("sync_stats").document(doc_id).get()
+        if doc.exists:
+            d = doc.to_dict()
+            stats.append({
+                "date": date,
+                "recent_successes": d.get("recent_successes", 0),
+                "oldest_successes": d.get("oldest_successes", 0),
+                "failures": d.get("failures", 0)
+            })
+        else:
+            stats.append({"date": date, "recent_successes": 0, "oldest_successes": 0, "failures": 0})
+
+    return render_template("sync_stats.html", stats=stats)

    new_password = request.form.get("new_password")
    confirm_password = request.form.get("confirm_password")
--- a/sync.py
+++ b/sync.py
@@ -76,25 +76,62 @@ def convert_to_pacific_time(date_str):

 def extract_domains_from_emails(emails: List[str]) -> List[str]:
    """Extract unique domains from a list of email addresses.
-    
+
    Args:
        emails (List[str]): List of email addresses
-        
+
    Returns:
        List[str]: List of unique domains extracted from the emails
    """
    if not emails:
        return []
-    
+
    domains = set()
    for email in emails:
        if email and '@' in email:
-            # Extract domain part after @
            domain = email.split('@')[1].lower()
            domains.add(domain)
-    
+
    return sorted(list(domains))

+
+def record_sync_stats(db, recent_successes: int, oldest_successes: int, failures: int):
+    """Record sync statistics for today in Firestore.
+
+    Args:
+        db: Firestore client
+        recent_successes: Number of recently active projects updated
+        oldest_successes: Number of oldest projects updated
+        failures: Number of failed updates
+    """
+    from datetime import datetime as dt
+    pacific = pytz.timezone('America/Los_Angeles')
+    today = dt.now(pacific).strftime('%Y-%m-%d')
+    doc_id = f"sync_{today}"
+
+    try:
+        doc_ref = db.collection("sync_stats").document(doc_id)
+        doc = doc_ref.get()
+        if doc.exists:
+            current = doc.to_dict()
+            doc_ref.update({
+                "recent_successes": current.get("recent_successes", 0) + recent_successes,
+                "oldest_successes": current.get("oldest_successes", 0) + oldest_successes,
+                "failures": current.get("failures", 0) + failures,
+                "updated_at": dt.now(pytz.UTC).isoformat()
+            })
+        else:
+            doc_ref.set({
+                "date": today,
+                "recent_successes": recent_successes,
+                "oldest_successes": oldest_successes,
+                "failures": failures,
+                "created_at": dt.now(pytz.UTC).isoformat()
+            })
+        print(f"[STATS] Recorded sync stats: recent={recent_successes}, oldest={oldest_successes}, failures={failures}")
+    except Exception as e:
+        print(f"[ERROR] Failed to record sync stats: {e}")
+
 from models.project_model import ProjectModel
 from filevine_client import FilevineClient

@@ -383,9 +420,9 @@ def get_oldest_unsynced_projects(db, fraction: float = 0.2) -> List[int]:
 def main():
    """Main function to fetch and sync projects"""
    import argparse
-    
+
    parser = argparse.ArgumentParser(description='Sync Filevine projects to Firestore')
-    parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single'], 
+    parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single'],
                        default='hybrid', help='Sync mode: full=all projects, last_n=recently active, oldest_percent=oldest by last_synced_at, hybrid=last_n+oldest_percent, single=one project')
    parser.add_argument('--days', type=int, default=14, help='Number of days for last_n mode (default: 14)')
    parser.add_argument('--percent', type=float, default=20.0, help='Percentage for oldest_percent mode (default: 20)')
@@ -401,28 +438,72 @@ def main():
        client.get_bearer_token()
        from app import db

+        recent_successes = 0
+        oldest_successes = 0
+        total_failures = 0
+
        if args.mode == 'full':
            print("[MODE] Full sync - fetching all projects")
            projects = client.list_all_projects()
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
        elif args.mode == 'last_n':
            days_ago = (datetime.now() - timedelta(days=args.days)).strftime('%Y-%m-%d')
            print(f"[MODE] Last {args.days} days - fetching active since {days_ago}")
            projects = client.list_all_projects(latest_activity_since=days_ago)
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
        elif args.mode == 'oldest_percent':
            fraction = args.percent / 100.0
            oldest_ids = get_oldest_unsynced_projects(db, fraction=fraction)
            print(f"[MODE] Oldest {args.percent}% - fetching {len(oldest_ids)} projects")
-            
+
            all_projects = client.list_all_projects()
            projects = [p for p in all_projects if p.get("projectId", {}).get("native") in set(oldest_ids)]
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, 0, success_count, fail_count)
+
        elif args.mode == 'single':
            print(f"[MODE] Single project - fetching project {args.project_id}")
            project_detail = client.fetch_project_detail(args.project_id)
            projects = [project_detail] if project_detail else []
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
        elif args.mode == 'hybrid':
            print("[MODE] Hybrid - active + oldest")

@@ -439,17 +520,22 @@ def main():

            all_projects = client.list_all_projects()
            projects = [p for p in all_projects if p.get("projectId", {}).get("native") in all_ids_to_sync]
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)

-        # Process projects in parallel
-        detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            # Classify successes by source
+            project_ids_synced = {r.get('ProjectId') for r in detailed_rows if r.get('ProjectId')}
+            recent_successes = len([pid for pid in project_ids_synced if pid in active_ids])
+            oldest_successes = len([pid for pid in project_ids_synced if pid in oldest_ids])

-        # Batch write all results to Firestore
-        documents = []
-        for row in detailed_rows:
-            if row.get('ProjectId'):
-                row['is_archived'] = (row.get('phase_name') == 'Archived')
-                documents.append((row.get('ProjectId'), row))
-        batch_write_to_firestore(db, "projects", documents)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+
+            total_failures = len(detailed_rows) - len(project_ids_synced)
+            record_sync_stats(db, recent_successes, oldest_successes, total_failures)

        print(f"[SYNC] Complete - {len(documents)} projects saved to Firestore")

--- a/templates/base.html
+++ b/templates/base.html
@@ -25,6 +25,7 @@
        {% set profile = get_user_profile(session.uid) %}
        {% if profile.is_admin %}
        <a href="/admin/users" class="text-sm text-slate-600 hover:text-slate-900">Admin Users</a>
+        <a href="/sync-stats" class="text-sm text-slate-600 hover:text-slate-900">Sync Stats</a>
        {% endif %}
        {% endif %}
        <a href="/logout" class="text-sm text-slate-600 hover:text-slate-900">Logout</a>
--- a/templates/sync_stats.html
+++ b/templates/sync_stats.html
@@ -0,0 +1,110 @@
+{% extends 'base.html' %}
+{% block content %}
+<div class="p-6">
+  <div class="mb-6">
+    <h1 class="text-2xl font-semibold text-slate-800">Sync Statistics</h1>
+    <p class="text-sm text-slate-500 mt-1">Last 14 days of sync activity</p>
+  </div>
+
+  <div class="bg-white rounded-lg shadow p-6">
+    <div class="h-80">
+      <canvas id="syncChart"></canvas>
+    </div>
+    <div class="flex justify-center gap-6 mt-4">
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-emerald-500"></span>
+        <span class="text-sm text-slate-600">Recent Updates</span>
+      </div>
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-sky-500"></span>
+        <span class="text-sm text-slate-600">Oldest Updates</span>
+      </div>
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-rose-500"></span>
+        <span class="text-sm text-slate-600">Failures</span>
+      </div>
+    </div>
+  </div>
+
+  <div class="mt-6 bg-white rounded-lg shadow overflow-hidden">
+    <table class="w-full text-sm">
+      <thead class="bg-slate-50 text-slate-600">
+        <tr>
+          <th class="px-4 py-3 text-left font-medium">Date</th>
+          <th class="px-4 py-3 text-right font-medium">Recent Updates</th>
+          <th class="px-4 py-3 text-right font-medium">Oldest Updates</th>
+          <th class="px-4 py-3 text-right font-medium">Failures</th>
+          <th class="px-4 py-3 text-right font-medium">Total</th>
+        </tr>
+      </thead>
+      <tbody class="divide-y divide-slate-100">
+        {% for stat in stats %}
+        <tr class="hover:bg-slate-50">
+          <td class="px-4 py-3 text-slate-800">{{ stat.date }}</td>
+          <td class="px-4 py-3 text-right text-emerald-600">{{ stat.recent_successes }}</td>
+          <td class="px-4 py-3 text-right text-sky-600">{{ stat.oldest_successes }}</td>
+          <td class="px-4 py-3 text-right text-rose-600">{{ stat.failures }}</td>
+          <td class="px-4 py-3 text-right font-medium text-slate-800">
+            {{ stat.recent_successes + stat.oldest_successes + stat.failures }}
+          </td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
+<script>
+  const ctx = document.getElementById('syncChart').getContext('2d');
+  const labels = {{ stats | map(attribute='date') | list | tojson }};
+  const recentSuccesses = {{ stats | map(attribute='recent_successes') | list | tojson }};
+  const oldestSuccesses = {{ stats | map(attribute='oldest_successes') | list | tojson }};
+  const failures = {{ stats | map(attribute='failures') | list | tojson }};
+
+  new Chart(ctx, {
+    type: 'bar',
+    data: {
+      labels: labels,
+      datasets: [
+        {
+          label: 'Recent Updates',
+          data: recentSuccesses,
+          backgroundColor: '#10b981',
+          stack: 'stack0'
+        },
+        {
+          label: 'Oldest Updates',
+          data: oldestSuccesses,
+          backgroundColor: '#0ea5e9',
+          stack: 'stack0'
+        },
+        {
+          label: 'Failures',
+          data: failures,
+          backgroundColor: '#f43f5e',
+          stack: 'stack0'
+        }
+      ]
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: { display: false }
+      },
+      scales: {
+        x: {
+          stacked: true,
+          grid: { display: false }
+        },
+        y: {
+          stacked: true,
+          beginAtZero: true,
+          ticks: { stepSize: 1 }
+        }
+      }
+    }
+  });
+</script>
+{% endblock %}