fixes for syncing

app.py

@@ -195,12 +195,37 @@ def require_password_reset():
     return render_template("require_password_reset.html")
 
 
-@app.route("/reset-password-submit", methods=["POST"])
+@app.route("/sync-stats")
 @login_required
-def reset_password_submit():
-    """Handle password reset form submission"""
-    uid = session.get("uid")
-    profile = get_user_profile(uid)
+def sync_stats():
+    """Display sync statistics as a stacked bar chart"""
+    from datetime import datetime, timedelta
+    import pytz
+
+    profile = get_user_profile(session['uid'])
+    if not profile.get("is_admin"):
+        abort(403, "Access denied. Admin privileges required.")
+
+    pacific = pytz.timezone('America/Los_Angeles')
+    today = datetime.now(pacific)
+    dates = [(today - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(13, -1, -1)]
+
+    stats = []
+    for date in dates:
+        doc_id = f"sync_{date}"
+        doc = db.collection("sync_stats").document(doc_id).get()
+        if doc.exists:
+            d = doc.to_dict()
+            stats.append({
+                "date": date,
+                "recent_successes": d.get("recent_successes", 0),
+                "oldest_successes": d.get("oldest_successes", 0),
+                "failures": d.get("failures", 0)
+            })
+        else:
+            stats.append({"date": date, "recent_successes": 0, "oldest_successes": 0, "failures": 0})
+
+    return render_template("sync_stats.html", stats=stats)
 
     new_password = request.form.get("new_password")
     confirm_password = request.form.get("confirm_password")

sync.py

@@ -74,27 +74,88 @@ def convert_to_pacific_time(date_str):
         return ''
 
 
+def parse_local_date(date_str):
+    """Parse a date string that is already in local (Pacific) time as YYYY-MM-DD.
+
+    Filevine date-picker fields store calendar dates as midnight UTC, but they
+    represent the user's local date. This function extracts just the date portion
+    without any timezone conversion.
+
+    Args:
+        date_str (str): ISO 8601 date string (e.g., "2026-05-29T00:00:00Z")
+
+    Returns:
+        str: Date formatted as YYYY-MM-DD, or empty string if input is empty
+    """
+    if not date_str:
+        return ''
+
+    try:
+        dt = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+        return dt.strftime('%Y-%m-%d')
+    except (ValueError, AttributeError) as e:
+        print(f"[WARN] Date parse failed for '{date_str}': {e}")
+        return ''
+
+
 def extract_domains_from_emails(emails: List[str]) -> List[str]:
     """Extract unique domains from a list of email addresses.
-    
+
     Args:
         emails (List[str]): List of email addresses
-        
+
     Returns:
         List[str]: List of unique domains extracted from the emails
     """
     if not emails:
         return []
-    
+
     domains = set()
     for email in emails:
         if email and '@' in email:
-            # Extract domain part after @
             domain = email.split('@')[1].lower()
             domains.add(domain)
-    
+
     return sorted(list(domains))
 
+
+def record_sync_stats(db, recent_successes: int, oldest_successes: int, failures: int):
+    """Record sync statistics for today in Firestore.
+
+    Args:
+        db: Firestore client
+        recent_successes: Number of recently active projects updated
+        oldest_successes: Number of oldest projects updated
+        failures: Number of failed updates
+    """
+    from datetime import datetime as dt
+    pacific = pytz.timezone('America/Los_Angeles')
+    today = dt.now(pacific).strftime('%Y-%m-%d')
+    doc_id = f"sync_{today}"
+
+    try:
+        doc_ref = db.collection("sync_stats").document(doc_id)
+        doc = doc_ref.get()
+        if doc.exists:
+            current = doc.to_dict()
+            doc_ref.update({
+                "recent_successes": current.get("recent_successes", 0) + recent_successes,
+                "oldest_successes": current.get("oldest_successes", 0) + oldest_successes,
+                "failures": current.get("failures", 0) + failures,
+                "updated_at": dt.now(pytz.UTC).isoformat()
+            })
+        else:
+            doc_ref.set({
+                "date": today,
+                "recent_successes": recent_successes,
+                "oldest_successes": oldest_successes,
+                "failures": failures,
+                "created_at": dt.now(pytz.UTC).isoformat()
+            })
+        print(f"[STATS] Recorded sync stats: recent={recent_successes}, oldest={oldest_successes}, failures={failures}")
+    except Exception as e:
+        print(f"[ERROR] Failed to record sync stats: {e}")
+
 from models.project_model import ProjectModel
 from filevine_client import FilevineClient
 
@@ -175,30 +236,30 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
                                    ), '')
 
         # Extract notice service and expiration dates
-        notice_service_date = convert_to_pacific_time(new_file_review.get("noticeServiceDate")) or ''
-        notice_expiration_date = convert_to_pacific_time(new_file_review.get("noticeExpirationDate")) or ''
+        notice_service_date = parse_local_date(new_file_review.get("noticeServiceDate")) or ''
+        notice_expiration_date = parse_local_date(new_file_review.get("noticeExpirationDate")) or ''
 
         # Extract daily rent damages
         daily_rent_damages = lease_info_np.get("dailyRentDamages") or dates_and_deadlines.get("dailyRentDamages") or ''
 
         # Extract default date
-        default_date = convert_to_pacific_time(dates_and_deadlines.get("defaultDate")) or ''
-        case_filed_date = convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled")) or ''
+        default_date = parse_local_date(dates_and_deadlines.get("defaultDate")) or ''
+        case_filed_date = parse_local_date(dates_and_deadlines.get("dateCaseFiled")) or ''
 
         # Extract motion hearing dates
-        demurrer_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("demurrerHearingDate")) or ''
-        motion_to_strike_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTSHearingDate")) or ''
-        motion_to_quash_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("mTQHearingDate")) or ''
-        other_motion_hearing_date = convert_to_pacific_time(dates_and_deadlines.get("otherMotion1HearingDate")) or ''
+        demurrer_hearing_date = parse_local_date(dates_and_deadlines.get("demurrerHearingDate")) or ''
+        motion_to_strike_hearing_date = parse_local_date(dates_and_deadlines.get("mTSHearingDate")) or ''
+        motion_to_quash_hearing_date = parse_local_date(dates_and_deadlines.get("mTQHearingDate")) or ''
+        other_motion_hearing_date = parse_local_date(dates_and_deadlines.get("otherMotion1HearingDate")) or ''
 
         # Extract MSC details
-        msc_date = convert_to_pacific_time(dates_and_deadlines.get("mSCDate")) or ''
+        msc_date = parse_local_date(dates_and_deadlines.get("mSCDate")) or ''
         msc_time = dates_and_deadlines.get("mSCTime") or ''  # Time field, not converting
         msc_address = dates_and_deadlines.get("mSCAddress") or ''
         msc_div_dept_room = dates_and_deadlines.get("mSCDeptDiv") or ''
 
         # Extract trial details
-        trial_date = convert_to_pacific_time(dates_and_deadlines.get("trialDate")) or ''
+        trial_date = parse_local_date(dates_and_deadlines.get("trialDate")) or ''
         trial_time = dates_and_deadlines.get("trialTime") or ''  # Time field, not converting
         trial_address = dates_and_deadlines.get("trialAddress") or ''
         trial_div_dept_room = dates_and_deadlines.get("trialDeptDivRoom") or ''
@@ -207,16 +268,16 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
         final_result = dates_and_deadlines.get("finalResultOfTrialMSCCa") or ''
 
         # Extract settlement details
-        date_of_settlement = convert_to_pacific_time(dates_and_deadlines.get("dateOfStipulation")) or ''
+        date_of_settlement = parse_local_date(dates_and_deadlines.get("dateOfStipulation")) or ''
         final_obligation = dates_and_deadlines.get("finalObligationUnderTheStip") or ''
         def_comply_stip = dates_and_deadlines.get("defendantsComplyWithStip") or ''
 
         # Extract judgment and writ details
-        judgment_date = convert_to_pacific_time(dates_and_deadlines.get("dateOfJudgment")) or ''
-        writ_issued_date = convert_to_pacific_time(dates_and_deadlines.get("writIssuedDate")) or ''
+        judgment_date = parse_local_date(dates_and_deadlines.get("dateOfJudgment")) or ''
+        writ_issued_date = parse_local_date(dates_and_deadlines.get("writIssuedDate")) or ''
 
         # Extract lockout and stay details
-        scheduled_lockout = convert_to_pacific_time(dates_and_deadlines.get("sheriffScheduledDate")) or ''
+        scheduled_lockout = parse_local_date(dates_and_deadlines.get("sheriffScheduledDate")) or ''
         oppose_stays = dates_and_deadlines.get("opposeStays") or ''
 
         # Extract premises safety and entry code
@@ -224,7 +285,7 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
         matter_gate_code = property_info.get("propertyEntryCodeOrInstructions") or ''
 
         # Extract possession recovered date
-        date_possession_recovered = convert_to_pacific_time(dates_and_deadlines.get("datePossessionRecovered")) or ''
+        date_possession_recovered = parse_local_date(dates_and_deadlines.get("datePossessionRecovered")) or ''
 
         # Extract attorney fees and costs
         attorney_fees = fees_and_costs.get("totalAttorneysFees") or ''
@@ -237,7 +298,7 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
             client=c.get("firstName", ""),
             matter_description=p.get("projectName", ""),
             defendant_1=defendant_one.get('fullName', 'Unknown'),
-            matter_open=convert_to_pacific_time(dates_and_deadlines.get("dateCaseFiled") or p.get("createdDate")),
+            matter_open=parse_local_date(dates_and_deadlines.get("dateCaseFiled") or p.get("createdDate")),
             notice_type=new_file_review.get("noticeType", '') or '',
             case_number=dates_and_deadlines.get('caseNumber', '') or '',
             premises_address=property_info.get("premisesAddressWithUnit", "") or '',
@@ -280,11 +341,11 @@ def process_project(index: int, total: int, project_data: dict, client: Filevine
             attorney_fees=attorney_fees,
             costs=costs,
             documents_url=matter_overview.get('documentShareFolderURL', '') or '',
-            service_attempt_date_1=convert_to_pacific_time(next(iter(service_info), {}).get('serviceDate')),
+            service_attempt_date_1=parse_local_date(next(iter(service_info), {}).get('serviceDate')),
             contacts=cs,
             project_email_address=p.get("projectEmailAddress", ""),
             number=p.get("number", "") or matter_overview.get('matterNumber', ''),
-            incident_date=convert_to_pacific_time(p.get("incidentDate") or detail.get("incidentDate")),
+            incident_date=parse_local_date(p.get("incidentDate") or detail.get("incidentDate")),
             project_id=pid,
             project_name=p.get("projectName") or detail.get("projectName"),
             project_url=p.get("projectUrl") or detail.get("projectUrl"),
@@ -383,9 +444,9 @@ def get_oldest_unsynced_projects(db, fraction: float = 0.2) -> List[int]:
 def main():
     """Main function to fetch and sync projects"""
     import argparse
-    
+
     parser = argparse.ArgumentParser(description='Sync Filevine projects to Firestore')
-    parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single'], 
+    parser.add_argument('--mode', choices=['full', 'last_n', 'oldest_percent', 'hybrid', 'single'],
                         default='hybrid', help='Sync mode: full=all projects, last_n=recently active, oldest_percent=oldest by last_synced_at, hybrid=last_n+oldest_percent, single=one project')
     parser.add_argument('--days', type=int, default=14, help='Number of days for last_n mode (default: 14)')
     parser.add_argument('--percent', type=float, default=20.0, help='Percentage for oldest_percent mode (default: 20)')
@@ -401,28 +462,72 @@ def main():
         client.get_bearer_token()
         from app import db
 
+        recent_successes = 0
+        oldest_successes = 0
+        total_failures = 0
+
         if args.mode == 'full':
             print("[MODE] Full sync - fetching all projects")
             projects = client.list_all_projects()
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
         elif args.mode == 'last_n':
             days_ago = (datetime.now() - timedelta(days=args.days)).strftime('%Y-%m-%d')
             print(f"[MODE] Last {args.days} days - fetching active since {days_ago}")
             projects = client.list_all_projects(latest_activity_since=days_ago)
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
         elif args.mode == 'oldest_percent':
             fraction = args.percent / 100.0
             oldest_ids = get_oldest_unsynced_projects(db, fraction=fraction)
             print(f"[MODE] Oldest {args.percent}% - fetching {len(oldest_ids)} projects")
-            
+
             all_projects = client.list_all_projects()
             projects = [p for p in all_projects if p.get("projectId", {}).get("native") in set(oldest_ids)]
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, 0, success_count, fail_count)
+
         elif args.mode == 'single':
             print(f"[MODE] Single project - fetching project {args.project_id}")
             project_detail = client.fetch_project_detail(args.project_id)
             projects = [project_detail] if project_detail else []
-        
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+            success_count = sum(1 for r in detailed_rows if r.get('ProjectId'))
+            fail_count = len(detailed_rows) - success_count
+            record_sync_stats(db, success_count, 0, fail_count)
+
         elif args.mode == 'hybrid':
             print("[MODE] Hybrid - active + oldest")
 
@@ -439,17 +544,22 @@ def main():
 
             all_projects = client.list_all_projects()
             projects = [p for p in all_projects if p.get("projectId", {}).get("native") in all_ids_to_sync]
+            detailed_rows = process_projects_parallel(projects, client, max_workers=10)
 
-        # Process projects in parallel
-        detailed_rows = process_projects_parallel(projects, client, max_workers=10)
+            # Classify successes by source
+            project_ids_synced = {r.get('ProjectId') for r in detailed_rows if r.get('ProjectId')}
+            recent_successes = len([pid for pid in project_ids_synced if pid in active_ids])
+            oldest_successes = len([pid for pid in project_ids_synced if pid in oldest_ids])
 
-        # Batch write all results to Firestore
-        documents = []
-        for row in detailed_rows:
-            if row.get('ProjectId'):
-                row['is_archived'] = (row.get('phase_name') == 'Archived')
-                documents.append((row.get('ProjectId'), row))
-        batch_write_to_firestore(db, "projects", documents)
+            documents = []
+            for row in detailed_rows:
+                if row.get('ProjectId'):
+                    row['is_archived'] = (row.get('phase_name') == 'Archived')
+                    documents.append((row.get('ProjectId'), row))
+            batch_write_to_firestore(db, "projects", documents)
+
+            total_failures = len(detailed_rows) - len(project_ids_synced)
+            record_sync_stats(db, recent_successes, oldest_successes, total_failures)
 
         print(f"[SYNC] Complete - {len(documents)} projects saved to Firestore")
 

templates/base.html

@@ -25,6 +25,7 @@
         {% set profile = get_user_profile(session.uid) %}
         {% if profile.is_admin %}
         <a href="/admin/users" class="text-sm text-slate-600 hover:text-slate-900">Admin Users</a>
+        <a href="/sync-stats" class="text-sm text-slate-600 hover:text-slate-900">Sync Stats</a>
         {% endif %}
         {% endif %}
         <a href="/logout" class="text-sm text-slate-600 hover:text-slate-900">Logout</a>

templates/sync_stats.html

@@ -0,0 +1,110 @@
+{% extends 'base.html' %}
+{% block content %}
+<div class="p-6">
+  <div class="mb-6">
+    <h1 class="text-2xl font-semibold text-slate-800">Sync Statistics</h1>
+    <p class="text-sm text-slate-500 mt-1">Last 14 days of sync activity</p>
+  </div>
+
+  <div class="bg-white rounded-lg shadow p-6">
+    <div class="h-80">
+      <canvas id="syncChart"></canvas>
+    </div>
+    <div class="flex justify-center gap-6 mt-4">
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-emerald-500"></span>
+        <span class="text-sm text-slate-600">Recent Updates</span>
+      </div>
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-sky-500"></span>
+        <span class="text-sm text-slate-600">Oldest Updates</span>
+      </div>
+      <div class="flex items-center gap-2">
+        <span class="w-4 h-4 rounded bg-rose-500"></span>
+        <span class="text-sm text-slate-600">Failures</span>
+      </div>
+    </div>
+  </div>
+
+  <div class="mt-6 bg-white rounded-lg shadow overflow-hidden">
+    <table class="w-full text-sm">
+      <thead class="bg-slate-50 text-slate-600">
+        <tr>
+          <th class="px-4 py-3 text-left font-medium">Date</th>
+          <th class="px-4 py-3 text-right font-medium">Recent Updates</th>
+          <th class="px-4 py-3 text-right font-medium">Oldest Updates</th>
+          <th class="px-4 py-3 text-right font-medium">Failures</th>
+          <th class="px-4 py-3 text-right font-medium">Total</th>
+        </tr>
+      </thead>
+      <tbody class="divide-y divide-slate-100">
+        {% for stat in stats %}
+        <tr class="hover:bg-slate-50">
+          <td class="px-4 py-3 text-slate-800">{{ stat.date }}</td>
+          <td class="px-4 py-3 text-right text-emerald-600">{{ stat.recent_successes }}</td>
+          <td class="px-4 py-3 text-right text-sky-600">{{ stat.oldest_successes }}</td>
+          <td class="px-4 py-3 text-right text-rose-600">{{ stat.failures }}</td>
+          <td class="px-4 py-3 text-right font-medium text-slate-800">
+            {{ stat.recent_successes + stat.oldest_successes + stat.failures }}
+          </td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
+<script>
+  const ctx = document.getElementById('syncChart').getContext('2d');
+  const labels = {{ stats | map(attribute='date') | list | tojson }};
+  const recentSuccesses = {{ stats | map(attribute='recent_successes') | list | tojson }};
+  const oldestSuccesses = {{ stats | map(attribute='oldest_successes') | list | tojson }};
+  const failures = {{ stats | map(attribute='failures') | list | tojson }};
+
+  new Chart(ctx, {
+    type: 'bar',
+    data: {
+      labels: labels,
+      datasets: [
+        {
+          label: 'Recent Updates',
+          data: recentSuccesses,
+          backgroundColor: '#10b981',
+          stack: 'stack0'
+        },
+        {
+          label: 'Oldest Updates',
+          data: oldestSuccesses,
+          backgroundColor: '#0ea5e9',
+          stack: 'stack0'
+        },
+        {
+          label: 'Failures',
+          data: failures,
+          backgroundColor: '#f43f5e',
+          stack: 'stack0'
+        }
+      ]
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: { display: false }
+      },
+      scales: {
+        x: {
+          stacked: true,
+          grid: { display: false }
+        },
+        y: {
+          stacked: true,
+          beginAtZero: true,
+          ticks: { stepSize: 1 }
+        }
+      }
+    }
+  });
+</script>
+{% endblock %}