diff --git a/.roo/rules/01-general.md b/.roo/rules/01-general.md index 5260f85..620de8f 100644 --- a/.roo/rules/01-general.md +++ b/.roo/rules/01-general.md @@ -13,7 +13,7 @@ Here are special rules you must follow: 11. Design docs go into docs/design/*.md. These docs are always kept up to date. 12. Before completing work, ensure that no design docs are left out of sync 13. Plans go into docs/plans/*.md. These may not be kept in sync, as they are just for brainstorming. -14. Database migrations are automatically created via `flask db migrate -m 'message'`. NEVER create migrations by hand. +14. ****IMPORTANT**** Database migrations are automatically created via `flask db migrate -m 'message'`. **NEVER** create migrations by hand. You should never have to read the contents of migrations/ 15. In the technical documentation, code should be used sparingly. Also, when needed, focus on the APIs, and only use snippets. diff --git a/app/imap_service.py b/app/imap_service.py index c905199..60b3347 100644 --- a/app/imap_service.py +++ b/app/imap_service.py @@ -251,6 +251,102 @@ class IMAPService: self.connection = None return [] + def get_folder_email_uids(self, folder_name: str) -> List[str]: + """Get the list of email UIDs in a specific folder.""" + try: + # Connect to IMAP server + self._connect() + + # Login + self.connection.login( + self.config.get('username', ''), + self.config.get('password', '') + ) + + # Select the folder + resp_code, content = self.connection.select(folder_name) + if resp_code != 'OK': + return [] + + # Get email UIDs + resp_code, content = self.connection.search(None, 'ALL') + if resp_code != 'OK': + return [] + + # Extract UIDs + email_uids = content[0].split() + uid_list = [uid.decode('utf-8') for uid in email_uids] + + # Close folder and logout + self.connection.close() + self.connection.logout() + self.connection = None + + return uid_list + + except Exception as e: + logging.error(f"Error getting email UIDs for folder {folder_name}: {str(e)}") + if self.connection: + try: + self.connection.logout() + except: + pass + self.connection = None + return [] + + def get_email_headers(self, folder_name: str, email_uid: str) -> Dict[str, str]: + """Get email headers for a specific email UID.""" + try: + # Connect to IMAP server + self._connect() + + # Login + self.connection.login( + self.config.get('username', ''), + self.config.get('password', '') + ) + + # Select the folder + resp_code, content = self.connection.select(folder_name) + if resp_code != 'OK': + return {} + + # Fetch email headers + resp_code, content = self.connection.fetch(email_uid, '(RFC822.HEADER)') + if resp_code != 'OK': + return {} + + # Parse the email headers + raw_email = content[0][1] + import email + msg = email.message_from_bytes(raw_email) + + # Extract headers + headers = { + 'subject': msg.get('Subject', 'No Subject'), + 'date': msg.get('Date', ''), + 'from': msg.get('From', ''), + 'to': msg.get('To', ''), + 'message_id': msg.get('Message-ID', '') + } + + # Close folder and logout + self.connection.close() + self.connection.logout() + self.connection = None + + return headers + + except Exception as e: + logging.error(f"Error getting email headers for UID {email_uid} in folder {folder_name}: {str(e)}") + if self.connection: + try: + self.connection.logout() + except: + pass + self.connection = None + return {} + def sync_folders(self) -> Tuple[bool, str]: """Sync IMAP folders with local database.""" try: diff --git a/app/models.py b/app/models.py index bd8f6b1..5a61f46 100644 --- a/app/models.py +++ b/app/models.py @@ -40,8 +40,29 @@ class Folder(Base): rule_text = db.Column(db.Text) priority = db.Column(db.Integer) organize_enabled = db.Column(db.Boolean, default=True) + folder_type = db.Column(db.String(20), default='destination', nullable=False) total_count = db.Column(db.Integer, default=0) pending_count = db.Column(db.Integer, default=0) + emails_count = db.Column(db.Integer, default=0) recent_emails = db.Column(db.JSON, default=list) # Store recent email subjects with dates - user = db.relationship('User', backref=db.backref('folders', lazy=True)) \ No newline at end of file + user = db.relationship('User', backref=db.backref('folders', lazy=True)) + +class ProcessedEmail(Base): + __tablename__ = 'processed_emails' + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False) + folder_id = db.Column(db.Integer, db.ForeignKey('folders.id'), nullable=False) + email_uid = db.Column(db.String(255), nullable=False) + folder_name = db.Column(db.String(255), nullable=False) + is_processed = db.Column(db.Boolean, default=False) + first_seen_at = db.Column(db.DateTime, default=datetime.utcnow) + processed_at = db.Column(db.DateTime, nullable=True) + created_at = db.Column(db.DateTime, default=datetime.utcnow) + updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + user = db.relationship('User', backref=db.backref('processed_emails', lazy=True)) + folder = db.relationship('Folder', backref=db.backref('processed_emails', lazy=True)) + + def __repr__(self): + return f'' \ No newline at end of file diff --git a/app/processed_emails_service.py b/app/processed_emails_service.py new file mode 100644 index 0000000..f604d22 --- /dev/null +++ b/app/processed_emails_service.py @@ -0,0 +1,235 @@ +from typing import List, Dict, Optional +from datetime import datetime +from app.models import db, ProcessedEmail, User, Folder +from sqlalchemy import and_, or_ + + +class ProcessedEmailsService: + def __init__(self, user: User): + self.user = user + + def get_pending_emails(self, folder_name: str) -> List[str]: + """Get list of email UIDs that are pending processing in a folder.""" + try: + pending_emails = ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + is_processed=False + ).all() + + return [email.email_uid for email in pending_emails] + except Exception as e: + print(f"Error getting pending emails: {str(e)}") + return [] + + def mark_email_processed(self, folder_name: str, email_uid: str) -> bool: + """Mark an email as processed.""" + try: + processed_email = ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + email_uid=email_uid + ).first() + + if processed_email: + processed_email.is_processed = True + processed_email.processed_at = datetime.utcnow() + processed_email.updated_at = datetime.utcnow() + db.session.commit() + return True + else: + # Get the folder object to set the folder_id + folder = Folder.query.filter_by( + user_id=self.user.id, + name=folder_name + ).first() + + if folder: + # Create a new record if it doesn't exist + new_record = ProcessedEmail( + user_id=self.user.id, + folder_id=folder.id, + folder_name=folder_name, + email_uid=email_uid, + is_processed=True, + processed_at=datetime.utcnow() + ) + db.session.add(new_record) + db.session.commit() + return True + else: + return False + + except Exception as e: + print(f"Error marking email as processed: {str(e)}") + db.session.rollback() + return False + + def mark_emails_processed(self, folder_name: str, email_uids: List[str]) -> int: + """Mark multiple emails as processed in bulk.""" + try: + updated_count = 0 + + # Update existing records + ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + is_processed=False + ).filter(ProcessedEmail.email_uid.in_(email_uids)).update( + { + 'is_processed': True, + 'processed_at': datetime.utcnow(), + 'updated_at': datetime.utcnow() + }, + synchronize_session=False + ) + + # Check for any email UIDs that don't have records yet + existing_uids = {email.email_uid for email in ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name + ).all()} + + missing_uids = set(email_uids) - existing_uids + if missing_uids: + # Get the folder object to set the folder_id + folder = Folder.query.filter_by( + user_id=self.user.id, + name=folder_name + ).first() + + if folder: + # Create new records for missing UIDs + new_records = [ + ProcessedEmail( + user_id=self.user.id, + folder_id=folder.id, + folder_name=folder_name, + email_uid=uid, + is_processed=True, + processed_at=datetime.utcnow() + ) + for uid in missing_uids + ] + db.session.bulk_save_objects(new_records) + + updated_count = len(email_uids) + db.session.commit() + return updated_count + + except Exception as e: + print(f"Error marking emails as processed: {str(e)}") + db.session.rollback() + return 0 + + def sync_folder_emails(self, folder_name: str, email_uids: List[str]) -> int: + """Sync email UIDs for a folder, adding new ones as pending.""" + try: + # Get existing UIDs for this folder + existing_uids = {email.email_uid for email in ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name + ).all()} + + # Find new UIDs that don't exist yet + new_uids = set(email_uids) - existing_uids + + # Create new records for new UIDs + if new_uids: + # Get the folder object to set the folder_id + folder = Folder.query.filter_by( + user_id=self.user.id, + name=folder_name + ).first() + + if folder: + new_records = [ + ProcessedEmail( + user_id=self.user.id, + folder_id=folder.id, + folder_name=folder_name, + email_uid=uid, + is_processed=False + ) + for uid in new_uids + ] + db.session.bulk_save_objects(new_records) + + # Get the folder to update counts + folder = Folder.query.filter_by( + user_id=self.user.id, + name=folder_name + ).first() + + if folder: + # Update pending count + pending_count = ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + is_processed=False + ).count() + folder.pending_count = pending_count + folder.total_count = len(email_uids) + db.session.commit() + + return len(new_uids) + + except Exception as e: + print(f"Error syncing folder emails: {str(e)}") + db.session.rollback() + return 0 + + def get_pending_count(self, folder_name: str) -> int: + """Get count of pending emails for a folder.""" + try: + count = ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + is_processed=False + ).count() + return count + except Exception as e: + print(f"Error getting pending count: {str(e)}") + return 0 + + def cleanup_old_records(self, folder_name: str, current_uids: List[str]) -> int: + """Remove records for emails that no longer exist in the folder.""" + try: + # Find records that no longer exist in the current UIDs + records_to_delete = ProcessedEmail.query.filter( + and_( + ProcessedEmail.user_id == self.user.id, + ProcessedEmail.folder_name == folder_name, + ~ProcessedEmail.email_uid.in_(current_uids) + ) + ).all() + + # Delete the records + for record in records_to_delete: + db.session.delete(record) + + deleted_count = len(records_to_delete) + db.session.commit() + + # Update folder counts after cleanup + folder = Folder.query.filter_by( + user_id=self.user.id, + name=folder_name + ).first() + + if folder: + pending_count = ProcessedEmail.query.filter_by( + user_id=self.user.id, + folder_name=folder_name, + is_processed=False + ).count() + folder.pending_count = pending_count + folder.total_count = len(current_uids) + db.session.commit() + + return deleted_count + + except Exception as e: + print(f"Error cleaning up old records: {str(e)}") + db.session.rollback() + return 0 \ No newline at end of file diff --git a/app/routes.py b/app/routes.py index 9f827c9..7a7750c 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,8 +1,9 @@ from flask import Blueprint, render_template, request, jsonify, make_response, flash, redirect, url_for from flask_login import login_required, current_user from app import db -from app.models import Folder, User +from app.models import Folder, User, ProcessedEmail from app.imap_service import IMAPService +from app.processed_emails_service import ProcessedEmailsService import uuid import logging @@ -13,7 +14,15 @@ main = Blueprint('main', __name__) def index(): # Get folders for the current authenticated user folders = Folder.query.filter_by(user_id=current_user.id).all() - return render_template('index.html', folders=folders) + + # Separate folders by type + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + return render_template('index.html', + folders=folders, + tidy_folders=tidy_folders, + destination_folders=destination_folders) @main.route('/api/folders/new', methods=['GET']) @login_required @@ -56,11 +65,15 @@ def add_folder(): return response # Create new folder for the current user + # Default to 'destination' type for manually created folders + folder_type = 'tidy' if name.strip().lower() == 'inbox' else 'destination' + folder = Folder( user_id=current_user.id, name=name.strip(), rule_text=rule_text.strip(), - priority=int(priority) if priority else 0 + priority=int(priority) if priority else 0, + folder_type=folder_type ) db.session.add(folder) @@ -69,8 +82,15 @@ def add_folder(): # Get updated list of folders for the current user folders = Folder.query.filter_by(user_id=current_user.id).all() - # Return the updated folders list HTML - response = make_response(render_template('partials/folders_list.html', folders=folders)) + # Get updated lists of folders by type + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + + response = make_response(tidy_section + destination_section) response.headers['HX-Trigger'] = 'close-modal' response.status_code = 201 return response @@ -81,6 +101,15 @@ def add_folder(): db.session.rollback() # Return error in modal errors = {'general': 'An unexpected error occurred. Please try again.'} + # Get updated lists of folders by type for error fallback + folders = Folder.query.filter_by(user_id=current_user.id).all() + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections as fallback + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + response = make_response(render_template('partials/folder_modal.html', errors=errors, name=name, rule_text=rule_text, priority=priority)) response.headers['HX-Retarget'] = '#folder-modal' response.headers['HX-Reswap'] = 'outerHTML' @@ -98,6 +127,9 @@ def delete_folder(folder_id): folders = Folder.query.filter_by(user_id=current_user.id).all() return render_template('partials/folders_list.html', folders=folders) + # Delete all associated processed emails first + ProcessedEmail.query.filter_by(folder_id=folder.id).delete() + # Delete the folder db.session.delete(folder) db.session.commit() @@ -105,8 +137,15 @@ def delete_folder(folder_id): # Get updated list of folders for the current user folders = Folder.query.filter_by(user_id=current_user.id).all() - # Return the updated folders list HTML - return render_template('partials/folders_list.html', folders=folders) + # Get updated lists of folders by type + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + + return tidy_section + destination_section except Exception as e: # Print unhandled exceptions to the console as required @@ -114,7 +153,14 @@ def delete_folder(folder_id): db.session.rollback() # Return the folders list unchanged folders = Folder.query.filter_by(user_id=current_user.id).all() - return render_template('partials/folders_list.html', folders=folders) + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + + return tidy_section + destination_section @main.route('/api/folders//toggle', methods=['PUT']) @login_required @@ -132,7 +178,11 @@ def toggle_folder_organize(folder_id): db.session.commit() # Return just the updated folder card HTML for this specific folder - return render_template('partials/folder_card.html', folder=folder) + # Use the appropriate template based on folder type + if folder.folder_type == 'tidy': + return render_template('partials/folder_card_tidy.html', folder=folder) + else: + return render_template('partials/folder_card_destination.html', folder=folder) except Exception as e: # Print unhandled exceptions to the console as required @@ -195,6 +245,15 @@ def update_folder(folder_id): # If there are validation errors, return the modal with errors if errors: + # Get updated lists of folders by type for error fallback + folders = Folder.query.filter_by(user_id=current_user.id).all() + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections as fallback + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + response = make_response(render_template('partials/folder_modal.html', folder=folder, errors=errors, name=name, rule_text=rule_text, priority=priority)) response.headers['HX-Retarget'] = '#folder-modal' response.headers['HX-Reswap'] = 'outerHTML' @@ -205,12 +264,26 @@ def update_folder(folder_id): folder.rule_text = rule_text.strip() folder.priority = int(priority) if priority else 0 + # Update folder type if the name changed to/from 'inbox' + if name.strip().lower() == 'inbox' and folder.folder_type != 'tidy': + folder.folder_type = 'tidy' + elif name.strip().lower() != 'inbox' and folder.folder_type != 'destination': + folder.folder_type = 'destination' + db.session.commit() # Get updated list of folders for the current user folders = Folder.query.filter_by(user_id=current_user.id).all() - response = make_response(render_template('partials/folders_list.html', folders=folders)) + # Get updated lists of folders by type + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + + response = make_response(tidy_section + destination_section) response.headers['HX-Trigger'] = 'close-modal' return response @@ -317,25 +390,104 @@ def test_imap_connection(): @main.route('/api/imap/sync', methods=['POST']) @login_required def sync_imap_folders(): - """Sync folders from IMAP server.""" + """Sync folders from IMAP server with processed email tracking.""" try: if not current_user.imap_config: return jsonify({'error': 'No IMAP configuration found. Please configure IMAP first.'}), 400 # Test connection first imap_service = IMAPService(current_user) - success, message = imap_service.sync_folders() - if success: - # Get updated list of folders - folders = Folder.query.filter_by(user_id=current_user.id).all() - return render_template('partials/folders_list.html', folders=folders) - else: - return jsonify({'error': message}), 400 + # Get folders from IMAP server + imap_folders = imap_service.get_folders() + + if not imap_folders: + return jsonify({'error': 'No folders found on IMAP server'}), 400 + + # Deduplicate folders by name to prevent creating multiple entries for the same folder + unique_folders = [] + seen_names = set() + for imap_folder in imap_folders: + folder_name = imap_folder['name'] + + # Skip special folders that might not be needed + if folder_name.lower() in ['sent', 'drafts', 'spam', 'trash']: + continue + + # Use case-insensitive comparison for deduplication + folder_name_lower = folder_name.lower() + if folder_name_lower not in seen_names: + unique_folders.append(imap_folder) + seen_names.add(folder_name_lower) + + # Process each unique folder + synced_count = 0 + print("HELLOOO") + processed_emails_service = ProcessedEmailsService(current_user) + + for imap_folder in unique_folders: + folder_name = imap_folder['name'].strip() + + # Handle nested folder names (convert slashes to underscores or keep as-is) + # According to requirements, nested folders should be created with slashes in the name + display_name = folder_name + + # Check if folder already exists + existing_folder = Folder.query.filter_by( + user_id=current_user.id, + name=display_name + ).first() + + if not existing_folder: + # Create new folder + # Determine folder type - inbox should be 'tidy', others 'destination' + folder_type = 'tidy' if folder_name.lower().strip() == 'inbox' else 'destination' + + new_folder = Folder( + user_id=current_user.id, + name=display_name, + rule_text=f"Auto-synced from IMAP folder: {folder_name}", + priority=0, # Default priority + folder_type=folder_type + ) + db.session.add(new_folder) + synced_count += 1 + else: + # Update existing folder with email counts and recent emails + # Get all email UIDs in this folder + email_uids = imap_service.get_folder_email_uids(folder_name) + + # Sync with processed emails service + new_emails_count = processed_emails_service.sync_folder_emails(display_name, email_uids) + print("NEW", new_emails_count) + + # Update counts + pending_count = processed_emails_service.get_pending_count(display_name) + existing_folder.pending_count = pending_count + existing_folder.total_count = len(email_uids) + + # Get the most recent emails for this folder + recent_emails = imap_service.get_recent_emails(folder_name, 3) + existing_folder.recent_emails = recent_emails + + db.session.commit() + + # Get updated list of folders + folders = Folder.query.filter_by(user_id=current_user.id).all() + tidy_folders = [folder for folder in folders if folder.folder_type == 'tidy'] + destination_folders = [folder for folder in folders if folder.folder_type == 'destination'] + + # Return both sections + tidy_section = render_template('partials/folders_to_tidy_section.html', tidy_folders=tidy_folders) + destination_section = render_template('partials/destination_folders_section.html', destination_folders=destination_folders) + + return tidy_section + destination_section except Exception as e: logging.exception("Error syncing IMAP folders: %s", e) print(e) + db.session.rollback() + return jsonify({'error': 'An unexpected error occurred'}), 500 @main.route('/api/folders', methods=['GET']) @login_required @@ -343,9 +495,13 @@ def get_folders(): """Get folders with optional filtering.""" # Get filter parameter from query string filter_type = request.args.get('filter', 'all') + folder_type = request.args.get('type', None) # Get folders for the current authenticated user - if filter_type == 'high': + if folder_type: + # Filter by folder type + folders = Folder.query.filter_by(user_id=current_user.id, folder_type=folder_type).all() + elif filter_type == 'high': # Get high priority folders (priority = 1) folders = Folder.query.filter_by(user_id=current_user.id, priority=1).all() elif filter_type == 'normal': @@ -355,4 +511,205 @@ def get_folders(): # Get all folders folders = Folder.query.filter_by(user_id=current_user.id).all() - return render_template('partials/folders_list.html', folders=folders) + # Check if we need to return a specific section + if folder_type == 'tidy': + return render_template('partials/folders_to_tidy_section.html', tidy_folders=folders) + elif folder_type == 'destination': + return render_template('partials/destination_folders_section.html', destination_folders=folders) + else: + return render_template('partials/folders_list.html', folders=folders) + + +# Processed Emails API Endpoints + +@main.route('/api/folders//pending-emails', methods=['GET']) +@login_required +def get_pending_emails(folder_id): + """Get pending emails for a folder with email metadata.""" + try: + # Find the folder and ensure it belongs to the current user + folder = Folder.query.filter_by(id=folder_id, user_id=current_user.id).first() + + if not folder: + return jsonify({'error': 'Folder not found'}), 404 + + # Get IMAP service to fetch email data + imap_service = IMAPService(current_user) + + # Get all email UIDs in the folder + all_uids = imap_service.get_folder_email_uids(folder.name) + + # Get processed emails service + processed_emails_service = ProcessedEmailsService(current_user) + + # Get pending email UIDs + pending_uids = processed_emails_service.get_pending_emails(folder.name) + + # Get email headers for pending emails + pending_emails = [] + for uid in pending_uids: + headers = imap_service.get_email_headers(folder.name, uid) + if headers: + # Add UID to the response + email_data = { + 'uid': uid, + 'subject': headers.get('subject', 'No Subject'), + 'date': headers.get('date', ''), + 'from': headers.get('from', ''), + 'to': headers.get('to', ''), + 'message_id': headers.get('message_id', '') + } + pending_emails.append(email_data) + + # Return the pending emails in a dialog format + response = make_response(render_template('partials/pending_emails_dialog.html', + folder=folder, + pending_emails=pending_emails)) + response.headers['HX-Trigger'] = 'open-modal' + return response + + except Exception as e: + logging.exception("Error getting pending emails: %s", e) + return jsonify({'error': 'An unexpected error occurred'}), 500 + +@main.route('/api/folders//emails//process', methods=['POST']) +@login_required +def mark_email_processed(folder_id, email_uid): + """Mark a specific email as processed.""" + try: + # Find the folder and ensure it belongs to the current user + folder = Folder.query.filter_by(id=folder_id, user_id=current_user.id).first() + + if not folder: + return jsonify({'error': 'Folder not found'}), 404 + + # Get processed emails service + processed_emails_service = ProcessedEmailsService(current_user) + + # Mark email as processed + success = processed_emails_service.mark_email_processed(folder.name, email_uid) + + if success: + # Get updated counts + pending_count = processed_emails_service.get_pending_count(folder.name) + + # Update the folder's count based on folder type + if folder.folder_type == 'tidy': + folder.pending_count = pending_count + elif folder.folder_type == 'destination': + # For destination folders, update emails_count + folder.emails_count = pending_count + + db.session.commit() + + # Return updated dialog body + response = make_response(render_template('partials/pending_emails_updated.html', + folder=folder, + uid=email_uid)) + response.headers['HX-Trigger'] = 'close-modal' + return response + else: + return jsonify({'error': 'Failed to mark email as processed'}), 500 + + except Exception as e: + logging.exception("Error marking email as processed: %s", e) + return jsonify({'error': 'An unexpected error occurred'}), 500 + +@main.route('/api/folders//sync-emails', methods=['POST']) +@login_required +def sync_folder_emails(folder_id): + """Sync emails for a specific folder with processed email tracking.""" + try: + # Find the folder and ensure it belongs to the current user + folder = Folder.query.filter_by(id=folder_id, user_id=current_user.id).first() + + if not folder: + return jsonify({'error': 'Folder not found'}), 404 + + # Get IMAP service to fetch email UIDs + imap_service = IMAPService(current_user) + + # Get all email UIDs in the folder + email_uids = imap_service.get_folder_email_uids(folder.name) + + if not email_uids: + return jsonify({'error': 'No emails found in folder'}), 404 + + # Get processed emails service + processed_emails_service = ProcessedEmailsService(current_user) + + # Sync emails with the processed emails service + new_emails_count = processed_emails_service.sync_folder_emails(folder.name, email_uids) + + # Get updated counts + pending_count = processed_emails_service.get_pending_count(folder.name) + + # Update the folder's counts based on folder type + if folder.folder_type == 'tidy': + folder.pending_count = pending_count + folder.total_count = len(email_uids) + elif folder.folder_type == 'destination': + # For destination folders, update emails_count + folder.emails_count = pending_count # Using pending_count as emails_count for destination folders + + db.session.commit() + + # Return success response with updated counts + return jsonify({ + 'success': True, + 'message': f'Synced {new_emails_count} new emails', + 'pending_count': pending_count, + 'total_count': len(email_uids) + }) + + except Exception as e: + logging.exception("Error syncing folder emails: %s", e) + return jsonify({'error': 'An unexpected error occurred'}), 500 + +@main.route('/api/folders//process-emails', methods=['POST']) +@login_required +def process_folder_emails(folder_id): + """Process multiple emails in a folder (mark as processed).""" + try: + # Find the folder and ensure it belongs to the current user + folder = Folder.query.filter_by(id=folder_id, user_id=current_user.id).first() + + if not folder: + return jsonify({'error': 'Folder not found'}), 404 + + # Get email UIDs from form data + email_uids = request.form.getlist('email_uids') + + if not email_uids: + return jsonify({'error': 'No email UIDs provided'}), 400 + + # Get processed emails service + processed_emails_service = ProcessedEmailsService(current_user) + + # Mark emails as processed + processed_count = processed_emails_service.mark_emails_processed(folder.name, email_uids) + + if processed_count > 0: + # Get updated counts + pending_count = processed_emails_service.get_pending_count(folder.name) + + # Update the folder's count based on folder type + if folder.folder_type == 'tidy': + folder.pending_count = pending_count + elif folder.folder_type == 'destination': + # For destination folders, update emails_count + folder.emails_count = pending_count + + db.session.commit() + + return jsonify({ + 'success': True, + 'message': f'Processed {processed_count} emails', + 'pending_count': pending_count + }) + else: + return jsonify({'error': 'No emails were processed'}), 400 + + except Exception as e: + logging.exception("Error processing folder emails: %s", e) + return jsonify({'error': 'An unexpected error occurred'}), 500 diff --git a/app/templates/index.html b/app/templates/index.html index f4d8690..ae06c67 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -74,18 +74,24 @@ -
+
{{ folders|length }}
Total Folders
-
0
-
Emails Processed
+
{{ tidy_folders|length }}
+
Folders to Tidy
-
0
-
Active Rules
+
{{ destination_folders|length }}
+
Destination Folders
+
+
+
+ {{ tidy_folders|sum(attribute='pending_count') }} +
+
Pending Emails
@@ -103,7 +109,8 @@
- {% include 'partials/folders_list.html' %} + {% include 'partials/folders_to_tidy_section.html' %} + {% include 'partials/destination_folders_section.html' %}
diff --git a/app/templates/partials/destination_folders_section.html b/app/templates/partials/destination_folders_section.html new file mode 100644 index 0000000..a48da3d --- /dev/null +++ b/app/templates/partials/destination_folders_section.html @@ -0,0 +1,35 @@ +
+
+
+
+

Destination Folders

+

Folders where emails are organized and stored

+
+ +
+
+ +
+ {% for folder in destination_folders %} + {% include 'partials/folder_card_destination.html' %} + {% else %} +
+
+ +
+

No destination folders yet

+

Create destination folders to organize your emails into categories.

+
+ +
+
+ {% endfor %} +
+
\ No newline at end of file diff --git a/app/templates/partials/folder_card.html b/app/templates/partials/folder_card.html index fce45a9..d9310e5 100644 --- a/app/templates/partials/folder_card.html +++ b/app/templates/partials/folder_card.html @@ -31,7 +31,20 @@
{{ folder.total_count }} emails + {% if folder.pending_count > 0 %} + + {% else %} {{ folder.pending_count }} pending + {% endif %}
{% if folder.priority == 1 %} High Priority diff --git a/app/templates/partials/folder_card_destination.html b/app/templates/partials/folder_card_destination.html new file mode 100644 index 0000000..a82159e --- /dev/null +++ b/app/templates/partials/folder_card_destination.html @@ -0,0 +1,41 @@ +
+ +
+
+

{{ folder.name }}

+
+ + +
+
+ + +
+
+ {{ folder.emails_count }} emails +
+
+ +
+

{{ folder.rule_text }}

+
+
+
\ No newline at end of file diff --git a/app/templates/partials/folder_card_tidy.html b/app/templates/partials/folder_card_tidy.html new file mode 100644 index 0000000..6a6bcfa --- /dev/null +++ b/app/templates/partials/folder_card_tidy.html @@ -0,0 +1,82 @@ +
+ +
+
+

{{ folder.name }}

+
+ + +
+
+ + +
+
+ {{ folder.total_count }} total + {% if folder.pending_count > 0 %} + + {% else %} + {{ folder.pending_count }} pending + {% endif %} + {{ folder.total_count - folder.pending_count }} processed +
+ {% if folder.priority == 1 %} + High Priority + {% elif folder.priority == -1 %} + Low Priority + {% else %} + Normal Priority + {% endif %} +
+ +
+

{{ folder.rule_text }}

+
+ +
+
+ Organize: + + + + +
+
+
+
\ No newline at end of file diff --git a/app/templates/partials/folders_to_tidy_section.html b/app/templates/partials/folders_to_tidy_section.html new file mode 100644 index 0000000..9738593 --- /dev/null +++ b/app/templates/partials/folders_to_tidy_section.html @@ -0,0 +1,35 @@ +
+
+
+
+

Folders to Tidy

+

Folders containing emails that need to be processed

+
+ +
+
+ +
+ {% for folder in tidy_folders %} + {% include 'partials/folder_card_tidy.html' %} + {% else %} +
+
+ +
+

No folders to tidy yet

+

Add your first folder to start organizing your emails.

+
+ +
+
+ {% endfor %} +
+
\ No newline at end of file diff --git a/app/templates/partials/pending_emails_dialog.html b/app/templates/partials/pending_emails_dialog.html new file mode 100644 index 0000000..f0b1b80 --- /dev/null +++ b/app/templates/partials/pending_emails_dialog.html @@ -0,0 +1,123 @@ + + + \ No newline at end of file diff --git a/app/templates/partials/pending_emails_updated.html b/app/templates/partials/pending_emails_updated.html new file mode 100644 index 0000000..09dd015 --- /dev/null +++ b/app/templates/partials/pending_emails_updated.html @@ -0,0 +1,33 @@ +
+ + Email {{ uid }} has been marked as processed successfully! +
+ +
+
+
+
Total Emails
+
{{ folder.total_count }}
+
+
+
Pending
+
{{ folder.pending_count }}
+
+
+
Processed
+
{{ folder.total_count - folder.pending_count }}
+
+
+
+ +
+ +
\ No newline at end of file diff --git a/docs/design/data-model.md b/docs/design/data-model.md index 7a2c319..99e1ff7 100644 --- a/docs/design/data-model.md +++ b/docs/design/data-model.md @@ -26,8 +26,10 @@ erDiagram text rule_text "Natural Language Rule" int priority "Processing Order" boolean organize_enabled "Default: True" + string folder_type "Default: 'destination'" int total_count "Default: 0" int pending_count "Default: 0" + int emails_count "Default: 0" json recent_emails "JSON Array" datetime created_at "Default: UTC Now" datetime updated_at "Default: UTC Now, On Update" @@ -83,8 +85,10 @@ The `Folder` entity stores email organization rules and metadata for each user's | rule_text | Text | Nullable | Natural language description of the folder rule | | priority | Integer | Nullable | Processing order (0=normal, 1=high) | | organize_enabled | Boolean | Default: True | Whether the organization rule is active | +| folder_type | String(20) | Default: 'destination' | Folder type: 'tidy' or 'destination' | | total_count | Integer | Default: 0 | Total number of emails in the folder | | pending_count | Integer | Default: 0 | Number of emails waiting to be processed | +| emails_count | Integer | Default: 0 | Number of emails moved to this destination folder | | recent_emails | JSON | Default: [] | Array of recent email metadata | | created_at | DateTime | Default: datetime.utcnow | Timestamp of folder creation | | updated_at | DateTime | Default: datetime.utcnow, On Update | Timestamp of last update | @@ -100,6 +104,9 @@ The `Folder` entity stores email organization rules and metadata for each user's - Folder name must be unique per user - Rule text can be null (for manually created folders) - Priority values: 0 (normal), 1 (high priority) +- Folder types: + - 'tidy': Folders containing emails to be processed (e.g., Inbox) + - 'destination': Folders that are targets for email organization (default) - Recent emails array stores JSON objects with subject and date information ## Data Constraints @@ -128,7 +135,8 @@ The `Folder` entity stores email organization rules and metadata for each user's - `User.created_at`, `User.updated_at`: Current UTC timestamp - `Folder.created_at`, `Folder.updated_at`: Current UTC timestamp - `Folder.organize_enabled`: True -- `Folder.total_count`, `Folder.pending_count`: 0 +- `Folder.folder_type`: 'destination' +- `Folder.total_count`, `Folder.pending_count`, `Folder.emails_count`: 0 - `Folder.recent_emails`: Empty array ## JSON Data Structures @@ -219,6 +227,49 @@ The `recent_emails` field stores an array of JSON objects: - Batch updates for email counts - JSON operations for recent emails metadata +## Folder Types + +The system supports two distinct types of folders, each with different purposes and behaviors: + +### Tidy Folders + +Folders with `folder_type = 'tidy'` are source folders that contain emails waiting to be processed and organized. + +**Characteristics:** +- Display pending and processed email counts +- Can have organization rules enabled/disabled +- Support viewing pending emails +- Example: Inbox folder + +**UI Representation:** +- Shows "pending count" and "processed count" badges +- Includes "View Pending" button if there are pending emails +- May include priority indicators + +### Destination Folders + +Folders with `folder_type = 'destination'` are target folders where emails are moved from other folders during organization. + +**Characteristics:** +- Display count of emails moved to this folder +- Typically don't have organization rules (or they're ignored) +- Focus on showing how many emails have been organized into them +- Example: "Projects", "Finance", "Personal" folders + +**UI Representation:** +- Shows "emails count" badge +- Simpler interface without pending/processed indicators +- Focus on folder management and viewing contents + +### Folder Type Determination + +Folder types are determined as follows: +- During IMAP synchronization: + - Folders named "inbox" (case-insensitive) are automatically set as 'tidy' + - All other folders are set as 'destination' +- Manually created folders default to 'destination' +- Folder type can be changed through administrative functions + ## Future Data Model Considerations ### Potential Enhancements diff --git a/docs/design/folder-types-ui.md b/docs/design/folder-types-ui.md new file mode 100644 index 0000000..12a71dc --- /dev/null +++ b/docs/design/folder-types-ui.md @@ -0,0 +1,228 @@ +# Folder Types UI Design + +## Overview + +This document outlines the UI design changes needed to support the new folder types feature: "Folders to Tidy" and "Destination Folders". The UI will be reorganized to clearly separate these two types of folders into distinct sections. + +## UI Structure + +### Main Page Layout + +The main page will be divided into two distinct sections: + +1. **Folders to Tidy Section** + - Contains folders with `folder_type = 'tidy'` + - Typically includes the inbox and other source folders + - Shows pending and processed email counts + - Has actions for viewing and processing emails + +2. **Destination Folders Section** + - Contains folders with `folder_type = 'destination'` + - Contains target folders for email organization + - Shows count of emails moved to each folder + - Focuses on folder management and viewing contents + +### Visual Design + +#### Section Headers +Each section will have a clear header with: +- Section title +- Brief description of the section's purpose +- Action button (e.g., "Add Folder" for destination folders) + +#### Section Styling +- Different background colors or borders to visually distinguish sections +- Consistent spacing and layout within each section +- Responsive grid layout that adapts to screen size + +## Folder Card Components + +### Tidy Folder Card + +For folders with `folder_type = 'tidy'`, the card will display: + +#### Header +- Folder name (prominent) +- Edit and delete buttons + +#### Count Badges +- Total emails count +- Pending emails count (with warning styling if > 0) +- Processed emails count (calculated as total - pending) + +#### Actions +- "View Pending" button (if pending count > 0) +- Organize toggle switch +- Priority badge + +#### Content +- Rule text description +- Recent email previews (if available) + +### Destination Folder Card + +For folders with `folder_type = 'destination'`, the card will display: + +#### Header +- Folder name (prominent) +- Edit and delete buttons + +#### Count Badge +- Emails count (number of emails moved to this folder) + +#### Content +- Rule text description (if applicable) + +#### Actions +- Basic folder management options +- No organize toggle (as it's not applicable) + +## UI Components + +### Section Templates + +#### Folders to Tidy Section +```html +
+
+

Folders to Tidy

+

Folders containing emails that need to be processed

+ +
+
+ {% for folder in tidy_folders %} + {% include 'partials/folder_card_tidy.html' %} + {% endfor %} +
+
+``` + +#### Destination Folders Section +```html +
+
+

Destination Folders

+

Folders where emails are organized and stored

+ +
+
+ {% for folder in destination_folders %} + {% include 'partials/folder_card_destination.html' %} + {% endfor %} +
+
+``` + +### Folder Card Templates + +#### Tidy Folder Card +```html +
+
+
+

{{ folder.name }}

+
+ + +
+
+ +
+ {{ folder.total_count }} total + {% if folder.pending_count > 0 %} + + {% else %} + {{ folder.pending_count }} pending + {% endif %} + {{ folder.total_count - folder.pending_count }} processed +
+ +
+

{{ folder.rule_text }}

+
+ +
+
+ Organize: + +
+ {% if folder.priority == 1 %} + High Priority + {% endif %} +
+
+
+``` + +#### Destination Folder Card +```html +
+
+
+

{{ folder.name }}

+
+ + +
+
+ +
+ {{ folder.emails_count }} emails +
+ +
+

{{ folder.rule_text }}

+
+
+
+``` + +## Styling Guidelines + +### Color Scheme +- **Tidy Folders**: Use warning colors (yellow/orange) for pending emails +- **Destination Folders**: Use primary colors (blue) for email counts +- Consistent use of DaisyUI badge classes for different states + +### Responsive Design +- Grid layout adjusts from 1 column on mobile to 3 columns on desktop +- Cards should have consistent width across all screen sizes +- Section headers should remain readable on mobile + +### Interactive Elements +- Buttons should have hover states +- Toggle switches should be clearly labeled +- Pending email count should be clickable to open dialog + +## Implementation Notes + +### Template Organization +- Create separate partial templates for each folder type +- Main index template will include both sections +- Use HTMX for dynamic updates without full page reloads + +### Data Loading +- Fetch folders filtered by type from the backend +- Pass filtered lists to appropriate section templates +- Maintain existing functionality for folder operations + +### Accessibility +- Ensure all interactive elements have proper ARIA labels +- Use semantic HTML5 elements for section structure +- Maintain keyboard navigation support \ No newline at end of file diff --git a/docs/design/processed-emails-spec.md b/docs/design/processed-emails-spec.md new file mode 100644 index 0000000..ed2ae9d --- /dev/null +++ b/docs/design/processed-emails-spec.md @@ -0,0 +1,261 @@ +# Processed Emails Feature Specification + +## Overview + +This document outlines the specification for implementing a feature to persistently track which emails have been processed by the Email Organizer system. The goal is to maintain a record of email processing status to avoid reprocessing the same emails during synchronization and provide accurate pending email counts. + +## Requirements + +### 1. Email Tracking Requirements + +- **Unique Email Identification**: Track emails using a unique identifier (UID) provided by the IMAP server, along with the folder name and user ID +- **Processing Status**: Mark emails as either "pending" (unprocessed) or "processed" +- **Minimal Data Storage**: Store only essential information - email UID, folder, user, and processing status - not email content, subjects, or bodies +- **Persistence**: Maintain processing status across application restarts and synchronization cycles +- **Efficient Lookup**: Quickly determine which emails in a folder are pending processing + +### 2. Synchronization Requirements + +- **Initial Sync**: During first synchronization of a folder, all emails should be marked as "pending" +- **Incremental Sync**: On subsequent syncs, only emails that haven't been processed should be identified as pending +- **Status Update**: When an email is processed, update its status from "pending" to "processed" +- **Cleanup**: Remove records for emails that no longer exist on the IMAP server (optional for future enhancement) + +### 3. Performance Requirements + +- **Efficient Storage**: Use appropriate database indexing for fast lookups +- **Minimal Memory Usage**: Store only essential data to keep memory footprint low +- **Batch Processing**: Support batch operations for processing multiple emails efficiently + +## Data Model Design + +### ProcessedEmails Table + +```mermaid +erDiagram + USER { + int id PK "Primary Key" + string email "Unique, Not Null" + string first_name "Not Null" + string last_name "Not Null" + string password_hash "Not Null" + json imap_config "JSON Configuration" + datetime created_at "Default: UTC Now" + datetime updated_at "Default: UTC Now, On Update" + } + + FOLDER { + int id PK "Primary Key" + int user_id FK "Foreign Key to User" + string name "Not Null" + text rule_text "Natural Language Rule" + int priority "Processing Order" + boolean organize_enabled "Default: True" + int total_count "Default: 0" + int pending_count "Default: 0" + json recent_emails "JSON Array" + datetime created_at "Default: UTC Now" + datetime updated_at "Default: UTC Now, On Update" + } + + PROCESSED_EMAIL { + int id PK "Primary Key" + int user_id FK "Foreign Key to User" + int folder_id FK "Foreign Key to Folder" + string email_uid "Not Null" "IMAP Email UID" + string folder_name "Not Null" "IMAP Folder Name" + boolean is_processed "Default: False" "Processing Status" + datetime first_seen_at "Default: UTC Now" "First seen during sync" + datetime processed_at "Nullable" "When email was processed" + datetime created_at "Default: UTC Now" + datetime updated_at "Default: UTC Now, On Update" + } + + USER ||--o{ FOLDER : "has" + USER ||--o{ PROCESSED_EMAIL : "has" + FOLDER ||--o{ PROCESSED_EMAIL : "has" +``` + +### Column Specifications + +| Table | Column | Data Type | Constraints | Description | +|-------|--------|-----------|--------------|-------------| +| PROCESSED_EMAIL | id | Integer | Primary Key, Autoincrement | Unique identifier for each processed email record | +| PROCESSED_EMAIL | user_id | Integer | Foreign Key to User, Not Null | Reference to the user who owns this email | +| PROCESSED_EMAIL | folder_id | Integer | Foreign Key to Folder, Not Null | Reference to the folder this email belongs to | +| PROCESSED_EMAIL | email_uid | String(255) | Not Null | Unique ID of the email from IMAP server | +| PROCESSED_EMAIL | folder_name | String(255) | Not Null | Name of the IMAP folder (for redundancy) | +| PROCESSED_EMAIL | is_processed | Boolean | Default: False | Processing status (false=pending, true=processed) | +| PROCESSED_EMAIL | first_seen_at | DateTime | Default: datetime.utcnow | First time this email was detected during sync | +| PROCESSED_EMAIL | processed_at | DateTime | Nullable | When the email was marked as processed | +| PROCESSED_EMAIL | created_at | DateTime | Default: datetime.utcnow | Record creation timestamp | +| PROCESSED_EMAIL | updated_at | DateTime | Default: datetime.utcnow, On Update | Record update timestamp | + +### Relationships + +- **User to ProcessedEmail**: One-to-many relationship - each user can have multiple processed email records +- **Folder to ProcessedEmail**: One-to-many relationship - each folder can have multiple processed email records +- **Composite Key**: The combination of (user_id, folder_name, email_uid) should be unique to prevent duplicate records + +### Database Indexes + +- Primary key index on `id` +- Foreign key indexes on `user_id` and `folder_id` +- Composite unique index on `(user_id, folder_name, email_uid)` +- Index on `folder_name` for faster folder-based queries +- Index on `is_processed` for filtering pending emails +- Index on `first_seen_at` for tracking recently added emails + +## Service Design + +### ProcessedEmailsService + +A new service class will be responsible for managing processed email records: + +```python +class ProcessedEmailsService: + def __init__(self, user: User): + self.user = user + + def get_pending_emails(self, folder_name: str) -> List[str]: + """Get list of email UIDs that are pending processing in a folder.""" + + def mark_email_processed(self, folder_name: str, email_uid: str) -> bool: + """Mark an email as processed.""" + + def mark_emails_processed(self, folder_name: str, email_uids: List[str]) -> int: + """Mark multiple emails as processed in bulk.""" + + def sync_folder_emails(self, folder_name: str, email_uids: List[str]) -> int: + """Sync email UIDs for a folder, adding new ones as pending.""" + + def get_pending_count(self, folder_name: str) -> int: + """Get count of pending emails for a folder.""" + + def cleanup_old_records(self, folder_name: str, current_uids: List[str]) -> int: + """Remove records for emails that no longer exist in the folder.""" +``` + +### IMAPService Integration + +The existing IMAP service will be enhanced to use the ProcessedEmailsService: + +```python +class IMAPService: + def __init__(self, user: User): + self.user = user + self.config = user.imap_config or {} + self.connection = None + self.processed_emails_service = ProcessedEmailsService(user) + + def get_folder_email_count(self, folder_name: str) -> int: + """Get the count of emails in a specific folder, considering processed status.""" + + def get_pending_emails(self, folder_name: str) -> List[str]: + """Get email UIDs that are pending processing.""" + + def sync_folders(self) -> Tuple[bool, str]: + """Sync IMAP folders with local database, tracking email processing status.""" +``` + +## API Endpoints + +### New HTMX Endpoints for Processed Email Management + +1. **Get Pending Emails for a Folder** + - Method: GET + - Path: `/api/folders//pending-emails` + - Response: An Dialog List of email metadata for pending emails (subject, date, UID), a button to preview the email (fetch it from the imap server) + +2. **Mark Email as Processed** + - Method: POST + - Path: `/api/folders//emails//process` + - Action: Mark a specific email as processed + - Response: Updated dialog body. + +## Workflow Integration + +### Email Processing Flow + +```mermaid +sequenceDiagram + participant U as User + participant B as Browser + participant M as Main Blueprint + participant I as IMAP Service + participant P as ProcessedEmails Service + participant DB as Database + + U->>B: Click "Sync Folders" + B->>M: POST /api/imap/sync + M->>I: Sync folders with processed email tracking + I->>I: Connect to IMAP server + I->>I: Get list of email UIDs for folder + I->>P: sync_folder_emails(folder_name, email_uids) + P->>DB: Create pending email records + P->>I: Return list of pending email UIDs + I->>M: Return sync results + M->>B: Update UI with pending counts +``` + +### Email Processing Status Update + +```mermaid +sequenceDiagram + participant U as User + participant B as Browser + participant M as Main Blueprint + participant P as ProcessedEmails Service + participant DB as Database + + U->>B: Trigger email processing + B->>M: POST /api/folders//process-emails + M->>P: mark_emails_processed(folder_name, email_uids) + P->>DB: Update email processing status + P->>M: Return success count + M->>B: Update UI with new counts +``` + +## Migration Strategy + +### Phase 1: Data Model Implementation +1. Create the `processed_emails` table with appropriate indexes +2. Implement the `ProcessedEmailsService` class +3. Add basic CRUD operations for email processing records + +### Phase 2: IMAP Service Integration +1. Update `IMAPService` to use `ProcessedEmailsService` +2. Modify folder synchronization to track email UIDs +3. Update email count methods to consider processing status + +### Phase 3: API and UI Integration +1. Add API endpoints for processed email management +2. Update UI to display accurate pending counts +3. Add bulk processing capabilities + +### Phase 4: Optimization and Cleanup +1. Implement batch processing for performance +2. Add periodic cleanup of orphaned records +3. Optimize database queries for large datasets + +## Security Considerations + +1. **Access Control**: Ensure users can only access their own email processing records +2. **Data Validation**: Validate all email UIDs and folder names to prevent injection attacks +3. **Rate Limiting**: Implement rate limiting for email processing endpoints to prevent abuse +4. **Data Privacy**: Ensure no sensitive email content is stored in the database + +## Performance Considerations + +1. **Database Indexing**: Proper indexing on frequently queried fields +2. **Batch Operations**: Use batch operations for processing multiple emails +3. **Memory Management**: Process emails in batches to avoid memory issues with large mailboxes +4. **Caching**: Consider caching frequently accessed email processing status + +## Future Enhancements + +1. **Email Movement Tracking**: Track when emails are moved between folders +2. **Processing History**: Maintain a history of email processing actions +3. **Email Deduplication**: Handle duplicate emails across folders +4. **Automated Cleanup**: Periodic cleanup of old or orphaned processing records +5. **Analytics**: Provide insights into email processing patterns and efficiency \ No newline at end of file diff --git a/docs/plans/folder-types-redesign.md b/docs/plans/folder-types-redesign.md new file mode 100644 index 0000000..76b03bf --- /dev/null +++ b/docs/plans/folder-types-redesign.md @@ -0,0 +1,143 @@ +# Folder Types Redesign Plan + +## Overview + +This plan outlines the redesign of the email organizer to support two distinct types of folders: +1. **Folders to Tidy**: These folders contain emails that need to be processed and organized +2. **Destination Folders**: These folders are targets for organizing emails from other folders + +## Current State + +Currently, all folders are treated the same way with: +- Pending and processed counts +- Organization rules +- Toggle for enabling/disabling organization + +The system doesn't differentiate between source folders (where emails are processed from) and destination folders (where emails are moved to). + +## Proposed Changes + +### 1. Data Model Updates + +#### Folder Model Changes + +Add a new field to the `Folder` model: +```python +folder_type = db.Column(db.String(20), default='destination', nullable=False) +``` + +Possible values: +- `'tidy'`: Folders that contain emails to be processed (e.g., Inbox) +- `'destination'`: Folders that are targets for email organization (default for new folders) + +#### New Field Requirements: +- Must be a string to allow for potential future folder types +- Default value should be 'destination' for backward compatibility +- Not nullable to ensure all folders have a defined type + +### 2. Business Logic Changes + +#### Folder Synchronization Logic +- When synchronizing IMAP folders: + - If folder name is 'inbox' (case-insensitive), set `folder_type = 'tidy'` + - For all other folders, set `folder_type = 'destination'` + - Existing folders will keep their current type unless explicitly changed + +#### Folder Count Logic +- **Folders to Tidy**: + - Show `pending_count` (emails waiting to be processed) + - Show `processed_count` (emails that have been processed) + - May show `total_count` (total emails in the folder) + +- **Destination Folders**: + - Show `emails_count` (number of emails that have been put into this folder) + - No need to show pending/processed counts + +### 3. UI Changes + +#### Main Page Layout +Create two distinct sections: +1. **Folders to Tidy Section** + - Contains folders with `folder_type = 'tidy'` + - Shows pending and processed counts + - May include special styling to indicate these are source folders + +2. **Destination Folders Section** + - Contains folders with `folder_type = 'destination'` + - Shows count of emails moved to this folder + - May include special styling to indicate these are target folders + +#### Folder Card Changes +- **Tidy Folder Card**: + - Display: Folder name, pending count, processed count + - Actions: View pending emails, toggle organization + +- **Destination Folder Card**: + - Display: Folder name, emails count + - Actions: View emails in folder (if implemented), basic folder management + +### 4. API Changes + +#### New Endpoints +- `/api/folders/filter?type={tidy|destination}` - Get folders filtered by type +- `/api/folders//type` - Update folder type (admin function) + +#### Existing Endpoint Updates +- `/api/folders` - Add filtering capability +- `/api/imap/sync` - Implement new folder type logic +- Folder-related templates should adapt based on folder type + +### 5. Migration Strategy + +#### Database Migration +1. Create a new migration to add the `folder_type` column +2. Set default value to 'destination' for existing folders +3. Special handling for 'inbox' folders to set type to 'tidy' + +#### Data Migration +- Scan all existing folders for user with 'inbox' in the name +- Set `folder_type = 'tidy'` for these folders +- Set `folder_type = 'destination'` for all other folders + +### 6. Implementation Steps + +1. **Phase 1: Data Model** + - Create migration for `folder_type` column + - Update `Folder` model + - Apply migration to database + +2. **Phase 2: Backend Logic** + - Update folder synchronization logic + - Modify folder query methods + - Update count calculation logic + +3. **Phase 3: Frontend UI** + - Create two new partial templates for folder sections + - Update folder card templates based on type + - Modify main page to show two sections + +4. **Phase 4: Testing** + - Test folder synchronization with new types + - Verify UI shows correct sections + - Test folder count calculations + +### 7. Future Considerations + +#### Potential Enhancements +- Add ability to change folder type through UI +- Implement email movement tracking +- Add analytics for email organization efficiency +- Support for additional folder types in the future + +#### Backward Compatibility +- Existing folders will be treated as destination folders +- No breaking changes to existing functionality +- Gradual rollout of new UI sections + +## Success Criteria + +1. Folders are correctly categorized as either 'tidy' or 'destination' +2. Inbox folders are automatically set as 'tidy' type +3. UI displays two distinct sections for folder types +4. Folder cards show appropriate counts based on type +5. All existing functionality continues to work \ No newline at end of file diff --git a/migrations/versions/29a9c4ac0fc6_processing_email.py b/migrations/versions/29a9c4ac0fc6_processing_email.py new file mode 100644 index 0000000..f88f8e6 --- /dev/null +++ b/migrations/versions/29a9c4ac0fc6_processing_email.py @@ -0,0 +1,42 @@ +"""processing email + +Revision ID: 29a9c4ac0fc6 +Revises: 9a88c7e94083 +Create Date: 2025-08-06 11:40:52.702637 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '29a9c4ac0fc6' +down_revision = '9a88c7e94083' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('processed_emails', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('folder_id', sa.Integer(), nullable=False), + sa.Column('email_uid', sa.String(length=255), nullable=False), + sa.Column('folder_name', sa.String(length=255), nullable=False), + sa.Column('is_processed', sa.Boolean(), nullable=True), + sa.Column('first_seen_at', sa.DateTime(), nullable=True), + sa.Column('processed_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['folder_id'], ['folders.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('processed_emails') + # ### end Alembic commands ### diff --git a/migrations/versions/7b6db971e3a4_add_folder_type_and_emails_count_fields_.py b/migrations/versions/7b6db971e3a4_add_folder_type_and_emails_count_fields_.py new file mode 100644 index 0000000..3390956 --- /dev/null +++ b/migrations/versions/7b6db971e3a4_add_folder_type_and_emails_count_fields_.py @@ -0,0 +1,47 @@ +"""Add folder_type and emails_count fields to Folder table + +Revision ID: 7b6db971e3a4 +Revises: 29a9c4ac0fc6 +Create Date: 2025-08-06 12:25:13.415140 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7b6db971e3a4' +down_revision = '29a9c4ac0fc6' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + # Add new columns as nullable first + op.add_column('folders', sa.Column('folder_type', sa.String(length=20), nullable=True)) + op.add_column('folders', sa.Column('emails_count', sa.Integer(), nullable=True)) + + # Set default values for existing records + # Set all existing folders to 'destination' type + op.execute("UPDATE folders SET folder_type = 'destination' WHERE folder_type IS NULL") + + # Set 'inbox' folders to 'tidy' type + op.execute("UPDATE folders SET folder_type = 'tidy' WHERE folder_type IS NULL AND name ILIKE '%inbox%'") + + # Now make folder_type NOT NULL + op.alter_column('folders', 'folder_type', nullable=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + # Make column nullable again before dropping + op.alter_column('folders', 'folder_type', nullable=True) + + # Drop the columns + op.drop_column('folders', 'emails_count') + op.drop_column('folders', 'folder_type') + + # ### end Alembic commands ### diff --git a/tests/conftest.py b/tests/conftest.py index d8a9d8e..c19a0d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,7 @@ def mock_user(app): first_name='Test', last_name='User', email='test@example.com', - password_hash=b'hashed_password' # Will be properly hashed in real tests + password_hash='hashed_password' # Will be properly hashed in real tests ) db.session.add(user) diff --git a/tests/test_folder_deletion.py b/tests/test_folder_deletion.py new file mode 100644 index 0000000..6e0cd55 --- /dev/null +++ b/tests/test_folder_deletion.py @@ -0,0 +1,78 @@ +import pytest +from app.models import User, Folder, ProcessedEmail, db +from app import create_app + + +class TestFolderDeletion: + def test_delete_folder_with_emails(self, app, mock_user, authenticated_client): + """Test that deleting a folder also deletes its associated emails.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some email records associated with this folder + email_records = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='456', + is_processed=False + ) + ] + db.session.bulk_save_objects(email_records) + db.session.commit() + + # Verify emails were created + assert ProcessedEmail.query.count() == 2 + + # Delete the folder + response = authenticated_client.delete(f'/api/folders/{folder.id}') + + # Verify the response is successful + assert response.status_code == 200 + + # Verify the folder is deleted + deleted_folder = Folder.query.filter_by(id=folder.id).first() + assert deleted_folder is None + + # Verify the associated emails are also deleted (cascade delete) + assert ProcessedEmail.query.count() == 0 + + def test_delete_folder_with_no_emails(self, app, mock_user, authenticated_client): + """Test that deleting a folder with no associated emails works normally.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Verify no emails are associated with this folder + assert ProcessedEmail.query.filter_by(folder_id=folder.id).count() == 0 + + # Delete the folder + response = authenticated_client.delete(f'/api/folders/{folder.id}') + + # Verify the response is successful + assert response.status_code == 200 + + # Verify the folder is deleted + deleted_folder = Folder.query.filter_by(id=folder.id).first() + assert deleted_folder is None \ No newline at end of file diff --git a/tests/test_imap_service.py b/tests/test_imap_service.py index 726fa0f..3db7e0a 100644 --- a/tests/test_imap_service.py +++ b/tests/test_imap_service.py @@ -102,6 +102,10 @@ class TestIMAPService: with patch('app.imap_service.imaplib.IMAP4_SSL') as mock_imap_ssl: mock_connection = mock_imap_ssl.return_value mock_connection.list.return_value = ('OK', []) + mock_connection.login.return_value = None + mock_connection.select.return_value = ('OK', [b'1']) + mock_connection.close.return_value = None + mock_connection.logout.return_value = None user = User(email='test@example.com', first_name='Test', last_name='User') user.imap_config = {'server': 'test.com', 'port': 993, 'username': 'user', 'password': 'pass'} @@ -119,6 +123,7 @@ class TestIMAPService: # Mock the IMAP connection to raise an exception with patch('app.imap_service.imaplib.IMAP4_SSL') as mock_imap_ssl: mock_connection = mock_imap_ssl.return_value + mock_connection.login.return_value = None mock_connection.list.side_effect = Exception("IMAP error") folders = imap_service.get_folders() @@ -193,22 +198,19 @@ class TestIMAPService: def test_sync_folders_no_folders(self, app): with app.app_context(): - # Mock the IMAP connection to return an empty list of folders - with patch('app.imap_service.imaplib.IMAP4_SSL') as mock_imap_ssl: - mock_connection = mock_imap_ssl.return_value - mock_connection.list.return_value = ('OK', []) + user = User(email='test@example.com', first_name='Test', last_name='User') + user.set_password('testpassword') + db.session.add(user) + db.session.commit() + user.imap_config = {'server': 'localhost', 'port': 5143, 'username': 'user1@example.com', 'password': 'password1', 'use_ssl': False} + imap_service = IMAPService(user) + + # Mock the get_folders method to return an empty list + with patch.object(imap_service, 'get_folders', return_value=[]): + success, message = imap_service.sync_folders() - user = User(email='test@example.com', first_name='Test', last_name='User') - user.set_password('testpassword') - db.session.add(user) - db.session.commit() - user.imap_config = {'server': 'localhost', 'port': 5143, 'username': 'user1@example.com', 'password': 'password1', 'use_ssl': False} - imap_service = IMAPService(user) - - success, message = imap_service.sync_folders() - - assert success is False - assert message == "No folders found on IMAP server" + assert success is False + assert message == "No folders found on IMAP server" def test_sync_folders_exception(self, app): with app.app_context(): @@ -220,12 +222,9 @@ class TestIMAPService: user.imap_config = {'server': 'localhost', 'port': 5143, 'username': 'user1@example.com', 'password': 'password1', 'use_ssl': False } imap_service = IMAPService(user) - # Mock the IMAP connection to raise an exception - with patch('app.imap_service.imaplib.IMAP4_SSL') as mock_imap_ssl: - mock_connection = mock_imap_ssl.return_value - mock_connection.list.side_effect = Exception("IMAP server error") - + # Mock the get_folders method to raise an exception + with patch.object(imap_service, 'get_folders', side_effect=Exception("IMAP server error")): success, message = imap_service.sync_folders() assert success is False - assert "No folders found on IMAP server" in message \ No newline at end of file + assert "Sync error: IMAP server error" in message \ No newline at end of file diff --git a/tests/test_processed_emails_routes.py b/tests/test_processed_emails_routes.py new file mode 100644 index 0000000..ab558c0 --- /dev/null +++ b/tests/test_processed_emails_routes.py @@ -0,0 +1,267 @@ +import pytest +from flask import url_for +from unittest.mock import patch +from app.models import User, Folder, ProcessedEmail, db +from app import create_app +from app.imap_service import IMAPService + + +class TestProcessedEmailsRoutes: + def test_get_pending_emails_success(self, app, mock_user, authenticated_client): + """Test get_pending_emails endpoint successfully.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some pending email records + pending_emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='456', + is_processed=False + ) + ] + db.session.bulk_save_objects(pending_emails) + db.session.commit() + + # Mock IMAP service to return email headers + with patch('app.routes.IMAPService') as mock_imap_service: + mock_imap_instance = mock_imap_service.return_value + mock_imap_instance.get_email_headers.side_effect = [ + { + 'subject': 'Test Subject 1', + 'date': '2023-01-01T12:00:00', + 'from': 'sender1@example.com', + 'to': 'recipient@example.com', + 'message_id': '' + }, + { + 'subject': 'Test Subject 2', + 'date': '2023-01-02T12:00:00', + 'from': 'sender2@example.com', + 'to': 'recipient@example.com', + 'message_id': '' + } + ] + + response = authenticated_client.get(f'/api/folders/{folder.id}/pending-emails') + + assert response.status_code == 200 + # The response should be HTML with the pending emails dialog + assert 'Test Subject 1' in response.get_data(as_text=True) + assert 'Test Subject 2' in response.get_data(as_text=True) + + def test_get_pending_emails_folder_not_found(self, app, mock_user, authenticated_client): + """Test get_pending_emails endpoint with non-existent folder.""" + with app.app_context(): + response = authenticated_client.get('/api/folders/999/pending-emails') + + assert response.status_code == 404 + assert 'Folder not found' in response.get_json()['error'] + + def test_mark_email_processed_success(self, app, mock_user, authenticated_client): + """Test mark_email_processed endpoint successfully.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create a pending email record + pending_email = ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ) + db.session.add(pending_email) + db.session.commit() + + response = authenticated_client.post(f'/api/folders/{folder.id}/emails/123/process') + + assert response.status_code == 200 + # The response should be HTML with the updated dialog + assert 'has been marked as processed successfully' in response.get_data(as_text=True) + + # Verify the email is marked as processed + updated_email = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder', + email_uid='123' + ).first() + assert updated_email.is_processed is True + + def test_mark_email_processed_folder_not_found(self, app, mock_user, authenticated_client): + """Test mark_email_processed endpoint with non-existent folder.""" + with app.app_context(): + # Login and make request + login_response = response = authenticated_client.post('/api/folders/999/emails/123/process') + + assert response.status_code == 404 + assert 'Folder not found' in response.get_json()['error'] + + def test_sync_folder_emails_success(self, app, mock_user, authenticated_client): + """Test sync_folder_emails endpoint successfully.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Mock IMAP service to return email UIDs + with patch('app.routes.IMAPService') as mock_imap_service: + mock_imap_instance = mock_imap_service.return_value + mock_imap_instance.get_folder_email_uids.return_value = ['123', '456', '789'] + + # Login and make request + response = authenticated_client.post(f'/api/folders/{folder.id}/sync-emails') + + assert response.status_code == 200 + data = response.get_json() + assert data['success'] is True + assert 'Synced 3 new emails' in data['message'] + assert data['pending_count'] == 3 + assert data['total_count'] == 3 + + # Verify records were created + records = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder' + ).all() + assert len(records) == 3 + + def test_sync_folder_emails_no_emails(self, app, mock_user, authenticated_client): + """Test sync_folder_emails endpoint with no emails found.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Mock IMAP service to return no email UIDs + with patch('app.routes.IMAPService') as mock_imap_service: + mock_imap_instance = mock_imap_service.return_value + mock_imap_instance.get_folder_email_uids.return_value = [] + + # Login and make request + response = authenticated_client.post(f'/api/folders/{folder.id}/sync-emails') + + assert response.status_code == 404 + assert 'No emails found in folder' in response.get_json()['error'] + + def test_sync_folder_emails_folder_not_found(self, app, mock_user, authenticated_client): + """Test sync_folder_emails endpoint with non-existent folder.""" + with app.app_context(): + # Login and make request + login_response = response = authenticated_client.post('/api/folders/999/sync-emails') + + assert response.status_code == 404 + assert 'Folder not found' in response.get_json()['error'] + + def test_process_folder_emails_success(self, app, mock_user, authenticated_client): + """Test process_folder_emails endpoint successfully.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some pending email records + pending_emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='456', + is_processed=False + ) + ] + db.session.bulk_save_objects(pending_emails) + db.session.commit() + + # Login and make request + login_response = response = authenticated_client.post(f'/api/folders/{folder.id}/process-emails', data={ + 'email_uids': ['123', '456'] + }) + + assert response.status_code == 200 + data = response.get_json() + assert data['success'] is True + assert 'Processed 2 emails' in data['message'] + assert data['pending_count'] == 0 + + # Verify emails are marked as processed + processed_emails = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder', + is_processed=True + ).all() + assert len(processed_emails) == 2 + + def test_process_folder_emails_no_uids(self, app, mock_user, authenticated_client): + """Test process_folder_emails endpoint with no email UIDs provided.""" + with app.app_context(): + # Create a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Login and make request + login_response = response = authenticated_client.post(f'/api/folders/{folder.id}/process-emails', data={}) + + assert response.status_code == 400 + assert 'No email UIDs provided' in response.get_json()['error'] + + def test_process_folder_emails_folder_not_found(self, app, mock_user, authenticated_client): + """Test process_folder_emails endpoint with non-existent folder.""" + with app.app_context(): + # Login and make request + login_response = response = authenticated_client.post('/api/folders/999/process-emails', data={ + 'email_uids': ['123', '456'] + }) + + assert response.status_code == 404 + assert 'Folder not found' in response.get_json()['error'] \ No newline at end of file diff --git a/tests/test_processed_emails_service.py b/tests/test_processed_emails_service.py new file mode 100644 index 0000000..e7f4aef --- /dev/null +++ b/tests/test_processed_emails_service.py @@ -0,0 +1,321 @@ +import pytest +from app.processed_emails_service import ProcessedEmailsService +from app.models import User, Folder, ProcessedEmail, db +from app import create_app +from datetime import datetime + + +class TestProcessedEmailsService: + def test_init_with_user(self, app, mock_user): + """Test ProcessedEmailsService initialization with a user.""" + service = ProcessedEmailsService(mock_user) + assert service.user == mock_user + + def test_get_pending_emails_empty(self, app, mock_user): + """Test get_pending_emails when no emails are pending.""" + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Test with no pending emails + pending_uids = service.get_pending_emails('Test Folder') + assert len(pending_uids) == 0 + + def test_get_pending_emails_with_data(self, app, mock_user): + """Test get_pending_emails when there are pending emails.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some pending email records + pending_emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='456', + is_processed=False + ) + ] + db.session.bulk_save_objects(pending_emails) + db.session.commit() + + # Test getting pending emails + pending_uids = service.get_pending_emails('Test Folder') + assert len(pending_uids) == 2 + assert '123' in pending_uids + assert '456' in pending_uids + + def test_mark_email_processed_existing(self, app, mock_user): + """Test mark_email_processed for an existing pending email.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create a pending email record + pending_email = ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ) + db.session.add(pending_email) + db.session.commit() + + # Mark email as processed + success = service.mark_email_processed('Test Folder', '123') + assert success is True + + # Verify the email is marked as processed + updated_email = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder', + email_uid='123' + ).first() + assert updated_email.is_processed is True + assert updated_email.processed_at is not None + + def test_mark_email_processed_new(self, app, mock_user): + """Test mark_email_processed for a new email record.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Mark email as processed (create new record) + success = service.mark_email_processed('Test Folder', '789') + assert success is True + + # Verify the record was created + new_email = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder', + email_uid='789' + ).first() + assert new_email is not None + assert new_email.is_processed is True + assert new_email.processed_at is not None + + def test_mark_emails_processed(self, app, mock_user): + """Test mark_emails_processed for multiple emails.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some pending email records + pending_emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='123', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='456', + is_processed=False + ) + ] + db.session.bulk_save_objects(pending_emails) + db.session.commit() + + # Mark multiple emails as processed + processed_count = service.mark_emails_processed('Test Folder', ['123', '456', '789']) + assert processed_count == 3 + + # Verify all emails are marked as processed + processed_uids = set() + for email in ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder' + ).all(): + processed_uids.add(email.email_uid) + assert email.is_processed is True + + assert '123' in processed_uids + assert '456' in processed_uids + assert '789' in processed_uids + + def test_sync_folder_emails(self, app, mock_user): + """Test sync_folder_emails for a folder.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Sync emails + email_uids = ['123', '456', '789'] + new_emails_count = service.sync_folder_emails('Test Folder', email_uids) + assert new_emails_count == 3 + + # Verify records were created + records = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder' + ).all() + assert len(records) == 3 + + # Verify folder counts + updated_folder = Folder.query.filter_by(id=folder.id).first() + assert updated_folder.total_count == 3 + assert updated_folder.pending_count == 3 + + def test_get_pending_count(self, app, mock_user): + """Test get_pending_count for a folder.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some pending and processed emails + emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='1', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='2', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='3', + is_processed=True + ) + ] + db.session.bulk_save_objects(emails) + db.session.commit() + + # Test pending count + pending_count = service.get_pending_count('Test Folder') + assert pending_count == 2 + + def test_cleanup_old_records(self, app, mock_user): + """Test cleanup_old_records for a folder.""" + with app.app_context(): + service = ProcessedEmailsService(mock_user) + + # Mock a folder + folder = Folder( + user_id=mock_user.id, + name='Test Folder', + rule_text='Test rule' + ) + db.session.add(folder) + db.session.commit() + + # Create some email records + emails = [ + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='1', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='2', + is_processed=False + ), + ProcessedEmail( + user_id=mock_user.id, + folder_id=folder.id, + folder_name='Test Folder', + email_uid='3', + is_processed=True + ) + ] + db.session.bulk_save_objects(emails) + db.session.commit() + + # Clean up records that don't exist in current UIDs + current_uids = ['1', '2'] + deleted_count = service.cleanup_old_records('Test Folder', current_uids) + assert deleted_count == 1 + + # Verify only existing records remain + remaining_records = ProcessedEmail.query.filter_by( + user_id=mock_user.id, + folder_name='Test Folder' + ).all() + assert len(remaining_records) == 2 + assert all(email.email_uid in current_uids for email in remaining_records) + + # Verify folder counts + updated_folder = Folder.query.filter_by(id=folder.id).first() + assert updated_folder.total_count == 2 + assert updated_folder.pending_count == 2 \ No newline at end of file