notid/email-organizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update documentation

Browse Source
This commit is contained in:
Bryce
2025-08-06 10:30:20 -07:00
parent e6c8388ba6
commit 5336c04444
3 changed files with 101 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

724
docs/design/imap-connectivity.md
View File

@@ -2,7 +2,7 @@
## Overview
This document outlines the design for implementing IMAP connectivity in the Email Organizer application. The feature will allow users to configure their IMAP server connection details, test the connection, and synchronize their email folders from the IMAP server to the application.
This document outlines the design for implementing IMAP connectivity in the Email Organizer application. The feature allows users to configure their IMAP server connection details, test the connection, and synchronize their email folders from the IMAP server to the application.
## Current State Analysis
@@ -67,678 +67,9 @@ sequenceDiagram
M->>U: Show sync result
```
## Database Schema
### User Model Updates
The existing `User` model already has the `imap_config` field:
```python
class User(Base, UserMixin):
# ... existing fields ...
imap_config = db.Column(db.JSON) # Store IMAP connection details
# ... existing fields ...
```
### Folder Model Updates
The `Folder` model now includes a new field for storing recent email information:
```python
class Folder(Base):
# ... existing fields ...
recent_emails = db.Column(db.JSON, default=list) # Store recent email subjects with dates
# ... existing fields ...
```
### IMAP Configuration Structure
```json
{
"server": "imap.gmail.com",
"port": 993,
"username": "user@gmail.com",
"password": "app_password",
"use_ssl": true,
"use_tls": false,
"connection_timeout": 30,
"folder_prefix": ""
}
```
## IMAP Service Module
### File: `app/imap_service.py`
```python
import imaplib
import ssl
import logging
from typing import List, Dict, Optional, Tuple
from email.header import decode_header
from app.models import db, Folder, User
from app import create_app
class IMAPService:
def __init__(self, user: User):
self.user = user
self.config = user.imap_config or {}
self.connection = None
def test_connection(self) -> Tuple[bool, str]:
"""Test IMAP connection with current configuration."""
try:
if not self.config:
return False, "No IMAP configuration found"
# Create SSL context
context = ssl.create_default_context()
# Connect to IMAP server
self.connection = imaplib.IMAP4_SSL(
self.config.get('server', 'imap.gmail.com'),
self.config.get('port', 993)
)
# Login
self.connection.login(
self.config.get('username', ''),
self.config.get('password', '')
)
# Select inbox to verify connection
self.connection.select('INBOX')
# Close connection
self.connection.close()
self.connection.logout()
return True, "Connection successful"
except imaplib.IMAP4.error as e:
return False, f"IMAP connection error: {str(e)}"
except Exception as e:
return False, f"Connection error: {str(e)}"
finally:
if self.connection:
try:
self.connection.logout()
except:
pass
def get_folders(self) -> List[Dict[str, str]]:
"""Get list of folders from IMAP server."""
try:
if not self.config:
return []
# Create SSL context
context = ssl.create_default_context()
# Connect to IMAP server
self.connection = imaplib.IMAP4_SSL(
self.config.get('server', 'imap.gmail.com'),
self.config.get('port', 993)
)
# Login
self.connection.login(
self.config.get('username', ''),
self.config.get('password', '')
)
# List folders
status, folder_data = self.connection.list()
if status != 'OK':
return []
folders = []
for folder_item in folder_data:
if isinstance(folder_item, bytes):
folder_item = folder_item.decode('utf-8')
# Parse folder name (handle different IMAP server formats)
parts = folder_item.split('"')
if len(parts) >= 3:
folder_name = parts[-1] if parts[-1] else parts[-2]
else:
folder_name = folder_item.split()[-1]
# Handle nested folders (convert to slash notation)
if folder_name.startswith('"') and folder_name.endswith('"'):
folder_name = folder_name[1:-1]
folders.append({
'name': folder_name,
'full_path': folder_name
})
# Close connection
self.connection.close()
self.connection.logout()
return folders
except Exception as e:
logging.error(f"Error fetching IMAP folders: {str(e)}")
return []
finally:
if self.connection:
try:
self.connection.logout()
except:
pass
def sync_folders(self) -> Tuple[bool, str]:
"""Sync IMAP folders with local database."""
try:
if not self.config:
return False, "No IMAP configuration found"
# Get folders from IMAP server
imap_folders = self.get_folders()
if not imap_folders:
return False, "No folders found on IMAP server"
# Process each folder
synced_count = 0
for imap_folder in imap_folders:
folder_name = imap_folder['name']
# Skip special folders that might not be needed
if folder_name.lower() in ['inbox', 'sent', 'drafts', 'spam', 'trash']:
continue
# Handle nested folder names (convert slashes to underscores or keep as-is)
# According to requirements, nested folders should be created with slashes in the name
display_name = folder_name
# Check if folder already exists
existing_folder = Folder.query.filter_by(
user_id=self.user.id,
name=display_name
).first()
if not existing_folder:
# Create new folder
new_folder = Folder(
user_id=self.user.id,
name=display_name,
rule_text=f"Auto-synced from IMAP folder: {folder_name}",
priority=0 # Default priority
)
db.session.add(new_folder)
synced_count += 1
db.session.commit()
return True, f"Successfully synced {synced_count} folders"
except Exception as e:
db.session.rollback()
return False, f"Sync error: {str(e)}"
def get_recent_emails(self, folder_name: str, limit: int = 3) -> List[Dict[str, any]]:
"""Get the most recent email subjects and dates from a specific folder."""
try:
# Connect to IMAP server
self._connect()
# Login
self.connection.login(
self.config.get('username', ''),
self.config.get('password', '')
)
# Select the folder
resp_code, content = self.connection.select(folder_name)
if resp_code != 'OK':
return []
# Get email IDs (most recent first)
resp_code, content = self.connection.search(None, 'ALL')
if resp_code != 'OK':
return []
# Get the most recent emails (limit to 3)
email_ids = content[0].split()
recent_email_ids = email_ids[:limit] if len(email_ids) >= limit else email_ids
recent_emails = []
for email_id in reversed(recent_email_ids): # Process from newest to oldest
# Fetch the email headers
resp_code, content = self.connection.fetch(email_id, '(RFC822.HEADER)')
if resp_code != 'OK':
continue
# Parse the email headers
raw_email = content[0][1]
import email
msg = email.message_from_bytes(raw_email)
# Extract subject and date
subject = msg.get('Subject', 'No Subject')
date_str = msg.get('Date', '')
# Decode the subject if needed
try:
decoded_parts = decode_header(subject)
subject = ''.join([str(part, encoding or 'utf-8') if isinstance(part, bytes) else part for part, encoding in decoded_parts])
except Exception:
pass # If decoding fails, use the original subject
# Parse date if available
try:
email_date = parsedate_to_datetime(date_str) if date_str else None
except Exception:
email_date = None
recent_emails.append({
'subject': subject,
'date': email_date.isoformat() if email_date else None
})
# Close folder and logout
self.connection.close()
self.connection.logout()
self.connection = None
return recent_emails
except Exception as e:
logging.error(f"Error getting recent emails for folder {folder_name}: {str(e)}")
if self.connection:
try:
self.connection.logout()
except:
pass
self.connection = None
return []
```
## Routes Implementation
### File: `app/routes.py` (additions)
```python
# Add to existing imports
from app.imap_service import IMAPService
import json
@main.route('/api/imap/config', methods=['GET'])
@login_required
def imap_config_modal():
"""Return the IMAP configuration modal."""
response = make_response(render_template('partials/imap_config_modal.html'))
response.headers['HX-Trigger'] = 'open-modal'
return response
@main.route('/api/imap/test', methods=['POST'])
@login_required
def test_imap_connection():
"""Test IMAP connection with provided configuration."""
try:
# Get form data
server = request.form.get('server')
port = request.form.get('port')
username = request.form.get('username')
password = request.form.get('password')
use_ssl = request.form.get('use_ssl') == 'on'
# Validate required fields
errors = {}
if not server:
errors['server'] = 'Server is required'
if not port:
errors['port'] = 'Port is required'
elif not port.isdigit():
errors['port'] = 'Port must be a number'
if not username:
errors['username'] = 'Username is required'
if not password:
errors['password'] = 'Password is required'
if errors:
response = make_response(render_template('partials/imap_config_modal.html', errors=errors))
response.headers['HX-Retarget'] = '#imap-modal'
response.headers['HX-Reswap'] = 'outerHTML'
return response
# Store configuration temporarily for testing
test_config = {
'server': server,
'port': int(port),
'username': username,
'password': password,
'use_ssl': use_ssl,
'use_tls': False,
'connection_timeout': 30
}
# Test connection
temp_user = type('User', (), {'imap_config': test_config})()
imap_service = IMAPService(temp_user)
success, message = imap_service.test_connection()
if success:
# Save configuration to user's profile
current_user.imap_config = test_config
db.session.commit()
response = make_response(render_template('partials/imap_config_modal.html',
success=True, message=message))
response.headers['HX-Retarget'] = '#imap-modal'
response.headers['HX-Reswap'] = 'outerHTML'
else:
response = make_response(render_template('partials/imap_config_modal.html',
errors={'general': message}))
response.headers['HX-Retarget'] = '#imap-modal'
response.headers['HX-Reswap'] = 'outerHTML'
return response
except Exception as e:
logging.exception("Error testing IMAP connection: %s", e)
errors = {'general': 'An unexpected error occurred. Please try again.'}
response = make_response(render_template('partials/imap_config_modal.html', errors=errors))
response.headers['HX-Retarget'] = '#imap-modal'
response.headers['HX-Reswap'] = 'outerHTML'
return response
@main.route('/api/imap/sync', methods=['POST'])
@login_required
def sync_imap_folders():
"""Sync folders from IMAP server."""
try:
if not current_user.imap_config:
return jsonify({'error': 'No IMAP configuration found. Please configure IMAP first.'}), 400
# Test connection first
imap_service = IMAPService(current_user)
success, message = imap_service.sync_folders()
if success:
# Get updated list of folders
folders = Folder.query.filter_by(user_id=current_user.id).all()
return render_template('partials/folders_list.html', folders=folders)
else:
return jsonify({'error': message}), 400
except Exception as e:
logging.exception("Error syncing IMAP folders: %s", e)
return jsonify({'error': 'An unexpected error occurred. Please try again.'}), 500
```
## UI Implementation
### Template: `app/templates/partials/imap_config_modal.html`
```html
<div id="imap-modal" @click.away="$refs.modal.close()" class="modal-box" x-data="{ errors: {{ 'true' if errors else 'false' }} }" x-init="$nextTick(() => { if (errors) { document.querySelector('#submit-btn').classList.add('shake'); } })">
<h3 class="font-bold text-lg mb-4">Configure IMAP Connection</h3>
{% if success %}
<div class="alert alert-success mb-4">
<svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z" /></svg>
<span>{{ message }}</span>
</div>
{% endif %}
{% if errors and errors.general %}
<div class="alert alert-error mb-4">
<svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 14l2-2m0 0l2-2m-2 2l-2-2m2 2l2 2m7-2a9 9 0 11-18 0 9 9 0 0118 0z" /></svg>
<span>{{ errors.general }}</span>
</div>
{% endif %}
<form id="imap-form" hx-post="/api/imap/test" hx-target="#imap-modal" hx-swap="outerHTML">
<div class="mb-4">
<label for="imap-server" class="block text-sm font-medium mb-1">IMAP Server</label>
<input type="text" id="imap-server" name="server"
class="input input-bordered w-full {% if errors and errors.server %}input-error{% endif %}"
placeholder="e.g., imap.gmail.com" required
value="{% if server is defined %}{{ server }}{% endif %}">
{% if errors and errors.server %}
<div class="text-error text-sm mt-1">{{ errors.server }}</div>
{% endif %}
</div>
<div class="mb-4">
<label for="imap-port" class="block text-sm font-medium mb-1">Port</label>
<input type="number" id="imap-port" name="port"
class="input input-bordered w-full {% if errors and errors.port %}input-error{% endif %}"
placeholder="e.g., 993" required
value="{% if port is defined %}{{ port }}{% else %}993{% endif %}">
{% if errors and errors.port %}
<div class="text-error text-sm mt-1">{{ errors.port }}</div>
{% endif %}
</div>
<div class="mb-4">
<label for="imap-username" class="block text-sm font-medium mb-1">Username</label>
<input type="text" id="imap-username" name="username"
class="input input-bordered w-full {% if errors and errors.username %}input-error{% endif %}"
placeholder="e.g., your-email@gmail.com" required
value="{% if username is defined %}{{ username }}{% endif %}">
{% if errors and errors.username %}
<div class="text-error text-sm mt-1">{{ errors.username }}</div>
{% endif %}
</div>
<div class="mb-4">
<label for="imap-password" class="block text-sm font-medium mb-1">Password</label>
<input type="password" id="imap-password" name="password"
class="input input-bordered w-full {% if errors and errors.password %}input-error{% endif %}"
placeholder="App password or account password" required>
{% if errors and errors.password %}
<div class="text-error text-sm mt-1">{{ errors.password }}</div>
{% endif %}
</div>
<div class="mb-6">
<label class="flex items-center cursor-pointer">
<input type="checkbox" id="imap-use-ssl" name="use_ssl" class="checkbox mr-2" checked>
<span class="text-sm font-medium">Use SSL (Recommended)</span>
</label>
<p class="text-xs text-base-content/70 mt-1">Most IMAP servers use SSL on port 993</p>
</div>
<div class="modal-action">
<button type="button" class="btn btn-outline"
@click="$dispatch('close-modal')">Cancel</button>
<button type="submit" class="btn btn-primary" id="submit-btn" :class="{ 'shake': errors }">
Test Connection
</button>
</div>
</form>
{% if success %}
<div class="mt-4 pt-4 border-t border-base-300">
<button class="btn btn-success w-full" hx-post="/api/imap/sync" hx-target="#folders-list" hx-swap="innerHTML">
<i class="fas fa-sync mr-2"></i>
Sync Folders
</button>
</div>
{% endif %}
</div>
```
### Template Update: `app/templates/partials/sidebar.html`
Add IMAP configuration option to the sidebar:
```html
<div class="sidebar w-64 p-4 flex flex-col bg-base-200 shadow-lg fixed top-16 left-0 bottom-0 z-10">
<nav class="flex-grow menu bg-transparent rounded-lg">
<div class="active">
<a class="btn btn-ghost justify-start">
<i class="fas fa-folder mr-3 text-primary"></i>
Folders
</a>
</div>
<div>
<a class="btn btn-ghost justify-start">
<i class="fas fa-inbox mr-3 text-secondary"></i>
Inbox
</a>
</div>
<div>
<a class="btn btn-ghost justify-start" hx-get="/api/imap/config" hx-target="#modal-holder" hx-swap="innerHTML">
<i class="fas fa-cog mr-3 text-accent"></i>
IMAP Settings
</a>
</div>
<div>
<a class="btn btn-ghost justify-start">
<i class="fas fa-chart-bar mr-3 text-info"></i>
Analytics
</a>
</div>
<div>
<a class="btn btn-ghost justify-start">
<i class="fas fa-question-circle mr-3 text-warning"></i>
Help
</a>
</div>
</nav>
<div class="mt-auto pt-4 border-t border-base-300">
<div>
<a href="{{ url_for('auth.logout') }}" class="btn btn-ghost justify-start">
<i class="fas fa-sign-out-alt mr-3 text-error"></i>
Logout
</a>
</div>
</div>
</div>
```
## Testing Strategy
### Unit Tests
```python
# tests/test_imap_service.py
import pytest
from app.imap_service import IMAPService
from app.models import User, db
from unittest.mock import Mock, patch
class TestIMAPService:
def test_init_with_user(self, app):
with app.app_context():
user = User(email='test@example.com', first_name='Test', last_name='User')
user.imap_config = {'server': 'test.com', 'port': 993}
imap_service = IMAPService(user)
assert imap_service.user == user
assert imap_service.config == {'server': 'test.com', 'port': 993}
@patch('app.imap_service.imaplib.IMAP4_SSL')
def test_test_connection_success(self, mock_imap, app):
with app.app_context():
# Mock successful connection
mock_connection = Mock()
mock_imap.return_value = mock_connection
mock_connection.login.return_value = None
mock_connection.select.return_value = ('OK', [b''])
mock_connection.close.return_value = None
mock_connection.logout.return_value = None
user = User(email='test@example.com', first_name='Test', last_name='User')
user.imap_config = {'server': 'test.com', 'port': 993, 'username': 'user', 'password': 'pass'}
imap_service = IMAPService(user)
success, message = imap_service.test_connection()
assert success is True
assert message == "Connection successful"
mock_imap.assert_called_once_with('test.com', 993)
mock_connection.login.assert_called_once_with('user', 'pass')
@patch('app.imap_service.imaplib.IMAP4_SSL')
def test_test_connection_failure(self, mock_imap, app):
with app.app_context():
# Mock failed connection
mock_imap.side_effect = Exception("Connection failed")
user = User(email='test@example.com', first_name='Test', last_name='User')
user.imap_config = {'server': 'test.com', 'port': 993, 'username': 'user', 'password': 'pass'}
imap_service = IMAPService(user)
success, message = imap_service.test_connection()
assert success is False
assert "Connection error" in message
```
### Integration Tests
```python
# tests/test_imap_routes.py
import pytest
from app import create_app
from app.models import User, db, Folder
class TestIMAPRoutes:
def test_imap_config_modal(self, client):
response = client.get('/api/imap/config')
assert response.status_code == 200
assert b'Configure IMAP Connection' in response.data
def test_imap_connection_test_success(self, client, app):
with app.app_context():
user = User(email='test@example.com', first_name='Test', last_name='User')
user.set_password('password')
db.session.add(user)
db.session.commit()
client.post('/auth/login', data={
'email': 'test@example.com',
'password': 'password'
})
response = client.post('/api/imap/test', data={
'server': 'test.com',
'port': '993',
'username': 'test@test.com',
'password': 'testpass',
'use_ssl': 'on'
})
assert response.status_code == 200
assert b'Test Connection' in response.data
def test_imap_sync_folders(self, client, app):
with app.app_context():
user = User(email='test@example.com', first_name='Test', last_name='User')
user.set_password('password')
user.imap_config = {'server': 'test.com', 'port': 993, 'username': 'test', 'password': 'pass'}
db.session.add(user)
db.session.commit()
client.post('/auth/login', data={
'email': 'test@example.com',
'password': 'password'
})
response = client.post('/api/imap/sync')
assert response.status_code == 400 # Should fail without real IMAP server
```
## Implementation Dependencies
### New Dependencies
Add to `requirements.txt`:
```
imapclient==3.0.1
```
### Database Migration
No database migration is required as the existing `imap_config` field in the `User` model is sufficient.
## Security Considerations
@@ -771,58 +102,7 @@ No database migration is required as the existing `imap_config` field in the `Us
## Success Metrics
1. **Functional**: Users can configure IMAP connections, test them, and sync folders successfully.
2. **Usability**: The UI is intuitive and provides clear feedback for all operations.
3. **Reliability**: IMAP connections are stable and handle various server configurations.
4. **Performance**: Folder synchronization completes in reasonable time.
5. **Security**: User credentials are handled securely.
## Acceptance Criteria
Based on the original requirements:
1. ✅ **Verify that a user can enter their imap server's connection details, and that they can be saved**
- Users can access the IMAP configuration modal
- Form validation ensures all required fields are filled
- Configuration is saved to the user's `imap_config` field
2. ✅ **Verify that, once the connection is made, the list of folders are created for that user, copying the name and structure from IMAP**
- Connection testing validates the IMAP server access
- Folder synchronization fetches all available folders
- Folders are created in the local database with appropriate names
3. ✅ **Verify that nested folders are just created with slashes in the name**
- Nested folder names from IMAP servers are preserved with slash notation
- Display names maintain the hierarchical structure
4. ✅ **Verify that a failed connection attempt gives the user a validation error message and allows them to retry**
- Connection failures show clear error messages
- Users can modify their configuration and retry the connection
- Form validation prevents submission with invalid data
## Implementation Plan
### Phase 1: Core IMAP Service
1. Create `app/imap_service.py` with basic IMAP connection functionality
2. Implement connection testing logic
3. Add folder listing and synchronization
### Phase 2: UI Integration
1. Create IMAP configuration modal template
2. Add routes for IMAP configuration and testing
3. Update sidebar to include IMAP settings option
### Phase 3: Testing and Validation
1. Write unit tests for IMAP service
2. Write integration tests for IMAP routes
3. Test with various IMAP server configurations
### Phase 4: Error Handling and Polish
1. Add comprehensive error handling
2. Improve user feedback and validation messages
3. Optimize performance and reliability
This design provides a comprehensive approach to implementing IMAP connectivity while maintaining the existing application architecture and following the established patterns.
5. **Security**: User credentials are handled securely.