Makes ai rule generation content work good.

This commit is contained in:
Bryce
2025-08-10 21:21:02 -07:00
parent 47a63d2eab
commit 0dac428217
12 changed files with 2129 additions and 8 deletions

View File

@@ -0,0 +1,307 @@
import pytest
from unittest.mock import patch, Mock
from app import create_app, db
from app.models import User, Folder, AIRuleCache
from bs4 import BeautifulSoup
class TestAIRuleUserFlow:
"""Test cases for the complete AI rule generation user flow."""
@pytest.fixture
def app(self):
"""Create and configure a test app."""
app = create_app('testing')
with app.app_context():
db.create_all()
yield app
db.drop_all()
@pytest.fixture
def client(self, app):
"""Create a test client."""
return app.test_client()
@pytest.fixture
def user(self, app):
"""Create a test user."""
with app.app_context():
user = User(
first_name='Test',
last_name='User',
email='test@example.com',
password_hash='hashed_password'
)
db.session.add(user)
db.session.commit()
return user
def test_folder_creation_modal_with_ai_controls(self, client, user):
"""Test that folder creation modal includes AI controls."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
response = client.get('/api/folders/new')
assert response.status_code == 200
# Check that AI controls are present
assert b'Generate Rule' in response.data
assert b'Multiple Options' in response.data
assert b'generate-rule' in response.data
def test_ai_rule_generation_in_modal(self, client, user):
"""Test AI rule generation within the folder creation modal."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Move emails from 'boss@company.com' to this folder",
{'quality_score': 85, 'model_used': 'test-model'}
)
# Simulate AI rule generation request
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that the response contains AI-generated rule
assert b'Move emails from' in response.data
assert b'generated-rule-text' in response.data
assert b'85%' in response.data
def test_multiple_rule_options_in_modal(self, client, user):
"""Test multiple rule options within the folder creation modal."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_multiple_rules.return_value = (
[
{'text': 'Move emails from boss@company.com', 'quality_score': 85},
{'text': 'Move emails with urgent subject', 'quality_score': 75},
{'text': 'Move emails from team members', 'quality_score': 70}
],
{'total_generated': 3}
)
# Simulate multiple rule generation request
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'multiple'
})
assert response.status_code == 200
# Check that multiple rules are displayed
assert b'Move emails from boss@company.com' in response.data
assert b'Move emails with urgent subject' in response.data
assert b'Move emails from team members' in response.data
assert b'85%' in response.data
assert b'75%' in response.data
assert b'70%' in response.data
def test_folder_creation_with_ai_rule(self, client, user):
"""Test folder creation using AI-generated rule."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Move emails from 'newsletter@company.com' to this folder",
{'quality_score': 90, 'model_used': 'test-model'}
)
# First, generate the rule
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Newsletters',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Then create folder with the generated rule
# We need to extract the rule from the response and use it
soup = BeautifulSoup(response.data, 'html.parser')
rule_text = soup.find(id='generated-rule-text').text.strip()
response = client.post('/api/folders', data={
'name': 'Newsletters',
'rule_text': rule_text,
'priority': '0'
})
assert response.status_code == 201
# Check that folder was created
folder = Folder.query.filter_by(name='Newsletters', user_id=user.id).first()
assert folder is not None
assert folder.rule_text == rule_text
def test_rule_quality_assessment(self, client, user):
"""Test rule quality assessment functionality."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.assess_rule_quality.return_value = {
'score': 75,
'grade': 'good',
'feedback': 'Good rule with room for improvement',
'assessed_at': '2023-01-01T00:00:00'
}
response = client.post('/api/folders/assess-rule', data={
'rule_text': 'Move emails from boss@company.com to this folder',
'folder_name': 'Work',
'folder_type': 'destination'
})
assert response.status_code == 200
# Check that quality assessment is displayed
assert b'75%' in response.data
assert b'generated-rule-text' in response.data
def test_error_handling_in_modal(self, client, user):
"""Test error handling within the modal."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
# Test with invalid inputs
response = client.post('/api/folders/generate-rule', data={
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that error is displayed in modal format
assert b'Folder name is required' in response.data
def test_fallback_rule_generation(self, client, user):
"""Test fallback rule generation when AI service fails."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service failure
mock_ai_service.generate_single_rule.return_value = (None, {'error': 'Service unavailable'})
mock_ai_service.get_fallback_rule.return_value = 'Move emails containing "Work" to this folder'
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that fallback rule is displayed
assert b'Move emails containing "Work" to this folder' in response.data
assert b'AI service unavailable' in response.data
def test_cache_usage_indicator(self, client, user):
"""Test that cache usage is properly indicated."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
# Create a cache entry
from datetime import datetime
cache_entry = AIRuleCache(
user_id=user.id,
folder_name='Work',
folder_type='destination',
rule_text='Cached rule',
rule_metadata={'quality_score': 90},
cache_key='test-key',
expires_at=datetime(2023, 12, 31, 23, 59, 59), # Future expiration
is_active=True
)
db.session.add(cache_entry)
db.session.commit()
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Make AI service return different rule to verify cache is used
mock_ai_service.generate_single_rule.return_value = (
'New rule',
{'quality_score': 95}
)
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that cached rule is used
assert b'Using cached rule' in response.data
assert b'Cached rule' in response.data
# New rule should not appear
assert b'New rule' not in response.data
def test_keyboard_navigation_support(self, client, user):
"""Test that keyboard navigation is supported."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
response = client.get('/api/folders/new')
assert response.status_code == 200
# Check that buttons have proper ARIA labels
assert b'aria-label' in response.data
assert b'Generate AI-powered email rule' in response.data
assert b'Generate multiple AI-powered email rule options' in response.data
def test_screen_reader_support(self, client, user):
"""Test that screen reader support is implemented."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Move emails from 'boss@company.com' to this folder",
{'quality_score': 85, 'model_used': 'test-model'}
)
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that screen reader support is present
assert b'role="status"' in response.data
assert b'aria-live="polite"' in response.data
assert b'sr-only' in response.data
def test_loading_states(self, client, user):
"""Test that loading states are properly handled."""
# Simulate logged-in user by setting session
with client.session_transaction() as sess:
sess['user_id'] = user.id
response = client.get('/api/folders/new')
assert response.status_code == 200
# Check that loading states are configured
assert b'data-loading-disable' in response.data
assert b'data-loading-class' in response.data
assert b'loading-spinner' in response.data

View File

@@ -0,0 +1,270 @@
import pytest
from unittest.mock import patch, Mock
from app import create_app, db
from app.models import User, Folder, AIRuleCache
from app.ai_service import AIService
class TestAIRuleEndpoints:
"""Test cases for AI rule generation API endpoints."""
@pytest.fixture
def app(self):
"""Create and configure a test app."""
app = create_app('testing')
with app.app_context():
db.create_all()
yield app
db.drop_all()
@pytest.fixture
def client(self, app):
"""Create a test client."""
return app.test_client()
@pytest.fixture
def user(self, app):
"""Create a test user."""
with app.app_context():
user = User(
first_name='Test',
last_name='User',
email='test@example.com',
password_hash='hashed_password'
)
db.session.add(user)
db.session.commit()
# Refresh the user to ensure it's attached to the session
db.session.refresh(user)
return user
@pytest.fixture
def authenticated_client(self, client, user):
"""Create a test client with authenticated user."""
with client.session_transaction() as sess:
sess['_user_id'] = str(user.id)
sess['_fresh'] = True
return client
def test_generate_rule_success(self, authenticated_client, user):
"""Test successful rule generation."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Move emails from 'boss@company.com' to this folder",
{'quality_score': 85, 'model_used': 'test-model'}
)
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Check that the response contains HTML
assert b'generated-rule-text' in response.data
assert b'Move emails from' in response.data
def test_generate_rule_missing_folder_name(self, authenticated_client, user):
"""Test rule generation with missing folder name."""
response = authenticated_client.post('/api/folders/generate-rule', data={
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Should return error in HTML format
assert b'Folder name is required' in response.data
def test_generate_rule_invalid_folder_type(self, authenticated_client, user):
"""Test rule generation with invalid folder type."""
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'invalid',
'rule_type': 'single'
})
assert response.status_code == 200
# Should return error in HTML format
assert b'Invalid folder type' in response.data
def test_generate_rule_multiple_options(self, authenticated_client, user):
"""Test multiple rule options generation."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_multiple_rules.return_value = (
[
{'text': 'Rule 1', 'quality_score': 85},
{'text': 'Rule 2', 'quality_score': 75}
],
{'total_generated': 2}
)
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'multiple'
})
assert response.status_code == 200
# Check that multiple rules are displayed
assert b'Rule 1' in response.data
assert b'Rule 2' in response.data
def test_generate_rule_ai_service_failure(self, authenticated_client, user):
"""Test rule generation when AI service fails."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service failure
mock_ai_service.generate_single_rule.return_value = (None, {'error': 'Service unavailable'})
mock_ai_service.get_fallback_rule.return_value = 'Fallback rule'
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Should return fallback rule
assert b'Fallback rule' in response.data
assert b'AI service unavailable' in response.data
def test_assess_rule_success(self, authenticated_client, user):
"""Test successful rule assessment."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.assess_rule_quality.return_value = {
'score': 85,
'grade': 'good',
'feedback': 'Good rule with room for improvement'
}
response = authenticated_client.post('/api/folders/assess-rule', data={
'rule_text': 'Move emails from boss@company.com to this folder',
'folder_name': 'Work',
'folder_type': 'destination'
})
assert response.status_code == 200
# Check that the response contains HTML
assert b'generated-rule-text' in response.data
assert b'85%' in response.data
def test_assess_rule_missing_inputs(self, authenticated_client, user):
"""Test rule assessment with missing inputs."""
response = authenticated_client.post('/api/folders/assess-rule', data={
'folder_name': 'Work',
'folder_type': 'destination'
})
assert response.status_code == 200
# Should return error in HTML format
assert b'Rule text is required' in response.data
def test_cache_functionality(self, authenticated_client, user):
"""Test rule caching functionality."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Cached rule",
{'quality_score': 90}
)
# First request - should generate new rule
response1 = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response1.status_code == 200
assert b'Cached rule' in response1.data
# Verify cache entry was created
cache_entry = AIRuleCache.query.filter_by(
user_id=user.id,
folder_name='Work',
folder_type='destination'
).first()
assert cache_entry is not None
assert cache_entry.rule_text == 'Cached rule'
# Second request - should use cached rule
with patch('app.routes.folders.ai_service') as mock_ai_service:
response2 = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response2.status_code == 200
assert b'Using cached rule' in response2.data
def test_cache_expiration(self, authenticated_client, user):
"""Test cache expiration functionality."""
with patch('app.routes.folders.ai_service') as mock_ai_service:
# Mock AI service response
mock_ai_service.generate_single_rule.return_value = (
"Expired rule",
{'quality_score': 90}
)
# Create expired cache entry
from datetime import datetime, timedelta
expired_entry = AIRuleCache(
user_id=user.id,
folder_name='Work',
folder_type='destination',
rule_text='Expired rule',
rule_metadata={'quality_score': 90},
cache_key='test-key',
expires_at=datetime.utcnow() - timedelta(hours=1),
is_active=True
)
db.session.add(expired_entry)
db.session.commit()
# Request should generate new rule despite cache entry
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Should not show cached message
assert b'Using cached rule' not in response.data
def test_unauthorized_access(self, client):
"""Test unauthorized access to AI rule endpoints."""
response = client.post('/api/folders/generate-rule', data={
'folder_name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
# Should be redirected to login
assert response.status_code == 302
def test_database_error_handling(self, authenticated_client, user):
"""Test handling of database errors."""
with patch('app.routes.folders.db.session.commit') as mock_commit:
# Mock database commit failure
mock_commit.side_effect = Exception("Database error")
with patch('app.routes.folders.ai_service') as mock_ai_service:
mock_ai_service.generate_single_rule.return_value = (
"Test rule",
{'quality_score': 85}
)
response = authenticated_client.post('/api/folders/generate-rule', data={
'name': 'Work',
'folder_type': 'destination',
'rule_type': 'single'
})
assert response.status_code == 200
# Should return error message
assert b'An unexpected error occurred' in response.data

View File

@@ -0,0 +1,206 @@
import pytest
from unittest.mock import Mock, patch
from app.ai_service import AIService
from datetime import datetime, timedelta
class TestAIService:
"""Test cases for the AI service functionality."""
def setup_method(self):
"""Set up test fixtures."""
self.ai_service = AIService()
# Set the attributes directly since they're not set in __init__ for tests
self.ai_service.api_key = 'test-api-key'
self.ai_service.model = 'test-model'
self.ai_service.api_url = 'https://api.openai.com/v1'
def test_init(self):
"""Test AI service initialization."""
assert self.ai_service.api_key == 'test-api-key'
assert self.ai_service.model == 'test-model'
assert self.ai_service.timeout == 30
assert self.ai_service.max_retries == 3
@patch('app.ai_service.requests.post')
def test_generate_single_rule_success(self, mock_post):
"""Test successful single rule generation."""
# Mock successful API response
mock_response = Mock()
mock_response.json.return_value = {
'choices': [{
'message': {
'content': 'Move emails from "boss@company.com" to this folder'
}
}]
}
mock_response.raise_for_status.return_value = None
mock_post.return_value = mock_response
rule_text, metadata = self.ai_service.generate_single_rule('Work', 'destination')
assert rule_text == 'Move emails from "boss@company.com" to this folder'
assert metadata is not None
assert 'quality_score' in metadata
assert 'model_used' in metadata
assert 'generated_at' in metadata
@patch('app.ai_service.requests.post')
def test_generate_single_rule_failure(self, mock_post):
"""Test single rule generation failure."""
# Mock API failure
mock_post.side_effect = Exception("API Error")
rule_text, metadata = self.ai_service.generate_single_rule('Work', 'destination')
assert rule_text is None
assert metadata is not None
assert 'error' in metadata
def test_assess_rule_quality(self):
"""Test rule quality assessment."""
rule_text = "Move emails from 'boss@company.com' to this folder"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
assert isinstance(score, int)
assert 0 <= score <= 100
def test_get_quality_grade(self):
"""Test quality grade determination."""
assert self.ai_service._get_quality_grade(90) == 'excellent'
assert self.ai_service._get_quality_grade(70) == 'good'
assert self.ai_service._get_quality_grade(50) == 'fair'
assert self.ai_service._get_quality_grade(30) == 'poor'
def test_generate_quality_feedback(self):
"""Test quality feedback generation."""
rule_text = "Move emails from 'boss@company.com' to this folder"
folder_name = "Work"
score = 85
feedback = self.ai_service._generate_quality_feedback(rule_text, folder_name, score)
assert isinstance(feedback, str)
assert len(feedback) > 0
def test_get_fallback_rule(self):
"""Test fallback rule generation."""
rule = self.ai_service.get_fallback_rule('Work', 'destination')
assert isinstance(rule, str)
assert len(rule) > 0
assert 'Work' in rule
def test_cache_key_generation(self):
"""Test cache key generation."""
# Access the static method directly since it's not a bound method
from app.ai_service import AIService
key1 = AIService.generate_cache_key('Work', 'destination', 'single')
key2 = AIService.generate_cache_key('Work', 'destination', 'single')
key3 = AIService.generate_cache_key('Personal', 'destination', 'single')
# Same inputs should produce same key
assert key1 == key2
# Different inputs should produce different keys
assert key1 != key3
def test_parse_multiple_rules_response(self):
"""Test parsing of multiple rules response."""
response_text = '''
{
"rules": [
{
"text": "Move emails from 'boss@company.com' to this folder",
"criteria": "Filters emails from specific sender"
},
{
"text": "Move emails with 'urgent' in subject to this folder",
"criteria": "Filters emails with urgent keywords"
}
]
}
'''
rules = self.ai_service._parse_multiple_rules_response(response_text)
assert len(rules) == 2
assert rules[0]['text'] == "Move emails from 'boss@company.com' to this folder"
assert rules[0]['criteria'] == "Filters emails from specific sender"
assert rules[1]['text'] == "Move emails with 'urgent' in subject to this folder"
assert rules[1]['criteria'] == "Filters emails with urgent keywords"
def test_parse_multiple_rules_response_manual(self):
"""Test manual parsing of multiple rules response."""
# Test with a more structured format that matches what the parser expects
response_text = '''{
"rules": [
{
"text": "Move emails from 'boss@company.com' to this folder",
"criteria": "Filters emails from specific sender"
},
{
"text": "Move emails with 'urgent' in subject to this folder",
"criteria": "Filters emails with urgent keywords"
}
]
}'''
rules = self.ai_service._parse_multiple_rules_response(response_text)
# Should parse JSON format correctly
assert len(rules) == 2
assert rules[0]['text'] == "Move emails from 'boss@company.com' to this folder"
assert rules[0]['criteria'] == "Filters emails from specific sender"
assert rules[1]['text'] == "Move emails with 'urgent' in subject to this folder"
assert rules[1]['criteria'] == "Filters emails with urgent keywords"
def test_short_rule_penalty(self):
"""Test that short rules get penalized."""
rule_text = "short"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
# Short rules should get low scores
assert score < 50
def test_long_rule_penalty(self):
"""Test that very long rules get penalized."""
rule_text = "This is a very long rule that exceeds the optimal length and should be penalized accordingly"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
# Very long rules should get lower scores (should be <= 80)
assert score <= 80
def test_specific_keyword_bonus(self):
"""Test that specific keywords get bonus points."""
rule_text = "Move emails from 'boss@company.com' to this folder"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
# Rules with specific keywords should get higher scores
assert score > 50
def test_action_word_bonus(self):
"""Test that action words get bonus points."""
rule_text = "Move emails from 'boss@company.com' to this folder"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
# Rules with action words should get higher scores
assert score > 50
def test_folder_relevance_bonus(self):
"""Test that folder name relevance gets bonus points."""
rule_text = "Move emails related to 'Work' projects to this folder"
folder_name = "Work"
score = self.ai_service._assess_rule_quality(rule_text, folder_name, 'destination')
# Rules relevant to folder name should get higher scores
assert score > 50