test-quality-review.py

#!/usr/bin/env python3
"""
Test Quality Reviewer
Analyzes test code for quality, coverage, maintainability, and best practices.

Usage:
    python test-quality-review.py <test_file_or_directory> [--output report.md]
    python test-quality-review.py tests/ --output test-quality-report.md
"""

import argparse
import sys
import re
from pathlib import Path
from datetime import datetime
from collections import defaultdict


AGENT_PROMPT = """
# Test Quality Review

You are an expert test code reviewer. Analyze test code for quality, coverage,
maintainability, and adherence to testing best practices.

## Review Focus

### Test Structure
- ✓ AAA (Arrange-Act-Assert) pattern
- ✓ One logical assertion per test
- ✓ Clear test naming
- ✓ Test independence
- ✓ Proper setup/teardown

### Test Naming
- ✓ Descriptive names (describe behavior)
- ✓ Consistent pattern
- ✓ Readable as sentences
- ✓ Include scenario and expected result

### Coverage
- ✓ Happy path tested
- ✓ Edge cases covered
- ✓ Error conditions tested
- ✓ Boundary values tested
- ✓ Critical paths verified

### Test Data
- ✓ Meaningful test data (not magic numbers)
- ✓ Test builders/factories
- ✓ Clear data setup
- ✓ Minimal necessary data

### Assertions
- ✓ Specific assertions
- ✓ Descriptive messages
- ✓ Testing behavior, not implementation
- ✓ No multiple unrelated assertions

### Mocks & Doubles
- ✓ Appropriate mock usage
- ✓ Not over-mocking
- ✓ Clear mock setup
- ✓ Verify behavior, not implementation

### Maintainability
- ✓ No duplication
- ✓ Test helpers extracted
- ✓ Readable (self-documenting)
- ✓ Not brittle
- ✓ Fast execution

## Test Smells to Flag

### Critical Smells
- ✗ Obscure Test (unclear intent)
- ✗ Eager Test (tests too much)
- ✗ Mystery Guest (hidden dependencies)
- ✗ Shared state between tests
- ✗ Tests that always pass
- ✗ Conditional test logic

### Common Smells
- ✗ Test code duplication
- ✗ Inappropriate mocking
- ✗ Fragile tests (brittle)
- ✗ Slow tests
- ✗ Poor naming (test1, testMethod)
- ✗ Missing assertions
- ✗ Assertion roulette (many assertions)

## Output Format

### Summary
- **Overall Score**: [1-10]
- **Test Maturity**: [Beginner/Intermediate/Advanced]
- **Coverage**: [Poor/Fair/Good/Excellent]
- **Maintainability**: [Poor/Fair/Good/Excellent]
- **Test Smells Found**: [count]

### Findings

#### [Severity]: [Smell Name]
**Location**: `TestClass.method:line`
**Issue**: [What's wrong]
**Impact**: [Why it matters]
**Example**: [Current code]
**Fix**: [How to improve]

### Coverage Analysis
- Happy paths: [✓/✗]
- Edge cases: [assessment]
- Error handling: [assessment]
- Missing scenarios: [list]

### Recommendations
1. [Specific improvement]
2. [Coverage gap to fill]
3. [Refactoring suggestion]

Provide specific, actionable feedback with code examples.
"""


def analyze_test_patterns(file_content, file_path):
    """Analyze test code for common patterns and smells."""
    smells = {
        'poor_naming': [],
        'no_assertions': [],
        'multiple_assertions': [],
        'sleep_calls': [],
        'test_duplication': [],
        'magic_numbers': []
    }

    lines = file_content.split('\n')

    # Track test methods
    test_methods = []
    current_test = None
    assertion_count = 0

    for line_num, line in enumerate(lines, 1):
        stripped = line.strip()

        # Detect test method start (various frameworks)
        if re.match(r'^\s*def\s+test_\w+', line):  # Python
            if current_test:
                if assertion_count == 0:
                    smells['no_assertions'].append(
                        f"{file_path}:{current_test['line']} - {current_test['name']}"
                    )
                elif assertion_count > 3:
                    smells['multiple_assertions'].append(
                        f"{file_path}:{current_test['line']} - {current_test['name']} ({assertion_count} assertions)"
                    )

            test_name = re.search(r'def\s+(test_\w+)', line).group(1)
            current_test = {'name': test_name, 'line': line_num}
            assertion_count = 0
            test_methods.append(test_name)

            # Check naming
            if len(test_name) < 10 or test_name.count('_') < 2:
                smells['poor_naming'].append(
                    f"{file_path}:{line_num} - '{test_name}' (too vague)"
                )

        # Detect assertions
        if re.search(r'assert|expect|should|verify', stripped, re.IGNORECASE):
            assertion_count += 1

        # Detect sleep/wait
        if re.search(r'sleep|wait|delay', stripped, re.IGNORECASE):
            smells['sleep_calls'].append(f"{file_path}:{line_num} - {stripped[:50]}")

        # Detect magic numbers in tests
        if current_test:
            numbers = re.findall(r'\b(\d{3,})\b', stripped)  # 3+ digit numbers
            if numbers and 'assert' not in stripped:
                smells['magic_numbers'].append(
                    f"{file_path}:{line_num} - {', '.join(numbers)}"
                )

    # Check for test duplication
    if test_methods:
        # Simple heuristic: very similar test names
        for i, name1 in enumerate(test_methods):
            for name2 in test_methods[i+1:]:
                base1 = name1.rsplit('_', 1)[0] if '_' in name1 else name1
                base2 = name2.rsplit('_', 1)[0] if '_' in name2 else name2
                if base1 == base2 and len(base1) > 10:
                    smells['test_duplication'].append(
                        f"{file_path} - Similar: {name1}, {name2}"
                    )
                    break

    return smells


def find_test_files(path: Path, patterns=None):
    """Find test files."""
    if patterns is None:
        patterns = ['*test*.py', '*_test.py', 'test_*.py', '*Test.java',
                   '*test.ts', '*spec.ts', '*_spec.rb']

    if path.is_file():
        return [path]

    files = []
    for pattern in patterns:
        files.extend(path.rglob(pattern))

    return sorted(set(files))


def read_code_file(file_path: Path):
    """Read file content."""
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            return f.read()
    except Exception as e:
        return f"Error reading: {e}"


def generate_review_prompt(files_content: dict, include_analysis: bool):
    """Generate review prompt."""
    prompt = AGENT_PROMPT + "\n\n## Test Code to Review\n\n"

    for file_path, content in files_content.items():
        prompt += f"### File: {file_path}\n\n"
        prompt += f"```\n{content}\n```\n\n"

    if include_analysis:
        prompt += "\n## Static Analysis Results\n\n"
        for file_path, content in files_content.items():
            smells = analyze_test_patterns(content, file_path)
            if any(smells.values()):
                prompt += f"### {file_path} - Detected Patterns\n\n"
                for smell_type, instances in smells.items():
                    if instances:
                        prompt += f"**{smell_type.replace('_', ' ').title()}**:\n"
                        for instance in instances[:5]:  # Limit examples
                            prompt += f"- {instance}\n"
                        prompt += "\n"

    return prompt


def main():
    parser = argparse.ArgumentParser(
        description='Review test code for quality and best practices'
    )
    parser.add_argument('path', help='Test file or directory to review')
    parser.add_argument('--output', '-o', help='Output file for report')
    parser.add_argument('--patterns',
                       default='*test*.py,test_*.py,*Test.java,*test.ts,*spec.ts',
                       help='Test file patterns (comma-separated)')
    parser.add_argument('--max-files', type=int, default=10,
                       help='Maximum files to review')
    parser.add_argument('--no-analysis', action='store_true',
                       help='Skip static analysis')

    args = parser.parse_args()

    path = Path(args.path)
    if not path.exists():
        print(f"Error: '{args.path}' not found", file=sys.stderr)
        sys.exit(1)

    # Find test files
    patterns = [p.strip() for p in args.patterns.split(',')]
    files = find_test_files(path, patterns)

    if not files:
        print(f"No test files found in '{args.path}'", file=sys.stderr)
        print(f"Patterns used: {patterns}")
        sys.exit(1)

    if len(files) > args.max_files:
        print(f"Found {len(files)} test files. Reviewing first {args.max_files}.")
        files = files[:args.max_files]
    else:
        print(f"Found {len(files)} test file(s) to review.")

    # Read files
    files_content = {}
    for file_path in files:
        content = read_code_file(file_path)
        files_content[str(file_path)] = content

    # Generate prompt
    review_prompt = generate_review_prompt(
        files_content,
        include_analysis=not args.no_analysis
    )

    # Output
    output_content = f"""# Test Quality Review Request
Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
Path: {args.path}
Test Files: {len(files)}

---

{review_prompt}

---

## Instructions for Claude

Please review the test code above for quality and best practices.
Focus on test structure, coverage, maintainability, and common test smells.
Provide specific suggestions with before/after examples.
"""

    if args.output:
        output_path = Path(args.output)
        with open(output_path, 'w', encoding='utf-8') as f:
            f.write(output_content)
        print(f"\n✓ Review prompt saved to: {output_path}")
        print(f"\nNext: Send to Claude for review")
    else:
        print("\n" + "="*80)
        print(output_content)
        print("="*80)

    print(f"\n📋 Test files reviewed:")
    for file_path in files:
        print(f"  - {file_path}")


if __name__ == '__main__':
    main()