BMad Agent Installation Test Script - Verifies all 12 agents are properly installed for GitHub Copilot

test-bmad-agents.sh

#!/bin/bash

# BMad Agent Installation Test Script
# Tests whether all 14 agents are properly installed for GitHub Copilot

set -e

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Expected agent count
# Note: Originally expected 14, but manifest contains 12 agents
# (1 core + 9 bmm + 2 bmad-custom)
EXPECTED_AGENT_COUNT=12

# Test results
TESTS_PASSED=0
TESTS_FAILED=0
WARNINGS=0

echo -e "${BLUE}================================================${NC}"
echo -e "${BLUE}  BMad Agent Installation Verification Test${NC}"
echo -e "${BLUE}================================================${NC}"
echo ""

# Function to print test result
pass_test() {
    echo -e "${GREEN}✓ PASS:${NC} $1"
    ((TESTS_PASSED++))
}

fail_test() {
    echo -e "${RED}✗ FAIL:${NC} $1"
    ((TESTS_FAILED++))
}

warn_test() {
    echo -e "${YELLOW}⚠ WARN:${NC} $1"
    ((WARNINGS++))
}

info() {
    echo -e "${BLUE}ℹ INFO:${NC} $1"
}

# Test 1: Check if .bmad directory exists
echo -e "\n${YELLOW}[Test 1]${NC} Checking if BMad is installed..."
if [ -d ".bmad" ]; then
    pass_test "BMad directory exists at .bmad/"
else
    fail_test "BMad directory not found - BMad is not installed"
    exit 1
fi

# Test 2: Check installation manifest
echo -e "\n${YELLOW}[Test 2]${NC} Checking installation manifest..."
if [ -f ".bmad/_cfg/manifest.yaml" ]; then
    pass_test "Installation manifest exists"

    # Check version
    VERSION=$(grep "version:" .bmad/_cfg/manifest.yaml | head -1 | awk '{print $2}')
    info "BMad version: $VERSION"

    # Check install date
    INSTALL_DATE=$(grep "installDate:" .bmad/_cfg/manifest.yaml | awk '{print $2}')
    info "Install date: $INSTALL_DATE"

    # Check modules
    info "Checking installed modules..."
    if grep -q "modules:" .bmad/_cfg/manifest.yaml; then
        MODULES=$(grep -A 5 "modules:" .bmad/_cfg/manifest.yaml | grep "^\s*-" | sed 's/^[[:space:]]*-[[:space:]]*//' | tr '\n' ', ' | sed 's/,$//')
        info "Installed modules: $MODULES"

        # Verify critical modules
        if grep -A 5 "modules:" .bmad/_cfg/manifest.yaml | grep -q "core"; then
            pass_test "Core module installed"
        else
            fail_test "Core module NOT installed"
        fi

        if grep -A 5 "modules:" .bmad/_cfg/manifest.yaml | grep -q "bmm"; then
            pass_test "BMM module installed"
        else
            fail_test "BMM module NOT installed (this would explain missing agents)"
        fi

        if grep -A 5 "modules:" .bmad/_cfg/manifest.yaml | grep -q "bmad-custom"; then
            pass_test "BMad-custom module installed"
        else
            warn_test "BMad-custom module NOT installed"
        fi
    fi

    # Check IDE configuration
    if grep -q "ides:" .bmad/_cfg/manifest.yaml; then
        IDES=$(grep -A 3 "ides:" .bmad/_cfg/manifest.yaml | grep "^\s*-" | sed 's/^[[:space:]]*-[[:space:]]*//' | tr '\n', ', ' | sed 's/,$//')
        info "Configured IDEs: $IDES"

        if grep -A 3 "ides:" .bmad/_cfg/manifest.yaml | grep -q "github-copilot"; then
            pass_test "GitHub Copilot IDE configured"
        else
            fail_test "GitHub Copilot IDE NOT configured"
        fi
    fi
else
    fail_test "Installation manifest not found"
fi

# Test 3: Check agent manifest
echo -e "\n${YELLOW}[Test 3]${NC} Checking agent manifest..."
if [ -f ".bmad/_cfg/agent-manifest.csv" ]; then
    pass_test "Agent manifest exists"

    # Count agents (subtract 1 for header)
    MANIFEST_AGENT_COUNT=$(($(wc -l < .bmad/_cfg/agent-manifest.csv) - 1))
    info "Agents in manifest: $MANIFEST_AGENT_COUNT"

    if [ "$MANIFEST_AGENT_COUNT" -eq "$EXPECTED_AGENT_COUNT" ]; then
        pass_test "Agent manifest contains all $EXPECTED_AGENT_COUNT agents"
    else
        fail_test "Agent manifest contains $MANIFEST_AGENT_COUNT agents, expected $EXPECTED_AGENT_COUNT"
    fi

    # List agent names
    info "Agents defined in manifest:"
    tail -n +2 .bmad/_cfg/agent-manifest.csv | cut -d',' -f1 | while read agent; do
        echo "  - $agent"
    done
else
    fail_test "Agent manifest not found"
fi

# Test 4: Check GitHub agents directory
echo -e "\n${YELLOW}[Test 4]${NC} Checking GitHub Copilot agent files..."
if [ -d ".github/agents" ]; then
    pass_test "GitHub agents directory exists"

    # Count agent files
    GITHUB_AGENT_COUNT=$(find .github/agents -name "*.agent.md" -type f | wc -l)
    info "GitHub agent files found: $GITHUB_AGENT_COUNT"

    if [ "$GITHUB_AGENT_COUNT" -eq "$EXPECTED_AGENT_COUNT" ]; then
        pass_test "All $EXPECTED_AGENT_COUNT agent files created in .github/agents/"
    elif [ "$GITHUB_AGENT_COUNT" -eq 1 ]; then
        fail_test "Only 1 agent file created (likely only bmad-master) - THIS IS THE BUG"
    else
        fail_test "$GITHUB_AGENT_COUNT agent files created, expected $EXPECTED_AGENT_COUNT"
    fi

    # List created agent files
    info "GitHub agent files:"
    find .github/agents -name "*.agent.md" -type f | sort | while read file; do
        basename "$file"
    done | sed 's/^/  - /'

    # Check if only bmad-master exists (the reported bug)
    if [ "$GITHUB_AGENT_COUNT" -eq 1 ]; then
        ONLY_FILE=$(find .github/agents -name "*.agent.md" -type f)
        if [[ "$ONLY_FILE" == *"bmad-master"* ]]; then
            fail_test "CONFIRMED BUG: Only bmad-master agent file exists"
        fi
    fi
else
    fail_test "GitHub agents directory not found - Copilot installation incomplete"
fi

# Test 5: Check source agent files
echo -e "\n${YELLOW}[Test 5]${NC} Checking source agent files..."

# Check core agents
if [ -d ".bmad/core/agents" ]; then
    CORE_AGENTS=$(find .bmad/core/agents -name "*.md" -type f | wc -l)
    info "Core agent files: $CORE_AGENTS"
    pass_test "Core agents directory exists"
else
    fail_test "Core agents directory not found"
fi

# Check BMM agents
if [ -d ".bmad/bmm/agents" ]; then
    BMM_AGENTS=$(find .bmad/bmm/agents -name "*.md" -type f | wc -l)
    info "BMM agent files: $BMM_AGENTS"
    pass_test "BMM agents directory exists"
else
    fail_test "BMM agents directory not found - this explains missing agents!"
fi

# Check custom agents
if [ -d ".bmad/bmad-custom/agents" ]; then
    CUSTOM_AGENTS=$(find .bmad/bmad-custom/agents -name "*.md" -type f | wc -l)
    info "Custom agent files: $CUSTOM_AGENTS"
    pass_test "Custom agents directory exists"
else
    warn_test "Custom agents directory not found"
fi

# Test 6: Verify specific expected agents
echo -e "\n${YELLOW}[Test 6]${NC} Verifying specific expected agents..."

EXPECTED_AGENTS=(
    "bmad-master"      # Core
    "analyst"          # BMM
    "architect"        # BMM
    "dev"              # BMM
    "pm"               # BMM
    "quick-flow-solo-dev"  # BMM
    "sm"               # BMM
    "tea"              # BMM
    "tech-writer"      # BMM
    "ux-designer"      # BMM
    "commit-poet"      # BMad-Custom
    "toolsmith"        # BMad-Custom
)

for agent in "${EXPECTED_AGENTS[@]}"; do
    if [ -f ".bmad/_cfg/agent-manifest.csv" ] && grep -q "^$agent," .bmad/_cfg/agent-manifest.csv; then
        # Agent in manifest, check if GitHub file exists
        GITHUB_FILE=$(find .github/agents -name "*$agent.agent.md" -type f 2>/dev/null | head -1)
        if [ -n "$GITHUB_FILE" ]; then
            pass_test "Agent '$agent' - manifest ✓ GitHub file ✓"
        else
            fail_test "Agent '$agent' - in manifest but NO GitHub file"
        fi
    else
        fail_test "Agent '$agent' - NOT in manifest"
    fi
done

# Test 7: Check VS Code settings
echo -e "\n${YELLOW}[Test 7]${NC} Checking VS Code configuration..."
if [ -f ".vscode/settings.json" ]; then
    pass_test "VS Code settings file exists"
    info "VS Code settings configured for Copilot integration"
else
    warn_test "VS Code settings file not found"
fi

# Test 8: Validate agent file structure
echo -e "\n${YELLOW}[Test 8]${NC} Validating agent file structure..."
if [ -d ".github/agents" ]; then
    INVALID_FILES=0
    for file in .github/agents/*.agent.md; do
        if [ -f "$file" ]; then
            # Check for frontmatter
            if grep -q "^---" "$file"; then
                # Check for agent-activation section
                if grep -q "<agent-activation" "$file"; then
                    continue
                else
                    warn_test "$(basename "$file") missing <agent-activation> section"
                    ((INVALID_FILES++))
                fi
            else
                warn_test "$(basename "$file") missing YAML frontmatter"
                ((INVALID_FILES++))
            fi
        fi
    done

    if [ "$INVALID_FILES" -eq 0 ]; then
        pass_test "All agent files have valid structure"
    else
        warn_test "$INVALID_FILES agent files have structural issues"
    fi
fi

# Summary
echo -e "\n${BLUE}================================================${NC}"
echo -e "${BLUE}  Test Summary${NC}"
echo -e "${BLUE}================================================${NC}"
echo -e "${GREEN}Tests Passed: $TESTS_PASSED${NC}"
echo -e "${RED}Tests Failed: $TESTS_FAILED${NC}"
echo -e "${YELLOW}Warnings: $WARNINGS${NC}"

echo ""
if [ "$TESTS_FAILED" -eq 0 ]; then
    echo -e "${GREEN}✓ All critical tests passed!${NC}"
    echo -e "${GREEN}All $EXPECTED_AGENT_COUNT agents are properly installed.${NC}"
    exit 0
else
    echo -e "${RED}✗ Some tests failed!${NC}"

    # Provide diagnostic suggestions
    echo -e "\n${YELLOW}Diagnostic Suggestions:${NC}"

    if [ "$GITHUB_AGENT_COUNT" -eq 1 ]; then
        echo -e "${YELLOW}→ Bug confirmed: Only bmad-master agent installed${NC}"
        echo -e "${YELLOW}→ Possible causes:${NC}"
        echo "  1. Installer only processed first agent from manifest"
        echo "  2. Loop/iteration error in GitHub Copilot handler"
        echo "  3. Module filtering issue (bmm/bmad-custom agents skipped)"
        echo "  4. Early exit/return in agent installation loop"
    fi

    if ! grep -A 5 "modules:" .bmad/_cfg/manifest.yaml 2>/dev/null | grep -q "bmm"; then
        echo -e "${YELLOW}→ BMM module not installed - this is the likely root cause${NC}"
        echo "  - BMM module contains 9 of the 12 expected agents"
        echo "  - Reinstall with BMM module enabled"
    fi

    echo ""
    echo -e "${YELLOW}Recommended actions:${NC}"
    echo "  1. Check installer logs for errors"
    echo "  2. Verify all modules selected during installation"
    echo "  3. Review .bmad/_cfg/manifest.yaml for missing modules"
    echo "  4. Check installer code: tools/cli/installers/lib/ide/github-copilot.js"
    echo "  5. Try reinstalling BMad with all modules selected"

    exit 1
fi