Script that reads agents-docs files and creates a Vercel-style compressed index

generate-agents-docs.sh

#!/bin/bash
# Generates a compressed docs index for AI agents from .docs folder
# See: https://vercel.com/blog/agents-md-outperforms-skills-in-our-agent-evals
#
# Usage: ./scripts/generate-agents-docs.sh [--preview]
#
# Pro tip: add to your `.husky/pre-push`, `.husky/post-merge`, `.husky/post-checkout` hooks.
#   >  echo "💽 (operation) Re-generate the docs index in AGENTS.md"
#   >  ./scripts/generate-agents-docs.sh

set -e

DOCS_DIR="./.agents-docs"
AGENTS_MD="AGENTS.md"
PREVIEW_ONLY=false
START_MARKER="<!-- AGENTS-DOCS-START -->"
END_MARKER="<!-- AGENTS-DOCS-END -->"

if [ "$1" = "--preview" ]; then
    PREVIEW_ONLY=true
fi

if [ ! -d "$DOCS_DIR" ]; then
    echo "Error: $DOCS_DIR directory not found" >&2
    exit 1
fi

# Get all directories containing .md files, sorted by depth (parent before child)
dirs=$(find "$DOCS_DIR" -name "*.md" -exec dirname {} \; | sort -u | awk -F'/' '{print NF-1, $0}' | sort -n | cut -d' ' -f2-)

# Build index content
INDEX="[Docs Index]|root: $DOCS_DIR"
INDEX+="|IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning"

for dir in $dirs; do
    rel_path="${dir#"$DOCS_DIR"/}"

    if [ "$rel_path" = "$DOCS_DIR" ]; then
        rel_path=""
    fi

    files=$(find "$dir" -maxdepth 1 -name "*.md" -type f -exec basename {} \; | sort | tr '\n' ',' | sed 's/,$//')

    if [ -z "$files" ]; then
        continue
    fi

    if [ -z "$rel_path" ] || [ "$rel_path" = "$DOCS_DIR" ]; then
        INDEX+="|{$files}"
    else
        INDEX+="|$rel_path:{$files}"
    fi
done

# Wrap in markers (single line)
BLOCK="${START_MARKER}${INDEX}${END_MARKER}"

if [ "$PREVIEW_ONLY" = true ]; then
    echo -n "$BLOCK"
    echo  # Add trailing newline for terminal display
    exit 0
fi

# Update CLAUDE.md
if [ ! -f "$AGENTS_MD" ]; then
    echo "Error: $AGENTS_MD not found" >&2
    exit 1
fi

# Check if markers exist
if grep -q "$START_MARKER" "$AGENTS_MD" && grep -q "$END_MARKER" "$AGENTS_MD"; then
    # Replace content between markers using temp files (awk -v can't handle multiline)
    block_file=$(mktemp)
    temp_file=$(mktemp)
    echo "$BLOCK" > "$block_file"
    awk -v start="$START_MARKER" -v end="$END_MARKER" -v blockfile="$block_file" '
        BEGIN { printing=1 }
        $0 ~ start { printing=0; while((getline line < blockfile) > 0) print line; next }
        $0 ~ end { printing=1; next }
        printing { print }
    ' "$AGENTS_MD" > "$temp_file"
    mv "$temp_file" "$AGENTS_MD"
    rm "$block_file"
    echo "Updated docs index in $AGENTS_MD"
else
    echo "" >> "$AGENTS_MD"
    echo "$BLOCK" >> "$AGENTS_MD"
    echo "Appended docs index to $AGENTS_MD"
fi

@Kingdutch thank you, will integrate that for sure! I also discovered TOON rencetly and will start using it from now on https://github.com/toon-format/toon