bootstrap-devops-agent-disk.sh

#!/usr/bin/env bash
set -euo pipefail

LOG="/var/log/bootstrap-devops-agent-disk.log"
exec > >(tee -a "$LOG") 2>&1

echo "===================================================="
echo "Bootstrap DevOps Agent Disk Start: $(date -Is)"
echo "===================================================="

# ====== CONFIG ======
MOUNTPOINT="/mnt/agentdisk"
WORKDIR="${MOUNTPOINT}/_work"
DOCKERROOT="${MOUNTPOINT}/docker"
DIAGROOT="${MOUNTPOINT}/agent_diag"

# Set to 1 ONLY if you want to reformat even when a filesystem exists (DANGEROUS)
FORCE_FORMAT="${FORCE_FORMAT:-0}"

# ====== Helpers ======
log() { echo "[$(date -Is)] $*"; }

require_root() {
  if [ "$(id -u)" -ne 0 ]; then
    log "ERROR: Run as root."
    exit 1
  fi
}

command_exists() { command -v "$1" >/dev/null 2>&1; }

# Identify root device (partition) backing /
get_root_source() {
  findmnt -n -o SOURCE /
}

# Given /dev/sda1 -> /dev/sda; /dev/nvme0n1p2 -> /dev/nvme0n1
get_parent_disk() {
  local src="$1"
  lsblk -no PKNAME "$src" 2>/dev/null | awk '{print "/dev/"$1}'
}

# Find first non-OS disk (type disk, not loop/rom, not the OS parent)
find_data_disk() {
  local os_disk="$1"
  # List real disks only
  # Example output: /dev/sda /dev/sdb /dev/sdc
  local disks
  disks=$(lsblk -dpno NAME,TYPE | awk '$2=="disk"{print $1}')

  # Pick first disk that's not the OS disk
  local d
  for d in $disks; do
    if [ "$d" != "$os_disk" ]; then
      echo "$d"
      return 0
    fi
  done

  return 1
}

# Create a single partition if none exists
ensure_partition() {
  local disk="$1"

  # If a partition already exists, use the first one
  local part
  part=$(lsblk -lnpo NAME,TYPE "$disk" | awk '$2=="part"{print $1}' | head -n 1 || true)
  if [ -n "${part:-}" ]; then
    echo "$part"
    return 0
  fi

  log "No partition found on $disk. Creating GPT + single partition..."
  if ! command_exists parted; then
    apt-get update
    apt-get install -y parted
  fi

  parted -s "$disk" mklabel gpt
  parted -s -a optimal "$disk" mkpart primary ext4 0% 100%

  # Wait for kernel to pick up new partition
  partprobe "$disk" || true
  sleep 2

  part=$(lsblk -lnpo NAME,TYPE "$disk" | awk '$2=="part"{print $1}' | head -n 1)
  if [ -z "${part:-}" ]; then
    log "ERROR: Partition creation failed on $disk."
    exit 1
  fi

  echo "$part"
}

# Format if needed
ensure_filesystem_ext4() {
  local part="$1"

  local fs
  fs=$(blkid -o value -s TYPE "$part" 2>/dev/null || true)

  if [ "${FORCE_FORMAT}" = "1" ]; then
    log "FORCE_FORMAT=1. Formatting $part as ext4 (DESTRUCTIVE)..."
    mkfs.ext4 -F "$part"
    return 0
  fi

  if [ -n "${fs:-}" ]; then
    log "$part already has filesystem: $fs. Skipping format."
    return 0
  fi

  log "$part has no filesystem. Formatting as ext4..."
  mkfs.ext4 -F "$part"
}

ensure_mount() {
  local part="$1"
  mkdir -p "$MOUNTPOINT"

  local uuid
  uuid=$(blkid -s UUID -o value "$part")

  # Add to fstab if not present
  if ! grep -q "$uuid" /etc/fstab; then
    log "Adding fstab entry for $part (UUID=$uuid) -> $MOUNTPOINT"
    echo "UUID=$uuid $MOUNTPOINT ext4 defaults,nofail 0 2" >> /etc/fstab
  else
    log "fstab already contains UUID=$uuid"
  fi

  log "Mounting all..."
  mount -a

  # Verify mounted
  if ! findmnt -n "$MOUNTPOINT" >/dev/null 2>&1; then
    log "ERROR: $MOUNTPOINT is not mounted after mount -a."
    exit 1
  fi

  log "Mounted $part at $MOUNTPOINT"
}

set_vsts_work_env() {
  log "Setting VSTS_AGENT_INPUT_WORK=$WORKDIR"
  if grep -q "^VSTS_AGENT_INPUT_WORK=" /etc/environment 2>/dev/null; then
    sed -i "s|^VSTS_AGENT_INPUT_WORK=.*|VSTS_AGENT_INPUT_WORK=$WORKDIR|g" /etc/environment
  else
    echo "VSTS_AGENT_INPUT_WORK=$WORKDIR" >> /etc/environment
  fi
}

configure_docker_data_root() {
  if ! command_exists docker; then
    log "Docker not found. Skipping docker root relocation (you can still preinstall Docker in the image)."
    return 0
  fi

  log "Configuring Docker data-root -> $DOCKERROOT"
  mkdir -p /etc/docker
  mkdir -p "$DOCKERROOT"

  cat >/etc/docker/daemon.json <<EOF
{
  "data-root": "$DOCKERROOT"
}
EOF

  # Try stop/start docker service if systemd has it
  if systemctl list-unit-files | grep -q "^docker.service"; then
    log "Stopping docker..."
    systemctl stop docker || true

    # Migrate old data if exists and dockerroot is empty
    if [ -d /var/lib/docker ] && [ ! -L /var/lib/docker ]; then
      if [ -z "$(ls -A "$DOCKERROOT" 2>/dev/null || true)" ]; then
        log "Migrating /var/lib/docker -> $DOCKERROOT (best effort)"
        rsync -aHAX --numeric-ids /var/lib/docker/ "$DOCKERROOT/" || true
      else
        log "$DOCKERROOT is not empty; skipping migration to avoid overwriting."
      fi
    fi

    log "Starting docker..."
    systemctl daemon-reload || true
    systemctl start docker || true

    log "Docker Root Dir:"
    docker info 2>/dev/null | sed -n 's/^ *Docker Root Dir: */Docker Root Dir: /p' || true
  else
    log "docker.service not found in systemd. Skipping service restart."
  fi
}

redirect_agent_diag() {
  # This directly targets your error: /agent/_diag/*.log no space
  if [ ! -d /agent ]; then
    log "/agent does not exist on this VM. Skipping /agent/_diag redirection."
    return 0
  fi

  mkdir -p "$DIAGROOT"
  chmod 777 "$DIAGROOT"

  if [ -L /agent/_diag ]; then
    log "/agent/_diag is already a symlink. Leaving as-is."
    return 0
  fi

  if [ -d /agent/_diag ]; then
    log "Migrating /agent/_diag -> $DIAGROOT (best effort)"
    rsync -aHAX --numeric-ids /agent/_diag/ "$DIAGROOT/" || true
    rm -rf /agent/_diag
  fi

  ln -s "$DIAGROOT" /agent/_diag
  log "Symlinked /agent/_diag -> $DIAGROOT"
}

restart_agent_services() {
  log "Restarting agent services if present..."
  systemctl daemon-reload || true
  systemctl restart "vsts.agent*" 2>/dev/null || true
  systemctl restart "azure-pipelines-agent*" 2>/dev/null || true
}

# ====== Main ======
require_root

log "Disk / mount snapshot (before):"
df -h || true
lsblk || true

ROOT_SRC="$(get_root_source)"
OS_DISK="$(get_parent_disk "$ROOT_SRC")"

log "Root source: $ROOT_SRC"
log "OS disk parent: $OS_DISK"

DATA_DISK="$(find_data_disk "$OS_DISK" || true)"
if [ -z "${DATA_DISK:-}" ]; then
  log "ERROR: Could not find a non-OS disk to use as data disk."
  exit 1
fi

log "Selected data disk: $DATA_DISK"

DATA_PART="$(ensure_partition "$DATA_DISK")"
log "Using data partition: $DATA_PART"

ensure_filesystem_ext4 "$DATA_PART"
ensure_mount "$DATA_PART"

# Create folders
log "Creating folders on $MOUNTPOINT"
mkdir -p "$WORKDIR" "$DOCKERROOT" "$DIAGROOT"
chmod 777 "$WORKDIR" "$DOCKERROOT" "$DIAGROOT"

set_vsts_work_env
configure_docker_data_root
redirect_agent_diag
restart_agent_services

log "Disk / mount snapshot (after):"
df -h || true
lsblk || true

log "Verify env:"
grep "^VSTS_AGENT_INPUT_WORK=" /etc/environment || true

log "Bootstrap complete: $(date -Is)"