Drumknott NixOS installation script

install.sh

#!/usr/bin/env bash
#
# Drumknott NixOS Installation Script
# Run from NixOS installer console (not SSH - causes crashes with ZFS)
#
# Storage Layout:
#   - tank: HDD mirror (primary storage, boot)
#   - NVMe: L2ARC cache + metadata special vdev
#   - EFI: On all 4 disks for boot from any disk
#
set -euo pipefail

echo "=== Drumknott NixOS Installer ==="
echo ""

# --- Configuration ---
HOSTNAME="drumknott"
DOMAIN="lab.unix.ie"
IP_ADDRESS="10.0.1.107"
GATEWAY="10.0.1.1"
NETWORK_INTERFACE="enp6s0"
SSH_PUBKEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJQEbUSPjTeZ6LV0JPZ82itJ8nINS3yxEMXUXqjvqOnY andre@sachs.ie"

# --- Discover disks ---
echo "=== Discovering disks ==="

# Find NVMe drives (for cache/metadata)
NVME_DISKS=($(lsblk -d -n -o NAME,TYPE | awk '$2=="disk" && $1~/^nvme/ {print "/dev/"$1}'))
echo "Found NVMe disks: ${NVME_DISKS[*]:-none}"

# Find SATA/SAS disks
SATA_DISKS=($(lsblk -d -n -o NAME,TYPE | awk '$2=="disk" && $1~/^sd[a-z]$/ {print "/dev/"$1}'))
echo "Found SATA disks: ${SATA_DISKS[*]:-none}"

# Exclude USB boot device
BOOT_DISK=$(mount | grep ' /iso ' | awk '{print $1}' | sed 's/[0-9]*$//' || true)
echo "Boot device (excluded): ${BOOT_DISK:-unknown}"

# Filter SATA disks - separate HDDs from SSDs by rotation
HDD_DISKS=()
SSD_DISKS=()
for disk in "${SATA_DISKS[@]:-}"; do
  if [[ -n "$disk" && "$disk" != "$BOOT_DISK" ]]; then
    rotational=$(cat /sys/block/$(basename $disk)/queue/rotational 2>/dev/null || echo "1")
    if [[ "$rotational" == "1" ]]; then
      HDD_DISKS+=("$disk")
    else
      SSD_DISKS+=("$disk")
    fi
  fi
done

echo "HDDs (for tank mirror): ${HDD_DISKS[*]:-none}"
echo "SATA SSDs (will wipe): ${SSD_DISKS[*]:-none}"
echo "NVMe (for cache/special): ${NVME_DISKS[*]:-none}"

# Validate disk counts
if [[ ${#HDD_DISKS[@]} -lt 2 ]]; then
  echo "ERROR: Expected at least 2 HDDs for mirror, found ${#HDD_DISKS[@]}"
  exit 1
fi

if [[ ${#NVME_DISKS[@]} -lt 1 ]]; then
  echo "ERROR: Expected at least 1 NVMe for cache, found ${#NVME_DISKS[@]}"
  exit 1
fi

echo ""
echo "Storage Layout:"
echo "  tank (mirror): ${HDD_DISKS[0]} + ${HDD_DISKS[1]}"
echo "  L2ARC cache:   ${NVME_DISKS[*]}"
echo "  EFI partitions: all disks"
echo ""
read -p "Press Enter to continue or Ctrl+C to abort..."

# --- Wipe all disks ---
echo ""
echo "=== Wiping disks ==="

# Wipe SATA SSDs (Micron drives user wanted removed)
for disk in "${SSD_DISKS[@]:-}"; do
  if [[ -n "$disk" && -b "$disk" ]]; then
    echo "Wiping SATA SSD $disk..."
    wipefs -af "$disk" 2>/dev/null || true
    dd if=/dev/zero of="$disk" bs=1M count=10 status=none 2>/dev/null || true
  fi
done

# Wipe HDDs
for disk in "${HDD_DISKS[@]}"; do
  echo "Wiping HDD $disk..."
  wipefs -af "$disk" 2>/dev/null || true
  dd if=/dev/zero of="$disk" bs=1M count=10 status=none 2>/dev/null || true
done

# Wipe NVMe disks
for disk in "${NVME_DISKS[@]}"; do
  echo "Wiping NVMe $disk..."
  wipefs -af "$disk" 2>/dev/null || true
  dd if=/dev/zero of="$disk" bs=1M count=10 status=none || true
done

echo "All disks wiped."

# --- Partition HDDs ---
echo ""
echo "=== Partitioning HDDs ==="

for disk in "${HDD_DISKS[@]}"; do
  echo "Partitioning $disk..."
  parted -s "$disk" mklabel gpt
  parted -s "$disk" mkpart ESP fat32 1MiB 1GiB
  parted -s "$disk" set 1 esp on
  parted -s "$disk" mkpart primary 1GiB 100%
done

# --- Partition NVMe drives ---
echo ""
echo "=== Partitioning NVMe drives ==="

for disk in "${NVME_DISKS[@]}"; do
  echo "Partitioning $disk..."
  parted -s "$disk" mklabel gpt
  parted -s "$disk" mkpart ESP fat32 1MiB 1GiB
  parted -s "$disk" set 1 esp on
  parted -s "$disk" mkpart cache 1GiB 100%   # Rest for L2ARC/special
done

# Wait for partitions to appear
sleep 2
partprobe

# --- Format all EFI partitions ---
echo ""
echo "=== Formatting EFI partitions ==="

EFI_COUNT=0
ALL_BOOT_PATHS=()

for disk in "${HDD_DISKS[@]}"; do
  part="${disk}1"
  EFI_COUNT=$((EFI_COUNT + 1))
  echo "Formatting $part as ESP${EFI_COUNT}..."
  mkfs.fat -F32 -n "ESP${EFI_COUNT}" "$part"
  ALL_BOOT_PATHS+=("/boot${EFI_COUNT}")
done

for disk in "${NVME_DISKS[@]}"; do
  part="${disk}p1"
  EFI_COUNT=$((EFI_COUNT + 1))
  echo "Formatting $part as ESP${EFI_COUNT}..."
  mkfs.fat -F32 -n "ESP${EFI_COUNT}" "$part"
  ALL_BOOT_PATHS+=("/boot${EFI_COUNT}")
done

echo "Created ${EFI_COUNT} EFI partitions."

# --- Get hostId BEFORE creating pool ---
echo ""
echo "=== Capturing hostId ==="
HOSTID=$(head -c 8 /etc/machine-id)
echo "HostID: $HOSTID"

# --- Create ZFS pool on HDDs ---
echo ""
echo "=== Creating ZFS pool 'tank' on HDDs ==="

zpool create -f \
  -o ashift=12 \
  -o autotrim=on \
  -O mountpoint=none \
  -O acltype=posixacl \
  -O compression=zstd \
  -O dnodesize=auto \
  -O normalization=formD \
  -O relatime=on \
  -O xattr=sa \
  -R /mnt \
  tank mirror "${HDD_DISKS[0]}2" "${HDD_DISKS[1]}2"

echo "Pool 'tank' created on HDD mirror."

# --- Add NVMe as L2ARC cache ---
echo ""
echo "=== Adding NVMe as L2ARC cache ==="

CACHE_PARTS=()
for disk in "${NVME_DISKS[@]}"; do
  CACHE_PARTS+=("${disk}p2")
done

zpool add tank cache "${CACHE_PARTS[@]}"
echo "Added ${#CACHE_PARTS[@]} NVMe devices as L2ARC cache."

# --- Create datasets with mountpoint=legacy ---
echo ""
echo "=== Creating ZFS datasets ==="

zfs create -o mountpoint=legacy tank/nixos
zfs create -o mountpoint=legacy tank/nixos/root
zfs create -o mountpoint=legacy tank/nixos/home
zfs create -o mountpoint=legacy tank/nixos/nix
zfs create -o mountpoint=legacy tank/nixos/var
zfs create -o mountpoint=legacy tank/nixos/var/log

echo "Datasets created with mountpoint=legacy."

# --- Mount filesystems ---
echo ""
echo "=== Mounting filesystems ==="

mount -t zfs tank/nixos/root /mnt
mkdir -p /mnt/{home,nix,var/log}
mount -t zfs tank/nixos/home /mnt/home
mount -t zfs tank/nixos/nix /mnt/nix
mount -t zfs tank/nixos/var /mnt/var
mount -t zfs tank/nixos/var/log /mnt/var/log

# Mount all EFI partitions
EFI_IDX=0
for disk in "${HDD_DISKS[@]}"; do
  EFI_IDX=$((EFI_IDX + 1))
  mkdir -p "/mnt/boot${EFI_IDX}"
  mount "${disk}1" "/mnt/boot${EFI_IDX}"
done
for disk in "${NVME_DISKS[@]}"; do
  EFI_IDX=$((EFI_IDX + 1))
  mkdir -p "/mnt/boot${EFI_IDX}"
  mount "${disk}p1" "/mnt/boot${EFI_IDX}"
done

echo "Filesystems mounted."
df -h /mnt /mnt/boot*

# --- Generate hardware config ---
echo ""
echo "=== Generating NixOS configuration ==="

nixos-generate-config --root /mnt

# --- Build mirroredBoots config ---
MIRRORED_BOOTS=""
for path in "${ALL_BOOT_PATHS[@]}"; do
  MIRRORED_BOOTS="${MIRRORED_BOOTS}      { devices = [ \"nodev\" ]; path = \"${path}\"; }\n"
done

# --- Write configuration.nix ---
echo ""
echo "=== Writing configuration.nix ==="

cat > /mnt/etc/nixos/configuration.nix << NIXEOF
{ config, lib, pkgs, ... }:

{
  imports = [ ./hardware-configuration.nix ];

  system.stateVersion = "25.11";

  # Boot - GRUB on all EFI partitions for boot from any disk
  boot.loader.grub = {
    enable = true;
    efiSupport = true;
    efiInstallAsRemovable = true;
    copyKernels = true;
    mirroredBoots = [
$(echo -e "$MIRRORED_BOOTS")    ];
  };
  boot.supportedFilesystems = [ "zfs" ];

  # Network
  networking = {
    hostName = "${HOSTNAME}";
    domain = "${DOMAIN}";
    hostId = "${HOSTID}";
    useDHCP = false;
    interfaces.${NETWORK_INTERFACE}.ipv4.addresses = [{
      address = "${IP_ADDRESS}";
      prefixLength = 24;
    }];
    defaultGateway = "${GATEWAY}";
    nameservers = [ "${GATEWAY}" ];
    firewall.enable = true;
    firewall.allowedTCPPorts = [ 22 ];
  };

  # Timezone & Locale
  time.timeZone = "Europe/Dublin";
  i18n.defaultLocale = "en_IE.UTF-8";

  # Nix
  nix.settings = {
    experimental-features = [ "nix-command" "flakes" ];
    auto-optimise-store = true;
  };

  # Allow unfree (NVIDIA)
  nixpkgs.config.allowUnfree = true;
  hardware.enableRedistributableFirmware = true;

  # NVIDIA GPU
  hardware.graphics.enable = true;
  services.xserver.videoDrivers = [ "nvidia" ];
  hardware.nvidia = {
    open = false;
    modesetting.enable = true;
    nvidiaSettings = false;
    nvidiaPersistenced = true;
  };
  hardware.nvidia-container-toolkit.enable = true;

  # SSH
  services.openssh = {
    enable = true;
    settings = {
      PermitRootLogin = "prohibit-password";
      PasswordAuthentication = false;
    };
  };

  # Users
  users.users.root.openssh.authorizedKeys.keys = [
    "${SSH_PUBKEY}"
  ];

  users.users.andre = {
    isNormalUser = true;
    extraGroups = [ "wheel" ];
    openssh.authorizedKeys.keys = [
      "${SSH_PUBKEY}"
    ];
  };

  security.sudo.wheelNeedsPassword = false;

  # ZFS maintenance
  services.zfs.autoScrub.enable = true;
  services.zfs.trim.enable = true;

  # Packages
  environment.systemPackages = with pkgs; [
    vim git htop tmux curl wget jq pciutils usbutils smartmontools lm_sensors nvtopPackages.full
  ];
}
NIXEOF

echo "Configuration written."

# --- Show pool status ---
echo ""
echo "=== Pool Status ==="
zpool status tank

# --- Verify setup ---
echo ""
echo "=== Verification ==="
echo "HostID in config: $(grep hostId /mnt/etc/nixos/configuration.nix | grep -oE '[0-9a-f]{8}')"
echo "Installer hostId: $HOSTID"
echo ""
echo "ZFS datasets:"
zfs list -o name,mountpoint
echo ""
echo "Mounts:"
mount | grep /mnt

# --- Install ---
echo ""
echo "=== Installing NixOS ==="
read -p "Press Enter to start installation or Ctrl+C to abort..."

nixos-install --no-root-passwd

echo ""
echo "=== Installation complete! ==="
echo ""
echo "Storage layout:"
echo "  - tank: HDD mirror with NVMe L2ARC cache"
echo "  - Boot: EFI on all ${EFI_COUNT} disks"
echo ""
echo "Remove the USB drive and reboot."
echo ""