ZFS Operational Prometheus node_exporter Stats

crontab-example.txt

# Match to your prometheus node_exporter --collector.textfile.directory
# add "sponge" commmand inline if needed, probably isn't here

# Replace pool0 with the pool you wish to monitor

* * * * * root /opt/zfs_space_used.sh pool0 > /var/tmp/node_exporter/zfs_space.prom
* * * * * root /opt/zpool_status_simple.sh pool0 > /var/tmp/node_exporter/zfs_status.prom

zfs_space_used.sh

#!/bin/bash

echo "# HELP node_zfs_custom_fs_used 100 bytes of space used"
echo "# TYPE node_zfs_custom_fs_used gauge"
zfs list -Hp -t filesystem -o name,referenced -r ${1} | awk '{print "node_zfs_custom_fs_used{zfs=\"" $1 "\"} " $2}'

echo "# HELP node_zfs_custom_fs_used_logical 100 bytes of space used"
echo "# TYPE node_zfs_custom_fs_used_logical gauge"
zfs list -Hp -t filesystem -o name,logicalreferenced -r ${1} | awk '{print "node_zfs_custom_fs_used_logical{zfs=\"" $1 "\"} " $2}'

zpool_status_simple.sh

#!/bin/bash
#
# usage: add cron job like:
#
# * * * * * root /opt/zpool_status_simple.sh pool0 > /var/lib/prometheus/node-exporter/zpool_status.prom
#
#

POOL=$1

STATE=`zpool status ${POOL} | grep "state:" | awk -F "state: " '{print $NF}'`
PERCENT_RESILVERED=`zpool status ${POOL} | grep "resilvered" | grep "% done" | awk '{print $3}' | awk -F "%" '{print $1}'`

echo "# HELP node_zfs_custom_percent_resilvered percent_resilvered, 100 is done"
echo "# TYPE node_zfs_custom_percent_resilvered gauge"

if [ $PERCENT_RESILVERED ] ; then
    echo "node_zfs_custom_percent_resilvered{zpool=\"${POOL}\"} ${PERCENT_RESILVERED}"
fi

# Googling around says the "right" way to do enum is like this:
# echo "node_zfs_custom_state{zpool=\"${POOL}\", state=\"${STATE}\"} 1"
#
# But I think this is dumb, idk, maybe I'm just dumb
#
# anyways, mystery-meat values work much better with
# state timeline in Grafana, regardless of how technically incorrect
# they may be
#
# So let's do it wrong and glitch our way to victory
#
# From: https://docs.oracle.com/cd/E19253-01/819-5461/gamno/index.html
#

echo "# HELP node_zfs_custom_state 1=ONLINE, 2=DEGRADED, 3=FAULTED, 4=OFFLINE, 5=UNAVAIL, 6=REMOVED, 255=UNKNOWN"
echo "# TYPE node_zfs_custom_state gauge"
case $STATE in
    ONLINE)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 1"
        ;;
    DEGRADED)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 2"
        ;;
    FAULTED)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 3"
        ;;
    OFFLINE)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 4"
        ;;
    UNAVAIL)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 5"
        ;;
    REMOVED)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 6"
        ;;
    *)
        echo "node_zfs_custom_state{zpool=\"${POOL}\"} 255"
        ;;
esac

echo "# HELP node_zfs_custom_fs_avail bytes available per zfs mount"
echo "# TYPE node_zfs_custom_fs_avail gauge"
zfs list -p | grep pool0 | awk '{print "node_zfs_custom_fs_avail\{zpool=\"" $1 "\"\} " $3}' 2> /dev/null