netmount_unmounter.sh

netmount_remounter.sh

#!/bin/bash
#
# NFS Unhanger
#
# Takes mounted filesystems from /proc/mounts
# Tries to df them in background
# If cannot in $TIMEOUT secs, kill and lazy unmount and try to remount
# If cannot remount, add to failed list
# Finally, retry re-mounting previously failed mounts, again with timeout
# Fin
#
# Run on cron, or something
#
# 2025-09-08: Paul Reece
#   Initial write. VERY MUCH a work in progress, alpha quality code
#   has not been extensively tested yet....
#

# look for CephFS and NFS based mounts
MATCH_REGEX="(nfs|ceph)"
LOGFILE=/var/log/hung_nfs.log
TIMEOUT=5

MyMounts=`grep -P ${MATCH_REGEX} /proc/mounts | awk '{print $2}'`

# flock boilerplate from man page
# don't run many instances of the same cron
[ "${FLOCKER}" != "$0" ] && exec env FLOCKER="$0" flock -en "$0" "$0" "$@" || :


#
# Try to df all mountpoints in background
#

echo $MyMounts
for mount in ${MyMounts}; do
    echo $mount
    df -h ${mount} &
done

sleep ${TIMEOUT}

# needed to clear state
jobs -l


if [ `jobs -l | wc -l` -gt 0 ]; then
    echo ""
    echo "cleaning up mountpoints still doing df..."
    echo ""
    echo ""

    mypids=`jobs -p | tr '\n' '|'`
    if [ ${#mypids} > 3 ]; then
        HungMounts=`ps auxf | grep -P "(${mypids})\b" | awk -F "df -h" '{print $NF}' | awk '{print $1}' | grep '^/'`
    else
        HungMounts=''
    fi

    for pid in `jobs -p`; do kill -9 ${pid} ; done;
    sleep 1
    for pid in `jobs -p`; do kill -9 ${pid} ; done;


    for mount in ${HungMounts}; do
        if [ ${#mount} -gt 4 ] ; then
            echo -n "`date -Isec`: unmounting : ${mount} " | tee -a ${LOGFILE}
            mountline=`grep -P " ${mount}\b" /proc/mounts`
            echo ${mountline} >> /tmp/failedmounts.txt
            echo ${mountline} | tee -a ${LOGFILE}
            umount -l ${mount}
            mount ${mountline} &
        fi
    done

    sleep ${TIMEOUT}
fi

jobs -l

echo ""
echo "retrying old mountpoints from /tmp/failedmounts.txt, if any..."
echo ""

for mountpoint in `cat /tmp/failedmounts.txt | awk '{print $2}'` ; do
    grep -P " ${mountpoint}\b" /proc/mounts
    if [ $? -eq 0 ]; then
        echo "`date -Isec`: found mountpoint mounted OK for ${mountpoint}, removing from failed file" | tee -a ${LOGFILE}
        sed -i "\~\ ${mountpoint}\b~D" /tmp/failedmounts.txt
    else
        mountline=`grep -P " ${mountpoint}\b" /tmp/failedmounts.txt | awk '{print $1 " " $2 " -t " $3 " -o " $4}'`
        grep -P "${mountpoint}\b" /etc/fstab
        if [ $? -eq 0 ]; then
            mount -v ${mountpoint} &
        else
            mount ${mountline} &
        fi
    fi
done

sleep ${TIMEOUT}

echo ""
echo "cleaning up..."
echo ""

jobs -l

for pid in `jobs -p`; do kill -9 ${pid} ; done;
sleep 1
for pid in `jobs -p`; do kill -9 ${pid} ; done;

killall -9 mount.nfs
killall -9 mount.nfs4
killall -9 mount.cifs
killall -9 mount.ceph
killall -9 mount.fuse.ceph