[git] commit-tracking program launching script with runtime code isolation

launch.sh

#!/bin/bash
set -e

# parse arguments
POSITIONAL_ARGS=()

while [[ $# -gt 0 ]]; do
  case $1 in
    -R|--restart)  # fresh restart of a job with the current commit (previous work will be lost)
      RESTART=Y
      shift
      ;;
    -r|--resume)   # restart a prematurely terminated job
      RESUME=Y
      shift
      ;;
    -e|--restore)  # restore code snapshot according to the tracked commit ID, and then restart
      RESTORE=Y
      shift
      ;;
    -i|--ignore)   # ignore new update since last commit
      IGNORE=Y
      shift
      ;;
    -n|--no-cleanup)
      NO_CLEANUP=Y
      shift
      ;;
    -o|--output_root)
      OUTPUT_ROOT="$2"
      shift
      shift
      ;;
    -u|--upload_root)
      UPLOAD_ROOT="$2"
      shift
      shift
      ;;
    -*|--*)
      echo "Unknown option $1"
      exit 1
      ;;
    *)
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift # past argument
      ;;
  esac
done

set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters

JOB_NAME=$1
EXTRA_JOB_ARGS=${@:2} # NOTE: don't put job-specific arguments here since EXTRA_JOB_ARGS won't be tracked!

# validate arguments
temp="$RESTART$RESUME$RESTORE"
num_y=${#temp}
if [ $num_y -gt 1 ]; then
    echo "[ERROR] -R, -r, and -e are mutually exclusive"
    exit 1
fi

# construct useful paths
REPO=$PWD
JOB_DIR=$REPO/runs/$JOB_NAME
TRACK_DIR=$JOB_DIR/.track
REPO_SNAPSHOT_PATH=$TRACK_DIR/repo_snapshot
HISTORY=$TRACK_DIR/history
if [ -z "$OUTPUT_ROOT" ]; then
    OUTPUT_DIR=$JOB_DIR/output
else
    OUTPUT_DIR=$OUTPUT_ROOT/$JOB_NAME/output
    OUTPUT_DIR_SOFT_LINK=$JOB_DIR/output
fi
START_SCRIPT=$JOB_DIR/start.sh   # the script must accept $OUTPUT_DIR as its first argument

# run some checks
if ! $(git rev-parse --is-inside-work-tree); then
    echo "[ERROR] not inside a git repository"
    exit 1
fi
if ! [ -f $START_SCRIPT ]; then
    echo "[ERROR] job not found"
    exit 1
fi
if [ -n "$UPLOAD_ROOT" ]; then
    if [ x$(type -t upload) != xfunction ]; then
        echo "[ERROR] upload function is not defined"
        exit 1
    fi
fi

# if -R|--restart, delete all previous work after user confirmation
if [ "$RESTART" = "Y" ]; then
    read -p "Previous work will be lost after restart. Continue? (y/n) " -n 1 -r
    echo
    if [[ $REPLY =~ ^[Yy]$ ]]
    then
        echo "[INFO] work restarting"
        rm -rf $OUTPUT_DIR
        rm -rf $OUTPUT_DIR_SOFT_LINK
        rm -rf $REPO_SNAPSHOT_PATH
        rm -f $TRACK_DIR/commit*
    else
        exit 0
    fi
fi

# create/restore track and code snapshot
if [ "$RESUME" = "Y" ]; then
    echo "[INFO] work resuming"
else
    if [ "$RESTORE" = "Y" ]; then
        rm -rf $REPO_SNAPSHOT_PATH
        git clone -s . $REPO_SNAPSHOT_PATH
        cd $REPO_SNAPSHOT_PATH
        git checkout $(cat $TRACK_DIR/commit)
        cd $REPO
    else
        # commit changes (if any)
        # note: changes in untracked files are ignored by diff-index
        if [ "$IGNORE" != "Y" ]; then
            git diff-index --quiet HEAD || git commit --interactive
        fi

        # set up environment
        if [ -z "$OUTPUT_DIR_SOFT_LINK" ]; then
            mkdir $OUTPUT_DIR
        else
            mkdir -p $OUTPUT_DIR
            ln -s $OUTPUT_DIR $OUTPUT_DIR_SOFT_LINK
        fi
        mkdir -p $TRACK_DIR
        echo $(git rev-parse HEAD) > $TRACK_DIR/commit
        git clone -s . $REPO_SNAPSHOT_PATH  # -s saves disk space
    fi
    # clone submodules (if any)
    clone='git clone -s -c protocol.file.allow=always . '$REPO_SNAPSHOT_PATH'/clone/$path'
    git submodule foreach "$clone"
    record_commit='echo $(git rev-parse HEAD)'" > $TRACK_DIR/commit_"'$(basename $PWD)'
    git submodule foreach "$record_commit"
fi

# mark the current run in the history
RUN_DIR=$HISTORY/$(date +%s)
mkdir -p $RUN_DIR
cp $START_SCRIPT $JOB_DIR/*.json $RUN_DIR
START_ARGS="$OUTPUT_DIR $EXTRA_JOB_ARGS"
echo $START_ARGS > $RUN_DIR/args.txt
cp $TRACK_DIR/commit* $RUN_DIR

# start program
cd $REPO_SNAPSHOT_PATH
(set -o pipefail && sh -e $START_SCRIPT $START_ARGS |& tee $RUN_DIR/log.txt)
echo "[INFO] work completed"
touch $RUN_DIR/done

# clean up
if [ "$NO_CLEANUP" != "Y" ]; then
    rm -rf $TRACK_DIR/repo_snapshot
    echo "[INFO] cleanup completed"
fi

# upload the current run using the `upload` function pre-defined by the user
if [ -n "$UPLOAD_ROOT" ]; then
    upload $JOB_DIR $UPLOAD_ROOT/$JOB_NAME
    if [ -n "$OUTPUT_DIR_SOFT_LINK" ]; then
        upload $OUTPUT_DIR $UPLOAD_ROOT/$JOB_NAME/output
    fi
fi