A shell script to build Chroma on USQCD clusters

chroma-build.sh

#!/usr/bin/env bash
# vim: ft=sh fdm=marker et sts=4 sw=4
#
# chroma-build.sh -- a shell script to build Chroma on USQCD clusters
#
# This is a script to build Chroma on the FNAL-USQCD clusters.
# The FNAL-USQCD network has multiple clusters with different processors
# and architectures.
#
# Basic Usage
# -----------
#
# To use this script, just set the host name of the target cluster
# either by setting the value of the $host variable in the script, or by
# setting it as an environment variable while calling the script:
#
#   host=<host> ./chroma-build.sh
#
# For most use cases, it suffices to edit the variables in the
# "Compilation options" section.
#
# SSH and Submodule Woes
# ----------------------
#
# Both QDP++ and Chroma are developed using Git with several Git
# submodules contained within the repository.  Some of these submodules
# contain other submodules.  Thus, it is important to recursively
# download these modules.
#
# Here comes a problem.  Chroma and QDP++ are hosted on GitHub, with
# git@github.com:... SSH repository URLs for the submodules.  If you've
# not set up SSH keys to use with GitHub [1,2], GitHub will not allow
# you to clone the repository.  For this script to work without any
# modification, you need to set up SSH keys with GitHub.
#
# Acknowledgments
# ---------------
#
# While writing and testing this script I benefited greatly from the
# build scripts written by Martin Ueding [3] and Evan Berkowitz [4].
#
#                                *****
#
# Written in 2019 by Manu Mannattil <manu.mannattil@gmail.com>
#
# [1]: https://help.github.com/en/articles/adding-a-new-ssh-key-to-your-github-account
# [2]: https://github.com/usqcd-software/qdpxx/pull/12
# [2]: https://github.com/HISKP-LQCD/chroma-auxiliary-scripts
# [3]: https://github.com/evanberkowitz/usqcd_install
#

# -----------------------------------------------------------------------------
# Compilation options
# -----------------------------------------------------------------------------

# Target host.  For more information see,
# https://www.usqcd.org/fnal/batch.html#resourcetypes
# https://www.usqcd.org/fnal/hardware.html
: ${host:=local}

# Directory to store Chroma/QDP++ builds and other libraries.
: ${build_dir:=$HOME/builds}

# Directory to store sources and compile them.
: ${source_dir:=$HOME/.cache/src/$host}

# Cache directory to store cloned repositories.
: ${source_cache:=$HOME/.cache/src/cache}

# Where to install stuff.
: ${prefix:=$build_dir/$host}

# Number of make jobs to run simultaneously.  This option will essentially be
# passed on to make as `make -j $make_jobs'.
: ${make_jobs:=$(grep ^processor /proc/cpuinfo | wc -l)}

# -----------------------------------------------------------------------------
# Build environment
# -----------------------------------------------------------------------------

# Bail out on the first error.
set -eu
set -o pipefail

mkdir -p "$prefix"
mkdir -p "$source_cache"
mkdir -p "$source_dir"

# Choose the microarchitecture based on the node we want to run Chroma on.
case "$host" in
    # 314-node cluster with dual-socket eight-core Intel E5-2650v2 "Ivy Bridge"
    # (2.6 GHz) processors and a quad-data-rate Infiniband fabric.
    pi) march="ivybridge" ;;

    # 224-node cluster with quad-socket eight-core Opteron 6320 (2.8 GHz)
    # processors and a quad-data-rate Infiniband fabric.
    # https://developer.amd.com/wordpress/media/2012/10/CompilerOptQuickRef-63004300.pdf
    bc) march="bdver2" ;;

    # 20-node cluster with dual-socket four-core 2.53 GHz Intel Xeon E5630
    # processors, two NVidia Tesla M2050 GPUs per node and a quad-data-rate
    # Infiniband fabric.
    exdsg) march="westmere" ;;

    # 196-node cluster with quad-socket eight-core AMD Opteron 6128 (2.0 GHz)
    # processors, 64 GB memory per node (2.0 GB/core), and a quad-data-rate
    # Infiniband fabric.
    # https://developer.amd.com/wordpress/media/2012/10/CompilerOptQuickRef-61004100.pdf
    ds) march="barcelona" ;;

    # For testing on lattice.fnal.gov.
    local) march="native" ;;

    # We can't proceed without knowing the target computer.
    *) echo >&2 "error: valid target host required; run as:"
       echo -e >&2 "\n  host=<host> ${0##/}"
       exit 1
esac

# Use lmod to set the environment.  Start by unloading all modules.
# NOTE: Instead of using lmod (i.e., module), one can also set the paths
# explicitly (see /usr/local).  That is very cumbersome however.
module purge

# Note that we shouldn't use OpenMPI 4.0.0 since it doesn't have the
# MPI_Type_struct function: https://github.com/open-mpi/ompi/issues/5920
module load gcc/8.1.0 mpi/openmpi/3.1.1 git/2.9.5

# Base flags used with all compilers.  (Actually, it's just libxml2 that uses
# these flags without the mpi flags.)
cflags="-O3 -std=c99 -march=$march"
cxxflags="-O3 -std=c++11 -march=$march"

# Flags included when using an OpenMPI compiler.
mpi_cflags="-fopenmp"
mpi_cxxflags="-fopenmp"

# -----------------------------------------------------------------------------
# Utility functions
# -----------------------------------------------------------------------------

color_info=$'\e[38;5;32m'
color_reset=$'\e[0m'

# Print an informational message.
info() {
    echo -e >&2 "${color_info}(*) ${0##*/}: $*${color_reset}"
}

# Like git clone, but caches the repository first.  Local Git mirrors can be
# set up using the `clone --mirror' option, but that doesn't support submodules.
git-cache-clone() {
    [[ -e "$source_dir/$2" ]] && {
        info "removing previous clone"
        rm -rf "$source_dir/$2"
    }

    pushd "$source_cache"
    if [[ -d "$source_cache/$2" ]]
    then
        info "cloning $2 using cached sources"
    else
        # Pass all options to git.  This way we can checkout a specific
        # branch using `-b <branch>'.
        git clone --recursive "$@"
    fi
    popd

    cp -r "$source_cache/$2" "$source_dir/$2"
}

# Download a compressed archive containing sources, but cache it first.
# Much like git-cache-clone, but for tarballs (and the like).
curl-cache-extract() {
    [[ -e "$source_dir/$2" ]] && {
        info "removing previous copy"
        rm -rf "$source_dir/$2"
    }

    pushd "$source_cache"
    if [[ -d "$source_cache/$2" ]]
    then
        info "copying $2 from cached sources"
    else
        local tar_opts="xvf"

        # Detect archive compression based on URL extension.
        case "$1" in
            *.bz2) tar_opts="${tar_opts}j"       ;;
            *.gz)  tar_opts="${tar_opts}z"       ;;
            *.xz)  tar_opts="${tar_opts}J"       ;;
            *.lz)  tar_opts="${tar_opts} --lzip" ;;
        esac
        mkdir -rp "$2"
        curl -q -#kL "$1" | tar $tar_opts - -C "$2" --strip-components=1
    fi
    popd

    cp -r "$source_cache/$2" "$source_dir/$2"
}

# Recursively autoreconfigure a repository.  This is required when the
# repository has multiple submodules (each containing other submodules).  We
# descent into the tree, and starting from the bottom, run autoreconf for each
# configure.ac file that's found.
recursive-autoreconf() {
    info "recursively autoreconfiguring $1"
    for configure_ac in $(find "$1" -type f -name configure.ac | sort | tac)
    do
        pushd "${configure_ac%/*}"
        autoreconf --force --install --verbose
        popd
    done
}

# Begin the build process.
pushd "$source_dir"

# -----------------------------------------------------------------------------
# libxml2
# -----------------------------------------------------------------------------

info "building libxml2"

# Use version 2.9.4 of libxml2.
git-cache-clone "https://gitlab.gnome.org/GNOME/libxml2.git" "libxml2" -b v2.9.4
pushd libxml2

./autogen.sh
./configure                            \
    --prefix="$prefix"                 \
    --with-output                      \
    --without-catalog                  \
    --without-docbook                  \
    --without-ftp                      \
    --without-history                  \
    --without-http                     \
    --without-iconv                    \
    --without-modules                  \
    --without-pattern                  \
    --without-python                   \
    --without-reader                   \
    --without-readline                 \
    --without-schemas                  \
    --without-schematron               \
    --without-threads                  \
    --without-writer                   \
    --without-xinclude                 \
    --without-xptr                     \
    --without-zlib                     \
    CFLAGS="$cflags"                   \
    CXXFLAGS="$cxxflags"

make -j "$make_jobs"
make install

popd
info "finished building libxml2"

# -----------------------------------------------------------------------------
# QMP
# -----------------------------------------------------------------------------

info "building QMP"

git-cache-clone "https://github.com/usqcd-software/qmp.git" "qmp"
recursive-autoreconf "qmp"
pushd "qmp"

./configure                            \
    --prefix="$prefix"                 \
    --with-qmp-comms-type=MPI          \
    CC="mpicc"                         \
    CFLAGS="$cflags $mpi_cflags"       \
    CXX="mpicxx"                       \
    CXXFLAGS="$cxxflags $mpi_cxxflags"

make -j "$make_jobs"
make install

popd
info "finished building QMP"

# -----------------------------------------------------------------------------
# QDP++
# -----------------------------------------------------------------------------

info "building QDP++"

git-cache-clone "https://github.com/usqcd-software/qdpxx.git" "qdpxx"
recursive-autoreconf "qdpxx"
pushd "qdpxx"

./configure                            \
    --prefix="$prefix"                 \
    --enable-openmp                    \
    --enable-parallel-arch="parscalar" \
    --enable-precision="double"        \
    --with-qmp="$prefix"               \
    CC="mpicc"                         \
    CFLAGS="$cflags $mpi_cflags"       \
    CXX="mpicxx"                       \
    CXXFLAGS="$cxxflags $mpi_cxxflags"

make -j "$make_jobs"
make install

popd
info "finished building QDP++"

# -----------------------------------------------------------------------------
# MDWF
# -----------------------------------------------------------------------------

# MDWF has newer versions [1].  But version 1.1.4 from Pochinski's MIT website
# is the only one that I could get to compile.  This also seems to be the only
# version distributed with C source files generated from .qa0 files.  For
# compiling newer versions, you will first have to get and compile #F [2] and
# then use it to compile qa0 [3].  You will then have to produce MDWF's
# C source files from the .qa0 files using the qa0 binary.  However, I couldn't
# get the final version to compile.  What am I doing wrong?
#
# [1]: https://usqcd.lns.mit.edu/gitweb/cgit.cgi/LHPC/Public/mdwf
# [2]: https://github.com/false-schemers/sharpF
# [3]: https://usqcd.lns.mit.edu/gitweb/cgit.cgi/LHPC/Public/qa0/

info "building MDWF"

curl-cache-extract "http://www.mit.edu/~avp/mdwf/mdwf-1.1.4.tar.bz2" "mdwf"
pushd "mdwf"

CFLAGS="-O3" ./configure \
    --target=c99-64      \
    --prefix="$prefix"   \
    --with-qmp="$prefix"

make -j "$make_jobs"
make install
popd

# -----------------------------------------------------------------------------
# Chroma
# -----------------------------------------------------------------------------

info "building Chroma"

git-cache-clone "https://github.com/JeffersonLab/chroma.git" "chroma"
recursive-autoreconf "chroma"
pushd "chroma"

./configure                            \
    --prefix="$prefix"                 \
    --enable-openmp                    \
    --with-qmp="$prefix"               \
    --with-qdp="$prefix"               \
    --with-mdwf="$prefix"              \
    CC="mpicc"                         \
    CFLAGS="$cflags $mpi_cflags"       \
    CXX="mpicxx"                       \
    CXXFLAGS="$cxxflags $mpi_cxxflags"

make -j "$make_jobs"
make install
popd

# ------------------------------------------------------------------------------
# Chroma utils
# ------------------------------------------------------------------------------

info "building Chroma utilities"

git-cache-clone "https://github.com/JeffersonLab/chroma_utils.git" "chroma-utils"
recursive-autoreconf "chroma-utils"
pushd "chroma-utils"

./configure                            \
    --prefix="$prefix"                 \
    CFLAGS="$cflags"                   \
    CXXFLAGS="$cxxflags"

make -j "$make_jobs"
make install

info "finished building Chroma for $host"
popd

# Write environment variables to a file.
cat >"$prefix/source.sh" <<EOF
module purge; module load gcc/8.1.0 mpi/openmpi/3.1.1
export PATH="$prefix/bin:\$PATH"
EOF

info "To set the environment source $prefix/source.sh."