Mount S3 as a directory using GeeseFS

mount-s3.sh

#!/bin/bash
set -e

INSTALLS=""
if command -v fio >/dev/null 2>&1; then
  echo "fio already installed"
else
  INSTALLS="$INSTALL fio"
fi

if command -v fusermount3 >/dev/null 2>&1; then
  echo "fuse3 already installed"
else
  INSTALLS="$INSTALLS fuse3"
fi

if [ "$INSTALLS" != "" ]; then
  echo "Installing fio and fuse3"
  DEBIAN_FRONTEND=noninteractive sudo apt install -y fio fuse3
fi

if command -v go >/dev/null 2>&1; then
  echo "Go is already installed"
else 
  echo "Installing Go"
  export GO_VERSION=1.26.0
  export TARGETARCH=amd64
  curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz | sudo tar -C /usr/local -xzf -
  echo 'export PATH="/usr/local/go/bin:/usr/local/bin:$PATH"' >> ~/.bashrc
  export PATH="/usr/local/go/bin:/usr/local/bin:$PATH"
fi

if command -v geesefs >/dev/null 2>&1; then
  echo "Geesefs is already installed"
else 
  echo "Installing geesefs"
  if [ ! -d geesefs ]; then
    git clone https://github.com/yandex-cloud/geesefs
  fi

  (cd geesefs && go build && sudo mv geesefs /usr/local/bin)
fi

S3_ENDPOINT=${S3_ENDPOINT:-"https://s3.${AWS_REGION:-"eu-west-1"}.amazonaws.com"}
BUCKET_NAME=${BUCKET_NAME:-"text-bucket"}
BUCKET_PATH=${BUCKET_PATH:-"test"}
MOUNT_POINT=${MOUNT_POINT:-"/mnt/data"}
#GEESEFS_OPTIONS="--debug_s3 --debug_fuse --log-file fuse.log"
GEESEFS_OPTIONS=${GEESEFS_OPTIONS:-"--ignore-fsync --no-checksum --multipart-age 4h --memory-limit 1024 --entry-limit 80000 --max-flushers 8 --max-parallel-parts 8 --part-sizes 25"}

if [ "${AWS_REGION}" != "" ]; then
  GEESEFS_OPTIONS="--region ${AWS_REGION} ${GEESEFS_OPTIONS}"
fi

# Fix fuse.conf to allow other users to mount the filesystem
if [[ "${GEESEFS_OPTIONS}" =~ "allow_other" ]]; then
  echo "Patching /etc/fuse.conf to allow other users to mount the filesystem"
  sudo sed -i -e "/^#allow_other/s/^#//" /etc/fuse.conf
fi

echo "Mounting ${BUCKET_NAME} path ${BUCKET_PATH} to ${MOUNT_POINT}"

if [ ! -d ${MOUNT_POINT} ]; then 
  sudo mkdir -p ${MOUNT_POINT}
  sudo chown $USER:$USER ${MOUNT_POINT}
fi

MOUNTED=$(findmnt -n -o SOURCE -M ${MOUNT_POINT} || true)

if [ ! -z "${MOUNTED}" ]; then
  echo "Unmounting ${MOUNT_POINT}, currently mounted as ${MOUNTED}"
  sudo umount -l ${MOUNT_POINT} || true
  sudo umount -f ${MOUNT_POINT} || true
fi

geesefs --version
echo "Mounting via geesefs"
geesefs ${GEESEFS_OPTIONS:-""} --subdomain \
  --endpoint "${S3_ENDPOINT}" "${BUCKET_NAME}:${BUCKET_PATH}" "${MOUNT_POINT}"

echo "Sanity check via fio"
(
  cd ${MOUNT_POINT} &&
  ls -la &&
  fio -name=fio -ioengine=libaio -direct=1 -bs=2M -iodepth=1 -fallocate=none -numjobs=1 -group_reporting -rw=write -size=64M -minimal &&
  rm -f fio.*
)

To mount, set env vars and run the script

export AWS_ACCESS_KEY_ID="zzz"
export AWS_SECRET_ACCESS_KEY="zzz"
export AWS_REGION=eu-west-1
# export S3_ENDPOINT="http://custom"
export BUCKET_NAME="mybucket"
export BUCKET_PATH="bucketpath"
export MOUNT_POINT="/home/ubuntu/data"
curl -fsSL https://gist.githubusercontent.com/huksley/44341276d7c269f092e10784959e8555/raw/s3-mount.sh | bash -

Disable some git features which might now work well with NFS/S3 backend FS

git config --global pack.threads 1
echo "git config --global pack.threads 1" >> ~/.bashrc