Migrate Nextcloud files from S3 object storage to filesystem

readme.md

About

This scripts allows to migrate the file structure of an existing Nextcloud installation from S3 object storage to the local filesystem.

Requirements

• Existing Nextcloud installation with a configured S3 Object Store
• jq
• mc

Usage

./nc_s3_to_filesystem.sh [<installDir>] [<migrateSingleUser>] [<webUser>] [<webGroup>]

Arguments:
  installDir         Path to local Nextcloud installation (default: /var/www/html); env: $NC_INSTALL_DIR
  migrateSingleUser  Optional flag to limit migration to the files of a single user; env: $NC_MIGRATE_USER
  webUser            System user managing the Nextcloud installation (extracted from chown); env: $NC_WEB_USER
  webGroup           System group managing the Nextcloud installation (extracted from chown); env: $NC_WEB_GROUP

Example

# Fetch files for all storages
./nc_s3_to_filesystem.sh /var/www/html

# Fetch files for a single user (alice)
./nc_s3_to_filesystem.sh /var/www/html alice

nc_s3_to_filesystem.sh

#!/usr/bin/env bash
# shellcheck disable=SC2155,SC2059
set -e

RED="\033[0;31m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
CYAN="\033[1;36m"
GRAY="\033[0;90m"
NC="\033[0m"

readonly scriptDir="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
readonly storagesCsv="${scriptDir}/storages.csv"
readonly filesCsvTemplate="${scriptDir}/files.%s.csv"
readonly phpBinary="${NC_PHP_BINARY:-php}"
readonly storageAppDataPrefix="object::store"

installDir="${NC_INSTALL_DIR:-$1}"
migrateUser="${NC_MIGRATE_USER:-$2}"
webUser="${NC_WEB_USER:-$3}"
webGroup="${NC_WEB_GROUP:-$4}"

checkedFiles=0
checkedFolders=0
skippedFiles=0
skippedFolders=0
copiedFiles=0
copiedFolders=0
temporaryFiles=()
cleanupDone=0
verboseCleanup=1

function _usage() {
  echo
  echo -e "Usage:"
  echo -e "  ${CYAN}$0${NC} [<installDir>] [<webUser>] [<webGroup>] [<migrateSingleUser>]"
  echo
  echo -e "Arguments:"
  echo -e "  ${GREEN}installDir${NC}         Path to local Nextcloud installation (default: /var/www/html); env: \$NC_INSTALL_DIR"
  echo -e "  ${GREEN}webUser${NC}            System user managing the Nextcloud installation (extracted from chown); env: \$NC_WEB_USER"
  echo -e "  ${GREEN}webGroup${NC}           System group managing the Nextcloud installation (extracted from chown); env: \$NC_WEB_GROUP"
  echo -e "  ${GREEN}migrateSingleUser${NC}  Optional flag to limit migration to the files of a single user; env: \$NC_MIGRATE_USER"
  echo
}

function _error() {
  local message="$1"
  local exitCode="${2:-1}"

  >&2 echo -e "${RED}${message}${NC}"
  exit "$exitCode"
}

function _success() {
  local message="$1"

  echo -e "  ${GREEN}✔︎ ${message}"
}

function _warning() {
  local message="$1"
  local level="${2:-3}"

  if [ "$level" -eq 2 ]; then
    echo -e "    ${YELLOW}${message}${NC}"
  elif [ "$level" -eq 3 ]; then
    echo -e "      ${YELLOW}${message}${NC}"
  fi
}

function _progress() {
  local message="$1"
  local level="${2:-1}"
  local print="${3:-1}"

  if [ "$print" -eq 0 ]; then
    return
  fi

  if [ "$level" -eq 1 ]; then
    echo -e "${CYAN}▶ ${message}...${NC}"
  elif [ "$level" -eq 2 ]; then
    echo -e "${GRAY}  ↳ ${message}${NC}"
  elif [ "$level" -eq 3 ]; then
    echo -e "${GRAY}    ∙ ${message}${NC}"
  fi
}

function _run() {
  if [ "$isWebUser" ]; then
    "$@"
  else
    sudo -u "$webUser" "$@"
  fi
}

function _get_config() {
  _run "$phpBinary" "$occFile" config:system:get "$@"
}

function _get_file_size() {
  local filename="$1"

  stat -c %s "$filename"
}

function _percentage() {
  local a="$1"
  local b="$2"
  local percentage="$(echo "scale=2; $a*100/$b" | bc)"

  echo "$percentage"
}

function _register_file() {
  temporaryFiles+=("$1")
}

function check_requirements() {
  _progress "Checking system requirements"

  if ! command -v mc >/dev/null 2>&1; then
    _error "mc is required, but not installed on your system."
  fi

  if ! command -v jq >/dev/null 2>&1; then
    _error "jq is required, but not installed on your system."
  fi
}

function lookup_installation() {
  _progress "Looking up Nextcloud installation"

  if [ -z "$installDir" ]; then
    printf "  Please enter the path to your Nextcloud installation [${GREEN}/var/www/html${NC}]: "
    read -r installDir

    if [ -z "$installDir" ]; then
      installDir="/var/www/html"
    fi
  fi

  occFile="${NC_OCC_FILE:-${installDir}/occ}"
  dataDir="${NC_DATA_DIR:-${installDir}/data}"

  # Validate install dir
  if [ ! -d "$installDir" ]; then
    _error "Path ${YELLOW}${installDir}${RED} does not exist or is not a directory."
    exit 1
  fi

  if [ ! -f "$occFile" ]; then
    _error "Nextcloud installation misses occ binary at ${YELLOW}${occFile}${RED}."
    exit 2
  fi

  if [ ! -d "$dataDir" ]; then
    _error "Data directory ${YELLOW}${dataDir}${RED} does not exist or is not a directory."
    exit 2
  fi

  _success "Found Nextcloud installation: ${installDir}"
  _success "Found occ binary: ${occFile}"
  _success "Found data directory: ${dataDir}"
}

function check_user_and_group() {
  _progress "Checking system user and group"

  if [ -z "$webUser" ]; then
    local currentUser="$(id -un)"
    webUser="$(stat -c '%U' "${installDir}")"

    if [[ "$currentUser" == "$webUser" ]]; then
      readonly isWebUser=1
    fi
  fi

  # Validate web user
  if ! id -u "$webUser" >/dev/null 2>&1; then
    _error "User ${YELLOW}${webUser}${NC} does not exist."
    exit 4
  fi

  # Get web group
  if [ -z "$webGroup" ]; then
    webGroup="$(stat -c '%G' "${installDir}")"
  fi

  # Validate web group
  if ! getent group "$webGroup" >/dev/null 2>&1; then
    _error "Group ${YELLOW}${webGroup}${NC} does not exist."
    exit 8
  fi

  _success "Detected system user: ${webUser}"
  _success "Detected system group: ${webGroup}"
}

function read_config() {
  _progress "Reading Nextcloud configuration (this may require your password)"

  # Database configuration
  readonly dbHost="$(_get_config dbhost)"
  readonly dbUser="$(_get_config dbuser)"
  readonly dbPass="$(_get_config dbpassword)"
  readonly dbName="$(_get_config dbname)"
  readonly dbPrefix="$(_get_config dbtableprefix)"

  local objectStorage="$(_get_config objectstore class || true)"

  # Fail if wrong or no object storage is configured
  if [[ "$objectStorage" != "OC\Files\ObjectStore\S3" ]]; then
    _error "S3 object storage is currently not configured for your Nextcloud installation."
  fi

  local useSsl="$(_get_config objectstore arguments use_ssl || echo 'true')"
  local hostProtocol="http"

  if [[ "$useSsl" == "true" ]]; then
    hostProtocol="https"
  fi

  # S3 Object Storage configuration
  readonly s3Host="${hostProtocol}://$(_get_config objectstore arguments hostname)"
  readonly s3Bucket="$(_get_config objectstore arguments bucket)"
  readonly s3AccessKey="$(_get_config objectstore arguments key)"
  readonly s3AccessSecret="$(_get_config objectstore arguments secret)"
}

function create_mc_alias() {
  _progress "Creating a temporary mc alias"

  readonly mcAlias="nc-migrate-$(head -c 32 /dev/urandom | sha256sum | cut -c1-16)"

  _run mc alias set "$mcAlias" "$s3Host" "$s3AccessKey" "$s3AccessSecret"
}

function query_storages() {
  _progress "Querying relevant file storages from database"

  local storageUserConstraint="object::user:%"
  local storageAppDataConstraint="OR id LIKE '${storageAppDataPrefix}:%'"

  if [ -n "$migrateUser" ]; then
    storageUserConstraint="object::user:${migrateUser}"
    storageAppDataConstraint=""
  fi

  mysql -h"$dbHost" -u"$dbUser" -p"$dbPass" "$dbName" --default-character-set=utf8mb4 -e "
  SELECT numeric_id, REPLACE(id, 'object::user:', '') AS id
  FROM ${dbPrefix}storages
  WHERE id LIKE '${storageUserConstraint}'
  ${storageAppDataConstraint};
  " | sed 's/\t/,/g' > "$storagesCsv"

  _register_file "$storagesCsv"
}

function fetch_files() {
  local storageId
  local userName
  local fileId
  local path
  local size
  local mtime
  local mimeType

  # Calculate total number of storages to process
  totalStorages="$(wc -l "$storagesCsv" | awk '{print $1}')"
  totalStorages=$((totalStorages - 1))
  local currentStorage=0

  while IFS=, read -r storageId userName; do
    # Skip header
    if [[ "$storageId" == "numeric_id" ]]; then
      continue
    fi

    currentStorage=$((currentStorage + 1))

    local targetDir="${dataDir}/${userName}"
    local filesCsv="$(printf "$filesCsvTemplate" "$storageId")"
    local storageHint="(user: ${userName})"

    # Handle appdata storages
    if [[ "$userName" == "$storageAppDataPrefix"* ]]; then
      targetDir="$dataDir"
      storageHint="(app data)"
    fi

    _progress "[$(_percentage "$currentStorage" "$totalStorages")] Fetching files for storage ${storageId} ${storageHint}"
    _progress "Target directory: ${GREEN}${targetDir}" 2

    # Select all files from database
    mysql -h"$dbHost" -u"$dbUser" -p"$dbPass" "$dbName" --default-character-set=utf8mb4 -e "
      SELECT fileid, path, size, mtime, mimetype
      FROM ${dbPrefix}filecache
      WHERE storage = ${storageId}
        AND path != ''
        AND name != '';
    " | sed 's/\t/,/g' > "$filesCsv"
    _register_file "$filesCsv"

    # Calculate total number of files to process
    local totalFiles="$(wc -l "$filesCsv" | awk '{print $1}')"
    totalFiles=$((totalFiles - 1))
    local currentFile=0

    if [ "$totalFiles" -le 0 ]; then
      _warning "Skipped: Found no objects" 2
    fi

    _progress "Found objects: ${totalFiles}" 2

    # Download files from object storage
    while IFS=, read -r fileId path size mtime mimeType; do
      # Skip header
      if [[ "$fileId" == "fileid" ]]; then
        continue
      fi

      currentFile=$((currentFile + 1))

      local sourceObjectId="urn:oid:${fileId}"
      local sourceObjectAlias="${mcAlias}/${s3Bucket}/${sourceObjectId}"
      local targetPath="${targetDir}/${path}"
      local targetOwner="${webUser}:${webGroup}"

      if [ "$mimeType" -eq 2 ]; then
        local fileType="folder"
      else
        local fileType="file"
      fi

      _progress "[$(_percentage "$currentFile" "$totalFiles")] ${GREEN}${sourceObjectId}${GRAY} => ${targetDir}/${GREEN}${path}${GRAY} (type: ${fileType})" 3

      # Create folder
      if [[ "$fileType" == "folder" ]]; then
        checkedFolders=$((checkedFolders+=1))

        # Skip existing directory
        if [ -d "$targetPath" ]; then
          _warning "Skipped: Folder already exists"
          skippedFolders=$((skippedFolders+=1))
          continue
        fi

        copiedFolders=$((copiedFolders+=1))

        _run mkdir -p "$targetPath"
        _run chown "$targetOwner" "$targetPath"
        _run chmod 755 "$targetPath"

        continue
      fi

      checkedFiles=$((checkedFiles+=1))

      local sourceObjectSize="$(_run mc stat "$sourceObjectAlias" --json | jq -r '.size // empty')"

      # Skip missing objects
      if [ -z "$sourceObjectSize" ]; then
        _warning "Skipped: Object no longer exists in bucket"
        skippedFiles=$((skippedFiles+=1))
        continue
      fi

      # Skip existing file
      if [ -f "$targetPath" ]; then
        local targetFileSize="$(_get_file_size "$targetPath")"

        if [ "$targetFileSize" -eq "$sourceObjectSize" ]; then
          _warning "Skipped: File already exists"
          skippedFiles=$((skippedFiles+=1))
          continue
        fi
      fi

      # Make sure parent folder exists
      local parentFolder="$(dirname "$targetPath")"
      if [ ! -d "$parentFolder" ]; then
        _run mkdir -p "$parentFolder"
        _run chown "$targetOwner" "$parentFolder"
        _run chmod 755 "$parentFolder"
      fi

      # Copy file from object storage to file system
      _run mc cp --preserve "$sourceObjectAlias" "$targetPath"

      # Change owner of copied file
      _run chown "$targetOwner" "$targetPath"

      # Change modification time of copied file
      _run touch -m -d "@${mtime}" "$targetPath"

      # Change permissions of modified file
      _run chmod 640 "$targetPath"

      # Validate file size
      local actualSize="$(_get_file_size "$targetPath")"
      if [ "$actualSize" -ne "$size" ]; then
        _error "Size mismatch, current: ${actualSize}, expected: ${size}"
      fi

      copiedFiles=$((copiedFiles+=1))
    done < "$filesCsv"
  done < "$storagesCsv"
}

function cleanup () {
  local exitCode=$?

  # Avoid performing cleanup multiple times
  if [ "$cleanupDone" -eq 1 ]; then
    return
  fi

  # Flag current cleanup
  cleanupDone=1

  # Early return if there's nothing to clean up
  if [ "${#temporaryFiles[@]}" -eq 0 ] && [ -z "$mcAlias" ]; then
    return
  fi

  _progress "Cleaning up temporary resources" 1 "$verboseCleanup"

  local file
  for file in "${temporaryFiles[@]}"; do
    if [ -e "$file" ]; then
      _progress "File: ${GREEN}${file}" 2 "$verboseCleanup"
      rm -f "$file"
    fi
  done

  if _run mc alias ls "$mcAlias" >/dev/null 2>&1; then
    _progress "mc alias: ${GREEN}${mcAlias}" 2 "$verboseCleanup"

    if [ "$verboseCleanup" -eq 1 ]; then
      _run mc alias rm "$mcAlias"
    else
      _run mc alias rm "$mcAlias" >/dev/null 2>&1
    fi
  fi

  exit "$exitCode"
}

function print_result() {
  echo
  echo -e "${GREEN}Finished with the following result:${NC}"
  echo -e "  ${CYAN}▶ Processed${NC} ${totalStorages} storage(s)."
  echo -e "  ${CYAN}▶ Processed${NC} ${checkedFiles} file(s) and ${checkedFolders} folder(s)."

  if [ "$skippedFiles" -gt 0 ] || [ "$skippedFolders" -gt 0 ]; then
    echo -e "  ${YELLOW}✘ Skipped${NC} ${skippedFiles} file(s) and ${skippedFolders} folder(s)."
  fi

  echo -e "  ${GREEN}✔︎ Copied${NC} ${copiedFiles} file(s) and ${copiedFolders} folder(s) from object storage to filesystem."

  # Since this is part of the default execution workflow,
  # we don't want the cleanup script to print any output.
  verboseCleanup=0
}

trap cleanup EXIT
trap cleanup ERR
trap cleanup SIGINT

# Print usage if requested
if [[ "$1" == "help" ]] || [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]]; then
  _usage
  exit
fi

check_requirements
lookup_installation
check_user_and_group
read_config
create_mc_alias
query_storages
fetch_files
print_result

When migrating from S3 to regular filesystem like in the script, isn't it necessary to update the connected MariaDB/MySQL?

Since with S3 backend the DB _filecache table works with IDs which are only saved in URI:...ID format on the S3 (and usernames are stored with object::user: prefix), but afterwards the files are saved with their concrete filenames in the nextcloud/data directory directly. Or is that no needed and the DB will edit the respective tables itself?