Ollama using AMD APU with ROCm

ollama_rocm_apu.sh

#!/bin/bash
# @parseArger-begin
# @parseArger-help "I send an SOS to the world" --option "help" --short-option "h"
# @parseArger-verbose --option "verbose" --level "0" --quiet-option "quiet"
_has_colors=0
if [ -t 1 ]; then # Check if stdout is a terminal
  ncolors=$(tput colors 2>/dev/null)
  if [ -n "$ncolors" ] && [ "$ncolors" -ge 8 ]; then
    _has_colors=1
  fi
fi
# @parseArger-declarations
# @parseArger opt rc-file "rcfile to add the env var export" --default-value "/home/didi/.bashrc"
# @parseArger opt directory "where to clone"
# @parseArger opt gfx "rocm target" --one-of "10.3.0" --one-of "11.0.0" --one-of "11.0.1" --one-of "9.0.0"
# @parseArger opt container-name "container name" --default-value "ollama"
# @parseArger opt container-image-name "container name" --default-value "ollama-gtt"
# @parseArger opt container-models-volume "container volume for model"
# @parseArger flag reboot "reboot to for group setup to be taken into account"
# @parseArger flag clone "clone the repo" --on
# @parseArger flag pull "pull repo, force --clone off"
# @parseArger flag docker "use docker"
# @parseArger flag podman "use podman"
# @parseArger flag dependencies "install dependencies" --on
# @parseArger flag container-prune "prune existing container images" --on
# @parseArger flag dependencies-confirm "must confirm when installing dependencies"
# @parseArger flag serve "start ollama server" --alias start
# @parseArger-declarations-end

# @parseArger-utils
_helpHasBeenPrinted=1
_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
# @parseArger-utils-end

# @parseArger-parsing

__cli_arg_count=$#

die() {
  local _ret=1
  if [[ -n "$2" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
    _ret="$2"
  fi
  test "${_PRINT_HELP:-no}" = yes && print_help >&2
  log "$1" -3 >&2
  exit "${_ret}"
}

begins_with_short_option() {
  local first_option all_short_options=''
  first_option="${1:0:1}"
  test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0
}

# POSITIONALS ARGUMENTS
_positionals=()
_optional_positionals=()
# OPTIONALS ARGUMENTS
_arg_rc_file="/home/didi/.bashrc"
_arg_directory=
_arg_gfx=
_one_of_arg_gfx=("10.3.0" "11.0.0" "11.0.1" "9.0.0")
_arg_container_name="ollama"
_arg_container_image_name="ollama-gtt"
_arg_container_models_volume=
# FLAGS
_arg_reboot="off"
_arg_clone="on"
_arg_pull="off"
_arg_docker="off"
_arg_podman="off"
_arg_dependencies="on"
_arg_container_prune="on"
_arg_dependencies_confirm="off"
_arg_serve="off"
# NESTED
_verbose_level="0"

print_help() {
  _triggerSCHelp=1

  if [[ "$_helpHasBeenPrinted" == "1" ]]; then
    _helpHasBeenPrinted=0
    echo -e "I send an SOS to the world:"
    echo -e "	--rc-file <rc-file>: rcfile to add the env var export [default: ' /home/didi/.bashrc ']"
    echo -e "	--directory <directory>: where to clone"
    echo -e "	--gfx <gfx>: rocm target [one of '10.3.0' '11.0.0' '11.0.1' '9.0.0']"
    echo -e "	--container-name <container-name>: container name [default: ' ollama ']"
    echo -e "	--container-image-name <container-image-name>: container name [default: ' ollama-gtt ']"
    echo -e "	--container-models-volume <container-models-volume>: container volume for model"
    echo -e "	--reboot|--no-reboot: reboot to for group setup to be taken into account"
    echo -e "	--clone|--no-clone: clone the repo, on by default (use --no-clone to turn it off)"
    echo -e "	--pull|--no-pull: pull repo, force --clone off"
    echo -e "	--docker|--no-docker: use docker"
    echo -e "	--podman|--no-podman: use podman"
    echo -e "	--dependencies|--no-dependencies: install dependencies, on by default (use --no-dependencies to turn it off)"
    echo -e "	--container-prune|--no-container-prune: prune existing container images, on by default (use --no-container-prune to turn it off)"
    echo -e "	--dependencies-confirm|--no-dependencies-confirm: must confirm when installing dependencies"
    echo -e "	--serve|--no-serve: start ollama server
		aliases: --start,"
    echo -e "Usage :
	$0 [--rc-file <value>] [--directory <value>] [--gfx <value>] [--container-name <value>] [--container-image-name <value>] [--container-models-volume <value>] [--[no-]reboot] [--[no-]clone] [--[no-]pull] [--[no-]docker] [--[no-]podman] [--[no-]dependencies] [--[no-]container-prune] [--[no-]dependencies-confirm] [--[no-]serve]"
  fi

}

log() {
  local _arg_msg="${1}"
  local _arg_level="${2:-0}"
  if [ "${_arg_level}" -le "${_verbose_level}" ]; then
    case "$_arg_level" in
    -3)
      _arg_COLOR="\033[0;31m"
      ;;
    -2)
      _arg_COLOR="\033[0;33m"
      ;;
    -1)
      _arg_COLOR="\033[1;33m"
      ;;
    1)
      _arg_COLOR="\033[0;32m"
      ;;
    2)
      _arg_COLOR="\033[1;36m"
      ;;
    3)
      _arg_COLOR="\033[0;36m"
      ;;
    *)
      _arg_COLOR="\033[0m"
      ;;
    esac
    if [ "${_has_colors}" == "1" ]; then
      echo -e "${_arg_COLOR}${_arg_msg}\033[0m"
    else
      echo "${_arg_msg}"
    fi
  fi
}

parse_commandline() {
  _positionals_count=0
  while test $# -gt 0; do
    _key="$1"
    case "$_key" in
    --rc-file)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_rc_file="$2"
      shift
      ;;
    --rc-file=*)
      _arg_rc_file="${_key##--rc-file=}"
      ;;

    --directory)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_directory="$2"
      shift
      ;;
    --directory=*)
      _arg_directory="${_key##--directory=}"
      ;;

    --gfx)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_gfx="$2"
      if [[ "${#_one_of_arg_gfx[@]}" -gt 0 ]]; then [[ "${_one_of_arg_gfx[*]}" =~ (^|[[:space:]])"$_arg_gfx"($|[[:space:]]) ]] || die "gfx must be one of: 10.3.0 11.0.0 11.0.1 9.0.0"; fi
      shift
      ;;
    --gfx=*)
      _arg_gfx="${_key##--gfx=}"
      if [[ "${#_one_of_arg_gfx[@]}" -gt 0 ]]; then [[ "${_one_of_arg_gfx[*]}" =~ (^|[[:space:]])"$_arg_gfx"($|[[:space:]]) ]] || die "gfx must be one of: 10.3.0 11.0.0 11.0.1 9.0.0"; fi
      ;;

    --container-name)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_container_name="$2"
      shift
      ;;
    --container-name=*)
      _arg_container_name="${_key##--container-name=}"
      ;;

    --container-image-name)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_container_image_name="$2"
      shift
      ;;
    --container-image-name=*)
      _arg_container_image_name="${_key##--container-image-name=}"
      ;;

    --container-models-volume)
      test $# -lt 2 && die "Missing value for the option: '$_key'" 1
      _arg_container_models_volume="$2"
      shift
      ;;
    --container-models-volume=*)
      _arg_container_models_volume="${_key##--container-models-volume=}"
      ;;

    --reboot)
      _arg_reboot="on"
      ;;
    --no-reboot)
      _arg_reboot="off"
      ;;
    --clone)
      _arg_clone="on"
      ;;
    --no-clone)
      _arg_clone="off"
      ;;
    --pull)
      _arg_pull="on"
      ;;
    --no-pull)
      _arg_pull="off"
      ;;
    --docker)
      _arg_docker="on"
      ;;
    --no-docker)
      _arg_docker="off"
      ;;
    --podman)
      _arg_podman="on"
      ;;
    --no-podman)
      _arg_podman="off"
      ;;
    --dependencies)
      _arg_dependencies="on"
      ;;
    --no-dependencies)
      _arg_dependencies="off"
      ;;
    --container-prune)
      _arg_container_prune="on"
      ;;
    --no-container-prune)
      _arg_container_prune="off"
      ;;
    --dependencies-confirm)
      _arg_dependencies_confirm="on"
      ;;
    --no-dependencies-confirm)
      _arg_dependencies_confirm="off"
      ;;
    --serve | --start)
      _arg_serve="on"
      ;;
    --no-serve)
      _arg_serve="off"
      ;;
    -h | --help)
      print_help
      exit 0
      ;;
    -h*)
      print_help
      exit 0
      ;;
    --verbose)
      if [ $# -lt 2 ]; then
        _verbose_level="$((_verbose_level + 1))"
      else
        _verbose_level="$2"
        shift
      fi
      ;;
    --quiet)
      if [ $# -lt 2 ]; then
        _verbose_level="$((_verbose_level - 1))"
      else
        _verbose_level="-$2"
        shift
      fi
      ;;

    *)
      _last_positional="$1"
      _positionals+=("$_last_positional")
      _positionals_count=$((_positionals_count + 1))
      ;;
    esac
    shift
  done
}

handle_passed_args_count() {
  local _required_args_string=""
  if [ "${_positionals_count}" -gt 0 ] && [ "$_helpHasBeenPrinted" == "1" ]; then
    _PRINT_HELP=yes die "FATAL ERROR: There were spurious positional arguments --- we expect at most 0 (namely: $_required_args_string), but got ${_positionals_count} (the last one was: '${_last_positional}').\n\t${_positionals[*]}" 1
  fi
  if [ "${_positionals_count}" -lt 0 ] && [ "$_helpHasBeenPrinted" == "1" ]; then
    _PRINT_HELP=yes die "FATAL ERROR: Not enough positional arguments - we require at least 0 (namely: $_required_args_string), but got only ${_positionals_count}.
	${_positionals[*]}" 1
  fi
}

assign_positional_args() {
  local _positional_name _shift_for=$1
  _positional_names=""
  shift "$_shift_for"
  for _positional_name in ${_positional_names}; do
    test $# -gt 0 || break
    eval "if [ \"\$_one_of${_positional_name}\" != \"\" ];then [[ \"\${_one_of${_positional_name}[*]}\" =~ \"\${1}\" ]];fi" || die "${_positional_name} must be one of: $(eval "echo \"\${_one_of${_positional_name}[*]}\"")" 1
    eval "$_positional_name=\${1}" || die "Error during argument parsing, possibly an ParseArger bug." 1
    shift
  done
}

print_debug() {
  print_help
  # shellcheck disable=SC2145
  echo "DEBUG: $0 $@"

  echo -e "	rc-file: ${_arg_rc_file}"
  echo -e "	directory: ${_arg_directory}"
  echo -e "	gfx: ${_arg_gfx}"
  echo -e "	container-name: ${_arg_container_name}"
  echo -e "	container-image-name: ${_arg_container_image_name}"
  echo -e "	container-models-volume: ${_arg_container_models_volume}"
  echo -e "	reboot: ${_arg_reboot}"
  echo -e "	clone: ${_arg_clone}"
  echo -e "	pull: ${_arg_pull}"
  echo -e "	docker: ${_arg_docker}"
  echo -e "	podman: ${_arg_podman}"
  echo -e "	dependencies: ${_arg_dependencies}"
  echo -e "	container-prune: ${_arg_container_prune}"
  echo -e "	dependencies-confirm: ${_arg_dependencies_confirm}"
  echo -e "	serve: ${_arg_serve}"

}

on_interrupt() {
  die Process aborted! 130
}

parse_commandline "$@"
handle_passed_args_count
assign_positional_args 1 "${_positionals[@]}"
trap on_interrupt INT

# @parseArger-parsing-end
# print_debug "$@"
# @parseArger-end

if [ "$_arg_dependencies" = "on" ]; then
  # Detect package manager and set appropriate commands and packages
  install_cmd=(install)
  if [ "$_arg_dependencies_confirm" != "on" ]; then
    install_cmd+=(-y)
  fi
  if command -v dnf &>/dev/null; then
    pkg_manager="dnf"
    dependencies=(go cmake gcc clang rocm-hip hipblas-devel rocblas-devel)
  elif command -v apt &>/dev/null; then
    pkg_manager="apt"
    dependencies=(golang cmake gcc clang rocm-hip-runtime rocm-hip-sdk rocblas hipblas)
  elif command -v pacman &>/dev/null; then
    pkg_manager="pacman"
    dependencies=(go cmake gcc clang rocm-hip-runtime rocm-hip-sdk rocblas hipblas)
    install_cmd=(-S)
    if [ "$_arg_dependencies_confirm" != "on" ]; then
      install_cmd+=(--noconfirm)
    fi
  else
    die "No supported package manager found (dnf, apt, or pacman)" 1
  fi
  _depCmd=(sudo "${pkg_manager}" "${install_cmd[@]}" "${dependencies[@]}")
  log "installing dependencies using ${pkg_manager}, needs sudo..." -1
  "${_depCmd[@]}"
fi

if [ "$_arg_clone" = "on" ] || [ "$_arg_pull" = "on" ]; then
  _git_cmd=(git)
  if [ "$_arg_clone" = "on" ]; then
    _git_cmd+=(clone git@github.com:Maciej-Mogilany/ollama.git -b AMD_APU_GTT_memory)
    if [ "$_arg_directory" != "" ]; then
      _git_cmd+=("${_arg_directory}")
    fi
    log "cloning repo" -1
    "${_git_cmd[@]}"
  elif [ "$_arg_pull" = "on" ]; then
    _git_cmd+=(pull origin AMD_GPU_GTT_memory)
    if [ "$_arg_directory" != "" ]; then
      _arg_directory="ollama"
    fi
    log "pulling repo" -1
    cd "${_arg_directory}" || die "Failed to cd into ${_arg_directory}" 1
    "${_git_cmd[@]}"
    cd -
  fi
fi

if [ "$_arg_directory" = "" ]; then
  _arg_directory="ollama"
fi
cd "${_arg_directory}" || die "Failed to cd into ${_arg_directory}" 1

log "checking if user is part of render and video groups" -1
_usermodCmd=(sudo usermod -a -G)
_toAddGroups=()
if ! groups "$USER" | grep -q "\brender\b"; then
  _toAddGroups+=(render)
fi
if ! groups "$USER" | grep -q "\bvideo\b"; then
  _toAddGroups+=(video)
fi
if [ "${#_toAddGroups[@]}" -gt 0 ]; then
  _gp=""
  for gp in "${_toAddGroups[@]}"; do
    _gp+=",${gp}"
  done
  log "adding user ${USER} to groups ${_gp}" -1
  "${_usermodCmd[@]}" "${_gp}" "${USER}"
fi

if [ "$_arg_docker" = "on" ] || [ "$_arg_podman" = "on" ]; then
  _ctnCmd=()
  if [ "$_arg_docker" = "on" ]; then
    _ctnCmd=(docker)
  elif [ "$_arg_podman" = "on" ]; then
    _ctnCmd=(podman)
  fi
  if [ "$_arg_container_prune" = "on" ]; then
    if "${_ctnCmd[0]}" ps -q --filter "name=${_arg_container_name}" | grep -q .; then
      log "stopping existing container ${_arg_container_name}" -1
      "${_ctnCmd[0]}" stop "${_arg_container_name}"
    fi
    log "removing existing container ${_arg_container_name}" -1
    "${_ctnCmd[0]}" rm -f "${_arg_container_name}"
    rm -rf /var/tmp/buildah-cache-1000
  fi

  _ctnCmd+=(build
    -f Dockerfile
    --no-cache
    --platform=linux/amd64
    --target runtime-rocm
    --build-arg=OLLAMA_SKIP_CUDA_GENERATE=1
    -t "${_arg_container_image_name}")

  log "building using ${_ctnCmd[0]}, this WILL take a GOOD LONG while" -1
  "${_ctnCmd[@]}"
  log "building done" -1
  log "command to run the container :" -1
  _cntRunCmd=("${_ctnCmd[0]}" run -d)
  if [ "$_arg_podman" = "on" ]; then
    _cntRunCmd+=(--replace --privileged)
  fi
  _cntRunCmd+=(--stop-signal=SIGKILL)
  _cntRunCmd+=(--name "${_arg_container_name}")
  _cntRunCmd+=(--device /dev/dri)
  _cntRunCmd+=(--device /dev/kfd)
  _cntRunCmd+=(-p 127.0.0.1:11434:11434)
  _cntRunCmd+=(-v ollama:/root/.ollama)
  if [ "$_arg_container_models_volume" != "" ]; then
    _cntRunCmd+=(-v "${_arg_container_models_volume}:/data/ollama:z")
  fi
  _cntRunCmd+=(-e OLLAMA_DEBUG=1)
  _cntRunCmd+=(-e OLLAMA_MAX_LOADED_MODELS=1)
  _cntRunCmd+=(-e OLLAMA_NUM_PARALLEL=1)
  _cntRunCmd+=(-e HSA_OVERRIDE_GFX_VERSION="${_arg_gfx}")
  _cntRunCmd+=("${_arg_container_image_name}")
  if [ "$_arg_reboot" = "on" ]; then
    log "rebooting to apply group changes" -1
    sudo reboot;
  else
    if [ "$_arg_serve" = "on" ]; then
      "${_cntRunCmd[@]}"
    else
      _cntrunOut="${_cntRunCmd[0]} ${_cntRunCmd[1]} ${_cntRunCmd[2]} \\"
      for cmd in "${_cntRunCmd[@]:3}"; do
        _cntrunOut+="\n\t$cmd \\"
      done
      log "${_cntrunOut}" -1
    fi
  fi
else
  log "building, this WILL take a while" -1
  go generate ./...
  go build .

  log "exporting environment variables to ${_arg_rc_file}" -1
  echo -e "
	export HSA_OVERRIDE_GFX_VERSION=${_arg_gfx};
	export ROCM_PATH=/usr/lib64/rocm;
	export HIP_PATH=\$ROCM_PATH;
	export LD_LIBRARY_PATH=\$ROCM_PATH/lib:\$ROCM_PATH/lib64:\$LD_LIBRARY_PATH;" >>"${_arg_rc_file}"

  if [ "$_arg_reboot" = "on" ]; then
    log "rebooting to apply group changes" -1
    sudo reboot;
  else
    if [ "$_arg_serve" = "on" ]; then
      ./ollama serve
    else
      log "command to run the server :" -1
      log "./ollama serve" -1
    fi
  fi
fi