POSIX sh script to create AVIF for a target SSIM score using binary search. Requires `imagemagick` and `cavif`.

avif.sh

#!/usr/bin/env -S unshare --mount --map-root-user /bin/sh
# shellcheck shell=sh

# Licensed under the Zero-Clause BSD terms: https://opensource.org/license/0bsd
## Requires ImageMagick and cavif


## -C: Fail if redirects try to overwrite an existing file.
## -e: Fail if any command fails (with exceptions).
## -u: Fail if an unset variable tries to be expanded.
## -f: No glob expansion.
set -Ceuf

perr() { printf "\a\033[1;31m‽\033[0;39m %s\033[0;39m\n" "${1-"$(cat)"}" >&2; }
clear_prog() { printf "\033]9;4;0\007" >&2; }
exit_clear() { clear_prog; exit 1; }
nexit() { printf "\a\033]777;notify;AVIFSSIM;%b\033\\" "$1" >&2; exit_clear; }
pquit() { arg="${1-"$(cat)"}"; perr "$arg"; nexit "$arg"; }
pstat() { printf "\033[1;34m%b\033[0;39m%s\033[0;39m\n" "$1" "${2-"$(cat)"}"; }


help() {
	cat <<"HELP"; exit 0
Usage: avifssim [OPTION]... SRC DEST
Convert an image into an AVIF image for a given SSIM target.

If DEST is omitted, DEST is set to 'SRC.avif'.

Options:
 -c, --clear-cache	Clear the cached SSIM values for all images.
 -d, --dry-run		Perform all operations except for writing the output.
 -f, --force		Overwrite existing destination files.
 -q, --quiet		Do not print to STDOUT or send notifications.
 -t, --target <num>	The target SSIM score as a decimal number, e.g.
			  100 = 100%
			  12.3456 = 12.3456%
			  0.123456 = 12.3456%
			  1 = 1%

 -h, --help		Display this help message, and then exit.
HELP
}




# Argument parsing
CLEAR=""	DRY=""		FORCE=""	TARGET="0.96"
SKIP=""		ARG_COUNT="0"
while [ "$ARG_COUNT" -lt "$#" ]; do
	if [ -z "$SKIP" ]; then
		case "$1" in
			-c|--clear-cache) CLEAR="1" ;;
			-d|--dry-run) DRY="1" ;;
			-f|--force) FORCE="1" ;;
			-q|--quiet)
				exec 1>/dev/null

				## Functions have to be redefined to be updated.
				### This `perr()` omits `\a`
				perr() { printf "\033[1;31m‽\033[0;39m %s\033[0;39m\n" "${1-"$(cat)"}" >&2; }
				pquit() { perr "$1"; exit_clear; }
				;;
			-t|--target)
				case "${2:-}" in
					[0-9]*) TARGET="$2"; shift ;;
					*) pquit "No number was given for '-t|--target'!" ;;
				esac
				;;
			-h|--help) help	;;
			--) SKIP="1" ;;
			-*) pquit "'$1' is not a known option!" ;;
			*) set -- "$@" "$1"; ARG_COUNT="$(( ARG_COUNT + 1 ))" ;;
		esac

		else set -- "$@" "$1"; ARG_COUNT="$(( ARG_COUNT + 1 ))"
	fi

	shift
done


## Validate arguments

# `[ cond ] && cmd` will carry over non-zero exit codes, so always use `||`
[ "$#" -le 2 ] || pquit "Too many arguments were given!"

## Don't try anything if `magick` or `cavif` are unavailable
command -v magick >/dev/null || pquit "The 'magick' command is required!"
command -v cavif >/dev/null || pquit "The 'cavif' command is required!"


## Initialise the cache directory, or re-use the existing one
set +f
for cache_path_exist in /tmp/avifssim.*; do
	[ -d "$cache_path_exist" ] || continue

	## Remove all cache directories if told to do so
	if [ -z "$CLEAR" ]; then
			cache_path="$cache_path_exist"
		else
			rm --recursive -- "$cache_path_exist"
	fi
done
set -f

cache_path="${cache_path:-"$(mktemp --dry-run --directory -t avifssim.XXXXXX)"}"
mkdir --parents -- "$cache_path"
exec 4<"$cache_path"
cache_path="/proc/self/fd/4"


### Setup a private TMPFS for this script to use, which is what 'unshare' is for.
#### 'nr_inodes' must be >= to 'max number of files + 1'
mount --types tmpfs --options nosuid,nodev,noexec,size=3G,nr_inodes=4 \
	avif-ssim /tmp/ || pquit "Failed to overmount '/tmp/'!"


### SRC
src_real="${1:?$(pquit "No source image file given!")}"
[ -s "$src_real" ] || pquit "'$src_real' is not a non-empty file!"

## Assign the SRC a file descriptor
exec 3<"$src_real"
src="/proc/self/fd/3"
src_real="$(realpath -- "$src_real")"


## Mimetype detection
mime="$(file --dereference --brief --mime-type -- "$src")"


## `cavif` only supports JPG & PNG for its input, so convert all other image
## formats to PNG
tmp_src="/tmp/src.png"
if [ "$mime" = "image/png" ] || [ "$mime" = "image/jpeg" ]; then
		cp -- "$src" "$tmp_src"
	else
		magick -- "$src" "$tmp_src"
fi


## Caching
### `magick` has non-deterministic output, so hash the SRC instead
hash="$(sha256sum -- "$src")"
hash="${hash%% *}"
hash_path="$cache_path/$hash.cache"


### DEST
dest_exists() {
	if [ -z "$FORCE" ] && [ -e "$1" ]; then
		pquit "'$1' already exists!"; fi
}

dest="${2:-"$src_real.avif"}"
dest="$(realpath -- "$dest")"
dest_exists "$dest"




### Target
ideal="${TARGET:-"0.96"}"

[ "$(magick -format "%[fx:${ideal}<1]\n" null: info:)" != 1 ] ||
	ideal="$(magick -format "%[fx:${ideal#*0}*100]" null: info:)"

pstat " Input ← " "$src_real"
pstat "Output ⇨ " "$dest"
pstat "Target ≥ " "$ideal%"
printf "\n"




# Search mode variables & functions
iteration="0"
threads="$(nproc)"
bad_quality="1"
quality="50"
break_quality="100"
upper_quality="$break_quality"
prev_trial="/tmp/trial.avif"
trial="$prev_trial.new"

tps_print() {
	printf "   \033[1m%b\033[0;39m \033[1m·\033[0m \033[2mquality=\033[0;39m%-2d  \033[2mssim=\033[0;39m%-8s  \033[2mbytes=\033[0;39m%d\n" \
		"$1" "$quality" "$score%" "$size"
}

evaluate() {
	meets_target="$(magick -format "%[fx:${score}>=${ideal}]" null: info:)"

	beats_prev_score="1"
	if [ -n "$upper_size" ]; then
		beats_prev_score="$(
			magick -format "%[fx:${score}>=${upper_score}]\n" \
				null: info:
		)"
	fi

	beats_size="0"
	if [ -z "$upper_size" ] || [ "$size" -lt "$upper_size" ]; then
		beats_size="1"; fi

	## Unlike '$remove', this variable gets operated on as a number in
	## `edgecase == 1` later on. It's simpler to not unset it
	edgecase="0"


	## Handle every combination of criteria.
	### Because '$ideal' is involved, this sum can't be considered for caching
	case "$(( meets_target + 2 * beats_prev_score + 4 * beats_size ))" in
		7)
			tps_print "\033[32mTPS"

			upper_quality="$quality"
			;;
		6)
			tps_print "\033[31mT\033[32mPS"

			remove="1"
			bad_quality="$(( quality + 1 ))"
			;;
		5)
			tps_print "\033[32mT\033[31mP\033[32mS"

			if [ "$size" -gt "$upper_size" ]; then
				remove="1"
				bad_quality="$(( quality + 1 ))"
					else
				upper_quality="$quality"
			fi
			;;
		4)
			tps_print "\033[31mTP\033[32mS"

			remove="1"
			bad_quality="$(( quality + 1 ))"
			;;
		3)
			tps_print "\033[32mTP\033[31mS"

			if [ "$size" -gt "$upper_size" ]; then
				remove="1"
				bad_quality="$(( quality + 1 ))"
					else
				upper_quality="$quality"
			fi
			;;
		2)
			tps_print "\033[1;31mT\033[32mP\033[31mS"

			remove="1"
			bad_quality="$(( quality + 1 ))"
			;;
		1)
			tps_print "\033[32mT\033[31mPS"

			remove="1"

			if [ "$(( quality + 1 ))" -eq "$upper_quality" ] ||
			   [ "$size" -ne "$upper_size" ]; then
				bad_quality="$(( quality + 1 ))"
					else
				edgecase="1"
			fi
			;;
		*)
			tps_print "\033[31mTPS"

			remove="1"
			bad_quality="$(( quality + 1 ))"
			;;
	esac

	[ -z "${remove:-}" ] || rm --force -- "$trial" ||:
	unset remove
}

encode() {
	## `heif-enc` has colour reproduction issues
#	heif-enc --avif --matrix_coefficients 1 --colour_primaries 1 \
#		--transfer_characteristic 1 --full_range_flag 1 \
#		--encoder rav1e -p speed=0,threads="$threads" \
#		--quality "$3" --output "$1" -- "$2" >/dev/null 2>&1 ||
#			pquit "Failed to create '$1' from '$2'!"

	## `--depth 8` reduces the final size; `--color rgb` increases it
	cavif --quiet --overwrite --threads "$threads" --speed 1 --depth 8 \
		--quality "$2" --output "$1" -- "$tmp_src"
}




# Use binary search to find the lowest quality image which meets or exceeds the
# target SSIM score
while [ "$bad_quality" -le "$(( upper_quality - 1 ))" ]; do
	iteration="$(( iteration + 1 ))"


	## Use the cached values if their available.
	### These variables hold the entire matching line
	{
		cache="$(sed --quiet "/^$quality/{p;q}" "$hash_path")" ||:
		upper_cache="$(sed --quiet "/^$upper_quality/{p;q}" "$hash_path")" ||:
	} 2>/dev/null

	### And these `read` commands split the lines into their values
	IFS="	" read -r _ score size <<-CACHE
		$cache
	CACHE

	IFS="	" read -r _ upper_score upper_size <<-UPPER_CACHE
		$upper_cache
	UPPER_CACHE


	## Create and score the trial if it's not present in the cache
	if [ -z "$cache" ]; then
		## Create the trial and rate it by score (SSIM) and size (bytes)
		encode "$trial" "$quality"

		## `-compare` must proceed `-metric`; otherwise, the specified
		## metric will be ignored.
		### The order of images does matter; however, for some images,
		### using the wrong order doesn't always affect the score
		score="$(
			magick "$src" "$trial" -metric ssim -compare \
				-format "%[fx:(1-%[distortion])*100]" info:
		)"

		size="$(wc --bytes <"$trial")"


		## Cache the "Quality-Score-Size" values
		cat <<-CACHE >>"$hash_path"
			$quality	$score	$size
		CACHE
	fi


	## Determine whether to remove '$trial', and how to search from here
	evaluate


	## If '$trial' still exists after `evaluate()`, then '$prev_trial'
	## should be replaced by it
	[ ! -e "$trial" ] || mv -- "$trial" "$prev_trial"


	## Handle any edge-cases & fail-safes with the search
	[ "$quality" -lt "$break_quality" ] || break

	quality="$((
		edgecase == 1
		? quality - 1
		: (bad_quality + upper_quality) / 2
	))"


	## Print a progress bar.
	### The maximum number of iterations for binary search should be 8.
	#### Without `>&2`, the updates could get redirected too
	printf "\033]9;4;1;%d\007" "$(( (iteration * 100) / 8 ))" >&2
done

clear_prog




# Create the final output
## If the best trial was found from the cache, actually create the trial
if [ ! -f "$prev_trial" ] && [ ! -f "$trial" ]; then
	[ "$upper_quality" -lt 100 ] || pquit "No trial met the SSIM target."

	encode "$prev_trial" "$upper_quality"
fi

final_score="$(
	magick "$src" "$prev_trial" -metric ssim -compare \
		-format "%[fx:(1-%[distortion])*100]" info:
)"

if [ "$(magick -format "%[fx:${final_score}>=${ideal}]" null: info:)" = 0 ] ||
   [ ! -f "$prev_trial" ] && [ ! -f "$trial" ]; then
	pquit "No trial met the SSIM target."; fi

final_size="$(wc --bytes <"$prev_trial")"

printf "\033[2mquality=\033[0;39m%-2d  \033[2mssim=\033[0;39m%-8s  \033[2mbytes=\033[0;39m%d" \
	"$upper_quality" "$final_score%" "$final_size" | pstat "\n Stats → "

dest_exists "$dest"
[ -n "$DRY" ] || cp -- "$prev_trial" "$dest"


## Alert the terminal (`\a`), and create a notification (`]777;notify`)
printf "\a\033]777;notify;AVIFSSIM;'%s' is finished! Score: (Q%d) %s. Size: %d bytes.\033\\" \
	"$dest" "$upper_quality" "$final_score%" "$final_size"