Demonstrator of deterministic output from llama.cpp

llama_cpp_deterministic.sh

#!/usr/bin/env bash

# This script demonstrates deterministic output from llama.cpp.
# Place it in an empty directory, run it and it will:
# 1. Clone llama.cpp
# 2. Build llama-cli, it takes a few minutes
# 3. Download a small model
# 4. Run the same prompt twice with the same settings
# 5. Compare the outputs and print whether they are identical

# Requirements:
# - git
# - Python 3
# - cmake (version 3.15 or higher)
# - A C++ compiler (e.g. g++ or clang++)
# - wget

# Usage:
# chmod +x llama_cpp_deterministic.sh
# ./llama_cpp_deterministic.sh

git clone https://github.com/ggerganov/llama.cpp

cd llama.cpp

cmake -B build
cmake --build build --config Release -j $(nproc) --target llama-cli

cd ..

wget https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf

LLAMA_CMD="./llama.cpp/build/bin/llama-cli \
  --model tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf \
  --seed 42 \
  --temp 0 \
  --threads 1 \
  --single-turn \
  --prompt \"What is a computer?\""

OUT1=$(mktemp)
OUT2=$(mktemp)

echo "=== Run 1 ==="
eval "$LLAMA_CMD" > "$OUT1" 2>/dev/null
cat "$OUT1"

echo ""
echo "=== Run 2 ==="
eval "$LLAMA_CMD" > "$OUT2" 2>/dev/null
cat "$OUT2"

echo ""
echo "=== Comparison ==="
if diff -q "$OUT1" "$OUT2" > /dev/null; then
    echo "PASS: Both outputs are identical."
else
    echo "FAIL: Outputs differ."
    diff "$OUT1" "$OUT2"
fi

rm "$OUT1" "$OUT2"