Zlib-ng benchmark workflow

benchmark.yml

name: Benchmark
on:
  issue_comment:
    types: [created]
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to benchmark (results posted as PR comment)'
        required: false
        type: number
      head_ref:
        description: 'Commit SHA or branch to benchmark (defaults to PR head)'
        required: false
        default: ''
        type: string
      base_ref:
        description: 'Base branch to compare against (defaults to PR base or develop)'
        required: false
        default: ''
        type: string

concurrency:
  group: benchmark
  cancel-in-progress: false

permissions:
  contents: read
  pull-requests: write

jobs:
  benchmark:
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (github.event.issue.pull_request &&
       contains(github.event.comment.body, '/benchmark'))
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Acknowledge benchmark request
        if: github.event_name == 'issue_comment'
        run: |
          gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
            -f content='rocket'

      - name: Resolve refs
        id: refs
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            PR_NUMBER="${{ inputs.pr_number }}"
            INPUT_HEAD="${{ inputs.head_ref }}"
            INPUT_BASE="${{ inputs.base_ref }}"
          else
            PR_NUMBER="${{ github.event.issue.number }}"
            # Parse optional commit from "/benchmark <commit>"
            INPUT_HEAD=$(echo "${{ github.event.comment.body }}" | grep -oP '/benchmark\s+\K\S+' || true)
            INPUT_BASE=""
          fi

          if [ -n "$PR_NUMBER" ]; then
            pr_json=$(gh pr view "$PR_NUMBER" --repo "${{ github.repository }}" --json baseRefName,headRefName,headRefOid)
            PR_BASE=$(echo "$pr_json" | jq -r .baseRefName)
            PR_HEAD_REF=$(echo "$pr_json" | jq -r .headRefName)
            PR_HEAD_SHA=$(echo "$pr_json" | jq -r .headRefOid)

            HEAD_REF="${INPUT_HEAD:-$PR_HEAD_SHA}"
            HEAD_LABEL="${INPUT_HEAD:-$PR_HEAD_REF}"
            BASE_REF="${INPUT_BASE:-$PR_BASE}"
          else
            HEAD_REF="$INPUT_HEAD"
            HEAD_LABEL="$INPUT_HEAD"
            BASE_REF="${INPUT_BASE:-develop}"
          fi

          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
          echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
          echo "head_label=$HEAD_LABEL" >> "$GITHUB_OUTPUT"
          echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"

      - name: Checkout head
        uses: actions/checkout@v4
        with:
          ref: ${{ steps.refs.outputs.head_ref }}
          path: head-source

      - name: Checkout base branch
        uses: actions/checkout@v4
        with:
          ref: ${{ steps.refs.outputs.base_ref }}
          path: base-source

      - name: Build base branch
        run: |
          cmake -S base-source -B base-build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
          cmake --build base-build --config Release -j$(nproc)

      - name: Build head
        run: |
          cmake -S head-source -B head-build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
          cmake --build head-build --config Release -j$(nproc)

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'

      - name: Checkout deflatebench
        uses: actions/checkout@v4
        with:
          repository: zlib-ng/deflatebench
          path: deflatebench

      - name: Install deflatebench requirements
        run: pip install -r deflatebench/requirements.txt

      - name: Cache test data
        id: cache-testdata
        uses: actions/cache@v4
        with:
          path: testdata/silesia-small.tar
          key: silesia-small

      - name: Download test data
        if: steps.cache-testdata.outputs.cache-hit != 'true'
        run: |
          mkdir -p testdata
          curl -L -o testdata/silesia-small.tar https://mirror.circlestorm.org/silesia-small.tar

      - name: Install benchmarking tools
        if: runner.os == 'Linux'
        run: |
          sudo apt-get -qq update
          sudo apt-get -qq install -y linux-tools-common linux-tools-$(uname -r)

      - name: Tune system for benchmarking
        if: runner.os == 'Linux'
        run: |
          BENCH_CPU=$(($(nproc) - 1))
          echo "BENCH_PREFIX=setarch $(uname -m) -R taskset -c $BENCH_CPU" >> "$GITHUB_ENV"

          # Set CPU governor to performance
          sudo cpupower frequency-set -g performance || true

          # Lock benchmark core to a fixed frequency
          MAX_FREQ=$(cat /sys/devices/system/cpu/cpu${BENCH_CPU}/cpufreq/cpuinfo_max_freq 2>/dev/null || echo "")
          if [ -n "$MAX_FREQ" ]; then
            sudo cpupower -c "$BENCH_CPU" frequency-set --min "$MAX_FREQ" --max "$MAX_FREQ" || true
          fi

          # Use tmpfs for deflatebench temp files to avoid disk I/O variance
          sudo mount -t tmpfs -o size=312M tmpfs /tmp

          # Disable turbo boost for consistent clocks
          if [ -f /sys/devices/system/cpu/intel_pstate/no_turbo ]; then
            echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
          elif [ -f /sys/devices/system/cpu/cpufreq/boost ]; then
            echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost
          fi

          # Disable transparent huge pages to avoid compaction latency spikes
          echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
          echo never | sudo tee /sys/kernel/mm/transparent_hugepage/defrag

          # Move IRQs off the benchmark core
          for irqdir in /proc/irq/[0-9]*; do
            echo "$(printf '%x' $(((1 << $(nproc)) - 1 - (1 << BENCH_CPU))))" \
              | sudo tee "$irqdir/smp_affinity" 2>/dev/null || true
          done

      - name: Drop filesystem caches
        if: runner.os == 'Linux'
        run: |
          sync
          echo 3 | sudo tee /proc/sys/vm/drop_caches

      - name: Benchmark base branch
        working-directory: deflatebench
        run: |
          $BENCH_PREFIX \
            python deflatebench.py --testtool ../base-build/minigzip --single \
            --file ../testdata/silesia-small.tar \
            --runs 5 --trimworst 2 --levels 1,3,6,9 \
            --benchmark both 2>&1 | sed -n '/^ Level/,$p' \
            > "$GITHUB_WORKSPACE/base-results.txt"

      - name: Benchmark head
        working-directory: deflatebench
        run: |
          $BENCH_PREFIX \
            python deflatebench.py --testtool ../head-build/minigzip --single \
            --file ../testdata/silesia-small.tar \
            --runs 5 --trimworst 2 --levels 1,3,6,9 \
            --benchmark both 2>&1 | sed -n '/^ Level/,$p' \
            > "$GITHUB_WORKSPACE/head-results.txt"

      - name: Restore system settings
        if: runner.os == 'Linux' && always()
        run: |
          # Restore CPU governor to ondemand
          sudo cpupower frequency-set -g ondemand || true

          # Re-enable turbo boost
          if [ -f /sys/devices/system/cpu/intel_pstate/no_turbo ]; then
            echo 0 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
          elif [ -f /sys/devices/system/cpu/cpufreq/boost ]; then
            echo 1 | sudo tee /sys/devices/system/cpu/cpufreq/boost
          fi

          # Re-enable transparent huge pages
          echo always | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
          echo always | sudo tee /sys/kernel/mm/transparent_hugepage/defrag

          # Unmount tmpfs
          sudo umount /tmp || true

      - name: Post benchmark results
        run: |
          PR_NUMBER="${{ steps.refs.outputs.pr_number }}"
          BASE_REF="${{ steps.refs.outputs.base_ref }}"
          HEAD_LABEL="${{ steps.refs.outputs.head_label }}"
          HEAD_REF="${{ steps.refs.outputs.head_ref }}"

          body=$(cat <<EOF
          ## Benchmark Results

          <details>
          <summary>Base branch (<code>${BASE_REF}</code>)</summary>

          \`\`\`
          $(cat base-results.txt)
          \`\`\`
          </details>

          <details open>
          <summary>Head (<code>${HEAD_LABEL}</code>)</summary>

          \`\`\`
          $(cat head-results.txt)
          \`\`\`
          </details>
          EOF
          )

          if [ -n "$PR_NUMBER" ]; then
            gh pr comment "$PR_NUMBER" --repo "${{ github.repository }}" --body "$body"
          else
            echo "$body" >> "$GITHUB_STEP_SUMMARY"
          fi