Analyze installer versions and their corresponding commits

analyze-installer-versions.sh

#!/bin/bash
# Analyze installer versions and their corresponding commits
# Similar to release image analysis, but for installer repository

set -e

INSTALLER_REPO="${1:-/Users/weli/works/oc-swarm/installer}"

echo "=========================================="
echo "Installer Version Analysis"
echo "=========================================="
echo "Repository: $INSTALLER_REPO"
echo ""

cd "$INSTALLER_REPO"

# Method 1: Check release branches
echo "=== Method 1: Release Branches ==="
for branch in release-4.16 release-4.17 release-4.18 release-4.19 release-4.20; do
    if git show-ref --verify --quiet refs/remotes/origin/$branch; then
        commit=$(git rev-parse origin/$branch)
        date=$(git log -1 --format="%cd" --date=short origin/$branch)
        echo "$branch: $commit ($date)"
    fi
done
echo ""

# Method 2: Check if bug commit exists in each branch
echo "=== Method 2: Bug Commit (4068682841) Check ==="
BUG_COMMIT="4068682841"
for branch in release-4.16 release-4.17 release-4.18 release-4.19 release-4.20; do
    if git branch -r --contains "$BUG_COMMIT" 2>/dev/null | grep -q "$branch"; then
        echo "$branch: ❌ Contains bug commit"
    else
        echo "$branch: ✅ No bug (safe)"
    fi
done
echo ""

# Method 3: Find commits that mention version numbers
echo "=== Method 3: Version-related Commits ==="
for version in 4.16 4.17 4.18 4.19 4.20; do
    echo "--- Version $version ---"
    git log --all --oneline --grep="$version" --grep="release.*$version" -i -3 --format="%h %ad %s" --date=short | head -3 || echo "  (no specific commits found)"
done
echo ""

# Method 4: Check default release image in code
echo "=== Method 4: Default Release Image in Code ==="
if [ -f "pkg/asset/releaseimage/default.go" ]; then
    echo "Current default release image:"
    grep -A 2 "defaultReleaseImageOriginal" pkg/asset/releaseimage/default.go | head -3 || echo "  (not found)"
else
    echo "  File not found"
fi
echo ""

# Method 5: Analyze by checking when release image changed
echo "=== Method 5: Release Image Changes ==="
git log --all --oneline --grep="release.*4\.[12]" -i --format="%h %ad %s" --date=short -- pkg/asset/releaseimage/default.go | head -10 || echo "  (no changes found)"
echo ""

# Method 6: Check tags
echo "=== Method 6: Version Tags ==="
git tag | grep -E "^v?4\.(1[6-9]|20)" | sort -V | tail -10
echo ""

# Method 7: Find commits before bug introduction
echo "=== Method 7: Commits Before Bug Introduction (2025-08-14) ==="
BUG_DATE="2025-08-14"
for branch in release-4.16 release-4.17 release-4.18 release-4.19; do
    if git show-ref --verify --quiet refs/remotes/origin/$branch; then
        last_before_bug=$(git log origin/$branch --before="$BUG_DATE" --oneline -1 --format="%h %ad %s" --date=short 2>/dev/null || echo "")
        if [ -n "$last_before_bug" ]; then
            echo "$branch: $last_before_bug"
        fi
    fi
done
echo ""

echo "=========================================="
echo "Summary"
echo "=========================================="
echo "To find installer commit for a specific OpenShift version:"
echo "1. Check release branch: git checkout release-4.19 && git log -1"
echo "2. Check release image: oc adm release info quay.io/openshift-release-dev/ocp-release:4.19.0-x86_64 --image-for=openshift-install"
echo "3. Check if bug exists: git branch -r --contains 4068682841 | grep release-4.19"
echo "=========================================="

Version	Branch	Latest Commit	Bug Status	Release Image Updated
4.16	release-4.16	0cba968788 (2026-01-16)	✅ No bug	-
4.17	release-4.17	64d3580eda (2025-11-07)	✅ No bug	2024-09-16
4.18	release-4.18	c9de42d6c4 (2026-01-22)	✅ No bug	2024-09-20
4.19	release-4.19	01a3127f01 (2026-01-23)	✅ No bug	2025-01-16
4.20	release-4.20	1a6e473ab1 (2026-01-22)	❌ Has bug	2025-06-16

Verified: Bug commit 4068682841 (2025-08-14) is NOT in release-4.16, 4.17, 4.18, or 4.19 branches.

cd /Users/weli/works/oc-swarm/installer && git checkout release-4.19 && git log -1 --oneline
branch 'release-4.19' set up to track 'origin/release-4.19'.
01a3127f01 Merge pull request #10248 from tlbueno/tb-4.19
Switched to a new branch 'release-4.19'
cd /Users/weli/works/oc-swarm/installer && hack/build.sh 2>&1 | tail -20
...
+ go build -mod=vendor -gcflags '' -ldflags ' -X github.com/openshift/installer/pkg/version.Raw=unreleased-master-12001-g01a3127f01c88a45c450e6790adb8d245634c542 -X github.com/openshift/installer/pkg/version.Commit=01a3127f01c88a45c450e6790adb8d245634c542 -X github.com/openshift/installer/pkg/version.defaultArch=amd64 -s -w' -tags ' release' -o bin/openshift-install ./cmd/openshift-install

cd /Users/weli/works/oc-swarm/installer && ./bin/openshift-install version
./bin/openshift-install unreleased-master-12001-g01a3127f01c88a45c450e6790adb8d245634c542
built from commit 01a3127f01c88a45c450e6790adb8d245634c542
release image registry.ci.openshift.org/origin/release:4.19
release architecture unknown
default architecture amd64
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown

Verification script:

#!/bin/bash
# Test zone consistency for OpenShift 4.19 installer
# Run 100 iterations to verify CAPI and MAPI zone allocation is deterministic

set -o errexit
set -o pipefail
set -o nounset

# Default values
INSTALLER="${1:-/Users/weli/works/oc-swarm/installer/bin/openshift-install}"
PULL_SECRET_FILE="${2:-}"
BASE_DOMAIN="${3:-qe.devcluster.openshift.com}"
REGION="${4:-us-east-1}"
ITERATIONS="${5:-100}"

WORK_DIR="/tmp/test-4.19-zone-consistency"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

usage() {
    cat << EOF
Usage: $0 [installer-path] [pull-secret-file] [base-domain] [region] [iterations]

Arguments:
  installer-path    Path to openshift-install binary (default: /Users/weli/works/oc-swarm/installer/bin/openshift-install)
  pull-secret-file  Path to pull secret JSON file (required)
  base-domain       Base domain for cluster (default: qe.devcluster.openshift.com)
  region            AWS region (default: us-east-1)
  iterations        Number of iterations (default: 100)

Example:
  $0 ./openshift-install ~/.pull-secret.json qe.devcluster.openshift.com us-east-1 100
EOF
    exit 1
}

# Check prerequisites
if [[ -z "$PULL_SECRET_FILE" || ! -f "$PULL_SECRET_FILE" ]]; then
    echo -e "${RED}Error: Pull secret file is required${NC}"
    usage
fi

if [[ ! -x "$INSTALLER" ]]; then
    echo -e "${RED}Error: Installer not found or not executable: $INSTALLER${NC}"
    exit 1
fi

if ! command -v yq &> /dev/null; then
    echo -e "${RED}Error: yq is required${NC}"
    exit 1
fi

# Read pull secret
PULL_SECRET=$(jq -c '.' "$PULL_SECRET_FILE" 2>/dev/null || cat "$PULL_SECRET_FILE")

# Generate temporary SSH key
TEMP_SSH_DIR=$(mktemp -d)
ssh-keygen -t ed25519 -f "$TEMP_SSH_DIR/temp_key" -N "" -q
SSH_PUB_KEY=$(cat "$TEMP_SSH_DIR/temp_key.pub")
rm -rf "$TEMP_SSH_DIR"

echo "=========================================="
echo "OCPBUGS-69923 Zone Consistency Test (4.19)"
echo "=========================================="
echo "Installer: $INSTALLER"
"$INSTALLER" version 2>/dev/null || echo "(version info unavailable)"
echo ""
echo "Region: $REGION"
echo "Iterations: $ITERATIONS"
echo "Work directory: $WORK_DIR"
echo "=========================================="
echo ""

TOTAL_FAILURES=0
FAILURE_DETAILS=""

for iteration in $(seq 1 "$ITERATIONS"); do
  echo "=========================================="
  echo "Iteration $iteration/$ITERATIONS"
  echo "=========================================="

  ITER_DIR="$WORK_DIR/iter-$iteration"
  rm -rf "$ITER_DIR"
  mkdir -p "$ITER_DIR"

  # Create install-config.yaml (without specifying zones - triggers the bug path)
  cat > "$ITER_DIR/install-config.yaml" << EOF
apiVersion: v1
baseDomain: ${BASE_DOMAIN}
metadata:
  name: test-4.19-${iteration}
controlPlane:
  architecture: amd64
  hyperthreading: Enabled
  name: master
  replicas: 3
compute:
- architecture: amd64
  hyperthreading: Enabled
  name: worker
  replicas: 3
platform:
  aws:
    region: ${REGION}
pullSecret: '${PULL_SECRET}'
sshKey: '${SSH_PUB_KEY}'
EOF

  # Generate manifests (show output like CI script)
  if ! "$INSTALLER" create manifests --dir "$ITER_DIR" 2>&1; then
    echo -e "${YELLOW}Warning: Iteration $iteration manifest generation failed${NC}"
    continue
  fi

  # Extract CAPI zones from cluster-api/machines/10_inframachine_*-master-*.yaml
  capi_zones=""
  for file in $(find "$ITER_DIR"/cluster-api/machines -name "10_inframachine_*-master-*.yaml" -type f 2>/dev/null | sort); do
    subnet_name=$(yq eval '.spec.subnet.filters[0].values[0]' "$file" 2>/dev/null || echo "")
    zone=$(echo "$subnet_name" | grep -oE "${REGION}[a-z]$" || echo "")
    if [[ -n "$zone" && "$zone" != "null" ]]; then
      capi_zones="${capi_zones} ${zone}"
    fi
  done
  capi_zones=$(echo "$capi_zones" | xargs)
  capi_count=$(echo "$capi_zones" | wc -w | xargs)

  # Extract MAPI zones from ControlPlaneMachineSet failureDomains
  mapi_zones=""
  mapi_count=0
  cpms_file="$ITER_DIR/openshift/99_openshift-machine-api_master-control-plane-machine-set.yaml"
  if [[ -f "$cpms_file" ]]; then
    idx=0
    while [[ $mapi_count -lt $capi_count ]]; do
      zone=$(yq eval ".spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[$idx].placement.availabilityZone" "$cpms_file" 2>/dev/null || echo "")
      if [[ -z "$zone" || "$zone" == "null" ]]; then
        break
      fi
      mapi_zones="${mapi_zones} ${zone}"
      mapi_count=$((mapi_count + 1))
      idx=$((idx + 1))
    done
  else
    echo -e "\n${YELLOW}Warning: Iteration $iteration - ControlPlaneMachineSet file not found${NC}"
    continue
  fi
  mapi_zones=$(echo "$mapi_zones" | xargs)

  # Compare
  echo "  CAPI zones (from cluster-api/machines/10_inframachine_*): $capi_zones"
  echo "  MAPI zones (from ControlPlaneMachineSet failureDomains): $mapi_zones"

  if [[ "$capi_zones" == "$mapi_zones" ]]; then
    echo "  PASS"
    # Clean up successful iteration
    rm -rf "$ITER_DIR"
  else
    echo "  ${RED}FAIL: zones mismatch - CAPI and MAPI have different zone assignments${NC}"
    TOTAL_FAILURES=$((TOTAL_FAILURES + 1))
    FAILURE_DETAILS="${FAILURE_DETAILS}\nIteration $iteration:\n  CAPI: $capi_zones\n  MAPI: $mapi_zones"
    # Keep failure directory for inspection
    mv "$ITER_DIR" "$WORK_DIR/fail-${iteration}"
  fi
done

echo ""
echo ""
echo "=========================================="
echo "Final Result: $ITERATIONS iterations completed"
echo "=========================================="
echo "Total failures: $TOTAL_FAILURES"

if [[ $TOTAL_FAILURES -gt 0 ]]; then
  failure_rate=$(echo "scale=2; $TOTAL_FAILURES * 100 / $ITERATIONS" | bc)
  echo -e "${RED}Failure rate: ${failure_rate}%${NC}"
  echo ""
  echo "Failure details:"
  echo -e "$FAILURE_DETAILS"
  echo ""
  echo "Failed iterations saved in: $WORK_DIR/fail-*"
  echo -e "${RED}Conclusion: BUG DETECTED in this installer version!${NC}"
else
  echo -e "${GREEN}Failure rate: 0%${NC}"
  echo -e "${GREEN}Conclusion: All iterations have consistent zone allocation${NC}"
  echo -e "${GREEN}This installer version does NOT have the bug.${NC}"
fi
echo "=========================================="

exit $TOTAL_FAILURES

Result:

cd /Users/weli/works/oc-swarm/my-openshift-workspace/OCPBUGS-69923 && ./test-4.19-zone-consistency.sh /Users/weli/works/oc-swarm/installer/bin/openshift-install ~/works/oc-swarm/openshift-versions/auth.json qe.devcluster.openshift.com us-east-1 10
==========================================
OCPBUGS-69923 Zone Consistency Test (4.19)
==========================================
Installer: /Users/weli/works/oc-swarm/installer/bin/openshift-install
/Users/weli/works/oc-swarm/installer/bin/openshift-install unreleased-master-12001-g01a3127f01c88a45c450e6790adb8d245634c542
built from commit 01a3127f01c88a45c450e6790adb8d245634c542
release image registry.ci.openshift.org/origin/release:4.19
release architecture unknown
default architecture amd64

Region: us-east-1
Iterations: 10
Work directory: /tmp/test-4.19-zone-consistency
==========================================

==========================================
Iteration 1/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:26Z 2026-01-27T07:08:26Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:26Z 2026-01-27T07:08:26Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-1/cluster-api, /tmp/test-4.19-zone-consistency/iter-1/manifests and /tmp/test-4.19-zone-consistency/iter-1/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 2/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:32Z 2026-01-27T07:08:32Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:32Z 2026-01-27T07:08:32Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-2/cluster-api, /tmp/test-4.19-zone-consistency/iter-2/manifests and /tmp/test-4.19-zone-consistency/iter-2/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 3/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:37Z 2026-01-27T07:08:37Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:37Z 2026-01-27T07:08:37Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-3/cluster-api, /tmp/test-4.19-zone-consistency/iter-3/manifests and /tmp/test-4.19-zone-consistency/iter-3/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 4/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:42Z 2026-01-27T07:08:42Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:42Z 2026-01-27T07:08:42Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-4/cluster-api, /tmp/test-4.19-zone-consistency/iter-4/manifests and /tmp/test-4.19-zone-consistency/iter-4/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 5/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:47Z 2026-01-27T07:08:47Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:47Z 2026-01-27T07:08:47Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-5/cluster-api, /tmp/test-4.19-zone-consistency/iter-5/manifests and /tmp/test-4.19-zone-consistency/iter-5/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 6/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:53Z 2026-01-27T07:08:53Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:53Z 2026-01-27T07:08:53Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-6/cluster-api, /tmp/test-4.19-zone-consistency/iter-6/manifests and /tmp/test-4.19-zone-consistency/iter-6/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 7/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:08:59Z 2026-01-27T07:08:59Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:08:59Z 2026-01-27T07:08:59Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-7/cluster-api, /tmp/test-4.19-zone-consistency/iter-7/manifests and /tmp/test-4.19-zone-consistency/iter-7/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 8/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:09:04Z 2026-01-27T07:09:04Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:09:04Z 2026-01-27T07:09:04Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-8/cluster-api, /tmp/test-4.19-zone-consistency/iter-8/manifests and /tmp/test-4.19-zone-consistency/iter-8/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 9/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:09:10Z 2026-01-27T07:09:10Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:09:10Z 2026-01-27T07:09:10Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-9/cluster-api, /tmp/test-4.19-zone-consistency/iter-9/manifests and /tmp/test-4.19-zone-consistency/iter-9/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 10/10
==========================================
level=warning msg=Release Image Architecture not detected. Release Image Architecture is unknown
level=info msg=Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials"
level=info msg=Consuming Install Config from target directory
level=info msg=Successfully populated MCS CA cert information: root-ca 2036-01-25T07:09:14Z 2026-01-27T07:09:14Z
level=info msg=Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:09:14Z 2026-01-27T07:09:14Z
level=info msg=Adding clusters...
level=info msg=Manifests created in: /tmp/test-4.19-zone-consistency/iter-10/cluster-api, /tmp/test-4.19-zone-consistency/iter-10/manifests and /tmp/test-4.19-zone-consistency/iter-10/openshift
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS


==========================================
Final Result: 10 iterations completed
==========================================
Total failures: 0
Failure rate: 0%
Conclusion: All iterations have consistent zone allocation
This installer version does NOT have the bug.
==========================================

Using the test script to test against a commit that contains the problem:

❯ cd /Users/weli/works/oc-swarm/my-openshift-workspace/OCPBUGS-69923 && LOG_LEVEL=debug ./test-4.19-zone-consistency.sh /Users/w
eli/works/oc-swarm/installer/bin/openshift-install ~/works/oc-swarm/openshift-versions/auth.json qe.devcluster.openshift.com us-
east-1 2
==========================================
OCPBUGS-69923 Zone Consistency Test (4.19)
==========================================
Installer: /Users/weli/works/oc-swarm/installer/bin/openshift-install
/Users/weli/works/oc-swarm/installer/bin/openshift-install unreleased-master-12138-g4068682841f807383c1ada67691f53cd1f2022bc-dirty
built from commit 4068682841f807383c1ada67691f53cd1f2022bc
release image registry.ci.openshift.org/origin/release:4.20
release architecture unknown
default architecture amd64

Region: us-east-1
Iterations: 2
Work directory: /tmp/test-4.19-zone-consistency
==========================================

==========================================
Iteration 1/2
==========================================
WARNING Release Image Architecture not detected. Release Image Architecture is unknown 
INFO Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials" 
INFO Credentials loaded from the AWS config using "SharedConfigCredentials: /Users/weli/.aws/credentials" provider 
INFO Consuming Install Config from target directory 
INFO Successfully populated MCS CA cert information: root-ca 2036-01-25T07:44:56Z 2026-01-27T07:44:56Z 
INFO Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:44:56Z 2026-01-27T07:44:56Z 
INFO Adding clusters...                           
INFO Manifests created in: /tmp/test-4.19-zone-consistency/iter-1/cluster-api, /tmp/test-4.19-zone-consistency/iter-1/manifests and /tmp/test-4.19-zone-consistency/iter-1/openshift 
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1a us-east-1b us-east-1c
  PASS
==========================================
Iteration 2/2
==========================================
WARNING Release Image Architecture not detected. Release Image Architecture is unknown 
INFO Credentials loaded from the "default" profile in file "/Users/weli/.aws/credentials" 
INFO Credentials loaded from the AWS config using "SharedConfigCredentials: /Users/weli/.aws/credentials" provider 
INFO Consuming Install Config from target directory 
INFO Successfully populated MCS CA cert information: root-ca 2036-01-25T07:45:09Z 2026-01-27T07:45:09Z 
INFO Successfully populated MCS TLS cert information: root-ca 2036-01-25T07:45:09Z 2026-01-27T07:45:09Z 
INFO Adding clusters...                           
INFO Manifests created in: /tmp/test-4.19-zone-consistency/iter-2/cluster-api, /tmp/test-4.19-zone-consistency/iter-2/manifests and /tmp/test-4.19-zone-consistency/iter-2/openshift 
  CAPI zones (from cluster-api/machines/10_inframachine_*): us-east-1a us-east-1b us-east-1c
  MAPI zones (from ControlPlaneMachineSet failureDomains): us-east-1f us-east-1a us-east-1b
  \033[0;31mFAIL: zones mismatch - CAPI and MAPI have different zone assignments\033[0m


==========================================
Final Result: 2 iterations completed
==========================================
Total failures: 1
Failure rate: 50.00%

Failure details:

Iteration 2:
  CAPI: us-east-1a us-east-1b us-east-1c
  MAPI: us-east-1f us-east-1a us-east-1b

Failed iterations saved in: /tmp/test-4.19-zone-consistency/fail-*
Conclusion: BUG DETECTED in this installer version!
==========================================