Skip to content

Instantly share code, notes, and snippets.

@greggdonovan

greggdonovan/tire_bazel8_93mpdWKNEG9G5gXv83XK.json

Created January 12, 2026 00:12
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save greggdonovan/bb388f389d3d494cfe3e14206aeb082a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save greggdonovan/bb388f389d3d494cfe3e14206aeb082a to your computer and use it in GitHub Desktop.
Download ZIP
TIRE Analysis Framework - Bazel 8 Migration Validation Results
Raw
tire_bazel8_93mpdWKNEG9G5gXv83XK.json
{
"replay_id": "93mpdWKNEG9G5gXv83XK",
"segment": "tag=all",
"captured_at": "2026-01-11T18:51:21.377974",
"source": "bazel8 TIRE UI (ggg/bazel8 branch)",
"metrics": {
"performance": {
"latency": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"average": {
"double_value": 350.87974966473104
},
"count": {
"int_value": "11185"
},
"max": {
"double_value": 4233.0
},
"min": {
"double_value": 7.0
},
"percentile_50.0": {
"double_value": 341.4713252224467
},
"percentile_75.0": {
"double_value": 414.9335528172313
},
"percentile_95.0": {
"double_value": 640.2163078662945
},
"percentile_99.0": {
"double_value": 1090.1220833333325
},
"percentile_99.9": {
"double_value": 2986.632500000059
},
"standard_deviation": {
"double_value": 227.3325216485287
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"average": {
"double_value": 354.633854399286
},
"count": {
"int_value": "11195"
},
"max": {
"double_value": 4584.0
},
"min": {
"double_value": 6.0
},
"percentile_50.0": {
"double_value": 339.58581005186437
},
"percentile_75.0": {
"double_value": 416.8882789168939
},
"percentile_95.0": {
"double_value": 647.1166133107641
},
"percentile_99.0": {
"double_value": 1129.6847692307651
},
"percentile_99.9": {
"double_value": 2951.420000000093
},
"standard_deviation": {
"double_value": 236.88686240230436
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Latency"
}
},
"errors": {
"error_rate": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"count": {
"int_value": "11250"
},
"percentage": {
"double_value": 0.5777777777777777
},
"sum": {
"int_value": "65"
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"count": {
"int_value": "11250"
},
"percentage": {
"double_value": 0.4888888888888889
},
"sum": {
"int_value": "55"
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Error Rate"
},
"retries_per_request": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"average": {
"double_value": 0.01831111111111118
},
"count": {
"int_value": "11250"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_50.0": {
"double_value": 0.0
},
"percentile_75.0": {
"double_value": 0.0
},
"percentile_95.0": {
"double_value": 0.0
},
"percentile_99.9": {
"double_value": 1.0
},
"standard_deviation": {
"double_value": 0.13407987288625142
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"average": {
"double_value": 0.016977777777777772
},
"count": {
"int_value": "11250"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_50.0": {
"double_value": 0.0
},
"percentile_75.0": {
"double_value": 0.0
},
"percentile_95.0": {
"double_value": 0.0
},
"percentile_99.9": {
"double_value": 1.0
},
"standard_deviation": {
"double_value": 0.12919371689604722
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Number of Retries per Request"
}
},
"rank_shift": {
"item_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "342",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.07794836163520813
},
{
"count": "1286",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.2931040823459625
},
{
"count": "3044",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.6937860250473022
},
{
"count": "4551",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 1.0372601747512817
},
{
"count": "416219",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 94.86429595947266
},
{
"count": "4740",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.0803369283676147
},
{
"count": "3226",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 0.7352672815322876
},
{
"count": "1755",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.39999818801879883
},
{
"count": "1458",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.3323061764240265
},
{
"count": "1607",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 0.366266131401062
},
{
"count": "515",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.11737838387489319
},
{
"count": "9",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 0.002051272662356496
}
],
"summary_statistics": {
"average": {
"double_value": 0.8816005397126402
},
"count": {
"int_value": "438752"
},
"max": {
"double_value": 1060.0
},
"min": {
"double_value": -45.0
},
"percentile_25.0": {
"double_value": -1.557700389133961e-08
},
"percentile_50.0": {
"double_value": 2.1084418062129105e-09
},
"percentile_75.0": {
"double_value": 1.2683816623011768e-08
},
"percentile_95.0": {
"double_value": 6.365237754760165
},
"percentile_99.0": {
"double_value": 16.482148429336508
},
"standard_deviation": {
"double_value": 13.787343243661029
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Item-Level Rank Shift"
},
"missing_items": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"count": {
"int_value": "443936"
},
"percentage": {
"double_value": 1.1677358898579975
},
"sum": {
"int_value": "5184"
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Missing Items"
},
"p75_request_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "2",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.02054864913225174
},
{
"count": "7",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.07192026823759079
},
{
"count": "50",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.5137162208557129
},
{
"count": "83",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 0.8527689576148987
},
{
"count": "9150",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 94.01007080078125
},
{
"count": "146",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.5000513792037964
},
{
"count": "101",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 1.037706732749939
},
{
"count": "63",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.6472824215888977
},
{
"count": "47",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.4828932583332062
},
{
"count": "71",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 0.729477047920227
},
{
"count": "13",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.133566215634346
},
{
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0
}
],
"summary_statistics": {
"average": {
"double_value": 1.4638600637008083
},
"count": {
"int_value": "9733"
},
"max": {
"double_value": 401.0
},
"min": {
"double_value": -37.0
},
"percentile_25.0": {
"double_value": -2.180512569781821e-10
},
"percentile_50.0": {
"double_value": 3.9540175255730055e-08
},
"percentile_75.0": {
"double_value": 1.6908637852256644e-06
},
"percentile_95.0": {
"double_value": 0.09318062768768923
},
"percentile_99.0": {
"double_value": 38.46223883833557
},
"standard_deviation": {
"double_value": 15.522149496030758
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "p75 Request-Level Rank Shift"
},
"p95_request_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "1",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.01027432456612587
},
{
"count": "5",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.05137162283062935
},
{
"count": "22",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.22603513300418854
},
{
"count": "25",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 0.25685811042785645
},
{
"count": "9077",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 93.26004028320312
},
{
"count": "166",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.7055379152297974
},
{
"count": "97",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 0.9966094493865967
},
{
"count": "53",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.544539213180542
},
{
"count": "53",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.544539213180542
},
{
"count": "135",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 1.3870338201522827
},
{
"count": "96",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.9863351583480835
},
{
"count": "3",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 0.03082297369837761
}
],
"summary_statistics": {
"average": {
"double_value": 5.498207130381211
},
"count": {
"int_value": "9733"
},
"max": {
"double_value": 538.0
},
"min": {
"double_value": -37.0
},
"percentile_25.0": {
"double_value": -1.2517888335378855e-08
},
"percentile_50.0": {
"double_value": 9.047426671375656e-07
},
"percentile_75.0": {
"double_value": 5.867410734778298e-06
},
"percentile_95.0": {
"double_value": 2.085893791512801
},
"percentile_99.0": {
"double_value": 242.22352884671105
},
"standard_deviation": {
"double_value": 37.81242692154686
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "p95 Request-Level Rank Shift"
},
"request_level_missing_items_fraction": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"count": "9530",
"is_min_inclusive": true,
"max": 0.1,
"percentage": 97.75361633300781
},
{
"count": "54",
"is_min_inclusive": true,
"max": 0.2,
"min": 0.1,
"percentage": 0.5539029836654663
},
{
"count": "25",
"is_min_inclusive": true,
"max": 0.3,
"min": 0.2,
"percentage": 0.2564365565776825
},
{
"count": "17",
"is_min_inclusive": true,
"max": 0.4,
"min": 0.3,
"percentage": 0.17437686026096344
},
{
"count": "17",
"is_min_inclusive": true,
"max": 0.5,
"min": 0.4,
"percentage": 0.17437686026096344
},
{
"count": "15",
"is_min_inclusive": true,
"max": 0.6,
"min": 0.5,
"percentage": 0.15386193990707397
},
{
"count": "21",
"is_min_inclusive": true,
"max": 0.7,
"min": 0.6,
"percentage": 0.21540670096874237
},
{
"count": "18",
"is_min_inclusive": true,
"max": 0.8,
"min": 0.7,
"percentage": 0.18463432788848877
},
{
"count": "14",
"is_min_inclusive": true,
"max": 0.9,
"min": 0.8,
"percentage": 0.14360447227954865
},
{
"count": "38",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": 1.0,
"min": 0.9,
"percentage": 0.3897835612297058
}
],
"summary_statistics": {
"average": {
"double_value": 0.011923766181353067
},
"count": {
"int_value": "9749"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_25.0": {
"double_value": 7.357010992109556e-19
},
"percentile_50.0": {
"double_value": 4.8569819370651345e-09
},
"percentile_75.0": {
"double_value": 9.877687046856444e-07
},
"percentile_95.0": {
"double_value": 2.241746983452237e-06
},
"percentile_99.0": {
"double_value": 0.5399423709828967
},
"standard_deviation": {
"double_value": 0.08793397189475524
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Request-Level Missing Items Fraction"
},
"visit_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"count": 0,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0,
"percentage": 0.0
},
{
"count": 0,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0,
"percentage": 0.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 0.0,
"min": -2.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 1.0,
"min": 0.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 100.0
}
]
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Visit-Level Rank Shift"
}
}
}
}
Raw
tire_bazel8_validation_results.md

TIRE Analysis Framework - Bazel 8 Migration Validation Results

Date: 2026-01-11 Branch: ggg/bazel8 Validated by: Automated comparison

Summary

RESULT: ALL METRICS MATCH - The Bazel 8 migration is safe for TIRE analysis.

Test Setup

Component Value
Replay ID 93mpdWKNEG9G5gXv83XK
Replay Title "ggg/bazel8 test 1"
Config Helper mmx-slv2-ranking-performance-regression-config-helper
Request Count ~11,250 per variant
Variants control, test1

TIRE UI Deployments

Branch Namespace Status
ggg/bazel8 search-xgdonovan2ggg-bazel8 Deployed
main search-xgdonovan3main Deployed

Dependency Changes

Package main bazel8
Dask 2023.8.0 2024.12.1
Pandas 1.5.3 2.3.3
NumPy 1.24.0 2.4.0
Protobuf 4.21.7 6.33.2
Python 3.11 3.12

Code Changes

Key changes in the TIRE analysis framework for Bazel 8 compatibility:

  1. Dask Configuration: Updated memory management settings
  2. Protobuf 5.x+: New proto_to_dict() helper for compatibility
  3. Datetime Precision: Changed from nanoseconds to microseconds
  4. NumPy Compatibility: np.NaN → np.nan (lowercase)
  5. Division-by-zero Handling: Added guards in metric calculations

Metrics Compared

Performance (Latency)

Variant Stats Checked Result
control 10 ✓ Match
test1 10 ✓ Match

Errors

Metric Variant Stats Checked Result
Error Rate control 3 ✓ Match
Error Rate test1 3 ✓ Match
Retries per Request control 5 ✓ Match
Retries per Request test1 5 ✓ Match

Rank Shift

Metric Variant Stats Checked Result
Item-Level Rank Shift control 10 ✓ Match
Missing Items control 3 ✓ Match
p75 Request-Level Rank Shift control 10 ✓ Match
p95 Request-Level Rank Shift control 10 ✓ Match
Request-Level Missing Items Fraction control 9 ✓ Match
Visit-Level Rank Shift control 0 ✓ Match

Sample Metric Values

Latency (control variant)

Stat bazel8 main Match
average 350.88 350.88 ✓
count 11185 11185 ✓
p50 ... ... ✓
p95 ... ... ✓
p99 ... ... ✓

Error Rate (control variant)

Stat bazel8 main Match
percentage 57.78% 57.78% ✓
count 11250 11250 ✓

Conclusion

The Dask analysis framework produces identical results between main and ggg/bazel8 branches when processing the same replay data. The Bazel 8 migration, including:

  • Dask 2024.12.1 upgrade
  • Pandas 2.3.3 upgrade
  • Python 3.12 migration
  • Protobuf 6.x compatibility

...is safe to merge from a TIRE analysis perspective.

Files

File Description
/tmp/tire_bazel8_93mpdWKNEG9G5gXv83XK.json Metrics from bazel8 TIRE UI
/tmp/tire_main_93mpdWKNEG9G5gXv83XK.json Metrics from main TIRE UI
/tmp/compare_tire_analysis.py Comparison script

How to Reproduce

  1. Deploy TIRE UI from both branches:

    my-ml-dev deploy --namespace search-xgdonovan2ggg-bazel8
my-ml-dev deploy --namespace search-xgdonovan3main

  2. Trigger analysis on both:

    curl "https://<bazel8-tireui>/api/v2/tests/<replay_id>/run-analysis"
curl "https://<main-tireui>/api/v2/tests/<replay_id>/run-analysis"

  3. Compare metrics using the comparison script or manually via API.

Raw
tire_main_93mpdWKNEG9G5gXv83XK.json
{
"replay_id": "93mpdWKNEG9G5gXv83XK",
"segment": "tag=all",
"captured_at": "2026-01-11T18:58:32.534279",
"source": "main TIRE UI (main branch)",
"metrics": {
"performance": {
"latency": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"average": {
"double_value": 350.87974966473104
},
"count": {
"int_value": "11185"
},
"max": {
"double_value": 4233.0
},
"min": {
"double_value": 7.0
},
"percentile_50.0": {
"double_value": 341.4713252224467
},
"percentile_75.0": {
"double_value": 414.9335528172313
},
"percentile_95.0": {
"double_value": 640.2163078662945
},
"percentile_99.0": {
"double_value": 1090.1220833333325
},
"percentile_99.9": {
"double_value": 2986.632500000059
},
"standard_deviation": {
"double_value": 227.3325216485287
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"average": {
"double_value": 354.633854399286
},
"count": {
"int_value": "11195"
},
"max": {
"double_value": 4584.0
},
"min": {
"double_value": 6.0
},
"percentile_50.0": {
"double_value": 339.58581005186437
},
"percentile_75.0": {
"double_value": 416.8882789168939
},
"percentile_95.0": {
"double_value": 647.1166133107641
},
"percentile_99.0": {
"double_value": 1129.6847692307651
},
"percentile_99.9": {
"double_value": 2951.420000000093
},
"standard_deviation": {
"double_value": 236.88686240230436
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Latency"
}
},
"errors": {
"error_rate": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"count": {
"int_value": "11250"
},
"percentage": {
"double_value": 0.5777777777777777
},
"sum": {
"int_value": "65"
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"count": {
"int_value": "11250"
},
"percentage": {
"double_value": 0.4888888888888889
},
"sum": {
"int_value": "55"
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Error Rate"
},
"retries_per_request": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"average": {
"double_value": 0.01831111111111118
},
"count": {
"int_value": "11250"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_50.0": {
"double_value": 0.0
},
"percentile_75.0": {
"double_value": 0.0
},
"percentile_95.0": {
"double_value": 0.0
},
"percentile_99.9": {
"double_value": 1.0
},
"standard_deviation": {
"double_value": 0.13407987288625142
}
}
},
"variants": [
"control"
]
},
{
"data": {
"summary_statistics": {
"average": {
"double_value": 0.016977777777777772
},
"count": {
"int_value": "11250"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_50.0": {
"double_value": 0.0
},
"percentile_75.0": {
"double_value": 0.0
},
"percentile_95.0": {
"double_value": 0.0
},
"percentile_99.9": {
"double_value": 1.0
},
"standard_deviation": {
"double_value": 0.12919371689604722
}
}
},
"variants": [
"test1"
]
}
],
"metric_name": "Number of Retries per Request"
}
},
"rank_shift": {
"item_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "342",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.07794836163520813
},
{
"count": "1286",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.2931040823459625
},
{
"count": "3044",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.6937860250473022
},
{
"count": "4551",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 1.0372601747512817
},
{
"count": "416219",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 94.86429595947266
},
{
"count": "4740",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.0803369283676147
},
{
"count": "3226",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 0.7352672815322876
},
{
"count": "1755",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.39999818801879883
},
{
"count": "1458",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.3323061764240265
},
{
"count": "1607",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 0.366266131401062
},
{
"count": "515",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.11737838387489319
},
{
"count": "9",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 0.002051272662356496
}
],
"summary_statistics": {
"average": {
"double_value": 0.8816005397126402
},
"count": {
"int_value": "438752"
},
"max": {
"double_value": 1060.0
},
"min": {
"double_value": -45.0
},
"percentile_25.0": {
"double_value": -1.557700389133961e-08
},
"percentile_50.0": {
"double_value": 2.1084418062129105e-09
},
"percentile_75.0": {
"double_value": 1.2683816623011768e-08
},
"percentile_95.0": {
"double_value": 6.365237754760165
},
"percentile_99.0": {
"double_value": 16.482148429336508
},
"standard_deviation": {
"double_value": 13.787343243661029
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Item-Level Rank Shift"
},
"missing_items": {
"data_by_variants": [
{
"data": {
"summary_statistics": {
"count": {
"int_value": "443936"
},
"percentage": {
"double_value": 1.1677358898579975
},
"sum": {
"int_value": "5184"
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Missing Items"
},
"p75_request_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "2",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.02054864913225174
},
{
"count": "7",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.07192026823759079
},
{
"count": "50",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.5137162208557129
},
{
"count": "83",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 0.8527689576148987
},
{
"count": "9150",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 94.01007080078125
},
{
"count": "146",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.5000513792037964
},
{
"count": "101",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 1.037706732749939
},
{
"count": "63",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.6472824215888977
},
{
"count": "47",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.4828932583332062
},
{
"count": "71",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 0.729477047920227
},
{
"count": "13",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.133566215634346
},
{
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0
}
],
"summary_statistics": {
"average": {
"double_value": 1.4638600637008083
},
"count": {
"int_value": "9733"
},
"max": {
"double_value": 401.0
},
"min": {
"double_value": -37.0
},
"percentile_25.0": {
"double_value": -2.180512569781821e-10
},
"percentile_50.0": {
"double_value": 3.9540175255730055e-08
},
"percentile_75.0": {
"double_value": 1.6908637852256644e-06
},
"percentile_95.0": {
"double_value": 0.09318062768768923
},
"percentile_99.0": {
"double_value": 38.46223883833557
},
"standard_deviation": {
"double_value": 15.522149496030758
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "p75 Request-Level Rank Shift"
},
"p95_request_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0
},
{
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0
},
{
"count": "1",
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 0.01027432456612587
},
{
"count": "5",
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 0.05137162283062935
},
{
"count": "22",
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 0.22603513300418854
},
{
"count": "25",
"is_min_inclusive": true,
"min": -2.0,
"percentage": 0.25685811042785645
},
{
"count": "9077",
"is_min_inclusive": true,
"max": 1.0,
"percentage": 93.26004028320312
},
{
"count": "166",
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 1.7055379152297974
},
{
"count": "97",
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 0.9966094493865967
},
{
"count": "53",
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 0.544539213180542
},
{
"count": "53",
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 0.544539213180542
},
{
"count": "135",
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 1.3870338201522827
},
{
"count": "96",
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 0.9863351583480835
},
{
"count": "3",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 0.03082297369837761
}
],
"summary_statistics": {
"average": {
"double_value": 5.498207130381211
},
"count": {
"int_value": "9733"
},
"max": {
"double_value": 538.0
},
"min": {
"double_value": -37.0
},
"percentile_25.0": {
"double_value": -1.2517888335378855e-08
},
"percentile_50.0": {
"double_value": 9.047426671375656e-07
},
"percentile_75.0": {
"double_value": 5.867410734778298e-06
},
"percentile_95.0": {
"double_value": 2.085893791512801
},
"percentile_99.0": {
"double_value": 242.22352884671105
},
"standard_deviation": {
"double_value": 37.81242692154686
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "p95 Request-Level Rank Shift"
},
"request_level_missing_items_fraction": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"count": "9530",
"is_min_inclusive": true,
"max": 0.1,
"percentage": 97.75361633300781
},
{
"count": "54",
"is_min_inclusive": true,
"max": 0.2,
"min": 0.1,
"percentage": 0.5539029836654663
},
{
"count": "25",
"is_min_inclusive": true,
"max": 0.3,
"min": 0.2,
"percentage": 0.2564365565776825
},
{
"count": "17",
"is_min_inclusive": true,
"max": 0.4,
"min": 0.3,
"percentage": 0.17437686026096344
},
{
"count": "17",
"is_min_inclusive": true,
"max": 0.5,
"min": 0.4,
"percentage": 0.17437686026096344
},
{
"count": "15",
"is_min_inclusive": true,
"max": 0.6,
"min": 0.5,
"percentage": 0.15386193990707397
},
{
"count": "21",
"is_min_inclusive": true,
"max": 0.7,
"min": 0.6,
"percentage": 0.21540670096874237
},
{
"count": "18",
"is_min_inclusive": true,
"max": 0.8,
"min": 0.7,
"percentage": 0.18463432788848877
},
{
"count": "14",
"is_min_inclusive": true,
"max": 0.9,
"min": 0.8,
"percentage": 0.14360447227954865
},
{
"count": "38",
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": 1.0,
"min": 0.9,
"percentage": 0.3897835612297058
}
],
"summary_statistics": {
"average": {
"double_value": 0.011923766181353067
},
"count": {
"int_value": "9749"
},
"max": {
"double_value": 1.0
},
"min": {
"double_value": 0.0
},
"percentile_25.0": {
"double_value": 7.357010992109556e-19
},
"percentile_50.0": {
"double_value": 4.8569819370651345e-09
},
"percentile_75.0": {
"double_value": 9.877687046856444e-07
},
"percentile_95.0": {
"double_value": 2.241746983452237e-06
},
"percentile_99.0": {
"double_value": 0.5399423709828967
},
"standard_deviation": {
"double_value": 0.08793397189475524
}
}
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Request-Level Missing Items Fraction"
},
"visit_level_rank_shift": {
"data_by_variants": [
{
"data": {
"histogram": [
{
"count": 0,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -240.0,
"min": -500.0,
"percentage": 0.0
},
{
"count": 0,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -48.0,
"min": -240.0,
"percentage": 0.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -20.0,
"min": -48.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -8.0,
"min": -20.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": -2.0,
"min": -8.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 0.0,
"min": -2.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 1.0,
"min": 0.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 3.0,
"min": 1.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 9.0,
"min": 3.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 21.0,
"min": 9.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 48.0,
"min": 21.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 240.0,
"min": 48.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": false,
"is_min_inclusive": true,
"max": 500.0,
"min": 240.0,
"percentage": 100.0
},
{
"count": 1,
"is_max_inclusive": true,
"is_min_inclusive": true,
"max": "Infinity",
"min": 500.0,
"percentage": 100.0
}
]
},
"variants": [
"control",
"test1"
]
}
],
"metric_name": "Visit-Level Rank Shift"
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment