Last active
December 18, 2025 15:38
-
-
Save schwesig/0f25d77f25a73abd547f28688525a347 to your computer and use it in GitHub Desktop.
S3_Storage_Observability
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # S3 on OpenStack NERC, migrated from minio on 2025-06-13 (minio started Jan 15 2025) | |
| ## 2025-09-23 03:00pm | |
| ### Bucket: acm-metrics | |
| Total Objects: 2512 | |
| Total Size: 477.3 GiB | |
| ### Bucket: acm-metrics-hypershift2 | |
| Total Objects: 1339 | |
| Total Size: 14.8 GiB | |
| ### Bucket: acm-metrics-test | |
| Total Objects: 1057 | |
| Total Size: 20.0 GiB | |
| ### Bucket: loki-logs-test | |
| Total Objects: 0 | |
| Total Size: 0 Bytes | |
| ### Bucket: open-telemetry | |
| Total Objects: 0 | |
| Total Size: 0 Bytes | |
| ### Bucket: open-telemetry-test | |
| Total Objects: 0 | |
| Total Size: 0 Bytes |
Author
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-08-24 | 7154521773 | 6823.08 | 6.66 |
| 2025-08-23 | 0 | 0.00 | 0.00 |
| 2025-08-22 | 7382739482 | 7040.73 | 6.88 |
| 2025-08-21 | 0 | 0.00 | 0.00 |
| 2025-08-20 | 7483204543 | 7136.54 | 6.97 |
| 2025-08-19 | 0 | 0.00 | 0.00 |
| 2025-08-18 | 7423028718 | 7079.15 | 6.91 |
| 2025-08-17 | 0 | 0.00 | 0.00 |
| 2025-08-16 | 7426459242 | 7082.42 | 6.92 |
| 2025-08-15 | 0 | 0.00 | 0.00 |
| 2025-08-14 | 2034804720 | 1940.54 | 1.90 |
| 2025-08-13 | 0 | 0.00 | 0.00 |
| 2025-08-12 | 262564722 | 250.40 | 0.24 |
| 2025-08-11 | 1632107933 | 1556.50 | 1.52 |
| 2025-08-10 | 1112547562 | 1061.01 | 1.04 |
| 2025-08-09 | 6862777600 | 6544.85 | 6.39 |
| 2025-08-08 | 6289871912 | 5998.49 | 5.86 |
| 2025-08-07 | 2196974 | 2.10 | 0.00 |
| 2025-08-06 | 6210674888 | 5922.96 | 5.78 |
| 2025-08-05 | 0 | 0.00 | 0.00 |
| 2025-08-04 | 6162319486 | 5876.85 | 5.74 |
| 2025-08-03 | 0 | 0.00 | 0.00 |
| 2025-08-02 | 6222776592 | 5934.50 | 5.80 |
| 2025-08-01 | 0 | 0.00 | 0.00 |
| 2025-07-31 | 34940195989 | 33321.57 | 32.54 |
| 2025-07-30 | 0 | 0.00 | 0.00 |
| 2025-07-29 | 0 | 0.00 | 0.00 |
| 2025-07-28 | 0 | 0.00 | 0.00 |
| 2025-07-27 | 0 | 0.00 | 0.00 |
| 2025-07-26 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-07-25 | 0 | 0.00 | 0.00 |
| 2025-07-24 | 0 | 0.00 | 0.00 |
| 2025-07-23 | 0 | 0.00 | 0.00 |
| 2025-07-22 | 0 | 0.00 | 0.00 |
| 2025-07-21 | 0 | 0.00 | 0.00 |
| 2025-07-20 | 0 | 0.00 | 0.00 |
| 2025-07-19 | 0 | 0.00 | 0.00 |
| 2025-07-18 | 0 | 0.00 | 0.00 |
| 2025-07-17 | 32729023794 | 31212.83 | 30.48 |
| 2025-07-16 | 0 | 0.00 | 0.00 |
| 2025-07-15 | 0 | 0.00 | 0.00 |
| 2025-07-14 | 0 | 0.00 | 0.00 |
| 2025-07-13 | 0 | 0.00 | 0.00 |
| 2025-07-12 | 0 | 0.00 | 0.00 |
| 2025-07-11 | 0 | 0.00 | 0.00 |
| 2025-07-10 | 0 | 0.00 | 0.00 |
| 2025-07-09 | 0 | 0.00 | 0.00 |
| 2025-07-08 | 0 | 0.00 | 0.00 |
| 2025-07-07 | 0 | 0.00 | 0.00 |
| 2025-07-06 | 0 | 0.00 | 0.00 |
| 2025-07-05 | 0 | 0.00 | 0.00 |
| 2025-07-04 | 0 | 0.00 | 0.00 |
| 2025-07-03 | 34190308430 | 32606.42 | 31.84 |
| 2025-07-02 | 0 | 0.00 | 0.00 |
| 2025-07-01 | 181069482265 | 172681.31 | 168.63 |
| 2025-06-30 | 0 | 0.00 | 0.00 |
| 2025-06-29 | 0 | 0.00 | 0.00 |
| 2025-06-28 | 0 | 0.00 | 0.00 |
| 2025-06-27 | 0 | 0.00 | 0.00 |
| 2025-06-26 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-06-25 | 0 | 0.00 | 0.00 |
| 2025-06-24 | 0 | 0.00 | 0.00 |
| 2025-06-23 | 0 | 0.00 | 0.00 |
| 2025-06-22 | 0 | 0.00 | 0.00 |
| 2025-06-21 | 0 | 0.00 | 0.00 |
| 2025-06-20 | 0 | 0.00 | 0.00 |
| 2025-06-19 | 2268854253 | 2163.75 | 2.11 |
| 2025-06-18 | 0 | 0.00 | 0.00 |
| 2025-06-17 | 0 | 0.00 | 0.00 |
| 2025-06-16 | 0 | 0.00 | 0.00 |
| 2025-06-15 | 0 | 0.00 | 0.00 |
| 2025-06-14 | 0 | 0.00 | 0.00 |
| 2025-06-13 | 0 | 0.00 | 0.00 |
| 2025-06-12 | 0 | 0.00 | 0.00 |
| 2025-06-11 | 0 | 0.00 | 0.00 |
| 2025-06-10 | 0 | 0.00 | 0.00 |
| 2025-06-09 | 0 | 0.00 | 0.00 |
| 2025-06-08 | 0 | 0.00 | 0.00 |
| 2025-06-07 | 0 | 0.00 | 0.00 |
| 2025-06-06 | 0 | 0.00 | 0.00 |
| 2025-06-05 | 0 | 0.00 | 0.00 |
| 2025-06-04 | 0 | 0.00 | 0.00 |
| 2025-06-03 | 0 | 0.00 | 0.00 |
| 2025-06-02 | 0 | 0.00 | 0.00 |
| 2025-06-01 | 0 | 0.00 | 0.00 |
| 2025-05-31 | 0 | 0.00 | 0.00 |
| 2025-05-30 | 0 | 0.00 | 0.00 |
| 2025-05-29 | 0 | 0.00 | 0.00 |
| 2025-05-28 | 0 | 0.00 | 0.00 |
| 2025-05-27 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-05-26 | 0 | 0.00 | 0.00 |
| 2025-05-25 | 0 | 0.00 | 0.00 |
| 2025-05-24 | 0 | 0.00 | 0.00 |
| 2025-05-23 | 0 | 0.00 | 0.00 |
| 2025-05-22 | 0 | 0.00 | 0.00 |
| 2025-05-21 | 0 | 0.00 | 0.00 |
| 2025-05-20 | 0 | 0.00 | 0.00 |
| 2025-05-19 | 0 | 0.00 | 0.00 |
| 2025-05-18 | 0 | 0.00 | 0.00 |
| 2025-05-17 | 0 | 0.00 | 0.00 |
| 2025-05-16 | 0 | 0.00 | 0.00 |
| 2025-05-15 | 0 | 0.00 | 0.00 |
| 2025-05-14 | 0 | 0.00 | 0.00 |
| 2025-05-13 | 0 | 0.00 | 0.00 |
| 2025-05-12 | 0 | 0.00 | 0.00 |
| 2025-05-11 | 0 | 0.00 | 0.00 |
| 2025-05-10 | 0 | 0.00 | 0.00 |
| 2025-05-09 | 0 | 0.00 | 0.00 |
| 2025-05-08 | 0 | 0.00 | 0.00 |
| 2025-05-07 | 0 | 0.00 | 0.00 |
| 2025-05-06 | 0 | 0.00 | 0.00 |
| 2025-05-05 | 0 | 0.00 | 0.00 |
| 2025-05-04 | 0 | 0.00 | 0.00 |
| 2025-05-03 | 0 | 0.00 | 0.00 |
| 2025-05-02 | 0 | 0.00 | 0.00 |
| 2025-05-01 | 0 | 0.00 | 0.00 |
| 2025-04-30 | 0 | 0.00 | 0.00 |
| 2025-04-29 | 0 | 0.00 | 0.00 |
| 2025-04-28 | 0 | 0.00 | 0.00 |
| 2025-04-27 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-04-26 | 0 | 0.00 | 0.00 |
| 2025-04-25 | 0 | 0.00 | 0.00 |
| 2025-04-24 | 0 | 0.00 | 0.00 |
| 2025-04-23 | 0 | 0.00 | 0.00 |
| 2025-04-22 | 0 | 0.00 | 0.00 |
| 2025-04-21 | 0 | 0.00 | 0.00 |
| 2025-04-20 | 0 | 0.00 | 0.00 |
| 2025-04-19 | 0 | 0.00 | 0.00 |
| 2025-04-18 | 0 | 0.00 | 0.00 |
| 2025-04-17 | 0 | 0.00 | 0.00 |
| 2025-04-16 | 0 | 0.00 | 0.00 |
| 2025-04-15 | 0 | 0.00 | 0.00 |
| 2025-04-14 | 0 | 0.00 | 0.00 |
| 2025-04-13 | 0 | 0.00 | 0.00 |
| 2025-04-12 | 0 | 0.00 | 0.00 |
| 2025-04-11 | 0 | 0.00 | 0.00 |
| 2025-04-10 | 0 | 0.00 | 0.00 |
| 2025-04-09 | 0 | 0.00 | 0.00 |
| 2025-04-08 | 0 | 0.00 | 0.00 |
| 2025-04-07 | 0 | 0.00 | 0.00 |
| 2025-04-06 | 0 | 0.00 | 0.00 |
| 2025-04-05 | 0 | 0.00 | 0.00 |
| 2025-04-04 | 0 | 0.00 | 0.00 |
| 2025-04-03 | 0 | 0.00 | 0.00 |
| 2025-04-02 | 0 | 0.00 | 0.00 |
| 2025-04-01 | 0 | 0.00 | 0.00 |
| 2025-03-31 | 0 | 0.00 | 0.00 |
| 2025-03-30 | 0 | 0.00 | 0.00 |
| 2025-03-29 | 0 | 0.00 | 0.00 |
| 2025-03-28 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-03-27 | 0 | 0.00 | 0.00 |
| 2025-03-26 | 0 | 0.00 | 0.00 |
| 2025-03-25 | 0 | 0.00 | 0.00 |
| 2025-03-24 | 0 | 0.00 | 0.00 |
| 2025-03-23 | 0 | 0.00 | 0.00 |
| 2025-03-22 | 0 | 0.00 | 0.00 |
| 2025-03-21 | 0 | 0.00 | 0.00 |
| 2025-03-20 | 0 | 0.00 | 0.00 |
| 2025-03-19 | 0 | 0.00 | 0.00 |
| 2025-03-18 | 0 | 0.00 | 0.00 |
| 2025-03-17 | 0 | 0.00 | 0.00 |
| 2025-03-16 | 0 | 0.00 | 0.00 |
| 2025-03-15 | 0 | 0.00 | 0.00 |
| 2025-03-14 | 0 | 0.00 | 0.00 |
| 2025-03-13 | 0 | 0.00 | 0.00 |
| 2025-03-12 | 0 | 0.00 | 0.00 |
| 2025-03-11 | 0 | 0.00 | 0.00 |
| 2025-03-10 | 0 | 0.00 | 0.00 |
| 2025-03-09 | 0 | 0.00 | 0.00 |
| 2025-03-08 | 0 | 0.00 | 0.00 |
| 2025-03-07 | 0 | 0.00 | 0.00 |
| 2025-03-06 | 0 | 0.00 | 0.00 |
| 2025-03-05 | 0 | 0.00 | 0.00 |
| 2025-03-04 | 0 | 0.00 | 0.00 |
| 2025-03-03 | 0 | 0.00 | 0.00 |
| 2025-03-02 | 0 | 0.00 | 0.00 |
| 2025-03-01 | 0 | 0.00 | 0.00 |
| 2025-02-28 | 0 | 0.00 | 0.00 |
| 2025-02-27 | 0 | 0.00 | 0.00 |
| 2025-02-26 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-02-25 | 0 | 0.00 | 0.00 |
| 2025-02-24 | 0 | 0.00 | 0.00 |
| 2025-02-23 | 0 | 0.00 | 0.00 |
| 2025-02-22 | 0 | 0.00 | 0.00 |
| 2025-02-21 | 0 | 0.00 | 0.00 |
| 2025-02-20 | 0 | 0.00 | 0.00 |
| 2025-02-19 | 0 | 0.00 | 0.00 |
| 2025-02-18 | 0 | 0.00 | 0.00 |
| 2025-02-17 | 0 | 0.00 | 0.00 |
| 2025-02-16 | 0 | 0.00 | 0.00 |
| 2025-02-15 | 0 | 0.00 | 0.00 |
| 2025-02-14 | 0 | 0.00 | 0.00 |
| 2025-02-13 | 0 | 0.00 | 0.00 |
| 2025-02-12 | 0 | 0.00 | 0.00 |
| 2025-02-11 | 0 | 0.00 | 0.00 |
| 2025-02-10 | 0 | 0.00 | 0.00 |
| 2025-02-09 | 0 | 0.00 | 0.00 |
| 2025-02-08 | 0 | 0.00 | 0.00 |
| 2025-02-07 | 0 | 0.00 | 0.00 |
| 2025-02-06 | 0 | 0.00 | 0.00 |
| 2025-02-05 | 0 | 0.00 | 0.00 |
| 2025-02-04 | 0 | 0.00 | 0.00 |
| 2025-02-03 | 0 | 0.00 | 0.00 |
| 2025-02-02 | 0 | 0.00 | 0.00 |
| 2025-02-01 | 0 | 0.00 | 0.00 |
| 2025-01-31 | 0 | 0.00 | 0.00 |
| 2025-01-30 | 0 | 0.00 | 0.00 |
| 2025-01-29 | 0 | 0.00 | 0.00 |
| 2025-01-28 | 0 | 0.00 | 0.00 |
| 2025-01-27 | 0 | 0.00 | 0.00 |
| 2025-01-26 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-01-25 | 0 | 0.00 | 0.00 |
| 2025-01-24 | 0 | 0.00 | 0.00 |
| 2025-01-23 | 0 | 0.00 | 0.00 |
| 2025-01-22 | 0 | 0.00 | 0.00 |
| 2025-01-21 | 0 | 0.00 | 0.00 |
| 2025-01-20 | 0 | 0.00 | 0.00 |
| 2025-01-19 | 0 | 0.00 | 0.00 |
| 2025-01-18 | 0 | 0.00 | 0.00 |
| 2025-01-17 | 0 | 0.00 | 0.00 |
| 2025-01-16 | 0 | 0.00 | 0.00 |
| 2025-01-15 | 0 | 0.00 | 0.00 |
| 2025-01-14 | 0 | 0.00 | 0.00 |
| 2025-01-13 | 0 | 0.00 | 0.00 |
| 2025-01-12 | 0 | 0.00 | 0.00 |
| 2025-01-11 | 0 | 0.00 | 0.00 |
| 2025-01-10 | 0 | 0.00 | 0.00 |
| 2025-01-09 | 0 | 0.00 | 0.00 |
| 2025-01-08 | 0 | 0.00 | 0.00 |
| 2025-01-07 | 0 | 0.00 | 0.00 |
| 2025-01-06 | 0 | 0.00 | 0.00 |
| 2025-01-05 | 0 | 0.00 | 0.00 |
| 2025-01-04 | 0 | 0.00 | 0.00 |
| 2025-01-03 | 0 | 0.00 | 0.00 |
| 2025-01-02 | 0 | 0.00 | 0.00 |
| 2025-01-01 | 0 | 0.00 | 0.00 |
| 2024-12-31 | 0 | 0.00 | 0.00 |
| 2024-12-30 | 0 | 0.00 | 0.00 |
Author
| Month | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-09 | 135088445444 | 128830.38 | 125.81 |
| 2025-08 | 94801336338 | 90409.60 | 88.29 |
| 2025-07 | 282929010478 | 269822.13 | 263.50 |
| 2025-06 | 2268854253 | 2163.75 | 2.11 |
| 2025-05 | 0 | 0.00 | 0.00 |
| 2025-04 | 0 | 0.00 | 0.00 |
Author
From 2025-01-15 to 2025-09-23 (~480GiB in 251 days):
Total Objects: 2512
Total Size: 477.3 GiB
Average per day:
~ 1.9 GiB
Author
MinIO
Jan, 15. 2025 to Apr, 23. 2025
99 Tage, 14 Wochen
acm-metrics
195,0GB of 2TB 1.408 Objects
--> ~ 2GB per day
Author
Final summary:
-
Migration: MinIO → OpenStack S3 on 2025-06-13
-
Time window analyzed: 2025-01-15 → 2025-09-23 (251 days)
-
acm-metrics bucket:
- Total size: 477.3 GiB
- Total objects: 2,512
- Average growth: ~1.9 GiB/day
-
MinIO phase (2025-01-15 → 2025-04-23, 99 days):
- Size: 195 GiB of 2 TiB quota
- Objects: 1,408
- Avg growth: ~2 GiB/day
Storage behavior is consistent across both MinIO and OpenStack S3, with average daily usage in the ~2 GiB/day range.
Author
echo "| Date | Bytes | MiB | GiB |"
echo "|------------|-----------|---------|---------|"
for i in {0..30}; do
DAY=$(date -d "-$i day" +%Y-%m-%d)
aws s3 ls s3://acm-metrics/ --recursive \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
| grep "$DAY" \
| awk -v d="$DAY" 'BEGIN{sum=0}
{sum += $3}
END {
mib = sum/1024/1024;
gib = sum/1024/1024/1024;
printf "| %s | %d | %.2f | %.2f |\n", d, sum, mib, gib
}'
done| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-10-01 | 21856686955 | 20844.16 | 20.36 |
| 2025-09-30 | 19635399317 | 18725.78 | 18.29 |
| 2025-09-29 | 18703535010 | 17837.08 | 17.42 |
| 2025-09-28 | 0 | 0.00 | 0.00 |
| 2025-09-27 | 8100073629 | 7724.83 | 7.54 |
| 2025-09-26 | 0 | 0.00 | 0.00 |
| 2025-09-25 | 1344009 | 1.28 | 0.00 |
| 2025-09-24 | 8158709682 | 7780.75 | 7.60 |
| 2025-09-23 | 7691775411 | 7335.45 | 7.16 |
| 2025-09-22 | 0 | 0.00 | 0.00 |
| 2025-09-21 | 1332099 | 1.27 | 0.00 |
| 2025-09-20 | 7814721048 | 7452.70 | 7.28 |
| 2025-09-19 | 7819468595 | 7457.23 | 7.28 |
| 2025-09-18 | 0 | 0.00 | 0.00 |
| 2025-09-17 | 7798931313 | 7437.64 | 7.26 |
| 2025-09-16 | 0 | 0.00 | 0.00 |
| 2025-09-15 | 1282086 | 1.22 | 0.00 |
| 2025-09-14 | 7772449357 | 7412.39 | 7.24 |
| 2025-09-13 | 8067753468 | 7694.01 | 7.51 |
| 2025-09-12 | 0 | 0.00 | 0.00 |
| 2025-09-11 | 1740468 | 1.66 | 0.00 |
| 2025-09-10 | 8527179815 | 8132.15 | 7.94 |
| 2025-09-09 | 1360669 | 1.30 | 0.00 |
| 2025-09-08 | 7915709723 | 7549.01 | 7.37 |
| 2025-09-07 | 1347687 | 1.29 | 0.00 |
| 2025-09-06 | 7817941954 | 7455.77 | 7.28 |
| 2025-09-05 | 7515077488 | 7166.94 | 7.00 |
| 2025-09-04 | 0 | 0.00 | 0.00 |
| 2025-09-03 | 7469928077 | 7123.88 | 6.96 |
| 2025-09-02 | 0 | 0.00 | 0.00 |
| 2025-09-01 | 7369493830 | 7028.10 | 6.86 |
Author
Merged Pull Requests Affecting Metrics (Chronological Order)
2025-09-05
- OCP-on-NERC/nerc-ocp-config#797
Add Thanos metrics to ACM observability
2025-09-24
- OCP-on-NERC/nerc-ocp-config#782
Add ACM metrics forkube_job_*(job start and completion)
2025-09-30
- OCP-on-NERC/nerc-ocp-config#792
Add OpenVino & Kueue metrics to ACM Observability
Summary:
In September 2025, ACM Observability was extended with:
- Thanos self-metrics (for observability stack health)
- Job lifecycle metrics (
kube_job_*for start & completion) - OpenVino and Kueue metrics (for AI workload tracking)
Author
timeline
title ACM Observability Metrics Additions - September 2025
2025-09-05 : Add Thanos metrics to ACM observability (PR #797)
2025-09-24 : Add ACM metrics for kube_job_* (job start and completion) (PR #782)
2025-09-30 : Add OpenVino & Kueue metrics to ACM Observability (PR #792)
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-10-06 | 5580752728 | 5322.22 | 5.20 |
| 2025-10-05 | 29335717959 | 27976.72 | 27.32 |
| 2025-10-04 | 13026871222 | 12423.39 | 12.13 |
| 2025-10-03 | 9534212741 | 9092.53 | 8.88 |
| 2025-10-02 | 0 | 0.00 | 0.00 |
| 2025-10-01 | 8197106200 | 7817.37 | 7.63 |
| 2025-09-30 | 0 | 0.00 | 0.00 |
| 2025-09-29 | 8092382951 | 7717.50 | 7.54 |
| 2025-09-28 | 0 | 0.00 | 0.00 |
| 2025-09-27 | 8100073629 | 7724.83 | 7.54 |
| 2025-09-26 | 0 | 0.00 | 0.00 |
| 2025-09-25 | 1344009 | 1.28 | 0.00 |
| 2025-09-24 | 8158709682 | 7780.75 | 7.60 |
| 2025-09-23 | 7691775411 | 7335.45 | 7.16 |
| 2025-09-22 | 0 | 0.00 | 0.00 |
| 2025-09-21 | 1332099 | 1.27 | 0.00 |
| 2025-09-20 | 7814721048 | 7452.70 | 7.28 |
| 2025-09-19 | 7819468595 | 7457.23 | 7.28 |
| 2025-09-18 | 0 | 0.00 | 0.00 |
| 2025-09-17 | 7798931313 | 7437.64 | 7.26 |
| 2025-09-16 | 0 | 0.00 | 0.00 |
| 2025-09-15 | 1282086 | 1.22 | 0.00 |
| 2025-09-14 | 7772449357 | 7412.39 | 7.24 |
| 2025-09-13 | 8067753468 | 7694.01 | 7.51 |
| 2025-09-12 | 0 | 0.00 | 0.00 |
| 2025-09-11 | 1740468 | 1.66 | 0.00 |
| 2025-09-10 | 8527179815 | 8132.15 | 7.94 |
| 2025-09-09 | 1360669 | 1.30 | 0.00 |
| 2025-09-08 | 7915709723 | 7549.01 | 7.37 |
| 2025-09-07 | 1347687 | 1.29 | 0.00 |
| 2025-09-06 | 7817941954 | 7455.77 | 7.28 |
Author
| Metric | Value |
|---|---|
| Average over all 31 days | 4.85 GiB/day |
| Total (31 days) | 150.18 GiB |
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-10-08 | 7497751099 | 7150.41 | 6.98 |
| 2025-10-07 | 29529351405 | 28161.38 | 27.50 |
| 2025-10-06 | 11332726783 | 10807.73 | 10.55 |
| 2025-10-05 | 9053864292 | 8634.44 | 8.43 |
| 2025-10-04 | 0 | 0.00 | 0.00 |
| 2025-10-03 | 7916092555 | 7549.37 | 7.37 |
| 2025-10-02 | 0 | 0.00 | 0.00 |
| 2025-10-01 | 8197106200 | 7817.37 | 7.63 |
| 2025-09-30 | 0 | 0.00 | 0.00 |
| 2025-09-29 | 8092382951 | 7717.50 | 7.54 |
| 2025-09-28 | 0 | 0.00 | 0.00 |
| 2025-09-27 | 8100073629 | 7724.83 | 7.54 |
| 2025-09-26 | 0 | 0.00 | 0.00 |
| 2025-09-25 | 1344009 | 1.28 | 0.00 |
| 2025-09-24 | 8158709682 | 7780.75 | 7.60 |
| 2025-09-23 | 7691775411 | 7335.45 | 7.16 |
| 2025-09-22 | 0 | 0.00 | 0.00 |
| 2025-09-21 | 1332099 | 1.27 | 0.00 |
| 2025-09-20 | 7814721048 | 7452.70 | 7.28 |
| 2025-09-19 | 7819468595 | 7457.23 | 7.28 |
| 2025-09-18 | 0 | 0.00 | 0.00 |
| 2025-09-17 | 7798931313 | 7437.64 | 7.26 |
| 2025-09-16 | 0 | 0.00 | 0.00 |
| 2025-09-15 | 1282086 | 1.22 | 0.00 |
| 2025-09-14 | 7772449357 | 7412.39 | 7.24 |
| 2025-09-13 | 8067753468 | 7694.01 | 7.51 |
| 2025-09-12 | 0 | 0.00 | 0.00 |
| 2025-09-11 | 1740468 | 1.66 | 0.00 |
| 2025-09-10 | 8527179815 | 8132.15 | 7.94 |
| 2025-09-09 | 1360669 | 1.30 | 0.00 |
| 2025-09-08 | 7915709723 | 7549.01 | 7.37 |
echo "| Date | Bytes | MiB | GiB |" ─╯
echo "|------------|-----------|---------|---------|"
for i in {0..30}; do
DAY=$(date -d "-$i day" +%Y-%m-%d)
aws s3 ls s3://acm-metrics/ --recursive \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
| grep "$DAY" \
| awk -v d="$DAY" 'BEGIN{sum=0}
{sum += $3}
END {
mib = sum/1024/1024;
gib = sum/1024/1024/1024;
printf "| %s | %d | %.2f | %.2f |\n", d, sum, mib, gib
}'
done
Author
| Metric | Value |
|---|---|
| Average over all 30 days | 3.11 GiB/day |
| Total (30 days) | 93.22 GiB |
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-09-07 | 1347687 | 1.29 | 0.00 |
| 2025-09-06 | 7817941954 | 7455.77 | 7.28 |
| 2025-09-05 | 7515077488 | 7166.94 | 7.00 |
| 2025-09-04 | 0 | 0.00 | 0.00 |
| 2025-09-03 | 7469928077 | 7123.88 | 6.96 |
| 2025-09-02 | 0 | 0.00 | 0.00 |
| 2025-09-01 | 7369493830 | 7028.10 | 6.86 |
| 2025-08-31 | 0 | 0.00 | 0.00 |
| 2025-08-30 | 6825113411 | 6508.94 | 6.36 |
| 2025-08-29 | 0 | 0.00 | 0.00 |
| 2025-08-28 | 1571072 | 1.50 | 0.00 |
| 2025-08-27 | 7043891557 | 6717.58 | 6.56 |
| 2025-08-26 | 7268164151 | 6931.46 | 6.77 |
| 2025-08-25 | 0 | 0.00 | 0.00 |
| 2025-08-24 | 7154521773 | 6823.08 | 6.66 |
| 2025-08-23 | 0 | 0.00 | 0.00 |
| 2025-08-22 | 7382739482 | 7040.73 | 6.88 |
| 2025-08-21 | 0 | 0.00 | 0.00 |
| 2025-08-20 | 7483204543 | 7136.54 | 6.97 |
| 2025-08-19 | 0 | 0.00 | 0.00 |
| 2025-08-18 | 7423028718 | 7079.15 | 6.91 |
| 2025-08-17 | 0 | 0.00 | 0.00 |
| 2025-08-16 | 7426459242 | 7082.42 | 6.92 |
| 2025-08-15 | 0 | 0.00 | 0.00 |
| 2025-08-14 | 2034804720 | 1940.54 | 1.90 |
| 2025-08-13 | 0 | 0.00 | 0.00 |
| 2025-08-12 | 262564722 | 250.40 | 0.24 |
| 2025-08-11 | 1632107933 | 1556.50 | 1.52 |
| 2025-08-10 | 1112547562 | 1061.01 | 1.04 |
| 2025-08-09 | 6862777600 | 6544.85 | 6.39 |
Author
| Metric | Value |
|---|---|
| Average over all 30 days | 2.87 GiB/day |
| Total (30 days) | 86.20 GiB |
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-08-08 | 6289871912 | 5998.49 | 5.86 |
| 2025-08-07 | 2196974 | 2.10 | 0.00 |
| 2025-08-06 | 6210674888 | 5922.96 | 5.78 |
| 2025-08-05 | 0 | 0.00 | 0.00 |
| 2025-08-04 | 6162319486 | 5876.85 | 5.74 |
| 2025-08-03 | 0 | 0.00 | 0.00 |
| 2025-08-02 | 6222776592 | 5934.50 | 5.80 |
| 2025-08-01 | 0 | 0.00 | 0.00 |
| 2025-07-31 | 34940195989 | 33321.57 | 32.54 |
| 2025-07-30 | 0 | 0.00 | 0.00 |
| 2025-07-29 | 0 | 0.00 | 0.00 |
| 2025-07-28 | 0 | 0.00 | 0.00 |
| 2025-07-27 | 0 | 0.00 | 0.00 |
| 2025-07-26 | 0 | 0.00 | 0.00 |
| 2025-07-25 | 0 | 0.00 | 0.00 |
| 2025-07-24 | 0 | 0.00 | 0.00 |
| 2025-07-23 | 0 | 0.00 | 0.00 |
| 2025-07-22 | 0 | 0.00 | 0.00 |
| 2025-07-21 | 0 | 0.00 | 0.00 |
| 2025-07-20 | 0 | 0.00 | 0.00 |
| 2025-07-19 | 0 | 0.00 | 0.00 |
| 2025-07-18 | 0 | 0.00 | 0.00 |
| 2025-07-17 | 32729023794 | 31212.83 | 30.48 |
| 2025-07-16 | 0 | 0.00 | 0.00 |
| 2025-07-15 | 0 | 0.00 | 0.00 |
| 2025-07-14 | 0 | 0.00 | 0.00 |
| 2025-07-13 | 0 | 0.00 | 0.00 |
| 2025-07-12 | 0 | 0.00 | 0.00 |
| 2025-07-11 | 0 | 0.00 | 0.00 |
| 2025-07-10 | 0 | 0.00 | 0.00 |
Author
2025-10-08 01-49 pm
data collection started Jan 15 2025
Bucket: acm-metrics
Total Objects: 2774
Total Size: 522.1 GiB
| Kennzahl | Wert |
|---|---|
| Zeitraum | 277 Tage |
| Gesamtdatenmenge | 522,1 GiB |
| Durchschnitt pro Tag | ≈ 1,89 GiB/Tag |
Bucket: acm-metrics-hypershift2
Total Objects: 1546
Total Size: 21.2 GiB
Bucket: acm-metrics-test
Total Objects: 1057
Total Size: 20.0 GiB
aws s3api list-objects-v2 \
--bucket acm-metrics \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
--output json --query "sum(Contents[].Size)"
aws s3api list-objects-v2 \
--bucket acm-metrics \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
--output json --query "sum(Contents[].Size)"
for b in acm-metrics acm-metrics-hypershift2 acm-metrics-test \
loki-logs loki-logs-test \
open-telemetry open-telemetry-test; do
echo "### Bucket: $b"
aws s3 ls s3://$b/ --recursive \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
--human-readable --summarize | tail -n 2
echo
done
Author
Bucket: acm-metrics
Total Objects: 3088
Total Size: 591.7 GiB
Bucket: acm-metrics-hypershift2
Total Objects: 1761
Total Size: 29.2 GiB
Bucket: acm-metrics-test
Total Objects: 1057
Total Size: 20.0 GiB
Bucket: loki-logs-test
Total Objects: 0
Total Size: 0 Bytes
Bucket: open-telemetry
Total Objects: 0
Total Size: 0 Bytes
Bucket: open-telemetry-test
Total Objects: 0
Total Size: 0 Bytes
Author
echo "| Date | Bytes | MiB | GiB |"
echo "|------------|-----------|---------|---------|"
for i in {0..30}; do
DAY=$(date -d "-$i day" +%Y-%m-%d)
aws s3 ls s3://acm-metrics/ --recursive \
--endpoint-url https://stack.nerc.mghpcc.org:13808 \
| grep "$DAY" \
| awk -v d="$DAY" 'BEGIN{sum=0}
{sum += $3}
END {
mib = sum/1024/1024;
gib = sum/1024/1024/1024;
printf "| %s | %d | %.2f | %.2f |\n", d, sum, mib, gib
}'
done
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-10-28 | 7857077874 | 7493.09 | 7.32 |
| 2025-10-27 | 32079892518 | 30593.77 | 29.88 |
| 2025-10-26 | 11677347734 | 11136.39 | 10.88 |
| 2025-10-25 | 10311231925 | 9833.56 | 9.60 |
| 2025-10-24 | 0 | 0.00 | 0.00 |
| 2025-10-23 | 9644908685 | 9198.10 | 8.98 |
| 2025-10-22 | 0 | 0.00 | 0.00 |
| 2025-10-21 | 8084740273 | 7710.21 | 7.53 |
| 2025-10-20 | 0 | 0.00 | 0.00 |
| 2025-10-19 | 8007399505 | 7636.45 | 7.46 |
| 2025-10-18 | 0 | 0.00 | 0.00 |
| 2025-10-17 | 8184415091 | 7805.27 | 7.62 |
| 2025-10-16 | 0 | 0.00 | 0.00 |
| 2025-10-15 | 8205295384 | 7825.18 | 7.64 |
| 2025-10-14 | 0 | 0.00 | 0.00 |
| 2025-10-13 | 8090056668 | 7715.28 | 7.53 |
| 2025-10-12 | 0 | 0.00 | 0.00 |
| 2025-10-11 | 8154914832 | 7777.13 | 7.59 |
| 2025-10-10 | 0 | 0.00 | 0.00 |
| 2025-10-09 | 7911879338 | 7545.36 | 7.37 |
| 2025-10-08 | 0 | 0.00 | 0.00 |
| 2025-10-07 | 7748094652 | 7389.16 | 7.22 |
| 2025-10-06 | 0 | 0.00 | 0.00 |
| 2025-10-05 | 7765105884 | 7405.38 | 7.23 |
| 2025-10-04 | 0 | 0.00 | 0.00 |
| 2025-10-03 | 7916092555 | 7549.37 | 7.37 |
| 2025-10-02 | 0 | 0.00 | 0.00 |
| 2025-10-01 | 8197106200 | 7817.37 | 7.63 |
| 2025-09-30 | 0 | 0.00 | 0.00 |
| 2025-09-29 | 8092382951 | 7717.50 | 7.54 |
| 2025-09-28 | 0 | 0.00 | 0.00 |
Author
| Date | Bytes | MiB | GiB |
|---|---|---|---|
| 2025-12-18 | 20450414264 | 19503.03 | 19.05 |
| 2025-12-17 | 17306328755 | 16504.60 | 16.12 |
| 2025-12-16 | 18704519716 | 17838.02 | 17.42 |
| 2025-12-15 | 0 | 0.00 | 0.00 |
| 2025-12-14 | 7729039245 | 7370.99 | 7.20 |
| 2025-12-13 | 0 | 0.00 | 0.00 |
| 2025-12-12 | 7800203628 | 7438.85 | 7.26 |
| 2025-12-11 | 0 | 0.00 | 0.00 |
| 2025-12-10 | 7751191610 | 7392.11 | 7.22 |
| 2025-12-09 | 0 | 0.00 | 0.00 |
| 2025-12-08 | 7652288120 | 7297.79 | 7.13 |
| 2025-12-07 | 0 | 0.00 | 0.00 |
| 2025-12-06 | 3239319 | 3.09 | 0.00 |
| 2025-12-05 | 7793628121 | 7432.58 | 7.26 |
| 2025-12-04 | 7729035512 | 7370.98 | 7.20 |
| 2025-12-03 | 0 | 0.00 | 0.00 |
| 2025-12-02 | 7672341312 | 7316.91 | 7.15 |
| 2025-12-01 | 0 | 0.00 | 0.00 |
| 2025-11-30 | 7446756997 | 7101.78 | 6.94 |
| 2025-11-29 | 0 | 0.00 | 0.00 |
| 2025-11-28 | 7386088120 | 7043.92 | 6.88 |
| 2025-11-27 | 0 | 0.00 | 0.00 |
| 2025-11-26 | 7459213237 | 7113.66 | 6.95 |
| 2025-11-25 | 0 | 0.00 | 0.00 |
| 2025-11-24 | 7285164975 | 6947.67 | 6.78 |
| 2025-11-23 | 0 | 0.00 | 0.00 |
| 2025-11-22 | 7360941402 | 7019.94 | 6.86 |
| 2025-11-21 | 0 | 0.00 | 0.00 |
| 2025-11-20 | 7436319653 | 7091.83 | 6.93 |
| 2025-11-19 | 0 | 0.00 | 0.00 |
| 2025-11-18 | 7289448962 | 6951.76 | 6.79 |
Author
#!/bin/bash
ENDPOINT="https://stack.nerc.mghpcc.org:13808"
echo "| Bucket | Objects | Size (GiB) |"
echo "|---------------------------|---------|------------|"
for bucket in acm-metrics acm-metrics-hypershift2 acm-metrics-test open-telemetry open-telemetry-test; do
result=$(aws s3 ls "s3://$bucket/" --recursive --summarize --endpoint-url "$ENDPOINT" 2>&1 | tail -2)
objects=$(echo "$result" | grep "Total Objects" | awk '{print $3}')
bytes=$(echo "$result" | grep "Total Size" | awk '{print $3}')
gib=$(echo "scale=2; ${bytes:-0}/1024/1024/1024" | bc)
printf "| %-25s | %7s | %10s |\n" "$bucket" "${objects:-0}" "$gib"
done
| Bucket | Objects | Size (GiB) |
|---|---|---|
| acm-metrics | 3610 | 708.58 |
| acm-metrics-hypershift2 | 2169 | 52.24 |
| acm-metrics-test | 1057 | 19.95 |
| open-telemetry | 0 | 0 |
| open-telemetry-test | 0 | 0 |
loki logs running into timeout (5min+, too many small files?)
Author
aws s3 ls --endpoint-url https://stack.nerc.mghpcc.org:13808
List all S3 buckets
| Erstellt | Bucket |
|---|---|
| 2025-06-13 | acm-metrics |
| 2025-08-06 | acm-metrics-hypershift2 |
| 2025-07-02 | acm-metrics-test |
| 2025-06-18 | loki-logs |
| 2025-07-02 | loki-logs-test |
| 2025-06-26 | open-telemetry |
| 2025-07-02 | open-telemetry-test |
Author
better skript
#!/bin/bash
BUCKET="${1:-acm-metrics}"
DAYS="${2:-30}"
ENDPOINT="https://stack.nerc.mghpcc.org:13808"
echo "Fetching listing for $BUCKET..."
LISTING=$(aws s3 ls "s3://$BUCKET/" --recursive --endpoint-url "$ENDPOINT" 2>&1)
echo ""
echo "| Date | Objects | Size (MiB) | Size (GiB) |"
echo "|------------|---------|------------|------------|"
for i in $(seq 0 $DAYS); do
DAY=$(date -d "-$i day" +%Y-%m-%d)
echo "$LISTING" | grep "$DAY" | awk -v d="$DAY" '
BEGIN{sum=0; count=0}
{sum += $3; count++}
END {
mib = sum/1024/1024
gib = sum/1024/1024/1024
printf "| %s | %7d | %10.2f | %10.2f |\n", d, count, mib, gib
}'
done
echo ""
echo "TOTAL:"
echo "$LISTING" | awk '
BEGIN{sum=0; count=0}
{sum += $3; count++}
END {
printf "Objects: %d, Size: %.2f GiB\n", count, sum/1024/1024/1024
}'
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
acm-metrics