S3_Storage_Observability

s3_storage_observability.txt

# S3 on OpenStack NERC, migrated from minio on 2025-06-13 (minio started Jan 15 2025)
## 2025-09-23 03:00pm
### Bucket: acm-metrics
Total Objects: 2512
   Total Size: 477.3 GiB

### Bucket: acm-metrics-hypershift2
Total Objects: 1339
   Total Size: 14.8 GiB

### Bucket: acm-metrics-test
Total Objects: 1057
   Total Size: 20.0 GiB

### Bucket: loki-logs-test
Total Objects: 0
   Total Size: 0 Bytes

### Bucket: open-telemetry
Total Objects: 0
   Total Size: 0 Bytes

### Bucket: open-telemetry-test
Total Objects: 0
   Total Size: 0 Bytes

echo "| Date       | Bytes     | MiB     | GiB     |"
echo "|------------|-----------|---------|---------|"

for i in {0..30}; do
  DAY=$(date -d "-$i day" +%Y-%m-%d)
  aws s3 ls s3://acm-metrics/ --recursive \
    --endpoint-url https://stack.nerc.mghpcc.org:13808 \
  | grep "$DAY" \
  | awk -v d="$DAY" 'BEGIN{sum=0}
       {sum += $3}
       END {
         mib = sum/1024/1024;
         gib = sum/1024/1024/1024;
         printf "| %s | %d | %.2f | %.2f |\n", d, sum, mib, gib
       }'
done

Date	Bytes	MiB	GiB
2025-10-28	7857077874	7493.09	7.32
2025-10-27	32079892518	30593.77	29.88
2025-10-26	11677347734	11136.39	10.88
2025-10-25	10311231925	9833.56	9.60
2025-10-24	0	0.00	0.00
2025-10-23	9644908685	9198.10	8.98
2025-10-22	0	0.00	0.00
2025-10-21	8084740273	7710.21	7.53
2025-10-20	0	0.00	0.00
2025-10-19	8007399505	7636.45	7.46
2025-10-18	0	0.00	0.00
2025-10-17	8184415091	7805.27	7.62
2025-10-16	0	0.00	0.00
2025-10-15	8205295384	7825.18	7.64
2025-10-14	0	0.00	0.00
2025-10-13	8090056668	7715.28	7.53
2025-10-12	0	0.00	0.00
2025-10-11	8154914832	7777.13	7.59
2025-10-10	0	0.00	0.00
2025-10-09	7911879338	7545.36	7.37
2025-10-08	0	0.00	0.00
2025-10-07	7748094652	7389.16	7.22
2025-10-06	0	0.00	0.00
2025-10-05	7765105884	7405.38	7.23
2025-10-04	0	0.00	0.00
2025-10-03	7916092555	7549.37	7.37
2025-10-02	0	0.00	0.00
2025-10-01	8197106200	7817.37	7.63
2025-09-30	0	0.00	0.00
2025-09-29	8092382951	7717.50	7.54
2025-09-28	0	0.00	0.00

Date	Bytes	MiB	GiB
2025-12-18	20450414264	19503.03	19.05
2025-12-17	17306328755	16504.60	16.12
2025-12-16	18704519716	17838.02	17.42
2025-12-15	0	0.00	0.00
2025-12-14	7729039245	7370.99	7.20
2025-12-13	0	0.00	0.00
2025-12-12	7800203628	7438.85	7.26
2025-12-11	0	0.00	0.00
2025-12-10	7751191610	7392.11	7.22
2025-12-09	0	0.00	0.00
2025-12-08	7652288120	7297.79	7.13
2025-12-07	0	0.00	0.00
2025-12-06	3239319	3.09	0.00
2025-12-05	7793628121	7432.58	7.26
2025-12-04	7729035512	7370.98	7.20
2025-12-03	0	0.00	0.00
2025-12-02	7672341312	7316.91	7.15
2025-12-01	0	0.00	0.00
2025-11-30	7446756997	7101.78	6.94
2025-11-29	0	0.00	0.00
2025-11-28	7386088120	7043.92	6.88
2025-11-27	0	0.00	0.00
2025-11-26	7459213237	7113.66	6.95
2025-11-25	0	0.00	0.00
2025-11-24	7285164975	6947.67	6.78
2025-11-23	0	0.00	0.00
2025-11-22	7360941402	7019.94	6.86
2025-11-21	0	0.00	0.00
2025-11-20	7436319653	7091.83	6.93
2025-11-19	0	0.00	0.00
2025-11-18	7289448962	6951.76	6.79

#!/bin/bash

ENDPOINT="https://stack.nerc.mghpcc.org:13808"

echo "| Bucket | Objects | Size (GiB) |"
echo "|---------------------------|---------|------------|"

for bucket in acm-metrics acm-metrics-hypershift2 acm-metrics-test open-telemetry open-telemetry-test; do
result=$(aws s3 ls "s3://$bucket/" --recursive --summarize --endpoint-url "$ENDPOINT" 2>&1 | tail -2)
objects=$(echo "$result" | grep "Total Objects" | awk '{print $3}')
bytes=$(echo "$result" | grep "Total Size" | awk '{print $3}')
gib=$(echo "scale=2; ${bytes:-0}/1024/1024/1024" | bc)
printf "| %-25s | %7s | %10s |\n" "$bucket" "${objects:-0}" "$gib"
done

Bucket	Objects	Size (GiB)
acm-metrics	3610	708.58
acm-metrics-hypershift2	2169	52.24
acm-metrics-test	1057	19.95
open-telemetry	0	0
open-telemetry-test	0	0

loki logs running into timeout (5min+, too many small files?)

aws s3 ls --endpoint-url https://stack.nerc.mghpcc.org:13808
List all S3 buckets

Erstellt	Bucket
2025-06-13	acm-metrics
2025-08-06	acm-metrics-hypershift2
2025-07-02	acm-metrics-test
2025-06-18	loki-logs
2025-07-02	loki-logs-test
2025-06-26	open-telemetry
2025-07-02	open-telemetry-test

better skript

#!/bin/bash

BUCKET="${1:-acm-metrics}"
DAYS="${2:-30}"
ENDPOINT="https://stack.nerc.mghpcc.org:13808"

echo "Fetching listing for $BUCKET..."
LISTING=$(aws s3 ls "s3://$BUCKET/" --recursive --endpoint-url "$ENDPOINT" 2>&1)

echo ""
echo "| Date | Objects | Size (MiB) | Size (GiB) |"
echo "|------------|---------|------------|------------|"

for i in $(seq 0 $DAYS); do
DAY=$(date -d "-$i day" +%Y-%m-%d)
echo "$LISTING" | grep "$DAY" | awk -v d="$DAY" '
BEGIN{sum=0; count=0}
{sum += $3; count++}
END {
mib = sum/1024/1024
gib = sum/1024/1024/1024
printf "| %s | %7d | %10.2f | %10.2f |\n", d, count, mib, gib
}'
done

echo ""
echo "TOTAL:"
echo "$LISTING" | awk '
BEGIN{sum=0; count=0}
{sum += $3; count++}
END {
printf "Objects: %d, Size: %.2f GiB\n", count, sum/1024/1024/1024
}'