K8s Job to clean Glacier Legacy vault

k8s-job-clean-glacier-legacy-vault.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: script
data:
  script.sh: |
    yum install -y jq findutils awscli

    SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
    cd "$SCRIPT_DIR"

    total=$(jq -r --stream 'select(.[0][-1]=="ArchiveId") | .[1]' "$INVENTORY_FILE" | wc -l);
    skip=$SKIP;
    total=$total GLACIER_VAULT=$GLACIER_VAULT xargs -P "$PARALLEL_JOBS" -n 2 sh -c '
      num="$1"; archive_id="$2"
      echo "[$num/$total] Delete $archive_id"
      aws glacier delete-archive --account-id - --vault-name $GLACIER_VAULT --archive-id="$archive_id"
    ' _ < <(jq -r --stream 'select(.[0][-1]=="ArchiveId") | .[1]' "$INVENTORY_FILE" | cat -n | tail -n +$skip)
---
apiVersion: v1
kind: Secret
metadata:
  name: amazonlinux-glacier-clean-legacy
type: Opaque
data:
  AWS_ACCESS_KEY_ID: <BASE64_ENCODED_VALUE>
  AWS_SECRET_ACCESS_KEY: <BASE64_ENCODED_VALUE>
---
# pvc to upload the json file
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: amazonlinux-glacier-clean-legacy
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi
# Note: Copy file using:
# kubectl cp inventory-file.json <POD_NAME_OF_JOB>:/var/glacier-job/inventory-file.json -c runner
---
apiVersion: batch/v1
kind: Job
metadata:
  name: amazonlinux-glacier-clean-legacy
spec:
  template:
    spec:
      containers:
      - name: runner
        image: amazonlinux:latest
        command: ["/bin/bash", "-c"]
        args:
        - |
          yum install -y tar
          [ -f "$INVENTORY_FILE" ] || echo "Waiting file $GLACIER_VAULT.json exists..."
          while [ ! -f "$INVENTORY_FILE" ]; do sleep 5; done
          echo "File $GLACIER_VAULT.json found! Running script..."
          exec /bin/bash /usr/local/bin/script.sh
        env:
        - name: AWS_REGION
          value: "us-east-1"
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: amazonlinux-glacier-clean-legacy
              key: AWS_ACCESS_KEY_ID
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: amazonlinux-glacier-clean-legacy
              key: AWS_SECRET_ACCESS_KEY
        - name: GLACIER_VAULT
          value: "<GLACIER_VAULT_NAME>"
        - name: SKIP
          value: "0"
        - name: PARALLEL_JOBS
          value: "50"
        - name: INVENTORY_FILE
          value: "/var/glacier-job/inventory-file.json"
        resources:
          limits:
            memory: "80Gi"
            cpu: "12000m"
          requests:
            memory: "2Gi"
            cpu: "2m"
        volumeMounts:
        - name: script
          mountPath: /usr/local/bin/script.sh
          subPath: script.sh
        - name: amazonlinux-glacier-clean-legacy
          mountPath: /var/glacier-job
      restartPolicy: Never
      volumes:
      - name: script
        configMap:
          name: script
      - name: amazonlinux-glacier-clean-legacy
        persistentVolumeClaim:
          claimName: amazonlinux-glacier-clean-legacy
  ttlSecondsAfterFinished: 2592000 # 30 days
  backoffLimit: 6