Download all the files in an S3 bucket

download_bucket.js

import {
  S3Client,
  paginateListObjectsV2,
  GetObjectCommand,
} from '@aws-sdk/client-s3';
import { createWriteStream } from 'node:fs';
import { mkdir } from 'node:fs/promises';

const createDirectory = async (newPath, existingPath) => {
  try {
    const url = new URL(newPath, existingPath);
    await mkdir(url, {
      recursive: true,
    });
  } catch (err) {
    console.log(`Error: Failed for create directory ${url}`, err);
  }
};

const downloadObjectsFromS3 = async (region, bucketPath) => {
  const s3 = new S3Client({
    region,
  });
  const Bucket = bucketPath.split('/').shift(0);
  const Prefix = bucketPath.split('/').slice(1, bucketPath.length).join('/');
  const paginatorConfig = {
    client: s3,
    pageSize: 1000,
  };
  const commandParams = {
    Bucket,
    Prefix,
  };
  const paginator = paginateListObjectsV2(paginatorConfig, commandParams);
  try {
    for await (const page of paginator) {
      const contents = page?.Contents;
      if (contents) {
        console.log(
          `Objects found in bucket ${bucketPath}\n`,
          JSON.stringify(contents, null, 2)
        );
        const cwd = import.meta.url;
        await createDirectory(Bucket, cwd);
        for (const obj of contents) {
          const Key = obj.Key;
          // A Key ending with / is a folder
          if (!Key.endsWith('/')) {
            try {
              console.log(
                `Downloading ${Key} from bucket ${Bucket} in region ${region}`
              );
              const data = await s3.send(new GetObjectCommand({ Bucket, Key }));
              data.Body.pipe(
                createWriteStream(new URL(Bucket + '/' + Key, cwd))
              );
            } catch (err) {
              console.log(
                `Error: Failed to download ${Key} from bucket ${Bucket} in region ${region}\n`,
                err
              );
            }
          } else {
            await createDirectory(Bucket + '/' + Key, cwd);
          }
        }
      } else {
        console.log(`No objects found in bucket ${Bucket}`);
      }
    }
  } catch (err) {
    console.log(
      `Error: Failed to list contents from bucket ${Bucket} in region ${region}\n`,
      err
    );
  }
};

const run = await downloadObjectsFromS3(
  'us-east-2',
  'infrastructured-test-bucket'
);