opendal-r2.py

import pandas as pd
import numpy as np

import opendal
 
# A ~300MB file with compression='none'
row_count = 18000000
df = pd.DataFrame({
    'A': np.random.randn(row_count),  # float64
    'B': np.random.randint(0, 100, size=row_count),  # int32
    'C': np.random.randn(row_count),  # float64
    'D': np.random.randint(0, 100, size=row_count)  # int32
})

# r2 info
access_key_id = 'xxxx'
secret_access_key = 'yyyy'
endpoint_url = 'https://zzzz.r2.cloudflarestorage.com'

# init operator
op = opendal.Operator("s3", bucket="bucket", region="auto", endpoint=endpoint_url, access_key_id=access_key_id, secret_access_key=secret_access_key)

r2_path = 'file.parquet'

# upload file
op.write(r2_path, df.to_parquet(engine='pyarrow', compression='none') )

print("File uploaded successfully to Cloudflare R2.")