Mastering Rust with Object Storage: Connect, Upload, Download, and Manage Large Files

Object storage is a fundamental cloud component; this article introduces the basic workflow and tips for using Rust with object storage.

Basic Connection

We use the AWS SDK for Rust (S3 SDK) as an example; the same code works with AWS, JD Cloud, and Alibaba Cloud.

Dependencies (Cargo.toml)

# oss
aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", branch = "main" }
aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", branch = "main" }
aws-types = { git = "https://github.com/awslabs/aws-sdk-rust", branch = "main", feature = ["hardcoded-credentials"] }
aws-credential-types = { git = "https://github.com/awslabs/aws-sdk-rust", branch = "main" }
aws-smithy-types = { git = "https://github.com/awslabs/aws-sdk-rust", branch = "main" }

Creating the client

let shared_config = SdkConfig::builder()
    .credentials_provider(SharedCredentialsProvider::new(Credentials::new(
        "LTAI5t7NPuPKsXm6UeSa1",
        "DGHuK03ESXQYqQ83buKMHs9NAwz",
        None,
        None,
        "Static",
    )))
    .endpoint_url("http://oss-cn-beijing.aliyuncs.com")
    .region(Region::new("oss-cn-beijing"))
    .build();
let s3_config_builder = aws_sdk_s3::config::Builder::from(&shared_config);
let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build());

The client requires the access key (AK), secret key (SK), endpoint URL, and region, which can be found in the provider’s documentation.

Object Listing

let mut obj_list = client
    .list_objects_v2()
    .bucket(bucket)
    .max_keys(max_keys)
    .prefix(prefix_str)
    .continuation_token(token_str);
let list = obj_list.send().await.unwrap();
println!("{:?}", list.contents());
println!("{:?}", list.next_continuation_token());

list_objects_v2 returns a paginated list; list.contents() gives the objects, and list.next_continuation_token() provides the token for the next page.

Uploading a File

let content = ByteStream::from("content in file".as_bytes());
let exp = aws_smithy_types::DateTime::from_secs(100);
let upload = client
    .put_object()
    .bucket("bucket")
    .key("/test/key")
    .expires(exp)
    .body(content);
upload.send().await.unwrap();

Specify bucket and key; the body accepts a ByteStream. The expires field is optional.

Downloading a File

let key = "/tmp/test/key".to_string();
let resp = client
    .get_object()
    .bucket("bucket")
    .key(&key)
    .send()
    .await.unwrap();
let data = resp.body.collect().await.unwrap();
let bytes = data.into_bytes();
let path = std::path::Path::new("/tmp/key");
if let Some(p) = path.parent() {
    std::fs::create_dir_all(p).unwrap();
}
let mut file = OpenOptions::new()
    .write(true)
    .truncate(true)
    .create(true)
    .open(path)
    .unwrap();
file.write_all(&bytes).unwrap();
file.flush().unwrap();

The get_object() call returns a ByteStream that can be written to a local file.

Deleting Files

let mut keys = vec![];
let key1 = ObjectIdentifier::builder()
    .set_key(Some("/tmp/key1".to_string()))
    .build();
let key2 = ObjectIdentifier::builder()
    .set_key(Some("/tmp/key2".to_string()))
    .build();
keys.push(key1);
keys.push(key2);
client
    .delete_objects()
    .bucket(bucket)
    .delete(Delete::builder().set_objects(Some(keys)).build())
    .send()
    .await
    .unwrap();

Delete objects in batch by constructing a vector of ObjectIdentifier and sending a Delete request.

Large File Upload

let mut file = fs::File::open("/tmp/file_name").unwrap();
let chunk_size = 1024 * 1024;
let mut part_number = 0;
let mut upload_parts: Vec<CompletedPart> = Vec::new();
// Initiate multipart upload
let multipart_upload_res = client
    .create_multipart_upload()
    .bucket("bucket")
    .key("/tmp/key")
    .send()
    .await.unwrap();
let upload_id = multipart_upload_res.upload_id().expect("upload id is None");
// Upload parts
loop {
    let mut buf = vec![0; chunk_size];
    let read_count = file.read(&mut buf)?;
    if read_count == 0 { break; }
    part_number += 1;
    let body = &buf[..read_count];
    let stream = ByteStream::from(body.to_vec());
    let upload_part_res = client
        .upload_part()
        .key(key)
        .bucket(bucket)
        .upload_id(upload_id)
        .body(stream)
        .part_number(part_number)
        .send()
        .await.unwrap();
    let completed_part = CompletedPart::builder()
        .e_tag(upload_part_res.e_tag.unwrap_or_default())
        .part_number(part_number)
        .build();
    upload_parts.push(completed_part);
    if read_count != chunk_size { break; }
}
// Complete multipart upload
let completed_multipart_upload = CompletedMultipartUpload::builder()
    .set_parts(Some(upload_parts))
    .build();
client
    .complete_multipart_upload()
    .bucket("bucket")
    .key(key)
    .multipart_upload(completed_multipart_upload)
    .upload_id(upload_id)
    .send()
    .await.unwrap();

For very large files, the code splits the file into chunks, uploads each part, records the CompletedPart, and finally merges them on the server.

Large File Download

let mut file = OpenOptions::new()
    .truncate(true)
    .create(true)
    .write(true)
    .open("/tmp/target_file");
let key = "/tmp/test/key".to_string();
let resp = client
    .get_object()
    .bucket("bucket")
    .key(&key)
    .send()
    .await.unwrap();
let content_len = resp.content_length();
let mut byte_stream_async_reader = resp.body.into_async_read();
let mut remaining = content_len as usize;
while remaining > 0 {
    let read_size = if remaining > chunk_size { chunk_size } else { remaining };
    let mut buffer = vec![0; read_size];
    byte_stream_async_reader.read_exact(&mut buffer).await.unwrap();
    file.write_all(&buffer).unwrap();
    remaining -= read_size;
}
file.flush().unwrap();

Downloading large files uses a streaming reader to write the data in chunks, reducing memory usage.

That concludes the overview of object storage with Rust.