Mastering Large File Uploads with Spring Boot: Chunked Upload Guide

Why Chunked Upload?

When a file exceeds 100 MB, traditional upload faces three main problems: unstable network transmission, server resource exhaustion, and high cost of failure.

Network instability: Long single requests are easy to interrupt.

Server resource exhaustion: Loading the whole file at once can cause out‑of‑memory errors.

High failure cost: The entire file must be re‑uploaded after an error.

Advantages of Chunked Upload

Reduces load per request.

Supports resumable uploads.

Allows concurrent uploading for higher efficiency.

Lowers memory pressure on the server.

Core Principle

Spring Boot Implementation

1. Core Dependencies

<dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
        <groupId>commons-io</groupId>
        <artifactId>commons-io</artifactId>
        <version>2.11.0</version>
    </dependency>
</dependencies>

2. Controller Logic

@RestController
@RequestMapping("/upload")
public class ChunkUploadController {

    private final String CHUNK_DIR = "uploads/chunks/";
    private final String FINAL_DIR = "uploads/final/";

    /** Initialize upload */
    @PostMapping("/init")
    public ResponseEntity<String> initUpload(@RequestParam String fileName,
                                             @RequestParam String fileMd5) {
        String uploadId = UUID.randomUUID().toString();
        Path chunkDir = Paths.get(CHUNK_DIR, fileMd5 + "_" + uploadId);
        try {
            Files.createDirectories(chunkDir);
        } catch (IOException e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("创建目录失败");
        }
        return ResponseEntity.ok(uploadId);
    }

    /** Upload a chunk */
    @PostMapping("/chunk")
    public ResponseEntity<String> uploadChunk(@RequestParam MultipartFile chunk,
                                            @RequestParam String uploadId,
                                            @RequestParam String fileMd5,
                                            @RequestParam Integer index) {
        String chunkName = "chunk_" + index + ".tmp";
        Path filePath = Paths.get(CHUNK_DIR, fileMd5 + "_" + uploadId, chunkName);
        try {
            chunk.transferTo(filePath);
            return ResponseEntity.ok("分块上传成功");
        } catch (IOException e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("分块保存失败");
        }
    }

    /** Merge chunks */
    @PostMapping("/merge")
    public ResponseEntity<String> mergeChunks(@RequestParam String fileName,
                                               @RequestParam String uploadId,
                                               @RequestParam String fileMd5) {
        File chunkDir = new File(CHUNK_DIR + fileMd5 + "_" + uploadId);
        File[] chunks = chunkDir.listFiles();
        if (chunks == null || chunks.length == 0) {
            return ResponseEntity.badRequest().body("无分块文件");
        }
        Arrays.sort(chunks, Comparator.comparingInt(f ->
                Integer.parseInt(f.getName().split("_")[1].split("\\.")[0])));
        Path finalPath = Paths.get(FINAL_DIR, fileName);
        try (BufferedOutputStream outputStream = new BufferedOutputStream(
                Files.newOutputStream(finalPath))) {
            for (File chunkFile : chunks) {
                Files.copy(chunkFile.toPath(), outputStream);
            }
            FileUtils.deleteDirectory(chunkDir);
            return ResponseEntity.ok("文件合并成功：" + finalPath);
        } catch (IOException e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("合并失败：" + e.getMessage());
        }
    }
}

3. High‑Performance Merge Optimization

// Use RandomAccessFile for large files
public void mergeFiles(File targetFile, List<File> chunkFiles) throws IOException {
    try (RandomAccessFile target = new RandomAccessFile(targetFile, "rw")) {
        byte[] buffer = new byte[1024 * 8];
        for (File chunk : chunkFiles) {
            try (RandomAccessFile src = new RandomAccessFile(chunk, "r")) {
                int bytesRead;
                while ((bytesRead = src.read(buffer)) != -1) {
                    target.write(buffer, 0, bytesRead);
                }
            }
        }
    }
}

Front‑End (Vue) Example

1. Chunk Processing

// 5 MB per chunk
const CHUNK_SIZE = 5 * 1024 * 1024;

function processFile(file) {
    const chunkCount = Math.ceil(file.size / CHUNK_SIZE);
    const chunks = [];
    for (let i = 0; i < chunkCount; i++) {
        const start = i * CHUNK_SIZE;
        const end = Math.min(file.size, start + CHUNK_SIZE);
        chunks.push(file.slice(start, end));
    }
    return chunks;
}

2. Upload with Progress

async function uploadFile(file) {
    const { data: uploadId } = await axios.post('/upload/init', {
        fileName: file.name,
        fileMd5: await calculateFileMD5(file)
    });
    const chunks = processFile(file);
    const total = chunks.length;
    let uploaded = 0;
    await Promise.all(chunks.map((chunk, index) => {
        const formData = new FormData();
        formData.append('chunk', chunk, `chunk_${index}`);
        formData.append('index', index);
        formData.append('uploadId', uploadId);
        formData.append('fileMd5', fileMd5);
        return axios.post('/upload/chunk', formData, {
            headers: { 'Content-Type': 'multipart/form-data' },
            onUploadProgress: () => {
                const percent = ((uploaded * 100) / total).toFixed(1);
                updateProgress(percent);
            }
        }).then(() => uploaded++);
    }));
    const result = await axios.post('/upload/merge', {
        fileName: file.name,
        uploadId,
        fileMd5
    });
    alert(`上传成功: ${result.data}`);
}

Enterprise‑Level Enhancements

1. Resumable Check API

@GetMapping("/check/{fileMd5}/{uploadId}")
public ResponseEntity<List<Integer>> getUploadedChunks(@PathVariable String fileMd5,
                                                      @PathVariable String uploadId) {
    Path chunkDir = Paths.get(CHUNK_DIR, fileMd5 + "_" + uploadId);
    if (!Files.exists(chunkDir)) {
        return ResponseEntity.ok(Collections.emptyList());
    }
    try {
        List<Integer> uploaded = Files.list(chunkDir)
                .map(p -> p.getFileName().toString())
                .filter(name -> name.startsWith("chunk_"))
                .map(name -> name.replace("chunk_", "").replace(".tmp", ""))
                .map(Integer::parseInt)
                .collect(Collectors.toList());
        return ResponseEntity.ok(uploaded);
    } catch (IOException e) {
        return ResponseEntity.status(500).body(Collections.emptyList());
    }
}

2. Chunk Signature Verification

@PostMapping("/chunk")
public ResponseEntity<?> uploadChunk(@RequestParam MultipartFile chunk,
                                     @RequestParam String sign) {
    String secretKey = "your-secret-key";
    String serverSign = HmacUtils.hmacSha256Hex(secretKey, chunk.getBytes());
    if (!serverSign.equals(sign)) {
        return ResponseEntity.status(403).body("签名验证失败");
    }
    // process chunk …
    return ResponseEntity.ok("分块上传成功");
}

3. MinIO Storage Integration

@Configuration
public class MinioConfig {
    @Bean
    public MinioClient minioClient() {
        return MinioClient.builder()
                .endpoint("http://minio:9000")
                .credentials("minio-access", "minio-secret")
                .build();
    }
}

@Service
public class MinioUploadService {
    @Autowired
    private MinioClient minioClient;

    public void uploadChunk(String bucket, String object,
                            InputStream chunkStream, long length) throws Exception {
        minioClient.putObject(
                PutObjectArgs.builder()
                        .bucket(bucket)
                        .object(object)
                        .stream(chunkStream, length, -1)
                        .build()
        );
    }
}

Performance Test Comparison

Traditional upload: >3 h, >10 GB memory, 100 % retry cost.

Chunked upload (single thread): 1.5 h, ~100 MB memory, ≈10 % retry cost.

Chunked upload (multi‑thread): 20 min, ~100 MB memory, <1 % retry cost.

Best Practices

Chunk Size Selection

Intranet: 10 MB‑20 MB

Mobile network: 1 MB‑5 MB

WAN: 500 KB‑1 MB

Scheduled Cleanup

@Scheduled(fixedRate = 24 * 60 * 60 * 1000) // daily
public void cleanTempFiles() {
    File tempDir = new File(CHUNK_DIR);
    FileUtils.deleteDirectory(tempDir);
}

Rate Limiting

spring:
  servlet:
    multipart:
      max-file-size: 100MB   # per chunk limit
      max-request-size: 100MB

Conclusion

Using Spring Boot for chunked file upload eliminates the core pain points of large‑file transmission; combined with resumable upload, chunk verification, and security controls, it enables a robust, enterprise‑grade solution that can be tuned for chunk size and concurrency to meet real‑world requirements.