Implementing a FIFO Export Queue for Large‑Scale Data Export in Java

The author introduces a business need: as database volume grows, full‑data exports can strain MySQL I/O and server resources, especially when multiple users export simultaneously. To mitigate this, a fixed‑size FIFO queue is proposed to serialize export tasks.

Key entities are defined: a user, an export queue, and an export execution method. The queue limits concurrent exports (max size 10) and records file locations for later download.

ExportQueue implementation

package com.example.system.config;

import com.example.system.api.domain.ExportUser;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.util.LinkedList;

@Slf4j
@Component
public class ExportQueue {

    private final int MAX_CAPACITY = 10; // queue max capacity
    private LinkedList<ExportUser> queue; // user queue

    public ExportQueue(LinkedList<ExportUser> queue) {
        this.queue = new LinkedList<>();
    }

    /**
     * Add a user to the queue
     */
    public synchronized LinkedList<ExportUser> add(ExportUser sysUser) {
        while (queue.size() >= MAX_CAPACITY) {
            try {
                log.info("当前排队人已满，请等待");
                wait();
            } catch (InterruptedException e) {
                e.getMessage();
            }
        }
        queue.add(sysUser);
        log.info("目前导出队列排队人数：" + queue.size());
        notifyAll();
        return queue;
    }

    /**
     * Get the next user in the queue
     */
    public synchronized ExportUser getNextSysUser() {
        while (queue.isEmpty()) {
            try {
                wait();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        ExportUser sysUser = queue.remove();
        notifyAll(); // wake up others
        return sysUser;
    }
}

AbstractExport class (EasyExcel integration)

package com.example.system.config;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.util.PageUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.example.system.api.domain.ExportUser;
import lombok.extern.slf4j.Slf4j;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.List;

@Slf4j
public abstract class AbstractExport<T, K> {

    public abstract void export(ExportUser sysUser) throws InterruptedException;

    /**
     * Export method
     */
    public void export(HttpServletResponse response, int pageSize, T t, Class<K> k, String fileName) throws Exception {
        ExcelWriter writer = null;
        try {
            writer = getExcelWriter(response, fileName);
            int total = this.countExport(t);
            int loopCount = PageUtil.totalPage(total, pageSize);
            BeanUtil.setProperty(t, "pageSize", pageSize);
            for (int i = 0; i < loopCount; i++) {
                BeanUtil.setProperty(t, "pageNum", PageUtil.getStart(i + 1, pageSize));
                List<K> kList = this.getExportDetail(t);
                WriteSheet writeSheet = EasyExcel.writerSheet(fileName).head(k).build();
                writer.write(kList, writeSheet);
            }
        } catch (Exception e) {
            String msg = "导出" + fileName + "异常";
            log.error(msg, e);
            throw new Exception(msg + e);
        } finally {
            if (writer != null) {
                writer.finish();
            }
        }
    }

    public com.alibaba.excel.ExcelWriter getExcelWriter(HttpServletResponse response, String fileName) throws IOException {
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("utf-8");
        String fileNameUtf = URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+", "%20");
        response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileNameUtf + ".xlsx");
        return EasyExcel.write(response.getOutputStream()).build();
    }

    public abstract void complexFillWithTable(T t, String fileName, HttpServletResponse response);
    public abstract int countExport(T t);
    public abstract List<K> getExportDetail(T t);
}

ExportImpl concrete service

package com.example.system.service.impl;

import com.alibaba.excel.ExcelWriter;
import com.example.system.api.domain.ExportUser;
import com.example.system.config.AbstractExport;
import com.example.system.config.ExportQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;

@Service
@Slf4j
public class ExportImpl extends AbstractExport {

    @Autowired
    private ExportQueue exportQueue;

    @Override
    public void export(ExportUser sysUser) throws InterruptedException {
        log.info("导出文件方法执行～～～～～～～～～");
        LinkedList<ExportUser> queue = exportQueue.add(sysUser);
        log.info("导出队列：" + queue);
        Thread.sleep(20000); // simulate export time
        ExportUser nextSysUser = exportQueue.getNextSysUser();
        log.info("移除后获取下一个排队的用户: " + nextSysUser.getUserName());
    }

    @Override
    public void export(HttpServletResponse response, int pageSize, Object o, Class k, String fileName) throws Exception {
        super.export(response, pageSize, o, k, fileName);
    }

    @Override
    public ExcelWriter getExcelWriter(HttpServletResponse response, String fileName) throws IOException {
        return super.getExcelWriter(response, fileName);
    }

    @Override
    public void complexFillWithTable(Object o, String fileName, HttpServletResponse response) {
        // not implemented
    }

    @Override
    public int countExport(Object o) {
        return 0;
    }

    @Override
    public List getExportDetail(Object o) {
        return null;
    }
}

Test controller to trigger export in separate threads

package com.example.system.controller;

import com.example.system.api.domain.ExportUser;
import com.example.system.service.impl.ExportImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/export")
@Slf4j
public class ExportController {

    @Autowired
    private ExportImpl export;

    @PostMapping("/exportFile")
    public void exportFile() {
        new Thread(new Runnable() {
            @SneakyThrows
            @Override
            public void run() {
                Thread thread1 = Thread.currentThread();
                ExportUser sysUser = new ExportUser();
                sysUser.setUserName(thread1.getName());
                export.export(sysUser);
            }
        }).start();
    }
}

The test results show that the queue caps at ten concurrent exports; additional requests wait until a slot frees. Two requests spaced ten seconds apart demonstrate that after the first export finishes, the next user is dequeued and processed.

In the conclusion, the author notes that further work is needed for file‑table design, OSS upload, download handling, and high‑concurrency edge cases. Alternative implementations such as Redis‑based queues are mentioned as possible extensions.