Kernel Perspective: Accelerating C++ File Transfer via System Call Optimizations

When C++ developers implement file transfer, logging, or network I/O, the business logic often works correctly, yet performance degrades dramatically for large files, high‑frequency small files, or high‑concurrency scenarios. The root cause is not C++ syntax but frequent user‑space ↔ kernel‑space transitions, over‑use of system calls, and multiple data copies.

Kernel view of a simple read‑write operation

During a read, the disk transfers data to the kernel page cache via DMA without CPU involvement, then the CPU copies the data from the page cache to the user buffer. During a write, data is copied from the user buffer to the kernel socket buffer before DMA sends it to the NIC. A single read‑write cycle therefore incurs four context switches and four copies (two DMA copies, two CPU copies), consuming CPU cycles and memory bandwidth.

Performance test of the traditional approach

The following program demonstrates the classic read/write loop and can be used to measure CPU and kernel‑time overhead when copying a large file:

#include <fcntl.h>
#include <unistd.h>
#include <cstdio>

int main() {
    int src_fd = open("source_file.txt", O_RDONLY);
    int dest_fd = open("target_file.txt", O_WRONLY | O_CREAT | O_TRUNC, 0644);
    if (src_fd == -1 || dest_fd == -1) { perror("open"); return -1; }
    const int BUF_SIZE = 4096;
    char buffer[BUF_SIZE];
    ssize_t read_len;
    while ((read_len = read(src_fd, buffer, BUF_SIZE)) > 0) {
        write(dest_fd, buffer, read_len);
    }
    close(src_fd);
    close(dest_fd);
    printf("File transfer completed
");
    return 0;
}

Running this code on a large file shows a noticeable rise in CPU usage and a large proportion of time spent in kernel mode, confirming the analysis above.

Solution 1 – Reduce memory copies (Zero‑Copy)

Zero‑copy removes the extra user↔kernel copies by mapping the file directly into the process address space with mmap and then writing the mapped region to the destination. This eliminates the read/write data copy path while keeping only the necessary system calls.

#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <cstdio>

int main() {
    int src_fd = open("source_file.dat", O_RDONLY);
    if (src_fd == -1) { perror("open source"); return -1; }
    struct stat file_st;
    if (fstat(src_fd, &file_st) == -1) { perror("fstat"); close(src_fd); return -1; }
    off_t file_size = file_st.st_size;
    if (file_size == 0) { printf("File empty
"); close(src_fd); return 0; }
    char *map_buf = (char*)mmap(NULL, file_size, PROT_READ, MAP_SHARED, src_fd, 0);
    if (map_buf == MAP_FAILED) { perror("mmap"); close(src_fd); return -1; }
    int dest_fd = open("target_file.dat", O_WRONLY | O_CREAT | O_TRUNC, 0644);
    if (dest_fd == -1) { perror("open dest"); munmap(map_buf, file_size); close(src_fd); return -1; }
    write(dest_fd, map_buf, file_size);
    munmap(map_buf, file_size);
    close(src_fd);
    close(dest_fd);
    printf("mmap zero‑copy file transfer completed
");
    return 0;
}

Solution 2 – Avoid context switches with asynchronous I/O

Asynchronous I/O lets the program issue a read or write request and continue executing other work while the kernel performs the operation in the background. On Linux, the POSIX AIO API ( aio_read, aio_write) can be used:

#include <aio.h>
#include <iostream>
#include <fcntl.h>
#include <unistd.h>

const int buffer_size = 1024;
char buffer[buffer_size];

int main() {
    int fd = open("test.txt", O_RDONLY);
    if (fd == -1) { perror("open"); return 1; }
    struct aiocb cb{};
    cb.aio_fildes = fd;
    cb.aio_buf = buffer;
    cb.aio_nbytes = buffer_size;
    cb.aio_offset = 0;
    if (aio_read(&cb) == -1) { perror("aio_read"); close(fd); return 1; }
    while (aio_error(&cb) == EINPROGRESS) { /* do other work */ }
    ssize_t bytes = aio_return(&cb);
    if (bytes == -1) { perror("aio_return"); }
    else { std::cout << "Read " << bytes << " bytes: " << buffer << std::endl; }
    close(fd);
    return 0;
}

This method reduces the time the thread spends blocked on I/O, but it introduces additional programming complexity because the program must manage the AIO control block and handle completion status.

Windows alternative – Overlapped I/O with I/O Completion Ports

On Windows, the combination of FILE_FLAG_OVERLAPPED and I/O Completion Ports (IOCP) provides a similar non‑blocking, high‑throughput solution. The code below creates an IOCP, opens a file with overlapped mode, issues an asynchronous read, and waits for completion via GetQueuedCompletionStatus:

#include <Windows.h>
#include <iostream>

#define BUFFER_SIZE 4096

typedef struct { OVERLAPPED overlapped; CHAR buffer[BUFFER_SIZE]; } IO_DATA, *LPIO_DATA;

int main() {
    HANDLE hPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if (hPort == NULL) { std::cout << "CreateIoCompletionPort failed, error: " << GetLastError() << std::endl; return -1; }
    HANDLE hFile = CreateFileA("test_win_file.dat", GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED, NULL);
    if (hFile == INVALID_HANDLE_VALUE) { std::cout << "Open file failed, error: " << GetLastError() << std::endl; CloseHandle(hPort); return -1; }
    CreateIoCompletionPort(hFile, hPort, (ULONG_PTR)hFile, 0);
    LPIO_DATA pIo = new IO_DATA{};
    ZeroMemory(&pIo->overlapped, sizeof(OVERLAPPED));
    BOOL ok = ReadFile(hFile, pIo->buffer, BUFFER_SIZE, NULL, &pIo->overlapped);
    if (!ok && GetLastError() != ERROR_IO_PENDING) { std::cout << "Async read failed, error: " << GetLastError() << std::endl; delete pIo; CloseHandle(hFile); CloseHandle(hPort); return -1; }
    std::cout << "Async read issued, main thread can continue..." << std::endl;
    DWORD bytes; ULONG_PTR key; LPOVERLAPPED lp;
    BOOL got = GetQueuedCompletionStatus(hPort, &bytes, &key, &lp, INFINITE);
    if (got && bytes > 0) {
        std::cout << "Async read succeeded, bytes: " << bytes << std::endl;
        std::cout << "Content: " << pIo->buffer << std::endl;
    } else {
        std::cout << "Async read failed, error: " << GetLastError() << std::endl;
    }
    delete pIo;
    CloseHandle(hFile);
    CloseHandle(hPort);
    return 0;
}

Both the POSIX and Windows asynchronous approaches eliminate the blocking wait present in the synchronous read/write loop, allowing the CPU to handle other tasks and improving overall throughput.

Takeaway

By understanding the kernel’s role in file I/O and applying zero‑copy or asynchronous techniques, developers can dramatically lower CPU consumption, increase throughput, and reduce latency for embedded file reads, backend services, or network‑based file transfers.