Unlocking Linux Performance: How mmap Bridges Memory and I/O

What is mmap?

mmap (memory mapping) is a Linux kernel system call that creates a direct mapping between a file (or device) and a process’s virtual address space. After a successful call the process can read or write the file by accessing ordinary memory, eliminating the extra copies between kernel buffers and user buffers that occur with read / write.

mmap API

void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);

addr : Desired start address; usually NULL so the kernel chooses a free region.

length : Number of bytes to map; must be >0 and page‑aligned.

prot : Protection flags – PROT_READ, PROT_WRITE, PROT_EXEC, PROT_NONE.

flags : Mapping type – MAP_SHARED (updates are written back and visible to other processes) or MAP_PRIVATE (copy‑on‑write, original file unchanged). Additional options include MAP_ANONYMOUS, MAP_FIXED, etc.

fd : File descriptor obtained from open(). Use -1 with MAP_ANONYMOUS for anonymous memory.

offset : Byte offset in the file where the mapping starts; must be a multiple of the system page size.

Simple file‑mapping example

#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

int main() {
    const char *path = "example.txt";
    int fd = open(path, O_RDWR);
    if (fd == -1) { perror("open"); exit(EXIT_FAILURE); }
    struct stat sb;
    if (fstat(fd, &sb) == -1) { perror("fstat"); close(fd); exit(EXIT_FAILURE); }
    char *addr = mmap(NULL, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
    if (addr == MAP_FAILED) { perror("mmap"); close(fd); exit(EXIT_FAILURE); }
    close(fd);                     // fd can be closed after mapping
    printf("%s", addr);           // read file content directly
    if (munmap(addr, sb.st_size) == -1) perror("munmap");
    return 0;
}

Mapping modes

MAP_SHARED : All processes that map the same file share the same physical pages. Writes are propagated to the underlying file and become visible to other processes.

MAP_PRIVATE : Each process gets a private copy‑on‑write view. Modifications are kept in a separate page and never reach the original file.

Why mmap matters

Reduces data copies – the process accesses the file directly in its address space.

Lowers system‑call overhead – only one mmap call is needed; subsequent accesses are ordinary memory reads/writes.

Enables efficient inter‑process sharing when MAP_SHARED is used.

Kernel‑level workflow

Virtual address allocation

When mmap is invoked, the kernel validates the arguments, finds a free virtual address range, and creates a vm_area_struct (VMA) that records start/end addresses, protection flags, and mapping type.

Page‑table setup

The kernel locates the file’s inode, translates the requested file offset to physical block numbers, and calls remap_pfn_range (or an equivalent) to install page‑table entries that map the selected virtual pages to the file’s physical pages.

Demand paging and page faults

Initially the pages are not resident. On first access a page‑fault occurs; the kernel allocates a physical page (or reads it from disk), updates the page table, and resumes the process. The typical fault‑handling steps are:

Determine the fault reason (unmapped, swapped out, protection violation).

Allocate or fetch the required page.

Load file data if needed.

Update the page‑table entry with correct permissions.

Wake the faulting process.

Zero‑copy I/O with mmap

By mapping a file and sending the mapped region directly with send(), a server can avoid copying data into an intermediate user buffer. The data path is reduced from four copies (traditional I/O) to three copies (disk → page cache → socket).

Static‑file server example

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>

#define STATIC_FILE_PATH "./static/index.html"
#define SERVER_PORT 8080
#define LISTEN_BACKLOG 10

static void error_exit(const char *msg) {
    perror(msg);
    exit(EXIT_FAILURE);
}

static ssize_t mmap_send_file(int sock_fd, const char *path) {
    int fd = open(path, O_RDONLY);
    if (fd == -1) error_exit("open");
    struct stat sb;
    if (fstat(fd, &sb) == -1) { close(fd); error_exit("fstat"); }
    void *addr = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0);
    if (addr == MAP_FAILED) { close(fd); error_exit("mmap"); }
    close(fd);                     // fd no longer needed
    ssize_t sent = send(sock_fd, addr, sb.st_size, 0);
    if (sent == -1) { munmap(addr, sb.st_size); error_exit("send"); }
    munmap(addr, sb.st_size);
    return sent;
}

int main() {
    int listen_fd = socket(AF_INET, SOCK_STREAM, 0);
    if (listen_fd == -1) error_exit("socket");
    struct sockaddr_in srv;
    memset(&srv, 0, sizeof(srv));
    srv.sin_family = AF_INET;
    srv.sin_port = htons(SERVER_PORT);
    srv.sin_addr.s_addr = htonl(INADDR_ANY);
    if (bind(listen_fd, (struct sockaddr *)&srv, sizeof(srv)) == -1) error_exit("bind");
    if (listen(listen_fd, LISTEN_BACKLOG) == -1) error_exit("listen");
    printf("Server listening on %d
", SERVER_PORT);
    while (1) {
        struct sockaddr_in cli;
        socklen_t cli_len = sizeof(cli);
        int conn = accept(listen_fd, (struct sockaddr *)&cli, &cli_len);
        if (conn == -1) {
            if (errno == EINTR) continue;
            error_exit("accept");
        }
        printf("Client %s:%d connected
", inet_ntoa(cli.sin_addr), ntohs(cli.sin_port));
        ssize_t n = mmap_send_file(conn, STATIC_FILE_PATH);
        if (n > 0) printf("Sent %zd bytes
", n);
        close(conn);
    }
    close(listen_fd);
    return 0;
}

Key points: always check the return value against MAP_FAILED, close the file descriptor after mapping, and call munmap when the region is no longer needed.

Managing mmap mappings

Memory‑management pitfalls

Mapping very large files consumes a large portion of the virtual address space, which can be scarce on 32‑bit systems. Map only the required portion, unmap promptly with munmap, and consider using memory pools for frequent small allocations.

Lifecycle

A mapping exists from a successful mmap call until munmap or process exit. The kernel maintains the page tables; the application must avoid out‑of‑bounds accesses and must call munmap to release resources.

Data synchronization

For MAP_SHARED mappings, modifications become visible to other processes immediately, but they are not guaranteed to be flushed to the underlying file until the kernel decides or the application calls msync:

#include <sys/mman.h>
int msync(void *addr, size_t len, int flags);

Use MS_SYNC for synchronous writes or MS_ASYNC for asynchronous writes.

#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

int main() {
    int fd = open("test.txt", O_RDWR);
    if (fd == -1) { perror("open"); return 1; }
    struct stat sb;
    if (fstat(fd, &sb) == -1) { perror("fstat"); close(fd); return 1; }
    char *addr = mmap(NULL, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
    if (addr == MAP_FAILED) { perror("mmap"); close(fd); return 1; }
    addr[0] = 'A';                     // modify
    if (msync(addr, sb.st_size, MS_SYNC) == -1) perror("msync");
    if (munmap(addr, sb.st_size) == -1) perror("munmap");
    close(fd);
    return 0;
}

Multithreaded access

When multiple threads share a mapping, protect concurrent writes with a mutex or semaphore to avoid race conditions.

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#define MAP_SIZE 1024
pthread_mutex_t mtx;
char *shared;

void *worker(void *arg) {
    pthread_mutex_lock(&mtx);
    shared[0] = 'X';
    pthread_mutex_unlock(&mtx);
    return NULL;
}

int main() {
    int fd = open("test.txt", O_RDWR|O_CREAT, 0664);
    if (fd == -1) { perror("open"); return 1; }
    ftruncate(fd, MAP_SIZE);
    shared = mmap(NULL, MAP_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
    if (shared == MAP_FAILED) { perror("mmap"); close(fd); return 1; }
    pthread_t th;
    pthread_mutex_init(&mtx, NULL);
    if (pthread_create(&th, NULL, worker, NULL) != 0) { perror("pthread_create"); munmap(shared, MAP_SIZE); close(fd); return 1; }
    pthread_join(th, NULL);
    pthread_mutex_destroy(&mtx);
    munmap(shared, MAP_SIZE);
    close(fd);
    return 0;
}

Common pitfalls

Access after unmapping

Using a pointer after munmap results in a segmentation fault.

#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

int main() {
    int fd = open("test.txt", O_RDONLY);
    if (fd == -1) { perror("open"); return 1; }
    struct stat sb;
    if (fstat(fd, &sb) == -1) { perror("fstat"); close(fd); return 1; }
    char *addr = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
    if (addr == MAP_FAILED) { perror("mmap"); close(fd); return 1; }
    munmap(addr, sb.st_size);
    printf("%c
", addr[0]);   // <-- crash
    close(fd);
    return 0;
}

Synchronization in multithreaded/multiprocess environments

Without proper locking, concurrent writes can corrupt data. Use mutexes, semaphores, or other synchronization primitives around shared accesses.