How Does Linux FS‑Cache Use Cookies to Speed Up Network Filesystems?

What Is FS‑Cache?

FS‑Cache (File System Cache) is a Linux kernel subsystem that provides a local‑disk cache for network filesystems, improving performance of slow network accesses. It can also be used by non‑network filesystems, e.g., the on‑demand read feature added to EROFS.

Cookie Concepts

FS‑Cache stores cached data in objects called cookies . Three types:

Cache cookie – represents the whole cache.

Volume cookie – represents a collection of files (e.g., a superblock).

Data file cookie – caches individual file data (inode).

Cache cookies represent the cache as a whole and are not normally visible to the netfs; the netfs gets a volume cookie to represent a collection of files (typically a superblock); and data file cookies are used to cache data (an inode).

Definitions are in linux/fs/fscache.h (NetFS interface) and linux/fs/fscache‑cache.h (backend interface).

Volume Cookie Management

The struct fscache_volume contains the key fields:

atomic_t n_cookies;          /* Number of data cookies in volume */
struct hlist_bl_node hash_link;  /* Link in hash table */
struct work_struct work;          /* Worker for async creation */
struct fscache_cache *cache;      /* The cache this volume belongs to */

Allocation initializes a work item:

static struct fscache_volume *fscache_alloc_volume(const char *volume_key,
                                                   const char *cache_name,
                                                   const void *coherency_data,
                                                   size_t coherency_len)
{
    …
    INIT_WORK(&volume->work, fscache_create_volume_work);
}

Creating a volume schedules the work:

void fscache_create_volume(struct fscache_volume *volume, bool wait)
{
    …
    if (!schedule_work(&volume->work))
        …
}

Insertion into the global hash table is performed by fscache_hash_volume(). On collision the code checks whether the existing volume is being relinquished; if so it waits before inserting the new volume.

static bool fscache_hash_volume(struct fscache_volume *candidate)
{
    …
    hlist_bl_lock(h);
    hlist_bl_for_each_entry(cursor, p, h, hash_link) {
        if (fscache_volume_same(candidate, cursor)) {
            if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags))
                goto collision;
            fscache_see_volume(cursor, fscache_volume_get_hash_collision);
            set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags);
            set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags);
            collidee_debug_id = cursor->debug_id;
            break;
        }
    }
    hlist_bl_add_head(&candidate->hash_link, h);
    hlist_bl_unlock(h);
    if (test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags))
        fscache_wait_on_volume_collision(candidate, collidee_debug_id);
    return true;
}

Removal clears the hash entry and wakes any pending volume:

static void fscache_unhash_volume(struct fscache_volume *volume)
{
    …
    hlist_bl_lock(h);
    hlist_bl_del(&volume->hash_link);
    if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags))
        fscache_wake_pending_volume(volume, h);
    hlist_bl_unlock(h);
}

Data File Cookie Management

The struct fscache_cookie (data file cookie) includes:

atomic_t            n_accesses;   /* Number of cache accesses in progress */
spinlock_t          lock;
struct fscache_volume *volume;   /* Parent volume */
struct hlist_bl_node hash_link; /* Link in hash table */
struct list_head    commit_link; /* Link in commit queue */
struct work_struct  work;        /* Commit/relinq/withdraw work */
unsigned long       unused_at;   /* Time when unused (jiffies) */
unsigned long       flags;
enum fscache_cookie_state state;

Allocation sets up the work item and puts the cookie into a quiescent state:

static struct fscache_cookie *fscache_alloc_cookie(...)
{
    …
    INIT_WORK(&cookie->work, fscache_cookie_worker);
    __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
}

The worker runs the state machine and finally releases the cookie:

static void fscache_cookie_worker(struct work_struct *work)
{
    struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
    fscache_see_cookie(cookie, fscache_cookie_see_work);
    fscache_cookie_state_machine(cookie);
    fscache_put_cookie(cookie, fscache_cookie_put_work);
}

When a cookie becomes unused it is moved to a global LRU list; unused_at records the timestamp.

static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
{
    cookie->unused_at = jiffies;
    spin_lock(&fscache_cookie_lru_lock);
    list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
    spin_unlock(&fscache_cookie_lru_lock);
}

The LRU worker iterates over the list, checks the timeout fscache_lru_cookie_timeout, and discards stale cookies.

static void fscache_cookie_lru_worker(struct work_struct *work)
{
    while (!list_empty(&fscache_cookie_lru)) {
        cookie = list_first_entry(&fscache_cookie_lru,
                                  struct fscache_cookie, commit_link);
        unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
        if (time_before(jiffies, unused_at)) {
            timer_reduce(&fscache_cookie_lru_timer, unused_at);
            break;
        }
        list_del_init(&cookie->commit_link);
        fscache_stat_d(&fscache_n_cookies_lru);
        spin_unlock(&fscache_cookie_lru_lock);
        fscache_cookie_lru_do_one(cookie);
    }
}

Cache Cookie Management

The struct fscache_cache (cache cookie) holds backend operations and bookkeeping:

const struct fscache_cache_ops *ops;
struct list_head cache_link;   /* Link in global cache list */
atomic_t n_volumes;           /* Number of active volumes */
enum fscache_cache_state state;

Acquiring a cache adds it to the global list fscache_caches:

struct fscache_cache *fscache_acquire_cache(const char *name)
{
    ASSERT(name);
    cache = fscache_lookup_cache(name, true);
    return cache;
}

Lookup inserts the cache into the list and increments the reference count.

struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache)
{
    …
    list_add_tail(&candidate->cache_link, &fscache_caches);
    …
    return candidate;
}

Releasing a cache decrements the reference count; when it reaches zero the cache is removed and its memory freed.

void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where)
{
    …
    zero = __refcount_dec_and_test(&cache->ref, &ref);
    if (zero) {
        down_write(&fscache_addremove_sem);
        list_del_init(&cache->cache_link);
        up_write(&fscache_addremove_sem);
        kfree(cache->name);
        kfree(cache);
    }
}

The cache cookie references a struct fscache_cache_ops supplied by the backend (e.g., CacheFiles):

const struct fscache_cache_ops cachefiles_cache_ops = {
    .name            = "cachefiles",
    .acquire_volume  = cachefiles_acquire_volume,
    .free_volume     = cachefiles_free_volume,
    .lookup_cookie   = cachefiles_lookup_cookie,
    .withdraw_cookie = cachefiles_withdraw_cookie,
    …
};

References

Kernel documentation for FS‑Cache: https://www.kernel.org/doc/html/latest/filesystems/caching/fscache.html

FS‑Cache design paper: https://people.redhat.com/dhowells/fscache/FS-Cache.pdf