How Python Implements Iterators: From __iter__ to __next__

Prologue

Any type that implements __iter__ is an iterable (e.g., str, list, dict, set). Numbers lack __iter__ and are therefore not iterable.

from typing import Iterable
print(isinstance("", Iterable), isinstance([], Iterable), isinstance(0, Iterable))  # True True False

Being iterable lets an object be used in a for loop, but not every for -compatible object is an iterable.

Iterator Creation

A class that defines __getitem__ can be indexed and also used in a for loop, yet it is not an iterable because it lacks __iter__:

class A:
    def __getitem__(self, item):
        return f"item: {item}"

a = A()
print(a["name"])          # item: name
for idx, val in enumerate(a):
    print(val)
    if idx == 5:
        break
# output shows items 0‑5
print(isinstance(a, Iterable))  # False

Iterator Internals

The built‑in iter function accepts one or two arguments. With a single argument it calls PyObject_GetIter to obtain an iterator; with two arguments it creates a callable iterator.

static PyObject *builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) {
    // argument checking omitted for brevity
    PyObject *object = args[0];
    PyObject *sentinel = (nargs < 2) ? NULL : args[1];
    return builtin_iter_impl(module, object, sentinel);
}

static PyObject *builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel) {
    if (sentinel == NULL)
        return PyObject_GetIter(object);
    if (!PyCallable_Check(object)) {
        PyErr_SetString(PyExc_TypeError, "iter(object, sentinel): object must be callable");
        return NULL;
    }
    return PyCallIter_New(object, sentinel);
}

PyObject_GetIter

looks up the type’s tp_iter slot (the C implementation of __iter__). If tp_iter is NULL, it falls back to __getitem__ via PySeqIter_New. Otherwise it calls the iterator‑producing function.

PyObject *PyObject_GetIter(PyObject *o) {
    PyTypeObject *t = Py_TYPE(o);
    getiterfunc f = t->tp_iter;
    if (f == NULL) {
        if (PySequence_Check(o))
            return PySeqIter_New(o);
        return type_error("'%.200s' object is not iterable", o);
    } else {
        PyObject *res = (*f)(o);
        if (res != NULL && !PyIter_Check(res)) {
            PyErr_Format(PyExc_TypeError, "iter() returned non‑iterator of type '%.100s'", Py_TYPE(res)->tp_name);
            Py_SETREF(res, NULL);
        }
        return res;
    }
}

For a list, the iterator type is list_iterator, represented by the C struct _PyListIterObject:

typedef struct {
    PyObject_HEAD
    Py_ssize_t it_index;
    PyListObject *it_seq;
} _PyListIterObject;

Using ctypes we can inspect the internal fields of a list iterator and see the index increase with each next call.

from ctypes import *
class PyObject(Structure):
    _fields_ = [("ob_refcnt", c_ssize_t), ("ob_size", c_void_p)]
class ListIterObject(PyObject):
    _fields_ = [("it_index", c_ssize_t), ("it_seq", POINTER(PyObject))]
it = iter([1, 2, 3])
it_obj = ListIterObject.from_address(id(it))
print(it_obj.it_index)   # 0
next(it)
print(it_obj.it_index)   # 1

How Iterator Iterates Elements

The built‑in next function forwards to the iterator’s tp_iternext slot (the C implementation of __next__). It also supports an optional default value.

static PyObject *builtin_next(PyObject *module, PyObject *const *args, Py_ssize_t nargs) {
    PyObject *iterator = args[0];
    PyObject *default_value = (nargs < 2) ? NULL : args[1];
    return builtin_next_impl(module, iterator, default_value);
}

static PyObject *builtin_next_impl(PyObject *module, PyObject *iterator, PyObject *default_value) {
    if (!PyIter_Check(iterator)) {
        PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", Py_TYPE(iterator)->tp_name);
        return NULL;
    }
    PyObject *res = (*Py_TYPE(iterator)->tp_iternext)(iterator);
    if (res != NULL)
        return res;
    if (default_value != NULL) {
        if (PyErr_Occurred()) {
            if (!PyErr_ExceptionMatches(PyExc_StopIteration))
                return NULL;
            PyErr_Clear();
        }
        return Py_NewRef(default_value);
    }
    if (PyErr_Occurred())
        return NULL;
    PyErr_SetNone(PyExc_StopIteration);
    return NULL;
}

For a list iterator, tp_iternext points to listiter_next, which returns the next element or NULL when the index reaches the list length, at which point it_seq is set to NULL and StopIteration is raised.

static PyObject *listiter_next(_PyListIterObject *it) {
    if (it->it_seq == NULL)
        return NULL;
    if (it->it_index < PyList_GET_SIZE(it->it_seq)) {
        PyObject *item = PyList_GET_ITEM(it->it_seq, it->it_index);
        ++it->it_index;
        return Py_NewRef(item);
    }
    it->it_seq = NULL;
    Py_DECREF(it->it_seq);
    return NULL;
}

Summary

Python’s iterator protocol is built on the __iter__ and __next__ methods, which map to the C slots tp_iter and tp_iternext. Every iterable object provides an iterator that wraps the original data with a simple index that increments on each call. The CPython implementation shows how the interpreter falls back to __getitem__ when __iter__ is absent, and how the iter built‑in can also create callable iterators with a sentinel value.