Python itertools Recipes: Practical Patterns for Efficient Iteration

Why itertools Matters

The itertools module provides building blocks for efficient looping. Every function returns an iterator, meaning it produces values on demand without storing them all in memory. When you need to process millions of records, chain multiple sequences, or generate combinations, itertools does it with minimal memory and maximum speed (the functions are implemented in C).

The Essential Functions

itertools.chain: Flatten Multiple Iterables

from itertools import chain

# Combine multiple lists without creating a new list
names = chain(["Alice", "Bob"], ["Charlie"], ["Diana", "Eve"])
for name in names:
    print(name)

# Flatten a list of lists
nested = [[1, 2], [3, 4], [5, 6]]
flat = list(chain.from_iterable(nested))
print(flat)  # [1, 2, 3, 4, 5, 6]

# Practical: combine results from multiple sources
def get_local_users():
    return ["admin", "user1"]

def get_remote_users():
    return ["remote1", "remote2"]

all_users = chain(get_local_users(), get_remote_users())

itertools.islice: Slice Any Iterator

Unlike regular slicing, islice works on any iterable, including generators:

from itertools import islice

def infinite_counter():
    n = 0
    while True:
        yield n
        n += 1

# Take the first 5 values
first_five = list(islice(infinite_counter(), 5))
print(first_five)  # [0, 1, 2, 3, 4]

# Skip 10, take 5
subset = list(islice(infinite_counter(), 10, 15))
print(subset)  # [10, 11, 12, 13, 14]

# Every other item from first 20
evens = list(islice(infinite_counter(), 0, 20, 2))
print(evens)  # [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]

itertools.groupby: Group Consecutive Elements

groupby groups consecutive elements that share a key. The data must be sorted by the same key first.

from itertools import groupby
from operator import itemgetter

# Group transactions by category
transactions = [
    {"category": "food", "amount": 12.50},
    {"category": "food", "amount": 8.00},
    {"category": "transport", "amount": 25.00},
    {"category": "transport", "amount": 15.00},
    {"category": "food", "amount": 30.00},
]

# Sort first! groupby only groups CONSECUTIVE equal elements
transactions.sort(key=itemgetter("category"))

for category, group in groupby(transactions, key=itemgetter("category")):
    items = list(group)
    total = sum(t["amount"] for t in items)
    print(f"{category}: ${total:.2f} ({len(items)} transactions)")

# Output:
# food: $50.50 (3 transactions)
# transport: $40.00 (2 transactions)

itertools.product: Cartesian Product

Replace nested loops with product:

from itertools import product

# Instead of nested loops
sizes = ["S", "M", "L"]
colors = ["red", "blue"]
materials = ["cotton", "polyester"]

for size, color, material in product(sizes, colors, materials):
    print(f"{size}-{color}-{material}")

# Generate all grid coordinates
grid = list(product(range(3), range(3)))
print(grid)
# [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2), (2,0), (2,1), (2,2)]

# Repeat an iterable
binary_3bit = list(product([0, 1], repeat=3))
# [(0,0,0), (0,0,1), (0,1,0), (0,1,1), (1,0,0), (1,0,1), (1,1,0), (1,1,1)]

itertools.combinations and permutations

from itertools import combinations, permutations

# All ways to choose 2 from 4
teams = ["Alice", "Bob", "Charlie", "Diana"]
pairs = list(combinations(teams, 2))
print(pairs)
# [('Alice','Bob'), ('Alice','Charlie'), ('Alice','Diana'),
#  ('Bob','Charlie'), ('Bob','Diana'), ('Charlie','Diana')]

# Order matters with permutations
orderings = list(permutations(["A", "B", "C"], 2))
print(orderings)
# [('A','B'), ('A','C'), ('B','A'), ('B','C'), ('C','A'), ('C','B')]

# combinations_with_replacement allows repeated picks
from itertools import combinations_with_replacement
dice = list(combinations_with_replacement(range(1, 7), 2))
print(f"{len(dice)} unique dice combinations")  # 21

Practical Recipes

Batching: Process Data in Chunks

from itertools import islice

def batched(iterable, n):
    """Yield successive n-sized chunks from iterable.
    Available as itertools.batched() in Python 3.12+"""
    iterator = iter(iterable)
    while True:
        batch = list(islice(iterator, n))
        if not batch:
            break
        yield batch

# Process records in batches of 100
records = range(1050)
for batch in batched(records, 100):
    print(f"Processing batch of {len(batch)} records")
    # send_to_api(batch)

In Python 3.12+, this is available as itertools.batched.

Sliding Window

from collections import deque
from itertools import islice

def sliding_window(iterable, n):
    """sliding_window('ABCDEF', 3) -> ABC BCD CDE DEF"""
    iterator = iter(iterable)
    window = deque(islice(iterator, n), maxlen=n)
    if len(window) == n:
        yield tuple(window)
    for item in iterator:
        window.append(item)
        yield tuple(window)

# Moving average
data = [10, 20, 30, 40, 50, 60, 70]
for window in sliding_window(data, 3):
    avg = sum(window) / len(window)
    print(f"{window} -> avg: {avg:.1f}")

In Python 3.12+, this is available as itertools.pairwise for window size 2.

Flattening Deeply Nested Structures

from itertools import chain

def flatten(nested):
    """Recursively flatten nested iterables (except strings)."""
    for item in nested:
        if hasattr(item, '__iter__') and not isinstance(item, (str, bytes)):
            yield from flatten(item)
        else:
            yield item

data = [1, [2, 3], [[4, 5], [6]], [[[7]]]]
print(list(flatten(data)))  # [1, 2, 3, 4, 5, 6, 7]

Round-Robin from Multiple Sources

from itertools import cycle, islice

def roundrobin(*iterables):
    """Visit items from each iterable in turn.
    roundrobin('ABC', 'D', 'EF') -> A D E B F C"""
    iterators = [iter(it) for it in iterables]
    active = list(iterators)
    while active:
        next_active = []
        for it in active:
            try:
                yield next(it)
            except StopIteration:
                pass
            else:
                next_active.append(it)
        active = next_active

result = list(roundrobin("ABC", "D", "EF"))
print(result)  # ['A', 'D', 'E', 'B', 'F', 'C']

Accumulate: Running Totals and More

from itertools import accumulate
import operator

# Running total
data = [1, 2, 3, 4, 5]
print(list(accumulate(data)))  # [1, 3, 6, 10, 15]

# Running maximum
temps = [72, 68, 75, 71, 78, 73, 80]
running_max = list(accumulate(temps, max))
print(running_max)  # [72, 72, 75, 75, 78, 78, 80]

# Running product
factorials = list(accumulate(range(1, 8), operator.mul))
print(factorials)  # [1, 2, 6, 24, 120, 720, 5040]

# Custom accumulator: running average
def running_avg(current_avg, new_value, _count=[0]):
    _count[0] += 1
    return current_avg + (new_value - current_avg) / _count[0]

Filtering with compress and filterfalse

from itertools import compress, filterfalse

data = ["Alice", "Bob", "Charlie", "Diana"]
mask = [True, False, True, False]

# Keep items where mask is True
selected = list(compress(data, mask))
print(selected)  # ['Alice', 'Charlie']

# Split into two groups by predicate
def partition(predicate, iterable):
    """Split iterable into (falsy, truthy) groups."""
    from itertools import tee
    t1, t2 = tee(iterable)
    return filterfalse(predicate, t1), filter(predicate, t2)

numbers = range(10)
evens, odds = partition(lambda x: x % 2, numbers)
print(list(evens))  # [0, 2, 4, 6, 8]
print(list(odds))   # [1, 3, 5, 7, 9]

Performance: itertools vs List Comprehensions

import sys
from itertools import chain

# List approach: creates intermediate list in memory
nested = [list(range(1000)) for _ in range(1000)]
flat_list = [x for sublist in nested for x in sublist]
print(f"List: {sys.getsizeof(flat_list)} bytes")

# itertools approach: lazy, minimal memory
flat_iter = chain.from_iterable(nested)
# flat_iter uses almost no memory until consumed

The itertools version processes one element at a time. For a million items, this difference is the difference between your program working and running out of memory.

Combining itertools with Other Modules

from itertools import starmap, repeat
from operator import mul
from functools import reduce

# starmap: apply function to pre-paired arguments
pairs = [(2, 3), (4, 5), (6, 7)]
products = list(starmap(mul, pairs))
print(products)  # [6, 20, 42]

# repeat: create infinite or fixed-count repetitions
threes = list(repeat(3, times=5))
print(threes)  # [3, 3, 3, 3, 3]

# Combine with map for scaling
data = [1, 2, 3, 4, 5]
scaled = list(starmap(mul, zip(data, repeat(10))))
print(scaled)  # [10, 20, 30, 40, 50]

Wrapping Up

The itertools module is one of Python’s most valuable standard library modules. Its functions compose naturally, use minimal memory, and run at C speed. The key recipes to remember are chain for flattening, groupby for grouping sorted data, product for replacing nested loops, islice for slicing generators, and batched (or the manual recipe) for chunking data. Master these patterns and you will write more efficient, more readable iteration code.