Skip to main content Brad's PyNotes

PEP 380 Yield From

TL;DR

PEP 380 introduced yield from syntax for delegating to subgenerators, simplifying generator composition and enabling generators to return values.

Interesting!

yield from was the foundation that made async/await possible - it established the delegation pattern that coroutines needed!

Basic Generator Delegation

python code snippet start

def inner_generator():
    yield 1
    yield 2
    yield 3

def outer_generator():
    yield 'start'
    yield from inner_generator()  # Delegate to subgenerator
    yield 'end'

# Usage
for value in outer_generator():
    print(value)
# Output: start, 1, 2, 3, end

python code snippet end

Before yield from

python code snippet start

# Manual delegation - verbose and error-prone
def manual_delegation():
    yield 'start'
    
    # Manually yield each value from subgenerator
    for value in inner_generator():
        yield value
    
    yield 'end'

# With yield from - clean and automatic
def clean_delegation():
    yield 'start'
    yield from inner_generator()  # Much simpler!
    yield 'end'

python code snippet end

Generator Return Values

python code snippet start

def subgenerator():
    yield 1
    yield 2
    return "finished!"  # Generators can return values

def main_generator():
    yield 'starting'
    result = yield from subgenerator()  # Capture return value
    yield f'subgenerator returned: {result}'

# Usage
for value in main_generator():
    print(value)
# Output: starting, 1, 2, subgenerator returned: finished!

python code snippet end

Two-Way Communication

python code snippet start

def echo_generator():
    """A generator that echoes sent values."""
    while True:
        received = yield
        if received is None:
            return "echo finished"
        yield f"echo: {received}"

def delegating_generator():
    yield "starting echo"
    result = yield from echo_generator()
    yield f"result: {result}"

# Usage with send()
gen = delegating_generator()
print(next(gen))  # starting echo

print(gen.send("hello"))  # echo: hello
print(gen.send("world"))  # echo: world

try:
    print(gen.send(None))  # Triggers return
except StopIteration:
    pass

python code snippet end

Flattening Nested Structures

python code snippet start

def flatten(nested_list):
    """Recursively flatten a nested list structure."""
    for item in nested_list:
        if isinstance(item, list):
            yield from flatten(item)  # Recursive delegation
        else:
            yield item

# Usage
nested = [1, [2, 3], [4, [5, 6]], 7]
flat = list(flatten(nested))
print(flat)  # [1, 2, 3, 4, 5, 6, 7]

python code snippet end

Tree Traversal

python code snippet start

class TreeNode:
    def __init__(self, value, children=None):
        self.value = value
        self.children = children or []

def traverse_tree(node):
    """Depth-first traversal using yield from."""
    yield node.value
    
    for child in node.children:
        yield from traverse_tree(child)  # Delegate to subtree

# Build a tree
root = TreeNode('A', [
    TreeNode('B', [TreeNode('D'), TreeNode('E')]),
    TreeNode('C', [TreeNode('F')])
])

# Traverse
for value in traverse_tree(root):
    print(value)  # A, B, D, E, C, F

python code snippet end

Exception Handling

python code snippet start

def might_fail():
    try:
        yield 1
        yield 2
        raise ValueError("Something went wrong!")
        yield 3  # Never reached
    except ValueError as e:
        yield f"Caught: {e}"
        return "recovered"

def exception_delegator():
    yield "starting"
    try:
        result = yield from might_fail()
        yield f"result: {result}"
    except Exception as e:
        yield f"outer caught: {e}"

# Usage
for value in exception_delegator():
    print(value)
# Output: starting, 1, 2, Caught: Something went wrong!, result: recovered

python code snippet end

Coroutine Simulation

python code snippet start

def coroutine_consumer():
    """Simulates a coroutine that processes values."""
    total = 0
    count = 0
    
    while True:
        value = yield
        if value is None:
            break
        total += value
        count += 1
        yield f"processed {value}, running total: {total}"
    
    return f"final average: {total/count if count else 0}"

def coroutine_delegator():
    print("Setting up consumer...")
    result = yield from coroutine_consumer()
    print(f"Consumer finished with: {result}")

# Usage
coro = coroutine_delegator()
next(coro)  # Prime the coroutine

print(coro.send(10))  # processed 10, running total: 10
print(coro.send(20))  # processed 20, running total: 30
print(coro.send(30))  # processed 30, running total: 60

try:
    coro.send(None)  # Signal completion
except StopIteration:
    pass

python code snippet end

Generator Composition

python code snippet start

def numbers(start, end):
    """Generate numbers in range."""
    for i in range(start, end):
        yield i

def squares(iterable):
    """Square each number."""
    for num in iterable:
        yield num ** 2

def even_only(iterable):
    """Filter even numbers."""
    for num in iterable:
        if num % 2 == 0:
            yield num

def pipeline():
    """Compose generators using yield from."""
    yield from even_only(squares(numbers(1, 10)))

# Usage
result = list(pipeline())
print(result)  # [4, 16, 36, 64]  (squares of even numbers)

python code snippet end

Real-World Example: File Processing

python code snippet start

import os

def process_file(filepath):
    """Process a single file."""
    try:
        with open(filepath, 'r') as f:
            line_count = sum(1 for _ in f)
        yield f"Processed {filepath}: {line_count} lines"
        return line_count
    except Exception as e:
        yield f"Error processing {filepath}: {e}"
        return 0

def process_directory(directory):
    """Process all files in directory."""
    total_lines = 0
    file_count = 0
    
    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)
        if os.path.isfile(filepath) and filename.endswith('.py'):
            file_count += 1
            lines = yield from process_file(filepath)
            total_lines += lines
    
    return f"Processed {file_count} files, {total_lines} total lines"

def file_processor(directory):
    """Main processor with summary."""
    yield "Starting file processing..."
    summary = yield from process_directory(directory)
    yield f"Summary: {summary}"

# Usage (if directory exists)
# for message in file_processor('.'):
#     print(message)

python code snippet end

Connection to async/await

python code snippet start

# yield from was the foundation for async/await
def old_style_coroutine():
    result = yield from some_async_operation()
    return result

# Modern equivalent
async def new_style_coroutine():
    result = await some_async_operation()
    return result

# The semantics are nearly identical!

python code snippet end

yield from transformed how we compose generators and laid the groundwork for Python’s async programming model - it’s the unsung hero behind modern asynchronous Python!

Reference: PEP 380 – Syntax for Delegating to a Subgenerator