PEP 380 Yield From
TL;DR
PEP 380 introduced yield from
syntax for delegating to subgenerators, simplifying generator composition and enabling generators to return values.
Interesting!
yield from
was the foundation that made async/await
possible - it established the delegation pattern that coroutines needed!
Basic Generator Delegation
python code snippet start
def inner_generator():
yield 1
yield 2
yield 3
def outer_generator():
yield 'start'
yield from inner_generator() # Delegate to subgenerator
yield 'end'
# Usage
for value in outer_generator():
print(value)
# Output: start, 1, 2, 3, end
python code snippet end
Before yield from
python code snippet start
# Manual delegation - verbose and error-prone
def manual_delegation():
yield 'start'
# Manually yield each value from subgenerator
for value in inner_generator():
yield value
yield 'end'
# With yield from - clean and automatic
def clean_delegation():
yield 'start'
yield from inner_generator() # Much simpler!
yield 'end'
python code snippet end
Generator Return Values
python code snippet start
def subgenerator():
yield 1
yield 2
return "finished!" # Generators can return values
def main_generator():
yield 'starting'
result = yield from subgenerator() # Capture return value
yield f'subgenerator returned: {result}'
# Usage
for value in main_generator():
print(value)
# Output: starting, 1, 2, subgenerator returned: finished!
python code snippet end
Two-Way Communication
python code snippet start
def echo_generator():
"""A generator that echoes sent values."""
while True:
received = yield
if received is None:
return "echo finished"
yield f"echo: {received}"
def delegating_generator():
yield "starting echo"
result = yield from echo_generator()
yield f"result: {result}"
# Usage with send()
gen = delegating_generator()
print(next(gen)) # starting echo
print(gen.send("hello")) # echo: hello
print(gen.send("world")) # echo: world
try:
print(gen.send(None)) # Triggers return
except StopIteration:
pass
python code snippet end
Flattening Nested Structures
python code snippet start
def flatten(nested_list):
"""Recursively flatten a nested list structure."""
for item in nested_list:
if isinstance(item, list):
yield from flatten(item) # Recursive delegation
else:
yield item
# Usage
nested = [1, [2, 3], [4, [5, 6]], 7]
flat = list(flatten(nested))
print(flat) # [1, 2, 3, 4, 5, 6, 7]
python code snippet end
Tree Traversal
python code snippet start
class TreeNode:
def __init__(self, value, children=None):
self.value = value
self.children = children or []
def traverse_tree(node):
"""Depth-first traversal using yield from."""
yield node.value
for child in node.children:
yield from traverse_tree(child) # Delegate to subtree
# Build a tree
root = TreeNode('A', [
TreeNode('B', [TreeNode('D'), TreeNode('E')]),
TreeNode('C', [TreeNode('F')])
])
# Traverse
for value in traverse_tree(root):
print(value) # A, B, D, E, C, F
python code snippet end
Exception Handling
python code snippet start
def might_fail():
try:
yield 1
yield 2
raise ValueError("Something went wrong!")
yield 3 # Never reached
except ValueError as e:
yield f"Caught: {e}"
return "recovered"
def exception_delegator():
yield "starting"
try:
result = yield from might_fail()
yield f"result: {result}"
except Exception as e:
yield f"outer caught: {e}"
# Usage
for value in exception_delegator():
print(value)
# Output: starting, 1, 2, Caught: Something went wrong!, result: recovered
python code snippet end
Coroutine Simulation
python code snippet start
def coroutine_consumer():
"""Simulates a coroutine that processes values."""
total = 0
count = 0
while True:
value = yield
if value is None:
break
total += value
count += 1
yield f"processed {value}, running total: {total}"
return f"final average: {total/count if count else 0}"
def coroutine_delegator():
print("Setting up consumer...")
result = yield from coroutine_consumer()
print(f"Consumer finished with: {result}")
# Usage
coro = coroutine_delegator()
next(coro) # Prime the coroutine
print(coro.send(10)) # processed 10, running total: 10
print(coro.send(20)) # processed 20, running total: 30
print(coro.send(30)) # processed 30, running total: 60
try:
coro.send(None) # Signal completion
except StopIteration:
pass
python code snippet end
Generator Composition
python code snippet start
def numbers(start, end):
"""Generate numbers in range."""
for i in range(start, end):
yield i
def squares(iterable):
"""Square each number."""
for num in iterable:
yield num ** 2
def even_only(iterable):
"""Filter even numbers."""
for num in iterable:
if num % 2 == 0:
yield num
def pipeline():
"""Compose generators using yield from."""
yield from even_only(squares(numbers(1, 10)))
# Usage
result = list(pipeline())
print(result) # [4, 16, 36, 64] (squares of even numbers)
python code snippet end
Real-World Example: File Processing
python code snippet start
import os
def process_file(filepath):
"""Process a single file."""
try:
with open(filepath, 'r') as f:
line_count = sum(1 for _ in f)
yield f"Processed {filepath}: {line_count} lines"
return line_count
except Exception as e:
yield f"Error processing {filepath}: {e}"
return 0
def process_directory(directory):
"""Process all files in directory."""
total_lines = 0
file_count = 0
for filename in os.listdir(directory):
filepath = os.path.join(directory, filename)
if os.path.isfile(filepath) and filename.endswith('.py'):
file_count += 1
lines = yield from process_file(filepath)
total_lines += lines
return f"Processed {file_count} files, {total_lines} total lines"
def file_processor(directory):
"""Main processor with summary."""
yield "Starting file processing..."
summary = yield from process_directory(directory)
yield f"Summary: {summary}"
# Usage (if directory exists)
# for message in file_processor('.'):
# print(message)
python code snippet end
Connection to async/await
python code snippet start
# yield from was the foundation for async/await
def old_style_coroutine():
result = yield from some_async_operation()
return result
# Modern equivalent
async def new_style_coroutine():
result = await some_async_operation()
return result
# The semantics are nearly identical!
python code snippet end
yield from
transformed how we compose generators and laid the groundwork for Python’s async programming model - it’s the unsung hero behind modern asynchronous Python!
Reference: PEP 380 – Syntax for Delegating to a Subgenerator