Skip to main content Brad's PyNotes

OS Module: Operating System Interface for File and Process Operations

TL;DR

The os module provides os.listdir(), os.makedirs(), os.environ, os.path operations, and process management functions for cross-platform system interactions and file operations.

Interesting!

The os module automatically adapts path separators for different operating systems - os.path.join() uses backslashes on Windows and forward slashes on Unix, making your code truly cross-platform without any changes.

File and Directory Operations

python code snippet start

import os

# Current working directory
current_dir = os.getcwd()
print(f"Current directory: {current_dir}")

# Change directory
os.chdir('/path/to/new/directory')

# List directory contents
files = os.listdir('.')
print(f"Files in current directory: {files}")

# List with full paths
for item in os.listdir('.'):
    full_path = os.path.join('.', item)
    if os.path.isfile(full_path):
        print(f"File: {item}")
    elif os.path.isdir(full_path):
        print(f"Directory: {item}")

python code snippet end

Creating and Removing Directories

python code snippet start

import os

# Create single directory
os.mkdir('new_folder')

# Create nested directories
os.makedirs('parent/child/grandchild', exist_ok=True)

# Remove empty directory
os.rmdir('empty_folder')

# Remove directory and all contents
import shutil
shutil.rmtree('folder_with_contents')

# Check if path exists
if os.path.exists('some_path'):
    print("Path exists")

if os.path.isdir('some_path'):
    print("It's a directory")
elif os.path.isfile('some_path'):
    print("It's a file")

python code snippet end

Path Manipulation with os.path

python code snippet start

import os

# Join paths (cross-platform)
file_path = os.path.join('home', 'user', 'documents', 'file.txt')
print(file_path)  # home/user/documents/file.txt (Unix) or home\user\documents\file.txt (Windows)

# Split path components
directory, filename = os.path.split('/home/user/document.txt')
print(f"Directory: {directory}")  # /home/user
print(f"Filename: {filename}")    # document.txt

# Split filename and extension
name, ext = os.path.splitext('document.txt')
print(f"Name: {name}")  # document
print(f"Extension: {ext}")  # .txt

# Get absolute path
abs_path = os.path.abspath('relative/path')
print(abs_path)

# Get directory name
dirname = os.path.dirname('/home/user/file.txt')
print(dirname)  # /home/user

# Get base name
basename = os.path.basename('/home/user/file.txt')
print(basename)  # file.txt

python code snippet end

File Information and Permissions

python code snippet start

import os
import stat
from datetime import datetime

# Get file statistics
file_stats = os.stat('myfile.txt')

print(f"Size: {file_stats.st_size} bytes")
print(f"Modified: {datetime.fromtimestamp(file_stats.st_mtime)}")
print(f"Created: {datetime.fromtimestamp(file_stats.st_ctime)}")

# Check file permissions
mode = file_stats.st_mode

if stat.S_ISDIR(mode):
    print("It's a directory")
elif stat.S_ISREG(mode):
    print("It's a regular file")

# Check specific permissions
if mode & stat.S_IRUSR:  # User read permission
    print("User can read")
if mode & stat.S_IWUSR:  # User write permission
    print("User can write")
if mode & stat.S_IXUSR:  # User execute permission
    print("User can execute")

# Change permissions
os.chmod('myfile.txt', stat.S_IRUSR | stat.S_IWUSR)  # Read and write for user only

python code snippet end

Environment Variables

python code snippet start

import os

# Access environment variables
home_dir = os.environ.get('HOME')  # Unix
user_profile = os.environ.get('USERPROFILE')  # Windows
path = os.environ.get('PATH')

print(f"Home directory: {home_dir}")

# Set environment variable
os.environ['MY_VAR'] = 'some_value'

# Get with default value
debug = os.environ.get('DEBUG', 'False')

# Check if variable exists
if 'HOME' in os.environ:
    print("HOME is set")

# Get all environment variables
for key, value in os.environ.items():
    print(f"{key}: {value}")

# Expand environment variables in paths
expanded = os.path.expandvars('$HOME/documents')  # Unix
expanded = os.path.expandvars('%USERPROFILE%\\documents')  # Windows

python code snippet end

Process Management

python code snippet start

import os
import sys

# Get current process ID
pid = os.getpid()
print(f"Current process ID: {pid}")

# Get parent process ID
ppid = os.getppid()
print(f"Parent process ID: {ppid}")

# Execute system commands
result = os.system('ls -la')  # Unix
result = os.system('dir')     # Windows

# Get command exit status
if result == 0:
    print("Command succeeded")
else:
    print("Command failed")

# Execute and capture output (use subprocess for modern Python)
import subprocess
result = subprocess.run(['ls', '-la'], capture_output=True, text=True)
print(result.stdout)

python code snippet end

Walking Directory Trees

python code snippet start

import os

# Walk through directory tree
for root, dirs, files in os.walk('/path/to/search'):
    print(f"Directory: {root}")
    
    # Process subdirectories
    for dir_name in dirs:
        full_dir_path = os.path.join(root, dir_name)
        print(f"  Subdirectory: {full_dir_path}")
    
    # Process files
    for file_name in files:
        full_file_path = os.path.join(root, file_name)
        file_size = os.path.getsize(full_file_path)
        print(f"  File: {full_file_path} ({file_size} bytes)")

# Find all Python files
def find_python_files(directory):
    python_files = []
    for root, dirs, files in os.walk(directory):
        for file in files:
            if file.endswith('.py'):
                python_files.append(os.path.join(root, file))
    return python_files

py_files = find_python_files('.')
print(f"Found {len(py_files)} Python files")

python code snippet end

Practical Examples

Directory Size Calculator

python code snippet start

import os

def get_directory_size(path):
    """Calculate total size of directory and all subdirectories"""
    total_size = 0
    
    for root, dirs, files in os.walk(path):
        for file in files:
            file_path = os.path.join(root, file)
            try:
                total_size += os.path.getsize(file_path)
            except (OSError, FileNotFoundError):
                # Handle broken symlinks or permission errors
                pass
    
    return total_size

def format_bytes(bytes):
    """Convert bytes to human readable format"""
    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
        if bytes < 1024.0:
            return f"{bytes:.2f} {unit}"
        bytes /= 1024.0
    return f"{bytes:.2f} PB"

# Usage
size = get_directory_size('.')
print(f"Directory size: {format_bytes(size)}")

python code snippet end

File Finder with Pattern Matching

python code snippet start

import os
import fnmatch

def find_files(directory, pattern):
    """Find all files matching pattern in directory tree"""
    matches = []
    
    for root, dirs, files in os.walk(directory):
        for filename in files:
            if fnmatch.fnmatch(filename, pattern):
                matches.append(os.path.join(root, filename))
    
    return matches

# Find all log files
log_files = find_files('/var/log', '*.log')
print(f"Found {len(log_files)} log files")

# Find all image files
image_patterns = ['*.jpg', '*.jpeg', '*.png', '*.gif']
for pattern in image_patterns:
    images = find_files('.', pattern)
    print(f"Found {len(images)} {pattern} files")

python code snippet end

Safe File Operations

python code snippet start

import os
import tempfile
import shutil

def safe_file_operation(source_path, operation_func):
    """Perform file operation with backup"""
    # Create backup
    backup_path = source_path + '.backup'
    
    try:
        # Copy original file
        shutil.copy2(source_path, backup_path)
        
        # Perform operation
        result = operation_func(source_path)
        
        # Remove backup if successful
        os.remove(backup_path)
        return result
        
    except Exception as e:
        # Restore from backup if operation failed
        if os.path.exists(backup_path):
            shutil.move(backup_path, source_path)
        raise e

# Example usage
def modify_file(filepath):
    with open(filepath, 'a') as f:
        f.write('\nModified content')

# safe_file_operation('important.txt', modify_file)

python code snippet end

Cross-Platform Path Handling

python code snippet start

import os

def ensure_path_exists(path):
    """Create directory path if it doesn't exist"""
    if not os.path.exists(path):
        os.makedirs(path, exist_ok=True)
        print(f"Created directory: {path}")

def get_config_dir():
    """Get platform-appropriate config directory"""
    if os.name == 'nt':  # Windows
        return os.path.join(os.environ['APPDATA'], 'MyApp')
    else:  # Unix-like
        return os.path.join(os.path.expanduser('~'), '.myapp')

def get_temp_file(prefix='temp_', suffix='.tmp'):
    """Get platform-appropriate temporary file"""
    temp_dir = tempfile.gettempdir()
    return os.path.join(temp_dir, prefix + str(os.getpid()) + suffix)

# Usage
config_dir = get_config_dir()
ensure_path_exists(config_dir)

temp_file = get_temp_file('data_', '.json')
print(f"Temp file: {temp_file}")

python code snippet end

Best Practices

python code snippet start

import os

# Use os.path.join() for cross-platform paths
# Good
path = os.path.join('home', 'user', 'file.txt')

# Bad (platform-specific)
path = 'home/user/file.txt'  # Won't work on Windows

# Check if path exists before operations
if os.path.exists(path):
    os.remove(path)

# Use exist_ok for makedirs
os.makedirs('path/to/directory', exist_ok=True)

# Use context managers for file operations
with open('file.txt', 'r') as f:
    content = f.read()

# Handle exceptions for file operations
try:
    os.remove('file.txt')
except FileNotFoundError:
    print("File already deleted")
except PermissionError:
    print("Permission denied")

# Use pathlib for modern Python (3.4+) - cleaner API
from pathlib import Path

# Modern approach
path = Path('home') / 'user' / 'file.txt'
if path.exists():
    print(f"File size: {path.stat().st_size}")

python code snippet end

Security Considerations

python code snippet start

import os

# Validate paths to prevent directory traversal
def safe_join(base_path, user_path):
    """Safely join paths to prevent directory traversal"""
    # Remove any leading slashes or relative components
    user_path = user_path.lstrip('/')
    user_path = os.path.normpath(user_path)
    
    # Join and resolve
    full_path = os.path.join(base_path, user_path)
    full_path = os.path.realpath(full_path)
    
    # Ensure the result is within base_path
    if not full_path.startswith(os.path.realpath(base_path)):
        raise ValueError("Path traversal detected")
    
    return full_path

# Example usage
try:
    safe_path = safe_join('/safe/directory', 'user/file.txt')  # OK
    safe_path = safe_join('/safe/directory', '../../../etc/passwd')  # Raises ValueError
except ValueError as e:
    print(f"Security error: {e}")

python code snippet end

The os module is fundamental for system programming in Python, providing the tools needed for file operations, process management, and cross-platform compatibility.

While os.path is still widely used, pathlib provides a modern alternative for file operations, and os module pairs well with sys module for system information .

Reference: Python OS Module Documentation