Python: Threading and ThreadPoolExecutor Examples

Table of Contents

• Basic Usage
• ThreadPoolExecutor
  • Basic Example
  • Using map() for Simple Parallelism
  • Error Handling with ThreadPoolExecutor

Basic Usage

from threading import Thread
from time import sleep, time

def slowFunction(time=5):
    """Create example of a slow function"""
    sleep(time)

t0 = time()
threads = []

for i in range(10):
    t = Thread(target=slowFunction, args=(1,))
    t.start()
    threads.append(t)
for t in threads:
    t.join(timeout=15)
print(f"Finished in: {time()-t0}")

Finished in: 1.0109996795654297

ThreadPoolExecutor

ThreadPoolExecutor provides a higher-level interface for managing thread pools, making it easier to parallelize tasks without manually managing threads.

Basic Example

from concurrent.futures import ThreadPoolExecutor, as_completed
from time import sleep, time

def process_task(task_id, duration):
    """Simulate a task that takes some time"""
    sleep(duration)
    return f"Task {task_id} completed in {duration}s"

t0 = time()

# Create a pool with 5 worker threads
with ThreadPoolExecutor(max_workers=5) as executor:
    # Submit tasks to the executor
    futures = [executor.submit(process_task, i, 1) for i in range(10)]

    # Wait for all tasks to complete and collect results
    for future in as_completed(futures):
        print(future.result())

print(f"Finished in: {time()-t0:.2f}s")
# Output: Finished in: ~2.00s (10 tasks / 5 workers)

Using map() for Simple Parallelism

from concurrent.futures import ThreadPoolExecutor
from time import sleep, time

def square(n):
    """Calculate square of a number (with simulated delay)"""
    sleep(0.1)
    return n * n

t0 = time()

with ThreadPoolExecutor(max_workers=4) as executor:
    numbers = range(20)
    results = list(executor.map(square, numbers))

print(f"Results: {results}")
print(f"Finished in: {time()-t0:.2f}s")
# Output: Finished in: ~0.50s (20 tasks / 4 workers)

Error Handling with ThreadPoolExecutor

from concurrent.futures import ThreadPoolExecutor, as_completed

def risky_operation(value):
    """Operation that might fail"""
    if value % 3 == 0:
        raise ValueError(f"Cannot process {value}")
    return value * 2

with ThreadPoolExecutor(max_workers=3) as executor:
    futures = {executor.submit(risky_operation, i): i for i in range(10)}

    for future in as_completed(futures):
        original_value = futures[future]
        try:
            result = future.result()
            print(f"Success: {original_value} -> {result}")
        except Exception as exc:
            print(f"Error processing {original_value}: {exc}")