Python Concurrency Techniques: Threads, Processes, Async, and Pools

Concurrent programming in Python allows multiple tasks to run simultaneously, improving performance and responsiveness. The following examples demonstrate common techniques.

1. Multithreading

import threading

def task():
    # 执行耗时任务
    pass

thread1 = threading.Thread(target=task)
thread2 = threading.Thread(target=task)
thread1.start()
thread2.start()
thread1.join()
thread2.join()

2. Multiprocessing

import multiprocessing

def task():
    # 执行耗时任务
    pass

process1 = multiprocessing.Process(target=task)
process2 = multiprocessing.Process(target=task)
process1.start()
process2.start()
process1.join()
process2.join()

3. ThreadPoolExecutor

import concurrent.futures

def task():
    # 执行任务
    pass

with concurrent.futures.ThreadPoolExecutor() as executor:
    tasks = [executor.submit(task) for _ in range(10)]
for future in concurrent.futures.as_completed(tasks):
    result = future.result()
    # 处理结果

4. ProcessPoolExecutor

import concurrent.futures

def task():
    # 执行任务
    pass

with concurrent.futures.ProcessPoolExecutor() as executor:
    tasks = [executor.submit(task) for _ in range(10)]
for future in concurrent.futures.as_completed(tasks):
    result = future.result()
    # 处理结果

5. Asyncio with gather

import asyncio

async def task():
    # 执行任务
    pass

async def main():
    tasks = [task() for _ in range(10)]
    await asyncio.gather(*tasks)

asyncio.run(main())

6. Asyncio with wait

import asyncio

async def task():
    # 执行任务
    pass

async def main():
    coroutines = [task() for _ in range(10)]
    await asyncio.wait(coroutines)

asyncio.run(main())

7. Thread‑based producer‑consumer model

import threading
import queue

def producer(queue):
    # 生产数据
    pass

def consumer(queue):
    while True:
        data = queue.get()
        if data is None:
            break
        # 消费数据

queue = queue.Queue()
thread1 = threading.Thread(target=producer, args=(queue,))
thread2 = threading.Thread(target=consumer, args=(queue,))
thread3 = threading.Thread(target=consumer, args=(queue,))
thread1.start()
thread2.start()
thread3.start()
thread1.join()
thread2.join()
thread3.join()

8. Process‑based producer‑consumer model

import multiprocessing
import queue

def producer(queue):
    # 生产数据
    pass

def consumer(queue):
    while True:
        data = queue.get()
        if data is None:
            break
        # 消费数据

queue = multiprocessing.Queue()
process1 = multiprocessing.Process(target=producer, args=(queue,))
process2 = multiprocessing.Process(target=consumer, args=(queue,))
process3 = multiprocessing.Process(target=consumer, args=(queue,))
process1.start()
process2.start()
process3.start()
process1.join()
process2.join()
process3.join()

9. Async event loop

import asyncio

async def task():
    # 执行任务
    pass

async def main():
    while True:
        await task()
        # 处理其他事务

asyncio.run(main())

10. Multithreaded network requests

import threading
import requests

def make_request(url):
    response = requests.get(url)
    # 处理响应

urls = ["https://example.com", "https://google.com", "https://github.com"]
threads = [threading.Thread(target=make_request, args=(url,)) for url in urls]
for thread in threads:
    thread.start()
for thread in threads:
    thread.join()

These examples illustrate how Python's concurrency mechanisms can be applied to improve performance in time‑consuming tasks, concurrent requests, and producer‑consumer scenarios.