understanding partial() with async/futures

the basic idea

partial "freezes" args in a fn so you don't have to pass them every time

from functools import partial

def add(a, b, c):
    return a + b + c

add_5_and_10 = partial(add, 5, 10)
add_5_and_10(3)  # returns 18 (same as add(5, 10, 3))

the problem: fetching from multiple APIs

imagine you need to fetch user data from 3 different API endpoints at the same time

here's the messy way:

import asyncio
from functools import partial
from concurrent.futures import ThreadPoolExecutor

def fetch_data(user_id, api_endpoint, timeout=30, retry=3, api_key="secret"):
    return f"Data from {api_endpoint} for user {user_id}"

async def get_user_data_messy(user_id):
    executor = ThreadPoolExecutor()
    loop = asyncio.get_event_loop()
    
    # repetition
    future1 = loop.run_in_executor(
        executor,
        lambda: fetch_data(user_id, "profile", 30, 3, "secret")
    )
    future2 = loop.run_in_executor(
        executor,
        lambda: fetch_data(user_id, "orders", 30, 3, "secret")
    )
    future3 = loop.run_in_executor(
        executor,
        lambda: fetch_data(user_id, "reviews", 30, 3, "secret")
    )
    
    results = await asyncio.gather(future1, future2, future3)
    return results

the clean way with partial:

async def get_user_data_clean(user_id):
    executor = ThreadPoolExecutor()
    loop = asyncio.get_event_loop()
    
    # common way
    fetcher = partial(
        fetch_data,
        user_id=user_id,
        timeout=30,
        retry=3,
        api_key="secret"
    )
    
    endpoints = ["profile", "orders", "reviews"]
    
    futures = [
        loop.run_in_executor(executor, partial(fetcher, api_endpoint=ep))
        for ep in endpoints
    ]
    
    results = await asyncio.gather(*futures)
    return results

why the double partial

loop.run_in_executor(executor, partial(fetcher, api_endpoint=ep))

here's what's actually happening:

# first partial: lock in the common stuff
fetcher = partial(fetch_data, user_id=user_id, timeout=30, retry=3, api_key="secret")

# second partial: add the specific endpoint
profile_fetcher = partial(fetcher, api_endpoint="profile")

# now profile_fetcher() is a zero-argument callable
# calling it is the same as: fetch_data(user_id, "profile", 30, 3, "secret")

seeing it run

import time

def fetch_data(user_id, api_endpoint, timeout=30, retry=3, api_key="secret"):
    time.sleep(1)  # pretend this is an API call
    return f"Data from {api_endpoint} for user {user_id}"

async def main():
    start = time.time()
    results = await get_user_data_clean(12345)
    print(f"completed in {time.time() - start:.2f}s")
    print(results)
    # completed in 1.01s  (all 3 APIs ran at the same time)
    # ['Data from profile for user 12345', 
    #  'Data from orders for user 12345',
    #  'Data from reviews for user 12345']

asyncio.run(main())