Source code for scrapy.utils.defer

"""
Helper functions for dealing with Twisted deferreds
"""

from __future__ import annotations

import asyncio
import contextvars
import inspect
import logging
import sys
import warnings
from asyncio import Future
from collections import deque
from collections.abc import Awaitable, Coroutine, Iterable, Iterator
from functools import wraps
from typing import (
    TYPE_CHECKING,
    Any,
    Concatenate,
    Generic,
    ParamSpec,
    Protocol,
    TypeVar,
    cast,
    overload,
)

from twisted.internet.defer import Deferred, DeferredList, FirstError, fail, succeed
from twisted.internet.task import Cooperator
from twisted.python import failure

from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import Request
from scrapy.utils.asyncio import (
    _get_running_or_installed_loop,
    is_asyncio_available,
    sleep,
)
from scrapy.utils.python import global_object_name

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Callable

    from twisted.python.failure import Failure

    # typing.TypeIs requires Python 3.13
    from typing_extensions import TypeIs

    class FailureWithRequest(Protocol):
        """A :class:`~twisted.python.failure.Failure` that has had a
        *request* attribute attached, identifying the request whose
        processing raised it.
        """

        request: Request


logger = logging.getLogger(__name__)

_T = TypeVar("_T")
_T2 = TypeVar("_T2")
_P = ParamSpec("_P")


_DEFER_DELAY = 0.1


def defer_fail(_failure: Failure) -> Deferred[Any]:  # pragma: no cover
    """Same as twisted.internet.defer.fail but delay calling errback until
    next reactor loop

    It delays by 100ms so reactor has a chance to go through readers and writers
    before attending pending delayed calls, so do not set delay to zero.
    """
    warnings.warn(
        "scrapy.utils.defer.defer_fail() is deprecated, use"
        " twisted.internet.defer.fail(), plus an explicit sleep if needed.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )

    from twisted.internet import reactor

    d: Deferred[Any] = Deferred()
    reactor.callLater(_DEFER_DELAY, d.errback, _failure)
    return d


def defer_succeed(result: _T) -> Deferred[_T]:  # pragma: no cover
    """Same as twisted.internet.defer.succeed but delay calling callback until
    next reactor loop

    It delays by 100ms so reactor has a chance to go through readers and writers
    before attending pending delayed calls, so do not set delay to zero.
    """
    warnings.warn(
        "scrapy.utils.defer.defer_succeed() is deprecated, use"
        " twisted.internet.defer.succeed(), plus an explicit sleep if needed.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )

    from twisted.internet import reactor

    d: Deferred[_T] = Deferred()
    reactor.callLater(_DEFER_DELAY, d.callback, result)
    return d


async def _process_pending_io() -> None:
    """Yield control until the event loop has gone through its readers and writers.

    Yielding twice is what makes that guarantee: the first yield can resume
    before the callbacks of the file descriptors that the poll found ready, and
    only the second one is certain to resume after them.
    """
    await sleep(0)
    await sleep(0)


async def _process_pending_io_before_callback() -> None:
    """Same as :func:`_process_pending_io`, for use right before a spider
    callback, where a real delay is needed on Windows.

    Console control events, which is what Ctrl-C and Ctrl-Break send, are
    delivered on a separate thread there, and their Python-level handler only
    runs once the main thread reaches a bytecode boundary, which it cannot do
    while parked in the blocking wait of the reactor or the event loop, as
    nothing interrupts that wait. So a signal sent during a slow callback is
    only handled at the next wake-up, e.g. the engine heartbeat 5 seconds
    later, which is late enough to miss a shutdown entirely. Yielding for a
    moment keeps a timed call pending, which bounds that wait, and by the time
    it elapses a shutdown is under way and keeps the loop busy on its own.

    A signal wake-up socket on Windows would remove the need for this:
    https://github.com/python/cpython/issues/67246
    https://github.com/python/cpython/issues/86849
    """
    if sys.platform == "win32":
        await sleep(_DEFER_DELAY)
        return
    await _process_pending_io()


def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover
    warnings.warn(
        "scrapy.utils.defer.defer_result() is deprecated, use"
        " twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
        " plus an explicit sleep if needed, or explicit reactor.callLater().",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )

    if isinstance(result, Deferred):
        return result

    from twisted.internet import reactor

    d: Deferred[Any] = Deferred()
    if isinstance(result, failure.Failure):
        reactor.callLater(_DEFER_DELAY, d.errback, result)
    else:
        reactor.callLater(_DEFER_DELAY, d.callback, result)
    return d


@overload
def mustbe_deferred(
    f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs
) -> Deferred[_T]: ...


@overload
def mustbe_deferred(
    f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs
) -> Deferred[_T]: ...


def mustbe_deferred(
    f: Callable[_P, Deferred[_T] | _T],
    *args: _P.args,
    **kw: _P.kwargs,
) -> Deferred[_T]:  # pragma: no cover
    """Same as twisted.internet.defer.maybeDeferred, but delay calling
    callback/errback to next reactor loop
    """
    warnings.warn(
        "scrapy.utils.defer.mustbe_deferred() is deprecated, use"
        " twisted.internet.defer.maybeDeferred(), with an explicit sleep if needed.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )
    result: _T | Deferred[_T] | Failure
    try:
        result = f(*args, **kw)
    except Exception:
        result = failure.Failure()
    return defer_result(result)


class _ParallelTasks:
    """Consumes *work* with no more than *count* concurrent cooperative tasks.

    Tasks are started on demand: :meth:`start` must be called every time a new
    item becomes available, so that a *count* much larger than the amount of
    work does not cost anything.

    :attr:`finished` fires once *work* is exhausted and every task is done.
    """

    def __init__(self, work: Iterator[Any], count: int):
        self._coop = Cooperator()
        self._work = work
        self._count = count
        self._running = 0
        self.finished: Deferred[None] = Deferred()

    def start(self) -> None:
        if self._running >= self._count:
            return
        self._running += 1
        self._coop.coiterate(self._work).addBoth(self._task_done)

    def _task_done(self, result: Any) -> None:
        self._running -= 1
        if isinstance(result, failure.Failure):
            # circular import
            from scrapy.utils.log import failure_to_exc_info  # noqa: PLC0415

            logger.error(
                "Error processing an item",
                exc_info=failure_to_exc_info(result),
            )
            # A failed task stops consuming work, which may not be exhausted
            # yet, so another one takes over.
            self.start()
            return
        # Only a running task can start another one, so once none is left no
        # more work can come.
        if not self._running:
            # The work usually references self.start, so keeping it would
            # leave a reference cycle that keeps the work arguments, e.g. a
            # response, alive until the next garbage collection.
            del self._work
            self.finished.callback(None)


def parallel(
    iterable: Iterable[_T],
    count: int,
    callable: Callable[Concatenate[_T, _P], _T2],  # noqa: A002
    *args: _P.args,
    **named: _P.kwargs,
) -> Deferred[None]:  # pragma: no cover
    """Execute a callable over the objects in the given iterable, in parallel,
    using no more than ``count`` concurrent calls.
    """

    def work() -> Iterator[_T2]:
        for elem in iterable:
            tasks.start()
            yield callable(elem, *args, **named)

    warnings.warn(
        "scrapy.utils.defer.parallel() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )

    tasks = _ParallelTasks(work(), count)
    tasks.start()
    return tasks.finished


class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_P, _T]):
    """A class that wraps an async iterable into a normal iterator suitable
    for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
    it calls the callable directly in the callback, instead of providing a more
    generic interface.

    On the outside, this class behaves as an iterator that yields Deferreds.
    Each Deferred is fired with the result of the callable which was called on
    the next result from aiterator. It raises StopIteration when aiterator is
    exhausted, as expected.

    Cooperator calls __next__() multiple times and waits on the Deferreds
    returned from it. As async generators (since Python 3.8) don't support
    awaiting on __anext__() several times in parallel, we need to serialize
    this. It's done by storing the Deferreds returned from __next__() and
    firing the oldest one when a result from __anext__() is available.

    The workflow:
    1. When __next__() is called for the first time, it creates a Deferred, stores it
    in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
    for self.aiterator.__anext__() and puts it into self.anext_deferred.
    2. If __next__() is called again before self.anext_deferred fires, more Deferreds
    are added to self.waiting_deferreds.
    3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
    clear self.anext_deferred.
    3.1. _callback() calls the callable passing the result value that it takes, pops a
    Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
    chains those Deferreds so that the waiting Deferred will fire when the result
    Deferred does, otherwise it fires it directly. This causes one awaiting task to
    receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
    called and self.anext_deferred is populated.
    3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
    self.aiterator is exhausted and so it sets self.finished and fires all
    self.waiting_deferreds. Other exceptions are propagated.
    4. If __next__() is called after __anext__() was handled, then if self.finished is
    True, it raises StopIteration, otherwise it acts like in step 2, but if
    self.anext_deferred is now empty is also populates it with a new __anext__().

    Note that CooperativeTask ignores the value returned from the Deferred that it waits
    for, so we fire them with None when needed.

    It may be possible to write an async iterator-aware replacement for
    Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
    goal.
    """

    def __init__(
        self,
        aiterable: AsyncIterator[_T],
        callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None],
        *callable_args: _P.args,
        **callable_kwargs: _P.kwargs,
    ):
        self.aiterator: AsyncIterator[_T] = aiter(aiterable)
        self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable_
        self.callable_args: tuple[Any, ...] = callable_args
        self.callable_kwargs: dict[str, Any] = callable_kwargs
        self.finished: bool = False
        self.waiting_deferreds: deque[Deferred[Any]] = deque()
        self.anext_deferred: Deferred[_T] | None = None
        # Cooperator ticks are scheduled through the reactor, which does not
        # preserve contextvars, so both anext() and the callable run in the
        # context captured here.
        self._context: contextvars.Context = contextvars.copy_context()
        # Called whenever aiterator produces a value, so that parallel_async()
        # can start a task for it.
        self.on_value: Callable[[], None] = lambda: None

    def _callback(self, result: _T) -> None:
        # This gets called when the result from aiterator.__anext__() is available.
        # It calls the callable on it and sends the result to the oldest waiting Deferred
        # (by chaining if the result is a Deferred too or by firing if not).
        self.anext_deferred = None
        self.on_value()
        callable_result = self._context.run(
            self.callable, result, *self.callable_args, **self.callable_kwargs
        )
        d = self.waiting_deferreds.popleft()
        if isinstance(callable_result, Deferred):
            callable_result.chainDeferred(d)
        else:
            d.callback(None)
        if self.waiting_deferreds:
            self._call_anext()

    def _errback(self, failure: Failure) -> None:
        # This gets called on any exceptions in aiterator.__anext__().
        # It handles StopAsyncIteration by stopping the iteration and reraises all others.
        self.anext_deferred = None
        failure.trap(StopAsyncIteration)
        self.finished = True
        for d in self.waiting_deferreds:
            d.callback(None)

    def _call_anext(self) -> None:
        # This starts waiting for the next result from aiterator.
        # If aiterator is exhausted, _errback will be called.
        self.anext_deferred = self._context.run(
            deferred_from_coro, anext(self.aiterator)
        )
        self.anext_deferred.addCallbacks(self._callback, self._errback)

    def __next__(self) -> Deferred[Any]:
        # This puts a new Deferred into self.waiting_deferreds and returns it.
        # It also calls __anext__() if needed.
        if self.finished:
            raise StopIteration
        d: Deferred[Any] = Deferred()
        self.waiting_deferreds.append(d)
        if not self.anext_deferred:
            self._call_anext()
        return d


def _parallel_async(
    async_iterable: AsyncIterator[_T],
    count: int,
    callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None],
    *args: _P.args,
    **named: _P.kwargs,
) -> Deferred[None]:
    """Like ``parallel`` but for async iterators"""
    work = _AsyncCooperatorAdapter(async_iterable, callable_, *args, **named)
    tasks = _ParallelTasks(work, count)
    work.on_value = tasks.start
    tasks.start()
    return tasks.finished


def parallel_async(
    async_iterable: AsyncIterator[_T],
    count: int,
    callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],  # noqa: A002
    *args: _P.args,
    **named: _P.kwargs,
) -> Deferred[None]:  # pragma: no cover
    warnings.warn(
        "scrapy.utils.defer.parallel_async() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )
    return _parallel_async(async_iterable, count, callable, *args, **named)


def process_chain(
    callbacks: Iterable[Callable[Concatenate[_T, _P], _T]],
    input: _T,  # noqa: A002
    *a: _P.args,
    **kw: _P.kwargs,
) -> Deferred[_T]:  # pragma: no cover
    """Return a Deferred built by chaining the given callbacks"""
    warnings.warn(
        "process_chain() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )
    d: Deferred[_T] = Deferred()
    for x in callbacks:
        d.addCallback(x, *a, **kw)
    d.callback(input)
    return d


def process_parallel(
    callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
    input: _T,  # noqa: A002
    *a: _P.args,
    **kw: _P.kwargs,
) -> Deferred[list[_T2]]:  # pragma: no cover
    """Return a Deferred with the output of all successful calls to the given
    callbacks
    """
    warnings.warn(
        "process_parallel() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )
    dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks]
    d: Deferred[list[tuple[bool, _T2]]] = DeferredList(
        dfds, fireOnOneErrback=True, consumeErrors=True
    )
    d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r])

    def eb(failure: Failure) -> Failure:
        assert isinstance(failure.value, FirstError)
        return failure.value.subFailure

    d2.addErrback(eb)
    return d2


def iter_errback(
    iterable: Iterable[_T],
    errback: Callable[Concatenate[Failure, _P], Any],
    *a: _P.args,
    **kw: _P.kwargs,
) -> Iterable[_T]:  # pragma: no cover
    """Wrap an iterable calling an errback if an error is caught while
    iterating it.
    """
    warnings.warn(
        "scrapy.utils.defer.iter_errback() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )
    it = iter(iterable)
    while True:
        try:
            yield next(it)
        except StopIteration:
            break
        except Exception:
            errback(failure.Failure(), *a, **kw)


async def _aiter_errback(
    aiterable: AsyncIterator[_T],
    errback: Callable[Concatenate[Exception, _P], Any],
    *a: _P.args,
    **kw: _P.kwargs,
) -> AsyncIterator[_T]:
    """Wrap an async iterable calling *errback* with the exception if one is
    raised while iterating it."""
    it = aiter(aiterable)
    while True:
        try:
            yield await anext(it)
        except StopAsyncIteration:
            break
        except Exception as exc:
            errback(exc, *a, **kw)


def _is_awaitable(o: Any) -> TypeIs[Awaitable[Any]]:
    """Return ``True`` if *o* is meant to be awaited.

    Request objects support ``await`` to send them, but one returned by a
    callable is a value to process, not something to await.
    """
    return inspect.isawaitable(o) and not isinstance(o, Request)


async def aiter_errback(
    aiterable: AsyncIterator[_T],
    errback: Callable[Concatenate[Failure, _P], Any],
    *a: _P.args,
    **kw: _P.kwargs,
) -> AsyncIterator[_T]:  # pragma: no cover
    """Wrap an async iterable calling an errback if an error is caught while
    iterating it."""
    warnings.warn(
        "scrapy.utils.defer.aiter_errback() is deprecated.",
        category=ScrapyDeprecationWarning,
        stacklevel=2,
    )

    def failure_errback(exc: Exception, /, *args: _P.args, **kwargs: _P.kwargs) -> Any:
        return errback(failure.Failure(exc), *args, **kwargs)

    async for item in _aiter_errback(aiterable, failure_errback, *a, **kw):
        yield item


@overload
def deferred_from_coro(o: Awaitable[_T]) -> Deferred[_T]: ...


@overload
def deferred_from_coro(o: _T2) -> _T2: ...


[docs] def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: """Convert a coroutine or other awaitable object into a Deferred, or return the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o if _is_awaitable(o): if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor return Deferred.fromFuture( asyncio.ensure_future(o, loop=_get_running_or_installed_loop()) ) return o
[docs] def deferred_f_from_coro_f( coro_f: Callable[_P, Awaitable[_T]], ) -> Callable[_P, Deferred[_T]]: """Convert a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. This is useful for callback chains, as callback functions are called with the previous callback result. """ @wraps(coro_f) def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[_T]: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f
def maybeDeferred_coro( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs ) -> Deferred[Any]: # pragma: no cover """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" warnings.warn( "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version.", ScrapyDeprecationWarning, stacklevel=2, ) return _maybeDeferred_coro(f, False, *args, **kw) def _maybeDeferred_coro( f: Callable[_P, Any], warn: bool, *args: _P.args, **kw: _P.kwargs ) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) except: # noqa: E722 return fail(failure.Failure(captureVars=Deferred.debug)) # when the deprecation period has ended we need to make sure the behavior # of the public maybeDeferred_coro() function isn't changed, or drop it in # the same release if isinstance(result, Deferred): if warn: warnings.warn( f"{global_object_name(f)} returned a Deferred, this is deprecated." f" Please refactor this function to return a coroutine.", ScrapyDeprecationWarning, stacklevel=2, ) return result if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) if isinstance(result, failure.Failure): # pragma: no cover if warn: warnings.warn( f"{global_object_name(f)} returned a Failure, this is deprecated." f" Please refactor this function to return a coroutine.", ScrapyDeprecationWarning, stacklevel=2, ) return fail(result) return succeed(result)
[docs] def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """Return an :class:`asyncio.Future` object that wraps *d*. This function requires an installed asyncio reactor or a running asyncio event loop, see :ref:`using-asyncio`. This function is needed because you cannot await on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy callables defined as coroutines <coroutine-support>`; you can only await on :class:`~asyncio.Future` objects. Wrapping ``Deferred`` objects into ``Future`` objects allows you to await on them: .. code-block:: python class MySpider(Spider): ... async def parse(self, response): deferred = some_dfd_helper() result = await deferred_to_future(deferred) .. versionchanged:: 2.14 This function no longer installs an asyncio loop if called before the Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised in this case. """ if not is_asyncio_available(): raise RuntimeError( "deferred_to_future() requires an installed asyncio reactor" " or a running asyncio event loop." ) return d.asFuture(_get_running_or_installed_loop())
[docs] def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: """Return *d* as an object that can be awaited from a :ref:`Scrapy callable defined as a coroutine <coroutine-support>`. This is similar to :func:`deferred_to_future`, but when a non-asyncio Twisted reactor is installed (which is deprecated), it returns *d* as is, because in that case it can be awaited directly. Use this function only if you need to support non-asyncio reactors. It will be deprecated in a later Scrapy version. """ if not is_asyncio_available(): return d return deferred_to_future(d)
def _schedule_coro(coro: Coroutine[Any, Any, Any]) -> None: """Schedule the coroutine as a task or a Deferred. This doesn't store the reference to the task/Deferred, so a better alternative is calling :func:`scrapy.utils.defer.deferred_from_coro`, keeping the result, and adding proper exception handling (e.g. errbacks) to it. """ if not is_asyncio_available(): Deferred.fromCoroutine(coro) return asyncio.create_task(coro) # noqa: RUF006 @overload def ensure_awaitable(o: Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]: ... @overload def ensure_awaitable(o: _T, _warn: str | None = None) -> Awaitable[_T]: ...
[docs] def ensure_awaitable(o: _T | Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]: """Convert any value to an awaitable object. For a :class:`~twisted.internet.defer.Deferred` object, use :func:`maybe_deferred_to_future` to wrap it into a suitable object. For an awaitable object of a different type, return it as is. For any other value, return a coroutine that completes with that value. .. versionadded:: 2.14 """ if isinstance(o, Deferred): if _warn: warnings.warn( f"{_warn} returned a Deferred, this is deprecated." f" Please refactor this function to return a coroutine.", ScrapyDeprecationWarning, stacklevel=2, ) return maybe_deferred_to_future(o) if _is_awaitable(o): return o async def coro() -> _T: return o return coro()
async def _maybe_coroutine( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs ) -> Any: """Call *f* and return its result, awaiting it first if it is awaitable. Raise :exc:`TypeError` if *f* returns a :class:`~twisted.internet.defer.Deferred`. """ result = f(*args, **kw) if isinstance(result, Deferred): raise TypeError( f"{global_object_name(f)} returned a Deferred, which is not supported." f" Please refactor this function to return a coroutine." ) if _is_awaitable(result): return await result return result