Source code for scrapy.utils.response

"""
This module provides some useful functions for working with
scrapy.http.Response objects
"""

from __future__ import annotations

import mimetypes
import os
import re
import tempfile
import warnings
import webbrowser
from pathlib import Path
from typing import TYPE_CHECKING, Any

import frostwork
from twisted.web import http

from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.headers import Headers
from scrapy.http.response.body_file import BodyFileResponse
from scrapy.http.response.text import MetaRefresh
from scrapy.utils.misc import load_object

if TYPE_CHECKING:
    from collections.abc import Callable, Iterable

    from scrapy.http import Response, TextResponse


def _body_size(response: Response) -> int:
    if isinstance(response, BodyFileResponse):
        return Path(response.body_file).stat().st_size
    return len(response.body)


def get_base_url(response: TextResponse) -> str:
    warnings.warn(
        "scrapy.utils.response.get_base_url() is deprecated, use "
        "TextResponse.base_url instead.",
        ScrapyDeprecationWarning,
        stacklevel=2,
    )
    return response.base_url


_IGNORABLE_TAGS = {"script", "noscript", "template"}


def _get_meta_refresh(response: TextResponse, *, scripting: bool) -> MetaRefresh | None:
    if scripting:
        return response.meta_refresh
    meta_refresh = frostwork.navigation(
        response.body,
        response.url,
        response.encoding,
        base_url=False,
        scripting=False,
    ).meta_refresh
    return None if meta_refresh is None else MetaRefresh(*meta_refresh)


def get_meta_refresh(
    response: TextResponse,
    ignore_tags: Iterable[str] = ("script", "noscript"),
) -> tuple[None, None] | tuple[float, str]:
    warnings.warn(
        "scrapy.utils.response.get_meta_refresh() is deprecated, use "
        "TextResponse.meta_refresh instead.",
        ScrapyDeprecationWarning,
        stacklevel=2,
    )
    ignore_tags = set(ignore_tags)
    if not ignore_tags <= _IGNORABLE_TAGS:
        raise ValueError(
            f"Unsupported ignore_tags: {sorted(ignore_tags - _IGNORABLE_TAGS)}"
        )
    meta_refresh = _get_meta_refresh(response, scripting="noscript" in ignore_tags)
    if meta_refresh is None:
        return None, None
    return meta_refresh.delay, meta_refresh.url


[docs] def response_from_dict(d: dict[str, Any]) -> Response: """Return a response built from the *d* dict, as returned by :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`. .. versionadded:: 2.19.0 If *d* does not indicate a response class, e.g. because it comes from a plain :class:`~scrapy.http.Response` object or predates :meth:`~scrapy.http.Response.to_dict`, the class is guessed with :attr:`~scrapy.responsetypes.responsetypes`. """ # Imported here to avoid a circular import. from scrapy.responsetypes import responsetypes # noqa: PLC0415 d = {**d, "headers": Headers(d.get("headers") or {})} response_cls: type[Response] = ( load_object(d["_class"]) if "_class" in d else responsetypes.from_args( headers=d["headers"], url=d["url"], body=d.get("body") ) ) return response_cls.from_dict(d)
def response_status_message(status: bytes | float | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, b"Unknown Status").decode() return f"{status_int} {message}" _DOCTYPE_RE = re.compile(rb"\s*<!doctype[^<>]*>", re.IGNORECASE)
[docs] def open_in_browser( response: Response, _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: """Open *response* in a local web browser, adjusting the `base tag`_ for external links to work, e.g. so that images and styles are displayed. .. _base tag: https://www.w3schools.com/tags/tag_base.asp For example: .. code-block:: python from scrapy.utils.response import open_in_browser def parse_details(self, response): if "item name" not in response.text: open_in_browser(response) On the Windows Subsystem for Linux, set the ``BROWSER`` environment variable to `wslview <https://github.com/wslutilities/wslu>`_ to open the response in a Windows browser, which cannot read Linux paths otherwise. """ # circular imports from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415 body = response.body if isinstance(response, HtmlResponse): # Web browsers move a base tag that precedes the head element into it, # so the head element does not need to be found, which is not always # possible. The base tag must come after the doctype declaration, if # any, to keep the browser out of quirks mode. It takes precedence over # any base tag of the response, and matches it when there is one. doctype = _DOCTYPE_RE.match(body) index = doctype.end() if doctype else 0 base_tag = f'<base href="{response.base_url}">'.encode() body = body[:index] + base_tag + body[index:] ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" else: content_type = (response.headers.get(b"Content-Type") or b"").decode("latin-1") mimetype = content_type.split(";")[0].strip().lower() ext = mimetypes.guess_extension(mimetype) or "" if not ext: raise TypeError(f"Unsupported response type: {response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) return _openfunc(f"file://{fname}")