"""
Functions for dealing with markup text
"""
from __future__ import annotations
import functools
import re
from html.entities import name2codepoint
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
from w3lib._util import _attr_re, _scannable, iter_tag_attributes, to_unicode
from w3lib.url import safe_url_string
if TYPE_CHECKING:
from collections.abc import Iterable
# Character references are ASCII digits only; \d would also match the Unicode
# decimal digits, which int() accepts but no HTML parser decodes.
_ent_re = re.compile(
r"&((?P<named>[a-z0-9]+)|#(?P<dec>[0-9]+)|#x(?P<hex>[a-f0-9]+))(?P<semicolon>;?)",
re.IGNORECASE,
)
# The text of a tag after its name, up to the angle bracket that ends the tag.
# A quoted attribute value is consumed whole, so that an angle bracket in it
# does not end the tag. A quote opens a value only right after the "=" of an
# attribute; anywhere else in a tag, HTML parsers take it as part of a name or
# of an unquoted value, and the tag still ends at the first ">". The runs of
# other characters stop at every "=", and what follows one is a double-quoted
# value, a single-quoted value or neither, which gives a tag body a single
# parse and hence nothing to backtrack into.
_TAG_BODY = r"""[^<>=]*(?:(?:=\s*"[^"]*"|=\s*'[^']*'|=(?!\s*["']))[^<>=]*)*"""
# Only a tag named the way an HTML element is gets its body read that way.
# Pairing quotes across a "<" that opens no tag, e.g. the one in "i<n" inside a
# script, would take the tag to the far side of the next quote, and past every
# tag in between. A name is pinned to its full length, as below, so that the
# "<" of "n<arguments.length" is left to the plain reading.
_TAG_NAME = r"""[a-zA-Z][a-zA-Z0-9]*(?![^ <>/])"""
# Anything else, a markup declaration or a tag named otherwise, keeps the plain
# reading: an apostrophe in a comment is text, and pairing it with a later
# quote would swallow the markup in between. A tag with no "=" before its
# first ">" has no quoted value that could hold one, so both readings agree on
# it, and the plain one comes first because it is the cheaper of the two and
# the case of most tags.
_tag_re = re.compile(
rf"""<[a-zA-Z/!][^<>=]*>|</?{_TAG_NAME}{_TAG_BODY}>|<[a-zA-Z/!][^<>]*>""",
re.ASCII,
)
# Tag syntax is ASCII, and re.ASCII holds the scan patterns of this module to
# it: "\s" matches the whitespace that separates markup and not, say, U+3000,
# and case-insensitive matching pairs no "s" with "\u017f" nor "k" with
# "\u212a". It is also what makes a pattern and its byte counterpart match the
# same markup.
# Any tag, up to where it ends or the text does. A quoted value left open by
# the end of the text is taken to the next angle bracket, which is where the
# tag it belongs to can no longer reach.
_tag_extent_source = rf"""<[a-zA-Z!/]{_TAG_BODY}(?:=\s*["'][^"'<>]*)?>?"""
_tag_extent_re = re.compile(_tag_extent_source, re.ASCII)
_tag_extent_bytes_re = re.compile(_tag_extent_source.encode())
_base_re = re.compile("<base", re.IGNORECASE | re.ASCII)
_base_bytes_re = re.compile(rb"<base", re.IGNORECASE)
# Scan for <base> tags, consuming comments and <script>/<noscript> content
# (where a browser never parses tags) along the way. Ignorable regions come
# first in the alternation, so a <base> inside one is consumed before it can
# match; unterminated regions swallow the rest of the document, as a browser
# does. Their content is consumed in runs of characters that cannot start the
# closing delimiter, so that the large inline scripts of real pages cost a
# tight loop per run rather than a match attempt per character.
#
# The text of a <base> tag after its name is captured whole and its attributes
# are then read one by one, as for <meta>.
_base_scan_re = re.compile(
rf"""
<!--[^-]*(?:-(?!->)[^-]*)*(?:-->|$)
| <(?P<t>script|noscript)\b{_TAG_BODY}>[^<]*(?:<(?!/(?P=t)>)[^<]*)*(?:</(?P=t)>|$)
| <base\s(?P<attrs>{_TAG_BODY})
""",
re.IGNORECASE | re.DOTALL | re.VERBOSE | re.ASCII,
)
# The refresh payload: ``3; url=...``. The url= part is required.
# The interval is ASCII digits only, as in the HTML refresh steps.
_meta_refresh_content_re = re.compile(
r"\s*(?P<int>([0-9]*\.)?[0-9]+)\s*;\s*url=\s*(?P<url>.*)",
re.DOTALL | re.IGNORECASE | re.ASCII,
)
_CDATA_START = "<![CDATA["
_CDATA_END = "]]>"
_tags_re = re.compile(
rf"""
</?(?P<named>{_TAG_NAME})
# a tag named the way an HTML element is, whose quoted
{_TAG_BODY}> # attribute values are read whole
|
</? # opening angle bracket, optional slash for a closing tag
(?P<name>[^ <>/]+)
# tag name (captured): a run of non-space, non-bracket chars,
(?![^ <>/]) # pinned to its maximal length by this lookahead so it can't
# overlap the run below and backtrack quadratically on an
# unterminated tag (a "<" with a long run and no ">")
[^<>]* # the rest of the tag: attributes, whitespace, etc.
> # closing angle bracket
""",
re.IGNORECASE | re.VERBOSE | re.ASCII,
)
_meta_re = re.compile("<meta", re.IGNORECASE | re.ASCII)
def _meta_scan_source(ignore_tags: tuple[str, ...]) -> str:
# Scan for <meta> tags, consuming comments and the content of the ignored
# tags along the way. Ignorable regions come first in the alternation, so a
# <meta> inside one is consumed before it can match; unterminated regions
# swallow the rest of the document, as a browser does. Their content is
# consumed in runs of characters that cannot start the closing delimiter,
# so that the large inline scripts of real pages cost a tight loop per run
# rather than a match attempt per character. The end tag closes on the tag
# name followed by whitespace, "/" or ">", as browsers treat it.
#
# The <meta> body is matched without the closing angle bracket, which is
# not required: a tag left unterminated by the next "<" or by the end of
# the text is still parsed, as browsers do.
alternatives = [r"<!--[^-]*(?:-(?!->)[^-]*)*(?:-->|$)"]
if ignore_tags:
tags = "|".join(re.escape(tag) for tag in ignore_tags)
alternatives.append(
rf"<(?P<t>{tags})\b{_TAG_BODY}>[^<]*(?:<(?!/(?P=t)[\s/>])[^<]*)*"
rf"(?:</(?P=t){_TAG_BODY}>?|$)"
)
alternatives.append(rf"<meta\s(?P<attrs>{_TAG_BODY})")
return "|".join(alternatives)
@functools.lru_cache(maxsize=256)
def _build_meta_scan_pattern(ignore_tags: tuple[str, ...]) -> re.Pattern[str]:
return re.compile(_meta_scan_source(ignore_tags), re.IGNORECASE | re.ASCII)
@functools.lru_cache(maxsize=256)
def _build_meta_scan_bytes_pattern(ignore_tags: tuple[str, ...]) -> re.Pattern[bytes]:
return re.compile(_meta_scan_source(ignore_tags).encode(), re.IGNORECASE)
HTML5_WHITESPACE = " \t\n\r\x0c"
[docs]
def replace_entities(
text: str | bytes,
keep: Iterable[str] = (),
remove_illegal: bool = True,
encoding: str = "utf-8",
) -> str:
r"""Remove entities from the given `text` by converting them to their
corresponding unicode character.
`text` can be a unicode string or a byte string encoded in the given
`encoding` (which defaults to 'utf-8').
If `keep` is passed (with a list of entity names) those entities will
be kept (they won't be removed).
It supports both numeric entities (``&#nnnn;`` and ``&#hhhh;``)
and named entities (such as `` `` or ``>``).
If `remove_illegal` is ``True``, entities that can't be converted are removed.
If `remove_illegal` is ``False``, entities that can't be converted are kept "as
is". For more information see the tests.
Always returns a unicode string (with the entities removed).
>>> import w3lib.html
>>> w3lib.html.replace_entities(b'Price: £100')
'Price: \xa3100'
>>> print(w3lib.html.replace_entities(b'Price: £100'))
Price: £100
>>>
"""
keep = frozenset(keep)
def convert_entity(m: re.Match[str]) -> str:
groups = m.groupdict()
number = None
if groups.get("dec"):
number = int(groups["dec"], 10)
elif groups.get("hex"):
number = int(groups["hex"], 16)
else:
# guaranteed to be named
entity_name = groups["named"]
if entity_name.lower() in keep:
return m.group(0)
number = name2codepoint.get(entity_name) or name2codepoint.get(
entity_name.lower()
)
if number is not None:
# A null or surrogate reference is a parse error that the tokenizer
# resolves to U+FFFD; chr() would instead emit a NUL or a lone
# surrogate, which is not a Unicode scalar value and fails to
# encode. Out-of-range references keep the remove_illegal handling.
# https://html.spec.whatwg.org/commit-snapshots/3e7b72c44ce144cee7db859cd0647af6646b6793/#numeric-character-reference-end-state
if number == 0 or 0xD800 <= number <= 0xDFFF:
return "\ufffd"
# Numeric character references in the 80-9F range are typically
# interpreted by browsers as representing the characters mapped
# to bytes 80-9F in the Windows-1252 encoding. For more info
# see: http://en.wikipedia.org/wiki/Character_encodings_in_HTML
try:
if 0x80 <= number <= 0x9F:
return bytes((number,)).decode("cp1252")
return chr(number)
except (ValueError, OverflowError):
pass
return "" if remove_illegal and groups.get("semicolon") else m.group(0)
return _ent_re.sub(convert_entity, to_unicode(text, encoding))
def has_entities(text: str | bytes, encoding: str | None = None) -> bool:
return bool(_ent_re.search(to_unicode(text, encoding)))
_REMOVECOMMENTS_RE = re.compile("<!--.*?(?:-->|$)", re.DOTALL)
def _remove_tag(
m: re.Match[str], which_ones: set[str] | tuple[()], keep: set[str] | tuple[()]
) -> str:
tag = (m.group("named") or m.group("name")).lower()
should_remove = tag in which_ones if which_ones else tag not in keep
return "" if should_remove else m.group(0)
@functools.lru_cache(maxsize=256)
def _build_remove_tags_pattern(tags_tuple: tuple[str, ...]) -> re.Pattern[str]:
tags = "|".join(re.escape(tag) for tag in tags_tuple)
# The end tag closes on the tag name followed by whitespace, "/" or ">",
# so `</script >`, `</script\n>`, `</script/>` and `</script foo>` all end
# the element, as browsers treat them. Requiring a bare `</tag>` left the
# content (and anything it hides, e.g. a <meta refresh>) in place. The
# trailing run stays [^<>]* so it can't cross into the next tag and match
# super-linearly.
pattern = rf"""
<(?P<tag>{tags})(?=[\s/>])(?:{_TAG_BODY}>|[^<>]*>)
.*?</(?P=tag)(?=[\s/>])[^<>]*>
|
<(?P<tag2>{tags})(?=[\s/>])(?:{_TAG_BODY}/>|[^<>]*/>)
"""
return re.compile(pattern, re.IGNORECASE | re.DOTALL | re.VERBOSE | re.ASCII)
[docs]
def remove_tags_with_content(
text: str | bytes, which_ones: Iterable[str] = (), encoding: str | None = None
) -> str:
"""Remove tags and their content.
`which_ones` is a tuple of which tags to remove including their content.
If is empty, returns the string unmodified.
>>> import w3lib.html
>>> doc = '<div><p><b>This is a link:</b> <a href="http://www.example.com">example</a></p></div>'
>>> w3lib.html.remove_tags_with_content(doc, which_ones=('b',))
'<div><p> <a href="http://www.example.com">example</a></p></div>'
>>>
"""
utext = to_unicode(text, encoding)
if not which_ones:
return utext
pattern = _build_remove_tags_pattern(tuple(sorted(set(which_ones))))
return pattern.sub("", utext)
[docs]
def replace_escape_chars(
text: str | bytes,
which_ones: Iterable[str] = ("\n", "\t", "\r"),
replace_by: str | bytes = "",
encoding: str | None = None,
) -> str:
r"""Remove escape characters.
`which_ones` is a tuple of which escape characters we want to remove.
By default removes ``\n``, ``\t``, ``\r``.
`replace_by` is the string to replace the escape characters by.
It defaults to ``''``, meaning the escape characters are removed.
"""
utext = to_unicode(text, encoding)
for ec in which_ones:
utext = utext.replace(ec, to_unicode(replace_by, encoding))
return utext
[docs]
def unquote_markup(
text: str | bytes,
keep: Iterable[str] = (),
remove_illegal: bool = True,
encoding: str | None = None,
) -> str:
"""
This function receives markup as a text (always a unicode string or
a UTF-8 encoded string) and does the following:
1. removes entities (except the ones in `keep`) from any part of it
that is not inside a CDATA
2. searches for CDATAs and extracts their text (if any) without modifying it.
3. removes the found CDATAs
"""
keep = frozenset(keep)
utext = to_unicode(text, encoding)
ret = []
offset = 0
# Scan for CDATA sections linearly.
while True:
start = utext.find(_CDATA_START, offset)
if start == -1:
break
data_start = start + len(_CDATA_START)
end = utext.find(_CDATA_END, data_start)
if end == -1:
# Unterminated CDATA
break
if offset < start:
ret.append(
replace_entities(
utext[offset:start],
keep=keep,
remove_illegal=remove_illegal,
)
)
ret.append(utext[data_start:end])
offset = end + len(_CDATA_END)
if offset < len(utext):
ret.append(
replace_entities(
utext[offset:],
keep=keep,
remove_illegal=remove_illegal,
)
)
return "".join(ret)
def _cut(text: str | bytes, max_scan: int | None) -> tuple[str | bytes, bool]:
"""Return *text* cut to its first *max_scan* characters or bytes, extended
to the end of a tag the cut splits, and whether anything was left out."""
if max_scan is None:
return text, False
if max_scan < 0:
raise ValueError(f"max_scan must not be negative, got {max_scan!r}")
if max_scan >= len(text):
return text, False
is_str = isinstance(text, str)
extent: re.Pattern[Any] = _tag_extent_re if is_str else _tag_extent_bytes_re
lt: Any = "<" if is_str else b"<"
# Only a tag that reaches past the cut is split by it, and reading it to
# its end is what keeps it whole. Half a tag parses as a whole one, down to
# a truncated url, hence reading it whole or not at all. The search runs
# backwards from the cut because the tag that straddles it, if any, is the
# one that starts closest to it.
start = max_scan
while (start := text.rfind(lt, 0, start)) != -1:
tag = extent.match(text, start)
if tag is None:
continue
if tag.end() > max_scan:
return text[: tag.end()], tag.end() < len(text)
break
return text[:max_scan], True
[docs]
def get_base_url(
text: str | bytes,
baseurl: str | bytes = "",
encoding: str = "utf-8",
*,
max_scan: int | None = None,
) -> str:
"""Return the base url if declared in the given HTML `text`,
relative to the given base url.
.. versionadded:: VERSION
The *max_scan* parameter.
If no base url is found, the given `baseurl` is returned.
*max_scan* is an upper bound on how much of *text* to look at, in
characters, or in bytes if *text* is a byte string, defaulting to ``None``,
i.e. no bound. It bounds work, not results: a tag that starts before the
limit is read to its end. ``0`` looks at nothing, and a negative value
raises :exc:`ValueError`.
"""
text, cut = _cut(text, max_scan)
# Most documents declare no base url, so ruling one out in the bytes saves
# decoding them. A hit falls through to the scan below, which decides: a
# byte sequence that spells "<base" is not necessarily a tag, e.g. in a
# multi-byte encoding it can be part of a character.
if (
isinstance(text, bytes)
and _scannable(encoding)
and not _base_bytes_re.search(text)
):
return safe_url_string(baseurl)
# A cut through bytes can split a character, which is then replaced.
utext = to_unicode(text, encoding, errors="replace" if cut else "strict")
if _base_re.search(utext):
for m in _base_scan_re.finditer(utext):
if (attrs := m.group("attrs")) is None:
continue
# iter_tag_attributes drops valueless attributes, but a valueless
# href counts as one, so read the attributes here to see it.
for attr in _attr_re.finditer(attrs):
if attr["name"].lower() != "href":
continue
# The first <base> with an href attribute sets the base URL,
# and the first href of that tag is the one read; an empty or
# valueless one leaves the fallback in place, so a later <base>
# is not read either way.
# https://html.spec.whatwg.org/commit-snapshots/3e7b72c44ce144cee7db859cd0647af6646b6793/#set-the-frozen-base-url
value = attr["double"] or attr["single"] or attr["bare"] or ""
if url := value.strip(HTML5_WHITESPACE):
return urljoin(
safe_url_string(baseurl),
safe_url_string(url, encoding=encoding),
)
return safe_url_string(baseurl)
return safe_url_string(baseurl)
def _refresh(attrs: str, baseurl: str, encoding: str) -> tuple[float, str] | None:
"""Return the interval and absolute url of the refresh that *attrs*, the
text of a <meta> tag after its name, declares, if it declares one."""
if "&" in attrs:
attrs = replace_entities(attrs)
has_refresh_pragma = False
interval: float | None = None
url: str | None = None
for name, value in iter_tag_attributes(attrs):
match name:
case "http-equiv":
if "refresh" in value.lower():
has_refresh_pragma = True
case "content":
if interval is None and (m := _meta_refresh_content_re.match(value)):
interval = float(m.group("int"))
url = m.group("url")
if not has_refresh_pragma or interval is None:
return None
assert url is not None
return interval, urljoin(baseurl, safe_url_string(url.strip(" \"'"), encoding))
[docs]
def strip_html5_whitespace(text: str) -> str:
r"""
Strip all leading and trailing space characters (as defined in
https://www.w3.org/TR/html5/infrastructure.html#space-character).
Such stripping is useful e.g. for processing HTML element attributes which
contain URLs, like ``href``, ``src`` or form ``action`` - HTML5 standard
defines them as "valid URL potentially surrounded by spaces"
or "valid non-empty URL potentially surrounded by spaces".
>>> strip_html5_whitespace(' hello\n')
'hello'
"""
return text.strip(HTML5_WHITESPACE)