Selector Class¶

def __getitem__(self, key: str) -> TextHandler:
    if self._is_text_node(self._root):
        raise TypeError("Text nodes do not have attributes")
    return self.attrib[key]

def __contains__(self, key: str) -> bool:
    if self._is_text_node(self._root):
        return False
    return key in self.attrib

def __getstate__(self) -> Any:
    # lxml don't like it :)
    raise TypeError("Can't pickle Selector objects")

def get_all_text(
    self,
    separator: str = "\n",
    strip: bool = False,
    ignore_tags: Tuple = (
        "script",
        "style",
    ),
    valid_values: bool = True,
) -> TextHandler:
    """Get all child strings of this element, concatenated using the given separator.

    :param separator: Strings will be concatenated using this separator.
    :param strip: If True, strings will be stripped before being concatenated.
    :param ignore_tags: A tuple of all tag names you want to ignore
    :param valid_values: If enabled, elements with text-content that is empty or only whitespaces will be ignored

    :return: A TextHandler
    """
    if self._is_text_node(self._root):
        return TextHandler(str(self._root))

    ignored_elements: set[Any] = set()
    if ignore_tags:
        ignored_elements.update(self._root.iter(*ignore_tags))

    _all_strings = []

    def append_text(text: str) -> None:
        processed_text = text.strip() if strip else text
        if not valid_values or processed_text.strip():
            _all_strings.append(processed_text)

    def is_visible_text_node(text_node: _ElementUnicodeResult) -> bool:
        parent = text_node.getparent()
        if parent is None:
            return False

        owner = parent.getparent() if text_node.is_tail else parent
        while owner is not None:
            if owner in ignored_elements:
                return False
            owner = owner.getparent()
        return True

    for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)):
        text = str(text_node)
        if text and is_visible_text_node(text_node):
            append_text(text)

    return cast(TextHandler, TextHandler(separator).join(_all_strings))

def urljoin(self, relative_url: str) -> str:
    """Join this Selector's url with a relative url to form an absolute full URL."""
    return urljoin(self.url, relative_url)

def prettify(self) -> TextHandler:
    """Return a prettified version of the element's inner html-code"""
    if self._is_text_node(self._root):
        return TextHandler(str(self._root))
    content = tostring(
        self._root,
        encoding=self.encoding,
        pretty_print=True,
        method="html",
        with_tail=False,
    )
    if isinstance(content, bytes):
        content = content.strip().decode(self.encoding)
    return TextHandler(content)

def has_class(self, class_name: str) -> bool:
    """Check if the element has a specific class
    :param class_name: The class name to check for
    :return: True if element has class with that name otherwise False
    """
    if self._is_text_node(self._root):
        return False
    return class_name in self._root.classes

def iterancestors(self) -> Generator["Selector", None, None]:
    """Return a generator that loops over all ancestors of the element, starting with the element's parent."""
    if self._is_text_node(self._root):
        return
    for ancestor in self._root.iterancestors():
        yield self.__element_convertor(ancestor)

def find_ancestor(self, func: Callable[["Selector"], bool]) -> Optional["Selector"]:
    """Loop over all ancestors of the element till one match the passed function
    :param func: A function that takes each ancestor as an argument and returns True/False
    :return: The first ancestor that match the function or ``None`` otherwise.
    """
    for ancestor in self.iterancestors():
        if func(ancestor):
            return ancestor
    return None

def get(self) -> TextHandler:
    """
    Serialize this element to a string.
    For text nodes, returns the text value. For HTML elements, returns the outer HTML.
    """
    if self._is_text_node(self._root):
        return TextHandler(str(self._root))
    return self.html_content

def getall(self) -> TextHandlers:
    """Return a single-element list containing this element's serialized string."""
    return TextHandlers([self.get()])

def __str__(self) -> str:
    if self._is_text_node(self._root):
        return str(self._root)
    return self.html_content

def __repr__(self) -> str:
    length_limit = 40

    if self._is_text_node(self._root):
        text = str(self._root)
        if len(text) > length_limit:
            text = text[:length_limit].strip() + "..."
        return f"<text='{text}'>"

    content = clean_spaces(self.html_content)
    if len(content) > length_limit:
        content = content[:length_limit].strip() + "..."
    data = f"<data='{content}'"

    if self.parent:
        parent_content = clean_spaces(self.parent.html_content)
        if len(parent_content) > length_limit:
            parent_content = parent_content[:length_limit].strip() + "..."

        data += f" parent='{parent_content}'"

    return data + ">"

def relocate(
    self,
    element: Union[Dict, HtmlElement, "Selector"],
    percentage: int = 40,
    selector_type: bool = False,
) -> Union[List[HtmlElement], "Selectors"]:
    """This function will search again for the element in the page tree, used automatically on page structure change

    :param element: The element we want to relocate in the tree
    :param percentage: The minimum percentage to accept and not going lower than that. Be aware that the percentage
     calculation depends solely on the page structure, so don't play with this number unless you must know
     what you are doing!
    :param selector_type: If True, the return result will be converted to `Selectors` object
    :return: List of pure HTML elements that got the highest matching score or 'Selectors' object
    """
    score_table: Dict[float, List[Any]] = {}
    # Note: `element` will most likely always be a dictionary at this point.
    if isinstance(element, self.__class__):
        element = element._root

    if issubclass(type(element), HtmlElement):
        element = _StorageTools.element_to_dict(element)

    for node in cast(List, _find_all_elements(self._root)):
        # Collect all elements in the page, then for each element get the matching score of it against the node.
        # Hence: the code doesn't stop even if the score was 100%
        # because there might be another element(s) left in page with the same score
        score = self.__calculate_similarity_score(cast(Dict, element), node)
        score_table.setdefault(score, []).append(node)

    if score_table:
        highest_probability = max(score_table.keys())
        if score_table[highest_probability] and highest_probability >= percentage:
            if log.getEffectiveLevel() < 20:
                # No need to execute this part if the logging level is not debugging
                log.debug(f"Highest probability was {highest_probability}%")
                log.debug("Top 5 best matching elements are: ")
                for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]:
                    log.debug(f"{percent} -> {self.__elements_convertor(score_table[percent])}")

            if not selector_type:
                return score_table[highest_probability]
            return self.__elements_convertor(score_table[highest_probability])
        log.warning(
            f"Adaptive relocation found no element above the {percentage}% threshold "
            f"(top score: {highest_probability}%). Lower `percentage` if this is the right element."
        )
    return []

def css(
    self,
    selector: str,
    identifier: str = "",
    adaptive: bool = False,
    auto_save: bool = False,
    percentage: int = 40,
) -> "Selectors":
    """Search the current tree with CSS3 selectors

    **Important:
    It's recommended to use the identifier argument if you plan to use a different selector later
    and want to relocate the same element(s)**

    :param selector: The CSS3 selector to be used.
    :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
    :param identifier: A string that will be used to save/retrieve element's data in adaptive,
     otherwise the selector will be used.
    :param auto_save: Automatically save new elements for `adaptive` later
    :param percentage: The minimum percentage to accept while `adaptive` is working and not going lower than that.
     Be aware that the percentage calculation depends solely on the page structure, so don't play with this
     number unless you must know what you are doing!

    :return: `Selectors` class.
    """
    if self._is_text_node(self._root):
        return Selectors()

    try:
        if not self.__adaptive_enabled or "," not in selector:
            # No need to split selectors in this case, let's save some CPU cycles :)
            xpath_selector = _css_to_xpath(selector)
            return self.xpath(
                xpath_selector,
                identifier or selector,
                adaptive,
                auto_save,
                percentage,
            )

        results = Selectors()
        for single_selector in split_selectors(selector):
            # I'm doing this only so the `save` function saves data correctly for combined selectors
            # Like using the ',' to combine two different selectors that point to different elements.
            xpath_selector = _css_to_xpath(single_selector.canonical())
            results += self.xpath(
                xpath_selector,
                identifier or single_selector.canonical(),
                adaptive,
                auto_save,
                percentage,
            )

        return Selectors(results)
    except (
        SelectorError,
        SelectorSyntaxError,
    ) as e:
        raise SelectorSyntaxError(f"Invalid CSS selector '{selector}': {str(e)}") from e

def xpath(
    self,
    selector: str,
    identifier: str = "",
    adaptive: bool = False,
    auto_save: bool = False,
    percentage: int = 40,
    **kwargs: Any,
) -> "Selectors":
    """Search the current tree with XPath selectors

    **Important:
    It's recommended to use the identifier argument if you plan to use a different selector later
    and want to relocate the same element(s)**

     Note: **Additional keyword arguments will be passed as XPath variables in the XPath expression!**

    :param selector: The XPath selector to be used.
    :param adaptive: Enabled will make the function try to relocate the element if it was 'saved' before
    :param identifier: A string that will be used to save/retrieve element's data in adaptive,
     otherwise the selector will be used.
    :param auto_save: Automatically save new elements for `adaptive` later
    :param percentage: The minimum percentage to accept while `adaptive` is working and not going lower than that.
     Be aware that the percentage calculation depends solely on the page structure, so don't play with this
     number unless you must know what you are doing!

    :return: `Selectors` class.
    """
    if self._is_text_node(self._root):
        return Selectors()

    try:
        if elements := self._root.xpath(selector, **kwargs):
            if not self.__adaptive_enabled and auto_save:
                log.warning(
                    "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
                )
            elif self.__adaptive_enabled and auto_save:
                self.save(elements[0], identifier or selector)

            return self.__handle_elements(elements)
        elif self.__adaptive_enabled:
            if adaptive:
                element_data = self.retrieve(identifier or selector)
                if element_data:
                    elements = self.relocate(element_data, percentage)
                    if elements and auto_save:
                        self.save(elements[0], identifier or selector)

            return self.__handle_elements(elements)
        else:
            if adaptive:
                log.warning(
                    "Argument `adaptive` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
                )
            elif auto_save:
                log.warning(
                    "Argument `auto_save` will be ignored because `adaptive` wasn't enabled on initialization. Check docs for more info."
                )

            return self.__handle_elements(elements)

    except (
        SelectorError,
        SelectorSyntaxError,
        XPathError,
        XPathEvalError,
    ) as e:
        raise SelectorSyntaxError(f"Invalid XPath selector: {selector}") from e

def find_all(
    self,
    *args: str | Iterable[str] | Pattern | Callable | Dict[str, str],
    **kwargs: str,
) -> "Selectors":
    """Find elements by filters of your creations for ease.

    :param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
    :param kwargs: The attributes you want to filter elements based on it.
    :return: The `Selectors` object of the elements or empty list
    """
    if self._is_text_node(self._root):
        return Selectors()

    if not args and not kwargs:
        raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.")

    attributes: Dict[str, Any] = dict()
    tags: Set[str] = set()
    patterns: Set[Pattern] = set()
    results, functions, selectors = Selectors(), [], []

    # Brace yourself for a wonderful journey!
    for arg in args:
        if isinstance(arg, str):
            tags.add(arg)

        elif type(arg) in (list, tuple, set):
            arg = cast(Iterable, arg)  # Type narrowing for type checkers like pyright
            if not all(map(lambda x: isinstance(x, str), arg)):
                raise TypeError("Nested Iterables are not accepted, only iterables of tag names are accepted")
            tags.update(set(arg))

        elif isinstance(arg, dict):
            if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in arg.items()]):
                raise TypeError(
                    "Nested dictionaries are not accepted, only string keys and string values are accepted"
                )
            attributes.update(arg)

        elif isinstance(arg, re_Pattern):
            patterns.add(arg)

        elif callable(arg):
            if len(signature(arg).parameters) > 0:
                functions.append(arg)
            else:
                raise TypeError(
                    "Callable filter function must have at least one argument to take `Selector` objects."
                )

        else:
            raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.')

    if not all([(isinstance(k, str) and isinstance(v, str)) for k, v in kwargs.items()]):
        raise TypeError("Only string values are accepted for arguments")

    for attribute_name, value in kwargs.items():
        # Only replace names for kwargs, replacing them in dictionaries doesn't make sense
        attribute_name = _whitelisted.get(attribute_name, attribute_name)
        attributes[attribute_name] = value

    # It's easier and faster to build a selector than traversing the tree
    tags = tags or set("*")
    for tag in tags:
        selector = tag
        for key, value in attributes.items():
            value = value.replace('"', r"\"")  # Escape double quotes in user input
            # Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
            selector += '[{}="{}"]'.format(key, value)
        if selector != "*":
            selectors.append(selector)

    if selectors:
        results = cast(Selectors, self.css(", ".join(selectors)))
        if results:
            # From the results, get the ones that fulfill passed regex patterns
            for pattern in patterns:
                results = results.filter(lambda e: e.text.re(pattern, check_match=True))

            # From the results, get the ones that fulfill passed functions
            for function in functions:
                results = results.filter(function)
    else:
        results = results or self.below_elements
        for pattern in patterns:
            results = results.filter(lambda e: e.text.re(pattern, check_match=True))

        # Collect an element if it fulfills the passed function otherwise
        for function in functions:
            results = results.filter(function)

    return results

def find(
    self,
    *args: str | Iterable[str] | Pattern | Callable | Dict[str, str],
    **kwargs: str,
) -> Optional["Selector"]:
    """Find elements by filters of your creations for ease, then return the first result. Otherwise return `None`.

    :param args: Tag name(s), iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.
    :param kwargs: The attributes you want to filter elements based on it.
    :return: The `Selector` object of the element or `None` if the result didn't match
    """
    for element in self.find_all(*args, **kwargs):
        return element
    return None

def save(self, element: HtmlElement, identifier: str) -> None:
    """Saves the element's unique properties to the storage for retrieval and relocation later

    :param element: The element itself that we want to save to storage, it can be a ` Selector ` or pure ` HtmlElement `
    :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
        the docs for more info.
    """
    if self.__adaptive_enabled and self._storage:
        target_element: Any = element
        if isinstance(target_element, self.__class__):
            target_element = target_element._root

        if self._is_text_node(target_element):
            target_element = target_element.getparent()

        self._storage.save(target_element, identifier)
    else:
        raise RuntimeError(
            "Can't use `adaptive` features while it's disabled globally, you have to start a new class instance."
        )

def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]:
    """Using the identifier, we search the storage and return the unique properties of the element

    :param identifier: This is the identifier that will be used to retrieve the element from the storage. See
        the docs for more info.
    :return: A dictionary of the unique properties
    """
    if self.__adaptive_enabled and self._storage:
        return self._storage.retrieve(identifier)

    raise RuntimeError(
        "Can't use `adaptive` features while it's disabled globally, you have to start a new class instance."
    )

def json(self) -> Dict:
    """Return JSON response if the response is jsonable otherwise throws error"""
    if self._is_text_node(self._root):
        return TextHandler(str(self._root)).json()
    if self._raw_body and isinstance(self._raw_body, (str, bytes)):
        if isinstance(self._raw_body, str):
            return TextHandler(self._raw_body).json()
        else:
            if TYPE_CHECKING:
                assert isinstance(self._raw_body, bytes)
            return TextHandler(self._raw_body.decode()).json()
    elif self.text:
        return self.text.json()
    else:
        return self.get_all_text(strip=True).json()

def re(
    self,
    regex: str | Pattern[str],
    replace_entities: bool = True,
    clean_match: bool = False,
    case_sensitive: bool = True,
) -> TextHandlers:
    """Apply the given regex to the current text and return a list of strings with the matches.

    :param regex: Can be either a compiled regular expression or a string.
    :param replace_entities: If enabled character entity references are replaced by their corresponding character
    :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
    :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it
    """
    return self.text.re(regex, replace_entities, clean_match, case_sensitive)

def re_first(
    self,
    regex: str | Pattern[str],
    default=None,
    replace_entities: bool = True,
    clean_match: bool = False,
    case_sensitive: bool = True,
) -> TextHandler:
    """Apply the given regex to text and return the first match if found, otherwise return the default value.

    :param regex: Can be either a compiled regular expression or a string.
    :param default: The default value to be returned if there is no match
    :param replace_entities: if enabled character entity references are replaced by their corresponding character
    :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
    :param case_sensitive: if disabled, the function will set the regex to ignore the letters case while compiling it
    """
    return self.text.re_first(regex, default, replace_entities, clean_match, case_sensitive)

def find_similar(
    self,
    similarity_threshold: float = 0.2,
    ignore_attributes: List | Tuple = (
        "href",
        "src",
    ),
    match_text: bool = False,
) -> "Selectors":
    """Find elements that are in the same tree depth in the page with the same tag name and same parent tag etc...
    then return the ones that match the current element attributes with a percentage higher than the input threshold.

    This function is inspired by AutoScraper and made for cases where you, for example, found a product div inside
    a products-list container and want to find other products using that element as a starting point EXCEPT
    this function works in any case without depending on the element type.

    :param similarity_threshold: The percentage to use while comparing element attributes.
        Note: Elements found before attributes matching/comparison will be sharing the same depth, same tag name,
        same parent tag name, and same grand parent tag name. So they are 99% likely to be correct unless you are
        extremely unlucky, then attributes matching comes into play, so don't play with this number unless
        you are getting the results you don't want.
        Also, if the current element doesn't have attributes and the similar element as well, then it's a 100% match.
    :param ignore_attributes: Attribute names passed will be ignored while matching the attributes in the last step.
        The default value is to ignore `href` and `src` as URLs can change a lot between elements, so it's unreliable
    :param match_text: If True, element text content will be taken into calculation while matching.
        Not recommended to use in normal cases, but it depends.

    :return: A ``Selectors`` container of ``Selector`` objects or empty list
    """
    if self._is_text_node(self._root):
        return Selectors()

    # We will use the elements' root from now on to get the speed boost of using Lxml directly
    root = self._root
    similar_elements = list()

    current_depth = len(list(root.iterancestors()))
    target_attrs = self.__get_attributes(root, ignore_attributes) if ignore_attributes else root.attrib

    path_parts = [self.tag]
    if (parent := root.getparent()) is not None:
        path_parts.insert(0, parent.tag)
        if (grandparent := parent.getparent()) is not None:
            path_parts.insert(0, grandparent.tag)

    xpath_path = "//{}".format("/".join(path_parts))
    potential_matches = root.xpath(f"{xpath_path}[count(ancestor::*) = {current_depth}]")

    for potential_match in potential_matches:
        if potential_match != root and self.__are_alike(
            root,
            target_attrs,
            potential_match,
            ignore_attributes,
            similarity_threshold,
            match_text,
        ):
            similar_elements.append(potential_match)

    return Selectors(map(self.__element_convertor, similar_elements))