diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index a13f345eec..c4a419c5ff 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -5,6 +5,7 @@ aexit aiter anext anextawaitable +annotationlib appendleft argcount arrayiterator diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4667f4ee17..a03de55068 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -4,7 +4,7 @@ This document provides guidelines for working with GitHub Copilot when contribut ## Project Overview -RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide: +RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0+ compatibility. The project aims to provide: - A complete Python-3 environment entirely in Rust (not CPython bindings) - A clean implementation without compatibility hacks diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b8263026d5..7fc4c65318 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,7 @@ env: test.test_multiprocessing_spawn.test_processes ENV_POLLUTING_TESTS_WINDOWS: >- # Python version targeted by the CI. - PYTHON_VERSION: "3.13.1" + PYTHON_VERSION: "3.14.2" X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index d48c5e4cfe..9d549602bc 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -13,7 +13,7 @@ name: Periodic checks/tasks env: CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - PYTHON_VERSION: "3.13.1" + PYTHON_VERSION: "3.14.2" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index c580e7d0ea..37cf56504d 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -9,7 +9,7 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.13.9" + default: "3.14.2" ref: description: Branch to commit to (leave empty for current branch) type: string diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 82364e9b81..f515bef1a1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -25,7 +25,7 @@ RustPython requires the following: stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.13 or higher +- CPython version 3.14 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 9eb6f0933b..d6673f6692 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,13 +1,16 @@ -from __future__ import annotations -import io import os import sys +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field, Field + COLORIZE = True + # types if False: - from typing import IO + from typing import IO, Self, ClassVar + _theme: Theme class ANSIColors: @@ -17,11 +20,13 @@ class ANSIColors: BLUE = "\x1b[34m" CYAN = "\x1b[36m" GREEN = "\x1b[32m" + GREY = "\x1b[90m" MAGENTA = "\x1b[35m" RED = "\x1b[31m" WHITE = "\x1b[37m" # more like LIGHT GRAY YELLOW = "\x1b[33m" + BOLD = "\x1b[1m" BOLD_BLACK = "\x1b[1;30m" # DARK GRAY BOLD_BLUE = "\x1b[1;34m" BOLD_CYAN = "\x1b[1;36m" @@ -60,13 +65,196 @@ class ANSIColors: INTENSE_BACKGROUND_YELLOW = "\x1b[103m" +ColorCodes = set() NoColors = ANSIColors() -for attr in dir(NoColors): +for attr, code in ANSIColors.__dict__.items(): if not attr.startswith("__"): + ColorCodes.add(code) setattr(NoColors, attr, "") +# +# Experimental theming support (see gh-133346) +# + +# - Create a theme by copying an existing `Theme` with one or more sections +# replaced, using `default_theme.copy_with()`; +# - create a theme section by copying an existing `ThemeSection` with one or +# more colors replaced, using for example `default_theme.syntax.copy_with()`; +# - create a theme from scratch by instantiating a `Theme` data class with +# the required sections (which are also dataclass instances). +# +# Then call `_colorize.set_theme(your_theme)` to set it. +# +# Put your theme configuration in $PYTHONSTARTUP for the interactive shell, +# or sitecustomize.py in your virtual environment or Python installation for +# other uses. Your applications can call `_colorize.set_theme()` too. +# +# Note that thanks to the dataclasses providing default values for all fields, +# creating a new theme or theme section from scratch is possible without +# specifying all keys. +# +# For example, here's a theme that makes punctuation and operators less prominent: +# +# try: +# from _colorize import set_theme, default_theme, Syntax, ANSIColors +# except ImportError: +# pass +# else: +# theme_with_dim_operators = default_theme.copy_with( +# syntax=Syntax(op=ANSIColors.INTENSE_BLACK), +# ) +# set_theme(theme_with_dim_operators) +# del set_theme, default_theme, Syntax, ANSIColors, theme_with_dim_operators +# +# Guarding the import ensures that your .pythonstartup file will still work in +# Python 3.13 and older. Deleting the variables ensures they don't remain in your +# interactive shell's global scope. + +class ThemeSection(Mapping[str, str]): + """A mixin/base class for theme sections. + + It enables dictionary access to a section, as well as implements convenience + methods. + """ + + # The two types below are just that: types to inform the type checker that the + # mixin will work in context of those fields existing + __dataclass_fields__: ClassVar[dict[str, Field[str]]] + _name_to_value: Callable[[str], str] + + def __post_init__(self) -> None: + name_to_value = {} + for color_name in self.__dataclass_fields__: + name_to_value[color_name] = getattr(self, color_name) + super().__setattr__('_name_to_value', name_to_value.__getitem__) + + def copy_with(self, **kwargs: str) -> Self: + color_state: dict[str, str] = {} + for color_name in self.__dataclass_fields__: + color_state[color_name] = getattr(self, color_name) + color_state.update(kwargs) + return type(self)(**color_state) + + @classmethod + def no_colors(cls) -> Self: + color_state: dict[str, str] = {} + for color_name in cls.__dataclass_fields__: + color_state[color_name] = "" + return cls(**color_state) + + def __getitem__(self, key: str) -> str: + return self._name_to_value(key) + + def __len__(self) -> int: + return len(self.__dataclass_fields__) + + def __iter__(self) -> Iterator[str]: + return iter(self.__dataclass_fields__) + + +@dataclass(frozen=True, kw_only=True) +class Argparse(ThemeSection): + usage: str = ANSIColors.BOLD_BLUE + prog: str = ANSIColors.BOLD_MAGENTA + prog_extra: str = ANSIColors.MAGENTA + heading: str = ANSIColors.BOLD_BLUE + summary_long_option: str = ANSIColors.CYAN + summary_short_option: str = ANSIColors.GREEN + summary_label: str = ANSIColors.YELLOW + summary_action: str = ANSIColors.GREEN + long_option: str = ANSIColors.BOLD_CYAN + short_option: str = ANSIColors.BOLD_GREEN + label: str = ANSIColors.BOLD_YELLOW + action: str = ANSIColors.BOLD_GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Syntax(ThemeSection): + prompt: str = ANSIColors.BOLD_MAGENTA + keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE + builtin: str = ANSIColors.CYAN + comment: str = ANSIColors.RED + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + op: str = ANSIColors.RESET + definition: str = ANSIColors.BOLD + soft_keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Traceback(ThemeSection): + type: str = ANSIColors.BOLD_MAGENTA + message: str = ANSIColors.MAGENTA + filename: str = ANSIColors.MAGENTA + line_no: str = ANSIColors.MAGENTA + frame: str = ANSIColors.MAGENTA + error_highlight: str = ANSIColors.BOLD_RED + error_range: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Unittest(ThemeSection): + passed: str = ANSIColors.GREEN + warn: str = ANSIColors.YELLOW + fail: str = ANSIColors.RED + fail_info: str = ANSIColors.BOLD_RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Theme: + """A suite of themes for all sections of Python. + + When adding a new one, remember to also modify `copy_with` and `no_colors` + below. + """ + argparse: Argparse = field(default_factory=Argparse) + syntax: Syntax = field(default_factory=Syntax) + traceback: Traceback = field(default_factory=Traceback) + unittest: Unittest = field(default_factory=Unittest) + + def copy_with( + self, + *, + argparse: Argparse | None = None, + syntax: Syntax | None = None, + traceback: Traceback | None = None, + unittest: Unittest | None = None, + ) -> Self: + """Return a new Theme based on this instance with some sections replaced. + + Themes are immutable to protect against accidental modifications that + could lead to invalid terminal states. + """ + return type(self)( + argparse=argparse or self.argparse, + syntax=syntax or self.syntax, + traceback=traceback or self.traceback, + unittest=unittest or self.unittest, + ) + + @classmethod + def no_colors(cls) -> Self: + """Return a new Theme where colors in all sections are empty strings. + + This allows writing user code as if colors are always used. The color + fields will be ANSI color code strings when colorization is desired + and possible, and empty strings otherwise. + """ + return cls( + argparse=Argparse.no_colors(), + syntax=Syntax.no_colors(), + traceback=Traceback.no_colors(), + unittest=Unittest.no_colors(), + ) + + def get_colors( colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None ) -> ANSIColors: @@ -76,22 +264,37 @@ def get_colors( return NoColors +def decolor(text: str) -> str: + """Remove ANSI color codes from a string.""" + for code in ColorCodes: + text = text.replace(code, "") + return text + + def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -108,5 +311,45 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() + + +default_theme = Theme() +theme_no_color = default_theme.no_colors() + + +def get_theme( + *, + tty_file: IO[str] | IO[bytes] | None = None, + force_color: bool = False, + force_no_color: bool = False, +) -> Theme: + """Returns the currently set theme, potentially in a zero-color variant. + + In cases where colorizing is not possible (see `can_colorize`), the returned + theme contains all empty strings in all color definitions. + See `Theme.no_colors()` for more information. + + It is recommended not to cache the result of this function for extended + periods of time because the user might influence theme selection by + the interactive shell, a debugger, or application-specific code. The + environment (including environment variable state and console configuration + on Windows) can also change in the course of the application life cycle. + """ + if force_color or (not force_no_color and + can_colorize(file=tty_file)): + return _theme + return theme_no_color + + +def set_theme(t: Theme) -> None: + global _theme + + if not isinstance(t, Theme): + raise ValueError(f"Expected Theme object, found {t}") + + _theme = t + + +set_theme(default_theme) diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 3e98489419..b70162d86d 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -132,6 +132,8 @@ 'BUILD_SET_FROM_TUPLES': 122, 'BUILD_TUPLE_FROM_ITER': 123, 'BUILD_TUPLE_FROM_TUPLES': 124, + 'BUILD_TEMPLATE': 125, + 'BUILD_INTERPOLATION': 126, 'CONTINUE': 128, 'JUMP_IF_FALSE_OR_POP': 129, 'JUMP_IF_TRUE_OR_POP': 130, diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 0000000000..a5788cdbfa --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,1143 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_from_class_namespace", + "get_annotations", + "annotations_to_string", + "type_repr", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + # These are always set to None here but may be non-None if a ForwardRef + # is created through __class__ assignment on a _Stringifier object. + self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). + self.__cell__ = None + self.__extra_names__ = None + # These are initially None but serve as a cache and may be set to a non-None + # value later. + self.__code__ = None + self.__ast_node__ = None + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + match format: + case Format.STRING: + return self.__forward_arg__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) + if isinstance(self.__cell__, types.CellType): + try: + return self.__cell__.cell_contents + except ValueError: + pass + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: + for param in type_params: + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + + if self.__extra_names__: + locals.update(self.__extra_names__) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + return locals[arg] + elif arg in globals: + return globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + elif is_forwardref_format: + return self + else: + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) + else: + code = self.__forward_code__ + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. + new_locals = _StringifierDict( + {**builtins.__dict__, **globals, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, + format=format, + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify(self.__cell__) + return result + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + try: + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + # Use "is" here because we use id() for this in __hash__ + # because dictionaries are not hashable. + and self.__globals__ is other.__globals__ + and self.__forward_is_class__ == other.__forward_is_class__ + and self.__cell__ == other.__cell__ + and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) + ) + + def __hash__(self): + return hash(( + self.__forward_arg__, + self.__forward_module__, + id(self.__globals__), # dictionaries are not hashable, so hash by identity + self.__forward_is_class__, + tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, + )) + + def __or__(self, other): + return types.UnionType[self, other] + + def __ror__(self, other): + return types.UnionType[other, self] + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_Template = type(t"") + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + extra_names=None, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__extra_names__ = extra_names + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): + return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + + return ast.Slice( + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names + else: + return self.__convert_to_ast(other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names or None, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + other = ast.Tuple(elts) + else: + other, extra_names = self.__convert_to_ast_getitem(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(self.__get_ast(), op, rhs), extra_names + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(new_other, op, self.__get_ast()), extra_names + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[rhs], + ), + extra_names, + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + +class _StringifierDict(dict): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + self.next_id = 1 + self.format = format + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + def transmogrify(self, cell_dict): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict + + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + + globals = _StringifierDict({}, format=format) + is_class = isinstance(owner, type) + closure, _ = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + if _is_evaluate: + return _stringify_single(annos) + return { + key: _stringify_single(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict( + namespace, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) + except Exception: + pass + else: + globals.transmogrify(cell_dict) + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + globals.transmogrify(cell_dict) + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + if not annotate.__closure__: + return None, None + new_closure = [] + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure), cell_dict + + +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) + else: + return repr(anno) + + +def get_annotate_from_class_namespace(obj): + """Retrieve the annotate function from a class namespace dictionary. + + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. + """ + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, module, or other object with + __annotate__ or __annotations__ attributes. + Passing any other object raises TypeError. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the Format enum or its integer equivalent. + For the VALUE format, the __annotations__ is tried first; if it + does not exist, the __annotate__ function is called. The + FORWARDREF format uses __annotations__ if it exists and can be + evaluated, and otherwise falls back to calling the __annotate__ function. + The SOURCE format tries __annotate__ first, and falls back to + using __annotations__, stringified using annotations_to_string(). + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we first look at __annotations__ + ann = _get_dunder_annotations(obj) + + # If it's not there, try __annotate__ instead + if ann is None: + ann = _get_and_call_annotate(obj, format) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except Exception: + pass + else: + if ann is not None: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is None: + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + ann = _get_dunder_annotations(obj) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return dict(ann) + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + if ann is not None: + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if ann is None: + if isinstance(obj, type) or callable(obj): + return {} + raise TypeError(f"{obj!r} does not have annotations") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) + for key, value in ann.items() + } + return return_value + + +def type_repr(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is intended as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) + if value is ...: + return "..." + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format. + + Always returns a fresh a dictionary. + """ + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } + + +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + +def _get_and_call_annotate(obj, format): + """Get the __annotate__ function and call it. + + May not return a fresh dictionary. + """ + annotate = getattr(obj, "__annotate__", None) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return ann + return None + + +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def _get_dunder_annotations(obj): + """Return the annotations for an object, checking that it is a dictionary. + + Does not return a fresh dictionary. + """ + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return ann diff --git a/Lib/argparse.py b/Lib/argparse.py index bd088ea0e6..88c1f5a7ef 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -159,18 +161,21 @@ class HelpFormatter(object): provided by the class are considered an implementation detail. """ - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - + def __init__( + self, + prog, + indent_increment=2, + max_help_position=24, + width=None, + color=True, + ): # default setting for width if width is None: import shutil width = shutil.get_terminal_size().columns width -= 2 + self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -187,6 +192,16 @@ def __init__(self, self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + def _set_color(self, color): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = lambda text: text + # =============================== # Section and indentation methods # =============================== @@ -225,7 +240,11 @@ def format_help(self): if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent heading_text = _('%(heading)s:') % dict(heading=self.heading) - heading = '%*s%s\n' % (current_indent, '', heading_text) + t = self.formatter._theme + heading = ( + f'{" " * current_indent}' + f'{t.heading}{heading_text}{t.reset}\n' + ) else: heading = '' @@ -262,7 +281,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -296,16 +315,23 @@ def _join_parts(self, part_strings): if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): + t = self._theme + if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: - usage = usage % dict(prog=self._prog) + usage = ( + t.prog_extra + + usage + % {"prog": f"{t.prog}{self._prog}{t.reset}{t.prog_extra}"} + + t.reset + ) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) + usage = f"{t.prog}{self._prog}{t.reset}" # if optionals and positionals are available, calculate usage elif usage is None: @@ -327,7 +353,7 @@ def _format_usage(self, usage, actions, groups, prefix): # wrap the usage parts if it's too long text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: + if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts opt_parts = self._get_actions_usage_parts(optionals, groups) @@ -343,12 +369,13 @@ def get_lines(parts, indent, prefix=None): else: line_len = indent_length - 1 for part in parts: - if line_len + 1 + len(part) > text_width and line: + part_len = len(self._decolor(part)) + if line_len + 1 + part_len > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = indent_length - 1 line.append(part) - line_len += len(part) + 1 + line_len += part_len + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: @@ -356,8 +383,9 @@ def get_lines(parts, indent, prefix=None): return lines # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) + prog_len = len(self._decolor(prog)) + if len(prefix) + prog_len <= 0.75 * text_width: + indent = ' ' * (len(prefix) + prog_len + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) lines.extend(get_lines(pos_parts, indent)) @@ -380,12 +408,18 @@ def get_lines(parts, indent, prefix=None): # join lines into usage usage = '\n'.join(lines) + usage = usage.removeprefix(prog) + usage = f"{t.prog}{prog}{t.reset}{usage}" + # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) + return f'{t.usage}{prefix}{t.reset}{usage}\n\n' def _format_actions_usage(self, actions, groups): return ' '.join(self._get_actions_usage_parts(actions, groups)) + def _is_long_option(self, string): + return len(string) > 2 + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() @@ -398,17 +432,18 @@ def _get_actions_usage_parts(self, actions, groups): continue try: - start = actions.index(group._group_actions[0]) + start = min(actions.index(item) for item in group._group_actions) except ValueError: continue else: end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: + if set(actions[start:end]) == set(group._group_actions): group_actions.update(group._group_actions) inserts[start, end] = group # collect all actions format strings parts = [] + t = self._theme for action in actions: # suppressed arguments are marked with None @@ -418,7 +453,11 @@ def _get_actions_usage_parts(self, actions, groups): # produce all arg strings elif not action.option_strings: default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) + part = ( + t.summary_action + + self._format_args(action, default) + + t.reset + ) # if it's in a group, strip the outer [] if action in group_actions: @@ -428,18 +467,26 @@ def _get_actions_usage_parts(self, actions, groups): # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option # if the Optional doesn't take a value, format is: # -s or --long if action.nargs == 0: part = action.format_usage() + part = f"{option_color}{part}{t.reset}" # if the Optional takes a value, format is: # -s ARGS or --long ARGS else: default = self._get_default_metavar_for_optional(action) args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) # make it look optional if it's not required or in a group if not action.required and action not in group_actions: @@ -486,6 +533,7 @@ def _format_action(self, action): help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) + action_header_no_color = self._decolor(action_header) # no help; start on same line and add a final newline if not action.help: @@ -493,9 +541,15 @@ def _format_action(self, action): action_header = '%*s%s\n' % tup # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header + elif len(action_header_no_color) <= action_width: + # calculate widths without color codes + action_header_color = action_header + tup = self._current_indent, '', action_width, action_header_no_color action_header = '%*s%-*s ' % tup + # swap in the colored header + action_header = action_header.replace( + action_header_no_color, action_header_color + ) indent_first = 0 # long action name; start on the next line @@ -528,23 +582,42 @@ def _format_action(self, action): return self._join_parts(parts) def _format_action_invocation(self, action): + t = self._theme + if not action.option_strings: default = self._get_default_metavar_for_positional(action) - return ' '.join(self._metavar_formatter(action, default)(1)) + return ( + t.action + + ' '.join(self._metavar_formatter(action, default)(1)) + + t.reset + ) else: + def color_option_strings(strings): + parts = [] + for s in strings: + if self._is_long_option(s): + parts.append(f"{t.long_option}{s}{t.reset}") + else: + parts.append(f"{t.short_option}{s}{t.reset}") + return parts + # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - return ', '.join(action.option_strings) + option_strings = color_option_strings(action.option_strings) + return ', '.join(option_strings) # if the Optional takes a value, format is: # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - return ', '.join(action.option_strings) + ' ' + args_string + option_strings = color_option_strings(action.option_strings) + args_string = ( + f"{t.label}{self._format_args(action, default)}{t.reset}" + ) + return ', '.join(option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: @@ -590,16 +663,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return self._get_help_string(action) % params + return help_string % params def _iter_indented_subactions(self, action): try: @@ -844,22 +920,16 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -867,38 +937,19 @@ def __init__(self, _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) @@ -1180,6 +1231,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1195,23 +1247,30 @@ def add_parser(self, name, *, deprecated=False, **kwargs): if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + # set color + if kwargs.get('color') is None: + kwargs['color'] = self._color + aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1276,7 +1335,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1293,6 +1352,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1396,7 +1461,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1449,7 +1514,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1465,27 +1531,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): + formatter = self._get_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1529,8 +1602,8 @@ def _add_container_actions(self, container): if group.title in title_group_map: # This branch could happen if a derived class added # groups with duplicated titles in __init__ - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1571,13 +1644,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required nargs = kwargs.get('nargs') - if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]: + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1590,11 +1665,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1610,8 +1683,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1627,8 +1700,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1667,10 +1740,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_formatter"): + formatter = self._get_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1702,14 +1791,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1720,7 +1802,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1730,14 +1812,29 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') + +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' class ArgumentParser(_AttributeHolder, _ActionsContainer): @@ -1760,6 +1857,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + - color - Allow color output in help messages (default: ``False``) """ def __init__(self, @@ -1775,19 +1875,18 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): - + exit_on_error=True, + *, + suggest_on_error=False, + color=True, + ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1795,6 +1894,8 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error + self.color = color add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1844,7 +1945,7 @@ def _get_kwargs(self): def add_subparsers(self, **kwargs): if self._subparsers is not None: - raise ArgumentError(None, _('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) @@ -1859,15 +1960,19 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') + formatter.add_usage(None, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + action._color = self.color + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -2498,7 +2603,6 @@ def _get_values(self, action, arg_strings): value = action.default if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2506,11 +2610,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2543,8 +2644,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2568,14 +2668,27 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) choices = action.choices - if choices is not None: - if isinstance(choices, str): - choices = iter(choices) - if value not in choices: - args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods @@ -2611,7 +2724,9 @@ def format_help(self): return formatter.format_help() def _get_formatter(self): - return self.formatter_class(prog=self.prog) + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color) + return formatter # ===================== # Help-printing methods diff --git a/Lib/calendar.py b/Lib/calendar.py index 8c1c646da4..18f76d52ff 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -428,6 +428,7 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): @@ -646,6 +647,117 @@ def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): return super().formatmonthname(theyear, themonth, withyear) + +class _CLIDemoCalendar(TextCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + highlight_day = self.highlight_day.day + else: + highlight_day = None + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month in months + ): + month_pos = months.index(self.highlight_day.month) + else: + month_pos = None + + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for k, cal in enumerate(row): + if j >= len(cal): + weeks.append('') + else: + day = ( + self.highlight_day.day if k == month_pos else None + ) + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + +class _CLIDemoLocaleCalendar(LocaleTextCalendar, _CLIDemoCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + # Support for old module level interface c = TextCalendar() @@ -698,7 +810,7 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( @@ -765,6 +877,7 @@ def main(args=None): sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": if options.month: @@ -781,23 +894,23 @@ def main(args=None): optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: - cal = LocaleTextCalendar(locale=locale) + cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) else: - cal = TextCalendar() + cal = _CLIDemoCalendar(highlight_day=today) cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if options.month is not None: + else: _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + result = cal.formatyear(today.year, **optdict) elif options.month is None: result = cal.formatyear(options.year, **optdict) else: diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index ab6d32478e..21bbfad0fe 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.2" +_PIP_VERSION = "25.3" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser(color=True) parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl similarity index 78% rename from Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl index 4db7e72071..755e1aa0c3 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328..4183c28a80 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/inspect.py b/Lib/inspect.py index 5a814f97b5..d2a0454695 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -220,13 +220,26 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: + # XXX: RUSTPYTHON + # PEP 649: Use descriptor access to get __annotations__ + # The __annotations__ attribute is now provided via a descriptor + # that calls __annotate__ to compute the annotations lazily + try: + ann = obj.__annotations__ + except AttributeError: ann = None + # Ensure we got this class's own annotations, not inherited + if ann is not None: + # Check if __annotations__ is in this class's __dict__ + # or is computed by a descriptor (both are valid) + obj_dict = getattr(obj, '__dict__', None) + if obj_dict is not None: + dict_ann = obj_dict.get('__annotations__', None) + # If it's a descriptor or cached dict, use what we got from attribute access + # If it's inherited (not in __dict__ at all and no __annotate__), return empty + # XXX: RUSTPYTHON - also check __annotate_func__ (PEP 649 descriptor) + if dict_ann is None and '__annotate__' not in obj_dict and '__annotations_cache__' not in obj_dict and not hasattr(obj, '__annotate_func__'): + ann = None obj_globals = None module_name = getattr(obj, '__module__', None) diff --git a/Lib/site.py b/Lib/site.py index 2983ca7154..5305d67b3b 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,7 +73,7 @@ import os import builtins import _sitebuiltins -import io +import _io as io import stat import errno @@ -95,6 +95,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -444,9 +450,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software - Foundation, and a cast of thousands for supporting Python - development. See www.python.org for more information.""") +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -574,7 +580,7 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass @@ -626,17 +632,17 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths @@ -646,7 +652,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -666,7 +672,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/string.py b/Lib/string/__init__.py similarity index 87% rename from Lib/string.py rename to Lib/string/__init__.py index 2eab6d4f59..eab5067c9b 100644 --- a/Lib/string.py +++ b/Lib/string/__init__.py @@ -49,11 +49,18 @@ def capwords(s, sep=None): #################################################################### -import re as _re -from collections import ChainMap as _ChainMap - _sentinel_dict = {} + +class _TemplatePattern: + # This descriptor is overwritten in ``Template._compile_pattern()``. + def __get__(self, instance, cls=None): + if cls is None: + return self + return cls._compile_pattern() +_TemplatePattern = _TemplatePattern() + + class Template: """A string class for supporting $-substitutions.""" @@ -64,14 +71,21 @@ class Template: # See https://bugs.python.org/issue31672 idpattern = r'(?a:[_a-z][_a-z0-9]*)' braceidpattern = None - flags = _re.IGNORECASE + flags = None # default: re.IGNORECASE + + pattern = _TemplatePattern # use a descriptor to compile the pattern def __init_subclass__(cls): super().__init_subclass__() - if 'pattern' in cls.__dict__: - pattern = cls.pattern - else: - delim = _re.escape(cls.delimiter) + cls._compile_pattern() + + @classmethod + def _compile_pattern(cls): + import re # deferred import, for performance + + pattern = cls.__dict__.get('pattern', _TemplatePattern) + if pattern is _TemplatePattern: + delim = re.escape(cls.delimiter) id = cls.idpattern bid = cls.braceidpattern or cls.idpattern pattern = fr""" @@ -82,7 +96,10 @@ def __init_subclass__(cls): (?P) # Other ill-formed delimiter exprs ) """ - cls.pattern = _re.compile(pattern, cls.flags | _re.VERBOSE) + if cls.flags is None: + cls.flags = re.IGNORECASE + pat = cls.pattern = re.compile(pattern, cls.flags | re.VERBOSE) + return pat def __init__(self, template): self.template = template @@ -105,7 +122,8 @@ def substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): # Check the most common path first. @@ -124,7 +142,8 @@ def safe_substitute(self, mapping=_sentinel_dict, /, **kws): if mapping is _sentinel_dict: mapping = kws elif kws: - mapping = _ChainMap(kws, mapping) + from collections import ChainMap + mapping = ChainMap(kws, mapping) # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') @@ -170,10 +189,6 @@ def get_identifiers(self): self.pattern) return ids -# Initialize Template.pattern. __init_subclass__() is automatically called -# only for subclasses, not for the Template class itself. -Template.__init_subclass__() - ######################################################################## # the Formatter class @@ -212,19 +227,20 @@ def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, # this is some markup, find the object and do # the formatting - # handle arg indexing when empty field_names are given. - if field_name == '': + # handle arg indexing when empty field first parts are given. + field_first, _ = _string.formatter_field_name_split(field_name) + if field_first == '': if auto_arg_index is False: raise ValueError('cannot switch from manual field ' 'specification to automatic field ' 'numbering') - field_name = str(auto_arg_index) + field_name = str(auto_arg_index) + field_name auto_arg_index += 1 - elif field_name.isdigit(): + elif isinstance(field_first, int): if auto_arg_index: - raise ValueError('cannot switch from manual field ' - 'specification to automatic field ' - 'numbering') + raise ValueError('cannot switch from automatic field ' + 'numbering to manual field ' + 'specification') # disable auto arg incrementing, if it gets # used later on, then an exception will be raised auto_arg_index = False diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py new file mode 100644 index 0000000000..8164872432 --- /dev/null +++ b/Lib/string/templatelib.py @@ -0,0 +1,33 @@ +"""Support for template string literals (t-strings).""" + +t = t"{0}" +Template = type(t) +Interpolation = type(t.interpolations[0]) +del t + +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + +def _template_unpickle(*args): + import itertools + + if len(args) != 2: + raise ValueError('Template expects tuple of length 2 to unpickle') + + strings, interpolations = args + parts = [] + for string, interpolation in itertools.zip_longest(strings, interpolations): + if string is not None: + parts.append(string) + if interpolation is not None: + parts.append(interpolation) + return Template(*parts) diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index b2f0bb1386..31dc60fec3 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import sys import unittest @@ -21,6 +22,42 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestTheme(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + class TestColorizeFunction(unittest.TestCase): def test_colorized_detection_checks_for_environment_variables(self): def check(env, fallback, expected): @@ -129,6 +166,17 @@ def test_colorized_detection_checks_for_file(self): file.isatty.return_value = False self.assertEqual(_colorize.can_colorize(file=file), False) + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index b7e995334f..beda61be8a 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,11 +1,13 @@ # Author: Steven J. Bethard . +import _colorize import contextlib import functools import inspect import io import operator import os +import py_compile import shutil import stat import sys @@ -16,12 +18,22 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) +from test.support import import_helper from test.support import os_helper +from test.support import script_helper from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -631,9 +643,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -777,48 +789,12 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 + def test_invalid_name(self): parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -1036,6 +1012,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -1101,8 +1078,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1135,8 +1112,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1149,9 +1126,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1811,27 +1788,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1885,7 +1878,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1898,8 +1890,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1919,7 +1911,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1938,8 +1930,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1977,8 +1969,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -2000,8 +1992,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -2015,8 +2007,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -2032,8 +2024,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -2045,7 +2037,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -2060,7 +2052,7 @@ def test_open_args(self): def test_invalid_file_type(self): with self.assertRaises(ValueError): - argparse.FileType('b')('-test') + FileType('b')('-test') class TestFileTypeMissingInitialization(TestCase): @@ -2071,7 +2063,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2256,6 +2248,130 @@ class TestActionExtend(ParserTestCase): ] +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + class TestInvalidAction(TestCase): """Test invalid user defined Action""" @@ -2269,17 +2385,31 @@ def test_invalid_type(self): self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) def test_modified_invalid_action(self): - parser = ErrorRaisingArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) action = parser.add_argument('--foo') # Someone got crazy and did this action.type = 1 - self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) # ================ # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -2287,16 +2417,17 @@ def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None, - aliases=False): + aliases=False, usage=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description', prefix_chars=prefix_chars) + prog='PROG', description='main description', usage=usage, + prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description') + prog='PROG', description='main description', usage=usage) parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( @@ -2310,7 +2441,7 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertRaisesRegex(argparse.ArgumentError, + self.assertRaisesRegex(ValueError, 'cannot have multiple subparser arguments', parser.add_subparsers) @@ -2333,7 +2464,8 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser - parser3_kwargs = dict(description='3 description') + parser3_kwargs = dict(description='3 description', + usage='PROG --foo bar 3 t ...') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) @@ -2355,6 +2487,47 @@ def test_parse_args_failures(self): args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) + def test_parse_args_failures_details(self): + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [-h] [--foo] bar {1,2,3} ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + self.parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + + def test_parse_args_failures_details_custom_usage(self): + parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...') + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + def test_parse_args(self): # check some non-failure cases: self.assertEqual( @@ -2508,18 +2681,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2655,6 +2816,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2796,10 +2980,43 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -2832,8 +3049,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2844,7 +3059,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2923,11 +3138,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2943,7 +3157,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2953,15 +3167,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2973,7 +3186,7 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) @@ -3011,6 +3224,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3049,6 +3263,29 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): self.maxDiff = None parser = ErrorRaisingArgumentParser(prog='PROG') @@ -3077,6 +3314,14 @@ def test_empty_group(self): with self.assertRaises(ValueError): parser.parse_args(['-h']) + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) + class MEMixin(object): def test_failures_when_not_required(self): @@ -3108,21 +3353,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -3444,55 +3693,6 @@ def get_parser(self, required): -c c help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - - def get_parser(self, required): - parser = ErrorRaisingArgumentParser(prog='PROG') - group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') - return parser - - usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] - ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) - ''' - - help = '''\ - - options: - -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F - ''' - - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None - class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): def get_parser(self, required=None): @@ -3843,11 +4043,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -3860,6 +4062,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4601,6 +4804,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -4663,25 +4867,6 @@ def test_all_suppressed_mutex_with_optional_nargs(self): usage = 'usage: PROG [-h]\n' self.assertEqual(parser.format_usage(), usage) - def test_nested_mutex_groups(self): - parser = argparse.ArgumentParser(prog='PROG') - g = parser.add_mutually_exclusive_group() - g.add_argument("--spam") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - gg = g.add_mutually_exclusive_group() - gg.add_argument("--hax") - gg.add_argument("--hox", help=argparse.SUPPRESS) - gg.add_argument("--hex") - g.add_argument("--eggs") - parser.add_argument("--num") - - usage = textwrap.dedent('''\ - usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] - [--num NUM] - ''') - self.assertEqual(parser.format_usage(), usage) - def test_long_mutex_groups_wrap(self): parser = argparse.ArgumentParser(prog='PROG') g = parser.add_mutually_exclusive_group() @@ -5301,11 +5486,61 @@ def custom_type(string): version = '' -class TestHelpUsageLongSubparserCommand(TestCase): - """Test that subparser commands are formatted correctly in help""" +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): maxDiff = None - def test_parent_help(self): + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=50) @@ -5371,29 +5606,45 @@ def test_missing_destination(self): self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') def test_invalid_prefix(self): - self.assertValueError('--foo', '+foo') + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') self.assertValueError('--foo', action="store-true", errmsg='unknown action') + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') + def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): @@ -5405,8 +5656,11 @@ def test_no_argument_actions(self): with self.subTest(attrs=attrs): self.assertTypeError('-x', action=action, **attrs) self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") self.assertTypeError('-x', action=action, nargs=0) - self.assertTypeError('x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments @@ -5426,7 +5680,7 @@ def test_more_than_one_argument_actions(self): self.assertValueError('-x', nargs=0, action=action, errmsg=f'nargs for {action_name} actions must be != 0') self.assertValueError('spam', nargs=0, action=action, - errmsg=f'nargs for {action_name} actions must be != 0') + errmsg='nargs for positionals must be != 0') # const is disallowed with non-optional arguments for nargs in [1, '*', '+']: @@ -5529,6 +5783,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -5558,20 +5813,25 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5791,6 +6051,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6563,9 +6824,10 @@ def test_nargs_zero(self): class TestImportStar(TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -6589,6 +6851,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6754,6 +7017,99 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + # ================= # Translation tests # ================= @@ -6764,6 +7120,257 @@ def test_translations(self): self.assertMsgidsEqual(argparse) +# =========== +# Color tests +# =========== + + +class TestColorized(TestCase): + maxDiff = None + + def setUp(self): + super().setUp() + # Ensure color even if ran with NO_COLOR=1 + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) + self.theme = _colorize.get_theme(force_color=True).argparse + + def test_argparse_color(self): + # Arrange: create a parser with a bit of everything + parser = argparse.ArgumentParser( + color=True, + description="Colorful help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prefix_chars="-+", + prog="PROG", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-v", "--verbose", action="store_true", help="more spam" + ) + group.add_argument( + "-q", "--quiet", action="store_true", help="less spam" + ) + parser.add_argument("x", type=int, help="the base") + parser.add_argument( + "y", type=int, help="the exponent", deprecated=True + ) + parser.add_argument( + "this_indeed_is_a_very_long_action_name", + type=int, + help="the exponent", + ) + parser.add_argument( + "-o", "--optional1", action="store_true", deprecated=True + ) + parser.add_argument("--optional2", help="pick one") + parser.add_argument("--optional3", choices=("X", "Y", "Z")) + parser.add_argument( + "--optional4", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional5", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional6", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "-p", + "--optional7", + choices=("Aaaaa", "Bbbbb", "Ccccc", "Ddddd"), + help="pick one", + ) + + parser.add_argument("+f") + parser.add_argument("++bar") + parser.add_argument("-+baz") + parser.add_argument("-c", "--count") + + subparsers = parser.add_subparsers( + title="subcommands", + description="valid subcommands", + help="additional help", + ) + subparsers.add_parser("sub1", deprecated=True, help="sub1 help") + sub2 = subparsers.add_parser("sub2", deprecated=True, help="sub2 help") + sub2.add_argument("--baz", choices=("X", "Y", "Z"), help="baz help") + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}-v{reset} | {short}-q{reset}] [{short}-o{reset}] [{long}--optional2 {label}OPTIONAL2{reset}] [{long}--optional3 {label}{{X,Y,Z}}{reset}] + [{long}--optional4 {label}{{X,Y,Z}}{reset}] [{long}--optional5 {label}{{X,Y,Z}}{reset}] [{long}--optional6 {label}{{X,Y,Z}}{reset}] + [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] [{long}-+baz {label}BAZ{reset}] + [{short}-c {label}COUNT{reset}] + {pos}x{reset} {pos}y{reset} {pos}this_indeed_is_a_very_long_action_name{reset} {pos}{{sub1,sub2}} ...{reset} + + Colorful help + + {heading}positional arguments:{reset} + {pos_b}x{reset} the base + {pos_b}y{reset} the exponent + {pos_b}this_indeed_is_a_very_long_action_name{reset} + the exponent + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}-v{reset}, {long_b}--verbose{reset} more spam (default: False) + {short_b}-q{reset}, {long_b}--quiet{reset} less spam (default: False) + {short_b}-o{reset}, {long_b}--optional1{reset} + {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} + pick one (default: None) + {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} + pick one (default: None) + {short_b}+f{reset} {label_b}F{reset} + {long_b}++bar{reset} {label_b}BAR{reset} + {long_b}-+baz{reset} {label_b}BAZ{reset} + {short_b}-c{reset}, {long_b}--count{reset} {label_b}COUNT{reset} + + {heading}subcommands:{reset} + valid subcommands + + {pos_b}{{sub1,sub2}}{reset} additional help + {pos_b}sub1{reset} sub1 help + {pos_b}sub2{reset} sub2 help + """ + ), + ) + + def test_argparse_color_usage(self): + # Arrange + parser = argparse.ArgumentParser( + add_help=False, + color=True, + description="Test prog and usage colors", + prog="PROG", + usage="[prefix] %(prog)s [suffix]", + ) + heading = self.theme.heading + prog = self.theme.prog + reset = self.theme.reset + usage = self.theme.prog_extra + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{usage}[prefix] {prog}PROG{reset}{usage} [suffix]{reset} + + Test prog and usage colors + """ + ), + ) + + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. RFile.seen = {} diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 38fd9ab95b..cbba54a3bf 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2010,21 +2010,18 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): + not NotImplemented class TestBreakpoint(unittest.TestCase): diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index df102fe198..35573fd9f0 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -987,10 +987,11 @@ def assertFailure(self, *args): self.assertCLIFails(*args) self.assertCmdFails(*args) + @support.force_not_colorized def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() @@ -1089,6 +1090,7 @@ def test_option_months(self): output = run('--months', '1', '2004') self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), output) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') @@ -1097,7 +1099,7 @@ def test_option_type(self): output = run('--type', 'text', '2004') self.assertEqual(output, conv(result_2004_text)) output = run('--type', 'html', '2004') - self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 456767bbe0..e543cc236c 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -207,8 +207,6 @@ def __eq__(self, other): self.assertIsNot(y, x) self.assertEqual(y.foo, x.foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_inst_getnewargs_ex(self): class C(int): def __new__(cls, *, foo): @@ -507,8 +505,6 @@ def __eq__(self, other): self.assertEqual(y.foo, x.foo) self.assertIsNot(y.foo, x.foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_deepcopy_inst_getnewargs_ex(self): class C(int): def __new__(cls, *, foo): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index b7f93d1bac..bf9b187557 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -698,7 +698,6 @@ def test_copy(self): dialect = csv.get_dialect(name) self.assertRaises(TypeError, copy.copy, dialect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 7420a49b8f..2ad302690c 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5258,7 +5258,6 @@ def _check_reduce(self, proto, obj, args=(), kwargs={}, state=None, self.assertEqual(obj.__reduce_ex__(proto), reduce_value) self.assertEqual(obj.__reduce__(), reduce_value) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_reduce(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) args = (-101, "spam") @@ -5382,7 +5381,6 @@ class C16(list): for proto in protocols: self._check_reduce(proto, obj, listitems=list(obj)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_method_lookup(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) class Picky: @@ -5515,7 +5513,6 @@ class E(C): y = pickle_copier.copy(x) self._assert_is_copy(x, y) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_reduce_copying(self): # Tests pickling and copying new-style classes and objects. global C1 diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a4b36a90d8..6d3c91b0b6 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,7 +6,6 @@ import test.support import unittest import unittest.mock -from importlib.resources.abc import Traversable from pathlib import Path import ensurepip diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 5a961711cc..21a3b8edd4 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2130,7 +2130,6 @@ class NEI(NamedInt, Enum): test_pickle_dump_load(self.assertIs, NEI.y) test_pickle_dump_load(self.assertIs, NEI) - @unittest.expectedFailure # TODO: RUSTPYTHON; fails on pickle def test_subclasses_with_getnewargs_ex(self): class NamedInt(int): __qualname__ = 'NamedInt' # needed for pickle protocol 4 diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 323f4ee4c6..aae5c2b1ce 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -374,6 +374,7 @@ class F(C, A): self.assertEqual(F.__annotations__, {}) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod @@ -403,6 +404,7 @@ def test_var_annot_in_module(self): with self.assertRaises(NameError): ann_module3.D_bad_ann(5) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_var_annot_simple_exec(self): gns = {}; lns= {} exec("'docstring'\n" diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 1c0c38ee04..e4d44e3d38 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -248,6 +248,7 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_chain_reducible(self): for oper in [copy.deepcopy] + picklecopiers: @@ -567,6 +568,7 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) @@ -601,6 +603,7 @@ def test_compress(self): next(testIntermediate) self.assertEqual(list(op(testIntermediate)), list(result2)) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) @@ -1035,6 +1038,7 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) @@ -1142,6 +1146,7 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_zip_longest_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -1365,6 +1370,7 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_product_pickling(self): # check copy, deepcopy, pickle @@ -1393,6 +1399,7 @@ def test_product_issue_25021(self): p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped self.assertRaises(StopIteration, next, p) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) @@ -1458,6 +1465,7 @@ def test_map(self): c = map(tupleize, 'abc', count()) self.pickletest(proto, c) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), @@ -1582,6 +1590,7 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] @@ -1941,6 +1950,7 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_accumulate_reducible(self): # check copy, deepcopy, pickle diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 1bac61f59e..f6fa88e650 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -409,8 +409,6 @@ def test_decompressor_bigmem(self, size): # Pickling raises an exception; there's no way to serialize an lzma_stream. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 07d9d38d6e..b6b9514362 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -745,11 +745,6 @@ def test_init(self): def test_issue5449(self): super().test_issue5449() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - def test_read(self): super().test_read() @@ -777,8 +772,6 @@ def test_truncate(self): def test_write(self): super().test_write() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -911,11 +904,6 @@ def test_newline_none(self): def test_newlines_property(self): super().test_newlines_property() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - def test_read(self): super().test_read() @@ -954,8 +942,6 @@ def test_widechar(self): self.assertEqual(memio.tell(), len(buf) * 2) self.assertEqual(memio.getvalue(), buf + buf) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -1006,21 +992,11 @@ def test_newline_cr(self): def test_newline_crlf(self): super().test_newline_crlf() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_default(self): - super().test_newline_default() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_newline_empty(self): super().test_newline_empty() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_lf(self): - super().test_newline_lf() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_newline_none(self): diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index b599c6d8c8..59c74fd0d4 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -293,8 +293,6 @@ class M(ModuleType): melon = Descr() self.assertRaises(RuntimeError, getattr, M("mymod"), "melon") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_create_annotations(self): # module objects lazy create their __annotations__ dict on demand. # the annotations dict is stored in module.__dict__. @@ -334,7 +332,11 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.typinganndata.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6..0e1a4fc50d 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -36,11 +36,15 @@ def test_default_annotations_exist(self): class C: pass self.assertEqual(C.__annotations__, {}) + # TODO: RustPython - test expectation changed in 3.14 due to PEP 649 + @unittest.expectedFailure def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + # TODO: RustPython - test expectation changed in 3.14 due to PEP 649 + @unittest.expectedFailure def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index ea51b9d091..0b634ea5dd 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -97,10 +97,6 @@ def dumps(self, arg, proto=None, **kwargs): def test_picklebuffer_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_picklebuffer_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_getattr(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bad_getattr() # TODO: RUSTPYTHON @unittest.expectedFailure @@ -135,16 +131,6 @@ def loads(self, buf, **kwds): def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_c_methods() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -239,11 +225,6 @@ def loads(self, buf, **kwds): def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_c_methods() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -259,11 +240,6 @@ def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test whe def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_load_python2_str_as_bytes() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 492f57cce2..f4ce1c75b8 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -97,16 +97,6 @@ def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_oob_buffers_writable_to_readonly() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_py_methods() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_complex_newobj_ex() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index df279bd965..56ed457882 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -8,6 +8,7 @@ import test.support from test import support from test.support.script_helper import assert_python_ok +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr @@ -308,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertTrue(site.getuserbase().startswith('xoxo')) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -319,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertTrue(user_site.startswith(site.USER_BASE)) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -362,11 +362,10 @@ def test_no_home_directory(self): environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertTrue(user_base.startswith('~' + os.sep)) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertTrue(user_site.startswith(user_base)) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -515,7 +514,7 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: @@ -578,6 +577,17 @@ def test_license_exists_at_url(self): code = e.code self.assertEqual(code, 200, msg="Can't find " + url) + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports("site", [ + "io", + "locale", + "traceback", + "atexit", + "warnings", + "textwrap", + ]) + class StartupImportTests(unittest.TestCase): @@ -843,12 +853,15 @@ def get_excepted_output(self, *args): return 10, None def invoke_command_line(self, *args): - args = ["-m", "site", *args] + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) with EnvironmentVarGuard() as env: env["PYTHONUTF8"] = "1" env["PYTHONIOENCODING"] = "utf-8" - proc = spawn_python(*args, text=True, env=env, + proc = spawn_python(*cmd_args, text=True, env=env, encoding='utf-8', errors='replace') output = kill_python(proc) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3e6c530cec..d96a3a6a5d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3803,6 +3803,7 @@ def __init__(self): self.assertNotIsInstance(B(), P) self.assertNotIsInstance(C(), P) + @unittest.expectedFailure # TODO: RUSTPYTHON; test doesn't include PEP 649 attrs def test_non_protocol_subclasses(self): class P(Protocol): x = 1 @@ -6868,6 +6869,7 @@ def test_get_type_hints_modules_forwardref(self): 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) + @unittest.expectedFailure # TODO: RUSTPYTHON; test expects outdated result def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) @@ -10328,7 +10330,6 @@ def test_special_attrs(self): TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_attrs2(self): # Forward refs provide a different introspection API. __name__ and # __qualname__ make little sense for forward refs as they can store diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 790c4d29ca..4d3cfd60b8 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1297,7 +1297,6 @@ def _makeResult(self): expected = ['startTestRun', 'stopTestRun'] self.assertEqual(events, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle_unpickle(self): # Issue #7197: a TextTestRunner should be (un)pickleable. This is # required by test_multiprocessing under Windows (in verbose mode). diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 0a75457ad8..c9f7b18340 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -747,15 +747,11 @@ def test_baddecompresscopy(self): self.assertRaises(ValueError, copy.copy, d) self.assertRaises(ValueError, copy.deepcopy, d) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): @@ -1006,8 +1002,6 @@ def testDecompress4G(self, size): compressed = None decompressed = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def testPickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b583..e1a1792cb4 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/typing.py b/Lib/typing.py index a7397356d6..77caac9eed 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -161,7 +161,17 @@ ] -def _type_convert(arg, module=None, *, allow_special_forms=False): +class _LazyAnnotationLib: + def __getattr__(self, attr): + global _lazy_annotationlib + import annotationlib + _lazy_annotationlib = annotationlib + return getattr(annotationlib, attr) + +_lazy_annotationlib = _LazyAnnotationLib() + + +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) @@ -170,7 +180,7 @@ def _type_convert(arg, module=None, *, allow_special_forms=False): return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -188,7 +198,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -1918,6 +1928,7 @@ class _TypingEllipsis: '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', '__annotate_func__', '__annotations_cache__', }) # These special attributes will be not collected as protocol members. @@ -2442,7 +2453,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = _lazy_annotationlib.get_annotations(base) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2476,7 +2487,10 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) + try: + hints = _lazy_annotationlib.get_annotations(obj) + except TypeError: + hints = getattr(obj, '__annotations__', None) if hints is None: # Return empty annotations for something that _could_ have them. if isinstance(obj, _allowed_types): @@ -2992,7 +3006,8 @@ def _make_nmtuple(name, types, module, defaults = ()): '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__', + '__annotate_func__', '__annotations_cache__'}) class NamedTupleMeta(type): @@ -3003,7 +3018,13 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif (annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = _lazy_annotationlib.call_annotate_function( + annotate, _lazy_annotationlib.Format.VALUE) + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -3158,16 +3179,26 @@ def __new__(cls, name, bases, ns, total=True): else: generic_base = () + ns_annotations = ns.pop('__annotations__', None) + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} - own_annotations = ns.get('__annotations__', {}) + if ns_annotations is not None: + own_annotate = None + own_annotations = ns_annotations + elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = _lazy_annotationlib.call_annotate_function( + own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + own_checked_annotations = { + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() @@ -3176,8 +3207,6 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -3189,8 +3218,7 @@ def __new__(cls, name, bases, ns, total=True): readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): + for annotation_key, annotation_type in own_checked_annotations.items(): qualifiers = set(_get_typeddict_qualifiers(annotation_type)) if Required in qualifiers: is_required = True @@ -3221,7 +3249,36 @@ def __new__(cls, name, bases, ns, total=True): f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = _lazy_annotationlib.call_annotate_function( + base_annotate, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = _lazy_annotationlib.call_annotate_function( + own_annotate, format, owner=tp_dict) + if format != _lazy_annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == _lazy_annotationlib.Format.STRING: + own = _lazy_annotationlib.annotations_to_string(own_annotations) + elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + own = own_checked_annotations + else: + raise NotImplementedError(format) + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) tp_dict.__readonly_keys__ = frozenset(readonly_keys) diff --git a/Lib/venv/scripts/nt/venvlauncher.exe b/Lib/venv/scripts/nt/venvlauncher.exe index 2439c22aa9..c6863b56e5 100644 Binary files a/Lib/venv/scripts/nt/venvlauncher.exe and b/Lib/venv/scripts/nt/venvlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvlaunchert.exe b/Lib/venv/scripts/nt/venvlaunchert.exe index 99f5f5e9fc..c12a7a869f 100644 Binary files a/Lib/venv/scripts/nt/venvlaunchert.exe and b/Lib/venv/scripts/nt/venvlaunchert.exe differ diff --git a/Lib/venv/scripts/nt/venvwlauncher.exe b/Lib/venv/scripts/nt/venvwlauncher.exe index 6c43c2e9d9..d0d3733266 100644 Binary files a/Lib/venv/scripts/nt/venvwlauncher.exe and b/Lib/venv/scripts/nt/venvwlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvwlaunchert.exe b/Lib/venv/scripts/nt/venvwlaunchert.exe index 74f40deb04..9456a9e9b4 100644 Binary files a/Lib/venv/scripts/nt/venvwlaunchert.exe and b/Lib/venv/scripts/nt/venvwlaunchert.exe differ diff --git a/README.md b/README.md index b3ddbe4e7c..c4f7bfb1d6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.13.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 0480f94b67..d479c0d0e6 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -24,13 +24,13 @@ use num_traits::{Num, ToPrimitive}; use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext, - ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, - FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement, + ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTString, ExprTuple, + ExprUnaryOp, FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement, InterpolatedStringElements, Keyword, MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, PatternMatchClass, PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, - Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, - TypeParams, UnaryOp, WithItem, + Stmt, StmtAnnAssign, StmtExpr, TString, TypeParam, TypeParamParamSpec, TypeParamTypeVar, + TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, visitor::{Visitor, walk_expr}, }; use ruff_text_size::{Ranged, TextRange}; @@ -130,6 +130,10 @@ struct Compiler { ctx: CompileContext, opts: CompileOpts, in_annotation: bool, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, + // PEP 649: Next index for conditional annotation tracking + next_conditional_annotation_index: u32, } enum DoneWithFuture { @@ -437,6 +441,8 @@ impl Compiler { }, opts, in_annotation: false, + in_conditional_block: false, + next_conditional_annotation_index: 0, } } @@ -676,6 +682,62 @@ impl Compiler { Ok(self.current_symbol_table()) } + /// Push the annotation symbol table from the next sub_table's annotation_block + /// The annotation_block is stored in the function's scope, which is the next sub_table + /// Returns true if annotation_block exists, false otherwise + fn push_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // The annotation_block is in the next sub_table (function scope) + let next_idx = current_table.next_sub_table; + if next_idx >= current_table.sub_tables.len() { + return false; + } + + let next_table = &mut current_table.sub_tables[next_idx]; + if let Some(annotation_block) = next_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Push the annotation symbol table for module/class level annotations + /// This takes annotation_block from the current symbol table (not sub_tables) + fn push_current_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // For modules/classes, annotation_block is directly in the current table + if let Some(annotation_block) = current_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Pop the annotation symbol table and restore it to the function scope's annotation_block + fn pop_annotation_symbol_table(&mut self) { + let annotation_table = self.symbol_table_stack.pop().expect("compiler bug"); + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // Restore to the next sub_table (function scope) where it came from + let next_idx = current_table.next_sub_table; + if next_idx < current_table.sub_tables.len() { + current_table.sub_tables[next_idx].annotation_block = Some(Box::new(annotation_table)); + } + } + /// Pop the current symbol table off the stack fn pop_symbol_table(&mut self) -> SymbolTable { self.symbol_table_stack.pop().expect("compiler bug") @@ -896,6 +958,12 @@ impl Compiler { cellvar_cache.insert("__classdict__".to_string()); } + // Handle implicit __conditional_annotations__ cell if needed + // Only for class scope - module scope uses NAME operations, not DEREF + if ste.has_conditional_annotations && scope_type == CompilerScope::Class { + cellvar_cache.insert("__conditional_annotations__".to_string()); + } + // Build freevars using dictbytype (FREE scope, offset by cellvars size) let mut freevar_cache = IndexSet::default(); let mut free_names: Vec<_> = ste @@ -933,6 +1001,12 @@ impl Compiler { 0, 0, ), + CompilerScope::Annotation => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, + 1, // annotation scope takes one argument (format) + 0, + ), }; // Get private name from parent scope @@ -1060,6 +1134,81 @@ impl Compiler { unwrap_internal(self, stack_top.finalize_code(&self.opts)) } + /// Exit annotation scope - similar to exit_scope but restores annotation_block to parent + fn exit_annotation_scope(&mut self) -> CodeObject { + self.pop_annotation_symbol_table(); + + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Enter annotation scope using the symbol table's annotation_block + /// Returns false if no annotation_block exists + fn enter_annotation_scope(&mut self, func_name: &str) -> CompileResult { + if !self.push_annotation_symbol_table() { + return Ok(false); + } + + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + let annotate_name = format!(""); + + self.enter_scope( + &annotate_name, + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Override arg_count since enter_scope sets it to 1 but we need the varnames + // setup to be correct too + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) + self.emit_format_validation()?; + + Ok(true) + } + + /// Emit format parameter validation for annotation scope + /// if format > VALUE_WITH_FAKE_GLOBALS (2): raise NotImplementedError + fn emit_format_validation(&mut self) -> CompileResult<()> { + use bytecode::ComparisonOperator::Greater; + + // Load format parameter (first local variable, index 0) + emit!(self, Instruction::LoadFast(0)); + + // Load VALUE_WITH_FAKE_GLOBALS constant (2) + self.emit_load_const(ConstantData::Integer { value: 2.into() }); + + // Compare: format > 2 + emit!(self, Instruction::CompareOp { op: Greater }); + + // Jump to body if format <= 2 (comparison is false) + let body_block = self.new_block(); + emit!(self, Instruction::PopJumpIfFalse { target: body_block }); + + // Raise NotImplementedError + let not_implemented_error = self.name("NotImplementedError"); + emit!(self, Instruction::LoadGlobal(not_implemented_error)); + emit!( + self, + Instruction::RaiseVarargs { + kind: bytecode::RaiseKind::Raise + } + ); + + // Body label - continue with annotation evaluation + self.switch_to_block(body_block); + + Ok(()) + } + /// Push a new fblock // = compiler_push_fblock fn push_fblock( @@ -1495,6 +1644,8 @@ impl Compiler { symbol_table: SymbolTable, ) -> CompileResult<()> { let size_before = self.code_stack.len(); + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); let (doc, statements) = split_doc(&body.body, &self.opts); @@ -1506,10 +1657,24 @@ impl Compiler { emit!(self, Instruction::StoreGlobal(doc)) } + // Handle annotations based on future_annotations flag if Self::find_ann(statements) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(statements)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } + // Compile all statements self.compile_statements(statements)?; assert_eq!(self.code_stack.len(), size_before); @@ -1524,10 +1689,25 @@ impl Compiler { body: &[Stmt], symbol_table: SymbolTable, ) -> CompileResult<()> { + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + // Handle annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(body)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + } } if let Some((last, body)) = body.split_last() { @@ -1650,6 +1830,7 @@ impl Compiler { Global, Deref, Name, + DictOrGlobals, // PEP 649: can_see_class_scope } let name = self.mangle(name); @@ -1666,16 +1847,18 @@ impl Compiler { // Determine the operation type based on symbol scope let is_function_like = self.ctx.in_func(); - // Look up the symbol, handling TypeParams scope specially - let (symbol_scope, _is_typeparams) = { + // Look up the symbol, handling TypeParams and Annotation scopes specially + let (symbol_scope, can_see_class_scope) = { let current_table = self.current_symbol_table(); let is_typeparams = current_table.typ == CompilerScope::TypeParams; + let is_annotation = current_table.typ == CompilerScope::Annotation; + let can_see_class = current_table.can_see_class_scope; // First try to find in current table let symbol = current_table.lookup(name.as_ref()); - // If not found and we're in TypeParams scope, try parent scope - let symbol = if symbol.is_none() && is_typeparams { + // If not found and we're in TypeParams or Annotation scope, try parent scope + let symbol = if symbol.is_none() && (is_typeparams || is_annotation) { self.symbol_table_stack .get(self.symbol_table_stack.len() - 2) // Try to get parent index .expect("Symbol has no parent! This is a compiler bug.") @@ -1684,14 +1867,46 @@ impl Compiler { symbol }; - (symbol.map(|s| s.scope), is_typeparams) + (symbol.map(|s| s.scope), can_see_class) + }; + + // Special handling for class scope implicit cell variables + // These are treated as Cell even if not explicitly marked in symbol table + // Only for LOAD operations - explicit stores like `__class__ = property(...)` + // should use STORE_NAME to store in class namespace dict + let symbol_scope = { + let current_table = self.current_symbol_table(); + if current_table.typ == CompilerScope::Class + && usage == NameUsage::Load + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { + Some(SymbolScope::Cell) + } else { + symbol_scope + } }; - let actual_scope = symbol_scope.ok_or_else(|| { - self.error(CodegenErrorType::SyntaxError(format!( - "The symbol '{name}' must be present in the symbol table" - ))) - })?; + // In annotation or type params scope, missing symbols are treated as global implicit + // This allows referencing global names like Union, Optional, etc. that are imported + // at module level but not explicitly bound in the function scope + let actual_scope = match symbol_scope { + Some(scope) => scope, + None => { + let current_table = self.current_symbol_table(); + if matches!( + current_table.typ, + CompilerScope::Annotation | CompilerScope::TypeParams + ) { + SymbolScope::GlobalImplicit + } else { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "the symbol '{name}' must be present in the symbol table" + )))); + } + } + }; // Determine operation type based on scope let op_type = match actual_scope { @@ -1705,7 +1920,11 @@ impl Compiler { } } SymbolScope::GlobalImplicit => { - if is_function_like { + // PEP 649: In annotation scope with class visibility, use DictOrGlobals + // to check classdict first before globals + if can_see_class_scope { + NameOp::DictOrGlobals + } else if is_function_like { NameOp::Global } else { NameOp::Name @@ -1765,6 +1984,25 @@ impl Compiler { }; self.emit_arg(idx, op); } + NameOp::DictOrGlobals => { + // PEP 649: First check classdict (from __classdict__ freevar), then globals + let idx = self.get_global_name_index(&name); + match usage { + NameUsage::Load => { + // Load __classdict__ first (it's a free variable in annotation scope) + let classdict_idx = self.get_free_var_index("__classdict__")?; + self.emit_arg(classdict_idx, Instruction::LoadDeref); + self.emit_arg(idx, Instruction::LoadFromDictOrGlobals); + } + // Store/Delete in annotation scope should use Name ops + NameUsage::Store => { + self.emit_arg(idx, Instruction::StoreName); + } + NameUsage::Delete => { + self.emit_arg(idx, Instruction::DeleteName); + } + } + } } Ok(()) @@ -2117,8 +2355,9 @@ impl Compiler { target, annotation, value, + simple, .. - }) => self.compile_annotated_assign(target, annotation, value.as_deref())?, + }) => self.compile_annotated_assign(target, annotation, value.as_deref(), *simple)?, Stmt::Delete(StmtDelete { targets, .. }) => { for target in targets { self.compile_delete(target)?; @@ -3306,12 +3545,19 @@ impl Compiler { // Set qualname self.set_qualname(); - // Handle docstring + // Handle docstring - store in co_consts[0] if present let (doc_str, body) = split_doc(body, &self.opts); - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + if let Some(doc) = &doc_str { + // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag + self.current_code_info() + .metadata + .consts + .insert_full(ConstantData::Str { + value: doc.to_string().into(), + }); + self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; + } + // If no docstring, don't add None to co_consts // Compile body statements self.compile_statements(body)?; @@ -3331,30 +3577,41 @@ impl Compiler { // Create function object with closure self.make_closure(code, funcflags)?; - // Handle docstring if present - if let Some(doc) = doc_str { - emit!(self, Instruction::Copy { index: 1_u32 }); - self.emit_load_const(ConstantData::Str { - value: doc.to_string().into(), - }); - emit!(self, Instruction::Swap { index: 2 }); - let doc_attr = self.name("__doc__"); - emit!(self, Instruction::StoreAttr { idx: doc_attr }); - } + // Note: docstring is now retrieved from co_consts[0] by the VM + // when HAS_DOCSTRING flag is set, so no runtime __doc__ assignment needed Ok(()) } - /// Compile function annotations - // = compiler_visit_annotations - fn visit_annotations( + /// Compile function annotations as a closure (PEP 649) + /// Returns true if an __annotate__ closure was created + /// Uses symbol table's annotation_block for proper scoping. + fn compile_annotations_closure( &mut self, + func_name: &str, parameters: &Parameters, returns: Option<&Expr>, - ) -> CompileResult { - let mut num_annotations = 0; + ) -> CompileResult { + // Try to enter annotation scope - returns false if no annotation_block exists + if !self.enter_annotation_scope(func_name)? { + return Ok(false); + } + + // Count annotations + let parameters_iter = core::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .chain(¶meters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + + let num_annotations: u32 = + u32::try_from(parameters_iter.filter(|p| p.annotation.is_some()).count()) + .expect("too many annotations") + + if returns.is_some() { 1 } else { 0 }; - // Handle parameter annotations + // Compile annotations inside the annotation scope let parameters_iter = core::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) @@ -3369,20 +3626,193 @@ impl Compiler { value: self.mangle(param.name.as_str()).into_owned().into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } } - // Handle return annotation last + // Handle return annotation if let Some(annotation) = returns { self.emit_load_const(ConstantData::Str { value: "return".into(), }); self.compile_annotation(annotation)?; - num_annotations += 1; } - Ok(num_annotations) + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + + // Exit the annotation scope and get the code object + let annotate_code = self.exit_annotation_scope(); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + Ok(true) + } + + /// Collect simple (non-conditional) annotations from module body + /// Returns list of (name, annotation_expr) pairs + fn collect_simple_annotations(body: &[Stmt]) -> Vec<(&str, &Expr)> { + let mut annotations = Vec::new(); + for stmt in body { + if let Stmt::AnnAssign(StmtAnnAssign { + target, + annotation, + simple, + .. + }) = stmt + && *simple + && let Expr::Name(ExprName { id, .. }) = target.as_ref() + { + annotations.push((id.as_str(), annotation.as_ref())); + } + } + annotations + } + + /// Compile module-level __annotate__ function (PEP 649) + /// Returns true if __annotate__ was created and stored + fn compile_module_annotate(&mut self, body: &[Stmt]) -> CompileResult { + // Collect simple annotations from module body first + let annotations = Self::collect_simple_annotations(body); + + if annotations.is_empty() { + return Ok(false); + } + + // Check if we have conditional annotations + let has_conditional = self.current_symbol_table().has_conditional_annotations; + + // Get parent scope type and name BEFORE pushing annotation symbol table + let parent_scope_type = self.current_symbol_table().typ; + let parent_name = self + .symbol_table_stack + .last() + .map(|t| t.name.as_str()) + .unwrap_or("module") + .to_owned(); + let scope_name = format!(""); + + // Try to push annotation symbol table from current scope + if !self.push_current_annotation_symbol_table() { + return Ok(false); + } + + // Enter annotation scope for code generation + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope(&scope_name, CompilerScope::Annotation, key, lineno.to_u32())?; + + // Add 'format' parameter to varnames + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + self.emit_format_validation()?; + + if has_conditional { + // PEP 649: Build dict incrementally, checking conditional annotations + // Start with empty dict + emit!(self, Instruction::BuildMap { size: 0 }); + + // Process each annotation + for (idx, (name, annotation)) in annotations.iter().enumerate() { + // Check if index is in __conditional_annotations__ + let not_set_block = self.new_block(); + + // LOAD_CONST index + self.emit_load_const(ConstantData::Integer { value: idx.into() }); + // Load __conditional_annotations__ from appropriate scope + // Class scope: LoadDeref (freevars), Module scope: LoadGlobal + if parent_scope_type == CompilerScope::Class { + let idx = self.get_free_var_index("__conditional_annotations__")?; + emit!(self, Instruction::LoadDeref(idx)); + } else { + let cond_annotations_name = self.name("__conditional_annotations__"); + emit!(self, Instruction::LoadGlobal(cond_annotations_name)); + } + // CONTAINS_OP (in) + emit!(self, Instruction::ContainsOp(bytecode::Invert::No)); + // POP_JUMP_IF_FALSE not_set + emit!( + self, + Instruction::PopJumpIfFalse { + target: not_set_block + } + ); + + // Annotation value + self.compile_annotation(annotation)?; + // COPY dict to TOS + emit!(self, Instruction::Copy { index: 2 }); + // LOAD_CONST name + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + // STORE_SUBSCR - dict[name] = value + emit!(self, Instruction::StoreSubscr); + + // not_set label + self.switch_to_block(not_set_block); + } + + // Return the dict + emit!(self, Instruction::ReturnValue); + } else { + // No conditional annotations - use simple BuildMap + let num_annotations = u32::try_from(annotations.len()).expect("too many annotations"); + + // Compile annotations inside the annotation scope + for (name, annotation) in annotations { + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + self.compile_annotation(annotation)?; + } + + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + } + + // Exit annotation scope - pop symbol table, restore to parent's annotation_block, and get code + let annotation_table = self.pop_symbol_table(); + // Restore annotation_block to module's symbol table + self.symbol_table_stack + .last_mut() + .expect("no module symbol table") + .annotation_block = Some(Box::new(annotation_table)); + // Exit code scope + let pop = self.code_stack.pop(); + let annotate_code = unwrap_internal( + self, + compiler_unwrap_option(self, pop).finalize_code(&self.opts), + ); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; + + // Store as __annotate_func__ for classes, __annotate__ for modules + let name = if parent_scope_type == CompilerScope::Class { + "__annotate_func__" + } else { + "__annotate__" + }; + self.store_name(name)?; + + Ok(true) } // = compiler_function @@ -3449,18 +3879,12 @@ impl Compiler { } } - // Compile annotations - let mut annotations_flag = bytecode::MakeFunctionFlags::empty(); - let num_annotations = self.visit_annotations(parameters, returns)?; - if num_annotations > 0 { - annotations_flag = bytecode::MakeFunctionFlags::ANNOTATIONS; - emit!( - self, - Instruction::BuildMap { - size: num_annotations, - } - ); - } + // Compile annotations as closure (PEP 649) + let annotations_flag = if self.compile_annotations_closure(name, parameters, returns)? { + bytecode::MakeFunctionFlags::ANNOTATE + } else { + bytecode::MakeFunctionFlags::empty() + }; // Compile function body let final_funcflags = funcflags | annotations_flag; @@ -3543,10 +3967,14 @@ impl Compiler { fn get_ref_type(&self, name: &str) -> Result { let table = self.symbol_table_stack.last().unwrap(); - // Special handling for __class__ and __classdict__ in class scope + // Special handling for __class__, __classdict__, and __conditional_annotations__ in class scope // This should only apply when we're actually IN a class body, // not when we're in a method nested inside a class. - if table.typ == CompilerScope::Class && (name == "__class__" || name == "__classdict__") { + if table.typ == CompilerScope::Class + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { return Ok(SymbolScope::Cell); } match table.lookup(name) { @@ -3669,6 +4097,16 @@ impl Compiler { ); } + // Set __annotate__ closure if present (PEP 649) + if flags.contains(bytecode::MakeFunctionFlags::ANNOTATE) { + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::ANNOTATE + } + ); + } + // Set kwdefaults if present if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { emit!( @@ -3799,9 +4237,31 @@ impl Compiler { emit!(self, Instruction::StoreName(dunder_type_params)); } - // Setup annotations if needed + // PEP 649: Initialize __classdict__ cell for class annotation scope + if self.current_symbol_table().needs_classdict { + let locals_name = self.name("locals"); + emit!(self, Instruction::LoadName(locals_name)); + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { nargs: 0 }); + let classdict_idx = self.get_cell_var_index("__classdict__")?; + emit!(self, Instruction::StoreDeref(classdict_idx)); + } + + // Handle class annotations based on future_annotations flag if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotations); + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict for class + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Initialize __conditional_annotations__ set if needed for class + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { size: 0 }); + self.store_name("__conditional_annotations__")?; + } + + // PEP 649: Generate __annotate__ function for class annotations + self.compile_module_annotate(body)?; + } } // 3. Compile the class body @@ -5457,31 +5917,55 @@ impl Compiler { target: &Expr, annotation: &Expr, value: Option<&Expr>, + simple: bool, ) -> CompileResult<()> { + // Perform the actual assignment first if let Some(value) = value { self.compile_expression(value)?; self.compile_store(target)?; } - // Annotations are only evaluated in a module or class. - if self.ctx.in_func() { - return Ok(()); - } - - // Compile annotation: - self.compile_annotation(annotation)?; - - if let Expr::Name(ExprName { id, .. }) = &target { - // Store as dict entry in __annotations__ dict: - let annotations = self.name("__annotations__"); - emit!(self, Instruction::LoadName(annotations)); - self.emit_load_const(ConstantData::Str { - value: self.mangle(id.as_str()).into_owned().into(), - }); - emit!(self, Instruction::StoreSubscr); - } else { - // Drop annotation if not assigned to simple identifier. - emit!(self, Instruction::PopTop); + // If we have a simple name in module or class scope, store annotation + if simple + && !self.ctx.in_func() + && let Expr::Name(ExprName { id, .. }) = target + { + if self.future_annotations { + // PEP 563: Store stringified annotation directly to __annotations__ + // Compile annotation as string + self.compile_annotation(annotation)?; + // Load __annotations__ + let annotations_name = self.name("__annotations__"); + emit!(self, Instruction::LoadName(annotations_name)); + // Load the variable name + self.emit_load_const(ConstantData::Str { + value: self.mangle(id.as_str()).into_owned().into(), + }); + // Store: __annotations__[name] = annotation + emit!(self, Instruction::StoreSubscr); + } else { + // PEP 649: Handle conditional annotations + if self.current_symbol_table().has_conditional_annotations { + // Determine if this annotation is conditional + let is_module = self.current_symbol_table().typ == CompilerScope::Module; + let is_conditional = is_module || self.in_conditional_block; + + if is_conditional { + // Get the current annotation index and increment + let annotation_index = self.next_conditional_annotation_index; + self.next_conditional_annotation_index += 1; + + // Add index to __conditional_annotations__ set + let cond_annotations_name = self.name("__conditional_annotations__"); + emit!(self, Instruction::LoadName(cond_annotations_name)); + self.emit_load_const(ConstantData::Integer { + value: annotation_index.into(), + }); + emit!(self, Instruction::SetAdd { i: 0_u32 }); + emit!(self, Instruction::PopTop); + } + } + } } Ok(()) @@ -6100,10 +6584,7 @@ impl Compiler { in_async_scope: false, }; - self.current_code_info() - .metadata - .consts - .insert_full(ConstantData::None); + // Lambda cannot have docstrings, so no None is added to co_consts self.compile_expression(body)?; self.emit_return_value(); @@ -6282,8 +6763,8 @@ impl Compiler { Expr::FString(fstring) => { self.compile_expr_fstring(fstring)?; } - Expr::TString(_) => { - return Err(self.error(CodegenErrorType::NotImplementedYet)); + Expr::TString(tstring) => { + self.compile_expr_tstring(tstring)?; } Expr::StringLiteral(string) => { let value = string.value.to_str(); @@ -7466,6 +7947,114 @@ impl Compiler { Ok(()) } + + fn compile_expr_tstring(&mut self, expr_tstring: &ExprTString) -> CompileResult<()> { + // TStringValue can contain multiple TString parts (implicit concatenation) + // Each TString part should be compiled and the results merged into a single Template + let tstring_value = &expr_tstring.value; + + // Collect all strings and compile all interpolations + let mut all_strings: Vec = Vec::new(); + let mut current_string = Wtf8Buf::new(); + let mut interp_count: u32 = 0; + + for tstring in tstring_value.iter() { + self.compile_tstring_into( + tstring, + &mut all_strings, + &mut current_string, + &mut interp_count, + )?; + } + + // Add trailing string + all_strings.push(std::mem::take(&mut current_string)); + + // Now build the Template: + // Stack currently has all interpolations from compile_tstring_into calls + + // 1. Build interpolations tuple from the interpolations on the stack + emit!(self, Instruction::BuildTuple { size: interp_count }); + + // 2. Load all string parts + let string_count: u32 = all_strings + .len() + .try_into() + .expect("t-string string count overflowed"); + for s in &all_strings { + self.emit_load_const(ConstantData::Str { value: s.clone() }); + } + + // 3. Build strings tuple + emit!(self, Instruction::BuildTuple { size: string_count }); + + // 4. Swap so strings is below interpolations: [interps, strings] -> [strings, interps] + emit!(self, Instruction::Swap { index: 2 }); + + // 5. Build the Template + emit!(self, Instruction::BuildTemplate); + + Ok(()) + } + + fn compile_tstring_into( + &mut self, + tstring: &TString, + strings: &mut Vec, + current_string: &mut Wtf8Buf, + interp_count: &mut u32, + ) -> CompileResult<()> { + for element in &tstring.elements { + match element { + InterpolatedStringElement::Literal(lit) => { + // Accumulate literal parts into current_string + current_string.push_str(&lit.value); + } + InterpolatedStringElement::Interpolation(interp) => { + // Finish current string segment + strings.push(std::mem::take(current_string)); + + // Compile the interpolation value + self.compile_expression(&interp.expression)?; + + // Load the expression source string + let expr_range = interp.expression.range(); + let expr_source = self.source_file.slice(expr_range); + self.emit_load_const(ConstantData::Str { + value: expr_source.to_string().into(), + }); + + // Determine conversion code + let conversion: u32 = match interp.conversion { + ConversionFlag::None => 0, + ConversionFlag::Str => 1, + ConversionFlag::Repr => 2, + ConversionFlag::Ascii => 3, + }; + + // Handle format_spec + let has_format_spec = interp.format_spec.is_some(); + if let Some(format_spec) = &interp.format_spec { + // Compile format_spec as a string using fstring element compilation + // Use default FStringFlags since format_spec syntax is independent of t-string flags + self.compile_fstring_elements( + FStringFlags::empty(), + &format_spec.elements, + )?; + } + + // Emit BUILD_INTERPOLATION + // oparg encoding: (conversion << 2) | has_format_spec + let oparg = (conversion << 2) | (has_format_spec as u32); + emit!(self, Instruction::BuildInterpolation { oparg }); + + *interp_count += 1; + } + } + } + + Ok(()) + } } trait EmitArg { diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 22b5bf358a..0bc76b897d 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -63,6 +63,17 @@ pub struct SymbolTable { /// Whether this comprehension scope should be inlined (PEP 709) /// True for list/set/dict comprehensions in non-generator expressions pub comp_inlined: bool, + + /// PEP 649: Reference to annotation scope for this block + /// Annotations are compiled as a separate `__annotate__` function + pub annotation_block: Option>, + + /// PEP 649: Whether this scope has conditional annotations + /// (annotations inside if/for/while/etc. blocks or at module level) + pub has_conditional_annotations: bool, + + /// Whether `from __future__ import annotations` is active + pub future_annotations: bool, } impl SymbolTable { @@ -80,6 +91,9 @@ impl SymbolTable { needs_classdict: false, can_see_class_scope: false, comp_inlined: false, + annotation_block: None, + has_conditional_annotations: false, + future_annotations: false, } } @@ -109,6 +123,8 @@ pub enum CompilerScope { Lambda, Comprehension, TypeParams, + /// PEP 649: Annotation scope for deferred evaluation + Annotation, } impl fmt::Display for CompilerScope { @@ -121,9 +137,8 @@ impl fmt::Display for CompilerScope { Self::Lambda => write!(f, "lambda"), Self::Comprehension => write!(f, "comprehension"), Self::TypeParams => write!(f, "type parameter"), + Self::Annotation => write!(f, "annotation"), // TODO missing types from the C implementation - // if self._table.type == _symtable.TYPE_ANNOTATION: - // return "annotation" // if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: // return "TypeVar bound" // if self._table.type == _symtable.TYPE_TYPE_ALIAS: @@ -246,7 +261,8 @@ impl core::fmt::Debug for SymbolTable { fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { let mut analyzer = SymbolTableAnalyzer::default(); // Discard the newfree set at the top level - it's only needed for propagation - let _newfree = analyzer.analyze_symbol_table(symbol_table)?; + // Pass None for class_entry at top level + let _newfree = analyzer.analyze_symbol_table(symbol_table, None)?; Ok(()) } @@ -269,6 +285,12 @@ fn drop_class_free(symbol_table: &mut SymbolTable, newfree: &mut HashSet if newfree.remove("__classdict__") { symbol_table.needs_classdict = true; } + + // Check if __conditional_annotations__ is in the free variables collected from children + // Remove it from free set - it's handled specially in class scope + if newfree.remove("__conditional_annotations__") { + symbol_table.has_conditional_annotations = true; + } } type SymbolMap = IndexMap; @@ -339,9 +361,11 @@ struct SymbolTableAnalyzer { impl SymbolTableAnalyzer { /// Analyze a symbol table and return the set of free variables. /// See symtable.c analyze_block(). + /// class_entry: PEP 649 - enclosing class symbols for annotation scopes fn analyze_symbol_table( &mut self, symbol_table: &mut SymbolTable, + class_entry: Option<&SymbolMap>, ) -> SymbolTableResult> { let symbols = core::mem::take(&mut symbol_table.symbols); let sub_tables = &mut *symbol_table.sub_tables; @@ -349,15 +373,43 @@ impl SymbolTableAnalyzer { // Collect free variables from all child scopes let mut newfree = HashSet::new(); + let annotation_block = &mut symbol_table.annotation_block; + + // PEP 649: Determine class_entry to pass to children + // If current scope is a class with annotation block that can_see_class_scope, + // we need to pass class symbols to the annotation scope + let is_class = symbol_table.typ == CompilerScope::Class; + + // Clone class symbols if needed for annotation scope (to avoid borrow conflict) + let class_symbols_for_ann = if is_class + && annotation_block + .as_ref() + .is_some_and(|b| b.can_see_class_scope) + { + Some(symbols.clone()) + } else { + None + }; + let mut info = (symbols, symbol_table.typ); self.tables.with_append(&mut info, |list| { let inner_scope = unsafe { &mut *(list as *mut _ as *mut Self) }; // Analyze sub scopes and collect their free variables for sub_table in sub_tables.iter_mut() { - let child_free = inner_scope.analyze_symbol_table(sub_table)?; + // Sub-scopes (functions, nested classes) don't inherit class_entry + let child_free = inner_scope.analyze_symbol_table(sub_table, None)?; // Propagate child's free variables to this scope newfree.extend(child_free); } + // PEP 649: Analyze annotation block if present + if let Some(annotation_table) = annotation_block { + // Pass class symbols to annotation scope if can_see_class_scope + let ann_class_entry = class_symbols_for_ann.as_ref().or(class_entry); + let child_free = + inner_scope.analyze_symbol_table(annotation_table, ann_class_entry)?; + // Propagate annotation's free variables to this scope + newfree.extend(child_free); + } Ok(()) })?; @@ -396,7 +448,7 @@ impl SymbolTableAnalyzer { // Analyze symbols in current scope for symbol in symbol_table.symbols.values_mut() { - self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?; + self.analyze_symbol(symbol, symbol_table.typ, sub_tables, class_entry)?; // Collect free variables from this scope // These will be propagated to the parent scope @@ -420,6 +472,7 @@ impl SymbolTableAnalyzer { symbol: &mut Symbol, st_typ: CompilerScope, sub_tables: &[SymbolTable], + class_entry: Option<&SymbolMap>, ) -> SymbolTableResult { if symbol .flags @@ -439,7 +492,8 @@ impl SymbolTableAnalyzer { // check if the name is already defined in any outer scope // therefore if scope_depth < 2 - || self.found_in_outer_scope(&symbol.name) != Some(SymbolScope::Free) + || self.found_in_outer_scope(&symbol.name, st_typ) + != Some(SymbolScope::Free) { return Err(SymbolTableError { error: format!("no binding for nonlocal '{}' found", symbol.name), @@ -465,11 +519,23 @@ impl SymbolTableAnalyzer { // all is well } SymbolScope::Unknown => { + // PEP 649: Check class_entry first (like analyze_name) + // If name is bound in enclosing class, mark as GlobalImplicit + if let Some(class_symbols) = class_entry + && let Some(class_sym) = class_symbols.get(&symbol.name) + { + // DEF_BOUND && !DEF_NONLOCAL -> GLOBAL_IMPLICIT + if class_sym.is_bound() && class_sym.scope != SymbolScope::Free { + symbol.scope = SymbolScope::GlobalImplicit; + return Ok(()); + } + } + // Try hard to figure out what the scope of this symbol is. let scope = if symbol.is_bound() { self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) .unwrap_or(SymbolScope::Local) - } else if let Some(scope) = self.found_in_outer_scope(&symbol.name) { + } else if let Some(scope) = self.found_in_outer_scope(&symbol.name, st_typ) { scope } else if self.tables.is_empty() { // Don't make assumptions when we don't know. @@ -485,18 +551,40 @@ impl SymbolTableAnalyzer { Ok(()) } - fn found_in_outer_scope(&mut self, name: &str) -> Option { + fn found_in_outer_scope(&mut self, name: &str, st_typ: CompilerScope) -> Option { let mut decl_depth = None; for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { if matches!(typ, CompilerScope::Module) - || matches!(typ, CompilerScope::Class if name != "__class__") + || matches!(typ, CompilerScope::Class if name != "__class__" && name != "__classdict__" && name != "__conditional_annotations__") + { + continue; + } + + // PEP 649: Annotation scope is conceptually a sibling of the function, + // not a child. Skip the immediate parent function scope when looking + // for outer variables from annotation scope. + if st_typ == CompilerScope::Annotation + && i == 0 + && matches!( + typ, + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda + ) { continue; } - // __class__ is implicitly declared in class scope - // This handles the case where super() is called in a nested class method - if name == "__class__" && matches!(typ, CompilerScope::Class) { + // __class__ and __classdict__ are implicitly declared in class scope + // This handles the case where nested scopes reference them + if (name == "__class__" || name == "__classdict__") + && matches!(typ, CompilerScope::Class) + { + decl_depth = Some(i); + break; + } + + // __conditional_annotations__ is implicitly declared in class scope + // for classes with conditional annotations + if name == "__conditional_annotations__" && matches!(typ, CompilerScope::Class) { decl_depth = Some(i); break; } @@ -657,6 +745,13 @@ impl SymbolTableAnalyzer { location: None, }); } + CompilerScope::Annotation => { + // Named expression is not allowed in annotation scope + return Err(SymbolTableError { + error: "named expression cannot be used within an annotation".to_string(), + location: None, + }); + } } Ok(()) } @@ -695,6 +790,8 @@ struct SymbolTableBuilder { in_comp_inner_loop_target: bool, // Scope info for error messages (e.g., "a TypeVar bound") scope_info: Option<&'static str>, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, } /// Enum to indicate in what mode an expression @@ -723,6 +820,7 @@ impl SymbolTableBuilder { in_type_alias: false, in_comp_inner_loop_target: false, scope_info: None, + in_conditional_block: false, }; this.enter_scope("top", CompilerScope::Module, 0); this @@ -733,6 +831,8 @@ impl SymbolTableBuilder { let mut symbol_table = self.tables.pop().unwrap(); // Save varnames for the top-level module scope symbol_table.varnames = self.current_varnames; + // Propagate future_annotations to the symbol table + symbol_table.future_annotations = self.future_annotations; analyze_symbol_table(&mut symbol_table)?; Ok(symbol_table) } @@ -782,6 +882,79 @@ impl SymbolTableBuilder { self.tables.last_mut().unwrap().sub_tables.push(table); } + /// Enter annotation scope (PEP 649) + /// Creates or reuses the annotation block for the current scope + fn enter_annotation_scope(&mut self, line_number: u32) { + let current = self.tables.last_mut().unwrap(); + let can_see_class_scope = current.typ == CompilerScope::Class; + let has_conditional = current.has_conditional_annotations; + + // Create annotation block if not exists + if current.annotation_block.is_none() { + let mut annotation_table = SymbolTable::new( + "__annotate__".to_owned(), + CompilerScope::Annotation, + line_number, + true, // is_nested + ); + // Annotation scope in class can see class scope + annotation_table.can_see_class_scope = can_see_class_scope; + // Add 'format' parameter + annotation_table.varnames.push("format".to_owned()); + current.annotation_block = Some(Box::new(annotation_table)); + } + + // Take the annotation block and push to stack for processing + let annotation_table = current.annotation_block.take().unwrap(); + self.tables.push(*annotation_table); + self.current_varnames.clear(); + + if can_see_class_scope && !self.future_annotations { + self.add_classdict_freevar(); + // Also add __conditional_annotations__ as free var if parent has conditional annotations + if has_conditional { + self.add_conditional_annotations_freevar(); + } + } + } + + /// Leave annotation scope (PEP 649) + /// Stores the annotation block back to parent instead of sub_tables + fn leave_annotation_scope(&mut self) { + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = core::mem::take(&mut self.current_varnames); + // Store back to parent's annotation_block (not sub_tables) + let parent = self.tables.last_mut().unwrap(); + parent.annotation_block = Some(Box::new(table)); + } + + fn add_classdict_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__classdict__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + fn add_conditional_annotations_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__conditional_annotations__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + fn line_index_start(&self, range: TextRange) -> u32 { self.source_file .to_source_code() @@ -830,15 +1003,58 @@ impl SymbolTableBuilder { } fn scan_annotation(&mut self, annotation: &Expr) -> SymbolTableResult { + let current_scope = self.tables.last().map(|t| t.typ); + + // PEP 649: Check if this is a conditional annotation + // Module-level: always conditional (module may be partially executed) + // Class-level: conditional only when inside if/for/while/etc. + if !self.future_annotations { + let is_conditional = matches!(current_scope, Some(CompilerScope::Module)) + || (matches!(current_scope, Some(CompilerScope::Class)) + && self.in_conditional_block); + + if is_conditional && !self.tables.last().unwrap().has_conditional_annotations { + self.tables.last_mut().unwrap().has_conditional_annotations = true; + // Register __conditional_annotations__ symbol in the scope (USE flag, not DEF) + self.register_name( + "__conditional_annotations__", + SymbolUsage::Used, + annotation.range(), + )?; + } + } + + // Create annotation scope for deferred evaluation + let line_number = self.line_index_start(annotation.range()); + self.enter_annotation_scope(line_number); + if self.future_annotations { - Ok(()) - } else { + // PEP 563: annotations are stringified at compile time + // Don't scan expression - symbols would fail to resolve + // Just create the annotation_block structure + self.leave_annotation_scope(); + return Ok(()); + } + + // PEP 649: scan expression for symbol references + // Class annotations are evaluated in class locals (not module globals) + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + let result = self.scan_expression(annotation, ExpressionContext::Load); + self.in_annotation = was_in_annotation; + + self.leave_annotation_scope(); + + // Module scope: re-scan to register symbols (builtins like str, int) + // Class scope: do NOT re-scan to preserve class-local symbol resolution + if matches!(current_scope, Some(CompilerScope::Module)) { let was_in_annotation = self.in_annotation; self.in_annotation = true; - let result = self.scan_expression(annotation, ExpressionContext::Load); + let _ = self.scan_expression(annotation, ExpressionContext::Load); self.in_annotation = was_in_annotation; - result } + + result } fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { @@ -873,9 +1089,23 @@ impl SymbolTableBuilder { }) => { self.scan_decorators(decorator_list, ExpressionContext::Load)?; self.register_ident(name, SymbolUsage::Assigned)?; - if let Some(expression) = returns { - self.scan_annotation(expression)?; - } + + // Save the parent's annotation_block before scanning function annotations, + // so function annotations don't interfere with parent scope annotations. + // This applies to both class scope (methods) and module scope (top-level functions). + let parent_scope_typ = self.tables.last().map(|t| t.typ); + let should_save_annotation_block = matches!( + parent_scope_typ, + Some(CompilerScope::Class) | Some(CompilerScope::Module) + ); + let saved_annotation_block = if should_save_annotation_block { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + // For generic functions, enter type_param block FIRST so that + // annotation scopes are nested inside and can see type parameters. if let Some(type_params) = type_params { self.enter_type_param_block( &format!("", name.as_str()), @@ -883,16 +1113,28 @@ impl SymbolTableBuilder { )?; self.scan_type_params(type_params)?; } + let has_return_annotation = if let Some(expression) = returns { + self.scan_annotation(expression)?; + true + } else { + false + }; self.enter_scope_with_parameters( name.as_str(), parameters, self.line_index_start(*range), + has_return_annotation, )?; self.scan_statements(body)?; self.leave_scope(); if type_params.is_some() { self.leave_scope(); } + + // Restore parent's annotation_block after processing the function + if let Some(block) = saved_annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } } Stmt::ClassDef(StmtClassDef { name, @@ -915,6 +1157,10 @@ impl SymbolTableBuilder { CompilerScope::Class, self.line_index_start(*range), ); + // Reset in_conditional_block for new class scope + // (each scope has its own conditional context) + let saved_in_conditional = self.in_conditional_block; + self.in_conditional_block = false; let prev_class = self.class_name.replace(name.to_string()); self.register_name("__module__", SymbolUsage::Assigned, *range)?; self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; @@ -922,6 +1168,7 @@ impl SymbolTableBuilder { self.register_name("__class__", SymbolUsage::Assigned, *range)?; self.scan_statements(body)?; self.leave_scope(); + self.in_conditional_block = saved_in_conditional; self.class_name = prev_class; if let Some(arguments) = arguments { self.scan_expressions(&arguments.args, ExpressionContext::Load)?; @@ -945,6 +1192,9 @@ impl SymbolTableBuilder { .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for elif in elif_else_clauses { if let Some(test) = &elif.test { @@ -952,6 +1202,7 @@ impl SymbolTableBuilder { } self.scan_statements(&elif.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::For(StmtFor { target, @@ -962,15 +1213,23 @@ impl SymbolTableBuilder { }) => { self.scan_expression(target, ExpressionContext::Store)?; self.scan_expression(iter, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::While(StmtWhile { test, body, orelse, .. }) => { self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { // No symbols here. @@ -1037,6 +1296,21 @@ impl SymbolTableBuilder { match &**target { Expr::Name(ast::ExprName { id, .. }) if *simple => { self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, *range)?; + // PEP 649: Register annotate function in module/class scope + let current_scope = self.tables.last().map(|t| t.typ); + match current_scope { + Some(CompilerScope::Module) => { + self.register_name("__annotate__", SymbolUsage::Assigned, *range)?; + } + Some(CompilerScope::Class) => { + self.register_name( + "__annotate_func__", + SymbolUsage::Assigned, + *range, + )?; + } + _ => {} + } } _ => { self.scan_expression(target, ExpressionContext::Store)?; @@ -1054,7 +1328,11 @@ impl SymbolTableBuilder { self.scan_expression(expression, ExpressionContext::Store)?; } } + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Try(StmtTry { body, @@ -1063,6 +1341,9 @@ impl SymbolTableBuilder { finalbody, .. }) => { + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; self.scan_statements(body)?; for handler in handlers { let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -1081,9 +1362,13 @@ impl SymbolTableBuilder { } self.scan_statements(orelse)?; self.scan_statements(finalbody)?; + self.in_conditional_block = saved_in_conditional_block; } Stmt::Match(StmtMatch { subject, cases, .. }) => { self.scan_expression(subject, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; for case in cases { self.scan_pattern(&case.pattern)?; if let Some(guard) = &case.guard { @@ -1091,6 +1376,7 @@ impl SymbolTableBuilder { } self.scan_statements(&case.body)?; } + self.in_conditional_block = saved_in_conditional_block; } Stmt::Raise(StmtRaise { exc, cause, .. }) => { if let Some(expression) = exc { @@ -1412,6 +1698,7 @@ impl SymbolTableBuilder { "lambda", parameters, self.line_index_start(expression.range()), + false, // lambdas have no return annotation )?; } else { self.enter_scope( @@ -1441,14 +1728,19 @@ impl SymbolTableBuilder { } } Expr::TString(tstring) => { - return Err(SymbolTableError { - error: "not yet implemented".into(), - location: Some( - self.source_file - .to_source_code() - .source_location(tstring.range.start(), PositionEncoding::Utf8), - ), - }); + // Scan t-string interpolation expressions (similar to f-strings) + for expr in tstring + .value + .elements() + .filter_map(|x| x.as_interpolation()) + { + self.scan_expression(&expr.expression, ExpressionContext::Load)?; + if let Some(format_spec) = &expr.format_spec { + for element in format_spec.elements.interpolations() { + self.scan_expression(&element.expression, ExpressionContext::Load)? + } + } + } } // Constants Expr::StringLiteral(_) @@ -1769,6 +2061,7 @@ impl SymbolTableBuilder { name: &str, parameters: &Parameters, line_number: u32, + has_return_annotation: bool, ) -> SymbolTableResult { // Evaluate eventual default parameters: for default in parameters @@ -1806,8 +2099,40 @@ impl SymbolTableBuilder { self.scan_annotation(annotation)?; } + // Check if this function has any annotations (parameter or return) + let has_param_annotations = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .any(|p| p.parameter.annotation.is_some()) + || parameters + .vararg + .as_ref() + .is_some_and(|p| p.annotation.is_some()) + || parameters + .kwarg + .as_ref() + .is_some_and(|p| p.annotation.is_some()); + + let has_any_annotations = has_param_annotations || has_return_annotation; + + // Take annotation_block if this function has any annotations. + // When in class scope, the class's annotation_block was saved before scanning + // function annotations, so the current annotation_block belongs to this function. + let annotation_block = if has_any_annotations { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + self.enter_scope(name, CompilerScope::Function, line_number); + // Move annotation_block to function scope only if we have one + if let Some(block) = annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } + // Fill scope with parameter names: self.scan_parameters(¶meters.posonlyargs)?; self.scan_parameters(¶meters.args)?; @@ -1836,6 +2161,20 @@ impl SymbolTableBuilder { .to_source_code() .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); + + // Check for forbidden names like __debug__ + if name == "__debug__" + && matches!( + role, + SymbolUsage::Parameter | SymbolUsage::AnnotationParameter | SymbolUsage::Assigned + ) + { + return Err(SymbolTableError { + error: "cannot assign to __debug__".to_owned(), + location, + }); + } + let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index a0054b2887..c59a64fea3 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -290,13 +290,16 @@ pub struct CodeObject { bitflags! { #[derive(Copy, Clone, Debug, PartialEq)] - pub struct CodeFlags: u16 { + pub struct CodeFlags: u32 { const OPTIMIZED = 0x0001; const NEWLOCALS = 0x0002; const VARARGS = 0x0004; const VARKEYWORDS = 0x0008; const GENERATOR = 0x0020; const COROUTINE = 0x0080; + /// If a code object represents a function and has a docstring, + /// this bit is set and the first item in co_consts is the docstring. + const HAS_DOCSTRING = 0x4000000; } } diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 3ebb3666ae..0f3d5b5f37 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -267,6 +267,19 @@ pub enum Instruction { BuildTupleFromTuples { size: Arg, } = 124, + /// Build a Template from strings tuple and interpolations tuple on stack. + /// Stack: [strings_tuple, interpolations_tuple] -> [template] + BuildTemplate = 125, + /// Build an Interpolation from value, expression string, and optional format_spec on stack. + /// + /// oparg encoding: (conversion << 2) | has_format_spec + /// - has_format_spec (bit 0): if 1, format_spec is on stack + /// - conversion (bits 2+): 0=None, 1=Str, 2=Repr, 3=Ascii + /// + /// Stack: [value, expression_str, format_spec?] -> [interpolation] + BuildInterpolation { + oparg: Arg, + } = 126, Continue { target: Arg