diff --git a/Lib/enum.py b/Lib/enum.py index 7cffb71863..3adb208c7e 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,12 +1,11 @@ import sys import builtins as bltns +from functools import partial from types import MappingProxyType, DynamicClassAttribute -from operator import or_ as _or_ -from functools import reduce __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', @@ -39,7 +38,7 @@ def _is_descriptor(obj): """ Returns True if obj is a descriptor, False otherwise. """ - return ( + return not isinstance(obj, partial) and ( hasattr(obj, '__get__') or hasattr(obj, '__set__') or hasattr(obj, '__delete__') @@ -63,8 +62,8 @@ def _is_sunder(name): return ( len(name) > 2 and name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' + name[1] != '_' and + name[-2] != '_' ) def _is_internal_class(cls_name, obj): @@ -83,7 +82,6 @@ def _is_private(cls_name, name): if ( len(name) > pat_len and name.startswith(pattern) - and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -158,7 +156,6 @@ def _dedent(text): Like textwrap.dedent. Rewritten because we cannot import textwrap. """ lines = text.split('\n') - blanks = 0 for i, ch in enumerate(lines[0]): if ch != ' ': break @@ -166,6 +163,11 @@ def _dedent(text): lines[j] = l[i:] return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() + class _auto_null: def __repr__(self): return '_auto_null' @@ -283,9 +285,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -313,72 +316,40 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -389,23 +360,19 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) - if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + if self._cls_name is not None and _is_private(self._cls_name, key): + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', - ): + '_add_alias_', '_add_value_alias_', + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -439,12 +406,17 @@ def __setitem__(self, key, value): elif isinstance(value, nonmember): # unwrap value here; it won't be processed by the below `else` value = value.value + elif isinstance(value, partial): + import warnings + warnings.warn('functools.partial will be a method descriptor ' + 'in future Python versions; wrap it in ' + 'enum.member() if you want to preserve the ' + 'old behavior', FutureWarning, stacklevel=2) elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -457,10 +429,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -475,12 +448,21 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singly + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -491,6 +473,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -502,8 +486,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: @@ -564,7 +547,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} - classdict['_unhashable_values_'] = [] + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -579,15 +564,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: - exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e - if exc is not None: - raise exc - # + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) + except Exception as e: + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) # @@ -706,7 +692,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -735,18 +721,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -760,10 +746,20 @@ def __contains__(cls, value): `value` is in `cls` if: 1) `value` is a member of `cls`, or 2) `value` is the value of one of the `cls`'s members. + 3) `value` is a pseudo-member (flags) """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + if issubclass(cls, Flag): + try: + result = cls._missing_(value) + return isinstance(result, cls) + except ValueError: + pass + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1059,7 +1055,55 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1125,12 +1169,17 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: + return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1168,6 +1217,34 @@ def __new__(cls, value): def __init__(self, *args, **kwds): pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) + @staticmethod def _generate_next_value_(name, start, count, last_values): """ @@ -1181,28 +1258,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in reversed(last_values): - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): @@ -1217,14 +1279,13 @@ def __str__(self): def __dir__(self): """ - Returns all members and all public methods + Returns public methods and other interesting attributes. """ - if self.__class__._member_type_ is object: - interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) - else: + interesting = set() + if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): - if name[0] != '_': + if name[0] != '_' and name not in self._member_map_: interesting.add(name) for cls in self.__class__.mro(): for name, obj in cls.__dict__.items(): @@ -1237,7 +1298,7 @@ def __dir__(self): else: # in case it was added by `dir(self)` interesting.discard(name) - else: + elif name not in self._member_map_: interesting.add(name) names = sorted( set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) @@ -1525,37 +1586,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) @@ -1622,7 +1696,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1665,7 +1739,7 @@ def _simple_enum(etype=Enum, *, boundary=None, use_args=None): Class decorator that converts a normal class into an :class:`Enum`. No safety checks are done, and some advanced behavior (such as :func:`__init_subclass__`) is not available. Enum creation can be faster - using :func:`simple_enum`. + using :func:`_simple_enum`. >>> from enum import Enum, _simple_enum >>> @_simple_enum(Enum) @@ -1696,7 +1770,9 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_hashable_values_'] = hashable_values = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1743,35 +1819,42 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member + hashable_values.append(value) if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ member_names.append(name) @@ -1793,37 +1876,53 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ @@ -1880,7 +1979,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) @@ -1908,7 +2007,8 @@ def __call__(self, enumeration): missed = [v for v in values if v not in member_values] if missed: missing_names.append(name) - missing_value |= reduce(_or_, missed) + for val in missed: + missing_value |= val if missing_names: if len(missing_names) == 1: alias = 'alias %s is missing' % missing_names[0] @@ -1957,7 +2057,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 30bd900fa8..c5b8d854d7 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -121,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -132,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -160,12 +164,14 @@ def phase2(self): # Distinguish files, directories, funnies ok = True try: a_stat = os.stat(a_path) - except OSError: + except (OSError, ValueError): + # See https://github.com/python/cpython/issues/122400 + # for the rationale for protecting against ValueError. # print('Can\'t stat', a_path, ':', why.args[1]) ok = False try: b_stat = os.stat(b_path) - except OSError: + except (OSError, ValueError): # print('Can\'t stat', b_path, ':', why.args[1]) ok = False @@ -184,7 +190,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -196,7 +202,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() @@ -280,12 +287,12 @@ def cmpfiles(a, b, common, shallow=True): # Return: # 0 for equal # 1 for different -# 2 for funny cases (can't stat, etc.) +# 2 for funny cases (can't stat, NUL bytes, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except (OSError, ValueError): return 2 diff --git a/Lib/fractions.py b/Lib/fractions.py index 88b418fe38..9d42e80987 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -139,6 +139,23 @@ def _round_to_figures(n, d, figures): return sign, significand, exponent +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + # Pattern for matching float-style format specifications; # supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. _FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" @@ -414,27 +431,42 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def __format__(self, format_spec, /): - """Format this fraction according to the given format specification.""" - - # Backwards compatiblility with existing formatting. - if not format_spec: - return str(self) + def _format_general(self, match): + """Helper method for __format__. + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ # Validate and parse the format specifier. - match = _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec) - if match is None: - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}" - ) - elif match["align"] is not None and match["zeropad"] is not None: - # Avoid the temptation to guess. - raise ValueError( - f"Invalid format specifier {format_spec!r} " - f"for object of type {type(self).__name__!r}; " - "can't use explicit alignment when zero-padding" - ) + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" fill = match["fill"] or " " align = match["align"] or ">" pos_sign = "" if match["sign"] == "-" else match["sign"] @@ -449,6 +481,9 @@ def __format__(self, format_spec, /): trim_point = not alternate_form exponent_indicator = "E" if presentation_type in "EFG" else "e" + if align == '=' and fill == '0': + zeropad = True + # Round to get the digits we need, figure out where to place the point, # and decide whether to use scientific notation. 'point_pos' is the # relative to the _end_ of the digit string: that is, it's the number @@ -530,7 +565,25 @@ def __format__(self, format_spec, /): else: # align == "=" return sign + padding + body - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -617,7 +670,7 @@ def forward(a, b): return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): + elif handle_complex and isinstance(b, complex): return fallback_operator(complex(a), b) else: return NotImplemented @@ -630,7 +683,7 @@ def reverse(b, a): return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): + elif handle_complex and isinstance(a, numbers.Complex): return fallback_operator(complex(a), complex(b)) else: return NotImplemented @@ -781,7 +834,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -789,14 +842,14 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) def __pow__(a, b): """a ** b @@ -825,8 +878,10 @@ def __pow__(a, b): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented def __rpow__(b, a): """a ** b""" diff --git a/Lib/ftplib.py b/Lib/ftplib.py index a56e0c3085..10c5d1ea08 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -900,11 +900,17 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): def test(): '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + Usage: ftplib [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... - -d dir - -l list - -p password + Options: + -d increase debugging level + -r[file] set alternate ~/.netrc file + + Commands: + -l[dir] list directory + -d[dir] change the current directory + -p toggle passive and active mode + file retrieve the file and write it to stdout ''' if len(sys.argv) < 2: @@ -930,15 +936,14 @@ def test(): netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") + print("Could not open account file -- using anonymous login.", + file=sys.stderr) else: try: userid, acct, passwd = netrcobj.authenticators(host) - except KeyError: + except (KeyError, TypeError): # no account for host - sys.stderr.write( - "No account -- using anonymous login.") + print("No account -- using anonymous login.", file=sys.stderr) ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': @@ -951,7 +956,9 @@ def test(): ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) + sys.stdout.buffer.write, 1024) + sys.stdout.buffer.flush() + sys.stdout.flush() ftp.quit() diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db9..8b4eb2fe74 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -53,7 +53,7 @@ def __init__(self, key, msg=None, digestmod=''): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") if _hashopenssl and isinstance(digestmod, (str, _functype)): try: diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 70da07ed2e..b00d9e8634 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() @@ -540,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -698,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: @@ -778,10 +834,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1089,10 +1146,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1146,7 +1217,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1169,7 +1244,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): @@ -1956,10 +2031,7 @@ def readlines(self, sizehint=None): def __iter__(self): """Iterate over lines.""" - while True: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def tell(self): diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff58..757fd01bec 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -15,32 +15,29 @@ def url2pathname(url): # become # C:\foo\bar\spam.foo import string, urllib.parse + if url[:3] == '///': + # URL has an empty authority section, so the path begins on the third + # character. + url = url[2:] + elif url[:12] == '//localhost/': + # Skip past 'localhost' authority. + url = url[11:] + if url[:3] == '///': + # Skip past extra slash before UNC drive in URL path. + url = url[1:] # Windows itself uses ":" even in URLs. url = url.replace(':', '|') if not '|' in url: # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) - url = url[2:] - components = url.split('/') # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) + return urllib.parse.unquote(url.replace('/', '\\')) comp = url.split('|') if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: error = 'Bad URL: ' + url raise OSError(error) drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path + tail = urllib.parse.unquote(comp[1].replace('/', '\\')) + return drive + ':' + tail def pathname2url(p): """OS-specific conversion from a file system path to a relative URL @@ -52,30 +49,21 @@ def pathname2url(p): import urllib.parse # First, clean up some special forms. We are going to sacrifice # the additional information anyway - if p[:4] == '\\\\?\\': + p = p.replace('\\', '/') + if p[:4] == '//?/': p = p[4:] - if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] + if p[:4].upper() == 'UNC/': + p = '//' + p[4:] elif p[1:2] != ':': raise OSError('Bad path: ' + p) if not ':' in p: - # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) + # No DOS drive specified, just quote the pathname + return urllib.parse.quote(p) comp = p.split(':', maxsplit=2) if len(comp) != 2 or len(comp[0]) > 1: error = 'Bad path: ' + p raise OSError(error) drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path + tail = urllib.parse.quote(comp[1]) + return '///' + drive + ':' + tail diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 0000000000..4b3edf535a --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,12 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +from ._abc import * +from ._local import * + +__all__ = (_abc.__all__ + + _local.__all__) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py new file mode 100644 index 0000000000..4d24146aa5 --- /dev/null +++ b/Lib/pathlib/_abc.py @@ -0,0 +1,930 @@ +""" +Abstract base classes for rich path objects. + +This module is published as a PyPI package called "pathlib-abc". + +This module is also a *PRIVATE* part of the Python standard library, where +it's developed alongside pathlib. If it finds success and maturity as a PyPI +package, it could become a public part of the standard library. + +Two base classes are defined here -- PurePathBase and PathBase -- that +resemble pathlib's PurePath and Path respectively. +""" + +import functools +from glob import _Globber, _no_recurse_symlinks +from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL +from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO + + +__all__ = ["UnsupportedOperation"] + +# +# Internals +# + +_WINERROR_NOT_READY = 21 # drive exists but is not accessible +_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 +_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself + +# EBADF - guard against macOS `stat` throwing EBADF +_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) + +_IGNORED_WINERRORS = ( + _WINERROR_NOT_READY, + _WINERROR_INVALID_NAME, + _WINERROR_CANT_RESOLVE_FILENAME) + +def _ignore_error(exception): + return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or + getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) + + +@functools.cache +def _is_case_sensitive(parser): + return parser.normcase('Aa') == 'Aa' + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is called on + a path object. + """ + pass + + +class ParserBase: + """Base class for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide PurePathBase functionality. Each PurePathBase + subclass references its path parser via a 'parser' class attribute. + + Every method in this base class raises an UnsupportedOperation exception. + """ + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" + + @property + def sep(self): + """The character used to separate path components.""" + raise UnsupportedOperation(self._unsupported_msg('sep')) + + def join(self, path, *paths): + """Join path segments.""" + raise UnsupportedOperation(self._unsupported_msg('join()')) + + def split(self, path): + """Split the path into a pair (head, tail), where *head* is everything + before the final path separator, and *tail* is everything after. + Either part may be empty. + """ + raise UnsupportedOperation(self._unsupported_msg('split()')) + + def splitdrive(self, path): + """Split the path into a 2-item tuple (drive, tail), where *drive* is + a device name or mount point, and *tail* is everything after the + drive. Either part may be empty.""" + raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + + def normcase(self, path): + """Normalize the case of the path.""" + raise UnsupportedOperation(self._unsupported_msg('normcase()')) + + def isabs(self, path): + """Returns whether the path is absolute, i.e. unaffected by the + current directory or drive.""" + raise UnsupportedOperation(self._unsupported_msg('isabs()')) + + +class PurePathBase: + """Base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, + __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path + joining methods accept only strings, not os.PathLike objects more broadly. + """ + + __slots__ = ( + # The `_raw_path` slot store a joined string path. This is set in the + # `__init__()` method. + '_raw_path', + + # The '_resolving' slot stores a boolean indicating whether the path + # is being processed by `PathBase.resolve()`. This prevents duplicate + # work from occurring when `resolve()` calls `stat()` or `readlink()`. + '_resolving', + ) + parser = ParserBase() + _globber = _Globber + + def __init__(self, path, *paths): + self._raw_path = self.parser.join(path, *paths) if paths else path + if not isinstance(self._raw_path, str): + raise TypeError( + f"path should be a str, not {type(self._raw_path).__name__!r}") + self._resolving = False + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + return self._raw_path + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + return self.parser.splitdrive(self.anchor)[0] + + @property + def root(self): + """The root of the path, if any.""" + return self.parser.splitdrive(self.anchor)[1] + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self._stack[0] + + @property + def name(self): + """The final path component, if any.""" + return self.parser.split(self._raw_path)[1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name + i = name.rfind('.') + if 0 < i < len(name) - 1: + return name[i:] + else: + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + name = self.name + if name.endswith('.'): + return [] + name = name.lstrip('.') + return ['.' + suffix for suffix in name.split('.')[1:]] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if 0 < i < len(name) - 1: + return name[:i] + else: + return name + + def with_name(self, name): + """Return a new path with the file name changed.""" + split = self.parser.split + if split(name)[0]: + raise ValueError(f"Invalid name {name!r}") + return self.with_segments(split(self._raw_path)[0], name) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not (suffix.startswith('.') and len(suffix) > 1): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not isinstance(other, PurePathBase): + other = self.with_segments(other) + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if not part or part == '.': + pass + elif not walk_up: + raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") + elif part == '..': + raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") + else: + parts0.append('..') + return self.with_segments('', *reversed(parts0)) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not isinstance(other, PurePathBase): + other = self.with_segments(other) + anchor0, parts0 = self._stack + anchor1, parts1 = other._stack + if anchor0 != anchor1: + return False + while parts0 and parts1 and parts0[-1] == parts1[-1]: + parts0.pop() + parts1.pop() + for part in parts1: + if part and part != '.': + return False + return True + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + anchor, parts = self._stack + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self._raw_path, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self._raw_path, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self._raw_path) + except TypeError: + return NotImplemented + + @property + def _stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = self.parser.split + path = self._raw_path + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + @property + def parent(self): + """The logical parent of the path.""" + path = self._raw_path + parent = self.parser.split(path)[0] + if path != parent: + parent = self.with_segments(parent) + parent._resolving = self._resolving + return parent + return self + + @property + def parents(self): + """A sequence of this path's logical parents.""" + split = self.parser.split + path = self._raw_path + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + return self.parser.isabs(self._raw_path) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + return str(self) + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not isinstance(path_pattern, PurePathBase): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + sep = path_pattern.parser.sep + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = self._globber(sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + globber = self._globber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern._pattern_str) + return match(self._pattern_str) is not None + + + +class PathBase(PurePathBase): + """Base class for concrete path objects. + + This class provides dummy implementations for many methods that derived + classes can override selectively; the default implementations raise + UnsupportedOperation. The most basic methods, such as stat() and open(), + directly raise UnsupportedOperation; these basic methods are called by + other methods such as is_dir() and read_text(). + + The Path class derives this class to implement local filesystem paths. + Users may derive their own classes to implement virtual filesystem paths, + such as paths in archive files or on remote storage systems. + """ + __slots__ = () + + # Maximum number of symlinks to follow in resolve() + _max_symlinks = 40 + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported" + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + raise UnsupportedOperation(self._unsupported_msg('stat()')) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return self.stat(follow_symlinks=False) + + + # Convenience functions for querying the stat results + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + try: + self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if not _ignore_error(e): + raise + return False + except ValueError: + # Non-encodable path + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + try: + parent_dev = self.parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = self.parent.stat().st_ino + return ino == parent_ino + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + try: + return S_ISLNK(self.lstat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist + return False + except ValueError: + # Non-encodable path + return False + + def is_junction(self): + """ + Whether this path is a junction. + """ + # Junctions are a Windows-only feature, not present in POSIX nor the + # majority of virtual filesystems. There is no cross-platform idiom + # to check for junctions (using stat().st_mode). + return False + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except OSError as e: + if not _ignore_error(e): + raise + # Path doesn't exist or is a broken symlink + # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) + return False + except ValueError: + # Non-encodable path + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + raise UnsupportedOperation(self._unsupported_msg('open()')) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb') as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + + def _glob_selector(self, parts, case_sensitive, recurse_symlinks): + if case_sensitive is None: + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) + return globber.selector(parts) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + anchor, parts = pattern._stack + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + select = self._glob_selector(parts, case_sensitive, recurse_symlinks) + return select(self) + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + if not isinstance(pattern, PurePathBase): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + try: + if child.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + raise UnsupportedOperation(self._unsupported_msg('absolute()')) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + # We call 'absolute()' rather than using 'os.getcwd()' directly to + # enable users to replace the implementation of 'absolute()' in a + # subclass and benefit from the new behaviour here. This works because + # os.path.abspath('.') == os.getcwd(). + return cls('').absolute() + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + raise UnsupportedOperation(self._unsupported_msg('expanduser()')) + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + return cls("~").expanduser() + + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + raise UnsupportedOperation(self._unsupported_msg('readlink()')) + readlink._supported = False + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + if self._resolving: + return self + path_root, parts = self._stack + path = self.with_segments(path_root) + try: + path = path.absolute() + except UnsupportedOperation: + path_tail = [] + else: + path_root, path_tail = path._stack + path_tail.reverse() + + # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported + # and (in non-strict mode) we can improve performance by not calling `stat()`. + querying = strict or getattr(self.readlink, '_supported', True) + link_count = 0 + while parts: + part = parts.pop() + if not part or part == '.': + continue + if part == '..': + if not path_tail: + if path_root: + # Delete '..' segment immediately following root + continue + elif path_tail[-1] != '..': + # Delete '..' segment and its predecessor + path_tail.pop() + continue + path_tail.append(part) + if querying and part != '..': + path = self.with_segments(path_root + self.parser.sep.join(path_tail)) + path._resolving = True + try: + st = path.stat(follow_symlinks=False) + if S_ISLNK(st.st_mode): + # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are + # encountered during resolution. + link_count += 1 + if link_count >= self._max_symlinks: + raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) + target_root, target_parts = path.readlink()._stack + # If the symlink target is absolute (like '/etc/hosts'), set the current + # path to its uppermost parent (like '/'). + if target_root: + path_root = target_root + path_tail.clear() + else: + path_tail.pop() + # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to + # the stack of unresolved path parts. + parts.extend(target_parts) + continue + elif parts and not S_ISDIR(st.st_mode): + raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) + except OSError: + if strict: + raise + else: + querying = False + return self.with_segments(path_root + self.parser.sep.join(path_tail)) + + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + raise UnsupportedOperation(self._unsupported_msg('touch()')) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + raise UnsupportedOperation(self._unsupported_msg('rename()')) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + raise UnsupportedOperation(self._unsupported_msg('replace()')) + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + raise UnsupportedOperation(self._unsupported_msg('chmod()')) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + raise UnsupportedOperation(self._unsupported_msg('unlink()')) + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + raise UnsupportedOperation(self._unsupported_msg('rmdir()')) + + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + raise UnsupportedOperation(self._unsupported_msg('owner()')) + + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + raise UnsupportedOperation(self._unsupported_msg('group()')) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) + + def as_uri(self): + """Return the path as a URI.""" + raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 0000000000..0188e7c772 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,861 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +import warnings +from glob import _StringGlobber +from itertools import chain +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from ._abc import UnsupportedOperation, PurePathBase, PathBase + + +__all__ = [ + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath(PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + _globber = _StringGlobber + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.parser.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def relative_to(self, other, /, *_deprecated, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other, /, *_deprecated): + """Return True if the path is relative to another path or False. + """ + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.write_text(self, data, encoding, errors, newline) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if pattern.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + parts = pattern._tail.copy() + if not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + raw = pattern._raw_path + if raw[-1] in (self.parser.sep, self.parser.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index a4c474006b..dccbec52aa 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -14,7 +14,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', + 'read_code', 'extend_path', 'ModuleInfo', ] @@ -23,20 +23,6 @@ ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' -def _get_spec(finder, name): - """Return the finder-specific module spec.""" - # Works with legacy finders. - try: - find_spec = finder.find_spec - except AttributeError: - loader = finder.find_module(name) - if loader is None: - return None - return importlib.util.spec_from_loader(name, loader) - else: - return find_spec(name) - - def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files @@ -184,6 +170,7 @@ def _iter_file_finder_modules(importer, prefix=''): iter_importer_modules.register( importlib.machinery.FileFinder, _iter_file_finder_modules) + try: import zipimport from zipimport import zipimporter @@ -231,6 +218,7 @@ def get_importer(path_item): The cache (or part of it) can be cleared manually if a rescan of sys.path_hooks is necessary. """ + path_item = os.fsdecode(path_item) try: importer = sys.path_importer_cache[path_item] except KeyError: @@ -282,6 +270,10 @@ def get_loader(module_or_name): If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. """ + warnings._deprecated("pkgutil.get_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] if module_or_name is None: @@ -306,6 +298,10 @@ def find_loader(fullname): importlib.util.find_spec that converts most failures to ImportError and only returns the loader rather than the full spec """ + warnings._deprecated("pkgutil.find_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) @@ -328,10 +324,10 @@ def extend_path(path, name): from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - This will add to the package's __path__ all subdirectories of - directories on sys.path named after the package. This is useful - if one wants to distribute different parts of a single logical - package as multiple directories. + For each directory on sys.path that has a subdirectory that + matches the package name, add the subdirectory to the package's + __path__. This is useful if one wants to distribute different + parts of a single logical package as multiple directories. It also looks for *.pkg files beginning where * matches the name argument. This feature is similar to *.pth files (see site.py), diff --git a/Lib/platform.py b/Lib/platform.py index c64c6d2c6f..8895177e32 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -10,7 +10,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -118,6 +119,10 @@ import sys import functools import itertools +try: + import _wmi +except ImportError: + _wmi = None ### Globals & Constants @@ -136,11 +141,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +157,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +190,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +206,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +253,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +298,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -310,44 +317,62 @@ def _syscmd_ver(system='', release='', version='', version = _norm_version(version) return system, release, version -_WIN32_CLIENT_RELEASES = { - (5, 0): "2000", - (5, 1): "XP", - # Strictly, 5.2 client is XP 64-bit, but platform.py historically - # has always called it 2003 Server - (5, 2): "2003Server", - (5, None): "post2003", - - (6, 0): "Vista", - (6, 1): "7", - (6, 2): "8", - (6, 3): "8.1", - (6, None): "post8.1", - - (10, 0): "10", - (10, None): "post10", -} - -# Server release name lookup will default to client names if necessary -_WIN32_SERVER_RELEASES = { - (5, 2): "2003Server", - (6, 0): "2008Server", - (6, 1): "2008ServerR2", - (6, 2): "2012Server", - (6, 3): "2012ServerR2", - (6, None): "post2012ServerR2", -} +def _wmi_query(table, *keys): + global _wmi + if not _wmi: + raise OSError("not supported") + table = { + "OS": "Win32_OperatingSystem", + "CPU": "Win32_Processor", + }[table] + try: + data = _wmi.exec_query("SELECT {} FROM {}".format( + ",".join(keys), + table, + )).split("\0") + except OSError: + _wmi = None + raise OSError("not supported") + split_data = (i.partition("=") for i in data) + dict_data = {i[0]: i[2] for i in split_data} + return (dict_data[k] for k in keys) + + +_WIN32_CLIENT_RELEASES = [ + ((10, 1, 0), "post11"), + ((10, 0, 22000), "11"), + ((6, 4, 0), "10"), + ((6, 3, 0), "8.1"), + ((6, 2, 0), "8"), + ((6, 1, 0), "7"), + ((6, 0, 0), "Vista"), + ((5, 2, 3790), "XP64"), + ((5, 2, 0), "XPMedia"), + ((5, 1, 0), "XP"), + ((5, 0, 0), "2000"), +] + +_WIN32_SERVER_RELEASES = [ + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), + ((10, 0, 20348), "2022Server"), + ((10, 0, 17763), "2019Server"), + ((6, 4, 0), "2016Server"), + ((6, 3, 0), "2012ServerR2"), + ((6, 2, 0), "2012Server"), + ((6, 1, 0), "2008ServerR2"), + ((6, 0, 0), "2008Server"), + ((5, 2, 0), "2003Server"), + ((5, 0, 0), "2000Server"), +] def win32_is_iot(): return win32_edition() in ('IoTUAP', 'NanoServer', 'WindowsCoreHeadless', 'IoTEdgeOS') def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -360,22 +385,40 @@ def win32_edition(): return None -def win32_ver(release='', version='', csd='', ptype=''): +def _win32_ver(version, csd, ptype): + # Try using WMI first, as this is the canonical source of data + try: + (version, product_type, ptype, spmajor, spminor) = _wmi_query( + 'OS', + 'Version', + 'ProductType', + 'BuildType', + 'ServicePackMajorVersion', + 'ServicePackMinorVersion', + ) + is_client = (int(product_type) == 1) + if spminor and spminor != '0': + csd = f'SP{spmajor}.{spminor}' + else: + csd = f'SP{spmajor}' + return version, csd, ptype, is_client + except OSError: + pass + + # Fall back to a combination of sys.getwindowsversion and "ver" try: from sys import getwindowsversion except ImportError: - return release, version, csd, ptype + return version, csd, ptype, True winver = getwindowsversion() + is_client = (getattr(winver, 'product_type', 1) == 1) try: - major, minor, build = map(int, _syscmd_ver()[2].split('.')) + version = _syscmd_ver()[2] + major, minor, build = map(int, version.split('.')) except ValueError: major, minor, build = winver.platform_version or winver[:3] - version = '{0}.{1}.{2}'.format(major, minor, build) - - release = (_WIN32_CLIENT_RELEASES.get((major, minor)) or - _WIN32_CLIENT_RELEASES.get((major, None)) or - release) + version = '{0}.{1}.{2}'.format(major, minor, build) # getwindowsversion() reflect the compatibility mode Python is # running under, and so the service pack value is only going to be @@ -387,17 +430,8 @@ def win32_ver(release='', version='', csd='', ptype=''): if csd[:13] == 'Service Pack ': csd = 'SP' + csd[13:] - # VER_NT_SERVER = 3 - if getattr(winver, 'product_type', None) == 3: - release = (_WIN32_SERVER_RELEASES.get((major, minor)) or - _WIN32_SERVER_RELEASES.get((major, None)) or - release) - try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -408,6 +442,18 @@ def win32_ver(release='', version='', csd='', ptype=''): except OSError: pass + return version, csd, ptype, is_client + +def win32_ver(release='', version='', csd='', ptype=''): + is_client = False + + version, csd, ptype, is_client = _win32_ver(version, csd, ptype) + + if version: + intversion = tuple(map(int, version.split('.'))) + releases = _WIN32_CLIENT_RELEASES if is_client else _WIN32_SERVER_RELEASES + release = next((r for v, r in releases if v <= intversion), release) + return release, version, csd, ptype @@ -452,8 +498,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -475,6 +545,8 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: import java.lang @@ -496,6 +568,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -562,12 +675,12 @@ def _platform(*args): platform = platform.replace('unknown', '') # Fold '--'s and remove trailing '-' - while 1: + while True: cleaned = platform.replace('--', '-') if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -608,7 +721,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -702,6 +815,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: @@ -726,6 +841,21 @@ def _get_machine_win32(): # http://www.geocities.com/rick_lively/MANUALS/ENV/MSWIN/PROCESSI.HTM # WOW64 processes mask the native architecture + try: + [arch, *_] = _wmi_query('CPU', 'Architecture') + except OSError: + pass + else: + try: + arch = ['x86', 'MIPS', 'Alpha', 'PowerPC', None, + 'ARM', 'ia64', None, None, + 'AMD64', None, None, 'ARM64', + ][int(arch)] + except (ValueError, IndexError): + pass + else: + if arch: + return arch return ( os.environ.get('PROCESSOR_ARCHITEW6432', '') or os.environ.get('PROCESSOR_ARCHITECTURE', '') @@ -739,7 +869,12 @@ def get(cls): return func() or '' def get_win32(): - return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + try: + manufacturer, caption = _wmi_query('CPU', 'Manufacturer', 'Caption') + except OSError: + return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + else: + return f'{caption}, {manufacturer}' def get_OpenVMS(): try: @@ -750,6 +885,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -904,6 +1047,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -971,32 +1123,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_ironpython_sys_version_parser = re.compile( - r'IronPython\s*' - r'([\d\.]+)' - r'(?: \(([\d\.]+)\))?' - r' on (.NET [\d\.]+)', re.ASCII) - -# IronPython covering 2.6 and 2.7 -_ironpython26_sys_version_parser = re.compile( - r'([\d.]+)\s*' - r'\(IronPython\s*' - r'[\d.]+\s*' - r'\(([\d.]+)\) on ([\w.]+ [\d.]+(?: \(\d+-bit\))?)\)' -) - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1028,28 +1154,16 @@ def _sys_version(sys_version=None): if result is not None: return result - # Parse it - if 'IronPython' in sys_version: - # IronPython - name = 'IronPython' - if sys_version.startswith('IronPython'): - match = _ironpython_sys_version_parser.match(sys_version) - else: - match = _ironpython26_sys_version_parser.match(sys_version) - - if match is None: - raise ValueError( - 'failed to parse IronPython sys.version: %s' % - repr(sys_version)) - - version, alt_version, compiler = match.groups() - buildno = '' - builddate = '' - - elif sys.platform.startswith('java'): + if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1061,8 +1175,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1071,20 +1190,21 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % repr(sys_version)) version, buildno, builddate, buildtime, compiler = \ match.groups() - - # XXX: RUSTPYTHON support - if "RustPython" in sys_version: - name = "RustPython" - else: - name = 'CPython' - + name = 'CPython' if builddate is None: builddate = '' elif buildtime: @@ -1115,7 +1235,6 @@ def python_implementation(): Currently, the following implementations are identified: 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), 'Jython' (Java implementation of Python), 'PyPy' (Python implementation of Python). @@ -1190,7 +1309,7 @@ def python_compiler(): _platform_cache = {} -def platform(aliased=0, terse=0): +def platform(aliased=False, terse=False): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). @@ -1222,11 +1341,14 @@ def platform(aliased=0, terse=0): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms @@ -1236,7 +1358,7 @@ def platform(aliased=0, terse=0): else: platform = _platform(system, release, version, csd) - elif system in ('Linux',): + elif system == 'Linux': # check for libc vs. glibc libcname, libcversion = libc_ver() platform = _platform(system, release, machine, processor, @@ -1267,13 +1389,6 @@ def platform(aliased=0, terse=0): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1288,10 +1403,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2eeebe4c9a..655c51eea3 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,6 +21,9 @@ Generate Plist example: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -28,22 +31,28 @@ anInt = 728, aDict = dict( anotherString = "", - aUnicodeValue = "M\xe4ssig, Ma\xdf", + aThirdString = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parse Plist example: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b''' + + foo + bar + + ''' + pl = plistlib.loads(plist) + print(pl["foo"]) """ __all__ = [ "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID" @@ -64,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -131,7 +143,7 @@ def _decode_base64(s): _dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -140,10 +152,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -152,7 +168,7 @@ def _date_to_string(d): def _escape(text): m = _controlCharPat.search(text) if m is not None: - raise ValueError("strings can't contains control characters; " + raise ValueError("strings can't contain control characters; " "use bytes instead") text = text.replace("\r\n", "\n") # convert DOS line endings text = text.replace("\r", "\n") # convert Mac line endings @@ -162,11 +178,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -178,8 +195,8 @@ def parse(self, fileobj): return self.root def handle_entity_decl(self, entity_name, is_parameter_entity, value, base, system_id, public_id, notation_name): - # Reject plist files with entity declarations to avoid XML vulnerabilies in expat. - # Regular plist files don't contain those declerations, and Apple's plutil tool does not + # Reject plist files with entity declarations to avoid XML vulnerabilities in expat. + # Regular plist files don't contain those declarations, and Apple's plutil tool does not # accept them either. raise InvalidFileException("XML entity declarations are not supported in plist files") @@ -199,7 +216,7 @@ def handle_data(self, data): def add_object(self, value): if self.current_key is not None: - if not isinstance(self.stack[-1], type({})): + if not isinstance(self.stack[-1], dict): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1][self.current_key] = value @@ -208,7 +225,7 @@ def add_object(self, value): # this is the root object self.root = value else: - if not isinstance(self.stack[-1], type([])): + if not isinstance(self.stack[-1], list): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1].append(value) @@ -232,7 +249,7 @@ def end_dict(self): self.stack.pop() def end_key(self): - if self.current_key or not isinstance(self.stack[-1], type({})): + if self.current_key or not isinstance(self.stack[-1], dict): raise ValueError("unexpected key at line %d" % self.parser.CurrentLineNumber) self.current_key = self.get_data() @@ -268,7 +285,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -312,13 +330,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -351,7 +370,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -452,8 +472,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -490,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -547,27 +580,24 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID @@ -579,7 +609,8 @@ def _read_object(self, ref): obj_refs = self._read_refs(s) result = [] self._objects[ref] = result - result.extend(self._read_object(x) for x in obj_refs) + for x in obj_refs: + result.append(self._read_object(x)) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -620,10 +651,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -769,7 +801,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -853,7 +890,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -871,32 +908,41 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ + if isinstance(value, str): + if fmt == FMT_BINARY: + raise TypeError("value must be bytes-like object when fmt is " + "FMT_BINARY") + value = value.encode() fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 1bb74c6d96..b3dd906cc7 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -398,6 +398,8 @@ def test_aware_datetime(self): self.assertEqual(tz.dst(t), t.replace(tzinfo=tz).dst()) + @unittest.skip('TODO: RUSTPYTHON') + # _pycodecs was deleted def test_pickle(self): for tz in self.ACDT, self.EST, timezone.min, timezone.max: for pickler, unpickler, proto in pickle_choices: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 32a3c1dee0..e9d4c23345 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -11,14 +11,15 @@ import builtins as bltns from collections import OrderedDict from datetime import date +from functools import partial from enum import Enum, EnumMeta, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember, _iter_bits_lsb +from enum import member, nonmember, _iter_bits_lsb, EnumDict from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support -from test.support import ALWAYS_EQ +from test.support import ALWAYS_EQ, REPO_ROOT from test.support import threading_helper from datetime import timedelta @@ -26,18 +27,42 @@ def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(enum)) - if os.path.exists('Doc/library/enum.rst'): + + lib_tests = os.path.join(REPO_ROOT, 'Doc/library/enum.rst') + if os.path.exists(lib_tests): tests.addTests(doctest.DocFileSuite( - '../../Doc/library/enum.rst', + lib_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) - if os.path.exists('Doc/howto/enum.rst'): + howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') + if os.path.exists(howto_tests) and sys.float_repr_style == 'short': tests.addTests(doctest.DocFileSuite( - '../../Doc/howto/enum.rst', + howto_tests, + module_relative=False, optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) return tests +def reraise_if_not_enum(*enum_types_or_exceptions): + from functools import wraps + + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + excs = [ + e + for e in enum_types_or_exceptions + if isinstance(e, Exception) + ] + if len(excs) == 1: + raise excs[0] + elif excs: + raise ExceptionGroup('Enum Exceptions', excs) + return func(*args, **kwargs) + return inner + return decorator + MODULE = __name__ SHORT_MODULE = MODULE.split('.')[-1] @@ -75,30 +100,42 @@ class FlagStooges(Flag): except Exception as exc: FlagStooges = exc -class FlagStoogesWithZero(Flag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStooges(IntFlag): - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStoogesWithZero(IntFlag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 +try: + class FlagStoogesWithZero(Flag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + FlagStoogesWithZero = exc + +try: + class IntFlagStooges(IntFlag): + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStooges = exc + +try: + class IntFlagStoogesWithZero(IntFlag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStoogesWithZero = exc # for pickle test and subclass tests -class Name(StrEnum): - BDFL = 'Guido van Rossum' - FLUFL = 'Barry Warsaw' +try: + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception as exc: + Name = exc try: Question = Enum('Question', 'who what when where why', module=__name__) @@ -140,7 +177,7 @@ class TestHelpers(unittest.TestCase): sunder_names = '_bad_', '_good_', '_what_ho_' dunder_names = '__mal__', '__bien__', '__que_que__' - private_names = '_MyEnum__private', '_MyEnum__still_private' + private_names = '_MyEnum__private', '_MyEnum__still_private', '_MyEnum___triple_private' private_and_sunder_names = '_MyEnum__private_', '_MyEnum__also_private_' random_names = 'okay', '_semi_private', '_weird__', '_MyEnum__' @@ -204,26 +241,35 @@ def __get__(self, instance, ownerclass): # for global repr tests -@enum.global_enum -class HeadlightsK(IntFlag, boundary=enum.KEEP): - OFF_K = 0 - LOW_BEAM_K = auto() - HIGH_BEAM_K = auto() - FOG_K = auto() +try: + @enum.global_enum + class HeadlightsK(IntFlag, boundary=enum.KEEP): + OFF_K = 0 + LOW_BEAM_K = auto() + HIGH_BEAM_K = auto() + FOG_K = auto() +except Exception as exc: + HeadlightsK = exc -@enum.global_enum -class HeadlightsC(IntFlag, boundary=enum.CONFORM): - OFF_C = 0 - LOW_BEAM_C = auto() - HIGH_BEAM_C = auto() - FOG_C = auto() +try: + @enum.global_enum + class HeadlightsC(IntFlag, boundary=enum.CONFORM): + OFF_C = 0 + LOW_BEAM_C = auto() + HIGH_BEAM_C = auto() + FOG_C = auto() +except Exception as exc: + HeadlightsC = exc -@enum.global_enum -class NoName(Flag): - ONE = 1 - TWO = 2 +try: + @enum.global_enum + class NoName(Flag): + ONE = 1 + TWO = 2 +except Exception as exc: + NoName = exc # tests @@ -399,10 +445,12 @@ def spam(cls): with self.assertRaises(AttributeError): del Season.SPRING.name + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance failed in 'BadSuper' def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -417,6 +465,7 @@ def test_basics(self): self.assertEqual(str(TE), "") self.assertEqual(format(TE), "") self.assertTrue(TE(5) is self.dupe2) + self.assertTrue(7 in TE) else: self.assertEqual(repr(TE), "") self.assertEqual(str(TE), "") @@ -469,6 +518,7 @@ def test_contains_tf(self): self.assertFalse('first' in MainEnum) val = MainEnum.dupe self.assertIn(val, MainEnum) + self.assertNotIn(float('nan'), MainEnum) # class OtherEnum(Enum): one = auto() @@ -1005,6 +1055,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -1292,7 +1358,7 @@ class Color(Enum): red = 1 green = 2 blue = 3 - def red(self): + def red(self): # noqa: F811 return 'red' # with self.assertRaises(TypeError): @@ -1300,13 +1366,12 @@ class Color(Enum): @enum.property def red(self): return 'redder' - red = 1 + red = 1 # noqa: F811 green = 2 blue = 3 + @reraise_if_not_enum(Theory) def test_enum_function_with_qualname(self): - if isinstance(Theory, Exception): - raise Theory self.assertEqual(Theory.__qualname__, 'spanish_inquisition') def test_enum_of_types(self): @@ -1369,12 +1434,10 @@ class Inner(Enum): [Outer.a, Outer.b, Outer.Inner], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'inner classes are still members', + ) def test_nested_classes_in_enum_are_not_members(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1439,6 +1502,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1460,6 +1544,22 @@ class Inner(Enum): [Outer.a, Outer.b, Outer.Inner], ) + # TODO: RUSTPYTHON + # AssertionError: FutureWarning not triggered + @unittest.expectedFailure + def test_partial(self): + def func(a, b=5): + return a, b + with self.assertWarnsRegex(FutureWarning, r'partial.*enum\.member') as cm: + class E(Enum): + a = 1 + b = partial(func) + self.assertEqual(cm.filename, __file__) + self.assertIsInstance(E.b, partial) + self.assertEqual(E.b(2), (2, 5)) + with self.assertWarnsRegex(FutureWarning, 'partial'): + self.assertEqual(E.a.b(2), (2, 5)) + def test_enum_with_value_name(self): class Huh(Enum): name = 1 @@ -1491,6 +1591,17 @@ class IntFlag1(IntFlag): self.assertIn(IntEnum1.X, IntFlag1) self.assertIn(IntFlag1.X, IntEnum1) + def test_contains_does_not_call_missing(self): + class AnEnum(Enum): + UNKNOWN = None + LUCKY = 3 + @classmethod + def _missing_(cls, *values): + return cls.UNKNOWN + self.assertTrue(None in AnEnum) + self.assertTrue(3 in AnEnum) + self.assertFalse(7 in AnEnum) + def test_inherited_data_type(self): class HexInt(int): __qualname__ = 'HexInt' @@ -1537,6 +1648,7 @@ class MyUnBrokenEnum(UnBrokenInt, Enum): test_pickle_dump_load(self.assertIs, MyUnBrokenEnum.I) test_pickle_dump_load(self.assertIs, MyUnBrokenEnum) + @reraise_if_not_enum(FloatStooges) def test_floatenum_fromhex(self): h = float.hex(FloatStooges.MOE.value) self.assertIs(FloatStooges.fromhex(h), FloatStooges.MOE) @@ -1657,8 +1769,8 @@ class ThreePart(Enum): self.assertIs(ThreePart((3, 3.0, 'three')), ThreePart.THREE) self.assertIs(ThreePart(3, 3.0, 'three'), ThreePart.THREE) - # TODO: RUSTPYTHON, AssertionError: is not - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: is not + @reraise_if_not_enum(IntStooges) def test_intenum_from_bytes(self): self.assertIs(IntStooges.from_bytes(b'\x00\x03', 'big'), IntStooges.MOE) with self.assertRaises(ValueError): @@ -1687,33 +1799,28 @@ def repr(self): class Huh(MyStr, MyInt, Enum): One = 1 + @reraise_if_not_enum(Stooges) def test_pickle_enum(self): - if isinstance(Stooges, Exception): - raise Stooges test_pickle_dump_load(self.assertIs, Stooges.CURLY) test_pickle_dump_load(self.assertIs, Stooges) + @reraise_if_not_enum(IntStooges) def test_pickle_int(self): - if isinstance(IntStooges, Exception): - raise IntStooges test_pickle_dump_load(self.assertIs, IntStooges.CURLY) test_pickle_dump_load(self.assertIs, IntStooges) + @reraise_if_not_enum(FloatStooges) def test_pickle_float(self): - if isinstance(FloatStooges, Exception): - raise FloatStooges test_pickle_dump_load(self.assertIs, FloatStooges.CURLY) test_pickle_dump_load(self.assertIs, FloatStooges) + @reraise_if_not_enum(Answer) def test_pickle_enum_function(self): - if isinstance(Answer, Exception): - raise Answer test_pickle_dump_load(self.assertIs, Answer.him) test_pickle_dump_load(self.assertIs, Answer) + @reraise_if_not_enum(Question) def test_pickle_enum_function_with_module(self): - if isinstance(Question, Exception): - raise Question test_pickle_dump_load(self.assertIs, Question.who) test_pickle_dump_load(self.assertIs, Question) @@ -1776,9 +1883,8 @@ class Season(Enum): [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], ) + @reraise_if_not_enum(Name) def test_subclassing(self): - if isinstance(Name, Exception): - raise Name self.assertEqual(Name.BDFL, 'Guido van Rossum') self.assertTrue(Name.BDFL, Name('Guido van Rossum')) self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) @@ -1817,6 +1923,27 @@ def test_wrong_inheritance_order(self): class Wrong(Enum, str): NotHere = 'error before this point' + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance INVALID in 'RgbColor' + def test_raise_custom_error_on_creation(self): + class InvalidRgbColorError(ValueError): + def __init__(self, r, g, b): + self.r = r + self.g = g + self.b = b + super().__init__(f'({r}, {g}, {b}) is not a valid RGB color') + + with self.assertRaises(InvalidRgbColorError): + class RgbColor(Enum): + RED = (255, 0, 0) + GREEN = (0, 255, 0) + BLUE = (0, 0, 255) + INVALID = (256, 0, 0) + + def __init__(self, r, g, b): + if not all(0 <= val <= 255 for val in (r, g, b)): + raise InvalidRgbColorError(r, g, b) + def test_intenum_transitivity(self): class number(IntEnum): one = 1 @@ -2003,8 +2130,7 @@ class NEI(NamedInt, Enum): test_pickle_dump_load(self.assertIs, NEI.y) test_pickle_dump_load(self.assertIs, NEI) - # TODO: RUSTPYTHON, fails on pickle - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fails on pickle def test_subclasses_with_getnewargs_ex(self): class NamedInt(int): __qualname__ = 'NamedInt' # needed for pickle protocol 4 @@ -2312,6 +2438,40 @@ class SomeTuple(tuple, Enum): globals()['SomeTuple'] = SomeTuple test_pickle_dump_load(self.assertIs, SomeTuple.first) + def test_tuple_subclass_with_auto_1(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(T, Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = auto(), 'for the money' + second = auto(), 'for the show' + third = auto(), 'for the music' + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.index, 1) + self.assertEqual(SomeEnum.second.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + + def test_tuple_subclass_with_auto_2(self): + from collections import namedtuple + T = namedtuple('T', 'index desc') + class SomeEnum(Enum): + __qualname__ = 'SomeEnum' # needed for pickle protocol 4 + first = T(auto(), 'for the money') + second = T(auto(), 'for the show') + third = T(auto(), 'for the music') + self.assertIs(type(SomeEnum.first), SomeEnum) + self.assertEqual(SomeEnum.third.value, (3, 'for the music')) + self.assertIsInstance(SomeEnum.third.value, T) + self.assertEqual(SomeEnum.first.value.index, 1) + self.assertEqual(SomeEnum.second.value.desc, 'for the show') + globals()['SomeEnum'] = SomeEnum + globals()['T'] = T + test_pickle_dump_load(self.assertIs, SomeEnum.first) + def test_duplicate_values_give_unique_enum_items(self): class AutoNumber(Enum): first = () @@ -2453,6 +2613,8 @@ class Test(Base2): self.assertEqual(Test.flash.flash, 'flashy dynamic') self.assertEqual(Test.flash.value, 1) + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance grene in 'Color' def test_no_duplicates(self): class UniqueEnum(Enum): def __init__(self, *args): @@ -2838,6 +3000,8 @@ def test_empty_globals(self): local_ls = {} exec(code, global_ns, local_ls) + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance one in 'FirstFailedStrEnum' def test_strenum(self): class GoodStrEnum(StrEnum): one = '1' @@ -2900,8 +3064,7 @@ class ThirdFailedStrEnum(StrEnum): one = '1' two = b'2', 'ascii', 9 - # TODO: RUSTPYTHON, fails on encoding testing : TypeError: Expected type 'str' but 'builtin_function_or_method' found - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fails on encoding testing : TypeError: Expected type 'str' but 'builtin_function_or_method' found def test_custom_strenum(self): class CustomStrEnum(str, Enum): pass @@ -2952,15 +3115,19 @@ class SecondFailedStrEnum(CustomStrEnum): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = 2 # this will become '2' - with self.assertRaisesRegex(TypeError, '.encoding. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (2|'encoding') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', sys.getdefaultencoding - with self.assertRaisesRegex(TypeError, '.errors. must be str, not '): + with self.assertRaisesRegex(TypeError, + r"argument (3|'errors') must be str, not "): class ThirdFailedStrEnum(CustomStrEnum): one = '1' two = b'2', 'ascii', 9 + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance key_type in 'Combined' def test_missing_value_error(self): with self.assertRaisesRegex(TypeError, "_value_ not set in __new__"): class Combined(str, Enum): @@ -3170,6 +3337,37 @@ class NTEnum(Enum): [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], ) + self.assertRaises(AttributeError, getattr, NTEnum.NONE, 'id') + # + class NTCEnum(TTuple, Enum): + NONE = 0, 0, [] + A = 1, 2, [4] + B = 2, 4, [0, 1, 2] + self.assertEqual(repr(NTCEnum.NONE), "") + self.assertEqual(NTCEnum.NONE.value, TTuple(id=0, a=0, blist=[])) + self.assertEqual(NTCEnum.NONE.id, 0) + self.assertEqual(NTCEnum.A.a, 2) + self.assertEqual(NTCEnum.B.blist, [0, 1 ,2]) + self.assertEqual( + [x.value for x in NTCEnum], + [TTuple(id=0, a=0, blist=[]), TTuple(id=1, a=2, blist=[4]), TTuple(id=2, a=4, blist=[0, 1, 2])], + ) + # + class NTDEnum(Enum): + def __new__(cls, id, a, blist): + member = object.__new__(cls) + member.id = id + member.a = a + member.blist = blist + return member + NONE = TTuple(0, 0, []) + A = TTuple(1, 2, [4]) + B = TTuple(2, 4, [0, 1, 2]) + self.assertEqual(repr(NTDEnum.NONE), "") + self.assertEqual(NTDEnum.NONE.id, 0) + self.assertEqual(NTDEnum.A.a, 2) + self.assertEqual(NTDEnum.B.blist, [0, 1 ,2]) + def test_flag_with_custom_new(self): class FlagFromChar(IntFlag): def __new__(cls, c): @@ -3216,6 +3414,8 @@ def __new__(cls, c): self.assertEqual(FlagFromChar.a, 158456325028528675187087900672) self.assertEqual(FlagFromChar.a|1, 158456325028528675187087900673) + @unittest.skip('TODO: RUSTPYTHON') + # RuntimeError: Error calling __set_name__ on '_proto_member' instance A in 'MyEnum' def test_init_exception(self): class Base: def __new__(cls, *args): @@ -3237,6 +3437,102 @@ def __new__(cls, value): member._value_ = Base(value) return member + def test_extra_member_creation(self): + class IDEnumMeta(EnumMeta): + def __new__(metacls, cls, bases, classdict, **kwds): + # add new entries to classdict + for name in classdict.member_names: + classdict[f'{name}_DESC'] = f'-{classdict[name]}' + return super().__new__(metacls, cls, bases, classdict, **kwds) + class IDEnum(StrEnum, metaclass=IDEnumMeta): + pass + class MyEnum(IDEnum): + ID = 'id' + NAME = 'name' + self.assertEqual(list(MyEnum), [MyEnum.ID, MyEnum.NAME, MyEnum.ID_DESC, MyEnum.NAME_DESC]) + + def test_add_alias(self): + class mixin: + @property + def ORG(self): + return 'huh' + class Color(mixin, Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_alias_('ROJO') + self.assertIs(Color.RED, Color['ROJO']) + self.assertIs(Color.RED, Color.ROJO) + Color.BLUE._add_alias_('ORG') + self.assertIs(Color.BLUE, Color['ORG']) + self.assertIs(Color.BLUE, Color.ORG) + self.assertEqual(Color.RED.ORG, 'huh') + self.assertEqual(Color.GREEN.ORG, 'huh') + self.assertEqual(Color.BLUE.ORG, 'huh') + self.assertEqual(Color.ORG.ORG, 'huh') + + def test_add_value_alias_after_creation(self): + class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + Color.RED._add_value_alias_(5) + self.assertIs(Color.RED, Color(5)) + + def test_add_value_alias_during_creation(self): + class Types(Enum): + Unknown = 0, + Source = 1, 'src' + NetList = 2, 'nl' + def __new__(cls, int_value, *value_aliases): + member = object.__new__(cls) + member._value_ = int_value + for alias in value_aliases: + member._add_value_alias_(alias) + return member + self.assertIs(Types(0), Types.Unknown) + self.assertIs(Types(1), Types.Source) + self.assertIs(Types('src'), Types.Source) + self.assertIs(Types(2), Types.NetList) + self.assertIs(Types('nl'), Types.NetList) + + def test_second_tuple_item_is_falsey(self): + class Cardinal(Enum): + RIGHT = (1, 0) + UP = (0, 1) + LEFT = (-1, 0) + DOWN = (0, -1) + self.assertIs(Cardinal(1, 0), Cardinal.RIGHT) + self.assertIs(Cardinal(-1, 0), Cardinal.LEFT) + + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) + + def test_empty_names(self): + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) + + def test_nonhashable_matches_hashable(self): # issue 125710 + class Directions(Enum): + DOWN_ONLY = frozenset({"sc"}) + UP_ONLY = frozenset({"cs"}) + UNRESTRICTED = frozenset({"sc", "cs"}) + self.assertIs(Directions({"sc"}), Directions.DOWN_ONLY) + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -3518,9 +3814,13 @@ def test_programatic_function_from_dict(self): self.assertIn(e, Perm) self.assertIs(type(e), Perm) + @reraise_if_not_enum( + FlagStooges, + FlagStoogesWithZero, + IntFlagStooges, + IntFlagStoogesWithZero, + ) def test_pickle(self): - if isinstance(FlagStooges, Exception): - raise FlagStooges test_pickle_dump_load(self.assertIs, FlagStooges.CURLY) test_pickle_dump_load(self.assertEqual, FlagStooges.CURLY|FlagStooges.MOE) @@ -3704,7 +4004,7 @@ class Color(StrMixin, AllMixin, Flag): self.assertEqual(Color.ALL.value, 7) self.assertEqual(str(Color.BLUE), 'blue') - @unittest.skip("TODO: RUSTPYTHON; flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_unique_composite(self): @@ -3826,6 +4126,7 @@ def test_type(self): self.assertTrue(isinstance(Open.WO | Open.RW, Open)) self.assertEqual(Open.WO | Open.RW, 3) + @reraise_if_not_enum(HeadlightsK) def test_global_repr_keep(self): self.assertEqual( repr(HeadlightsK(0)), @@ -3840,6 +4141,7 @@ def test_global_repr_keep(self): '%(m)s.HeadlightsK(8)' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(HeadlightsC) def test_global_repr_conform1(self): self.assertEqual( repr(HeadlightsC(0)), @@ -3854,13 +4156,17 @@ def test_global_repr_conform1(self): '%(m)s.OFF_C' % {'m': SHORT_MODULE}, ) + # TODO: RUSTPYTHON + # AssertionError: '__main__.ONE' != 'test_enum.ONE' + @unittest.expectedFailure + @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') - - # TODO: RUSTPYTHON, format(NewPerm.R) does not use __str__ - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; format(NewPerm.R) does not use __str__ def test_format(self): Perm = self.Perm self.assertEqual(format(Perm.R, ''), '4') @@ -4223,7 +4529,7 @@ class Color(StrMixin, AllMixin, IntFlag): self.assertEqual(Color.ALL.value, 7) self.assertEqual(str(Color.BLUE), 'blue') - @unittest.skip("TODO: RUSTPYTHON; flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_unique_composite(self): @@ -4548,35 +4854,28 @@ class Color(Enum): red = 'red' blue = 2 green = auto() - yellow = auto() - self.assertEqual(list(Color), - [Color.red, Color.blue, Color.green, Color.yellow]) + self.assertEqual(list(Color), [Color.red, Color.blue, Color.green]) self.assertEqual(Color.red.value, 'red') self.assertEqual(Color.blue.value, 2) self.assertEqual(Color.green.value, 3) - self.assertEqual(Color.yellow.value, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', + ) def test_auto_garbage_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, "unable to increment 'red'"): class Color(Enum): red = 'red' blue = auto() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'mixed types with auto() will raise in 3.13', + ) def test_auto_garbage_corrected_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, 'unable to sort non-numeric values'): class Color(Enum): red = 'red' blue = 2 @@ -4604,9 +4903,9 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Color.blue.value, 'blue') @unittest.skipIf( - python_version < (3, 13), - 'inner classes are still members', - ) + python_version < (3, 13), + 'auto() will return highest value + 1 in 3.13', + ) def test_auto_with_aliases(self): class Color(Enum): red = auto() @@ -4670,8 +4969,6 @@ def _generate_next_value_(name, start, count, last): self.assertEqual(Huh.TWO.value, (2, 2)) self.assertEqual(Huh.THREE.value, (3, 3, 3)) -class TestEnumTypeSubclassing(unittest.TestCase): - pass expected_help_output_with_docs = """\ Help on class Color in module %s: @@ -4702,22 +4999,23 @@ class Color(enum.Enum) | The value of the Enum member. | | ---------------------------------------------------------------------- - | Methods inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: | - | __contains__(value) from enum.EnumType + | __contains__(value) | Return True if `value` is in `cls`. | | `value` is in `cls` if: | 1) `value` is a member of `cls`, or | 2) `value` is the value of one of the `cls`'s members. + | 3) `value` is a pseudo-member (flags) | - | __getitem__(name) from enum.EnumType + | __getitem__(name) | Return the member matching `name`. | - | __iter__() from enum.EnumType + | __iter__() | Return members in definition order. | - | __len__() from enum.EnumType + | __len__() | Return the number of members (no aliases) | | ---------------------------------------------------------------------- @@ -4742,11 +5040,11 @@ class Color(enum.Enum) | | Data and other attributes defined here: | - | YELLOW = + | CYAN = | | MAGENTA = | - | CYAN = + | YELLOW = | | ---------------------------------------------------------------------- | Data descriptors inherited from enum.Enum: @@ -4756,7 +5054,18 @@ class Color(enum.Enum) | value | | ---------------------------------------------------------------------- - | Data descriptors inherited from enum.EnumType: + | Static methods inherited from enum.EnumType: + | + | __contains__(value) + | + | __getitem__(name) + | + | __iter__() + | + | __len__() + | + | ---------------------------------------------------------------------- + | Readonly properties inherited from enum.EnumType: | | __members__""" @@ -4769,8 +5078,7 @@ class Color(Enum): MAGENTA = 2 YELLOW = 3 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pydoc(self): # indirectly test __objclass__ if StrEnum.__doc__ is None: @@ -4903,8 +5211,7 @@ def test_inspect_signatures(self): ]), ) - # TODO: RUSTPYTHON, len is often/always > 256 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; len is often/always > 256 def test_test_simple_enum(self): @_simple_enum(Enum) class SimpleColor: @@ -4921,12 +5228,14 @@ class CheckedColor(Enum): @bltns.property def zeroth(self): return 'zeroed %s' % self.name - self.assertTrue(_test_simple_enum(CheckedColor, SimpleColor) is None) + _test_simple_enum(CheckedColor, SimpleColor) SimpleColor.MAGENTA._value_ = 9 self.assertRaisesRegex( TypeError, "enum mismatch", _test_simple_enum, CheckedColor, SimpleColor, ) + # + # class CheckedMissing(IntFlag, boundary=KEEP): SIXTY_FOUR = 64 ONE_TWENTY_EIGHT = 128 @@ -4943,8 +5252,78 @@ class Missing: ALL = 2048 + 128 + 64 + 12 M = Missing self.assertEqual(list(CheckedMissing), [M.SIXTY_FOUR, M.ONE_TWENTY_EIGHT, M.TWENTY_FORTY_EIGHT]) - # _test_simple_enum(CheckedMissing, Missing) + # + # + class CheckedUnhashable(Enum): + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), CheckedUnhashable) + self.assertIn('python', CheckedUnhashable) + self.assertEqual(CheckedUnhashable.name.value, 'python') + self.assertEqual(CheckedUnhashable.name.name, 'name') + # + @_simple_enum() + class Unhashable: + ONE = dict() + TWO = set() + name = 'python' + self.assertIn(dict(), Unhashable) + self.assertIn('python', Unhashable) + self.assertEqual(Unhashable.name.value, 'python') + self.assertEqual(Unhashable.name.name, 'name') + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): @@ -5031,7 +5410,7 @@ def test_convert_value_lookup_priority(self): filter=lambda x: x.startswith('CONVERT_TEST_')) # We don't want the reverse lookup value to vary when there are # multiple possible names for a given value. It should always - # report the first lexigraphical name in that case. + # report the first lexicographical name in that case. self.assertEqual(test_type(5).name, 'CONVERT_TEST_NAME_A') def test_convert_int(self): @@ -5117,6 +5496,37 @@ def test_convert_repr_and_str(self): self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5') +class TestEnumDict(unittest.TestCase): + def test_enum_dict_in_metaclass(self): + """Test that EnumDict is usable as a class namespace""" + class Meta(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return EnumDict(cls) + + class MyClass(metaclass=Meta): + a = 1 + + with self.assertRaises(TypeError): + a = 2 # duplicate + + with self.assertRaises(ValueError): + _a_sunder_ = 3 + + def test_enum_dict_standalone(self): + """Test that EnumDict is usable on its own""" + enumdict = EnumDict() + enumdict['a'] = 1 + + with self.assertRaises(TypeError): + enumdict['a'] = 'other value' + + # Only MutableMapping interface is overridden for now. + # If this stops passing, update the documentation. + enumdict |= {'a': 'other value'} + self.assertEqual(enumdict['a'], 'other value') + + # helpers def enum_dir(cls): @@ -5151,7 +5561,7 @@ def member_dir(member): allowed.add(name) else: allowed.discard(name) - else: + elif name not in member._member_map_: allowed.add(name) return sorted(allowed) diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 9b5ac12bcc..8f0c1a3e25 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -8,11 +9,24 @@ from test.support import os_helper +def _create_file_shallow_equal(template_path, new_path): + """create a file with the same size and mtime but different content.""" + shutil.copy2(template_path, new_path) + with open(new_path, 'r+b') as f: + next_char = bytearray(f.read(1)) + next_char[0] = (next_char[0] + 1) % 256 + f.seek(0) + f.write(next_char) + shutil.copystat(template_path, new_path) + assert os.stat(new_path).st_size == os.stat(template_path).st_size + assert os.stat(new_path).st_mtime == os.stat(template_path).st_mtime + class FileCompareTestCase(unittest.TestCase): def setUp(self): self.name = os_helper.TESTFN self.name_same = os_helper.TESTFN + '-same' self.name_diff = os_helper.TESTFN + '-diff' + self.name_same_shallow = os_helper.TESTFN + '-same-shallow' data = 'Contents of file go here.\n' for name in [self.name, self.name_same, self.name_diff]: with open(name, 'w', encoding="utf-8") as output: @@ -20,12 +34,19 @@ def setUp(self): with open(self.name_diff, 'a+', encoding="utf-8") as output: output.write('An extra line.\n') + + for name in [self.name_same, self.name_diff]: + shutil.copystat(self.name, name) + + _create_file_shallow_equal(self.name, self.name_same_shallow) + self.dir = tempfile.gettempdir() def tearDown(self): os.unlink(self.name) os.unlink(self.name_same) os.unlink(self.name_diff) + os.unlink(self.name_same_shallow) def test_matching(self): self.assertTrue(filecmp.cmp(self.name, self.name), @@ -36,12 +57,17 @@ def test_matching(self): "Comparing file to identical file fails") self.assertTrue(filecmp.cmp(self.name, self.name_same, shallow=False), "Comparing file to identical file fails") + self.assertTrue(filecmp.cmp(self.name, self.name_same_shallow), + "Shallow identical files should be considered equal") def test_different(self): self.assertFalse(filecmp.cmp(self.name, self.name_diff), "Mismatched files compare as equal") self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + self.assertFalse(filecmp.cmp(self.name, self.name_same_shallow, + shallow=False), + "Mismatched file to shallow identical file compares as equal") def test_cache_clear(self): first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) @@ -56,6 +82,8 @@ def setUp(self): self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + self.dir_diff_file = os.path.join(tmpdir, 'dir-diff-file') + self.dir_same_shallow = os.path.join(tmpdir, 'dir-same-shallow') # Another dir is created under dir_same, but it has a name from the # ignored list so it should not affect testing results. @@ -63,7 +91,17 @@ def setUp(self): self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): + + shutil.rmtree(self.dir, True) + os.mkdir(self.dir) + subdir_path = os.path.join(self.dir, 'subdir') + os.mkdir(subdir_path) + dir_file_path = os.path.join(self.dir, "file") + with open(dir_file_path, 'w', encoding="utf-8") as output: + output.write(data) + + for dir in (self.dir_same, self.dir_same_shallow, + self.dir_diff, self.dir_diff_file): shutil.rmtree(dir, True) os.mkdir(dir) subdir_path = os.path.join(dir, 'subdir') @@ -72,14 +110,25 @@ def setUp(self): fn = 'FiLe' # Verify case-insensitive comparison else: fn = 'file' - with open(os.path.join(dir, fn), 'w', encoding="utf-8") as output: - output.write(data) + + file_path = os.path.join(dir, fn) + + if dir is self.dir_same_shallow: + _create_file_shallow_equal(dir_file_path, file_path) + else: + shutil.copy2(dir_file_path, file_path) with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: output.write('An extra file.\n') + # Add different file2 with respect to dir_diff + with open(os.path.join(self.dir_diff_file, 'file2'), 'w', encoding="utf-8") as output: + output.write('Different contents.\n') + + def tearDown(self): - for dir in (self.dir, self.dir_same, self.dir_diff): + for dir in (self.dir, self.dir_same, self.dir_diff, + self.dir_same_shallow, self.dir_diff_file): shutil.rmtree(dir) def test_default_ignores(self): @@ -102,25 +151,65 @@ def test_cmpfiles(self): shallow=False), "Comparing directory to same fails") - # Add different file2 - with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - - self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_same, + self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_diff_file, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails") + def test_cmpfiles_invalid_names(self): + # See https://github.com/python/cpython/issues/122400. + for file, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long filename'), + ]: + for other_dir in [self.dir, self.dir_same, self.dir_diff]: + with self.subTest(f'cmpfiles: {desc}', other_dir=other_dir): + res = filecmp.cmpfiles(self.dir, other_dir, [file]) + self.assertTupleEqual(res, ([], [], [file])) + + def test_dircmp_invalid_names(self): + for bad_dir, desc in [ + ('\x00', 'NUL bytes dirname'), + (f'Top{os.sep}Mid\x00', 'dirname with embedded NUL bytes'), + ("\uD834\uDD1E", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long dirname'), + ]: + d1 = filecmp.dircmp(self.dir, bad_dir) + d2 = filecmp.dircmp(bad_dir, self.dir) + for target in [ + # attributes where os.listdir() raises OSError or ValueError + 'left_list', 'right_list', + 'left_only', 'right_only', 'common', + ]: + with self.subTest(f'dircmp(ok, bad): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d1, target) + with self.subTest(f'dircmp(bad, ok): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d2, target) def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" self.assertEqual(sorted(actual), sorted(expected)) + def test_dircmp_identical_directories(self): + self._assert_dircmp_identical_directories() + self._assert_dircmp_identical_directories(shallow=False) + + def test_dircmp_different_file(self): + self._assert_dircmp_different_file() + self._assert_dircmp_different_file(shallow=False) - def test_dircmp(self): + def test_dircmp_different_directories(self): + self._assert_dircmp_different_directories() + self._assert_dircmp_different_directories(shallow=False) + + def _assert_dircmp_identical_directories(self, **options): # Check attributes for comparison of two identical directories left_dir, right_dir = self.dir, self.dir_same - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) if self.caseinsensitive: @@ -142,9 +231,10 @@ def test_dircmp(self): ] self._assert_report(d.report, expected_report) + def _assert_dircmp_different_directories(self, **options): # Check attributes for comparison of two different directories (right) left_dir, right_dir = self.dir, self.dir_diff - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'subdir']) @@ -164,12 +254,8 @@ def test_dircmp(self): self._assert_report(d.report, expected_report) # Check attributes for comparison of two different directories (left) - left_dir, right_dir = self.dir, self.dir_diff - shutil.move( - os.path.join(self.dir_diff, 'file2'), - os.path.join(self.dir, 'file2') - ) - d = filecmp.dircmp(left_dir, right_dir) + left_dir, right_dir = self.dir_diff, self.dir + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'file2', 'subdir']) @@ -180,27 +266,65 @@ def test_dircmp(self): self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, []) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), - "Only in {} : ['file2']".format(self.dir), + "diff {} {}".format(self.dir_diff, self.dir), + "Only in {} : ['file2']".format(self.dir_diff), "Identical files : ['file']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) - # Add different file2 - with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - d = filecmp.dircmp(self.dir, self.dir_diff) + + def _assert_dircmp_different_file(self, **options): + # A different file2 + d = filecmp.dircmp(self.dir_diff, self.dir_diff_file, **options) self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, ['file2']) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), + "diff {} {}".format(self.dir_diff, self.dir_diff_file), "Identical files : ['file']", "Differing files : ['file2']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) + def test_dircmp_no_shallow_different_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow, shallow=False) + self.assertEqual(d.same_files, []) + self.assertEqual(d.diff_files, ['file']) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Differing files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_same_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow) + self.assertEqual(d.same_files, ['file']) + self.assertEqual(d.diff_files, []) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Identical files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + # TODO: RUSTPYTHON + # AssertionError: "dircmp\.__init__\(\)\ takes\ from\ 3\ to\ 5\ positional\ arguments\ but\ 6\ were\ given" does not match "dircmp.__init__() takes 5 positional arguments but 6 were given + @unittest.expectedFailure + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 5c74e36a18..67c9b98d67 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -2,7 +2,7 @@ import cmath from decimal import Decimal -# from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, adjust_int_max_str_digits import math import numbers import operator @@ -19,7 +19,7 @@ #locate file with float format test values test_dir = os.path.dirname(__file__) or os.curdir -format_testfile = os.path.join(test_dir, 'formatfloat_testcases.txt') +format_testfile = os.path.join(test_dir, 'mathdata', 'formatfloat_testcases.txt') class DummyFloat(object): """Dummy float class for testing comparisons with Fractions""" @@ -97,7 +97,7 @@ def typed_approx_eq(a, b): class Symbolic: """Simple non-numeric class for testing mixed arithmetic. - It is not Integral, Rational, Real or Complex, and cannot be conveted + It is not Integral, Rational, Real or Complex, and cannot be converted to int, float or complex. but it supports some arithmetic operations. """ def __init__(self, value): @@ -331,7 +331,7 @@ def testInit(self): self.assertRaises(TypeError, F, 3, 1j) self.assertRaises(TypeError, F, 1, 2, 3) - # @requires_IEEE_754 + @requires_IEEE_754 def testInitFromFloat(self): self.assertEqual((5, 2), _components(F(2.5))) self.assertEqual((0, 1), _components(F(-0.0))) @@ -357,12 +357,14 @@ def testInitFromDecimal(self): def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((5, 1), _components(F("005"))) self.assertEqual((3, 2), _components(F("3/2"))) self.assertEqual((3, 2), _components(F("3 / 2"))) self.assertEqual((3, 2), _components(F(" \n +3/2"))) self.assertEqual((-3, 2), _components(F("-3/2 "))) - self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((13, 2), _components(F(" 0013/002 \n "))) self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((16, 5), _components(F("003.2"))) self.assertEqual((-16, 5), _components(F(" -3.2 "))) self.assertEqual((-3, 1), _components(F(" -3. "))) self.assertEqual((3, 5), _components(F(" .6 "))) @@ -381,116 +383,102 @@ def testFromString(self): self.assertRaisesMessage( ZeroDivisionError, "Fraction(3, 0)", F, "3/0") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '3/'", - F, "3/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/2'", - F, "/2") - self.assertRaisesMessage( - # Denominators don't need a sign. - ValueError, "Invalid literal for Fraction: '3/+2'", - F, "3/+2") - self.assertRaisesMessage( - # Imitate float's parsing. - ValueError, "Invalid literal for Fraction: '+ 3/2'", - F, "+ 3/2") - self.assertRaisesMessage( - # Avoid treating '.' as a regex special character. - ValueError, "Invalid literal for Fraction: '3a2'", - F, "3a2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3/7.2'", - F, "3/7.2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3.2/7'", - F, "3.2/7") - self.assertRaisesMessage( - # Allow 3. and .3, but not . - ValueError, "Invalid literal for Fraction: '.'", - F, ".") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_'", - F, "_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1'", - F, "_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2'", - F, "1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/_'", - F, "/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1_/'", - F, "1_/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1/'", - F, "_1/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2/'", - F, "1__2/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_'", - F, "1/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_1'", - F, "1/_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/1__2'", - F, "1/1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1._111'", - F, "1._111") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1__1'", - F, "1.1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+_1'", - F, "1.1e+_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+1__1'", - F, "1.1e+1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.dd'", - F, "123.dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.5_dd'", - F, "123.5_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: 'dd.5'", - F, "dd.5") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '7_dd'", - F, "7_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/dd'", - F, "1/dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/123_dd'", - F, "1/123_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789edd'", - F, "789edd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789e2_dd'", - F, "789e2_dd") + + def check_invalid(s): + msg = "Invalid literal for Fraction: " + repr(s) + self.assertRaisesMessage(ValueError, msg, F, s) + + check_invalid("3/") + check_invalid("/2") + # Denominators don't need a sign. + check_invalid("3/+2") + check_invalid("3/-2") + # Imitate float's parsing. + check_invalid("+ 3/2") + check_invalid("- 3/2") + # Avoid treating '.' as a regex special character. + check_invalid("3a2") + # Don't accept combinations of decimals and rationals. + check_invalid("3/7.2") + check_invalid("3.2/7") + # No space around dot. + check_invalid("3 .2") + check_invalid("3. 2") + # No space around e. + check_invalid("3.2 e1") + check_invalid("3.2e 1") + # Fractional part don't need a sign. + check_invalid("3.+2") + check_invalid("3.-2") + # Only accept base 10. + check_invalid("0x10") + check_invalid("0x10/1") + check_invalid("1/0x10") + check_invalid("0x10.") + check_invalid("0x10.1") + check_invalid("1.0x10") + check_invalid("1.0e0x10") + # Only accept decimal digits. + check_invalid("³") + check_invalid("³/2") + check_invalid("3/²") + check_invalid("³.2") + check_invalid("3.²") + check_invalid("3.2e²") + check_invalid("¼") + # Allow 3. and .3, but not . + check_invalid(".") + check_invalid("_") + check_invalid("_1") + check_invalid("1__2") + check_invalid("/_") + check_invalid("1_/") + check_invalid("_1/") + check_invalid("1__2/") + check_invalid("1/_") + check_invalid("1/_1") + check_invalid("1/1__2") + check_invalid("1._111") + check_invalid("1.1__1") + check_invalid("1.1e+_1") + check_invalid("1.1e+1__1") + check_invalid("123.dd") + check_invalid("123.5_dd") + check_invalid("dd.5") + check_invalid("7_dd") + check_invalid("1/dd") + check_invalid("1/123_dd") + check_invalid("789edd") + check_invalid("789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '" + val + "'", - F, val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/" + val + "'", - F, "1/" + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1." + val + "'", - F, "1." + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1+e" + val + "'", - F, "1.1+e" + val) + check_invalid(val) + check_invalid("1/" + val) + check_invalid("1." + val) + check_invalid("." + val) + check_invalid("1.1+e" + val) + check_invalid("1.1e" + val) + + def test_limit_int(self): + maxdigits = 5000 + with adjust_int_max_str_digits(maxdigits): + msg = 'Exceeds the limit' + val = '1' * maxdigits + num = (10**maxdigits - 1)//9 + self.assertEqual((num, 1), _components(F(val))) + self.assertRaisesRegex(ValueError, msg, F, val + '1') + self.assertEqual((num, 2), _components(F(val + '/2'))) + self.assertRaisesRegex(ValueError, msg, F, val + '1/2') + self.assertEqual((1, num), _components(F('1/' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1/1' + val) + self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), + _components(F('1.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1.1' + val) + self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1e1' + val) + self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) + self.assertRaisesRegex(ValueError, msg, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) @@ -922,21 +910,21 @@ def testMixedPower(self): self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) - self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(2.25, 0.0)) - self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) - self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('1.5 ** X')) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) - self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('1.5 ** X')) + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) def testMixingWithDecimal(self): @@ -1165,12 +1153,50 @@ def denominator(self): self.assertEqual(type(f.denominator), myint) def test_format_no_presentation_type(self): - # Triples (fraction, specification, expected_result) + # Triples (fraction, specification, expected_result). testcases = [ - (F(1, 3), '', '1/3'), - (F(-1, 3), '', '-1/3'), - (F(3), '', '3'), - (F(-3), '', '-3'), + # Explicit sign handling + (F(2, 3), '+', '+2/3'), + (F(-2, 3), '+', '-2/3'), + (F(3), '+', '+3'), + (F(-3), '+', '-3'), + (F(2, 3), ' ', ' 2/3'), + (F(-2, 3), ' ', '-2/3'), + (F(3), ' ', ' 3'), + (F(-3), ' ', '-3'), + (F(2, 3), '-', '2/3'), + (F(-2, 3), '-', '-2/3'), + (F(3), '-', '3'), + (F(-3), '-', '-3'), + # Padding + (F(0), '5', ' 0'), + (F(2, 3), '5', ' 2/3'), + (F(-2, 3), '5', ' -2/3'), + (F(2, 3), '0', '2/3'), + (F(2, 3), '1', '2/3'), + (F(2, 3), '2', '2/3'), + # Alignment + (F(2, 3), '<5', '2/3 '), + (F(2, 3), '>5', ' 2/3'), + (F(2, 3), '^5', ' 2/3 '), + (F(2, 3), '=5', ' 2/3'), + (F(-2, 3), '<5', '-2/3 '), + (F(-2, 3), '>5', ' -2/3'), + (F(-2, 3), '^5', '-2/3 '), + (F(-2, 3), '=5', '- 2/3'), + # Fill + (F(2, 3), 'X>5', 'XX2/3'), + (F(-2, 3), '.<5', '-2/3.'), + (F(-2, 3), '\n^6', '\n-2/3\n'), + # Thousands separators + (F(1234, 5679), ',', '1,234/5,679'), + (F(-1234, 5679), '_', '-1_234/5_679'), + (F(1234567), '_', '1_234_567'), + (F(-1234567), ',', '-1,234,567'), + # Alternate form forces a slash in the output + (F(123), '#', '123/1'), + (F(-123), '#', '-123/1'), + (F(0), '#', '0/1'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1385,11 +1411,8 @@ def test_format_f_presentation_type(self): (F('-1234.5678'), '08,.0f', '-001,235'), (F('-1234.5678'), '09,.0f', '-0,001,235'), # Corner-case - zero-padding specified through fill and align - # instead of the zero-pad character - in this case, treat '0' as a - # regular fill character and don't attempt to insert commas into - # the filled portion. This differs from the int and float - # behaviour. - (F('1234.5678'), '0=12,.2f', '00001,234.57'), + # instead of the zero-pad character. + (F('1234.5678'), '0=12,.2f', '0,001,234.57'), # Corner case where it's not clear whether the '0' indicates zero # padding or gives the minimum width, but there's still an obvious # answer to give. We want this to work in case the minimum width @@ -1534,13 +1557,17 @@ def test_invalid_formats(self): '.%', # Z instead of z for negative zero suppression 'Z.2f' + # z flag not supported for general formatting + 'z', + # zero padding not supported for general formatting + '05', ] for spec in invalid_specs: with self.subTest(spec=spec): with self.assertRaises(ValueError): format(fraction, spec) - # @requires_IEEE_754 + @requires_IEEE_754 def test_float_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: @@ -1564,6 +1591,36 @@ def test_float_format_testfile(self): self.assertEqual(float(format(f, fmt2)), float(rhs)) self.assertEqual(float(format(-f, fmt2)), float('-' + rhs)) + # TODO: RUSTPYTHON + # TypeError: '%' not supported between instances of 'Fraction' and 'complex' + @unittest.expectedFailure + def test_complex_handling(self): + # See issue gh-102840 for more details. + + a = F(1, 2) + b = 1j + message = "unsupported operand type(s) for %s: '%s' and '%s'" + # test forward + self.assertRaisesMessage(TypeError, + message % ("%", "Fraction", "complex"), + operator.mod, a, b) + self.assertRaisesMessage(TypeError, + message % ("//", "Fraction", "complex"), + operator.floordiv, a, b) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "Fraction", "complex"), + divmod, a, b) + # test reverse + self.assertRaisesMessage(TypeError, + message % ("%", "complex", "Fraction"), + operator.mod, b, a) + self.assertRaisesMessage(TypeError, + message % ("//", "complex", "Fraction"), + operator.floordiv, b, a) + self.assertRaisesMessage(TypeError, + message % ("divmod()", "complex", "Fraction"), + divmod, b, a) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index b3e4e776a4..ae4ec5dbfb 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,6 +18,7 @@ from unittest import TestCase, skipUnless from test import support +from test.support import requires_subprocess from test.support import threading_helper from test.support import socket_helper from test.support import warnings_helper @@ -32,7 +33,7 @@ DEFAULT_ENCODING = 'utf-8' # the dummy data returned by server over the data channel when # RETR, LIST, NLST, MLSD commands are issued -RETR_DATA = 'abcde12345\r\n' * 1000 + 'non-ascii char \xAE\r\n' +RETR_DATA = 'abcde\xB9\xB2\xB3\xA4\xA6\r\n' * 1000 LIST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' NLST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n' MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" @@ -67,11 +68,11 @@ class DummyDTPHandler(asynchat.async_chat): def __init__(self, conn, baseclass): asynchat.async_chat.__init__(self, conn) self.baseclass = baseclass - self.baseclass.last_received_data = '' + self.baseclass.last_received_data = bytearray() self.encoding = baseclass.encoding def handle_read(self): - new_data = self.recv(1024).decode(self.encoding, 'replace') + new_data = self.recv(1024) self.baseclass.last_received_data += new_data def handle_close(self): @@ -80,7 +81,7 @@ def handle_close(self): # (behaviour witnessed with test_data_connection) if not self.dtp_conn_closed: self.baseclass.push('226 transfer complete') - self.close() + self.shutdown() self.dtp_conn_closed = True def push(self, what): @@ -94,6 +95,9 @@ def push(self, what): def handle_error(self): default_error_handler() + def shutdown(self): + self.close() + class DummyFTPHandler(asynchat.async_chat): @@ -107,7 +111,7 @@ def __init__(self, conn, encoding=DEFAULT_ENCODING): self.in_buffer = [] self.dtp = None self.last_received_cmd = None - self.last_received_data = '' + self.last_received_data = bytearray() self.next_response = '' self.next_data = None self.rest = None @@ -226,7 +230,7 @@ def cmd_type(self, arg): def cmd_quit(self, arg): self.push('221 quit ok') - self.close() + self.shutdown() def cmd_abor(self, arg): self.push('226 abor ok') @@ -313,7 +317,7 @@ def handle_accepted(self, conn, addr): self.handler_instance = self.handler(conn, encoding=self.encoding) def handle_connect(self): - self.close() + self.shutdown() handle_read = handle_connect def writable(self): @@ -325,8 +329,8 @@ def handle_error(self): if ssl is not None: - CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") - CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + CERTFILE = os.path.join(os.path.dirname(__file__), "certdata", "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "certdata", "pycacert.pem") class SSLConnection(asyncore.dispatcher): """An asyncore.dispatcher subclass supporting TLS/SSL.""" @@ -425,12 +429,12 @@ def recv(self, buffer_size): def handle_error(self): default_error_handler() - def close(self): + def shutdown(self): if (isinstance(self.socket, ssl.SSLSocket) and self.socket._sslobj is not None): self._do_ssl_shutdown() else: - super(SSLConnection, self).close() + self.close() class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): @@ -542,8 +546,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') @@ -589,37 +593,44 @@ def test_quit(self): def test_abort(self): self.client.abort() + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_retrbinary(self): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.check_data(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.check_data(b''.join(received), + RETR_DATA.encode(self.client.encoding)) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_retrbinary_rest(self): - def callback(data): - received.append(data.decode(self.client.encoding)) for rest in (0, 10, 20): received = [] - self.client.retrbinary('retr', callback, rest=rest) - self.check_data(''.join(received), RETR_DATA[rest:]) + self.client.retrbinary('retr', received.append, rest=rest) + self.check_data(b''.join(received), + RETR_DATA[rest:].encode(self.client.encoding)) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_retrlines(self): received = [] self.client.retrlines('retr', received.append) self.check_data(''.join(received), RETR_DATA.replace('\r\n', '')) - @unittest.skip("TODO: RUSTPYTHON; weird limiting to 8192, something w/ buffering?") + @unittest.skip('TODO: RUSTPYTHON; weird limiting to 8192, something w/ buffering?') def test_storbinary(self): f = io.BytesIO(RETR_DATA.encode(self.client.encoding)) self.client.storbinary('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) self.client.storbinary('stor', f, callback=lambda x: flag.append(None)) self.assertTrue(flag) + @unittest.skip('TODO: RUSTPYTHON') + # ssl_error.SSLWantReadError: The operation did not complete (read) def test_storbinary_rest(self): data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding) f = io.BytesIO(data) @@ -628,11 +639,14 @@ def test_storbinary_rest(self): self.client.storbinary('stor', f, rest=r) self.assertEqual(self.server.handler_instance.rest, str(r)) + @unittest.skip('TODO: RUSTPYTHON') + # ssl_error.SSLWantReadError: The operation did not complete (read) def test_storlines(self): data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding) f = io.BytesIO(data) self.client.storlines('stor', f) - self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + self.check_data(self.server.handler_instance.last_received_data, + RETR_DATA.encode(self.server.encoding)) # test new callback arg flag = [] f.seek(0) @@ -644,15 +658,21 @@ def test_storlines(self): with warnings_helper.check_warnings(('', BytesWarning), quiet=True): self.assertRaises(TypeError, self.client.storlines, 'stor foo', f) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_nlst(self): self.client.nlst() self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1]) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_dir(self): l = [] - self.client.dir(lambda x: l.append(x)) + self.client.dir(l.append) self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_mlsd(self): list(self.client.mlsd()) list(self.client.mlsd(path='/')) @@ -839,6 +859,8 @@ def test_storlines_too_long(self): f = io.BytesIO(b'x' * self.client.maxline * 2) self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + @unittest.skip('TODO: RUSTPYTHON') + # TimeoutError: The read operation timed out def test_encoding_param(self): encodings = ['latin-1', 'utf-8'] for encoding in encodings: @@ -890,12 +912,10 @@ def test_makepasv(self): def test_transfer(self): def retr(): - def callback(data): - received.append(data.decode(self.client.encoding)) received = [] - self.client.retrbinary('retr', callback) - self.assertEqual(len(''.join(received)), len(RETR_DATA)) - self.assertEqual(''.join(received), RETR_DATA) + self.client.retrbinary('retr', received.append) + self.assertEqual(b''.join(received), + RETR_DATA.encode(self.client.encoding)) self.client.set_pasv(True) retr() self.client.set_pasv(False) @@ -903,7 +923,7 @@ def callback(data): @skipUnless(ssl, "SSL not available") -@unittest.skip("TODO: RUSTPYTHON; figure out why do_handshake() is throwing 'ssl session has been shut down'. SslSession object?") +@requires_subprocess() class TestTLS_FTPClassMixin(TestFTPClass): """Repeat TestFTPClass tests starting the TLS layer for both control and data connections first. @@ -920,7 +940,7 @@ def setUp(self, encoding=DEFAULT_ENCODING): @skipUnless(ssl, "SSL not available") -@unittest.skip("TODO: RUSTPYTHON; fix ssl") +@requires_subprocess() class TestTLS_FTPClass(TestCase): """Specific TLS_FTP class tests.""" @@ -1004,6 +1024,8 @@ def test_context(self): self.assertIs(sock.context, ctx) self.assertIsInstance(sock, ssl.SSLSocket) + @unittest.skip('TODO: RUSTPYTHON') + # ssl_error.SSLWantReadError: The operation did not complete (read) def test_ccc(self): self.assertRaises(ValueError, self.client.ccc) self.client.login(secure=True) diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 74ebcb2fe7..8dea56b1c0 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -39,8 +39,7 @@ def wrapper(*args, **kwargs): class TestVectorsTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def assert_hmac_internals( self, h, digest, hashname, digest_size, block_size ): @@ -50,8 +49,7 @@ def assert_hmac_internals( self.assertEqual(h.digest_size, digest_size) self.assertEqual(h.block_size, block_size) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def assert_hmac( self, key, data, digest, hashfunc, hashname, digest_size, block_size ): @@ -126,8 +124,7 @@ def assert_hmac( h, digest, hashname, digest_size, block_size ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. @@ -170,8 +167,7 @@ def md5test(key, data, digest): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha1', openssl=True) def test_sha_vectors(self): def shatest(key, data, digest): @@ -331,26 +327,22 @@ def hmactest(key, data, hexdigests): '134676fb6de0446065c97440fa8c6a58', }) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha224', openssl=True) def test_sha224_rfc4231(self): self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha256', openssl=True) def test_sha256_rfc4231(self): self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha384', openssl=True) def test_sha384_rfc4231(self): self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha512', openssl=True) def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) @@ -389,8 +381,7 @@ def test_with_digestmod_no_default(self): with self.assertRaisesRegex(TypeError, r'required.*digestmod'): hmac.HMAC(key, msg=data, digestmod='') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_with_fallback(self): cache = getattr(hashlib, '__builtin_constructor_cache') try: @@ -408,8 +399,7 @@ class ConstructorTestCase(unittest.TestCase): "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha256') def test_normal(self): # Standard constructor call. @@ -466,8 +456,7 @@ def test_withmodule(self): except Exception: self.fail("Constructor call with hashlib.sha256 raised exception.") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(C_HMAC is not None, 'need _hashlib') def test_internal_types(self): # internal types like _hashlib.C_HMAC are not constructable @@ -475,8 +464,7 @@ def test_internal_types(self): with self.assertRaisesRegex(TypeError, "immutable type"): C_HMAC.value = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(sha256_module is not None, 'need _sha256') def test_with_sha256_module(self): h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256) @@ -489,8 +477,7 @@ def test_with_sha256_module(self): class SanityTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha256') def test_exercise_all_methods(self): # Exercising all methods once. @@ -505,6 +492,14 @@ def test_exercise_all_methods(self): self.fail("Exception raised during normal usage of HMAC class.") +class UpdateTestCase(unittest.TestCase): + @hashlib_helper.requires_hashdigest('sha256') + def test_with_str_update(self): + with self.assertRaises(TypeError): + h = hmac.new(b"key", digestmod='sha256') + h.update("invalid update") + + class CopyTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha256') @@ -532,8 +527,7 @@ def test_realcopy_old(self): "No real copy of the attribute 'outer'.") self.assertIs(h1._hmac, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl") @hashlib_helper.requires_hashdigest('sha256') def test_realcopy_hmac(self): @@ -542,8 +536,7 @@ def test_realcopy_hmac(self): h2 = h1.copy() self.assertTrue(id(h1._hmac) != id(h2._hmac)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha256') def test_equality(self): # Testing if the copy has the same digests. @@ -555,8 +548,7 @@ def test_equality(self): self.assertEqual(h1.hexdigest(), h2.hexdigest(), "Hexdigest of copy doesn't match original hexdigest.") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('sha256') def test_equality_new(self): # Testing if the copy has the same digests with hmac.new(). @@ -574,8 +566,7 @@ def test_equality_new(self): class CompareDigestTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_hmac_compare_digest(self): self._test_compare_digest(hmac.compare_digest) if openssl_compare_digest is not None: @@ -586,8 +577,7 @@ def test_hmac_compare_digest(self): def test_operator_compare_digest(self): self._test_compare_digest(operator_compare_digest) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(openssl_compare_digest is None, "test requires _hashlib") def test_openssl_compare_digest(self): self._test_compare_digest(openssl_compare_digest) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 51fafd89ce..8f3fd8ec2c 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1,9 +1,16 @@ import builtins -import contextlib import errno import glob +import json import importlib.util from importlib._bootstrap_external import _get_sourcefile +from importlib.machinery import ( + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, +) import marshal import os import py_compile @@ -15,27 +22,125 @@ import textwrap import threading import time +import types import unittest from unittest import mock +import _imp from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi) + STDLIB_DIR, + swap_attr, + swap_item, + cpython_only, + is_apple_mobile, + is_emscripten, + is_wasi, + run_in_subinterp, + run_in_subinterp_with_config, + Py_TRACE_REFS, + requires_gil_enabled, + Py_GIL_DISABLED, + force_not_colorized_test_class, +) from test.support.import_helper import ( - forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) + forget, make_legacy_pyc, unlink, unload, ready_to_import, + DirsOnSysPath, CleanImport, import_module) from test.support.os_helper import ( - TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir) + TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper +from test.support.testcase import ExtraAssertions from test.test_importlib.util import uncache from types import ModuleType +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None skip_if_dont_write_bytecode = unittest.skipIf( sys.dont_write_bytecode, "test meaningful only when writing bytecode") + +def _require_loader(module, loader, skip): + if isinstance(module, str): + module = __import__(module) + + MODULE_KINDS = { + BuiltinImporter: 'built-in', + ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', + FrozenImporter: 'frozen', + SourceFileLoader: 'pure Python', + } + + expected = loader + assert isinstance(expected, type), expected + expected = MODULE_KINDS[expected] + + actual = module.__spec__.loader + if not isinstance(actual, type): + actual = type(actual) + actual = MODULE_KINDS[actual] + + if actual != expected: + err = f'expected module to be {expected}, got {module.__spec__}' + if skip: + raise unittest.SkipTest(err) + raise Exception(err) + return module + +def require_builtin(module, *, skip=False): + module = _require_loader(module, BuiltinImporter, skip) + assert module.__spec__.origin == 'built-in', module.__spec__ + +def require_extension(module, *, skip=False): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) + +def require_frozen(module, *, skip=True): + module = _require_loader(module, FrozenImporter, skip) + assert module.__spec__.origin == 'frozen', module.__spec__ + +def require_pure_python(module, *, skip=False): + _require_loader(module, SourceFileLoader, skip) + +def create_extension_loader(modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + +def import_extension_from_file(modname, filename, *, put_in_sys_modules=True): + loader = create_extension_loader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + if put_in_sys_modules: + sys.modules[modname] = module + return module + + def remove_files(name): for f in (name + ".py", name + ".pyc", @@ -45,28 +150,222 @@ def remove_files(name): rmtree('__pycache__') -@contextlib.contextmanager -def _ready_to_import(name=None, source=""): - # sets up a temporary directory and removes it - # creates the module file - # temporarily clears the module from sys.modules (if any) - # reverts or removes the module when cleaning up - name = name or "spam" - with temp_dir() as tempdir: - path = script_helper.make_script(tempdir, name, source) - old_module = sys.modules.pop(name, None) +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + + +if _testsinglephase is not None: + def restore__testsinglephase(*, _orig=_testsinglephase): + # We started with the module imported and want to restore + # it to its nominal state. + sys.modules.pop('_testsinglephase', None) + _orig._clear_globals() + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) + import _testsinglephase + + +def requires_singlephase_init(meth): + """Decorator to skip if single-phase init modules are not supported.""" + if not isinstance(meth, type): + def meth(self, _meth=meth): + try: + return _meth(self) + finally: + restore__testsinglephase() + meth = cpython_only(meth) + msg = "gh-117694: free-threaded build does not currently support single-phase init modules in sub-interpreters" + meth = requires_gil_enabled(msg)(meth) + return unittest.skipIf(_testsinglephase is None, + 'test requires _testsinglephase module')(meth) + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + +class ModuleSnapshot(types.SimpleNamespace): + """A representation of a module for testing. + + Fields: + + * id - the module's object ID + * module - the actual module or an adequate substitute + * __file__ + * __spec__ + * name + * origin + * ns - a copy (dict) of the module's __dict__ (or None) + * ns_id - the object ID of the module's __dict__ + * cached - the sys.modules[mod.__spec__.name] entry (or None) + * cached_id - the object ID of the sys.modules entry (or None) + + In cases where the value is not available (e.g. due to serialization), + the value will be None. + """ + _fields = tuple('id module ns ns_id cached cached_id'.split()) + + @classmethod + def from_module(cls, mod): + name = mod.__spec__.name + cached = sys.modules.get(name) + return cls( + id=id(mod), + module=mod, + ns=types.SimpleNamespace(**mod.__dict__), + ns_id=id(mod.__dict__), + cached=cached, + cached_id=id(cached), + ) + + SCRIPT = textwrap.dedent(''' + {imports} + + name = {name!r} + + {prescript} + + mod = {name} + + {body} + + {postscript} + ''') + IMPORTS = textwrap.dedent(''' + import sys + ''').strip() + SCRIPT_BODY = textwrap.dedent(''' + # Capture the snapshot data. + cached = sys.modules.get(name) + snapshot = dict( + id=id(mod), + module=dict( + __file__=mod.__file__, + __spec__=dict( + name=mod.__spec__.name, + origin=mod.__spec__.origin, + ), + ), + ns=None, + ns_id=id(mod.__dict__), + cached=None, + cached_id=id(cached) if cached else None, + ) + ''').strip() + CLEANUP_SCRIPT = textwrap.dedent(''' + # Clean up the module. + sys.modules.pop(name, None) + ''').strip() + + @classmethod + def build_script(cls, name, *, + prescript=None, + import_first=False, + postscript=None, + postcleanup=False, + ): + if postcleanup is True: + postcleanup = cls.CLEANUP_SCRIPT + elif isinstance(postcleanup, str): + postcleanup = textwrap.dedent(postcleanup).strip() + postcleanup = cls.CLEANUP_SCRIPT + os.linesep + postcleanup + else: + postcleanup = '' + prescript = textwrap.dedent(prescript).strip() if prescript else '' + postscript = textwrap.dedent(postscript).strip() if postscript else '' + + if postcleanup: + if postscript: + postscript = postscript + os.linesep * 2 + postcleanup + else: + postscript = postcleanup + + if import_first: + prescript += textwrap.dedent(f''' + + # Now import the module. + assert name not in sys.modules + import {name}''') + + return cls.SCRIPT.format( + imports=cls.IMPORTS.strip(), + name=name, + prescript=prescript.strip(), + body=cls.SCRIPT_BODY.strip(), + postscript=postscript, + ) + + @classmethod + def parse(cls, text): + raw = json.loads(text) + mod = raw['module'] + mod['__spec__'] = types.SimpleNamespace(**mod['__spec__']) + raw['module'] = types.SimpleNamespace(**mod) + return cls(**raw) + + @classmethod + def from_subinterp(cls, name, interpid=None, *, pipe=None, **script_kwds): + if pipe is not None: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + pipe = os.pipe() try: - sys.path.insert(0, tempdir) - yield name, path - sys.path.remove(tempdir) + return cls._from_subinterp(name, interpid, pipe, script_kwds) finally: - if old_module is not None: - sys.modules[name] = old_module - elif name in sys.modules: - del sys.modules[name] + r, w = pipe + os.close(r) + os.close(w) + + @classmethod + def _from_subinterp(cls, name, interpid, pipe, script_kwargs): + r, w = pipe + + # Build the script. + postscript = textwrap.dedent(f''' + # Send the result over the pipe. + import json + import os + os.write({w}, json.dumps(snapshot).encode()) + + ''') + _postscript = script_kwargs.get('postscript') + if _postscript: + _postscript = textwrap.dedent(_postscript).lstrip() + postscript += _postscript + script_kwargs['postscript'] = postscript.strip() + script = cls.build_script(name, **script_kwargs) + + # Run the script. + if interpid is None: + ret = run_in_subinterp(script) + if ret != 0: + raise AssertionError(f'{ret} != 0') + else: + _interpreters.run_string(interpid, script) + + # Parse the results. + text = os.read(r, 1000) + return cls.parse(text.decode()) -class ImportTests(unittest.TestCase): +@force_not_colorized_test_class +class ImportTests(unittest.TestCase, ExtraAssertions): def setUp(self): remove_files(TESTFN) @@ -87,8 +386,6 @@ def test_from_import_missing_attr_raises_ImportError(self): with self.assertRaises(ImportError): from importlib import something_that_should_not_exist_anywhere - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_has_name_and_path(self): with CleanImport('os'): import os @@ -100,15 +397,19 @@ def test_from_import_missing_attr_has_name_and_path(self): @cpython_only def test_from_import_missing_attr_has_name_and_so_path(self): - import _testcapi + _testcapi = import_module("_testcapi") with self.assertRaises(ImportError) as cm: from _testcapi import i_dont_exist self.assertEqual(cm.exception.name, '_testcapi') if hasattr(_testcapi, "__file__"): - self.assertEqual(cm.exception.path, _testcapi.__file__) + # The path on the exception is strictly the spec origin, not the + # module's __file__. For most cases, these are the same; but on + # iOS, the Framework relocation process results in the exception + # being raised from the spec location. + self.assertEqual(cm.exception.path, _testcapi.__spec__.origin) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*(\.(so|pyd))?\)" ) else: self.assertEqual( @@ -123,19 +424,15 @@ def test_from_import_missing_attr_has_name(self): self.assertEqual(cm.exception.name, '_warning') self.assertIsNone(cm.exception.path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_path_is_canonical(self): with self.assertRaises(ImportError) as cm: from os.path import i_dont_exist self.assertIn(cm.exception.name, {'posixpath', 'ntpath'}) self.assertIsNotNone(cm.exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_star_invalid_type(self): import re - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") globals = {} @@ -144,7 +441,7 @@ def test_from_import_star_invalid_type(self): ): exec(f"from {name} import *", globals) self.assertNotIn(b"invalid_type", globals) - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("globals()[b'invalid_type'] = object()") globals = {} @@ -161,18 +458,18 @@ def test_case_sensitivity(self): import RAnDoM def test_double_const(self): - # Another brief digression to test the accuracy of manifest float - # constants. - from test import double_const # don't blink -- that *was* the test + # Importing double_const checks that float constants + # serialiazed by marshal as PYC files don't lose precision + # (SF bug 422177). + from test.test_import.data import double_const + unload('test.test_import.data.double_const') + from test.test_import.data import double_const # noqa: F811 def test_import(self): def test_with_extension(ext): # The extension is normally ".py", perhaps ".pyw". source = TESTFN + ext - if is_jython: - pyc = TESTFN + "$py.class" - else: - pyc = TESTFN + ".pyc" + pyc = TESTFN + ".pyc" with open(source, "w", encoding='utf-8') as f: print("# This tests Python's ability to import a", @@ -274,7 +571,7 @@ def test_import_name_binding(self): import test as x import test.support self.assertIs(x, test, x.__name__) - self.assertTrue(hasattr(test.support, "__file__")) + self.assertHasAttr(test.support, "__file__") # import x.y.z as w binds z as w import test.support as y @@ -295,7 +592,7 @@ def test_issue31286(self): # import in a 'for' loop resulted in segmentation fault for i in range(2): - import test.support.script_helper as x + import test.support.script_helper as x # noqa: F811 def test_failing_reload(self): # A failing reload should leave the module object in sys.modules. @@ -345,7 +642,7 @@ def test_file_to_source(self): sys.path.insert(0, os.curdir) try: mod = __import__(TESTFN) - self.assertTrue(mod.__file__.endswith('.py')) + self.assertEndsWith(mod.__file__, '.py') os.remove(source) del sys.modules[TESTFN] make_legacy_pyc(source) @@ -427,8 +724,6 @@ def test_from_import_message_for_existing_module(self): with self.assertRaisesRegex(ImportError, "^cannot import name 'bogus'"): from re import bogus - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_AttributeError(self): # Issue #24492: trying to import an attribute that raises an # AttributeError should lead to an ImportError. @@ -457,7 +752,6 @@ def test_issue31492(self): with self.assertRaises(AttributeError): os.does_not_exist - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Flaky') @threading_helper.requires_working_threading() def test_concurrency(self): # bpo 38091: this is a hack to slow down the code that calls @@ -493,7 +787,6 @@ def run(): finally: del sys.path[0] - @unittest.expectedFailure @unittest.skipUnless(sys.platform == "win32", "Windows-specific") def test_dll_dependency_import(self): from _winapi import GetModuleFileName @@ -539,6 +832,443 @@ def test_dll_dependency_import(self): env=env, cwd=os.path.dirname(pyexe)) + def test_issue105979(self): + # this used to crash + with self.assertRaises(ImportError) as cm: + _imp.get_frozen_object("x", b"6\'\xd5Cu\x12") + self.assertIn("Frozen object named 'x' is invalid", + str(cm.exception)) + + def test_frozen_module_from_import_error(self): + with self.assertRaises(ImportError) as cm: + from os import this_will_never_exist + self.assertIn( + f"cannot import name 'this_will_never_exist' from 'os' ({os.__file__})", + str(cm.exception), + ) + with self.assertRaises(ImportError) as cm: + from sys import this_will_never_exist + self.assertIn( + "cannot import name 'this_will_never_exist' from 'sys' (unknown location)", + str(cm.exception), + ) + + scripts = [ + """ +import os +os.__spec__.has_location = False +os.__file__ = [] +from os import this_will_never_exist +""", + """ +import os +os.__spec__.has_location = False +del os.__file__ +from os import this_will_never_exist +""", + """ +import os +os.__spec__.origin = [] +os.__file__ = [] +from os import this_will_never_exist +""" + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"cannot import name 'this_will_never_exist' " + b"from 'os' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_non_module_from_import_error(self): + prefix = """ +import sys +class NotAModule: ... +nm = NotAModule() +nm.symbol = 123 +sys.modules["not_a_module"] = nm +from not_a_module import symbol +""" + scripts = [ + prefix + "from not_a_module import missing_symbol", + prefix + "nm.__spec__ = []\nfrom not_a_module import missing_symbol", + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"ImportError: cannot import name 'missing_symbol' from " + b"'' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_script_shadowing_stdlib(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no error at all when using -P + popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') + + tmp_child = os.path.join(tmp, "child") + os.mkdir(tmp_child) + + # test the logic with different cwd + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + def test_package_shadowing_stdlib_module(self): + script_errors = [ + ( + "fractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + os.mkdir(os.path.join(tmp, "fractions")) + with open( + os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8' + ) as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write("import fractions; fractions.shadowing_module\n") + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*[\\/]fractions[\\/]+__init__.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'main', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no shadowing at all when using -P + popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") + + def test_script_shadowing_third_party(self): + script_errors = [ + ( + "import numpy\nnumpy.array", + rb"AttributeError: module 'numpy' has no attribute 'array'" + ), + ( + "from numpy import array", + rb"ImportError: cannot import name 'array' from 'numpy'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*numpy.py' if it has the " + rb"same name as a library you intended to import\)\s+\Z" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_maybe_not_shadowing_third_party(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write("this_script_does_not_attempt_to_import_numpy = True") + + expected_error = ( + rb"AttributeError: module 'numpy' has no attribute 'attr'\s+\Z" + ) + popen = script_helper.spawn_python('-c', 'import numpy; numpy.attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + expected_error = ( + rb"ImportError: cannot import name 'attr' from 'numpy' \(.*\)\s+\Z" + ) + popen = script_helper.spawn_python('-c', 'from numpy import attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_edge_cases(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + + # Unhashable str subclass + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + fractions.Fraction +except TypeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout.rstrip(), b"unhashable type: 'substr'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + from fractions import Fraction +except TypeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout.rstrip(), b"unhashable type: 'substr'") + + # Various issues with sys module + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +sys.path = [0] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +sys.path = [0] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + # Various issues with origin + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') + @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') + def test_script_shadowing_stdlib_cwd_failure(self): + with os_helper.temp_dir() as tmp: + subtmp = os.path.join(tmp, "subtmp") + os.mkdir(subtmp) + with open(os.path.join(subtmp, "main.py"), "w", encoding='utf-8') as f: + f.write(f""" +import sys +assert sys.path[0] == '' + +import os +import shutil +shutil.rmtree(os.getcwd()) + +os.does_not_exist +""") + # Use -c to ensure sys.path[0] is "" + popen = script_helper.spawn_python("-c", "import main", cwd=subtmp) + stdout, stderr = popen.communicate() + expected_error = rb"AttributeError: module 'os' has no attribute 'does_not_exist'" + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_sys_path_modification(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write('import sys; sys.path.insert(0, "this_folder_does_not_exist")\n') + f.write(script) + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -552,7 +1282,7 @@ class FilePermissionTests(unittest.TestCase): ) def test_creation_mode(self): mask = 0o022 - with temp_umask(mask), _ready_to_import() as (name, path): + with temp_umask(mask), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) module = __import__(name) if not os.path.exists(cached_path): @@ -571,7 +1301,7 @@ def test_creation_mode(self): def test_cached_mode_issue_2051(self): # permissions of .pyc should match those of .py, regardless of mask mode = 0o600 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -587,7 +1317,7 @@ def test_cached_mode_issue_2051(self): @os_helper.skip_unless_working_chmod def test_cached_readonly(self): mode = 0o400 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -602,7 +1332,7 @@ def test_cached_readonly(self): def test_pyc_always_writable(self): # Initially read-only .pyc files on Windows used to cause problems # with later updates, see issue #6074 for details - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): # Write a Python file, make it read-only and import it with open(path, 'w', encoding='utf-8') as f: f.write("x = 'original'\n") @@ -684,8 +1414,6 @@ def test_basics(self): self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_incorrect_code_name(self): py_compile.compile(self.file_name, dfile="another_module.py") mod = self.import_module() @@ -721,7 +1449,7 @@ def test_foreign_code(self): self.assertEqual(mod.constant.co_filename, foreign_code.co_filename) -class PathsTests(unittest.TestCase): +class PathsTests(unittest.TestCase, ExtraAssertions): SAMPLES = ('test', 'test\u00e4\u00f6\u00fc\u00df', 'test\u00e9\u00e8', 'test\u00b0\u00b3\u00b2') path = TESTFN @@ -771,11 +1499,11 @@ def test_UNC_path(self): self.fail("could not import 'test_unc_path' from %r: %r" % (unc, e)) self.assertEqual(mod.testdata, 'test_unc_path') - self.assertTrue(mod.__file__.startswith(unc), mod.__file__) + self.assertStartsWith(mod.__file__, unc) unload("test_unc_path") -class RelativeImportTests(unittest.TestCase): +class RelativeImportTests(unittest.TestCase, ExtraAssertions): def tearDown(self): unload("test.relimport") @@ -784,7 +1512,7 @@ def tearDown(self): def test_relimport_star(self): # This will import * from .test_import. from .. import relimport - self.assertTrue(hasattr(relimport, "RelativeImportTests")) + self.assertHasAttr(relimport, "RelativeImportTests") def test_issue3221(self): # Note for mergers: the 'absolute' tests from the 2.x branch @@ -843,10 +1571,36 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_override_builtin(self): # Test that overriding builtins.__import__ can bypass sys.modules. import os @@ -1075,7 +1829,7 @@ def test_symlinked_dir_importable(self): @cpython_only -class ImportlibBootstrapTests(unittest.TestCase): +class ImportlibBootstrapTests(unittest.TestCase, ExtraAssertions): # These tests check that importlib is bootstrapped. def test_frozen_importlib(self): @@ -1088,7 +1842,7 @@ def test_frozen_importlib_is_bootstrap(self): self.assertIs(mod, _bootstrap) self.assertEqual(mod.__name__, 'importlib._bootstrap') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap.py') def test_frozen_importlib_external_is_bootstrap_external(self): from importlib import _bootstrap_external @@ -1096,7 +1850,7 @@ def test_frozen_importlib_external_is_bootstrap_external(self): self.assertIs(mod, _bootstrap_external) self.assertEqual(mod.__name__, 'importlib._bootstrap_external') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap_external.py') def test_there_can_be_only_one(self): # Issue #15386 revealed a tricky loophole in the bootstrapping @@ -1306,8 +2060,6 @@ def exec_module(*args): else: importlib.SourceLoader.exec_module = old_exec_module - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE') def test_unencodable_filename(self): # Issue #11619: The Python parser and the import machinery must not @@ -1358,8 +2110,6 @@ def test_rebinding(self): from test.test_import.data.circular_imports.subpkg import util self.assertIs(util.util, rebinding.util) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binding(self): try: import test.test_import.data.circular_imports.binding @@ -1370,8 +2120,6 @@ def test_crossreference1(self): import test.test_import.data.circular_imports.use import test.test_import.data.circular_imports.source - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_crossreference2(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.source @@ -1391,8 +2139,14 @@ def test_circular_from_import(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_circular_import(self): + with self.assertRaisesRegex( + AttributeError, + r"partially initialized module 'test.test_import.data.circular_imports.import_cycle' " + r"from '.*' has no attribute 'some_attribute' \(most likely due to a circular import\)" + ): + import test.test_import.data.circular_imports.import_cycle + def test_absolute_circular_submodule(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.subpkg2.parent @@ -1403,6 +2157,37 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) + @requires_singlephase_init + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_circular(self): + """Regression test for gh-123950 + + Import a single-phase-init module that imports itself + from the PyInit_* function (before it's added to sys.modules). + Manages its own cache (which is `static`, and so incompatible + with multiple interpreters or interpreter reset). + """ + name = '_testsinglephase_circular' + helper_name = 'test.test_import.data.circular_imports.singlephase' + with uncache(name, helper_name): + filename = _testsinglephase.__file__ + # We don't put the module in sys.modules: that the *inner* + # import should do that. + mod = import_extension_from_file(name, filename, + put_in_sys_modules=False) + + self.assertEqual(mod.helper_mod_name, helper_name) + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + self.assertNotIn(name, sys.modules) + self.assertNotIn(helper_name, sys.modules) + self.assertIs(mod.clear_static_var(), mod) + _testinternalcapi.clear_extension('_testsinglephase_circular', + mod.__spec__.origin) + def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") @@ -1417,6 +2202,1187 @@ def test_unwritable_module(self): unwritable.x = 42 +class SubinterpImportTests(unittest.TestCase): + + RUN_KWARGS = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + # Isolation-related config values aren't included here. + ) + ISOLATED = dict( + use_main_obmalloc=False, + gil=2, + ) + NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} + NOT_ISOLATED['gil'] = 1 + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def pipe(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + if hasattr(os, 'set_blocking'): + os.set_blocking(r, False) + return (r, w) + + def import_script(self, name, fd, filename=None, check_override=None): + override_text = '' + if check_override is not None: + override_text = f''' + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import {loader} + import os, sys + {override_text} + loader = {loader}({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + + def run_here(self, name, filename=None, *, + check_singlephase_setting=False, + check_singlephase_override=None, + isolated=False, + ): + """ + Try importing the named module in a subinterpreter. + + The subinterpreter will be in the current process. + The module will have already been imported in the main interpreter. + Thus, for extension/builtin modules, the module definition will + have been loaded already and cached globally. + + "check_singlephase_setting" determines whether or not + the interpreter will be configured to check for modules + that are not compatible with use in multiple interpreters. + + This should always return "okay" for all modules if the + setting is False (with no override). + """ + __import__(name) + + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=check_singlephase_setting, + ) + + r, w = self.pipe() + script = self.import_script(name, w, filename, + check_singlephase_override) + + ret = run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + return os.read(r, 100) + + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): + # Verify that the named module may be imported in a subinterpreter. + # (See run_here() for more info.) + out = self.run_here(name, filename, + check_singlephase_setting=strict, + isolated=isolated, + ) + self.assertEqual(out, b'okay') + + def check_incompatible_here(self, name, filename=None, *, isolated=False): + # Differences from check_compatible_here(): + # * verify that import fails + # * "strict" is always True + out = self.run_here(name, filename, + check_singlephase_setting=True, + isolated=isolated, + ) + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def check_compatible_fresh(self, name, *, strict=False, isolated=False): + # Differences from check_compatible_here(): + # * subinterpreter in a new process + # * module has never been imported before in that process + # * this tests importing the module for the first time + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=strict, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert ( + {name!r} in sys.builtin_module_names or + {name!r} not in sys.modules + ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual(out, b'okay') + + def check_incompatible_fresh(self, name, *, isolated=False): + # Differences from check_compatible_fresh(): + # * verify that import fails + # * "strict" is always True + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=True, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_builtin_compat(self): + # For now we avoid using sys or builtins + # since they still don't implement multi-phase init. + module = '_imp' + require_builtin(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @cpython_only + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_frozen_compat(self): + module = '_frozen_importlib' + require_frozen(module, skip=True) + if __import__(module).__spec__.origin != 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly not frozen') + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @requires_singlephase_init + def test_single_init_extension_compat(self): + module = '_testsinglephase' + require_extension(module) + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_incompatible_here(module) + with self.subTest(f'{module}: strict, fresh'): + self.check_incompatible_fresh(module) + with self.subTest(f'{module}: isolated, fresh'): + self.check_incompatible_fresh(module, isolated=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_compat(self): + # Module with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + module = '_testmultiphase' + require_extension(module) + + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + # Module with Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + # and Py_MOD_GIL_NOT_USED + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + + # _test_shared_gil_only: + # Explicit Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED (default) + # and Py_MOD_GIL_NOT_USED + # _test_no_multiple_interpreter_slot: + # No Py_mod_multiple_interpreters slot + # and Py_MOD_GIL_NOT_USED + for modname in ('_test_shared_gil_only', + '_test_no_multiple_interpreter_slot'): + with self.subTest(modname=modname): + + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, + isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here( + modname, filename, strict=False, isolated=False) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_python_compat(self): + module = 'threading' + require_pure_python(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @requires_singlephase_init + def test_singlephase_check_with_setting_and_override(self): + module = '_testsinglephase' + require_extension(module) + + def check_compatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertEqual(out, b'okay') + + def check_incompatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertNotEqual(out, b'okay') + + with self.subTest('config: check enabled; override: enabled'): + check_incompatible(True, 1) + with self.subTest('config: check enabled; override: use config'): + check_incompatible(True, 0) + with self.subTest('config: check enabled; override: disabled'): + check_compatible(True, -1) + + with self.subTest('config: check disabled; override: enabled'): + check_incompatible(False, 1) + with self.subTest('config: check disabled; override: use config'): + check_compatible(False, 0) + with self.subTest('config: check disabled; override: disabled'): + check_compatible(False, -1) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_isolated_config(self): + module = 'threading' + require_pure_python(module) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True, isolated=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True, isolated=True) + + @requires_subinterpreters + @requires_singlephase_init + def test_disallowed_reimport(self): + # See https://github.com/python/cpython/issues/104621. + script = textwrap.dedent(''' + import _testsinglephase + print(_testsinglephase) + ''') + interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + +class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ + + @classmethod + def from_module(cls, mod): + self = super().from_module(mod) + self.summed = mod.sum(1, 2) + self.lookedup = mod.look_up_self() + self.lookedup_id = id(self.lookedup) + self.state_initialized = mod.state_initialized() + if hasattr(mod, 'initialized_count'): + self.init_count = mod.initialized_count() + return self + + SCRIPT_BODY = ModuleSnapshot.SCRIPT_BODY + textwrap.dedent(''' + snapshot['module'].update(dict( + int_const=mod.int_const, + str_const=mod.str_const, + _module_initialized=mod._module_initialized, + )) + snapshot.update(dict( + summed=mod.sum(1, 2), + lookedup_id=id(mod.look_up_self()), + state_initialized=mod.state_initialized(), + init_count=mod.initialized_count(), + has_spam=hasattr(mod, 'spam'), + spam=getattr(mod, 'spam', None), + )) + ''').rstrip() + + @classmethod + def parse(cls, text): + self = super().parse(text) + if not self.has_spam: + del self.spam + del self.has_spam + return self + + +@requires_singlephase_init +class SinglephaseInitTests(unittest.TestCase, ExtraAssertions): + + NAME = '_testsinglephase' + + @classmethod + def setUpClass(cls): + spec = importlib.util.find_spec(cls.NAME) + cls.LOADER = type(spec.loader) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin + + # Start fresh. + cls.clean_up() + + def tearDown(self): + # Clean up the module. + self.clean_up() + + @classmethod + def clean_up(cls): + name = cls.NAME + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, cls.ORIGIN) + + ######################### + # helpers + + def add_module_cleanup(self, name): + def clean_up(): + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.ORIGIN) + self.addCleanup(clean_up) + + def _load_dynamic(self, name, path): + """ + Load an extension module. + """ + # This is essentially copied from the old imp module. + from importlib._bootstrap import _load + loader = self.LOADER(name, path) + + # Issue bpo-24748: Skip the sys.modules check in _load_module_shim; + # always load new extension. + spec = importlib.util.spec_from_file_location(name, path, + loader=loader) + return _load(spec) + + def load(self, name): + try: + already_loaded = self.already_loaded + except AttributeError: + already_loaded = self.already_loaded = {} + assert name not in already_loaded + mod = self._load_dynamic(name, self.ORIGIN) + self.assertNotIn(mod, already_loaded.values()) + already_loaded[name] = mod + return types.SimpleNamespace( + name=name, + module=mod, + snapshot=TestSinglePhaseSnapshot.from_module(mod), + ) + + def re_load(self, name, mod): + assert sys.modules[name] is mod + assert mod.__dict__ == mod.__dict__ + reloaded = self._load_dynamic(name, self.ORIGIN) + return types.SimpleNamespace( + name=name, + module=reloaded, + snapshot=TestSinglePhaseSnapshot.from_module(reloaded), + ) + + # subinterpreters + + def add_subinterpreter(self): + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' + import sys + import _testinternalcapi + ''')) + def clean_up(): + _interpreters.exec(interpid, textwrap.dedent(f''' + name = {self.NAME!r} + if name in sys.modules: + sys.modules.pop(name)._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''')) + _interpreters.destroy(interpid) + self.addCleanup(clean_up) + return interpid + + def import_in_subinterp(self, interpid=None, *, + postscript=None, + postcleanup=False, + ): + name = self.NAME + + if postcleanup: + import_ = 'import _testinternalcapi' if interpid is None else '' + postcleanup = f''' + {import_} + mod._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''' + + try: + pipe = self._pipe + except AttributeError: + r, w = pipe = self._pipe = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + snapshot = TestSinglePhaseSnapshot.from_subinterp( + name, + interpid, + pipe=pipe, + import_first=True, + postscript=postscript, + postcleanup=postcleanup, + ) + + return types.SimpleNamespace( + name=name, + module=None, + snapshot=snapshot, + ) + + # checks + + def check_common(self, loaded): + isolated = False + + mod = loaded.module + if not mod: + # It came from a subinterpreter. + isolated = True + mod = loaded.snapshot.module + # mod.__name__ might not match, but the spec will. + self.assertEqual(mod.__spec__.name, loaded.name) + self.assertEqual(mod.__file__, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) + if not isolated: + self.assertIsSubclass(mod.error, Exception) + self.assertEqual(mod.int_const, 1969) + self.assertEqual(mod.str_const, 'something different') + self.assertIsInstance(mod._module_initialized, float) + self.assertGreater(mod._module_initialized, 0) + + snap = loaded.snapshot + self.assertEqual(snap.summed, 3) + if snap.state_initialized is not None: + self.assertIsInstance(snap.state_initialized, float) + self.assertGreater(snap.state_initialized, 0) + if isolated: + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snap.lookedup_id, snap.id) + self.assertEqual(snap.cached_id, snap.id) + with self.assertRaises(AttributeError): + snap.spam + else: + self.assertIs(snap.lookedup, mod) + self.assertIs(snap.cached, mod) + + def check_direct(self, loaded): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_indirect(self, loaded, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not loaded.module + assert orig.__name__ != loaded.name + self.assertNotEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_basic(self, loaded, expected_init_count): + # m_size == -1 + # The module loads fresh the first time and copies m_copy after. + snap = loaded.snapshot + self.assertIsNot(snap.state_initialized, None) + self.assertIsInstance(snap.init_count, int) + self.assertGreater(snap.init_count, 0) + self.assertEqual(snap.init_count, expected_init_count) + + def check_with_reinit(self, loaded): + # m_size >= 0 + # The module loads fresh every time. + pass + + def check_fresh(self, loaded): + """ + The module had not been loaded before (at least since fully reset). + """ + snap = loaded.snapshot + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snap.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + + def check_semi_fresh(self, loaded, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + snap = loaded.snapshot + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertNotEqual(snap.id, prev.snapshot.id) + self.assertEqual(snap.init_count, prev.snapshot.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertNotEqual(snap.state_initialized, + base.snapshot.state_initialized) + self.assertNotEqual(snap.state_initialized, + prev.snapshot.state_initialized) + + def check_copied(self, loaded, base): + """ + The module had been loaded before and never reset. + """ + snap = loaded.snapshot + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertEqual(snap.init_count, base.snapshot.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertEqual(snap.state_initialized, + base.snapshot.state_initialized) + + ######################### + # the tests + + def test_cleared_globals(self): + loaded = self.load(self.NAME) + _testsinglephase = loaded.module + init_before = _testsinglephase.state_initialized() + + _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() + init_count = _testsinglephase.initialized_count() + + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) + self.assertEqual(init_count, -1) + + def test_variants(self): + # Exercise the most meaningful variants described in Python/import.c. + self.maxDiff = None + + # Check the "basic" module. + + name = self.NAME + expected_init_count = 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + basic = loaded.module + + # Check its indirect variants. + + name = f'{self.NAME}_basic_wrapper' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_indirect(loaded, basic) + self.check_basic(loaded, expected_init_count) + + # Currently PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), loaded.module) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't change after this point. + basic_lookedup = loaded.module + + # Check its direct variant. + + name = f'{self.NAME}_basic_copy' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{self.NAME}_with_reinit' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIs(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{self.NAME}_with_state' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + self.addCleanup(loaded.module._clear_module_state) + + self.check_common(loaded) + self.assertIsNot(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + def test_basic_reloaded(self): + # m_copy is copied into the existing module object. + # Global state is not changed. + self.maxDiff = None + + for name in [ + self.NAME, # the "basic" module + f'{self.NAME}_basic_wrapper', # the indirect variant + f'{self.NAME}_basic_copy', # the direct variant + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIs(reloaded.module, loaded.module) + self.assertIs(reloaded.module.__dict__, loaded.module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(reloaded.snapshot.lookedup, + loaded.snapshot.lookedup) + self.assertEqual(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + self.assertEqual(reloaded.snapshot.init_count, + loaded.snapshot.init_count) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + def test_with_reinit_reloaded(self): + # The module's m_init func is run again. + self.maxDiff = None + + # Keep a reference around. + basic = self.load(self.NAME) + + for name, has_state in [ + (f'{self.NAME}_with_reinit', False), # m_size == 0 + (f'{self.NAME}_with_state', True), # m_size > 0 + ]: + self.add_module_cleanup(name) + with self.subTest(name=name, has_state=has_state): + loaded = self.load(name) + if has_state: + self.addCleanup(loaded.module._clear_module_state) + + reloaded = self.re_load(name, loaded.module) + if has_state: + self.addCleanup(reloaded.module._clear_module_state) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIsNot(reloaded.module, loaded.module) + self.assertNotEqual(reloaded.module.__dict__, + loaded.module.__dict__) + self.assertIs(reloaded.snapshot.lookedup, reloaded.module) + if loaded.snapshot.state_initialized is None: + self.assertIs(reloaded.snapshot.state_initialized, None) + else: + self.assertGreater(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) + + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + # Also, we test with a single-phase module that has global state, + # which is shared by all interpreters. + + @no_rerun(reason="module state is not cleared (see gh-140657)") + @requires_subinterpreters + def test_basic_multiple_interpreters_main_no_reset(self): + # without resetting; already loaded in main interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + main_loaded = self.load(self.NAME) + _testsinglephase = main_loaded.module + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + + self.check_common(main_loaded) + self.check_fresh(main_loaded) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # At this point: + # * alive in 1 interpreter (main) + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy was copied from the main interpreter (was NULL) + # * module's global state was initialized + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp() + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 1 interpreter (main) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (claered when the interpreter was destroyed) + # (was from main interpreter) + # * module's global state was updated, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 2 interpreters (main, interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 3 interpreters (main, interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") + @requires_subinterpreters + def test_basic_multiple_interpreters_deleted_no_reset(self): + # without resetting; already loaded in a deleted interpreter + + if Py_TRACE_REFS: + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # First, load in the main interpreter but then completely clear it. + loaded_main = self.load(self.NAME) + loaded_main.module._clear_globals() + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) + + # At this point: + # * alive in 0 interpreters + # * module def loaded already + # * module def was in _PyRuntime.imports.extensions, but cleared + # * mod init func ran for the first time (since reset, at least) + # * m_copy was set, but cleared (was NULL) + # * module's global state was initialized but cleared + + # Start with an interpreter that gets destroyed right away. + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') + self.check_common(base) + self.check_fresh(base) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset) + # * m_copy is still set (owned by main interpreter) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded_interp1 = self.import_in_subinterp(interpid1) + self.check_common(loaded_interp1) + self.check_copied(loaded_interp1, base) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + # Use a subinterpreter while the previous one is still alive. + loaded_interp2 = self.import_in_subinterp(interpid2) + self.check_common(loaded_interp2) + self.check_copied(loaded_interp2, loaded_interp1) + + # At this point: + # * alive in 2 interpreters (interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + @requires_subinterpreters + def test_basic_multiple_interpreters_reset_each(self): + # resetting between each interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''', + postcleanup=True, + ) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy is NULL (claered when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was initialized, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 2 interpreters (interp2, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was initialized, not reset + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_import/data/circular_imports/import_cycle.py b/Lib/test/test_import/data/circular_imports/import_cycle.py new file mode 100644 index 0000000000..cd9507b5f6 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/import_cycle.py @@ -0,0 +1,3 @@ +import test.test_import.data.circular_imports.import_cycle as m + +m.some_attribute diff --git a/Lib/test/test_import/data/circular_imports/singlephase.py b/Lib/test/test_import/data/circular_imports/singlephase.py new file mode 100644 index 0000000000..05618bc72f --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/singlephase.py @@ -0,0 +1,13 @@ +"""Circular import involving a single-phase-init extension. + +This module is imported from the _testsinglephase_circular module from +_testsinglephase, and imports that module again. +""" + +import importlib +import _testsinglephase +from test.test_import import import_extension_from_file + +name = '_testsinglephase_circular' +filename = _testsinglephase.__file__ +mod = import_extension_from_file(name, filename) diff --git a/Lib/test/test_import/data/double_const.py b/Lib/test/test_import/data/double_const.py new file mode 100644 index 0000000000..67852aaf98 --- /dev/null +++ b/Lib/test/test_import/data/double_const.py @@ -0,0 +1,30 @@ +from test.support import TestFailed + +# A test for SF bug 422177: manifest float constants varied way too much in +# precision depending on whether Python was loading a module for the first +# time, or reloading it from a precompiled .pyc. The "expected" failure +# mode is that when test_import imports this after all .pyc files have been +# erased, it passes, but when test_import imports this from +# double_const.pyc, it fails. This indicates a woeful loss of precision in +# the marshal format for doubles. It's also possible that repr() doesn't +# produce enough digits to get reasonable precision for this box. + +PI = 3.14159265358979324 +TWOPI = 6.28318530717958648 + +PI_str = "3.14159265358979324" +TWOPI_str = "6.28318530717958648" + +# Verify that the double x is within a few bits of eval(x_str). +def check_ok(x, x_str): + assert x > 0.0 + x2 = eval(x_str) + assert x2 > 0.0 + diff = abs(x - x2) + # If diff is no larger than 3 ULP (wrt x2), then diff/8 is no larger + # than 0.375 ULP, so adding diff/8 to x2 should have no effect. + if x2 + (diff / 8.) != x2: + raise TestFailed("Manifest const %s lost too much precision " % x_str) + +check_ok(PI, PI_str) +check_ok(TWOPI, TWOPI_str) diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 0000000000..7033c22a71 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 0000000000..cd7b30db15 --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 0000000000..d8af60ab38 --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 0000000000..c861417aec --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py new file mode 100644 index 0000000000..940baf3941 --- /dev/null +++ b/Lib/test/test_mailbox.py @@ -0,0 +1,2495 @@ +import os +import sys +import time +import stat +import socket +import email +import email.message +import re +import io +import tempfile +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import refleak_helper +from test.support import socket_helper +from test.support.testcase import ExtraAssertions +import unittest +import textwrap +import mailbox +import glob + + +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + +class TestBase: + + all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage, + mailbox.mboxMessage, mailbox.MHMessage, + mailbox.BabylMessage, mailbox.MMDFMessage) + + def _check_sample(self, msg): + # Inspect a mailbox.Message representation of the sample message + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + for key, value in _sample_headers: + self.assertIn(value, msg.get_all(key)) + self.assertTrue(msg.is_multipart()) + self.assertEqual(len(msg.get_payload()), len(_sample_payloads)) + for i, payload in enumerate(_sample_payloads): + part = msg.get_payload(i) + self.assertIsInstance(part, email.message.Message) + self.assertNotIsInstance(part, mailbox.Message) + self.assertEqual(part.get_payload(), payload) + + def _delete_recursively(self, target): + # Delete a file or delete a directory recursively + if os.path.isdir(target): + os_helper.rmtree(target) + elif os.path.exists(target): + os_helper.unlink(target) + + +class TestMailbox(TestBase): + + maxDiff = None + + _factory = None # Overridden by subclasses to reuse tests + _template = 'From: foo\n\n%s\n' + + def setUp(self): + self._path = os_helper.TESTFN + self._delete_recursively(self._path) + self._box = self._factory(self._path) + + def tearDown(self): + self._box.close() + self._delete_recursively(self._path) + + def test_add(self): + # Add copies of a sample message + keys = [] + keys.append(self._box.add(self._template % 0)) + self.assertEqual(len(self._box), 1) + keys.append(self._box.add(mailbox.Message(_sample_message))) + self.assertEqual(len(self._box), 2) + keys.append(self._box.add(email.message_from_string(_sample_message))) + self.assertEqual(len(self._box), 3) + keys.append(self._box.add(io.BytesIO(_bytes_sample_message))) + self.assertEqual(len(self._box), 4) + keys.append(self._box.add(_sample_message)) + self.assertEqual(len(self._box), 5) + keys.append(self._box.add(_bytes_sample_message)) + self.assertEqual(len(self._box), 6) + with self.assertWarns(DeprecationWarning): + keys.append(self._box.add( + io.TextIOWrapper(io.BytesIO(_bytes_sample_message), encoding="utf-8"))) + self.assertEqual(len(self._box), 7) + self.assertEqual(self._box.get_string(keys[0]), self._template % 0) + for i in (1, 2, 3, 4, 5, 6): + self._check_sample(self._box[keys[i]]) + + _nonascii_msg = textwrap.dedent("""\ + From: foo + Subject: Falinaptár házhozszállítással. Már rendeltél? + + 0 + """) + + def test_add_invalid_8bit_bytes_header(self): + key = self._box.add(self._nonascii_msg.encode('latin-1')) + self.assertEqual(len(self._box), 1) + self.assertEqual(self._box.get_bytes(key), + self._nonascii_msg.encode('latin-1')) + + def test_invalid_nonascii_header_as_string(self): + subj = self._nonascii_msg.splitlines()[1] + key = self._box.add(subj.encode('latin-1')) + self.assertEqual(self._box.get_string(key), + 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' + 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') + + def test_add_nonascii_string_header_raises(self): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(self._nonascii_msg) + self._box.flush() + self.assertEqual(len(self._box), 0) + self.assertMailboxEmpty() + + def test_add_that_raises_leaves_mailbox_empty(self): + class CustomError(Exception): ... + exc_msg = "a fake error" + + def raiser(*args, **kw): + raise CustomError(exc_msg) + support.patch(self, email.generator.BytesGenerator, 'flatten', raiser) + with self.assertRaisesRegex(CustomError, exc_msg): + self._box.add(email.message_from_string("From: Alphöso")) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + _non_latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: báz + Subject: Maintenant je vous présente mon collègue, le pouf célèbre + \tJean de Baddie + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Да, они летят. + """).encode('utf-8') + + def test_add_8bit_body(self): + key = self._box.add(self._non_latin_bin_msg) + self.assertEqual(self._box.get_bytes(key), + self._non_latin_bin_msg) + with self._box.get_file(key) as f: + self.assertEqual(f.read(), + self._non_latin_bin_msg.replace(b'\n', + os.linesep.encode())) + self.assertEqual(self._box[key].get_payload(), + "Да, они летят.\n") + + def test_add_binary_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_binary_nonascii_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(self._non_latin_bin_msg) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + self._non_latin_bin_msg.split(b'\n')) + + def test_add_text_file_warns(self): + with tempfile.TemporaryFile('w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + with self.assertWarns(DeprecationWarning): + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_StringIO_warns(self): + with self.assertWarns(DeprecationWarning): + key = self._box.add(io.StringIO(self._template % "0")) + self.assertEqual(self._box.get_string(key), self._template % "0") + + def test_add_nonascii_StringIO_raises(self): + with self.assertWarns(DeprecationWarning): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(io.StringIO(self._nonascii_msg)) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + def test_remove(self): + # Remove messages using remove() + self._test_remove_or_delitem(self._box.remove) + + def test_delitem(self): + # Remove messages using __delitem__() + self._test_remove_or_delitem(self._box.__delitem__) + + def _test_remove_or_delitem(self, method): + # (Used by test_remove() and test_delitem().) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + method(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self.assertRaises(KeyError, lambda: method(key0)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + key2 = self._box.add(self._template % 2) + self.assertEqual(len(self._box), 2) + method(key2) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key2]) + self.assertRaises(KeyError, lambda: method(key2)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + method(key1) + self.assertEqual(len(self._box), 0) + self.assertRaises(KeyError, lambda: self._box[key1]) + self.assertRaises(KeyError, lambda: method(key1)) + + def test_discard(self, repetitions=10): + # Discard messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get(self): + # Retrieve messages using get() + key0 = self._box.add(self._template % 0) + msg = self._box.get(key0) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertIsNone(self._box.get('foo')) + self.assertIs(self._box.get('foo', False), False) + self._box.close() + self._box = self._factory(self._path) + key1 = self._box.add(self._template % 1) + msg = self._box.get(key1) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '1\n') + + def test_getitem(self): + # Retrieve message using __getitem__() + key0 = self._box.add(self._template % 0) + msg = self._box[key0] + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertRaises(KeyError, lambda: self._box['foo']) + self._box.discard(key0) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get_message(self): + # Get Message representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + msg0 = self._box.get_message(key0) + self.assertIsInstance(msg0, mailbox.Message) + self.assertEqual(msg0['from'], 'foo') + self.assertEqual(msg0.get_payload(), '0\n') + self._check_sample(self._box.get_message(key1)) + + def test_get_bytes(self): + # Get bytes representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_bytes(key0), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1), _bytes_sample_message) + + def test_get_string(self): + # Get string representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_string(key0), self._template % 0) + self.assertEqual(self._box.get_string(key1).split('\n'), + _sample_message.split('\n')) + + def test_get_file(self): + # Get file representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + with self._box.get_file(key0) as file: + data0 = file.read() + with self._box.get_file(key1) as file: + data1 = file.read() + self.assertEqual(data0.decode('ascii').replace(os.linesep, '\n'), + self._template % 0) + self.assertEqual(data1.decode('ascii').replace(os.linesep, '\n'), + _sample_message) + + def test_get_file_can_be_closed_twice(self): + # Issue 11700 + key = self._box.add(_sample_message) + f = self._box.get_file(key) + f.close() + f.close() + + def test_iterkeys(self): + # Get keys using iterkeys() + self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False) + + def test_keys(self): + # Get keys using keys() + self._check_iteration(self._box.keys, do_keys=True, do_values=False) + + def test_itervalues(self): + # Get values using itervalues() + self._check_iteration(self._box.itervalues, do_keys=False, + do_values=True) + + def test_iter(self): + # Get values using __iter__() + self._check_iteration(self._box.__iter__, do_keys=False, + do_values=True) + + def test_values(self): + # Get values using values() + self._check_iteration(self._box.values, do_keys=False, do_values=True) + + def test_iteritems(self): + # Get keys and values using iteritems() + self._check_iteration(self._box.iteritems, do_keys=True, + do_values=True) + + def test_items(self): + # Get keys and values using items() + self._check_iteration(self._box.items, do_keys=True, do_values=True) + + def _check_iteration(self, method, do_keys, do_values, repetitions=10): + for value in method(): + self.fail("Not empty") + keys, values = [], [] + for i in range(repetitions): + keys.append(self._box.add(self._template % i)) + values.append(self._template % i) + if do_keys and not do_values: + returned_keys = list(method()) + elif do_values and not do_keys: + returned_values = list(method()) + else: + returned_keys, returned_values = [], [] + for key, value in method(): + returned_keys.append(key) + returned_values.append(value) + if do_keys: + self.assertEqual(len(keys), len(returned_keys)) + self.assertEqual(set(keys), set(returned_keys)) + if do_values: + count = 0 + for value in returned_values: + self.assertEqual(value['from'], 'foo') + self.assertLess(int(value.get_payload()), repetitions) + count += 1 + self.assertEqual(len(values), count) + + def test_contains(self): + # Check existence of keys using __contains__() + self.assertNotIn('foo', self._box) + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key0) + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key1) + self.assertNotIn(key1, self._box) + self.assertNotIn(key0, self._box) + self.assertNotIn('foo', self._box) + + def test_len(self, repetitions=10): + # Get message count + keys = [] + for i in range(repetitions): + self.assertEqual(len(self._box), i) + keys.append(self._box.add(self._template % i)) + self.assertEqual(len(self._box), i + 1) + for i in range(repetitions): + self.assertEqual(len(self._box), repetitions - i) + self._box.remove(keys[i]) + self.assertEqual(len(self._box), repetitions - i - 1) + + def test_set_item(self): + # Modify messages using __setitem__() + key0 = self._box.add(self._template % 'original 0') + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._box[key0] = self._template % 'changed 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self._box[key1] = self._template % 'changed 1' + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self._box[key0] = _sample_message + self._check_sample(self._box[key0]) + self._box[key1] = self._box[key0] + self._check_sample(self._box[key1]) + self._box[key0] = self._template % 'original 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self._check_sample(self._box[key1]) + self.assertRaises(KeyError, + lambda: self._box.__setitem__('foo', 'bar')) + self.assertRaises(KeyError, lambda: self._box['foo']) + self.assertEqual(len(self._box), 2) + + def test_clear(self, iterations=10): + # Remove all messages using clear() + keys = [] + for i in range(iterations): + self._box.add(self._template % i) + for i, key in enumerate(keys): + self.assertEqual(self._box.get_string(key), self._template % i) + self._box.clear() + self.assertEqual(len(self._box), 0) + for i, key in enumerate(keys): + self.assertRaises(KeyError, lambda: self._box.get_string(key)) + + def test_pop(self): + # Get and remove a message using pop() + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key0).get_payload(), '0\n') + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + key2 = self._box.add(self._template % 2) + self.assertIn(key2, self._box) + self.assertEqual(self._box.pop(key2).get_payload(), '2\n') + self.assertNotIn(key2, self._box) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key1).get_payload(), '1\n') + self.assertNotIn(key1, self._box) + self.assertEqual(len(self._box), 0) + + def test_popitem(self, iterations=10): + # Get and remove an arbitrary (key, message) using popitem() + keys = [] + for i in range(10): + keys.append(self._box.add(self._template % i)) + seen = [] + for i in range(10): + key, msg = self._box.popitem() + self.assertIn(key, keys) + self.assertNotIn(key, seen) + seen.append(key) + self.assertEqual(int(msg.get_payload()), keys.index(key)) + self.assertEqual(len(self._box), 0) + for key in keys: + self.assertRaises(KeyError, lambda: self._box[key]) + + def test_update(self): + # Modify multiple messages using update() + key0 = self._box.add(self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + key2 = self._box.add(self._template % 'original 2') + self._box.update({key0: self._template % 'changed 0', + key2: _sample_message}) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._check_sample(self._box[key2]) + self._box.update([(key2, self._template % 'changed 2'), + (key1, self._template % 'changed 1'), + (key0, self._template % 'original 0')]) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self.assertEqual(self._box.get_string(key2), + self._template % 'changed 2') + self.assertRaises(KeyError, + lambda: self._box.update({'foo': 'bar', + key0: self._template % "changed 0"})) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % "changed 0") + self.assertEqual(self._box.get_string(key1), + self._template % "changed 1") + self.assertEqual(self._box.get_string(key2), + self._template % "changed 2") + + def test_flush(self): + # Write changes to disk + self._test_flush_or_close(self._box.flush, True) + + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + + def test_lock_unlock(self): + # Lock and unlock the mailbox + self.assertFalse(os.path.exists(self._get_lock_path())) + self._box.lock() + self.assertTrue(os.path.exists(self._get_lock_path())) + self._box.unlock() + self.assertFalse(os.path.exists(self._get_lock_path())) + + def test_close(self): + # Close mailbox and flush changes to disk + self._test_flush_or_close(self._box.close, False) + + def _test_flush_or_close(self, method, should_call_close): + contents = [self._template % i for i in range(3)] + self._box.add(contents[0]) + self._box.add(contents[1]) + self._box.add(contents[2]) + oldbox = self._box + method() + if should_call_close: + self._box.close() + self._box = self._factory(self._path) + keys = self._box.keys() + self.assertEqual(len(keys), 3) + for key in keys: + self.assertIn(self._box.get_string(key), contents) + oldbox.close() + + def test_dump_message(self): + # Write message representations to disk + for input in (email.message_from_string(_sample_message), + _sample_message, io.BytesIO(_bytes_sample_message)): + output = io.BytesIO() + self._box._dump_message(input, output) + self.assertEqual(output.getvalue(), + _bytes_sample_message.replace(b'\n', os.linesep.encode())) + output = io.BytesIO() + self.assertRaises(TypeError, + lambda: self._box._dump_message(None, output)) + + def _get_lock_path(self): + # Return the path of the dot lock file. May be overridden. + return self._path + '.lock' + + +class TestMailboxSuperclass(TestBase, unittest.TestCase): + + def test_notimplemented(self): + # Test that all Mailbox methods raise NotImplementedException. + box = mailbox.Mailbox('path') + self.assertRaises(NotImplementedError, lambda: box.add('')) + self.assertRaises(NotImplementedError, lambda: box.remove('')) + self.assertRaises(NotImplementedError, lambda: box.__delitem__('')) + self.assertRaises(NotImplementedError, lambda: box.discard('')) + self.assertRaises(NotImplementedError, lambda: box.__setitem__('', '')) + self.assertRaises(NotImplementedError, lambda: box.iterkeys()) + self.assertRaises(NotImplementedError, lambda: box.keys()) + self.assertRaises(NotImplementedError, lambda: box.itervalues().__next__()) + self.assertRaises(NotImplementedError, lambda: box.__iter__().__next__()) + self.assertRaises(NotImplementedError, lambda: box.values()) + self.assertRaises(NotImplementedError, lambda: box.iteritems().__next__()) + self.assertRaises(NotImplementedError, lambda: box.items()) + self.assertRaises(NotImplementedError, lambda: box.get('')) + self.assertRaises(NotImplementedError, lambda: box.__getitem__('')) + self.assertRaises(NotImplementedError, lambda: box.get_message('')) + self.assertRaises(NotImplementedError, lambda: box.get_string('')) + self.assertRaises(NotImplementedError, lambda: box.get_bytes('')) + self.assertRaises(NotImplementedError, lambda: box.get_file('')) + self.assertRaises(NotImplementedError, lambda: '' in box) + self.assertRaises(NotImplementedError, lambda: box.__contains__('')) + self.assertRaises(NotImplementedError, lambda: box.__len__()) + self.assertRaises(NotImplementedError, lambda: box.clear()) + self.assertRaises(NotImplementedError, lambda: box.pop('')) + self.assertRaises(NotImplementedError, lambda: box.popitem()) + self.assertRaises(NotImplementedError, lambda: box.update((('', ''),))) + self.assertRaises(NotImplementedError, lambda: box.flush()) + self.assertRaises(NotImplementedError, lambda: box.lock()) + self.assertRaises(NotImplementedError, lambda: box.unlock()) + self.assertRaises(NotImplementedError, lambda: box.close()) + + +class TestMaildir(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory) + + def setUp(self): + TestMailbox.setUp(self) + if (os.name == 'nt') or (sys.platform == 'cygwin'): + self._box.colon = '!' + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(os.path.join(self._path, 'tmp')), []) + + def test_add_MM(self): + # Add a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_info('foo') + key = self._box.add(msg) + self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' % + (key, self._box.colon)))) + + def test_get_MM(self): + # Get a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + msg_returned = self._box.get_message(key) + self.assertIsInstance(msg_returned, mailbox.MaildirMessage) + self.assertEqual(msg_returned.get_subdir(), 'cur') + self.assertEqual(msg_returned.get_flags(), 'FR') + + def test_set_MM(self): + # Set with a MaildirMessage instance + msg0 = mailbox.MaildirMessage(self._template % 0) + msg0.set_flags('TP') + key = self._box.add(msg0) + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'PT') + msg1 = mailbox.MaildirMessage(self._template % 1) + self._box[key] = msg1 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), '') + self.assertEqual(msg_returned.get_payload(), '1\n') + msg2 = mailbox.MaildirMessage(self._template % 2) + msg2.set_info('2,S') + self._box[key] = msg2 + self._box[key] = self._template % 3 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'S') + self.assertEqual(msg_returned.get_payload(), '3\n') + + def test_consistent_factory(self): + # Add a message. + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + + # Create new mailbox with + class FakeMessage(mailbox.MaildirMessage): + pass + box = mailbox.Maildir(self._path, factory=FakeMessage) + box.colon = self._box.colon + msg2 = box.get_message(key) + self.assertIsInstance(msg2, FakeMessage) + + def test_initialize_new(self): + # Initialize a non-existent mailbox + self.tearDown() + self._box = mailbox.Maildir(self._path) + self._check_basics() + self._delete_recursively(self._path) + self._box = self._factory(self._path, factory=None) + self._check_basics() + + def test_initialize_existing(self): + # Initialize an existing mailbox + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + self._box = mailbox.Maildir(self._path) + self._check_basics() + + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + + def _check_basics(self, factory=None): + # (Used by test_open_new() and test_open_existing().) + self.assertEqual(self._box._path, os.path.abspath(self._path)) + self.assertEqual(self._box._factory, factory) + for subdir in '', 'tmp', 'new', 'cur': + path = os.path.join(self._path, subdir) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_clean(self): + # Remove old files from 'tmp' + foo_path = os.path.join(self._path, 'tmp', 'foo') + bar_path = os.path.join(self._path, 'tmp', 'bar') + with open(foo_path, 'w', encoding='utf-8') as f: + f.write("@") + with open(bar_path, 'w', encoding='utf-8') as f: + f.write("@") + self._box.clean() + self.assertTrue(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + foo_stat = os.stat(foo_path) + os.utime(foo_path, (time.time() - 129600 - 2, + foo_stat.st_mtime)) + self._box.clean() + self.assertFalse(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + + def test_create_tmp(self, repetitions=10): + # Create files in tmp directory + hostname = socket.gethostname() + if '/' in hostname: + hostname = hostname.replace('/', r'\057') + if ':' in hostname: + hostname = hostname.replace(':', r'\072') + pid = os.getpid() + pattern = re.compile(r"(?P