[New-bugs-announce] [issue41451] Class with __weakref__ slot cannot inherit from multiple typing.Generic classes

[Joshua Bronson]

New submission from Joshua Bronson <jabronson at gmail.com>:

This appears to be a bug in Python 3.6 that I hit while trying to add type hints to my bidirectional mapping library (https://bidict.rtfd.io).

Pasting a working, minimal repro below for easier inline viewing, and also attaching for easier downloading and running.

Please let me know if there is a workaround that would allow me to continue to support Python 3.6 after adding type hints without having to remove the use of slots and weak references. I spent a while trying to find one first but was then encouraged to report this by @ethanhs.

Thanks in advance for any pointers you may have.

#!/usr/bin/env python3
"""Repro for Python 3.6 slots + weakref + typing.Generic bug."""

from typing import Iterator, Mapping, MutableMapping, TypeVar
from weakref import ref


KT1 = TypeVar("KT1")
KT2 = TypeVar("KT2")


class Invertible(Mapping[KT1, KT2]):
    """A one-element mapping that is generic in two key types with a reference to its inverse.

    ...which in turn holds a (weak) reference back to it.

        >>> element = Invertible("H", 1)
        >>> element
        <Invertible key1='H' key2=1>
        >>> element.inverse
        <Invertible key1=1 key2='H'>
        >>> element.inverse.inverse
        <Invertible key1='H' key2=1>
        >>> element.inverse.inverse is element
        True

        >>> dict(element.items())
        {'H': 1}
        >>> dict(element.inverse.items())
        {1: 'H'}
        >>> list(element)
        ['H']

    Uses the __slots__ optimization, and uses weakrefs for references in one direction
    to avoid strong reference cycles. And still manages to support pickling to boot!

        >>> from pickle import dumps, loads
        >>> pickled = dumps(element)
        >>> roundtripped = loads(pickled)
        >>> roundtripped
        <Invertible key1='H' key2=1>

    """

    # Each instance has (either a strong or a weak) reference to its
    # inverse instance, which has a (weak or strong) reference back.
    __slots__ = ("_inverse_strong", "_inverse_weak", "__weakref__", "key1", "key2")

    def __init__(self, key1: KT1, key2: KT2) -> None:
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        self.key1 = inverse.key2 = key1
        self.key2 = inverse.key1 = key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)

    def __len__(self) -> int:
        return 1

    def __iter__(self) -> Iterator[KT1]:
        yield self.key1

    def __getitem__(self, key: KT1) -> KT2:
        if key == self.key1:
            return self.key2
        raise KeyError(key)

    def __repr__(self) -> str:
        return f"<{self.__class__.__name__} key1={self.key1!r} key2={self.key2!r}>"

    @property
    def inverse(self) -> "Invertible[KT2, KT1]":
        """The inverse instance."""
        if self._inverse_strong is not None:
            return self._inverse_strong
        inverse = self._inverse_weak()
        if inverse is not None:
            return inverse
        # Refcount of referent must have dropped to zero,
        # as in `Invertible().inverse.inverse`, so init a new one.
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        inverse.key2 = self.key1
        inverse.key1 = self.key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)
        return inverse

    def __getstate__(self) -> dict:
        """Needed to enable pickling due to use of __slots__ and weakrefs."""
        state = {}
        for cls in self.__class__.__mro__:
            slots = getattr(cls, '__slots__', ())
            for slot in slots:
                if hasattr(self, slot):
                    state[slot] = getattr(self, slot)
        # weakrefs can't be pickled.
        state.pop('_inverse_weak', None)  # Added back in __setstate__ below.
        state.pop('__weakref__', None)  # Not added back in __setstate__. Python manages this one.
        return state

    def __setstate__(self, state) -> None:
        """Needed because use of __slots__ would prevent unpickling otherwise."""
        for slot, value in state.items():
            setattr(self, slot, value)
        self._inverse_weak = None
        self._inverse_strong = inverse = self.__class__.__new__(self.__class__)
        inverse.key2 = self.key1
        inverse.key1 = self.key2
        inverse._inverse_strong = None
        inverse._inverse_weak = ref(self)


# So far so good, but now let's make a mutable version.
# 
# The following class definition works on Python > 3.6, but fails on 3.6 with
# TypeError: __weakref__ slot disallowed: either we already got one, or __itemsize__ != 0

class MutableInvertible(Invertible[KT1, KT2], MutableMapping[KT1, KT2]):
    """Works on > 3.6, but we don't even get this far on Python 3.6:

        >>> MutableInvertible("H", 1)
        <MutableInvertible key1='H' key2=1>

    """

    __slots__ = ()

    def __setitem__(self, key1: KT1, key2: KT2) -> None:
        self.key1 = self.inverse.key2 = key1
        self.key2 = self.inverse.key1 = key2

    def __delitem__(self, key: KT1) -> None:
        raise KeyError(key)


if __name__ == "__main__":
    import doctest
    doctest.testmod()

----------
components: Library (Lib)
files: repro.py
messages: 374649
nosy: jab
priority: normal
severity: normal
status: open
title: Class with __weakref__ slot cannot inherit from multiple typing.Generic classes
type: behavior
versions: Python 3.6
Added file: https://bugs.python.org/file49352/repro.py

_______________________________________
Python tracker <report at bugs.python.org>
<https://bugs.python.org/issue41451>
_______________________________________