Fix crash with generic class definition in function (#13678)

Fixes #12112.

The reason why mypy was crashing with a "Must not defer during final
iteration" error in the following snippet:

    from typing import TypeVar

    def test() -> None:
        T = TypeVar('T', bound='Foo')

        class Foo:
            def bar(self, foo: T) -> None:
                pass

...was because mypy did not seem to be updating the types of the `bar`
callable on each pass: the `bind_function_type_variables` method in
`typeanal.py` always returned the _old_ type variables instead of using
the new updated ones we found by calling `self.lookup_qualified(...)`.

This in turn prevented us from making any forward progress when mypy
generated a CallableType containing a placedholder type variable. So, we
repeated the semanal passes until we hit the limit and crashed.

I opted to fix this by having the function always return the newly-bound
TypeVarLikeType instead. (Hopefully this is safe -- the way mypy likes
mutating types always makes it hard to reason about this sort of stuff).

Interestingly, my fix for this bug introduced a regression in one of our
existing tests:

    from typing import NamedTuple, TypeVar

    T = TypeVar("T")
    NT = NamedTuple("NT", [("key", int), ("value", T)])

    # Test thinks the revealed type should be:
# def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1,
fallback=__main__.NT[T`-1]]
    #
    # ...but we started seeing:
# def [T, _NT <: Tuple[builtins.int, T`-1]] (key: builtins.int, value:
T`-1) -> Tuple[builtins.int, T`-1, fallback=test.WTF[T`-1]]
    reveal_type(NT)

What seems to be happening here is that during the first pass, we add
two type vars to the `tvar_scope` inside `bind_function_type_variables`:
`T` with id -1 and `_NT` with id -2.

But in the second pass, we lose track of the `T` typevar definition
and/or introduce a fresh scope somewhere and infer `_NT` with id -1
instead?

So now mypy thinks there are two type variables associated with this
NamedTuple, which results in the screwed-up type definition.

I wasn't really sure how to fix this, but I thought it was weird that:

1. We were using negative IDs instead of positive ones. (Class typevars
are supposed to use the latter).
2. We weren't wrapping this whole thing in a new tvar scope frame, given
we're nominally synthesizing a new class.

So I did that, and the tests started passing?

I wasn't able to repro this issue for TypedDicts, but opted to introduce
a new tvar scope frame there as well for consistency.

mypy/semanal.py

@@ -2752,30 +2752,32 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool:
             return False
         lvalue = s.lvalues[0]
         name = lvalue.name
-        internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple(
-            s.rvalue, name, self.is_func_scope()
-        )
-        if internal_name is None:
-            return False
-        if isinstance(lvalue, MemberExpr):
-            self.fail("NamedTuple type as an attribute is not supported", lvalue)
-            return False
-        if internal_name != name:
-            self.fail(
-                'First argument to namedtuple() should be "{}", not "{}"'.format(
-                    name, internal_name
-                ),
-                s.rvalue,
-                code=codes.NAME_MATCH,
+        namespace = self.qualified_name(name)
+        with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
+            internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple(
+                s.rvalue, name, self.is_func_scope()
             )
+            if internal_name is None:
+                return False
+            if isinstance(lvalue, MemberExpr):
+                self.fail("NamedTuple type as an attribute is not supported", lvalue)
+                return False
+            if internal_name != name:
+                self.fail(
+                    'First argument to namedtuple() should be "{}", not "{}"'.format(
+                        name, internal_name
+                    ),
+                    s.rvalue,
+                    code=codes.NAME_MATCH,
+                )
+                return True
+            # Yes, it's a valid namedtuple, but defer if it is not ready.
+            if not info:
+                self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
+            else:
+                self.setup_type_vars(info.defn, tvar_defs)
+                self.setup_alias_type_vars(info.defn)
             return True
-        # Yes, it's a valid namedtuple, but defer if it is not ready.
-        if not info:
-            self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
-        else:
-            self.setup_type_vars(info.defn, tvar_defs)
-            self.setup_alias_type_vars(info.defn)
-        return True
 
     def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool:
         """Check if s defines a typed dict."""
@@ -2789,22 +2791,24 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool:
             return False
         lvalue = s.lvalues[0]
         name = lvalue.name
-        is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict(
-            s.rvalue, name, self.is_func_scope()
-        )
-        if not is_typed_dict:
-            return False
-        if isinstance(lvalue, MemberExpr):
-            self.fail("TypedDict type as attribute is not supported", lvalue)
-            return False
-        # Yes, it's a valid typed dict, but defer if it is not ready.
-        if not info:
-            self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
-        else:
-            defn = info.defn
-            self.setup_type_vars(defn, tvar_defs)
-            self.setup_alias_type_vars(defn)
-        return True
+        namespace = self.qualified_name(name)
+        with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
+            is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict(
+                s.rvalue, name, self.is_func_scope()
+            )
+            if not is_typed_dict:
+                return False
+            if isinstance(lvalue, MemberExpr):
+                self.fail("TypedDict type as attribute is not supported", lvalue)
+                return False
+            # Yes, it's a valid typed dict, but defer if it is not ready.
+            if not info:
+                self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
+            else:
+                defn = info.defn
+                self.setup_type_vars(defn, tvar_defs)
+                self.setup_alias_type_vars(defn)
+            return True
 
     def analyze_lvalues(self, s: AssignmentStmt) -> None:
         # We cannot use s.type, because analyze_simple_literal_type() will set it.

mypy/semanal_namedtuple.py

@@ -321,11 +321,12 @@ def parse_namedtuple_args(
     ) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]):
         """Parse a namedtuple() call into data needed to construct a type.
 
-        Returns a 5-tuple:
+        Returns a 6-tuple:
         - List of argument names
         - List of argument types
         - List of default values
         - First argument of namedtuple
+        - All typevars found in the field definition
         - Whether all types are ready.
 
         Return None if the definition didn't typecheck.

mypy/typeanal.py

@@ -1331,29 +1331,29 @@ def bind_function_type_variables(
     ) -> Sequence[TypeVarLikeType]:
         """Find the type variables of the function type and bind them in our tvar_scope"""
         if fun_type.variables:
+            defs = []
             for var in fun_type.variables:
                 var_node = self.lookup_qualified(var.name, defn)
                 assert var_node, "Binding for function type variable not found within function"
                 var_expr = var_node.node
                 assert isinstance(var_expr, TypeVarLikeExpr)
-                self.tvar_scope.bind_new(var.name, var_expr)
-            return fun_type.variables
+                binding = self.tvar_scope.bind_new(var.name, var_expr)
+                defs.append(binding)
+            return defs
         typevars = self.infer_type_variables(fun_type)
         # Do not define a new type variable if already defined in scope.
         typevars = [
             (name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn)
         ]
-        defs: list[TypeVarLikeType] = []
+        defs = []
         for name, tvar in typevars:
             if not self.tvar_scope.allow_binding(tvar.fullname):
                 self.fail(
                     f'Type variable "{name}" is bound by an outer class',
                     defn,
                     code=codes.VALID_TYPE,
                 )
-            self.tvar_scope.bind_new(name, tvar)
-            binding = self.tvar_scope.get_binding(tvar.fullname)
-            assert binding is not None
+            binding = self.tvar_scope.bind_new(name, tvar)
             defs.append(binding)
 
         return defs

test-data/unit/check-classes.test

@@ -1060,6 +1060,35 @@ def f() -> None:
     a.g(a) # E: Too many arguments for "g" of "A"
 [targets __main__, __main__.f]
 
+[case testGenericClassWithinFunction]
+from typing import TypeVar
+
+def test() -> None:
+    T = TypeVar('T', bound='Foo')
+    class Foo:
+        def returns_int(self) -> int:
+            return 0
+
+        def bar(self, foo: T) -> T:
+            x: T = foo
+            reveal_type(x)  # N: Revealed type is "T`-1"
+            reveal_type(x.returns_int())  # N: Revealed type is "builtins.int"
+            return foo
+    reveal_type(Foo.bar)  # N: Revealed type is "def [T <: __main__.Foo@5] (self: __main__.Foo@5, foo: T`-1) -> T`-1"
+
+[case testGenericClassWithInvalidTypevarUseWithinFunction]
+from typing import TypeVar
+
+def test() -> None:
+    T = TypeVar('T', bound='Foo')
+    class Foo:
+        invalid: T  # E: Type variable "T" is unbound \
+                    # N: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class) \
+                    # N: (Hint: Use "T" in function signature to bind "T" inside a function)
+
+        def bar(self, foo: T) -> T:
+            pass
+
 [case testConstructNestedClass]
 import typing
 class A:

test-data/unit/check-namedtuple.test

@@ -1284,7 +1284,7 @@ from typing import NamedTuple, TypeVar
 
 T = TypeVar("T")
 NT = NamedTuple("NT", [("key", int), ("value", T)])
-reveal_type(NT)  # N: Revealed type is "def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=__main__.NT[T`-1]]"
+reveal_type(NT)  # N: Revealed type is "def [T] (key: builtins.int, value: T`1) -> Tuple[builtins.int, T`1, fallback=__main__.NT[T`1]]"
 
 nts: NT[str]
 reveal_type(nts)  # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]"

test-data/unit/check-typeddict.test

@@ -2550,7 +2550,7 @@ from typing import TypedDict, TypeVar
 
 T = TypeVar("T")
 TD = TypedDict("TD", {"key": int, "value": T})
-reveal_type(TD)  # N: Revealed type is "def [T] (*, key: builtins.int, value: T`-1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`-1})"
+reveal_type(TD)  # N: Revealed type is "def [T] (*, key: builtins.int, value: T`1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`1})"
 
 tds: TD[str]
 reveal_type(tds)  # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.str})"