Sync to upstream/release/644 (#1432)

In this update we improve overall stability of the new type solver and
address some type inference issues with it.

If you use the new solver and want to use all new fixes included in this
release, you have to reference an additional Luau flag:
```c++
LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease)
```
And set its value to `644`:
```c++
DFInt::LuauTypeSolverRelease.value = 644; // Or a higher value for future updates
```

## New Solver
* Fixed a debug assertion failure in autocomplete (Fixes #1391)
* Fixed type function distribution issue which transformed `len<>` and
`unm<>` into `not<>` (Fixes #1416)
* Placed a limit on the possible normalized table intersection size as a
temporary measure to avoid hangs and out-of-memory issues for complex
type refinements
* Internal recursion limits are now respected in the subtyping
operations and in autocomplete, to avoid stack overflow crashes
* Fixed false positive errors on assignments to tables whose indexers
are unions of strings
* Fixed memory corruption crashes in subtyping of generic types
containing other generic types in their bounds

---

Internal Contributors:

Co-authored-by: Aaron Weiss <[email protected]>
Co-authored-by: Andy Friesen <[email protected]>
Co-authored-by: Vyacheslav Egorov <[email protected]>

Analysis/include/Luau/ConstraintGenerator.h

@@ -321,6 +321,11 @@ struct ConstraintGenerator
      */
     void checkFunctionBody(const ScopePtr& scope, AstExprFunction* fn);
 
+    // Specializations of 'resolveType' below
+    TypeId resolveReferenceType(const ScopePtr& scope, AstType* ty, AstTypeReference* ref, bool inTypeArguments, bool replaceErrorWithFresh);
+    TypeId resolveTableType(const ScopePtr& scope, AstType* ty, AstTypeTable* tab, bool inTypeArguments, bool replaceErrorWithFresh);
+    TypeId resolveFunctionType(const ScopePtr& scope, AstType* ty, AstTypeFunction* fn, bool inTypeArguments, bool replaceErrorWithFresh);
+
     /**
      * Resolves a type from its AST annotation.
      * @param scope the scope that the type annotation appears within.

Analysis/include/Luau/Subtyping.h

@@ -96,6 +96,22 @@ struct SubtypingEnvironment
         DenseHashSet<TypeId> upperBound{nullptr};
     };
 
+    /* For nested subtyping relationship tests of mapped generic bounds, we keep the outer environment immutable */
+    SubtypingEnvironment* parent = nullptr;
+
+    /// Applies `mappedGenerics` to the given type.
+    /// This is used specifically to substitute for generics in type function instances.
+    std::optional<TypeId> applyMappedGenerics(NotNull<BuiltinTypes> builtinTypes, NotNull<TypeArena> arena, TypeId ty);
+
+    const TypeId* tryFindSubstitution(TypeId ty) const;
+    const SubtypingResult* tryFindSubtypingResult(std::pair<TypeId, TypeId> subAndSuper) const;
+
+    bool containsMappedType(TypeId ty) const;
+    bool containsMappedPack(TypePackId tp) const;
+
+    GenericBounds& getMappedTypeBounds(TypeId ty);
+    TypePackId* getMappedPackBounds(TypePackId tp);
+
     /*
      * When we encounter a generic over the course of a subtyping test, we need
      * to tentatively map that generic onto a type on the other side.
@@ -112,10 +128,6 @@ struct SubtypingEnvironment
     DenseHashMap<TypeId, TypeId> substitutions{nullptr};
 
     DenseHashMap<std::pair<TypeId, TypeId>, SubtypingResult, TypePairHash> ephemeralCache{{}};
-
-    /// Applies `mappedGenerics` to the given type.
-    /// This is used specifically to substitute for generics in type function instances.
-    std::optional<TypeId> applyMappedGenerics(NotNull<BuiltinTypes> builtinTypes, NotNull<TypeArena> arena, TypeId ty);
 };
 
 struct Subtyping

Analysis/src/Autocomplete.cpp

@@ -13,7 +13,12 @@
 #include <unordered_set>
 #include <utility>
 
-LUAU_FASTFLAG(LuauSolverV2);
+LUAU_FASTFLAG(LuauSolverV2)
+LUAU_FASTFLAG(LuauAutocompleteNewSolverLimit)
+
+LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease)
+LUAU_FASTINT(LuauTypeInferIterationLimit)
+LUAU_FASTINT(LuauTypeInferRecursionLimit)
 
 static const std::unordered_set<std::string> kStatementStartingKeywords =
     {"while", "if", "local", "repeat", "function", "do", "for", "return", "break", "continue", "type", "export"};
@@ -144,6 +149,12 @@ static bool checkTypeMatch(TypeId subTy, TypeId superTy, NotNull<Scope> scope, T
 
     if (FFlag::LuauSolverV2)
     {
+        if (FFlag::LuauAutocompleteNewSolverLimit)
+        {
+            unifierState.counters.recursionLimit = FInt::LuauTypeInferRecursionLimit;
+            unifierState.counters.iterationLimit = FInt::LuauTypeInferIterationLimit;
+        }
+
         Subtyping subtyping{builtinTypes, NotNull{typeArena}, NotNull{&normalizer}, NotNull{&iceReporter}};
 
         return subtyping.isSubtype(subTy, superTy, scope).isSubtype;
@@ -199,6 +210,9 @@ static TypeCorrectKind checkTypeCorrectKind(
     {
         for (TypeId id : itv->parts)
         {
+            if (DFInt::LuauTypeSolverRelease >= 644)
+                id = follow(id);
+
             if (const FunctionType* ftv = get<FunctionType>(id); ftv && checkFunctionType(ftv))
             {
                 return TypeCorrectKind::CorrectFunctionResult;