Sync to upstream/release/548 (#699)

- Fix rare type checking bugs with invalid generic types escaping the
module scope
- Fix type checking of variadic type packs in certain cases
- Implement type normalization, which resolves a large set of various
issues with unions/intersections in type checker
- Improve parse errors for trailing commas in function calls and type
lists
- Reduce profiling skew when using --profile with very high frequencies
- Improve performance of `lua_getinfo` (`debug.info`, `debug.traceback`
and profiling overhead are now 20% faster/smaller)
- Improve performance of polymorphic comparisons (1-2% lift on some
benchmarks)
- Improve performance of closure creation (1-2% lift on some benchmarks)
- Improve string comparison performance (4% lift on string sorting)

Analysis/include/Luau/ConstraintSolver.h

@@ -7,6 +7,7 @@
 #include "Luau/Constraint.h"
 #include "Luau/TypeVar.h"
 #include "Luau/ToString.h"
+#include "Luau/Normalize.h"
 
 #include <vector>
 
@@ -44,6 +45,7 @@ struct ConstraintSolver
     TypeArena* arena;
     NotNull<SingletonTypes> singletonTypes;
     InternalErrorReporter iceReporter;
+    NotNull<Normalizer> normalizer;
     // The entire set of constraints that the solver is trying to resolve.
     std::vector<NotNull<Constraint>> constraints;
     NotNull<Scope> rootScope;
@@ -74,9 +76,12 @@ struct ConstraintSolver
 
     DcrLogger* logger;
 
-    explicit ConstraintSolver(TypeArena* arena, NotNull<SingletonTypes> singletonTypes, NotNull<Scope> rootScope, ModuleName moduleName,
+    explicit ConstraintSolver(NotNull<Normalizer> normalizer, NotNull<Scope> rootScope, ModuleName moduleName,
         NotNull<ModuleResolver> moduleResolver, std::vector<RequireCycle> requireCycles, DcrLogger* logger);
 
+    // Randomize the order in which to dispatch constraints
+    void randomize(unsigned seed);
+
     /**
      * Attempts to dispatch all pending constraints and reach a type solution
      * that satisfies all of the constraints.
@@ -85,8 +90,9 @@ struct ConstraintSolver
 
     bool done();
 
-    /** Attempt to dispatch a constraint.  Returns true if it was successful.
-     * If tryDispatch() returns false, the constraint remains in the unsolved set and will be retried later.
+    /** Attempt to dispatch a constraint.  Returns true if it was successful. If
+     * tryDispatch() returns false, the constraint remains in the unsolved set
+     * and will be retried later.
      */
     bool tryDispatch(NotNull<const Constraint> c, bool force);
 

Analysis/include/Luau/Error.h

@@ -16,7 +16,7 @@ struct TypeMismatch
     TypeMismatch() = default;
     TypeMismatch(TypeId wantedType, TypeId givenType);
     TypeMismatch(TypeId wantedType, TypeId givenType, std::string reason);
-    TypeMismatch(TypeId wantedType, TypeId givenType, std::string reason, TypeError error);
+    TypeMismatch(TypeId wantedType, TypeId givenType, std::string reason, std::optional<TypeError> error);
 
     TypeId wantedType = nullptr;
     TypeId givenType = nullptr;

Analysis/include/Luau/Frontend.h

@@ -83,8 +83,13 @@ struct FrontendOptions
     // is complete.
     bool retainFullTypeGraphs = false;
 
-    // Run typechecking only in mode required for autocomplete (strict mode in order to get more precise type information)
+    // Run typechecking only in mode required for autocomplete (strict mode in
+    // order to get more precise type information)
     bool forAutocomplete = false;
+
+    // If not empty, randomly shuffle the constraint set before attempting to
+    // solve.  Use this value to seed the random number generator.
+    std::optional<unsigned> randomizeConstraintResolutionSeed;
 };
 
 struct CheckResult

Analysis/include/Luau/Normalize.h

@@ -1,9 +1,9 @@
 // This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
 #pragma once
 
-#include "Luau/Module.h"
 #include "Luau/NotNull.h"
 #include "Luau/TypeVar.h"
+#include "Luau/UnifierSharedState.h"
 
 #include <memory>
 
@@ -29,4 +29,231 @@ std::pair<TypePackId, bool> normalize(
 std::pair<TypePackId, bool> normalize(TypePackId ty, NotNull<Module> module, NotNull<SingletonTypes> singletonTypes, InternalErrorReporter& ice);
 std::pair<TypePackId, bool> normalize(TypePackId ty, const ModulePtr& module, NotNull<SingletonTypes> singletonTypes, InternalErrorReporter& ice);
 
+class TypeIds
+{
+private:
+    std::unordered_set<TypeId> types;
+    std::vector<TypeId> order;
+    std::size_t hash = 0;
+
+public:
+    using iterator = std::vector<TypeId>::iterator;
+    using const_iterator = std::vector<TypeId>::const_iterator;
+
+    TypeIds(const TypeIds&) = delete;
+    TypeIds(TypeIds&&) = default;
+    TypeIds() = default;
+    ~TypeIds() = default;
+    TypeIds& operator=(TypeIds&&) = default;
+
+    void insert(TypeId ty);
+    /// Erase every element that does not also occur in tys
+    void retain(const TypeIds& tys);
+    void clear();
+
+    iterator begin();
+    iterator end();
+    const_iterator begin() const;
+    const_iterator end() const;
+    iterator erase(const_iterator it);
+
+    size_t size() const;
+    bool empty() const;
+    size_t count(TypeId ty) const;
+
+    template<class Iterator>
+    void insert(Iterator begin, Iterator end)
+    {
+        for (Iterator it = begin; it != end; ++it)
+            insert(*it);
+    }
+
+    bool operator ==(const TypeIds& there) const;
+    size_t getHash() const;
+};
+
+} // namespace Luau
+
+template<> struct std::hash<Luau::TypeIds>
+{
+    std::size_t operator()(const Luau::TypeIds& tys) const
+    {
+        return tys.getHash();
+    }
+};
+
+template<> struct std::hash<const Luau::TypeIds*>
+{
+    std::size_t operator()(const Luau::TypeIds* tys) const
+    {
+        return tys->getHash();
+    }
+};
+
+template<> struct std::equal_to<Luau::TypeIds>
+{
+    bool operator()(const Luau::TypeIds& here, const Luau::TypeIds& there) const
+    {
+        return here == there;
+    }
+};
+
+template<> struct std::equal_to<const Luau::TypeIds*>
+{
+    bool operator()(const Luau::TypeIds* here, const Luau::TypeIds* there) const
+    {
+        return *here == *there;
+    }
+};
+
+namespace Luau
+{
+
+// A normalized string type is either `string` (represented by `nullopt`)
+// or a union of string singletons.
+using NormalizedStringType = std::optional<std::map<std::string, TypeId>>;
+
+// A normalized function type is either `never` (represented by `nullopt`)
+// or an intersection of function types.
+// NOTE: type normalization can fail on function types with generics
+// (e.g. because we do not support unions and intersections of generic type packs),
+// so this type may contain `error`.
+using NormalizedFunctionType = std::optional<TypeIds>;
+
+// A normalized generic/free type is a union, where each option is of the form (X & T) where
+// * X is either a free type or a generic
+// * T is a normalized type.
+struct NormalizedType;
+using NormalizedTyvars = std::unordered_map<TypeId, std::unique_ptr<NormalizedType>>;
+
+// A normalized type is either any, unknown, or one of the form P | T | F | G where
+// * P is a union of primitive types (including singletons, classes and the error type)
+// * T is a union of table types
+// * F is a union of an intersection of function types
+// * G is a union of generic/free normalized types, intersected with a normalized type
+struct NormalizedType
+{
+    // The top part of the type.
+    // This type is either never, unknown, or any.
+    // If this type is not never, all the other fields are null.
+    TypeId tops;
+
+    // The boolean part of the type.
+    // This type is either never, boolean type, or a boolean singleton.
+    TypeId booleans;
+
+    // The class part of the type.
+    // Each element of this set is a class, and none of the classes are subclasses of each other.
+    TypeIds classes;
+
+    // The error part of the type.
+    // This type is either never or the error type.
+    TypeId errors;
+
+    // The nil part of the type.
+    // This type is either never or nil.
+    TypeId nils;
+
+    // The number part of the type.
+    // This type is either never or number.
+    TypeId numbers;
+
+    // The string part of the type.
+    // This may be the `string` type, or a union of singletons.
+    NormalizedStringType strings = std::map<std::string,TypeId>{};
+
+    // The thread part of the type.
+    // This type is either never or thread.
+    TypeId threads;
+
+    // The (meta)table part of the type.
+    // Each element of this set is a (meta)table type.
+    TypeIds tables;
+
+    // The function part of the type.
+    NormalizedFunctionType functions;
+
+    // The generic/free part of the type.
+    NormalizedTyvars tyvars;
+
+    NormalizedType(NotNull<SingletonTypes> singletonTypes);
+
+    NormalizedType(const NormalizedType&) = delete;
+    NormalizedType(NormalizedType&&) = default;
+    NormalizedType() = delete;
+    ~NormalizedType() = default;
+    NormalizedType& operator=(NormalizedType&&) = default;
+    NormalizedType& operator=(NormalizedType&) = delete;
+};
+
+class Normalizer
+{
+    std::unordered_map<TypeId, std::unique_ptr<NormalizedType>> cachedNormals;
+    std::unordered_map<const TypeIds*, TypeId> cachedIntersections;
+    std::unordered_map<const TypeIds*, TypeId> cachedUnions;
+    std::unordered_map<const TypeIds*, std::unique_ptr<TypeIds>> cachedTypeIds;
+    bool withinResourceLimits();
+
+public:
+    TypeArena* arena;
+    NotNull<SingletonTypes> singletonTypes;
+    NotNull<UnifierSharedState> sharedState;
+
+    Normalizer(TypeArena* arena, NotNull<SingletonTypes> singletonTypes, NotNull<UnifierSharedState> sharedState);
+    Normalizer(const Normalizer&) = delete;
+    Normalizer(Normalizer&&) = delete;
+    Normalizer() = delete;
+    ~Normalizer() = default;
+    Normalizer& operator=(Normalizer&&) = delete;
+    Normalizer& operator=(Normalizer&) = delete;
+
+    // If this returns null, the typechecker should emit a "too complex" error
+    const NormalizedType* normalize(TypeId ty);
+    void clearNormal(NormalizedType& norm);
+
+    // ------- Cached TypeIds
+    TypeId unionType(TypeId here, TypeId there);
+    TypeId intersectionType(TypeId here, TypeId there);
+    const TypeIds* cacheTypeIds(TypeIds tys);
+    void clearCaches();
+
+    // ------- Normalizing unions
+    void unionTysWithTy(TypeIds& here, TypeId there);
+    TypeId unionOfTops(TypeId here, TypeId there);
+    TypeId unionOfBools(TypeId here, TypeId there);
+    void unionClassesWithClass(TypeIds& heres, TypeId there);
+    void unionClasses(TypeIds& heres, const TypeIds& theres);
+    void unionStrings(NormalizedStringType& here, const NormalizedStringType& there);
+    std::optional<TypePackId> unionOfTypePacks(TypePackId here, TypePackId there);
+    std::optional<TypeId> unionOfFunctions(TypeId here, TypeId there);
+    std::optional<TypeId> unionSaturatedFunctions(TypeId here, TypeId there);
+    void unionFunctionsWithFunction(NormalizedFunctionType& heress, TypeId there);
+    void unionFunctions(NormalizedFunctionType& heress, const NormalizedFunctionType& theress);
+    void unionTablesWithTable(TypeIds& heres, TypeId there);
+    void unionTables(TypeIds& heres, const TypeIds& theres);
+    bool unionNormals(NormalizedType& here, const NormalizedType& there, int ignoreSmallerTyvars = -1);
+    bool unionNormalWithTy(NormalizedType& here, TypeId there, int ignoreSmallerTyvars = -1);
+
+    // ------- Normalizing intersections
+    void intersectTysWithTy(TypeIds& here, TypeId there);
+    TypeId intersectionOfTops(TypeId here, TypeId there);
+    TypeId intersectionOfBools(TypeId here, TypeId there);
+    void intersectClasses(TypeIds& heres, const TypeIds& theres);
+    void intersectClassesWithClass(TypeIds& heres, TypeId there);
+    void intersectStrings(NormalizedStringType& here, const NormalizedStringType& there);
+    std::optional<TypePackId> intersectionOfTypePacks(TypePackId here, TypePackId there);
+    std::optional<TypeId> intersectionOfTables(TypeId here, TypeId there);
+    void intersectTablesWithTable(TypeIds& heres, TypeId there);
+    void intersectTables(TypeIds& heres, const TypeIds& theres);
+    std::optional<TypeId> intersectionOfFunctions(TypeId here, TypeId there);
+    void intersectFunctionsWithFunction(NormalizedFunctionType& heress, TypeId there);
+    void intersectFunctions(NormalizedFunctionType& heress, const NormalizedFunctionType& theress);
+    bool intersectTyvarsWithTy(NormalizedTyvars& here, TypeId there);
+    bool intersectNormals(NormalizedType& here, const NormalizedType& there, int ignoreSmallerTyvars = -1);
+    bool intersectNormalWithTy(NormalizedType& here, TypeId there);
+
+    // -------- Convert back from a normalized type to a type
+    TypeId typeFromNormal(const NormalizedType& norm);
+};
+
 } // namespace Luau

Analysis/include/Luau/TypeInfer.h

@@ -234,6 +234,8 @@ struct TypeChecker
     TypeId anyify(const ScopePtr& scope, TypeId ty, Location location);
     TypePackId anyify(const ScopePtr& scope, TypePackId ty, Location location);
 
+    TypePackId anyifyModuleReturnTypePackGenerics(TypePackId ty);
+
     void reportError(const TypeError& error);
     void reportError(const Location& location, TypeErrorData error);
     void reportErrors(const ErrorVec& errors);
@@ -359,6 +361,7 @@ struct TypeChecker
     InternalErrorReporter* iceHandler;
 
     UnifierSharedState unifierState;
+    Normalizer normalizer;
 
     std::vector<RequireCycle> requireCycles;
 

Analysis/include/Luau/Unifiable.h

@@ -96,7 +96,7 @@ struct Free
     bool forwardedTypeAlias = false;
 
 private:
-    static int nextIndex;
+    static int DEPRECATED_nextIndex;
 };
 
 template<typename Id>
@@ -127,7 +127,7 @@ struct Generic
     bool explicitName = false;
 
 private:
-    static int nextIndex;
+    static int DEPRECATED_nextIndex;
 };
 
 struct Error

Analysis/include/Luau/Unifier.h

@@ -9,6 +9,7 @@
 #include "Luau/TxnLog.h"
 #include "Luau/TypeArena.h"
 #include "Luau/UnifierSharedState.h"
+#include "Normalize.h"
 
 #include <unordered_set>
 
@@ -52,6 +53,7 @@ struct Unifier
 {
     TypeArena* const types;
     NotNull<SingletonTypes> singletonTypes;
+    NotNull<Normalizer> normalizer;
     Mode mode;
 
     NotNull<Scope> scope; // const Scope maybe
@@ -60,13 +62,14 @@ struct Unifier
     Location location;
     Variance variance = Covariant;
     bool anyIsTop = false; // If true, we consider any to be a top type.  If false, it is a familiar but weird mix of top and bottom all at once.
+    bool normalize; // Normalize unions and intersections if necessary
     bool useScopes = false; // If true, we use the scope hierarchy rather than TypeLevels
     CountMismatch::Context ctx = CountMismatch::Arg;
 
     UnifierSharedState& sharedState;
 
-    Unifier(TypeArena* types, NotNull<SingletonTypes> singletonTypes, Mode mode, NotNull<Scope> scope, const Location& location, Variance variance,
-        UnifierSharedState& sharedState, TxnLog* parentLog = nullptr);
+    Unifier(NotNull<Normalizer> normalizer, Mode mode, NotNull<Scope> scope, const Location& location, Variance variance,
+        TxnLog* parentLog = nullptr);
 
     // Test whether the two type vars unify.  Never commits the result.
     ErrorVec canUnify(TypeId subTy, TypeId superTy);
@@ -84,6 +87,7 @@ struct Unifier
     void tryUnifyTypeWithUnion(TypeId subTy, TypeId superTy, const UnionTypeVar* uv, bool cacheEnabled, bool isFunctionCall);
     void tryUnifyTypeWithIntersection(TypeId subTy, TypeId superTy, const IntersectionTypeVar* uv);
     void tryUnifyIntersectionWithType(TypeId subTy, const IntersectionTypeVar* uv, TypeId superTy, bool cacheEnabled, bool isFunctionCall);
+    void tryUnifyNormalizedTypes(TypeId subTy, TypeId superTy, const NormalizedType& subNorm, const NormalizedType& superNorm, std::string reason, std::optional<TypeError> error = std::nullopt);
     void tryUnifyPrimitives(TypeId subTy, TypeId superTy);
     void tryUnifySingletons(TypeId subTy, TypeId superTy);
     void tryUnifyFunctions(TypeId subTy, TypeId superTy, bool isFunctionCall = false);
@@ -92,6 +96,8 @@ struct Unifier
     void tryUnifyWithMetatable(TypeId subTy, TypeId superTy, bool reversed);
     void tryUnifyWithClass(TypeId subTy, TypeId superTy, bool reversed);
 
+    TypePackId tryApplyOverloadedFunction(TypeId function, const NormalizedFunctionType& overloads, TypePackId args);
+
     TypeId widen(TypeId ty);
     TypePackId widen(TypePackId tp);