Sync to upstream/release/658 (#1625)

## What's Changed ### General - Allow types of tables to diverge after using `table.clone` (fixes #1617). - Allow 2-argument vector.create in Luau. - Fix a crash when suggesting autocomplete after encountering parsing errors. - Add lua_tolstringatom C API which returns the string length (whether or not the atom exists) and which extends the existing lua_tostringatom function the same way lua_tolstring/lua_tostring do. - Luau now retains the DFGs of typechecked modules. ### Magic Functions Migration Note We've made a change to the API used to define magic functions. Previously, we had a set of function pointers on each `FunctionType` that would be invoked by the type inference engine at the correct point. The problem we'd run into is that they were all `std::function`s, we'd grown quite a few of them, and Luau allocates tens of thousands of types as it performs type inference. This adds up to a large amount of memory for data that isn't used by 99% of types. To slim things down a bit, we've replaced all of those `std::function`s with a single `shared_ptr` to a new interface called `MagicFunction`. This slims down the memory footprint of each type by about 50 bytes. The virtual methods of `MagicFunction` have roughly 1:1 correspondence with the old interface, so updating things should not be too difficult: * `FunctionType::magicFunction` is now `MagicFunction::handleOldSolver` * `FunctionType::dcrMagicFunction` is now `MagicFunction::infer` * `FunctionType::dcrMagicRefinement` is now `MagicFunction::refine` * `FunctionType::dcrMagicTypeCheck` is now `MagicFunction::typeCheck` **Full Changelog**: 0.657...0.658 --- Co-authored-by: Andy Friesen <[email protected]> Co-authored-by: Ariel Weiss <[email protected]> Co-authored-by: Aviral Goel <[email protected]> Co-authored-by: Hunter Goldstein <[email protected]> Co-authored-by: Talha Pathan <[email protected]> Co-authored-by: Varun Saini <[email protected]> Co-authored-by: Vighnesh Vijay <[email protected]> Co-authored-by: Vyacheslav Egorov <[email protected]>
luau-lang · Jan 24, 2025 · c13b5b7 · c13b5b7
1 parent 6061a14
commit c13b5b7
Show file tree

Hide file tree

Showing 37 changed files with 747 additions and 356 deletions.
diff --git a/Analysis/include/Luau/BuiltinDefinitions.h b/Analysis/include/Luau/BuiltinDefinitions.h
@@ -65,10 +65,7 @@ TypeId makeFunction( // Polymorphic
     bool checked = false
 );
 
-void attachMagicFunction(TypeId ty, MagicFunction fn);
-void attachDcrMagicFunction(TypeId ty, DcrMagicFunction fn);
-void attachDcrMagicRefinement(TypeId ty, DcrMagicRefinement fn);
-void attachDcrMagicFunctionTypeCheck(TypeId ty, DcrMagicFunctionTypeCheck fn);
+void attachMagicFunction(TypeId ty, std::shared_ptr<MagicFunction> fn);
 Property makeProperty(TypeId ty, std::optional<std::string> documentationSymbol = std::nullopt);
 void assignPropDocumentationSymbols(TableType::Props& props, const std::string& baseName);
 

diff --git a/Analysis/include/Luau/ConstraintSolver.h b/Analysis/include/Luau/ConstraintSolver.h
@@ -166,7 +166,7 @@ struct ConstraintSolver
      **/
     void finalizeTypeFunctions();
 
-    bool isDone();
+    bool isDone() const;
 
 private:
     /**
@@ -298,10 +298,10 @@ struct ConstraintSolver
     // FIXME: This use of a boolean for the return result is an appalling
     // interface.
     bool blockOnPendingTypes(TypeId target, NotNull<const Constraint> constraint);
-    bool blockOnPendingTypes(TypePackId target, NotNull<const Constraint> constraint);
+    bool blockOnPendingTypes(TypePackId targetPack, NotNull<const Constraint> constraint);
 
     void unblock(NotNull<const Constraint> progressed);
-    void unblock(TypeId progressed, Location location);
+    void unblock(TypeId ty, Location location);
     void unblock(TypePackId progressed, Location location);
     void unblock(const std::vector<TypeId>& types, Location location);
     void unblock(const std::vector<TypePackId>& packs, Location location);
@@ -336,7 +336,7 @@ struct ConstraintSolver
      * @param location the location where the require is taking place; used for
      * error locations.
      **/
-    TypeId resolveModule(const ModuleInfo& module, const Location& location);
+    TypeId resolveModule(const ModuleInfo& info, const Location& location);
 
     void reportError(TypeErrorData&& data, const Location& location);
     void reportError(TypeError e);

diff --git a/Analysis/include/Luau/DataFlowGraph.h b/Analysis/include/Luau/DataFlowGraph.h
@@ -6,6 +6,7 @@
 #include "Luau/ControlFlow.h"
 #include "Luau/DenseHash.h"
 #include "Luau/Def.h"
+#include "Luau/NotNull.h"
 #include "Luau/Symbol.h"
 #include "Luau/TypedAllocator.h"
 
@@ -48,13 +49,13 @@ struct DataFlowGraph
     const RefinementKey* getRefinementKey(const AstExpr* expr) const;
 
 private:
-    DataFlowGraph() = default;
+    DataFlowGraph(NotNull<DefArena> defArena, NotNull<RefinementKeyArena> keyArena);
 
     DataFlowGraph(const DataFlowGraph&) = delete;
     DataFlowGraph& operator=(const DataFlowGraph&) = delete;
 
-    DefArena defArena;
-    RefinementKeyArena keyArena;
+    NotNull<DefArena> defArena;
+    NotNull<RefinementKeyArena> keyArena;
 
     DenseHashMap<const AstExpr*, const Def*> astDefs{nullptr};
 
@@ -110,30 +111,22 @@ using ScopeStack = std::vector<DfgScope*>;
 
 struct DataFlowGraphBuilder
 {
-    static DataFlowGraph build(AstStatBlock* root, NotNull<struct InternalErrorReporter> handle);
-
-    /**
-     * This method is identical to the build method above, but returns a pair of dfg, scopes as the data flow graph
-     * here is intended to live on the module between runs of typechecking. Before, the DFG only needed to live as
-     * long as the typecheck, but in a world with incremental typechecking, we need the information on the dfg to incrementally
-     * typecheck small fragments of code.
-     * @param block - pointer to the ast to build the dfg for
-     * @param handle - for raising internal errors while building the dfg
-     */
-    static std::pair<std::shared_ptr<DataFlowGraph>, std::vector<std::unique_ptr<DfgScope>>> buildShared(
+    static DataFlowGraph build(
         AstStatBlock* block,
-        NotNull<InternalErrorReporter> handle
+        NotNull<DefArena> defArena,
+        NotNull<RefinementKeyArena> keyArena,
+        NotNull<struct InternalErrorReporter> handle
     );
 
 private:
-    DataFlowGraphBuilder() = default;
+    DataFlowGraphBuilder(NotNull<DefArena> defArena, NotNull<RefinementKeyArena> keyArena);
 
     DataFlowGraphBuilder(const DataFlowGraphBuilder&) = delete;
     DataFlowGraphBuilder& operator=(const DataFlowGraphBuilder&) = delete;
 
     DataFlowGraph graph;
-    NotNull<DefArena> defArena{&graph.defArena};
-    NotNull<RefinementKeyArena> keyArena{&graph.keyArena};
+    NotNull<DefArena> defArena;
+    NotNull<RefinementKeyArena> keyArena;
 
     struct InternalErrorReporter* handle = nullptr;
 

diff --git a/Analysis/include/Luau/EqSatSimplificationImpl.h b/Analysis/include/Luau/EqSatSimplificationImpl.h
@@ -105,6 +105,9 @@ struct TTable
     std::vector<Id> storage;
 };
 
+template <typename L>
+using Node = EqSat::Node<L>;
+
 using EType = EqSat::Language<
     TNil,
     TBoolean,
@@ -171,6 +174,9 @@ struct Subst
     Id eclass;
     Id newClass;
 
+    // The node into eclass which is boring, if any
+    std::optional<size_t> boringIndex;
+
     std::string desc;
 
     Subst(Id eclass, Id newClass, std::string desc = "");
@@ -211,6 +217,7 @@ struct Simplifier
     void subst(Id from, Id to);
     void subst(Id from, Id to, const std::string& ruleName);
     void subst(Id from, Id to, const std::string& ruleName, const std::unordered_map<Id, size_t>& forceNodes);
+    void subst(Id from, size_t boringIndex, Id to, const std::string& ruleName, const std::unordered_map<Id, size_t>& forceNodes);
 
     void unionClasses(std::vector<Id>& hereParts, Id there);
 
@@ -295,13 +302,13 @@ QueryIterator<Tag>::QueryIterator(EGraph* egraph_, Id eclass)
 
     for (const auto& enode : ecl.nodes)
     {
-        if (enode.index() < idx)
+        if (enode.node.index() < idx)
             ++index;
         else
             break;
     }
 
-    if (index >= ecl.nodes.size() || ecl.nodes[index].index() != idx)
+    if (index >= ecl.nodes.size() || ecl.nodes[index].node.index() != idx)
     {
         egraph = nullptr;
         index = 0;
@@ -331,7 +338,7 @@ std::pair<const Tag*, size_t> QueryIterator<Tag>::operator*() const
     EGraph::EClassT& ecl = (*egraph)[eclass];
 
     LUAU_ASSERT(index < ecl.nodes.size());
-    auto& enode = ecl.nodes[index];
+    auto& enode = ecl.nodes[index].node;
     Tag* result = enode.template get<Tag>();
     LUAU_ASSERT(result);
     return {result, index};
@@ -343,12 +350,16 @@ QueryIterator<Tag>& QueryIterator<Tag>::operator++()
 {
     const auto& ecl = (*egraph)[eclass];
 
-    ++index;
-    if (index >= ecl.nodes.size() || ecl.nodes[index].index() != EType::VariantTy::getTypeId<Tag>())
+    do
     {
-        egraph = nullptr;
-        index = 0;
-    }
+        ++index;
+        if (index >= ecl.nodes.size() || ecl.nodes[index].node.index() != EType::VariantTy::getTypeId<Tag>())
+        {
+            egraph = nullptr;
+            index = 0;
+            break;
+        }
+    } while (ecl.nodes[index].boring);
 
     return *this;
 }

diff --git a/Analysis/include/Luau/FragmentAutocomplete.h b/Analysis/include/Luau/FragmentAutocomplete.h
@@ -17,8 +17,8 @@ struct FrontendOptions;
 
 enum class FragmentTypeCheckStatus
 {
-    Success,
     SkipAutocomplete,
+    Success,
 };
 
 struct FragmentAutocompleteAncestryResult
@@ -56,7 +56,7 @@ struct FragmentAutocompleteResult
 
 FragmentAutocompleteAncestryResult findAncestryForFragmentParse(AstStatBlock* root, const Position& cursorPos);
 
-FragmentParseResult parseFragment(
+std::optional<FragmentParseResult> parseFragment(
     const SourceModule& srcModule,
     std::string_view src,
     const Position& cursorPos,

diff --git a/Analysis/include/Luau/Module.h b/Analysis/include/Luau/Module.h
@@ -139,6 +139,11 @@ struct Module
     TypePackId returnType = nullptr;
     std::unordered_map<Name, TypeFun> exportedTypeBindings;
 
+    // Arenas related to the DFG must persist after the DFG no longer exists, as
+    // Module objects maintain raw pointers to objects in these arenas.
+    DefArena defArena;
+    RefinementKeyArena keyArena;
+
     bool hasModuleScope() const;
     ScopePtr getModuleScope() const;
 

diff --git a/Analysis/include/Luau/Type.h b/Analysis/include/Luau/Type.h
@@ -131,14 +131,14 @@ struct BlockedType
     BlockedType();
     int index;
 
-    Constraint* getOwner() const;
-    void setOwner(Constraint* newOwner);
-    void replaceOwner(Constraint* newOwner);
+    const Constraint* getOwner() const;
+    void setOwner(const Constraint* newOwner);
+    void replaceOwner(const Constraint* newOwner);
 
 private:
     // The constraint that is intended to unblock this type. Other constraints
     // should block on this constraint if present.
-    Constraint* owner = nullptr;
+    const Constraint* owner = nullptr;
 };
 
 struct PrimitiveType
@@ -279,9 +279,6 @@ struct WithPredicate
     }
 };
 
-using MagicFunction = std::function<std::optional<
-    WithPredicate<TypePackId>>(struct TypeChecker&, const std::shared_ptr<struct Scope>&, const class AstExprCall&, WithPredicate<TypePackId>)>;
-
 struct MagicFunctionCallContext
 {
     NotNull<struct ConstraintSolver> solver;
@@ -291,7 +288,6 @@ struct MagicFunctionCallContext
     TypePackId result;
 };
 
-using DcrMagicFunction = std::function<bool(MagicFunctionCallContext)>;
 struct MagicRefinementContext
 {
     NotNull<Scope> scope;
@@ -308,8 +304,29 @@ struct MagicFunctionTypeCheckContext
     NotNull<Scope> checkScope;
 };
 
-using DcrMagicRefinement = void (*)(const MagicRefinementContext&);
-using DcrMagicFunctionTypeCheck = std::function<void(const MagicFunctionTypeCheckContext&)>;
+struct MagicFunction
+{
+    virtual std::optional<WithPredicate<TypePackId>> handleOldSolver(struct TypeChecker&, const std::shared_ptr<struct Scope>&, const class AstExprCall&, WithPredicate<TypePackId>) = 0;
+
+    // Callback to allow custom typechecking of builtin function calls whose argument types
+    // will only be resolved after constraint solving. For example, the arguments to string.format
+    // have types that can only be decided after parsing the format string and unifying
+    // with the passed in values, but the correctness of the call can only be decided after
+    // all the types have been finalized.
+    virtual bool infer(const MagicFunctionCallContext&) = 0;
+    virtual void refine(const MagicRefinementContext&) {}
+
+    // If a magic function needs to do its own special typechecking, do it here.
+    // Returns true if magic typechecking was performed.  Return false if the
+    // default typechecking logic should run.
+    virtual bool typeCheck(const MagicFunctionTypeCheckContext&)
+    {
+        return false;
+    }
+
+    virtual ~MagicFunction() {}
+};
+
 struct FunctionType
 {
     // Global monomorphic function
@@ -367,16 +384,7 @@ struct FunctionType
     Scope* scope = nullptr;
     TypePackId argTypes;
     TypePackId retTypes;
-    MagicFunction magicFunction = nullptr;
-    DcrMagicFunction dcrMagicFunction = nullptr;
-    DcrMagicRefinement dcrMagicRefinement = nullptr;
-
-    // Callback to allow custom typechecking of builtin function calls whose argument types
-    // will only be resolved after constraint solving. For example, the arguments to string.format
-    // have types that can only be decided after parsing the format string and unifying
-    // with the passed in values, but the correctness of the call can only be decided after
-    // all the types have been finalized.
-    DcrMagicFunctionTypeCheck dcrMagicTypeCheck = nullptr;
+    std::shared_ptr<MagicFunction> magic = nullptr;
 
     bool hasSelf;
     // `hasNoFreeOrGenericTypes` should be true if and only if the type does not have any free or generic types present inside it.

diff --git a/Analysis/include/Luau/VisitType.h b/Analysis/include/Luau/VisitType.h
@@ -85,6 +85,8 @@ struct GenericTypeVisitor
     {
     }
 
+    virtual ~GenericTypeVisitor() {}
+
     virtual void cycle(TypeId) {}
     virtual void cycle(TypePackId) {}
 

diff --git a/Analysis/src/AstQuery.cpp b/Analysis/src/AstQuery.cpp
@@ -13,6 +13,8 @@
 
 LUAU_FASTFLAG(LuauSolverV2)
 
+LUAU_FASTFLAG(LuauExtendStatEndPosWithSemicolon)
+
 namespace Luau
 {
 
@@ -41,11 +43,26 @@ struct AutocompleteNodeFinder : public AstVisitor
 
     bool visit(AstStat* stat) override
     {
-        if (stat->location.begin < pos && pos <= stat->location.end)
+        if (FFlag::LuauExtendStatEndPosWithSemicolon)
         {
-            ancestry.push_back(stat);
-            return true;
+            // Consider 'local myLocal = 4;|' and 'local myLocal = 4', where '|' is the cursor position. In both cases, the cursor position is equal
+            // to `AstStatLocal.location.end`. However, in the first case (semicolon), we are starting a new statement, whilst in the second case
+            // (no semicolon) we are still part of the AstStatLocal, hence the different comparison check.
+            if (stat->location.begin < pos && (stat->hasSemicolon ? pos < stat->location.end : pos <= stat->location.end))
+            {
+                ancestry.push_back(stat);
+                return true;
+            }
         }
+        else
+        {
+            if (stat->location.begin < pos && pos <= stat->location.end)
+            {
+                ancestry.push_back(stat);
+                return true;
+            }
+        }
+
         return false;
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -85,6 +85,8 @@ struct GenericTypeVisitor @@
         {
         }
+        virtual ~GenericTypeVisitor() {}
         virtual void cycle(TypeId) {}
         virtual void cycle(TypePackId) {}
@@ Expand Down @@