Skip to content

Commit

Permalink
CodeGen: Optimize vector ops for X64 when the source is computed (#1174)
Browse files Browse the repository at this point in the history 
With the TAG_VECTOR change, we can now confidently distinguish cases
when the .w component
contains TVECTOR tag from cases where it doesn't: loads and tag ops
produce the tag, whereas
other instructions don't.

We now take advantage of this fact and only apply vandps with a mask
when we need to.

It would be possible to use a positive filter (explicitly checking for
source coming from ADD_VEC
et al), but there are more instructions to check this way and this is
purely an optimization so
it is allowed to be conservative (as in, the cost of a mistake here is a
potential slowdown,
not a correctness issue).

Additionally, this change only performs vandps once when the arguments
are the same instead
of doing it twice.

On the function that computes a polynomial approximation this change
makes it ~20% faster on Zen4.
  • Loading branch information
@zeux
zeux authored Mar 1, 2024
1 parent c932485 commit cc51e61
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 24 deletions.
68 changes: 44 additions & 24 deletions CodeGen/src/IrLoweringX64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "lgc.h"

LUAU_FASTFLAG(LuauCodegenVectorTag)
LUAU_FASTFLAGVARIABLE(LuauCodegenVectorOptAnd, false)

namespace Luau
{
Expand Down Expand Up @@ -603,13 +604,13 @@ void IrLoweringX64::lowerInst(IrInst& inst, uint32_t index, const IrBlock& next)
{
inst.regX64 = regs.allocRegOrReuse(SizeX64::xmmword, index, {inst.a, inst.b});

ScopedRegX64 tmp1{regs, SizeX64::xmmword};
ScopedRegX64 tmp2{regs, SizeX64::xmmword};
ScopedRegX64 tmp1{regs};
ScopedRegX64 tmp2{regs};

RegisterX64 tmpa = vecOp(inst.a, tmp1);
RegisterX64 tmpb = (inst.a == inst.b) ? tmpa : vecOp(inst.b, tmp2);

// Fourth component is the tag number which is interpreted as a denormal and has to be filtered out
build.vandps(tmp1.reg, regOp(inst.a), vectorAndMaskOp());
build.vandps(tmp2.reg, regOp(inst.b), vectorAndMaskOp());
build.vaddps(inst.regX64, tmp1.reg, tmp2.reg);
build.vaddps(inst.regX64, tmpa, tmpb);

if (!FFlag::LuauCodegenVectorTag)
build.vorps(inst.regX64, inst.regX64, vectorOrMaskOp());
Expand All @@ -619,13 +620,13 @@ void IrLoweringX64::lowerInst(IrInst& inst, uint32_t index, const IrBlock& next)
{
inst.regX64 = regs.allocRegOrReuse(SizeX64::xmmword, index, {inst.a, inst.b});

ScopedRegX64 tmp1{regs, SizeX64::xmmword};
ScopedRegX64 tmp2{regs, SizeX64::xmmword};
ScopedRegX64 tmp1{regs};
ScopedRegX64 tmp2{regs};

// Fourth component is the tag number which is interpreted as a denormal and has to be filtered out
build.vandps(tmp1.reg, regOp(inst.a), vectorAndMaskOp());
build.vandps(tmp2.reg, regOp(inst.b), vectorAndMaskOp());
build.vsubps(inst.regX64, tmp1.reg, tmp2.reg);
RegisterX64 tmpa = vecOp(inst.a, tmp1);
RegisterX64 tmpb = (inst.a == inst.b) ? tmpa : vecOp(inst.b, tmp2);

build.vsubps(inst.regX64, tmpa, tmpb);
if (!FFlag::LuauCodegenVectorTag)
build.vorps(inst.regX64, inst.regX64, vectorOrMaskOp());
break;
Expand All @@ -634,13 +635,13 @@ void IrLoweringX64::lowerInst(IrInst& inst, uint32_t index, const IrBlock& next)
{
inst.regX64 = regs.allocRegOrReuse(SizeX64::xmmword, index, {inst.a, inst.b});

ScopedRegX64 tmp1{regs, SizeX64::xmmword};
ScopedRegX64 tmp2{regs, SizeX64::xmmword};
ScopedRegX64 tmp1{regs};
ScopedRegX64 tmp2{regs};

RegisterX64 tmpa = vecOp(inst.a, tmp1);
RegisterX64 tmpb = (inst.a == inst.b) ? tmpa : vecOp(inst.b, tmp2);

// Fourth component is the tag number which is interpreted as a denormal and has to be filtered out
build.vandps(tmp1.reg, regOp(inst.a), vectorAndMaskOp());
build.vandps(tmp2.reg, regOp(inst.b), vectorAndMaskOp());
build.vmulps(inst.regX64, tmp1.reg, tmp2.reg);
build.vmulps(inst.regX64, tmpa, tmpb);
if (!FFlag::LuauCodegenVectorTag)
build.vorps(inst.regX64, inst.regX64, vectorOrMaskOp());
break;
Expand All @@ -649,13 +650,13 @@ void IrLoweringX64::lowerInst(IrInst& inst, uint32_t index, const IrBlock& next)
{
inst.regX64 = regs.allocRegOrReuse(SizeX64::xmmword, index, {inst.a, inst.b});

ScopedRegX64 tmp1{regs, SizeX64::xmmword};
ScopedRegX64 tmp2{regs, SizeX64::xmmword};
ScopedRegX64 tmp1{regs};
ScopedRegX64 tmp2{regs};

RegisterX64 tmpa = vecOp(inst.a, tmp1);
RegisterX64 tmpb = (inst.a == inst.b) ? tmpa : vecOp(inst.b, tmp2);

// Fourth component is the tag number which is interpreted as a denormal and has to be filtered out
build.vandps(tmp1.reg, regOp(inst.a), vectorAndMaskOp());
build.vandps(tmp2.reg, regOp(inst.b), vectorAndMaskOp());
build.vdivps(inst.regX64, tmp1.reg, tmp2.reg);
build.vdivps(inst.regX64, tmpa, tmpb);
if (!FFlag::LuauCodegenVectorTag)
build.vpinsrd(inst.regX64, inst.regX64, build.i32(LUA_TVECTOR), 3);
break;
Expand Down Expand Up @@ -2234,6 +2235,24 @@ OperandX64 IrLoweringX64::bufferAddrOp(IrOp bufferOp, IrOp indexOp)
return noreg;
}

RegisterX64 IrLoweringX64::vecOp(IrOp op, ScopedRegX64& tmp)
{
if (FFlag::LuauCodegenVectorOptAnd && FFlag::LuauCodegenVectorTag)
{
IrInst source = function.instOp(op);
CODEGEN_ASSERT(source.cmd != IrCmd::SUBSTITUTE); // we don't process substitutions

// source that comes from memory or from tag instruction has .w = TVECTOR, which is denormal
// to avoid performance degradation on some CPUs we mask this component to produce zero
// otherwise we conservatively assume the vector is a result of a well formed math op so .w is a normal number or zero
if (source.cmd != IrCmd::LOAD_TVALUE && source.cmd != IrCmd::TAG_VECTOR)
return regOp(op);
}
tmp.alloc(SizeX64::xmmword);
build.vandps(tmp.reg, regOp(op), vectorAndMaskOp());
return tmp.reg;
}

IrConst IrLoweringX64::constOp(IrOp op) const
{
return function.constOp(op);
Expand Down Expand Up @@ -2279,6 +2298,7 @@ OperandX64 IrLoweringX64::vectorAndMaskOp()

OperandX64 IrLoweringX64::vectorOrMaskOp()
{
CODEGEN_ASSERT(!FFlag::LuauCodegenVectorTag);
if (vectorOrMask.base == noreg)
vectorOrMask = build.u32x4(0, 0, 0, LUA_TVECTOR);

Expand Down
1 change: 1 addition & 0 deletions CodeGen/src/IrLoweringX64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ struct IrLoweringX64
OperandX64 memRegTagOp(IrOp op);
RegisterX64 regOp(IrOp op);
OperandX64 bufferAddrOp(IrOp bufferOp, IrOp indexOp);
RegisterX64 vecOp(IrOp op, ScopedRegX64& tmp);

IrConst constOp(IrOp op) const;
uint8_t tagOp(IrOp op) const;
Expand Down

0 comments on commit cc51e61

Please sign in to comment.