CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)

{

dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);

if (nodeIndex != UINT_MAX)

dataLog(", node @", nodeIndex);

dataLog(".\n");

CRASH();

LowerDFGToLLVM(State& state)

: m_graph(state.graph)

, m_ftlState(state)

, m_heaps(state.context)

, m_out(state.context)

, m_state(state.graph)

, m_interpreter(state.graph, m_state)

, m_stackmapIDs(0)

, m_tbaaKind(mdKindID(state.context, "tbaa"))

, m_tbaaStructKind(mdKindID(state.context, "tbaa.struct"))

{

}

void lower()

{

CString name;

if (verboseCompilationEnabled()) {

name = toCString(

"jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),

"_", codeBlock()->hash());

} else

name = "jsBody";

m_graph.m_dominators.computeIfNecessary(m_graph);

m_ftlState.module =

moduleCreateWithNameInContext(name.data(), m_ftlState.context);

m_ftlState.function = addFunction(

m_ftlState.module, name.data(), functionType(m_out.int64));

setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);

if (isX86() && Options::llvmDisallowAVX()) {

// AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x

// slower. It should be disabled.

addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");

}

if (verboseCompilationEnabled())

dataLog("Function ready, beginning lowering.\n");

m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);

m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));

LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));

m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));

LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments"));

for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {

m_highBlock = m_graph.block(blockIndex);

if (!m_highBlock)

continue;

m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));

}

m_out.appendTo(m_prologue, stackOverflow);

createPhiVariables();

auto preOrder = m_graph.blocksInPreOrder();

LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));

m_captured = m_out.add(

m_out.ptrToInt(capturedAlloca, m_out.intPtr),

m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));

m_ftlState.capturedStackmapID = m_stackmapIDs++;

m_out.call(

m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),

m_out.int32Zero, capturedAlloca);

// If we have any CallVarargs then we nee to have a spill slot for it.

bool hasVarargs = false;

for (BasicBlock* block : preOrder) {

for (Node* node : *block) {

switch (node->op()) {

case CallVarargs:

case CallForwardVarargs:

case ConstructVarargs:

case ConstructForwardVarargs:

hasVarargs = true;

break;

default:

break;

}

}

}

if (hasVarargs) {

LValue varargsSpillSlots = m_out.alloca(

arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded()));

m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++;

m_out.call(

m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID),

m_out.int32Zero, varargsSpillSlots);

}

// We should not create any alloca's after this point, since they will cease to

// be mem2reg candidates.

m_callFrame = m_out.ptrToInt(

m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);

m_tagTypeNumber = m_out.constInt64(TagTypeNumber);

m_tagMask = m_out.constInt64(TagMask);

m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));

m_out.branch(

didOverflowStack(), rarely(stackOverflow), usually(checkArguments));

m_out.appendTo(stackOverflow, m_handleExceptions);

m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));

m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;

m_out.call(

m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),

m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));

m_out.unreachable();

m_out.appendTo(m_handleExceptions, checkArguments);

m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;

m_out.call(

m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),

m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));

m_out.unreachable();

m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));

availabilityMap().clear();

availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);

for (unsigned i = codeBlock()->numParameters(); i--;) {

availabilityMap().m_locals.argument(i) =

Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));

}

m_node = nullptr;

m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);

for (unsigned i = codeBlock()->numParameters(); i--;) {

Node* node = m_graph.m_arguments[i];

VirtualRegister operand = virtualRegisterForArgument(i);

LValue jsValue = m_out.load64(addressFor(operand));

if (node) {

DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);

// This is a hack, but it's an effective one. It allows us to do CSE on the

// primordial load of arguments. This assumes that the GetLocal that got put in

// place of the original SetArgument doesn't have any effects before it. This

// should hold true.

m_loadedArgumentValues.add(node, jsValue);

}

switch (m_graph.m_argumentFormats[i]) {

case FlushedInt32:

speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));

break;

case FlushedBoolean:

speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));

break;

case FlushedCell:

speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));

break;

case FlushedJSValue:

break;

default:

DFG_CRASH(m_graph, node, "Bad flush format for argument");

break;

}

}

m_out.jump(lowBlock(m_graph.block(0)));

for (BasicBlock* block : preOrder)

compileBlock(block);

if (Options::dumpLLVMIR())

dumpModule(m_ftlState.module);

if (verboseCompilationEnabled())

m_ftlState.dumpState("after lowering");

if (validationEnabled())

verifyModule(m_ftlState.module);

}

void createPhiVariables()

{

for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {

BasicBlock* block = m_graph.block(blockIndex);

if (!block)

continue;

for (unsigned nodeIndex = block->size(); nodeIndex--;) {

Node* node = block->at(nodeIndex);

if (node->op() != Phi)

continue;

LType type;

switch (node->flags() & NodeResultMask) {

case NodeResultDouble:

type = m_out.doubleType;

break;

case NodeResultInt32:

type = m_out.int32;

break;

case NodeResultInt52:

type = m_out.int64;

break;

case NodeResultBoolean:

type = m_out.boolean;

break;

case NodeResultJS:

type = m_out.int64;

break;

default:

DFG_CRASH(m_graph, node, "Bad Phi node result type");

break;

}

m_phis.add(node, buildAlloca(m_out.m_builder, type));

}

}

}

void compileBlock(BasicBlock* block)

{

if (!block)

return;

if (verboseCompilationEnabled())

dataLog("Compiling block ", *block, "\n");

m_highBlock = block;

LBasicBlock lowBlock = m_blocks.get(m_highBlock);

m_nextHighBlock = 0;

for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {

m_nextHighBlock = m_graph.block(nextBlockIndex);

if (m_nextHighBlock)

break;

}

m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;

// All of this effort to find the next block gives us the ability to keep the

// generated IR in roughly program order. This ought not affect the performance

// of the generated code (since we expect LLVM to reorder things) but it will

// make IR dumps easier to read.

m_out.appendTo(lowBlock, m_nextLowBlock);

if (Options::ftlCrashes())

m_out.trap();

if (!m_highBlock->cfaHasVisited) {

if (verboseCompilationEnabled())

dataLog("Bailing because CFA didn't reach.\n");

crash(m_highBlock->index, UINT_MAX);

return;

}

m_availabilityCalculator.beginBlock(m_highBlock);

m_state.reset();

m_state.beginBasicBlock(m_highBlock);

for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {

if (!compileNode(m_nodeIndex))

break;

}

}

void safelyInvalidateAfterTermination()

{

if (verboseCompilationEnabled())

dataLog("Bailing.\n");

crash();

// Invalidate dominated blocks. Under normal circumstances we would expect

// them to be invalidated already. But you can have the CFA become more

// precise over time because the structures of objects change on the main

// thread. Failing to do this would result in weird crashes due to a value

// being used but not defined. Race conditions FTW!

for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {

BasicBlock* target = m_graph.block(blockIndex);

if (!target)

continue;

if (m_graph.m_dominators.dominates(m_highBlock, target)) {

if (verboseCompilationEnabled())

dataLog("Block ", *target, " will bail also.\n");

target->cfaHasVisited = false;

}

}

}

bool compileNode(unsigned nodeIndex)

{

if (!m_state.isValid()) {

safelyInvalidateAfterTermination();

return false;

}

m_node = m_highBlock->at(nodeIndex);

m_origin = m_node->origin;

if (verboseCompilationEnabled())

dataLog("Lowering ", m_node, "\n");

m_availableRecoveries.resize(0);

m_interpreter.startExecuting();

switch (m_node->op()) {

case Upsilon:

compileUpsilon();

break;

case Phi:

compilePhi();

break;

case JSConstant:

break;

case DoubleConstant:

compileDoubleConstant();

break;

case Int52Constant:

compileInt52Constant();

break;

case DoubleRep:

compileDoubleRep();

break;

case DoubleAsInt32:

compileDoubleAsInt32();

break;

case ValueRep:

compileValueRep();

break;

case Int52Rep:

compileInt52Rep();

break;

case ValueToInt32:

compileValueToInt32();

break;

case BooleanToNumber:

compileBooleanToNumber();

break;

case ExtractOSREntryLocal:

compileExtractOSREntryLocal();

break;

case GetStack:

compileGetStack();

break;

case PutStack:

compilePutStack();

break;

case Check:

compileNoOp();

break;

case ToThis:

compileToThis();

break;

case ValueAdd:

compileValueAdd();

break;

case StrCat:

compileStrCat();

break;

case ArithAdd:

case ArithSub:

compileArithAddOrSub();

break;

case ArithClz32:

compileArithClz32();

break;

case ArithMul:

compileArithMul();

break;

case ArithDiv:

compileArithDiv();

break;

case ArithMod:

compileArithMod();

break;

case ArithMin:

case ArithMax:

compileArithMinOrMax();

break;

case ArithAbs:

compileArithAbs();

break;

case ArithSin:

compileArithSin();

break;

case ArithCos:

compileArithCos();

break;

case ArithPow:

compileArithPow();

break;

case ArithRound:

compileArithRound();

break;

case ArithSqrt:

compileArithSqrt();

break;

case ArithLog:

compileArithLog();

break;

case ArithFRound:

compileArithFRound();

break;

case ArithNegate:

compileArithNegate();

break;

case BitAnd:

compileBitAnd();

break;

case BitOr:

compileBitOr();

break;

case BitXor:

compileBitXor();

break;

case BitRShift:

compileBitRShift();

break;

case BitLShift:

compileBitLShift();

break;

case BitURShift:

compileBitURShift();

break;

case UInt32ToNumber:

compileUInt32ToNumber();

break;

case CheckStructure:

compileCheckStructure();

break;

case CheckCell:

compileCheckCell();

break;

case CheckNotEmpty:

compileCheckNotEmpty();

break;

case CheckBadCell:

compileCheckBadCell();

break;

case CheckIdent:

compileCheckIdent();

break;

case GetExecutable:

compileGetExecutable();

break;

case ArrayifyToStructure:

compileArrayifyToStructure();

break;

case PutStructure:

compilePutStructure();

break;

case GetById:

compileGetById();

break;

case In:

compileIn();

break;

case PutById:

case PutByIdDirect:

compilePutById();

break;

case GetButterfly:

compileGetButterfly();

break;

case ConstantStoragePointer:

compileConstantStoragePointer();

break;

case GetIndexedPropertyStorage:

compileGetIndexedPropertyStorage();

break;

case CheckArray:

compileCheckArray();

break;

case GetArrayLength:

compileGetArrayLength();

break;

case CheckInBounds:

compileCheckInBounds();

break;

case GetByVal:

compileGetByVal();

break;

case GetMyArgumentByVal:

compileGetMyArgumentByVal();

break;

case PutByVal:

case PutByValAlias:

case PutByValDirect:

compilePutByVal();

break;

case ArrayPush:

compileArrayPush();

break;

case ArrayPop:

compileArrayPop();

break;

case CreateActivation:

compileCreateActivation();

break;

case NewFunction:

case NewArrowFunction:

compileNewFunction();

break;

case CreateDirectArguments:

compileCreateDirectArguments();

break;

case CreateScopedArguments:

compileCreateScopedArguments();

break;

case CreateClonedArguments:

compileCreateClonedArguments();

break;

case NewObject:

compileNewObject();

break;

case NewArray:

compileNewArray();

break;

case NewArrayBuffer:

compileNewArrayBuffer();

break;

case NewArrayWithSize:

compileNewArrayWithSize();

break;

case GetTypedArrayByteOffset:

compileGetTypedArrayByteOffset();

break;

case AllocatePropertyStorage:

compileAllocatePropertyStorage();

break;

case ReallocatePropertyStorage:

compileReallocatePropertyStorage();

break;

case ToString:

case CallStringConstructor:

compileToStringOrCallStringConstructor();

break;

case ToPrimitive:

compileToPrimitive();

break;

case MakeRope:

compileMakeRope();

break;

case StringCharAt:

compileStringCharAt();

break;

case StringCharCodeAt:

compileStringCharCodeAt();

break;

case GetByOffset:

case GetGetterSetterByOffset:

compileGetByOffset();

break;

case GetGetter:

compileGetGetter();

break;

case GetSetter:

compileGetSetter();

break;

case MultiGetByOffset:

compileMultiGetByOffset();

break;

case PutByOffset:

compilePutByOffset();

break;

case MultiPutByOffset:

compileMultiPutByOffset();

break;

case GetGlobalVar:

case GetGlobalLexicalVariable:

compileGetGlobalVariable();

break;

case PutGlobalVariable:

compilePutGlobalVariable();

break;

case NotifyWrite:

compileNotifyWrite();

break;

case GetCallee:

compileGetCallee();

break;

case GetArgumentCount:

compileGetArgumentCount();

break;

case GetScope:

compileGetScope();

break;

case LoadArrowFunctionThis:

compileLoadArrowFunctionThis();

break;

case SkipScope:

compileSkipScope();

break;

case GetClosureVar:

compileGetClosureVar();

break;

case PutClosureVar:

compilePutClosureVar();

break;

case GetFromArguments:

compileGetFromArguments();

break;

case PutToArguments:

compilePutToArguments();

break;

case CompareEq:

compileCompareEq();

break;

case CompareStrictEq:

compileCompareStrictEq();

break;

case CompareLess:

compileCompareLess();

break;

case CompareLessEq:

compileCompareLessEq();

break;

case CompareGreater:

compileCompareGreater();

break;

case CompareGreaterEq:

compileCompareGreaterEq();

break;

case LogicalNot:

compileLogicalNot();

break;

case Call:

case Construct:

compileCallOrConstruct();

break;

case CallVarargs:

case CallForwardVarargs:

case ConstructVarargs:

case ConstructForwardVarargs:

compileCallOrConstructVarargs();

break;

case LoadVarargs:

compileLoadVarargs();

break;

case ForwardVarargs:

compileForwardVarargs();

break;

case Jump:

compileJump();

break;

case Branch:

compileBranch();

break;

case Switch:

compileSwitch();

break;

case Return:

compileReturn();

break;

case ForceOSRExit:

compileForceOSRExit();

break;

case Throw:

case ThrowReferenceError:

compileThrow();

break;

case InvalidationPoint:

compileInvalidationPoint();

break;

case IsUndefined:

compileIsUndefined();

break;

case IsBoolean:

compileIsBoolean();

break;

case IsNumber:

compileIsNumber();

break;

case IsString:

compileIsString();

break;

case IsObject:

compileIsObject();

break;

case IsObjectOrNull:

compileIsObjectOrNull();

break;

case IsFunction:

compileIsFunction();

break;

case TypeOf:

compileTypeOf();

break;

case CheckHasInstance:

compileCheckHasInstance();

break;

case InstanceOf:

compileInstanceOf();

break;

case CountExecution:

compileCountExecution();

break;

case StoreBarrier:

compileStoreBarrier();

break;

case HasIndexedProperty:

compileHasIndexedProperty();

break;

case HasGenericProperty:

compileHasGenericProperty();

break;

case HasStructureProperty:

compileHasStructureProperty();

break;

case GetDirectPname:

compileGetDirectPname();

break;

case GetEnumerableLength:

compileGetEnumerableLength();

break;

case GetPropertyEnumerator:

compileGetPropertyEnumerator();

break;

case GetEnumeratorStructurePname:

compileGetEnumeratorStructurePname();

break;

case GetEnumeratorGenericPname:

compileGetEnumeratorGenericPname();

break;

case ToIndexString:

compileToIndexString();

break;

case CheckStructureImmediate:

compileCheckStructureImmediate();

break;

case MaterializeNewObject:

compileMaterializeNewObject();

break;

case MaterializeCreateActivation:

compileMaterializeCreateActivation();

break;

case CheckWatchdogTimer:

compileCheckWatchdogTimer();

break;

case PhantomLocal:

case LoopHint:

case MovHint:

case ZombieHint:

case ExitOK:

case PhantomNewObject:

case PhantomNewFunction:

case PhantomCreateActivation:

case PhantomDirectArguments:

case PhantomClonedArguments:

case PutHint:

case BottomValue:

case KillStack:

break;

default:

DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");

break;

}

if (m_node->isTerminal())

return false;

if (!m_state.isValid()) {

safelyInvalidateAfterTermination();

return false;

}

m_availabilityCalculator.executeNode(m_node);

m_interpreter.executeEffects(nodeIndex);

return true;

}

void compileUpsilon()

{

LValue destination = m_phis.get(m_node->phi());

switch (m_node->child1().useKind()) {

case DoubleRepUse:

m_out.set(lowDouble(m_node->child1()), destination);

break;

case Int32Use:

case KnownInt32Use:

m_out.set(lowInt32(m_node->child1()), destination);

break;

case Int52RepUse:

m_out.set(lowInt52(m_node->child1()), destination);

break;

case BooleanUse:

case KnownBooleanUse:

m_out.set(lowBoolean(m_node->child1()), destination);

break;

case CellUse:

case KnownCellUse:

m_out.set(lowCell(m_node->child1()), destination);

break;

case UntypedUse:

m_out.set(lowJSValue(m_node->child1()), destination);

break;

default:

DFG_CRASH(m_graph, m_node, "Bad use kind");

break;

}

}

void compilePhi()

{

LValue source = m_phis.get(m_node);

switch (m_node->flags() & NodeResultMask) {

case NodeResultDouble:

setDouble(m_out.get(source));

break;

case NodeResultInt32:

setInt32(m_out.get(source));

break;

case NodeResultInt52:

setInt52(m_out.get(source));

break;

case NodeResultBoolean:

setBoolean(m_out.get(source));

break;

case NodeResultJS:

setJSValue(m_out.get(source));

break;

default:

DFG_CRASH(m_graph, m_node, "Bad use kind");

break;

}

}

void compileDoubleConstant()

{

setDouble(m_out.constDouble(m_node->asNumber()));

}

void compileInt52Constant()

{

int64_t value = m_node->asMachineInt();

setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));

setStrictInt52(m_out.constInt64(value));

}

void compileDoubleRep()

{

switch (m_node->child1().useKind()) {

case RealNumberUse: {

LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);

LValue doubleValue = unboxDouble(value);

LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("DoubleRep RealNumberUse int case"));

LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("DoubleRep continuation"));

ValueFromBlock fastResult = m_out.anchor(doubleValue);

m_out.branch(

m_out.doubleEqual(doubleValue, doubleValue),

usually(continuation), rarely(intCase));

LBasicBlock lastNext = m_out.appendTo(intCase, continuation);

FTL_TYPE_CHECK(

jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,

isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));

ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));

m_out.jump(continuation);

m_out.appendTo(continuation, lastNext);

setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult));

return;

}

case NotCellUse:

case NumberUse: {

bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;

LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);

LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case"));

LBasicBlock doubleTesting = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing double case"));

LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case"));

LBasicBlock nonDoubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing undefined case"));

LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation"));

m_out.branch(

isNotInt32(value, provenType(m_node->child1())),

unsure(doubleTesting), unsure(intCase));

LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);

ValueFromBlock intToDouble = m_out.anchor(

m_out.intToDouble(unboxInt32(value)));

m_out.jump(continuation);

m_out.appendTo(doubleTesting, doubleCase);

LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));

m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));

m_out.appendTo(doubleCase, nonDoubleCase);

ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));