/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com)     |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/

#include "hphp/runtime/vm/jit/mc-generator.h"
#include "hphp/runtime/vm/jit/vtune-jit.h"

#include <cinttypes>
#include <assert.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#ifndef _MSC_VER
#include <unwind.h>
#endif

#include <algorithm>
#include <exception>
#include <memory>
#include <queue>
#include <string>
#include <strstream>
#include <unordered_set>
#include <vector>

#include <folly/Format.h>
#include <folly/MapUtil.h>
#include <folly/Optional.h>
#include <folly/String.h>

#include "hphp/util/abi-cxx.h"
#include "hphp/util/safe-cast.h"
#include "hphp/util/asm-x64.h"
#include "hphp/util/bitops.h"
#include "hphp/util/code-cache.h"
#include "hphp/util/cycles.h"
#include "hphp/util/debug.h"
#include "hphp/util/disasm.h"
#include "hphp/util/logger.h"
#include "hphp/util/maphuge.h"
#include "hphp/util/meta.h"
#include "hphp/util/process.h"
#include "hphp/util/rank.h"
#include "hphp/util/repo-schema.h"
#include "hphp/util/ringbuffer.h"
#include "hphp/util/service-data.h"
#include "hphp/util/timer.h"
#include "hphp/util/trace.h"

#include "hphp/runtime/base/arch.h"
#include "hphp/runtime/base/execution-context.h"
#include "hphp/runtime/base/rds.h"
#include "hphp/runtime/base/runtime-option.h"
#include "hphp/runtime/base/stats.h"
#include "hphp/runtime/base/strings.h"
#include "hphp/runtime/base/zend-string.h"
#include "hphp/runtime/ext/generator/ext_generator.h"
#include "hphp/runtime/ext/std/ext_std_function.h"
#include "hphp/runtime/server/source-root-info.h"
#include "hphp/runtime/vm/blob-helper.h"
#include "hphp/runtime/vm/bytecode.h"
#include "hphp/runtime/vm/debug/debug.h"
#include "hphp/runtime/vm/func.h"
#include "hphp/runtime/vm/hhbc-codec.h"
#include "hphp/runtime/vm/jit/align.h"
#include "hphp/runtime/vm/jit/check.h"
#include "hphp/runtime/vm/jit/code-gen-helpers.h"
#include "hphp/runtime/vm/jit/debug-guards.h"
#include "hphp/runtime/vm/jit/func-guard.h"
#include "hphp/runtime/vm/jit/func-prologue.h"
#include "hphp/runtime/vm/jit/inlining-decider.h"
#include "hphp/runtime/vm/jit/irgen.h"
#include "hphp/runtime/vm/jit/irlower.h"
#include "hphp/runtime/vm/jit/normalized-instruction.h"
#include "hphp/runtime/vm/jit/opt.h"
#include "hphp/runtime/vm/jit/print.h"
#include "hphp/runtime/vm/jit/prof-data.h"
#include "hphp/runtime/vm/jit/recycle-tc.h"
#include "hphp/runtime/vm/jit/region-selection.h"
#include "hphp/runtime/vm/jit/service-requests.h"
#include "hphp/runtime/vm/jit/smashable-instr.h"
#include "hphp/runtime/vm/jit/srcdb.h"
#include "hphp/runtime/vm/jit/timer.h"
#include "hphp/runtime/vm/jit/translate-region.h"
#include "hphp/runtime/vm/jit/translator-inline.h"
#include "hphp/runtime/vm/jit/vasm-gen.h"
#include "hphp/runtime/vm/jit/vasm-instr.h"
#include "hphp/runtime/vm/jit/relocation.h"
#include "hphp/runtime/vm/member-operations.h"
#include "hphp/runtime/vm/php-debug.h"
#include "hphp/runtime/vm/repo.h"
#include "hphp/runtime/vm/runtime.h"
#include "hphp/runtime/vm/srckey.h"
#include "hphp/runtime/vm/treadmill.h"
#include "hphp/runtime/vm/type-profile.h"
#include "hphp/runtime/vm/unwind.h"

namespace HPHP { namespace jit {

TRACE_SET_MOD(mcg);

using namespace reg;
using namespace Trace;
using std::max;

#define TPC(n) "jit_" #n,
static const char* const kPerfCounterNames[] = {
  TRANS_PERF_COUNTERS
};
#undef TPC

#ifdef __APPLE__
// Clang believes that it can force s_perfCounters into 16-byte alignment, and
// thus emit an inlined version of memcpy later in this file using SSE
// instructions which require  such alignment. It can, in fact, do this --
// except due to what is as far as I can tell a linker bug on OS X, ld doesn't
// actually lay this out with 16 byte alignment, and so the SSE instructions
// crash. In order to work around this, tell clang to force it to only 8 byte
// alignment, which causes it to emit an inlined version of memcpy which does
// not assume 16-byte alignment. (Perversely, it also tickles the ld bug
// differently such that it actually gets 16-byte alignment :\)
alignas(8)
#endif
__thread int64_t s_perfCounters[tpc_num_counters];

static __thread size_t s_initialTCSize;

// The global MCGenerator object.
MCGenerator* mcg;

CallSpec MCGenerator::getDtorCall(DataType type) {
  switch (type) {
    case KindOfString:
      return CallSpec::method(&StringData::release);
    case KindOfArray:
      return CallSpec::method(&ArrayData::release);
    case KindOfObject:
      return CallSpec::method(
        RuntimeOption::EnableObjDestructCall
          ? &ObjectData::release
          : &ObjectData::releaseNoObjDestructCheck
      );
    case KindOfResource:
      return CallSpec::method(&ResourceHdr::release);
    case KindOfRef:
      return CallSpec::method(&RefData::release);
    DT_UNCOUNTED_CASE:
    case KindOfClass:
      break;
  }
  not_reached();
}

///////////////////////////////////////////////////////////////////////////////

namespace {
/*
 * Convenience class for creating TransLocs and TransRecs for new translations.
 *
 * Records the beginning and end of a translation and stores the size of the
 * cold and frozen regions in the first 4 bytes of their respective regions.
 */
struct TransLocMaker {
  explicit TransLocMaker(CodeCache& c) : cache(c) {}

  /*
   * Record the start of a translation, and reserve space at the top of cold
   * and frozen (if they aren't the same) to record sizes.
   */
  void markStart() {
    loc.setMainStart(cache.main().frontier());
    loc.setColdStart(cache.cold().frontier());
    loc.setFrozenStart(cache.frozen().frontier());

    cache.cold().dword(0);
    if (&cache.cold() != &cache.frozen()) cache.frozen().dword(0);
  }

  /*
   * Record the end of a translation, storing the size of cold and frozen,
   * returns a TransLoc representing the translation.
   */
  TransLoc markEnd() {
    uint32_t* coldSize   = (uint32_t*)loc.coldStart();
    uint32_t* frozenSize = (uint32_t*)loc.frozenStart();
    *coldSize   = cache  .cold().frontier() - loc.coldStart();
    *frozenSize = cache.frozen().frontier() - loc.frozenStart();
    loc.setMainSize(cache.main().frontier() - loc.mainStart());

    return loc;
  }

  /*
   * Create a TransRec for the translation, markEnd() should be called prior to
   * calling rec().
   */
  TransRec rec(
      SrcKey                      sk,
      TransKind                   kind,
      RegionDescPtr               region  = RegionDescPtr(),
      std::vector<TransBCMapping> bcmap   = std::vector<TransBCMapping>(),
      Annotations&&               annot   = Annotations(),
      bool                        llvm    = false,
      bool                        hasLoop = false) const {
    auto& cold = cache.realCold();
    auto& frozen = cache.realFrozen();
    TCA coldStart = cold.frontier();
    TCA frozenStart = frozen.frontier();
    size_t coldSize = 0;
    size_t frozenSize = 0;

    if (&cache.cold() == &cold) {
      coldStart = loc.coldCodeStart();
      coldSize  = loc.coldCodeSize();
    }
    if (&cache.frozen() == &frozen) {
      frozenStart = loc.frozenCodeStart();
      frozenSize  = loc.frozenCodeSize();
    }

    return TransRec(sk, kind,
                    loc.mainStart(), loc.mainSize(),
                    coldStart, coldSize,
                    frozenStart, frozenSize,
                    std::move(region), std::move(bcmap),
                    std::move(annot), llvm, hasLoop);
  }

private:
  CodeCache& cache;
  TransLoc loc;
};
}

///////////////////////////////////////////////////////////////////////////////

bool shouldPGOFunc(const Func& func) {
  if (!RuntimeOption::EvalJitPGO) return false;

  // JITing pseudo-mains requires extra checks that blow the IR.  PGO
  // can significantly increase the size of the regions, so disable it for
  // pseudo-mains (so regions will be just tracelets).
  if (func.isPseudoMain()) return false;

  if (!RuntimeOption::EvalJitPGOHotOnly) return true;
  return func.attrs() & AttrHot;
}

bool MCGenerator::profileSrcKey(SrcKey sk) const {
  if (!shouldPGOFunc(*sk.func())) return false;
  if (m_tx.profData()->optimized(sk.funcID())) return false;
  if (m_tx.profData()->profiling(sk.funcID())) return true;

  // Don't start profiling new functions if the size of either main or
  // prof is already above Eval.JitAMaxUsage.
  auto tcUsage = std::max(code.mainUsed(), code.profUsed());
  if (tcUsage >= CodeCache::AMaxUsage) {
    return false;
  }

  return requestCount() <= RuntimeOption::EvalJitProfileRequests;
}

/*
 * Invalidate the SrcDB entries for func's SrcKeys that have any
 * Profile translation.
 */
void MCGenerator::invalidateFuncProfSrcKeys(const Func* func) {
  assertx(RuntimeOption::EvalJitPGO);
  FuncId funcId = func->getFuncId();
  for (auto tid : m_tx.profData()->funcProfTransIDs(funcId)) {
    invalidateSrcKey(m_tx.profData()->transSrcKey(tid));
  }
}

TCA MCGenerator::retranslate(const TranslArgs& args) {
  auto sr = m_tx.getSrcDB().find(args.sk);
  always_assert(sr);
  bool locked = sr->tryLock();
  SCOPE_EXIT {
    if (locked) sr->freeLock();
  };
  if (isDebuggerAttachedProcess() && m_tx.isSrcKeyInBL(args.sk)) {
    // We are about to translate something known to be blacklisted by
    // debugger, exit early
    SKTRACE(1, args.sk, "retranslate abort due to debugger\n");
    return nullptr;
  }
  LeaseHolder writer(Translator::WriteLease());
  if (!writer || !shouldTranslate(args.sk.func())) return nullptr;
  if (!locked) {
    // Even though we knew above that we were going to skip
    // doing another translation, we wait until we get the
    // write lease, to avoid spinning through the tracelet
    // guards again and again while another thread is writing
    // to it.
    return sr->getTopTranslation();
  }
  if (sr->translations().size() > RuntimeOption::EvalJitMaxTranslations) {
    always_assert(sr->translations().size() ==
                  RuntimeOption::EvalJitMaxTranslations + 1);
    return sr->getTopTranslation();
  }
  SKTRACE(1, args.sk, "retranslate\n");

  m_tx.setMode(profileSrcKey(args.sk) ? TransKind::Profile : TransKind::Live);
  SCOPE_EXIT{ m_tx.setMode(TransKind::Invalid); };

  auto start = translate(args);

  // In PGO mode, we free all the profiling data once the TC is full.
  if (RuntimeOption::EvalJitPGO &&
      code.mainUsed() >= CodeCache::AMaxUsage) {
    m_tx.profData()->free();
  }

  return start;
}

TCA MCGenerator::retranslateOpt(TransID transId, bool align) {
  LeaseHolder writer(Translator::WriteLease());
  if (!writer) return nullptr;
  if (isDebuggerAttachedProcess()) return nullptr;

  TRACE(1, "retranslateOpt: transId = %u\n", transId);

  SCOPE_EXIT { m_tx.setMode(TransKind::Invalid); };

  if (!m_tx.profData()->hasTransRec(transId)) return nullptr;

  always_assert(m_tx.profData()->transRegion(transId) != nullptr);

  auto func   = m_tx.profData()->transFunc(transId);
  auto funcId = func->getFuncId();
  auto sk     = m_tx.profData()->transSrcKey(transId);

  if (m_tx.profData()->optimized(funcId)) return nullptr;
  m_tx.profData()->setOptimized(funcId);

  bool setFuncBody = func->getDVFunclets().size() == 0;

  func->setFuncBody(m_tx.uniqueStubs.funcBodyHelperThunk);

  // Invalidate SrcDB's entries for all func's SrcKeys.
  invalidateFuncProfSrcKeys(func);

  // Regenerate the prologues and DV funclets before the actual function body.
  TCA start = regeneratePrologues(func, sk);

  // Regionize func and translate all its regions.
  m_tx.setMode(TransKind::Optimize);
  std::vector<RegionDescPtr> regions;
  regionizeFunc(func, this, regions);

  for (auto region : regions) {
    m_tx.setMode(TransKind::Optimize);
    always_assert(!region->empty());
    auto regionSk = region->start();
    auto translArgs = TranslArgs{regionSk, align};
    translArgs.region = region;

    if (setFuncBody && regionSk.offset() == func->base()) {
      translArgs.setFuncBody = true;
      setFuncBody = false;
    }
    auto regionStart = translate(translArgs);
    if (start == nullptr && regionSk == sk) {
      start = regionStart;
    }
  }

  // In PGO mode, we free all the profiling data once the TC is full.
  if (RuntimeOption::EvalJitPGO &&
      code.mainUsed() >= CodeCache::AMaxUsage) {
    m_tx.profData()->free();
  }

  return start;
}

static bool liveFrameIsPseudoMain() {
  auto const ar = vmfp();
  if (!(ar->func()->attrs() & AttrMayUseVV)) return false;
  return ar->hasVarEnv() && ar->getVarEnv()->isGlobalScope();
}

/*
 * Find or create a translation for sk. Returns TCA of "best" current
 * translation. May return NULL if it is currently impossible to create
 * a translation.
 */
TCA
MCGenerator::getTranslation(const TranslArgs& args) {
  auto sk = args.sk;
  sk.func()->validate();
  SKTRACE(2, sk,
          "getTranslation: curUnit %s funcId %x offset %d\n",
          sk.unit()->filepath()->data(),
          sk.funcID(),
          sk.offset());
  SKTRACE(2, sk, "   funcId: %x \n", sk.func()->getFuncId());

  if (liveFrameIsPseudoMain() && !RuntimeOption::EvalJitPseudomain) {
    SKTRACE(2, sk, "punting on pseudoMain\n");
    return nullptr;
  }
  if (const SrcRec* sr = m_tx.getSrcDB().find(sk)) {
    TCA tca = sr->getTopTranslation();
    if (tca) {
      SKTRACE(2, sk, "getTranslation: found %p\n", tca);
      return tca;
    }
  }
  return createTranslation(args);
}

int
MCGenerator::numTranslations(SrcKey sk) const {
  if (const SrcRec* sr = m_tx.getSrcDB().find(sk)) {
    return sr->translations().size();
  }
  return 0;
}

const StaticString
  s_php_errormsg("php_errormsg"),
  s_http_response_header("http_response_header");

bool MCGenerator::shouldTranslateNoSizeLimit(const Func* func) const {
  // If we've hit Eval.JitGlobalTranslationLimit, then we stop translating.
  if (m_numTrans >= RuntimeOption::EvalJitGlobalTranslationLimit) {
    return false;
  }

  // Do not translate functions from units marked as interpret-only.
  if (func->unit()->isInterpretOnly()) {
    return false;
  }

  /*
   * We don't support JIT compiling functions that use some super-dynamic php
   * variables.
   */
  if (func->lookupVarId(s_php_errormsg.get()) != -1 ||
      func->lookupVarId(s_http_response_header.get()) != -1) {
    return false;
  }

  return true;
}

bool MCGenerator::shouldTranslate(const Func* func) const {
  if (!shouldTranslateNoSizeLimit(func)) return false;
  // Otherwise, follow the Eval.JitAMaxUsage limit.  However, we do
  // allow Optimize translations past that limit.
  return code.mainUsed() < CodeCache::AMaxUsage ||
         m_tx.mode() == TransKind::Optimize;
}


static void populateLiveContext(RegionContext& ctx) {
  typedef RegionDesc::Location L;

  const ActRec*     const fp {vmfp()};
  const TypedValue* const sp {vmsp()};

  always_assert(ctx.func == fp->m_func);

  for (uint32_t i = 0; i < fp->m_func->numLocals(); ++i) {
    ctx.liveTypes.push_back(
      { L::Local{i}, typeFromTV(frame_local(fp, i)) }
    );
  }

  int32_t stackOff = 0;
  visitStackElems(
    fp, sp, ctx.bcOffset,
    [&](const ActRec* ar, Offset) {
      // TODO(#2466980): when it's a Cls, we should pass the Class* in
      // the Type.
      auto const objOrCls =
        ar->hasThis()  ? Type::SubObj(ar->getThis()->getVMClass()) :
        ar->hasClass() ? TCls
                       : TNullptr;

      ctx.preLiveARs.push_back({
        stackOff,
        ar->m_func,
        objOrCls
      });
      FTRACE(2, "added prelive ActRec {}\n", show(ctx.preLiveARs.back()));
      stackOff += kNumActRecCells;
    },
    [&](const TypedValue* tv) {
      ctx.liveTypes.push_back(
        { L::Stack{ctx.spOffset - stackOff}, typeFromTV(tv) }
      );
      stackOff++;
      FTRACE(2, "added live type {}\n", show(ctx.liveTypes.back()));
    }
  );
}

TCA
MCGenerator::createTranslation(const TranslArgs& args) {
  if (!shouldTranslate(args.sk.func())) return nullptr;

  /*
   * Try to become the writer. We delay this until we *know* we will have
   * a need to create new translations, instead of just trying to win the
   * lottery at the dawn of time. Hopefully lots of requests won't require
   * any new translation.
   */
  auto sk = args.sk;
  LeaseHolder writer(Translator::WriteLease());
  if (!writer || !shouldTranslate(args.sk.func())) return nullptr;

  if (auto sr = m_tx.getSrcDB().find(sk)) {
    TCA tca = sr->getTopTranslation();
    if (tca) {
      // Handle extremely unlikely race; someone may have just already
      // added the first instance of this SrcRec while we did a
      // non-blocking wait on the write lease.
      return tca;
    } else {
      // Since we are holding the write lease, we know that sk is properly
      // initialized, except that it has no translations (due to
      // replaceOldTranslations)
      return retranslate(args);
    }
  }

  auto const srcRecSPOff = [&] () -> folly::Optional<FPInvOffset> {
    if (sk.resumed()) return folly::none;
    return liveSpOff();
  }();

  // We put retranslate requests at the end of our slab to more frequently
  //   allow conditional jump fall-throughs
  TCA astart          = code.main().frontier();
  TCA realColdStart   = code.realCold().frontier();
  TCA realFrozenStart = code.realFrozen().frontier();
  TCA req;
  if (!RuntimeOption::EvalEnableReusableTC) {
    req = svcreq::emit_persistent(code.cold(),
                                  srcRecSPOff,
                                  REQ_RETRANSLATE,
                                  sk.offset(),
                                  TransFlags().packed);
  } else {
    auto const stubsize = svcreq::stub_size();
    auto newStart = code.cold().allocInner(stubsize);
    if (!newStart) {
      newStart = code.cold().frontier();
    }
    // Ensure that the anchor translation is a known size so that it can be
    // reclaimed when the function is freed
    req = svcreq::emit_ephemeral(code.cold(),
                                 (TCA)newStart,
                                 srcRecSPOff,
                                 REQ_RETRANSLATE,
                                 sk.offset(),
                                 TransFlags().packed);
  }
  SKTRACE(1, sk, "inserting anchor translation for (%p,%d) at %p\n",
          sk.unit(), sk.offset(), req);
  SrcRec* sr = m_tx.getSrcRec(sk);
  sr->setFuncInfo(sk.func());
  sr->setAnchorTranslation(req);

  if (srcRecSPOff) always_assert(sr->nonResumedSPOff() == *srcRecSPOff);

  size_t asize = code.main().frontier() - astart;
  size_t realColdSize   = code.realCold().frontier()   - realColdStart;
  size_t realFrozenSize = code.realFrozen().frontier() - realFrozenStart;
  assertx(asize == 0);
  if (realColdSize && RuntimeOption::EvalDumpTCAnchors) {
    TransRec tr(sk,
                TransKind::Anchor,
                astart, asize, realColdStart, realColdSize,
                realFrozenStart, realFrozenSize);
    m_tx.addTranslation(tr);
    if (RuntimeOption::EvalJitUseVtuneAPI) {
      reportTraceletToVtune(sk.unit(), sk.func(), tr);
    }

    if (m_tx.profData()) {
      m_tx.profData()->addTransNonProf(TransKind::Anchor, sk);
    }
    assertx(!m_tx.isTransDBEnabled() ||
           m_tx.getTransRec(realColdStart)->kind == TransKind::Anchor);
  }

  return retranslate(args);
}

TCA
MCGenerator::lookupTranslation(SrcKey sk) const {
  if (SrcRec* sr = m_tx.getSrcDB().find(sk)) {
    return sr->getTopTranslation();
  }
  return nullptr;
}

TCA
MCGenerator::translate(const TranslArgs& args) {
  INC_TPC(translate);

  assertx(((uintptr_t)vmsp() & (sizeof(Cell) - 1)) == 0);
  assertx(((uintptr_t)vmfp() & (sizeof(Cell) - 1)) == 0);
  assertx(m_tx.mode() != TransKind::Invalid);
  SCOPE_EXIT{ m_tx.setMode(TransKind::Invalid); };

  if (!shouldTranslate(args.sk.func())) return nullptr;

  auto func = const_cast<Func*>(args.sk.func());
  CodeCache::Selector cbSel(CodeCache::Selector::Args(code)
                            .profile(m_tx.mode() == TransKind::Profile)
                            .hot(RuntimeOption::EvalHotFuncCount &&
                                 (func->attrs() & AttrHot) && m_tx.useAHot()));

  auto start = translateWork(args);

  if (args.setFuncBody) {
    func->setFuncBody(start);
  }
  SKTRACE(1, args.sk, "translate moved head from %p to %p\n",
          getTopTranslation(args.sk), start);
  return start;
}

TCA MCGenerator::getFuncBody(Func* func) {
  TCA tca = func->getFuncBody();
  if (tca != m_tx.uniqueStubs.funcBodyHelperThunk) return tca;

  DVFuncletsVec dvs = func->getDVFunclets();

  if (dvs.size()) {
    LeaseHolder writer(Translator::WriteLease());
    if (!writer) return nullptr;
    tca = func->getFuncBody();
    if (tca != m_tx.uniqueStubs.funcBodyHelperThunk) return tca;
    tca = genFuncBodyDispatch(func, dvs);
    func->setFuncBody(tca);
  } else {
    SrcKey sk(func, func->base(), false);
    auto args = TranslArgs{sk, false};
    args.setFuncBody = true;
    tca = mcg->getTranslation(args);
  }

  return tca;
}

void MCGenerator::smashPrologueGuards(AtomicLowPtr<uint8_t>* prologues,
                                      int numPrologues, const Func* func) {
  for (int i = 0; i < numPrologues; i++) {
    auto const guard = funcGuardFromPrologue(prologues[i], func);
    if (funcGuardMatches(guard, func)) {
      clobberFuncGuard(guard, func);
    }
  }
}

/*
 * funcPrologue --
 *
 * Given a callee and a number of args, match up to the callee's
 * argument expectations and dispatch.
 *
 * Call/return hand-shaking is a bit funny initially. At translation time,
 * we don't necessarily know what function we're calling. For instance,
 *
 *   f(g());
 *
 * Will lead to a set of basic blocks like:
 *
 * b1: pushfuncd "f"
 *     pushfuncd "g"
 *     fcall
 * b2: fcall
 *
 * The fcall labelled "b2" above may not be statically bindable in our
 * execution model.
 *
 * We decouple the call work into a per-callsite portion, responsible
 * for recording the return address, and a per-(callee, numArgs) portion,
 * responsible for fixing up arguments and dispatching to remaining
 * code. We call the per-callee portion a "prologue."
 *
 * Also, we are called from two distinct environments. From REQ_BIND_CALL,
 * we're running "between" basic blocks, with all VM registers sync'ed.
 * However, we're also called in the middle of basic blocks, when dropping
 * entries into func->m_prologues. So don't go around using the
 * translation-time values of vmfp()/vmsp(), since they have an
 * unpredictable relationship to the source.
 */
bool
MCGenerator::checkCachedPrologue(const Func* func, int paramIdx,
                                 TCA& prologue) const {
  prologue = (TCA)func->getPrologue(paramIdx);
  if (prologue != m_tx.uniqueStubs.fcallHelperThunk) {
    TRACE(1, "cached prologue %s(%d) -> cached %p\n",
          func->fullName()->data(), paramIdx, prologue);
    assertx(isValidCodeAddress(prologue));
    return true;
  }
  return false;
}

TCA MCGenerator::emitFuncPrologue(Func* func, int argc) {
  const int nparams = func->numNonVariadicParams();
  const int paramIndex = argc <= nparams ? argc : nparams + 1;

  auto const funcBody = SrcKey { func, func->getEntryForNumArgs(argc), false };

  CodeCache::Selector cbSel(CodeCache::Selector::Args(code)
                            .profile(m_tx.mode() == TransKind::Proflogue)
                            .hot(RuntimeOption::EvalHotFuncCount &&
                                 (func->attrs() & AttrHot) && m_tx.useAHot()));
  assertx(m_fixups.empty());

  TCA mainOrig = code.main().frontier();

  // If we're close to a cache line boundary, just burn some space to
  // try to keep the func and its body on fewer total lines.
  align(code.main(), Alignment::CacheLineRoundUp, AlignContext::Dead);

  TransLocMaker maker(code);
  maker.markStart();

  // Careful: this isn't necessarily the real entry point. For funcIsMagic
  // prologues, this is just a possible prologue.
  TCA aStart = code.main().frontier();

  // Give the prologue a TransID if we have profiling data.
  auto transID = m_tx.profData()
    ? m_tx.profData()->addTransPrologue(m_tx.mode(), funcBody, paramIndex)
    : kInvalidTransID;

  TCA start = genFuncPrologue(transID, func, argc);

  auto loc = maker.markEnd();
  if (RuntimeOption::EvalEnableReusableTC) {
    TCA UNUSED ms = loc.mainStart(), me = loc.mainEnd(),
               cs = loc.coldStart(), ce = loc.coldEnd(),
               fs = loc.frozenStart(), fe = loc.frozenEnd(),
               oldStart = start;
    bool did_relocate = relocateNewTranslation(loc, code, &start);

    if (did_relocate) {
      FTRACE_MOD(reusetc, 1,
                 "Relocated prologue for func {} (id = {}) "
                 "from M[{}, {}], C[{}, {}], F[{}, {}] to M[{}, {}] "
                 "C[{}, {}] F[{}, {}] orig start @ {} new start @ {}\n",
                 func->fullName()->data(), func->getFuncId(),
                 ms, me, cs, ce, fs, fe, loc.mainStart(), loc.mainEnd(),
                 loc.coldStart(), loc.coldEnd(), loc.frozenStart(),
                 loc.frozenEnd(), oldStart, start);
    } else {
      FTRACE_MOD(reusetc, 1,
                 "Created prologue for func {} (id = {}) at "
                 "M[{}, {}], C[{}, {}], F[{}, {}] start @ {}\n",
                 func->fullName()->data(), func->getFuncId(),
                 ms, me, cs, ce, fs, fe, oldStart);
    }

    recordFuncPrologue(func, loc);
    if (loc.mainStart() != aStart) {
      code.main().setFrontier(mainOrig); // we may have shifted to align
    }
  }
  if (RuntimeOption::EvalPerfRelocate) {
    GrowableVector<IncomingBranch> incomingBranches;
    recordPerfRelocMap(loc.mainStart(), loc.mainEnd(),
                       loc.coldCodeStart(), loc.coldEnd(),
                       funcBody, paramIndex,
                       incomingBranches,
                       m_fixups);
  }
  m_fixups.process(nullptr);

  assertx(funcGuardMatches(funcGuardFromPrologue(start, func), func));
  assertx(isValidCodeAddress(start));

  TRACE(2, "funcPrologue mcg %p %s(%d) setting prologue %p\n",
        this, func->fullName()->data(), argc, start);
  func->setPrologue(paramIndex, start);

  assertx(m_tx.mode() == TransKind::Prologue ||
          m_tx.mode() == TransKind::Proflogue);

  auto tr = maker.rec(funcBody, m_tx.mode());
  m_tx.addTranslation(tr);
  if (RuntimeOption::EvalJitUseVtuneAPI) {
    reportTraceletToVtune(func->unit(), func, tr);
  }


  recordGdbTranslation(funcBody, func, code.main(), aStart, false, true);
  recordBCInstr(OpFuncPrologue, loc.mainStart(), loc.mainEnd(), false);

  m_numTrans++;
  assertx(m_numTrans <= RuntimeOption::EvalJitGlobalTranslationLimit);

  return start;
}

TCA MCGenerator::getFuncPrologue(Func* func, int nPassed, ActRec* ar,
                                 bool forRegeneratePrologue) {
  func->validate();
  TRACE(1, "funcPrologue %s(%d)\n", func->fullName()->data(), nPassed);
  int const numParams = func->numNonVariadicParams();
  int paramIndex = nPassed <= numParams ? nPassed : numParams + 1;

  // Do a quick test before grabbing the write lease
  TCA prologue;
  if (checkCachedPrologue(func, paramIndex, prologue)) return prologue;

  Offset entry = func->getEntryForNumArgs(nPassed);
  SrcKey funcBody(func, entry, false);

  LeaseHolder writer(Translator::WriteLease());
  if (!writer) return nullptr;

  // If we're regenerating a prologue, and we want to check shouldTranslate()
  // but ignore the code size limits.  We still want to respect the global
  // translation limit and other restrictions, though.
  if (forRegeneratePrologue) {
    if (!shouldTranslateNoSizeLimit(func)) return nullptr;
  } else {
    if (!shouldTranslate(func)) return nullptr;
  }

  // Double check the prologue array now that we have the write lease
  // in case another thread snuck in and set the prologue already.
  if (checkCachedPrologue(func, paramIndex, prologue)) return prologue;

  // We're coming from a BIND_CALL service request, so enable
  // profiling if we haven't optimized the function entry yet.
  assertx(m_tx.mode() == TransKind::Invalid ||
         m_tx.mode() == TransKind::Prologue);
  if (m_tx.mode() == TransKind::Invalid && profileSrcKey(funcBody)) {
    m_tx.setMode(TransKind::Proflogue);
  } else {
    m_tx.setMode(TransKind::Prologue);
  }
  SCOPE_EXIT{ m_tx.setMode(TransKind::Invalid); };

  try {
    return emitFuncPrologue(func, nPassed);
  } catch (const DataBlockFull& dbFull) {

    // Fail hard if the block isn't code.hot.
    always_assert_flog(dbFull.name == "hot",
                       "data block = {}\nmessage: {}\n",
                       dbFull.name, dbFull.what());

    // Otherwise, fall back to code.main and retry.
    assertx(m_tx.useAHot());
    m_tx.setUseAHot(false);
    m_fixups.clear();
    try {
      return emitFuncPrologue(func, nPassed);
    } catch (const DataBlockFull& dbFull) {
      always_assert_flog(0, "data block = {}\nmessage: {}\n",
                         dbFull.name, dbFull.what());
    }
  }
}

/**
 * Given the proflogueTransId for a TransProflogue translation,
 * regenerate the prologue (as a TransPrologue).  Returns the starting
 * address for the translation corresponding to triggerSk, if such
 * translation is generated; otherwise returns nullptr.
 */
TCA MCGenerator::regeneratePrologue(TransID prologueTransId, SrcKey triggerSk) {
  Func* func = m_tx.profData()->transFunc(prologueTransId);
  int  nArgs = m_tx.profData()->prologueArgs(prologueTransId);

  // Regenerate the prologue.
  func->resetPrologue(nArgs);
  m_tx.setMode(TransKind::Prologue);
  SCOPE_EXIT { m_tx.setMode(TransKind::Invalid); };
  auto const start = getFuncPrologue(
    func,
    nArgs,
    nullptr /* ActRec */,
    true /* regeneratePrologue */
  );
  if (!start) return nullptr;

  func->setPrologue(nArgs, start);

  // Smash callers of the old prologue with the address of the new one.
  PrologueCallersRec* pcr =
    m_tx.profData()->prologueCallers(prologueTransId);
  for (TCA toSmash : pcr->mainCallers()) {
    smashCall(toSmash, start);
  }

  // If the prologue has a matching guard, then smash its guard-callers as
  // well.
  auto const guard = funcGuardFromPrologue(start, func);
  if (funcGuardMatches(guard, func)) {
    for (TCA toSmash : pcr->guardCallers()) {
      smashCall(toSmash, guard);
    }
  }
  pcr->clearAllCallers();

  // If this prologue has a DV funclet, then generate a translation for the DV
  // funclet right after the prologue.
  TCA triggerSkStart = nullptr;
  if (nArgs < func->numNonVariadicParams()) {
    auto paramInfo = func->params()[nArgs];
    if (paramInfo.hasDefaultValue()) {
      m_tx.setMode(TransKind::Optimize);
      SrcKey funcletSK(func, paramInfo.funcletOff, false);
      auto funcletTransId = m_tx.profData()->dvFuncletTransId(func, nArgs);
      if (funcletTransId != kInvalidTransID) {
        invalidateSrcKey(funcletSK);
        auto args = TranslArgs{funcletSK, false};
        args.transId = funcletTransId;
        auto dvStart = translate(args);
        if (dvStart && !triggerSkStart && funcletSK == triggerSk) {
          triggerSkStart = dvStart;
        }
        // Flag that this translation has been retranslated, so that
        // it's not retranslated again along with the function body.
        m_tx.profData()->setOptimized(funcletSK);
      }
    }
  }

  return triggerSkStart;
}

/**
 * Regenerate all prologues of func that were previously generated.
 * The prologues are sorted in ascending order of profile counters.
 * For prologues with corresponding DV funclets, their corresponding
 * DV funclet will be regenerated right after them.  The idea is to
 * generate the function body right after calling this function, so
 * that all prologues are placed right before it, and with the hottest
 * prologues closer to it.
 *
 * Returns the starting address for the translation corresponding to
 * triggerSk, if such translation is generated; otherwise returns
 * nullptr.
 */
TCA MCGenerator::regeneratePrologues(Func* func, SrcKey triggerSk) {
  TCA triggerStart = nullptr;
  std::vector<TransID> prologTransIDs;

  for (int nArgs = 0; nArgs < func->numPrologues(); nArgs++) {
    TransID tid = m_tx.profData()->prologueTransId(func, nArgs);
    if (tid != kInvalidTransID) {
      prologTransIDs.push_back(tid);
    }
  }

  std::sort(prologTransIDs.begin(), prologTransIDs.end(),
          [&](TransID t1, TransID t2) -> bool {
            // This will sort in ascending order.
            return m_tx.profData()->transCounter(t2) >
                   m_tx.profData()->transCounter(t1);
          });

  // Next, we're going to regenerate each prologue along with its DV
  // funclet.  We consider the option of either including the DV
  // funclets in the same region as the function body or not.
  // Including them in the same region enables some type information
  // to flow and thus can eliminate some stores and type checks, but
  // it can also increase the code size by duplicating the whole
  // function body.  Therefore, we keep the DV inits separate if both
  // (a) the function has multiple proflogues, and (b) the size of the
  // function is above a certain threshold.
  //
  // The mechanism used to keep the function body separate from the DV
  // init is to temporarily mark the SrcKey for the function body as
  // already optimized.  (The region selectors break a region whenever
  // they hit a SrcKey that has already been optimized.)
  SrcKey funcBodySk(func, func->base(), false);
  if (prologTransIDs.size() > 1 &&
      func->past() - func->base() > RuntimeOption::EvalJitPGOMaxFuncSizeDupBody)
  {
    m_tx.profData()->setOptimized(funcBodySk);
  }
  SCOPE_EXIT{ m_tx.profData()->clearOptimized(funcBodySk); };

  for (TransID tid : prologTransIDs) {
    TCA start = regeneratePrologue(tid, triggerSk);
    if (triggerStart == nullptr && start != nullptr) {
      triggerStart = start;
    }
  }

  return triggerStart;
}

/*
 * bindJmp --
 *
 *   Runtime service handler that patches a jmp to the translation of
 *   u:dest from toSmash.
 */
TCA
MCGenerator::bindJmp(TCA toSmash, SrcKey destSk, ServiceRequest req,
                     TransFlags trflags, bool& smashed) {
  auto args = TranslArgs{destSk, false};
  args.flags = trflags;
  auto tDest = getTranslation(args);
  if (!tDest) return nullptr;

  LeaseHolder writer(Translator::WriteLease());
  if (!writer) return tDest;

  SrcRec* sr = m_tx.getSrcRec(destSk);
  // The top translation may have changed while we waited for the
  // write lease, so read it again.  If it was replaced with a new
  // translation, then bind to the new one.  If it was invalidated,
  // then don't bind the jump.
  tDest = sr->getTopTranslation();
  if (tDest == nullptr) return nullptr;

  if (req == REQ_BIND_ADDR) {
    auto addr = reinterpret_cast<TCA*>(toSmash);
    if (*addr == tDest) {
      // Already smashed
      return tDest;
    }
    sr->chainFrom(IncomingBranch::addr(addr));
    smashed = true;
    return tDest;
  }

  DecodedInstruction di(toSmash);
  if (di.isBranch() && !di.isJmp()) {
    auto const target = smashableJccTarget(toSmash);
    assertx(target);

    // Return if already smashed.
    if (target == tDest) return tDest;
    sr->chainFrom(IncomingBranch::jccFrom(toSmash));
  } else {
    auto const target = smashableJmpTarget(toSmash);
    assertx(target);

    // Return if already smashed.
    if (!target || target == tDest) return tDest;
    sr->chainFrom(IncomingBranch::jmpFrom(toSmash));
  }

  smashed = true;
  return tDest;
}

/*
 * When we end a tracelet with a conditional jump, emitCondJmp first emits:
 *
 *   1:         j<CC> stubJmpccFirst
 *              jmp   stubJmpccFirst
 *
 * Our "taken" argument tells us whether the branch at 1: was taken or
 * not; and therefore which of offTaken and offNotTaken to continue executing.
 * If we did take the branch, we now rewrite the code so that the branch is
 * straightened. This predicts that subsequent executions will go the same way
 * as the first execution.
 *
 *              jn<CC> stubJmpccSecond:offNotTaken
 *              nop5   ; fallthru, or jmp if there's already a translation.
 * offTaken:
 *
 * If we did not take the branch, we leave the sense of the condition
 * intact, while patching it up to go to the unexplored code:
 *
 *              j<CC> stubJmpccSecond:offTaken
 *              nop5
 * offNotTaken:
 */
TCA
MCGenerator::bindJccFirst(TCA jccAddr, SrcKey skTaken, SrcKey skNotTaken,
                          bool taken, bool& smashed) {
  LeaseHolder writer(Translator::WriteLease());
  if (!writer) return nullptr;

  auto const skWillExplore = taken ? skTaken : skNotTaken;
  auto const skWillDefer = taken ? skNotTaken : skTaken;
  auto const dest = skWillExplore;
  auto cc = smashableJccCond(jccAddr);

  TRACE(3, "bindJccFirst: explored %d, will defer %d; "
           "overwriting cc%02x taken %d\n",
        skWillExplore.offset(), skWillDefer.offset(), cc, taken);
  always_assert(skTaken.resumed() == skNotTaken.resumed());

  // We want the branch to point to whichever side has not been explored yet.
  if (taken) cc = ccNegate(cc);

  auto& cb = code.blockFor(jccAddr);

  // It's not clear where the IncomingBranch should go to if cb is frozen.
  assertx(&cb != &code.frozen());

  auto const jmpAddr = jccAddr + smashableJccLen();
  auto const afterAddr = jmpAddr + smashableJmpLen();

  // Can we just directly fall through?
  bool const fallThru = afterAddr == cb.frontier() &&
                        !m_tx.getSrcDB().find(dest);

  auto const tDest = getTranslation(TranslArgs{dest, !fallThru});
  if (!tDest) return nullptr;

  auto const jmpTarget = smashableJmpTarget(jmpAddr);
  if (jmpTarget != smashableJccTarget(jccAddr)) {
    // Someone else already smashed this one.  Ideally we would just re-execute
    // from jccAddr---except the status flags will have been trashed.
    return tDest;
  }

  auto const stub = svcreq::emit_bindjmp_stub(
    code.frozen(),
    liveSpOff(),
    jccAddr,
    skWillDefer,
    TransFlags{}
  );

  mcg->cgFixups().process(nullptr);
  smashed = true;
  assertx(Translator::WriteLease().amOwner());

  /*
   * Roll over the jcc and the jmp/fallthru. E.g., from:
   *
   *     toSmash:    jcc   <jmpccFirstStub>
   *     toSmash+6:  jmp   <jmpccFirstStub>
   *     toSmash+11: <probably the new translation == tDest>
   *
   * to:
   *
   *     toSmash:    j[n]z <jmpccSecondStub>
   *     toSmash+6:  nop5
   *     toSmash+11: newHotness
   */
  smashJcc(jccAddr, stub, cc);
  m_tx.getSrcRec(dest)->chainFrom(IncomingBranch::jmpFrom(jmpAddr));

  TRACE(5, "bindJccFirst: overwrote with cc%02x taken %d\n", cc, taken);
  return tDest;
}

namespace {

class FreeRequestStubTrigger {
  TCA m_stub;
 public:
  explicit FreeRequestStubTrigger(TCA stub) : m_stub(stub) {
    TRACE(3, "FreeStubTrigger @ %p, stub %p\n", this, m_stub);
  }
  void operator()() {
    TRACE(3, "FreeStubTrigger: Firing @ %p , stub %p\n", this, m_stub);
    if (mcg->freeRequestStub(m_stub) != true) {
      // If we can't free the stub, enqueue again to retry.
      TRACE(3, "FreeStubTrigger: write lease failed, requeueing %p\n", m_stub);
      Treadmill::enqueue(FreeRequestStubTrigger(m_stub));
    }
  }
};
}

#ifdef DEBUG

struct DepthGuard {
  static __thread int m_depth;
  DepthGuard()  { m_depth++; TRACE(2, "DepthGuard: %d {\n", m_depth); }
  ~DepthGuard() { TRACE(2, "DepthGuard: %d }\n", m_depth); m_depth--; }

  bool depthOne() const { return m_depth == 1; }
};
__thread int DepthGuard::m_depth;

#else

struct DepthGuard { bool depthOne() const { return false; } };

#endif

void
MCGenerator::enterTC(TCA start, ActRec* stashedAR) {
  if (debug) {
    fflush(stdout);
    fflush(stderr);
  }
  DepthGuard d;

  assertx(isValidCodeAddress(start));
  assertx(((uintptr_t)vmsp() & (sizeof(Cell) - 1)) == 0);
  assertx(((uintptr_t)vmfp() & (sizeof(Cell) - 1)) == 0);

  Translator::WriteLease().gremlinUnlock();
  assertx(!Translator::WriteLease().amOwner());

  INC_TPC(enter_tc);
  if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
    auto skData = SrcKey{liveFunc(), vmpc(), liveResumed()}.toAtomicInt();
    Trace::ringbufferEntry(RBTypeEnterTC, skData, (uint64_t)start);
  }

  tl_regState = VMRegState::DIRTY;
  enterTCImpl(start, stashedAR);
  tl_regState = VMRegState::CLEAN;
  assertx(isValidVMStackAddress(vmsp()));

  if (debug) {
    // Debugging code: cede the write lease half the time.
    if (RuntimeOption::EvalJitStressLease) {
      if (d.depthOne() && (rand() % 2) == 0) {
        Translator::WriteLease().gremlinLock();
      }
    }
  }

  vmfp() = nullptr;
}

TCA MCGenerator::handleServiceRequest(svcreq::ReqInfo& info) noexcept {
  FTRACE(1, "handleServiceRequest {}\n", svcreq::to_name(info.req));

  assert_native_stack_aligned();
  tl_regState = VMRegState::CLEAN; // partially a lie: vmpc() isn't synced

  if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
    Trace::ringbufferEntry(
      RBTypeServiceReq, (uint64_t)info.req, (uint64_t)info.args[0].tca
    );
  }

  TCA start = nullptr;
  SrcKey sk;
  auto smashed = false;

  switch (info.req) {
    case REQ_BIND_JMP:
    case REQ_BIND_ADDR: {
      auto const toSmash = info.args[0].tca;
      sk = SrcKey::fromAtomicInt(info.args[1].sk);
      auto const trflags = info.args[2].trflags;
      start = bindJmp(toSmash, sk, info.req, trflags, smashed);
      break;
    }

    case REQ_BIND_JCC_FIRST: {
      auto toSmash = info.args[0].tca;
      auto skTaken = SrcKey::fromAtomicInt(info.args[1].sk);
      auto skNotTaken = SrcKey::fromAtomicInt(info.args[2].sk);
      auto taken = info.args[3].boolVal;
      sk = taken ? skTaken : skNotTaken;
      start = bindJccFirst(toSmash, skTaken, skNotTaken, taken, smashed);
      break;
    }

    case REQ_RETRANSLATE: {
      INC_TPC(retranslate);
      sk = SrcKey{liveFunc(), info.args[0].offset, liveResumed()};
      auto trflags = info.args[1].trflags;
      auto args = TranslArgs{sk, true};
      args.flags = trflags;
      start = retranslate(args);
      SKTRACE(2, sk, "retranslated @%p\n", start);
      break;
    }

    case REQ_RETRANSLATE_OPT: {
      sk = SrcKey::fromAtomicInt(info.args[0].sk);
      auto transID = info.args[1].transID;
      start = retranslateOpt(transID, false);
      SKTRACE(2, sk, "retranslated-OPT: transId = %d  start: @%p\n", transID,
              start);
      break;
    }

    case REQ_POST_INTERP_RET: {
      // This is only responsible for the control-flow aspect of the Ret:
      // getting to the destination's translation, if any.
      auto ar = info.args[0].ar;
      auto caller = info.args[1].ar;
      assertx(caller == vmfp());
      // If caller is a resumable (aka a generator) then whats likely happened
      // here is that we're resuming a yield from. That expression happens to
      // cause an assumption that we made earlier to be violated (that `ar` is
      // on the stack), so if we detect this situation we need to fix up the
      // value of `ar`.
      if (UNLIKELY(caller->resumed() &&
                   caller->func()->isNonAsyncGenerator())) {
        auto gen = frame_generator(caller);
        if (gen->m_delegate.m_type == KindOfObject) {
          auto delegate = gen->m_delegate.m_data.pobj;
          // We only checked that our delegate is an object, but we can't get
          // into this situation if the object itself isn't a Generator
          assert(delegate->getVMClass() == Generator::getClass());
          // Ok so we're in a `yield from` situation, we know our ar is garbage.
          // The ar that we're looking for is the ar of the delegate generator,
          // so grab that here.
          ar = Generator::fromObject(delegate)->actRec();
        }
      }
      Unit* destUnit = caller->func()->unit();
      // Set PC so logging code in getTranslation doesn't get confused.
      vmpc() = destUnit->at(caller->m_func->base() + ar->m_soff);
      sk = SrcKey{caller->func(), vmpc(), caller->resumed()};
      start = getTranslation(TranslArgs{sk, true});
      TRACE(3, "REQ_POST_INTERP_RET: from %s to %s\n",
            ar->m_func->fullName()->data(),
            caller->m_func->fullName()->data());
      break;
    }

    case REQ_POST_DEBUGGER_RET: {
      auto fp = vmfp();
      auto caller = fp->func();
      vmpc() = caller->unit()->at(caller->base() +
                                  unwindRdsInfo->debuggerReturnOff);
      FTRACE(3, "REQ_DEBUGGER_RET: pc {} in {}\n",
             vmpc(), fp->func()->fullName()->data());
      sk = SrcKey{fp->func(), vmpc(), fp->resumed()};
      start = getTranslation(TranslArgs{sk, true});
      break;
    }
  }

  if (smashed && info.stub) {
    Treadmill::enqueue(FreeRequestStubTrigger(info.stub));
  }

  if (start == nullptr) {
    vmpc() = sk.unit()->at(sk.offset());
    start = m_tx.uniqueStubs.interpHelperSyncedPC;
  }

  if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
    auto skData = sk.valid() ? sk.toAtomicInt() : uint64_t(-1LL);
    Trace::ringbufferEntry(RBTypeResumeTC, skData, (uint64_t)start);
  }

  tl_regState = VMRegState::DIRTY;
  return start;
}

TCA MCGenerator::handleBindCall(TCA toSmash,
                                ActRec* calleeFrame,
                                bool isImmutable) {
  Func* func = const_cast<Func*>(calleeFrame->m_func);
  int nArgs = calleeFrame->numArgs();
  TRACE(2, "bindCall %s, ActRec %p\n", func->fullName()->data(), calleeFrame);
  TCA start = getFuncPrologue(func, nArgs);
  TRACE(2, "bindCall -> %p\n", start);
  if (start && !isImmutable) {
    // We dont know we're calling the right function, so adjust start to point
    // to the dynamic check of ar->m_func.
    start = funcGuardFromPrologue(start, func);
  } else {
    TRACE(2, "bindCall immutably %s -> %p\n", func->fullName()->data(), start);
  }

  if (start && !RuntimeOption::EvalFailJitPrologs) {
    LeaseHolder writer(Translator::WriteLease());
    if (writer) {
      // Someone else may have changed the func prologue while we waited for
      // the write lease, so read it again.
      start = getFuncPrologue(func, nArgs);
      if (start && !isImmutable) start = funcGuardFromPrologue(start, func);

      if (start && smashableCallTarget(toSmash) != start) {
        assertx(smashableCallTarget(toSmash));
        TRACE(2, "bindCall smash %p -> %p\n", toSmash, start);
        smashCall(toSmash, start);

        bool is_profiled = false;
        // For functions to be PGO'ed, if their current prologues are still
        // profiling ones (living in code.prof()), then save toSmash as a
        // caller to the prologue, so that it can later be smashed to call a
        // new prologue when it's generated.
        int calleeNumParams = func->numNonVariadicParams();
        int calledPrologNumArgs = (nArgs <= calleeNumParams ?
                                   nArgs :  calleeNumParams + 1);
        if (code.prof().contains(start) && !m_tx.profData()->freed()) {
          if (isImmutable) {
            m_tx.profData()->addPrologueMainCaller(
              func, calledPrologNumArgs, toSmash);
          } else {
            m_tx.profData()->addPrologueGuardCaller(
              func, calledPrologNumArgs, toSmash);
          }
          is_profiled = true;
        }

        // We need to be able to reclaim the function prologues once the unit
        // associated with this function is treadmilled-- so record all of the
        // callers that will need to be re-smashed
        if (RuntimeOption::EvalEnableReusableTC) {
          if (debug || !isImmutable) {
            recordFuncCaller(func, toSmash, isImmutable,
                             is_profiled, calledPrologNumArgs);
          }
        }
      }
    }
  } else {
    // We couldn't get a prologue address. Return a stub that will finish
    // entering the callee frame in C++, then call handleResume at the callee's
    // entry point.
    start = m_tx.uniqueStubs.fcallHelperThunk;
  }

  return start;
}

TCA MCGenerator::handleResume(bool interpFirst) {
  assert_native_stack_aligned();
  FTRACE(1, "handleResume({})\n", interpFirst);

  if (!vmRegsUnsafe().pc) return m_tx.uniqueStubs.callToExit;

  tl_regState = VMRegState::CLEAN;

  auto sk = SrcKey{liveFunc(), vmpc(), liveResumed()};
  TCA start;
  if (interpFirst) {
    start = nullptr;
    INC_TPC(interp_bb_force);
  } else {
    start = getTranslation(TranslArgs(sk, true));
  }

  vmJitCalledFrame() = vmfp();
  SCOPE_EXIT { vmJitCalledFrame() = nullptr; };

  // If we can't get a translation at the current SrcKey, interpret basic
  // blocks until we end up somewhere with a translation (which we may have
  // created, if the lease holder dropped it).
  while (!start) {
    INC_TPC(interp_bb);
    if (auto retAddr = HPHP::dispatchBB()) {
      start = retAddr;
      break;
    }

    assertx(vmpc());
    sk = SrcKey{liveFunc(), vmpc(), liveResumed()};
    start = getTranslation(TranslArgs{sk, true});
  }

  if (Trace::moduleEnabled(Trace::ringbuffer, 1)) {
    auto skData = sk.valid() ? sk.toAtomicInt() : uint64_t(-1LL);
    Trace::ringbufferEntry(RBTypeResumeTC, skData, (uint64_t)start);
  }

  tl_regState = VMRegState::DIRTY;
  return start;
}

void handleStackOverflow(ActRec* calleeAR) {
  /*
   * First synchronize registers.
   *
   * We're called in two situations: either this is the first frame after a
   * re-entry, in which case calleeAR->m_sfp is enterTCHelper's native stack,
   * or we're called in the middle of one VM entry (from a func prologue).  We
   * want to raise the exception from the caller's FCall instruction in the
   * second case, and in the first case we have to raise in a special way
   * inside this re-entry.
   *
   * Either way the stack depth is below the calleeAR by numArgs, because we
   * haven't run func prologue duties yet.
   */
  auto& unsafeRegs = vmRegsUnsafe();
  auto const isReentry = calleeAR == vmFirstAR();
  auto const arToSync = isReentry ? calleeAR : calleeAR->m_sfp;
  unsafeRegs.fp = arToSync;
  unsafeRegs.stack.top() =
    reinterpret_cast<Cell*>(calleeAR) - calleeAR->numArgs();
  auto const func_base = arToSync->func()->base();
  // calleeAR m_soff is 0 in the re-entry case, so we'll set pc to the func
  // base.  But it also doesn't matter because we're going to throw a special
  // VMReenterStackOverflow in that case so the unwinder won't worry about it.
  unsafeRegs.pc = arToSync->func()->unit()->at(func_base + calleeAR->m_soff);
  tl_regState = VMRegState::CLEAN;

  if (!isReentry) {
    /*
     * The normal case - we were called via FCall, or FCallArray.  We need to
     * construct the pc of the fcall from the return address (which will be
     * after the fcall). Because fcall is a variable length instruction, and
     * because we sometimes delete instructions from the instruction stream, we
     * need to use fpi regions to find the fcall.
     */
    const FPIEnt* fe = liveFunc()->findPrecedingFPI(
      liveUnit()->offsetOf(vmpc()));
    vmpc() = liveUnit()->at(fe->m_fcallOff);
    assertx(isFCallStar(peek_op(vmpc())));
    raise_error("Stack overflow");
  } else {
    /*
     * We were called via re-entry.  Leak the params and the actrec, and tell
     * the unwinder that there's nothing left to do in this "entry".
     *
     * Also, the caller hasn't set up the m_invName area on the ActRec (unless
     * it was a magic call), since it's the prologue's responsibility if it's a
     * non-magic call.  We can just null it out since we're fatalling.
     */
    vmsp() = reinterpret_cast<Cell*>(calleeAR + 1);
    calleeAR->setVarEnv(nullptr);
    throw VMReenterStackOverflow();
  }
  not_reached();
}

///////////////////////////////////////////////////////////////////////////////

bool checkCalleeStackOverflow(const ActRec* calleeAR) {
  auto const func = calleeAR->func();
  auto const limit = func->maxStackCells() + kStackCheckPadding;

  const void* const needed_top =
    reinterpret_cast<const TypedValue*>(calleeAR) - limit;

  const void* const limit_addr =
    static_cast<char*>(vmRegsUnsafe().stack.getStackLowAddress()) +
    Stack::sSurprisePageSize;

  return needed_top < limit_addr;
}

void handlePossibleStackOverflow(ActRec* calleeAR) {
  assert_native_stack_aligned();

  // If it's not an overflow, it was probably a surprise flag trip.  But we
  // can't assert that it is because background threads are allowed to clear
  // surprise bits concurrently, so it could be cleared again by now.
  if (!checkCalleeStackOverflow(calleeAR)) return;
  auto const func = calleeAR->func();

  /*
   * Stack overflows in this situation are a slightly different case than
   * handleStackOverflow:
   *
   * A function prologue already did all the work to prepare to enter the
   * function, but then it found out it didn't have enough room on the stack.
   * It may even have written uninits deeper than the stack base (but we limit
   * it to sSurprisePageSize, so it's harmless).
   *
   * Most importantly, it might have pulled args /off/ the eval stack and
   * shoved them into an ExtraArgs on the calleeAR, or into an array for a
   * variadic capture param.  We need to get things into an appropriate state
   * for handleStackOverflow to be able to synchronize things to throw from the
   * PC of the caller's FCall.
   *
   * We don't actually need to make sure the stack is the right depth for the
   * FCall: the unwinder will expect to see a pre-live ActRec (and we'll set it
   * up so it will), but it doesn't care how many args (or what types of args)
   * are below it on the stack.
   *
   * It is tempting to try to free the ExtraArgs structure here, but it's ok to
   * not to:
   *
   *     o We're about to raise an uncatchable fatal, which will end the
   *       request.  We leak ExtraArgs in other similar situations for this too
   *       (e.g. if called via FCallArray and then a stack overflow happens).
   *
   *     o If we were going to free the ExtraArgs structure, we'd need to make
   *       sure we can re-enter the VM right now, which means performing a
   *       manual fixup first.  (We aren't in a situation where we can do a
   *       normal VMRegAnchor fixup right now.)  But moreover we shouldn't be
   *       running destructors if a fatal is happening anyway, so we don't want
   *       that either.
   *
   * So, all that boils down to this: we ignore the extra args field (the
   * unwinder will not consult the ExtraArgs field because it believes the
   * ActRec is pre-live).  And set calleeAR->m_numArgs to indicate how many
   * things are actually on the stack (so handleStackOverflow knows what to set
   * the vmsp to)---we just set it to the function's numLocals, which might
   * mean decreffing some uninits unnecessarily, but that's ok.
   */

  if (debug && func->attrs() & AttrMayUseVV && calleeAR->getExtraArgs()) {
    calleeAR->trashVarEnv();
  }
  calleeAR->setNumArgs(calleeAR->m_func->numLocals());
  handleStackOverflow(calleeAR);
}

/*
 * Support for the stub freelist.
 */
TCA FreeStubList::maybePop() {
  StubNode* ret = m_list;
  if (ret) {
    TRACE(1, "alloc stub %p\n", ret);
    m_list = ret->m_next;
    ret->m_freed = ~kStubFree;
  }
  return (TCA)ret;
}

void FreeStubList::push(TCA stub) {
  /*
   * A freed stub may be released by Treadmill more than once if multiple
   * threads execute the service request before it is freed. We detect
   * duplicates by marking freed stubs
   */
  StubNode* n = reinterpret_cast<StubNode*>(stub);
  if (n->m_freed == kStubFree) {
    TRACE(1, "already freed stub %p\n", stub);
    return;
  }
  n->m_freed = kStubFree;
  n->m_next = m_list;
  TRACE(1, "free stub %p (-> %p)\n", stub, m_list);
  m_list = n;
}

bool
MCGenerator::freeRequestStub(TCA stub) {
  LeaseHolder writer(Translator::WriteLease());
  /*
   * If we can't acquire the write lock, the caller
   * (FreeRequestStubTrigger) retries
   */
  if (!writer) return false;
  assertx(code.frozen().contains(stub));
  m_debugInfo.recordRelocMap(stub, 0, "FreeStub");
  m_freeStubs.push(stub);
  return true;
}

TCA MCGenerator::getFreeStub(CodeBlock& frozen, CodeGenFixups* fixups,
                             bool* isReused) {
  TCA ret = m_freeStubs.maybePop();
  if (isReused) *isReused = ret;

  if (ret) {
    Stats::inc(Stats::Astub_Reused);
    always_assert(m_freeStubs.peek() == nullptr ||
                  code.isValidCodeAddress(m_freeStubs.peek()));
    TRACE(1, "recycle stub %p\n", ret);
  } else {
    ret = frozen.frontier();
    Stats::inc(Stats::Astub_New);
    TRACE(1, "alloc new stub %p\n", ret);
  }

  if (fixups) {
    fixups->m_reusedStubs.emplace_back(ret);
  }
  return ret;
}

TCA MCGenerator::getTranslatedCaller() const {
  DECLARE_FRAME_POINTER(fp);
  ActRec* framePtr = fp;  // can't directly mutate the register-mapped one
  for (; framePtr; framePtr = framePtr->m_sfp) {
    TCA rip = (TCA)framePtr->m_savedRip;
    if (isValidCodeAddress(rip)) {
      return rip;
    }
  }
  return nullptr;
}

void
MCGenerator::syncWork() {
  assertx(tl_regState == VMRegState::DIRTY);
  m_fixupMap.fixup(g_context.getNoCheck());
  tl_regState = VMRegState::CLEAN;
  Stats::inc(Stats::TC_Sync);
}

// Get the address of the literal val in the global data section.
// If it's not there, add it to the map in m_fixups, which will
// be committed to m_literals when m_fixups.process() is called.
const uint64_t*
MCGenerator::allocLiteral(uint64_t val) {
  auto it = m_literals.find(val);
  if (it != m_literals.end()) {
    assertx(*it->second == val);
    return it->second;
  }
  auto& pending = m_fixups.m_literals;
  it = pending.find(val);
  if (it != pending.end()) {
    assertx(*it->second == val);
    return it->second;
  }
  auto addr = allocData<uint64_t>(sizeof(uint64_t), 1);
  *addr = val;
  return pending[val] = addr;
}

bool
MCGenerator::reachedTranslationLimit(SrcKey sk,
                                     const SrcRec& srcRec) const {
  if (srcRec.translations().size() == RuntimeOption::EvalJitMaxTranslations) {
    INC_TPC(max_trans);
    if (debug && Trace::moduleEnabled(Trace::mcg, 2)) {
      const auto& tns = srcRec.translations();
      TRACE(1, "Too many (%zd) translations: %s, BC offset %d\n",
            tns.size(), sk.unit()->filepath()->data(),
            sk.offset());
      SKTRACE(2, sk, "{\n");
      TCA topTrans = srcRec.getTopTranslation();
      for (size_t i = 0; i < tns.size(); ++i) {
        const TransRec* rec = m_tx.getTransRec(tns[i].mainStart());
        assertx(rec);
        SKTRACE(2, sk, "%zd %p\n", i, tns[i].mainStart());
        if (tns[i].mainStart() == topTrans) {
          SKTRACE(2, sk, "%zd: *Top*\n", i);
        }
        if (rec->kind == TransKind::Anchor) {
          SKTRACE(2, sk, "%zd: Anchor\n", i);
        } else {
          SKTRACE(2, sk, "%zd: guards {\n", i);
          for (unsigned j = 0; j < rec->guards.size(); ++j) {
            FTRACE(2, "{}\n", rec->guards[j]);
          }
          SKTRACE(2, sk, "%zd } guards\n", i);
        }
      }
      SKTRACE(2, sk, "} /* Too many translations */\n");
    }
    return true;
  }

  return false;
}

void
MCGenerator::recordSyncPoint(CodeAddress frontier, Fixup fix) {
  m_fixups.m_pendingFixups.push_back(PendingFixup(frontier, fix));
}

/*
 * Equivalent to container.clear(), but guarantees to free
 * any memory associated with the container (eg clear
 * doesn't affect std::vector's capacity).
 */
template <typename T> void ClearContainer(T& container) {
  T().swap(container);
}

void
CodeGenFixups::process_only(
  GrowableVector<IncomingBranch>* inProgressTailBranches) {
  for (uint32_t i = 0; i < m_pendingFixups.size(); i++) {
    TCA tca = m_pendingFixups[i].m_tca;
    assertx(mcg->isValidCodeAddress(tca));
    mcg->fixupMap().recordFixup(tca, m_pendingFixups[i].m_fixup);
  }
  ClearContainer(m_pendingFixups);

  auto& ctm = mcg->catchTraceMap();
  for (auto const& pair : m_pendingCatchTraces) {
    if (auto pos = ctm.find(pair.first)) {
      *pos = pair.second;
    } else {
      ctm.insert(pair.first, pair.second);
    }
  }
  ClearContainer(m_pendingCatchTraces);

  for (auto const& elm : m_pendingJmpTransIDs) {
    mcg->getJmpToTransIDMap()[elm.first] = elm.second;
  }
  ClearContainer(m_pendingJmpTransIDs);

  mcg->literals().insert(m_literals.begin(), m_literals.end());
  ClearContainer(m_literals);

  if (inProgressTailBranches) {
    m_inProgressTailJumps.swap(*inProgressTailBranches);
  }
  assertx(m_inProgressTailJumps.empty());

  for (auto& stub : m_reusedStubs) {
    mcg->getDebugInfo()->recordRelocMap(stub, 0, "NewStub");
  }
  ClearContainer(m_reusedStubs);
}

void CodeGenFixups::clear() {
  ClearContainer(m_pendingFixups);
  ClearContainer(m_pendingCatchTraces);
  ClearContainer(m_pendingJmpTransIDs);
  ClearContainer(m_reusedStubs);
  ClearContainer(m_addressImmediates);
  ClearContainer(m_codePointers);
  ClearContainer(m_bcMap);
  ClearContainer(m_alignFixups);
  ClearContainer(m_inProgressTailJumps);
  ClearContainer(m_literals);
}

bool CodeGenFixups::empty() const {
  return
    m_pendingFixups.empty() &&
    m_pendingCatchTraces.empty() &&
    m_pendingJmpTransIDs.empty() &&
    m_reusedStubs.empty() &&
    m_addressImmediates.empty() &&
    m_codePointers.empty() &&
    m_bcMap.empty() &&
    m_alignFixups.empty() &&
    m_inProgressTailJumps.empty() &&
    m_literals.empty();
}

TCA MCGenerator::translateWork(const TranslArgs& args) {
  Timer _t(Timer::translate);
  auto sk = args.sk;

  SKTRACE(1, sk, "translateWork\n");
  assertx(m_tx.getSrcDB().find(sk));

  TCA mainOrig = code.main().frontier();

  if (args.align) {
    // Align without registering fixups; we do so manually after translating
    // the region because we may hit retries and need to roll back.
    align(code.main(), Alignment::CacheLine, AlignContext::Dead, false);
  }

  TransLocMaker maker(code);
  TCA        start             = code.main().frontier();
  TCA DEBUG_ONLY frozenStart   = code.frozen().frontier();
  SrcRec&    srcRec            = *m_tx.getSrcRec(sk);
  TransKind  transKindToRecord = TransKind::Interp;
  UndoMarker undo[] = {
    UndoMarker{code.main()},
    UndoMarker{code.cold()},
    UndoMarker{code.frozen()},
    UndoMarker{code.data()},
  };
  m_annotations.clear();

  setUseLLVM(
    RuntimeOption::EvalJitLLVM > 1 ||
    (RuntimeOption::EvalJitLLVM && m_tx.mode() == TransKind::Optimize)
  );
  SCOPE_EXIT { setUseLLVM(false); };

  auto resetState = [&] {
    for (auto& u : undo) u.undo();
    m_fixups.clear();
  };

  auto assertCleanState = [&] {
    assertx(code.main().frontier() == start);
    assertx(code.frozen().frontier() == frozenStart);
    assertx(m_fixups.empty());
  };

  FPInvOffset initSpOffset =
    args.region ? args.region->entry()->initialSpOffset()
                : liveSpOff();

  PostConditions pconds;
  RegionDescPtr region;
  bool hasLoop = false;

  if (!reachedTranslationLimit(sk, srcRec)) {
    // Attempt to create a region at this SrcKey
    if (m_tx.mode() == TransKind::Optimize) {
      assertx(RuntimeOption::EvalJitPGO);
      region = args.region;
      if (region) {
        assertx(!region->empty());
      } else {
        assertx(isValidTransID(args.transId));
        region = selectHotRegion(args.transId, this);
        assertx(region);
        if (region && region->empty()) region = nullptr;
      }
    } else {
      assertx(m_tx.mode() == TransKind::Profile ||
              m_tx.mode() == TransKind::Live);
      RegionContext rContext { sk.func(), sk.offset(), liveSpOff(),
                               sk.resumed() };
      FTRACE(2, "populating live context for region\n");
      populateLiveContext(rContext);
      region = selectRegion(rContext, m_tx.mode());
    }

    auto result = TranslateResult::Retry;
    TranslateRetryContext retry;
    initSpOffset = region ? region->entry()->initialSpOffset()
                          : liveSpOff();
    while (region && result == TranslateResult::Retry) {
      auto const profTransID = RuntimeOption::EvalJitPGO
        ? m_tx.profData()->curTransID()
        : kInvalidTransID;
      auto const transContext = TransContext(profTransID, sk, initSpOffset);

      IRGS irgs { transContext, args.flags };
      FTRACE(1, "{}{:-^40}{}\n",
             color(ANSI_COLOR_BLACK, ANSI_BGCOLOR_GREEN),
             " HHIR during translation ",
             color(ANSI_COLOR_END));

      try {
        assertCleanState();
        maker.markStart();

        result = translateRegion(irgs, *region, retry, args.flags, pconds);
        hasLoop = RuntimeOption::EvalJitLoops && cfgHasLoop(irgs.unit);
        FTRACE(2, "translateRegion finished with result {}\n", show(result));
      } catch (const std::exception& e) {
        FTRACE(1, "translateRegion failed with '{}'\n", e.what());
        result = TranslateResult::Failure;
      }

      if (result != TranslateResult::Success) {
        // Translation failed or will be retried. Free resources for this
        // trace, rollback the translation cache frontiers, and discard any
        // pending fixups.
        resetState();
      }

      if (result == TranslateResult::Failure) {
        // If the region translator failed, clear `region' to fall
        // back to the interpreter.
        FTRACE(1, "translateRegion: failed on region:\n{}\n", show(*region));
        region.reset();
      }
    }

    if (!region) m_tx.setMode(TransKind::Interp);

    if (result == TranslateResult::Success) {
      assertx(m_tx.mode() == TransKind::Live    ||
             m_tx.mode() == TransKind::Profile ||
             m_tx.mode() == TransKind::Optimize);
      transKindToRecord = m_tx.mode();
    }
  }

  if (transKindToRecord == TransKind::Interp) {
    assertCleanState();
    maker.markStart();

    FTRACE(1, "emitting dispatchBB interp request for failed "
      "translation (spOff = {})\n", initSpOffset.offset);
    vwrap(code.main(),
          [&] (Vout& v) { emitInterpReq(v, sk, initSpOffset); },
          CodeKind::Helper);
    // Fall through.
  }

  auto loc = maker.markEnd();

  if (args.align) {
    m_fixups.m_alignFixups.emplace(
      loc.mainStart(),
      std::make_pair(Alignment::CacheLine, AlignContext::Dead)
    );
  }

  if (RuntimeOption::EvalEnableReusableTC) {
    TCA UNUSED ms = loc.mainStart(), me = loc.mainEnd(),
               cs = loc.coldStart(), ce = loc.coldEnd(),
               fs = loc.frozenStart(), fe = loc.frozenEnd(),
               oldStart = start;
    bool did_relocate = relocateNewTranslation(loc, code);

    if (did_relocate) {
      FTRACE_MOD(reusetc, 1,
                 "Relocated translation for func {} (id = {})  @ sk({}) "
                 "from M[{}, {}], C[{}, {}], F[{}, {}] to M[{}, {}] "
                 "C[{}, {}] F[{}, {}]\n",
                 sk.func()->fullName()->data(), sk.func()->getFuncId(),
                 sk.offset(), ms, me, cs, ce, fs, fe, loc.mainStart(),
                 loc.mainEnd(), loc.coldStart(), loc.coldEnd(),
                 loc.frozenStart(), loc.frozenEnd());
    } else {
      FTRACE_MOD(reusetc, 1,
                 "Created translation for func {} (id = {}) "
                 " @ sk({}) at M[{}, {}], C[{}, {}], F[{}, {}]\n",
                 sk.func()->fullName()->data(), sk.func()->getFuncId(),
                 sk.offset(), ms, me, cs, ce, fs, fe);
    }

    if (loc.mainStart() != start) {
      code.main().setFrontier(mainOrig); // we may have shifted to align
    }
  }

  if (RuntimeOption::EvalProfileBC) {
    auto* unit = sk.unit();
    TransBCMapping prev{};
    for (auto& cur : m_fixups.m_bcMap) {
      if (!cur.aStart) continue;
      if (prev.aStart) {
        if (prev.bcStart < unit->bclen()) {
          recordBCInstr(uint32_t(unit->getOp(prev.bcStart)),
                        prev.aStart, cur.aStart, false);
        }
      } else {
        recordBCInstr(OpTraceletGuard, loc.mainStart(), cur.aStart, false);
      }
      prev = cur;
    }
  }

  recordGdbTranslation(sk, sk.func(), code.main(), loc.mainStart(),
                       false, false);
  recordGdbTranslation(sk, sk.func(), code.cold(), loc.coldStart(),
                       false, false);
  if (RuntimeOption::EvalJitPGO) {
    if (transKindToRecord == TransKind::Profile) {
      always_assert(region);
      m_tx.profData()->addTransProfile(region, pconds);
    } else {
      m_tx.profData()->addTransNonProf(transKindToRecord, sk);
    }
  }

  auto tr = maker.rec(sk, transKindToRecord, region, m_fixups.m_bcMap,
                      std::move(m_annotations), useLLVM(), hasLoop);
  m_tx.addTranslation(tr);
  if (RuntimeOption::EvalJitUseVtuneAPI) {
    reportTraceletToVtune(sk.unit(), sk.func(), tr);
  }

  if (RuntimeOption::EvalPerfRelocate) {
    recordPerfRelocMap(loc.mainStart(), loc.mainEnd(),
                       loc.coldCodeStart(), loc.coldEnd(),
                       sk, -1,
                       srcRec.tailFallbackJumps(),
                       m_fixups);
  }
  GrowableVector<IncomingBranch> inProgressTailBranches;
  m_fixups.process(&inProgressTailBranches);
  m_annotations.clear();

  // SrcRec::newTranslation() makes this code reachable. Do this last;
  // otherwise there's some chance of hitting in the reader threads whose
  // metadata is not yet visible.
  TRACE(1, "newTranslation: %p  sk: (func %d, bcOff %d)\n",
        loc.mainStart(), sk.funcID(), sk.offset());
  srcRec.newTranslation(loc, inProgressTailBranches);

  TRACE(1, "mcg: %zd-byte tracelet\n", (ssize_t)loc.mainSize());
  if (Trace::moduleEnabledRelease(Trace::tcspace, 1)) {
    Trace::traceRelease("%s", getUsageString().c_str());
  }

  // Report jit maturity based on the amount of code emitted.
  auto percent = code.mainUsed() * 100 / CodeCache::AMaxUsage;
  if (percent > 100) percent = 100;
  ServiceData::createCounter("jit.maturity")->setValue(percent);

  return loc.mainStart();
}

void MCGenerator::traceCodeGen(IRGS& irgs) {
  auto& unit = irgs.unit;

  auto finishPass = [&](const char* msg, int level) {
    printUnit(level, unit, msg, nullptr, irgs.irb->guards());
    assertx(checkCfg(unit));
  };

  finishPass(" after initial translation ", kIRLevel);

  always_assert_flog(
    IMPLIES(cfgHasLoop(unit), RuntimeOption::EvalJitLoops),
    "IRUnit has loop but Eval.JitLoops=0"
  );

  optimize(unit, *irgs.irb, m_tx.mode());
  finishPass(" after optimizing ", kOptLevel);

  always_assert(this == mcg);
  irlower::genCode(unit);

  m_numTrans++;
  assertx(m_numTrans <= RuntimeOption::EvalJitGlobalTranslationLimit);
}

MCGenerator::MCGenerator()
  : m_numTrans(0)
  , m_catchTraceMap(128)
  , m_useLLVM(false)
{
  TRACE(1, "MCGenerator@%p startup\n", this);
  mcg = this;

  m_unwindRegistrar = register_unwind_region(code.base(), code.codeSize());

  static bool profileUp = false;
  if (!profileUp) {
    profileInit();
    profileUp = true;
  }

  if (Trace::moduleEnabledRelease(Trace::printir) &&
      !RuntimeOption::EvalJit) {
    Trace::traceRelease("TRACE=printir is set but the jit isn't on. "
                        "Did you mean to run with -vEval.Jit=1?\n");
  }
  if (Trace::moduleEnabledRelease(Trace::llvm_count, 1) ||
      RuntimeOption::EvalJitLLVMCounters) {
    g_bytecodesVasm.bind();
    g_bytecodesLLVM.bind();
  }
}

void MCGenerator::initUniqueStubs() {
  // Put the following stubs into ahot, rather than a.
  CodeCache::Selector cbSel(CodeCache::Selector::Args(code).
                            hot(m_tx.useAHot()));
  m_tx.uniqueStubs.emitAll();
  m_fixups.process(nullptr); // in case we generated literals
}

void MCGenerator::registerCatchBlock(CTCA ip, TCA block) {
  FTRACE(1, "registerCatchBlock: afterCall: {} block: {}\n", ip, block);
  m_fixups.m_pendingCatchTraces.emplace_back(ip, block);
}

folly::Optional<TCA> MCGenerator::getCatchTrace(CTCA ip) const {
  TCA* found = m_catchTraceMap.find(ip);
  if (found && *found != kInvalidCatchTrace) return *found;
  return folly::none;
}

void MCGenerator::codeEmittedThisRequest(size_t& requestEntry,
                                         size_t& now) const {
  requestEntry = s_initialTCSize;
  now = code.totalUsed();
}

namespace {
__thread std::unordered_map<const ActRec*, TCA>* tl_debuggerCatches{nullptr};
}

void pushDebuggerCatch(const ActRec* fp) {
  if (!tl_debuggerCatches) {
    tl_debuggerCatches = new std::unordered_map<const ActRec*, TCA>();
  }

  auto optCatchBlock = mcg->getCatchTrace(TCA(fp->m_savedRip));
  always_assert(optCatchBlock && *optCatchBlock);
  auto catchBlock = *optCatchBlock;
  FTRACE(1, "Pushing debugger catch {} with fp {}\n", catchBlock, fp);
  tl_debuggerCatches->emplace(fp, catchBlock);
}

TCA popDebuggerCatch(const ActRec* fp) {
  always_assert(tl_debuggerCatches);
  auto const it = tl_debuggerCatches->find(fp);
  always_assert(it != tl_debuggerCatches->end());
  auto const catchBlock = it->second;
  tl_debuggerCatches->erase(it);
  FTRACE(1, "Popped debugger catch {} for fp {}\n", catchBlock, fp);
  return catchBlock;
}

void MCGenerator::requestInit() {
  tl_regState = VMRegState::CLEAN;
  Timer::RequestInit();
  memset(&s_perfCounters, 0, sizeof(s_perfCounters));
  Stats::init();
  s_initialTCSize = code.totalUsed();
}

void MCGenerator::requestExit() {
  always_assert(!Translator::WriteLease().amOwner());
  TRACE_MOD(txlease, 2, "%" PRIx64 " write lease stats: %15" PRId64
            " kept, %15" PRId64 " grabbed\n",
            Process::GetThreadIdForTrace(), Translator::WriteLease().m_hintKept,
            Translator::WriteLease().m_hintGrabbed);
  Stats::dump();
  Stats::clear();
  Timer::RequestExit();

  if (Trace::moduleEnabledRelease(Trace::mcgstats, 1)) {
    Trace::traceRelease("MCGenerator perf counters for %s:\n",
                        g_context->getRequestUrl(50).c_str());
    for (int i = 0; i < tpc_num_counters; i++) {
      Trace::traceRelease("%-20s %10" PRId64 "\n",
                          kPerfCounterNames[i], s_perfCounters[i]);
    }
    Trace::traceRelease("\n");
  }

  if (Trace::moduleEnabledRelease(Trace::llvm_count, 1)) {
    auto llvm = *g_bytecodesLLVM;
    auto total = llvm + *g_bytecodesVasm;
    Trace::ftraceRelease(
      "{:9} / {:9} bytecodes ({:6.2f}%) handled by LLVM backend for {}\n",
      llvm, total, llvm * 100.0 / total, g_context->getRequestUrl(50)
    );
  }

  delete tl_debuggerCatches;
  tl_debuggerCatches = nullptr;
}

void
MCGenerator::getPerfCounters(Array& ret) {
  for (int i = 0; i < tpc_num_counters; i++) {
    // Until Perflab can automatically scale the values we give it to
    // an appropriate range, we have to fudge these numbers so they
    // look more like reasonable hardware counter values.
    ret.set(String::FromCStr(kPerfCounterNames[i]),
            s_perfCounters[i] * 1000);
  }

  for (auto const& pair : Timer::Counters()) {
    if (pair.second.total == 0 && pair.second.count == 0) continue;

    ret.set(String("jit_time_") + pair.first, pair.second.total);
  }

  if (RuntimeOption::EvalJitLLVMCounters) {
    ret.set(String("jit_instr_vasm"), *g_bytecodesVasm);
    ret.set(String("jit_instr_llvm"), *g_bytecodesLLVM);
  }
}

MCGenerator::~MCGenerator() {
}

static Debug::TCRange rangeFrom(const CodeBlock& cb, const TCA addr,
                                bool isAcold) {
  assertx(cb.contains(addr));
  return Debug::TCRange(addr, cb.frontier(), isAcold);
}

void MCGenerator::recordBCInstr(uint32_t op,
                                const TCA addr,
                                const TCA end,
                                bool cold) {
  if (addr != end) {
    m_debugInfo.recordBCInstr(
      Debug::TCRange(addr, end, cold), op);
  }
}

void MCGenerator::recordGdbTranslation(SrcKey sk,
                                       const Func* srcFunc,
                                       const CodeBlock& cb,
                                       const TCA start,
                                       bool exit,
                                       bool inPrologue) {
  if (start != cb.frontier()) {
    assertx(Translator::WriteLease().amOwner());
    if (!RuntimeOption::EvalJitNoGdb) {
      m_debugInfo.recordTracelet(rangeFrom(cb, start, &cb == &code.cold()),
                                 srcFunc,
                                 srcFunc->unit() ?
                                   srcFunc->unit()->at(sk.offset()) : nullptr,
                                 exit, inPrologue);
    }
    if (RuntimeOption::EvalPerfPidMap) {
      m_debugInfo.recordPerfMap(rangeFrom(cb, start, &cb == &code.cold()),
                                srcFunc, exit, inPrologue);
    }
  }
}

void MCGenerator::recordGdbStub(const CodeBlock& cb,
                                const TCA start,
                                const std::string& name) {
  if (RuntimeOption::EvalJitNoGdb) return;
  m_debugInfo.recordStub(rangeFrom(cb, start, &cb == &code.cold()), name);
}

std::vector<UsageInfo> MCGenerator::getUsageInfo() {
  std::vector<UsageInfo> tcUsageInfo;
  code.forEachBlock([&](const char* name, const CodeBlock& a) {
    tcUsageInfo.emplace_back(UsageInfo{std::string("code.") + name,
                             a.used(),
                             a.capacity(),
                             true});
  });
  tcUsageInfo.emplace_back(UsageInfo{
      "data",
      code.data().used(),
      code.data().capacity(),
      true});
  tcUsageInfo.emplace_back(UsageInfo{
      "RDS",
      rds::usedBytes(),
      RuntimeOption::EvalJitTargetCacheSize * 3 / 4,
      false});
  tcUsageInfo.emplace_back(UsageInfo{
      "RDSLocal",
      rds::usedLocalBytes(),
      RuntimeOption::EvalJitTargetCacheSize * 3 / 4,
      false});
  tcUsageInfo.emplace_back(UsageInfo{
      "persistentRDS",
      rds::usedPersistentBytes(),
      RuntimeOption::EvalJitTargetCacheSize / 4,
      false});
  return tcUsageInfo;
}

std::string MCGenerator::getUsageString() {
  std::string usage;
  size_t totalBlockSize = 0;
  size_t totalBlockCapacity = 0;
  auto addRow = [&](UsageInfo blockUsageInfo) {
    auto percent = blockUsageInfo.m_capacity ?
      100 * blockUsageInfo.m_used / blockUsageInfo.m_capacity : 0;
    usage += folly::format("mcg: {:9} bytes ({}%) in {}\n",
                           blockUsageInfo.m_used,
                           percent,
                           blockUsageInfo.m_name).str();
    if (blockUsageInfo.m_global) {
      totalBlockSize += blockUsageInfo.m_used;
      totalBlockCapacity += blockUsageInfo.m_capacity;
    }
  };
  auto tcUsageInfo = getUsageInfo();
  for_each(tcUsageInfo.begin(), tcUsageInfo.end(), addRow);
  addRow(UsageInfo{"total", totalBlockSize, totalBlockCapacity, false});
  return usage;
}

std::string MCGenerator::getTCAddrs() {
  std::string addrs;
  code.forEachBlock([&](const char* name, const CodeBlock& a) {
      addrs += folly::format("{}: {}\n", name, a.base()).str();
  });
  return addrs;
}

bool MCGenerator::addDbgGuards(const Unit* unit) {
  // TODO refactor
  // It grabs the write lease and iterates through whole SrcDB...
  struct timespec tsBegin, tsEnd;
  {
    BlockingLeaseHolder writer(Translator::WriteLease());
    if (!writer) {
      return false;
    }
    assertx(mcg->cgFixups().empty());
    HPHP::Timer::GetMonotonicTime(tsBegin);
    // Doc says even find _could_ invalidate iterator, in pactice it should
    // be very rare, so go with it now.
    for (SrcDB::const_iterator it = m_tx.getSrcDB().begin();
         it != m_tx.getSrcDB().end(); ++it) {
      SrcKey const sk = SrcKey::fromAtomicInt(it->first);
      // We may have a SrcKey to a deleted function. NB: this may miss a
      // race with deleting a Func. See task #2826313.
      if (!Func::isFuncIdValid(sk.funcID())) continue;
      SrcRec* sr = it->second;
      if (sr->unitMd5() == unit->md5() &&
          !sr->hasDebuggerGuard() &&
          m_tx.isSrcKeyInBL(sk)) {
        addDbgGuardImpl(sk, sr);
      }
    }
    mcg->cgFixups().process(nullptr);
  }
  HPHP::Timer::GetMonotonicTime(tsEnd);
  int64_t elapsed = gettime_diff_us(tsBegin, tsEnd);
  if (Trace::moduleEnabledRelease(Trace::mcg, 5)) {
    Trace::traceRelease("addDbgGuards got lease for %" PRId64 " us\n", elapsed);
  }
  return true;
}

bool MCGenerator::addDbgGuard(const Func* func, Offset offset, bool resumed) {
  SrcKey sk(func, offset, resumed);
  {
    if (SrcRec* sr = m_tx.getSrcDB().find(sk)) {
      if (sr->hasDebuggerGuard()) {
        return true;
      }
    } else {
      // no translation yet
      return true;
    }
  }
  if (debug) {
    if (!m_tx.isSrcKeyInBL(sk)) {
      TRACE(5, "calling addDbgGuard on PC that is not in blacklist");
      return false;
    }
  }
  BlockingLeaseHolder writer(Translator::WriteLease());
  if (!writer) {
    return false;
  }
  assertx(mcg->cgFixups().empty());
  {
    if (SrcRec* sr = m_tx.getSrcDB().find(sk)) {
      addDbgGuardImpl(sk, sr);
    }
  }
  mcg->cgFixups().process(nullptr);
  return true;
}

bool MCGenerator::dumpTCCode(const char* filename) {
#define OPEN_FILE(F, SUFFIX)                                    \
  std::string F ## name = std::string(filename).append(SUFFIX); \
  FILE* F = fopen(F ## name .c_str(),"wb");                     \
  if (F == nullptr) return false;                               \
  SCOPE_EXIT{ fclose(F); };

  OPEN_FILE(ahotFile,       "_ahot");
  OPEN_FILE(aFile,          "_a");
  OPEN_FILE(aprofFile,      "_aprof");
  OPEN_FILE(acoldFile,      "_acold");
  OPEN_FILE(afrozenFile,    "_afrozen");
  OPEN_FILE(helperAddrFile, "_helpers_addrs.txt");

#undef OPEN_FILE

  // dump starting from the hot region
  size_t count = code.hot().used();
  bool result = (fwrite(code.hot().base(), 1, count, ahotFile) == count);
  if (result) {
    count = code.realMain().used();
    result = (fwrite(code.realMain().base(), 1, count, aFile) == count);
  }
  if (result) {
    count = code.prof().used();
    result = (fwrite(code.prof().base(), 1, count, aprofFile) == count);
  }
  if (result) {
    count = code.realCold().used();
    result = (fwrite(code.realCold().base(), 1, count, acoldFile) == count);
  }
  if (result) {
    count = code.frozen().used();
    result = (fwrite(code.frozen().base(), 1, count, afrozenFile) == count);
  }
  return result;
}

// Returns true on success
bool MCGenerator::dumpTC(bool ignoreLease /* =false */) {
  folly::Optional<BlockingLeaseHolder> writer;
  if (!ignoreLease) {
    writer.emplace(Translator::WriteLease());
    if (!*writer) return false;
  }
  bool success = dumpTCData();
  if (success) {
    success = dumpTCCode("/tmp/tc_dump");
  }
  return success;
}

// Returns true on success
bool tc_dump(bool ignoreLease /* =false */) {
  return mcg && mcg->dumpTC(ignoreLease);
}

// Returns true on success
bool MCGenerator::dumpTCData() {
  gzFile tcDataFile = gzopen("/tmp/tc_data.txt.gz", "w");
  if (!tcDataFile) return false;

  if (!gzprintf(tcDataFile,
                "repo_schema      = %s\n"
                "ahot.base        = %p\n"
                "ahot.frontier    = %p\n"
                "a.base           = %p\n"
                "a.frontier       = %p\n"
                "aprof.base       = %p\n"
                "aprof.frontier   = %p\n"
                "acold.base       = %p\n"
                "acold.frontier   = %p\n"
                "afrozen.base     = %p\n"
                "afrozen.frontier = %p\n\n",
                kRepoSchemaId,
                code.hot().base(), code.hot().frontier(),
                code.realMain().base(), code.realMain().frontier(),
                code.prof().base(), code.prof().frontier(),
                code.realCold().base(), code.realCold().frontier(),
                code.frozen().base(), code.frozen().frontier())) {
    return false;
  }

  if (!gzprintf(tcDataFile, "total_translations = %zu\n\n",
                m_tx.getCurrentTransID())) {
    return false;
  }

  for (TransID t = 0; t < m_tx.getCurrentTransID(); t++) {
    if (gzputs(tcDataFile,
               m_tx.getTransRec(t)->print(m_tx.getTransCounter(t)).c_str()) ==
        -1) {
      return false;
    }
  }

  gzclose(tcDataFile);
  return true;
}

void MCGenerator::invalidateSrcKey(SrcKey sk) {
  assertx(!RuntimeOption::RepoAuthoritative || RuntimeOption::EvalJitPGO);
  assertx(Translator::WriteLease().amOwner());
  /*
   * Reroute existing translations for SrcKey to an as-yet indeterminate
   * new one.
   */
  SrcRec* sr = m_tx.getSrcDB().find(sk);
  assertx(sr);
  /*
   * Since previous translations aren't reachable from here, we know we
   * just created some garbage in the TC. We currently have no mechanism
   * to reclaim this.
   */
  FTRACE_MOD(reusetc, 1, "Replacing translations from func {} (id = {}) "
             "@ sk({}) in SrcRec addr={}\n",
             sk.func()->fullName()->data(), sk.func()->getFuncId(), sk.offset(),
             (void*)sr);
  Trace::Indent _i;
  sr->replaceOldTranslations();
}

void MCGenerator::setJmpTransID(TCA jmp) {
  if (m_tx.mode() != TransKind::Profile) return;

  TransID transId = m_tx.profData()->curTransID();
  FTRACE(5, "setJmpTransID: adding {} => {}\n", jmp, transId);
  m_fixups.m_pendingJmpTransIDs.emplace_back(jmp, transId);
}

void emitIncStat(Vout& v, Stats::StatCounter stat, int n, bool force) {
  if (!force && !Stats::enabled()) return;
  intptr_t disp = uintptr_t(&Stats::tl_counters[stat]) - tlsBase();
  v << addqim{n, Vptr{baseless(disp), Vptr::FS}, v.makeReg()};
}

}}
