You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
254 lines
9.6 KiB
254 lines
9.6 KiB
//===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This pass applies cache prefetch instructions based on a profile. The pass
|
|
// assumes DiscriminateMemOps ran immediately before, to ensure debug info
|
|
// matches the one used at profile generation time. The profile is encoded in
|
|
// afdo format (text or binary). It contains prefetch hints recommendations.
|
|
// Each recommendation is made in terms of debug info locations, a type (i.e.
|
|
// nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a
|
|
// memory operand (see X86DiscriminateMemOps). The prefetch will be made for
|
|
// a location at that memory operand + the delta specified in the
|
|
// recommendation.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "X86.h"
|
|
#include "X86InstrBuilder.h"
|
|
#include "X86InstrInfo.h"
|
|
#include "X86MachineFunctionInfo.h"
|
|
#include "X86Subtarget.h"
|
|
#include "llvm/CodeGen/MachineModuleInfo.h"
|
|
#include "llvm/IR/DebugInfoMetadata.h"
|
|
#include "llvm/ProfileData/SampleProf.h"
|
|
#include "llvm/ProfileData/SampleProfReader.h"
|
|
#include "llvm/Transforms/IPO/SampleProfile.h"
|
|
using namespace llvm;
|
|
using namespace sampleprof;
|
|
|
|
static cl::opt<std::string>
|
|
PrefetchHintsFile("prefetch-hints-file",
|
|
cl::desc("Path to the prefetch hints profile. See also "
|
|
"-x86-discriminate-memops"),
|
|
cl::Hidden);
|
|
namespace {
|
|
|
|
class X86InsertPrefetch : public MachineFunctionPass {
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override;
|
|
bool doInitialization(Module &) override;
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
|
struct PrefetchInfo {
|
|
unsigned InstructionID;
|
|
int64_t Delta;
|
|
};
|
|
typedef SmallVectorImpl<PrefetchInfo> Prefetches;
|
|
bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI,
|
|
Prefetches &prefetches) const;
|
|
|
|
public:
|
|
static char ID;
|
|
X86InsertPrefetch(const std::string &PrefetchHintsFilename);
|
|
StringRef getPassName() const override {
|
|
return "X86 Insert Cache Prefetches";
|
|
}
|
|
|
|
private:
|
|
std::string Filename;
|
|
std::unique_ptr<SampleProfileReader> Reader;
|
|
};
|
|
|
|
using PrefetchHints = SampleRecord::CallTargetMap;
|
|
|
|
// Return any prefetching hints for the specified MachineInstruction. The hints
|
|
// are returned as pairs (name, delta).
|
|
ErrorOr<PrefetchHints> getPrefetchHints(const FunctionSamples *TopSamples,
|
|
const MachineInstr &MI) {
|
|
if (const auto &Loc = MI.getDebugLoc())
|
|
if (const auto *Samples = TopSamples->findFunctionSamples(Loc))
|
|
return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc),
|
|
Loc->getBaseDiscriminator());
|
|
return std::error_code();
|
|
}
|
|
|
|
// The prefetch instruction can't take memory operands involving vector
|
|
// registers.
|
|
bool IsMemOpCompatibleWithPrefetch(const MachineInstr &MI, int Op) {
|
|
Register BaseReg = MI.getOperand(Op + X86::AddrBaseReg).getReg();
|
|
Register IndexReg = MI.getOperand(Op + X86::AddrIndexReg).getReg();
|
|
return (BaseReg == 0 ||
|
|
X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) ||
|
|
X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg)) &&
|
|
(IndexReg == 0 ||
|
|
X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg) ||
|
|
X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg));
|
|
}
|
|
|
|
} // end anonymous namespace
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
char X86InsertPrefetch::ID = 0;
|
|
|
|
X86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename)
|
|
: MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {}
|
|
|
|
/// Return true if the provided MachineInstruction has cache prefetch hints. In
|
|
/// that case, the prefetch hints are stored, in order, in the Prefetches
|
|
/// vector.
|
|
bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
|
|
const MachineInstr &MI,
|
|
Prefetches &Prefetches) const {
|
|
assert(Prefetches.empty() &&
|
|
"Expected caller passed empty PrefetchInfo vector.");
|
|
static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
|
|
{"_nta_", X86::PREFETCHNTA},
|
|
{"_t0_", X86::PREFETCHT0},
|
|
{"_t1_", X86::PREFETCHT1},
|
|
{"_t2_", X86::PREFETCHT2},
|
|
};
|
|
static const char *SerializedPrefetchPrefix = "__prefetch";
|
|
|
|
const ErrorOr<PrefetchHints> T = getPrefetchHints(TopSamples, MI);
|
|
if (!T)
|
|
return false;
|
|
int16_t max_index = -1;
|
|
// Convert serialized prefetch hints into PrefetchInfo objects, and populate
|
|
// the Prefetches vector.
|
|
for (const auto &S_V : *T) {
|
|
StringRef Name = S_V.getKey();
|
|
if (Name.consume_front(SerializedPrefetchPrefix)) {
|
|
int64_t D = static_cast<int64_t>(S_V.second);
|
|
unsigned IID = 0;
|
|
for (const auto &HintType : HintTypes) {
|
|
if (Name.startswith(HintType.first)) {
|
|
Name = Name.drop_front(HintType.first.size());
|
|
IID = HintType.second;
|
|
break;
|
|
}
|
|
}
|
|
if (IID == 0)
|
|
return false;
|
|
uint8_t index = 0;
|
|
Name.consumeInteger(10, index);
|
|
|
|
if (index >= Prefetches.size())
|
|
Prefetches.resize(index + 1);
|
|
Prefetches[index] = {IID, D};
|
|
max_index = std::max(max_index, static_cast<int16_t>(index));
|
|
}
|
|
}
|
|
assert(max_index + 1 >= 0 &&
|
|
"Possible overflow: max_index + 1 should be positive.");
|
|
assert(static_cast<size_t>(max_index + 1) == Prefetches.size() &&
|
|
"The number of prefetch hints received should match the number of "
|
|
"PrefetchInfo objects returned");
|
|
return !Prefetches.empty();
|
|
}
|
|
|
|
bool X86InsertPrefetch::doInitialization(Module &M) {
|
|
if (Filename.empty())
|
|
return false;
|
|
|
|
LLVMContext &Ctx = M.getContext();
|
|
ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr =
|
|
SampleProfileReader::create(Filename, Ctx);
|
|
if (std::error_code EC = ReaderOrErr.getError()) {
|
|
std::string Msg = "Could not open profile: " + EC.message();
|
|
Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg,
|
|
DiagnosticSeverity::DS_Warning));
|
|
return false;
|
|
}
|
|
Reader = std::move(ReaderOrErr.get());
|
|
Reader->read();
|
|
return true;
|
|
}
|
|
|
|
void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
|
|
AU.setPreservesAll();
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
}
|
|
|
|
bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
|
|
if (!Reader)
|
|
return false;
|
|
const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction());
|
|
if (!Samples)
|
|
return false;
|
|
|
|
bool Changed = false;
|
|
|
|
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
|
|
SmallVector<PrefetchInfo, 4> Prefetches;
|
|
for (auto &MBB : MF) {
|
|
for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) {
|
|
auto Current = MI;
|
|
++MI;
|
|
|
|
int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags);
|
|
if (Offset < 0)
|
|
continue;
|
|
unsigned Bias = X86II::getOperandBias(Current->getDesc());
|
|
int MemOpOffset = Offset + Bias;
|
|
// FIXME(mtrofin): ORE message when the recommendation cannot be taken.
|
|
if (!IsMemOpCompatibleWithPrefetch(*Current, MemOpOffset))
|
|
continue;
|
|
Prefetches.clear();
|
|
if (!findPrefetchInfo(Samples, *Current, Prefetches))
|
|
continue;
|
|
assert(!Prefetches.empty() &&
|
|
"The Prefetches vector should contain at least a value if "
|
|
"findPrefetchInfo returned true.");
|
|
for (auto &PrefInfo : Prefetches) {
|
|
unsigned PFetchInstrID = PrefInfo.InstructionID;
|
|
int64_t Delta = PrefInfo.Delta;
|
|
const MCInstrDesc &Desc = TII->get(PFetchInstrID);
|
|
MachineInstr *PFetch =
|
|
MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true);
|
|
MachineInstrBuilder MIB(MF, PFetch);
|
|
|
|
assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 &&
|
|
X86::AddrIndexReg == 2 && X86::AddrDisp == 3 &&
|
|
X86::AddrSegmentReg == 4 &&
|
|
"Unexpected change in X86 operand offset order.");
|
|
|
|
// This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc.
|
|
// FIXME(mtrofin): consider adding a:
|
|
// MachineInstrBuilder::set(unsigned offset, op).
|
|
MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg())
|
|
.addImm(
|
|
Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm())
|
|
.addReg(
|
|
Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg())
|
|
.addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() +
|
|
Delta)
|
|
.addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg)
|
|
.getReg());
|
|
|
|
if (!Current->memoperands_empty()) {
|
|
MachineMemOperand *CurrentOp = *(Current->memoperands_begin());
|
|
MIB.addMemOperand(MF.getMachineMemOperand(
|
|
CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize()));
|
|
}
|
|
|
|
// Insert before Current. This is because Current may clobber some of
|
|
// the registers used to describe the input memory operand.
|
|
MBB.insert(Current, PFetch);
|
|
Changed = true;
|
|
}
|
|
}
|
|
}
|
|
return Changed;
|
|
}
|
|
|
|
FunctionPass *llvm::createX86InsertPrefetchPass() {
|
|
return new X86InsertPrefetch(PrefetchHintsFile);
|
|
}
|