summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Target/BPF/BPF.h5
-rw-r--r--llvm/lib/Target/BPF/BPFAbstrctMemberAccess.cpp482
-rw-r--r--llvm/lib/Target/BPF/BPFAsmPrinter.cpp20
-rw-r--r--llvm/lib/Target/BPF/BPFCORE.h24
-rw-r--r--llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp163
-rw-r--r--llvm/lib/Target/BPF/BPFTargetMachine.cpp12
-rw-r--r--llvm/lib/Target/BPF/BTF.h63
-rw-r--r--llvm/lib/Target/BPF/BTFDebug.cpp489
-rw-r--r--llvm/lib/Target/BPF/BTFDebug.h63
-rw-r--r--llvm/lib/Target/BPF/CMakeLists.txt2
10 files changed, 1268 insertions, 55 deletions
diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h
index 6c75fff9c7a..d311fc15409 100644
--- a/llvm/lib/Target/BPF/BPF.h
+++ b/llvm/lib/Target/BPF/BPF.h
@@ -15,11 +15,16 @@
namespace llvm {
class BPFTargetMachine;
+ModulePass *createBPFAbstractMemberAccess();
+
FunctionPass *createBPFISelDag(BPFTargetMachine &TM);
+FunctionPass *createBPFMISimplifyPatchablePass();
FunctionPass *createBPFMIPeepholePass();
FunctionPass *createBPFMIPreEmitPeepholePass();
FunctionPass *createBPFMIPreEmitCheckingPass();
+void initializeBPFAbstractMemberAccessPass(PassRegistry&);
+void initializeBPFMISimplifyPatchablePass(PassRegistry&);
void initializeBPFMIPeepholePass(PassRegistry&);
void initializeBPFMIPreEmitPeepholePass(PassRegistry&);
void initializeBPFMIPreEmitCheckingPass(PassRegistry&);
diff --git a/llvm/lib/Target/BPF/BPFAbstrctMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstrctMemberAccess.cpp
new file mode 100644
index 00000000000..51d4cbc8a42
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFAbstrctMemberAccess.cpp
@@ -0,0 +1,482 @@
+//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass abstracted struct/union member accesses in order to support
+// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program
+// which can run on different kernels. In particular, if bpf program tries to
+// access a particular kernel data structure member, the details of the
+// intermediate member access will be remembered so bpf loader can do
+// necessary adjustment right before program loading.
+//
+// For example,
+//
+// struct s {
+// int a;
+// int b;
+// };
+// struct t {
+// struct s c;
+// int d;
+// };
+// struct t e;
+//
+// For the member access e.c.b, the compiler will generate code
+// &e + 4
+//
+// The compile-once run-everywhere instead generates the following code
+// r = 4
+// &e + r
+// The "4" in "r = 4" can be changed based on a particular kernel version.
+// For example, on a particular kernel version, if struct s is changed to
+//
+// struct s {
+// int new_field;
+// int a;
+// int b;
+// }
+//
+// By repeating the member access on the host, the bpf loader can
+// adjust "r = 4" as "r = 8".
+//
+// This feature relies on the following three intrinsic calls:
+// addr = preserve_array_access_index(base, dimension, index)
+// addr = preserve_union_access_index(base, di_index)
+// !llvm.preserve.access.index <union_ditype>
+// addr = preserve_struct_access_index(base, gep_index, di_index)
+// !llvm.preserve.access.index <struct_ditype>
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPF.h"
+#include "BPFCORE.h"
+#include "BPFTargetMachine.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/User.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+#define DEBUG_TYPE "bpf-abstract-member-access"
+
+namespace llvm {
+const std::string BPFCoreSharedInfo::AmaAttr = "btf_ama";
+const std::string BPFCoreSharedInfo::PatchableExtSecName =
+ ".BPF.patchable_externs";
+} // namespace llvm
+
+using namespace llvm;
+
+namespace {
+
+class BPFAbstractMemberAccess final : public ModulePass {
+ StringRef getPassName() const override {
+ return "BPF Abstract Member Access";
+ }
+
+ bool runOnModule(Module &M) override;
+
+public:
+ static char ID;
+ BPFAbstractMemberAccess() : ModulePass(ID) {}
+
+private:
+ enum : uint32_t {
+ BPFPreserveArrayAI = 1,
+ BPFPreserveUnionAI = 2,
+ BPFPreserveStructAI = 3,
+ };
+
+ std::map<std::string, GlobalVariable *> GEPGlobals;
+ // A map to link preserve_*_access_index instrinsic calls.
+ std::map<CallInst *, std::pair<CallInst *, uint32_t>> AIChain;
+ // A map to hold all the base preserve_*_access_index instrinsic calls.
+ // The base call is not an input of any other preserve_*_access_index
+ // intrinsics.
+ std::map<CallInst *, uint32_t> BaseAICalls;
+
+ bool doTransformation(Module &M);
+
+ void traceAICall(CallInst *Call, uint32_t Kind);
+ void traceBitCast(BitCastInst *BitCast, CallInst *Parent, uint32_t Kind);
+ void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, uint32_t Kind);
+ void collectAICallChains(Module &M, Function &F);
+
+ bool IsPreserveDIAccessIndexCall(const CallInst *Call, uint32_t &Kind);
+ bool removePreserveAccessIndexIntrinsic(Module &M);
+ void replaceWithGEP(std::vector<CallInst *> &CallList,
+ uint32_t NumOfZerosIndex, uint32_t DIIndex);
+
+ Value *computeBaseAndAccessStr(CallInst *Call, std::string &AccessStr,
+ std::string &AccessKey, uint32_t Kind,
+ MDNode *&TypeMeta);
+ bool getAccessIndex(const Value *IndexValue, uint64_t &AccessIndex);
+ bool transformGEPChain(Module &M, CallInst *Call, uint32_t Kind);
+};
+} // End anonymous namespace
+
+char BPFAbstractMemberAccess::ID = 0;
+INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE,
+ "abstracting struct/union member accessees", false, false)
+
+ModulePass *llvm::createBPFAbstractMemberAccess() {
+ return new BPFAbstractMemberAccess();
+}
+
+bool BPFAbstractMemberAccess::runOnModule(Module &M) {
+ LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");
+
+ // Bail out if no debug info.
+ if (empty(M.debug_compile_units()))
+ return false;
+
+ return doTransformation(M);
+}
+
+/// Check whether a call is a preserve_*_access_index intrinsic call or not.
+bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
+ uint32_t &Kind) {
+ if (!Call)
+ return false;
+
+ const auto *GV = dyn_cast<GlobalValue>(Call->getCalledValue());
+ if (!GV)
+ return false;
+ if (GV->getName().startswith("llvm.preserve.array.access.index")) {
+ Kind = BPFPreserveArrayAI;
+ return true;
+ }
+ if (GV->getName().startswith("llvm.preserve.union.access.index")) {
+ Kind = BPFPreserveUnionAI;
+ return true;
+ }
+ if (GV->getName().startswith("llvm.preserve.struct.access.index")) {
+ Kind = BPFPreserveStructAI;
+ return true;
+ }
+
+ return false;
+}
+
+void BPFAbstractMemberAccess::replaceWithGEP(std::vector<CallInst *> &CallList,
+ uint32_t DimensionIndex,
+ uint32_t GEPIndex) {
+ for (auto Call : CallList) {
+ uint32_t Dimension = 1;
+ if (DimensionIndex > 0)
+ Dimension = cast<ConstantInt>(Call->getArgOperand(DimensionIndex))
+ ->getZExtValue();
+
+ Constant *Zero =
+ ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0);
+ SmallVector<Value *, 4> IdxList;
+ for (unsigned I = 0; I < Dimension; ++I)
+ IdxList.push_back(Zero);
+ IdxList.push_back(Call->getArgOperand(GEPIndex));
+
+ auto *GEP = GetElementPtrInst::CreateInBounds(Call->getArgOperand(0),
+ IdxList, "", Call);
+ Call->replaceAllUsesWith(GEP);
+ Call->eraseFromParent();
+ }
+}
+
+bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Module &M) {
+ std::vector<CallInst *> PreserveArrayIndexCalls;
+ std::vector<CallInst *> PreserveUnionIndexCalls;
+ std::vector<CallInst *> PreserveStructIndexCalls;
+ bool Found = false;
+
+ for (Function &F : M)
+ for (auto &BB : F)
+ for (auto &I : BB) {
+ auto *Call = dyn_cast<CallInst>(&I);
+ uint32_t Kind;
+ if (!IsPreserveDIAccessIndexCall(Call, Kind))
+ continue;
+
+ Found = true;
+ if (Kind == BPFPreserveArrayAI)
+ PreserveArrayIndexCalls.push_back(Call);
+ else if (Kind == BPFPreserveUnionAI)
+ PreserveUnionIndexCalls.push_back(Call);
+ else
+ PreserveStructIndexCalls.push_back(Call);
+ }
+
+ // do the following transformation:
+ // . addr = preserve_array_access_index(base, dimension, index)
+ // is transformed to
+ // addr = GEP(base, dimenion's zero's, index)
+ // . addr = preserve_union_access_index(base, di_index)
+ // is transformed to
+ // addr = base, i.e., all usages of "addr" are replaced by "base".
+ // . addr = preserve_struct_access_index(base, gep_index, di_index)
+ // is transformed to
+ // addr = GEP(base, 0, gep_index)
+ replaceWithGEP(PreserveArrayIndexCalls, 1, 2);
+ replaceWithGEP(PreserveStructIndexCalls, 0, 1);
+ for (auto Call : PreserveUnionIndexCalls) {
+ Call->replaceAllUsesWith(Call->getArgOperand(0));
+ Call->eraseFromParent();
+ }
+
+ return Found;
+}
+
+void BPFAbstractMemberAccess::traceAICall(CallInst *Call, uint32_t Kind) {
+ for (User *U : Call->users()) {
+ Instruction *Inst = dyn_cast<Instruction>(U);
+ if (!Inst)
+ continue;
+
+ if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
+ traceBitCast(BI, Call, Kind);
+ } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
+ uint32_t CIKind;
+ if (IsPreserveDIAccessIndexCall(CI, CIKind)) {
+ AIChain[CI] = std::make_pair(Call, Kind);
+ traceAICall(CI, CIKind);
+ } else {
+ BaseAICalls[Call] = Kind;
+ }
+ } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
+ if (GI->hasAllZeroIndices())
+ traceGEP(GI, Call, Kind);
+ else
+ BaseAICalls[Call] = Kind;
+ }
+ }
+}
+
+void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
+ CallInst *Parent, uint32_t Kind) {
+ for (User *U : BitCast->users()) {
+ Instruction *Inst = dyn_cast<Instruction>(U);
+ if (!Inst)
+ continue;
+
+ if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
+ traceBitCast(BI, Parent, Kind);
+ } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
+ uint32_t CIKind;
+ if (IsPreserveDIAccessIndexCall(CI, CIKind)) {
+ AIChain[CI] = std::make_pair(Parent, Kind);
+ traceAICall(CI, CIKind);
+ } else {
+ BaseAICalls[Parent] = Kind;
+ }
+ } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
+ if (GI->hasAllZeroIndices())
+ traceGEP(GI, Parent, Kind);
+ else
+ BaseAICalls[Parent] = Kind;
+ }
+ }
+}
+
+void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
+ uint32_t Kind) {
+ for (User *U : GEP->users()) {
+ Instruction *Inst = dyn_cast<Instruction>(U);
+ if (!Inst)
+ continue;
+
+ if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
+ traceBitCast(BI, Parent, Kind);
+ } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
+ uint32_t CIKind;
+ if (IsPreserveDIAccessIndexCall(CI, CIKind)) {
+ AIChain[CI] = std::make_pair(Parent, Kind);
+ traceAICall(CI, CIKind);
+ } else {
+ BaseAICalls[Parent] = Kind;
+ }
+ } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
+ if (GI->hasAllZeroIndices())
+ traceGEP(GI, Parent, Kind);
+ else
+ BaseAICalls[Parent] = Kind;
+ }
+ }
+}
+
+void BPFAbstractMemberAccess::collectAICallChains(Module &M, Function &F) {
+ AIChain.clear();
+ BaseAICalls.clear();
+
+ for (auto &BB : F)
+ for (auto &I : BB) {
+ uint32_t Kind;
+ auto *Call = dyn_cast<CallInst>(&I);
+ if (!IsPreserveDIAccessIndexCall(Call, Kind) ||
+ AIChain.find(Call) != AIChain.end())
+ continue;
+
+ traceAICall(Call, Kind);
+ }
+}
+
+/// Get access index from the preserve_*_access_index intrinsic calls.
+bool BPFAbstractMemberAccess::getAccessIndex(const Value *IndexValue,
+ uint64_t &AccessIndex) {
+ const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue);
+ if (!CV)
+ return false;
+
+ AccessIndex = CV->getValue().getZExtValue();
+ return true;
+}
+
+/// Compute the base of the whole preserve_*_access_index chains, i.e., the base
+/// pointer of the first preserve_*_access_index call, and construct the access
+/// string, which will be the name of a global variable.
+Value *BPFAbstractMemberAccess::computeBaseAndAccessStr(CallInst *Call,
+ std::string &AccessStr,
+ std::string &AccessKey,
+ uint32_t Kind,
+ MDNode *&TypeMeta) {
+ Value *Base = nullptr;
+ std::vector<uint64_t> AccessIndices;
+ uint64_t TypeNameIndex = 0;
+ std::string LastTypeName;
+
+ while (Call) {
+ // Base of original corresponding GEP
+ Base = Call->getArgOperand(0);
+
+ // Type Name
+ std::string TypeName;
+ MDNode *MDN;
+ if (Kind == BPFPreserveUnionAI || Kind == BPFPreserveStructAI) {
+ MDN = Call->getMetadata(LLVMContext::MD_preserve_access_index);
+ if (!MDN)
+ return nullptr;
+
+ DIType *Ty = dyn_cast<DIType>(MDN);
+ if (!Ty)
+ return nullptr;
+
+ TypeName = Ty->getName();
+ }
+
+ // Access Index
+ uint64_t AccessIndex;
+ uint32_t ArgIndex = (Kind == BPFPreserveUnionAI) ? 1 : 2;
+ if (!getAccessIndex(Call->getArgOperand(ArgIndex), AccessIndex))
+ return nullptr;
+
+ AccessIndices.push_back(AccessIndex);
+ if (TypeName.size()) {
+ TypeNameIndex = AccessIndices.size() - 1;
+ LastTypeName = TypeName;
+ TypeMeta = MDN;
+ }
+
+ Kind = AIChain[Call].second;
+ Call = AIChain[Call].first;
+ }
+
+ // The intial type name is required.
+ // FIXME: if the initial type access is an array index, e.g.,
+ // &a[3].b.c, only one dimentional array is supported.
+ if (!LastTypeName.size() || AccessIndices.size() > TypeNameIndex + 2)
+ return nullptr;
+
+ // Construct the type string AccessStr.
+ for (unsigned I = 0; I < AccessIndices.size(); ++I)
+ AccessStr = std::to_string(AccessIndices[I]) + ":" + AccessStr;
+
+ if (TypeNameIndex == AccessIndices.size() - 1)
+ AccessStr = "0:" + AccessStr;
+
+ // Access key is the type name + access string, uniquely identifying
+ // one kernel memory access.
+ AccessKey = LastTypeName + ":" + AccessStr;
+
+ return Base;
+}
+
+/// Call/Kind is the base preserve_*_access_index() call. Attempts to do
+/// transformation to a chain of relocable GEPs.
+bool BPFAbstractMemberAccess::transformGEPChain(Module &M, CallInst *Call,
+ uint32_t Kind) {
+ std::string AccessStr, AccessKey;
+ MDNode *TypeMeta = nullptr;
+ Value *Base =
+ computeBaseAndAccessStr(Call, AccessStr, AccessKey, Kind, TypeMeta);
+ if (!Base)
+ return false;
+
+ // Do the transformation
+ // For any original GEP Call and Base %2 like
+ // %4 = bitcast %struct.net_device** %dev1 to i64*
+ // it is transformed to:
+ // %6 = load __BTF_0:sk_buff:0:0:2:0:
+ // %7 = bitcast %struct.sk_buff* %2 to i8*
+ // %8 = getelementptr i8, i8* %7, %6
+ // %9 = bitcast i8* %8 to i64*
+ // using %9 instead of %4
+ // The original Call inst is removed.
+ BasicBlock *BB = Call->getParent();
+ GlobalVariable *GV;
+
+ if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) {
+ GV = new GlobalVariable(M, Type::getInt64Ty(BB->getContext()), false,
+ GlobalVariable::ExternalLinkage, NULL, AccessStr);
+ GV->addAttribute(BPFCoreSharedInfo::AmaAttr);
+ // Set the metadata (debuginfo types) for the global.
+ if (TypeMeta)
+ GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta);
+ GEPGlobals[AccessKey] = GV;
+ } else {
+ GV = GEPGlobals[AccessKey];
+ }
+
+ // Load the global variable.
+ auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV);
+ BB->getInstList().insert(Call->getIterator(), LDInst);
+
+ // Generate a BitCast
+ auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext()));
+ BB->getInstList().insert(Call->getIterator(), BCInst);
+
+ // Generate a GetElementPtr
+ auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()),
+ BCInst, LDInst);
+ BB->getInstList().insert(Call->getIterator(), GEP);
+
+ // Generate a BitCast
+ auto *BCInst2 = new BitCastInst(GEP, Call->getType());
+ BB->getInstList().insert(Call->getIterator(), BCInst2);
+
+ Call->replaceAllUsesWith(BCInst2);
+ Call->eraseFromParent();
+
+ return true;
+}
+
+bool BPFAbstractMemberAccess::doTransformation(Module &M) {
+ bool Transformed = false;
+
+ for (Function &F : M) {
+ // Collect PreserveDIAccessIndex Intrinsic call chains.
+ // The call chains will be used to generate the access
+ // patterns similar to GEP.
+ collectAICallChains(M, F);
+
+ for (auto &C : BaseAICalls)
+ Transformed = transformGEPChain(M, C.first, C.second) || Transformed;
+ }
+
+ return removePreserveAccessIndexIntrinsic(M) || Transformed;
+}
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
index 129a8c0704d..e61e7346805 100644
--- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -38,7 +38,7 @@ class BPFAsmPrinter : public AsmPrinter {
public:
explicit BPFAsmPrinter(TargetMachine &TM,
std::unique_ptr<MCStreamer> Streamer)
- : AsmPrinter(TM, std::move(Streamer)) {}
+ : AsmPrinter(TM, std::move(Streamer)), BTF(nullptr) {}
StringRef getPassName() const override { return "BPF Assembly Printer"; }
bool doInitialization(Module &M) override;
@@ -49,6 +49,9 @@ public:
const char *ExtraCode, raw_ostream &O) override;
void EmitInstruction(const MachineInstr *MI) override;
+
+private:
+ BTFDebug *BTF;
};
} // namespace
@@ -57,8 +60,10 @@ bool BPFAsmPrinter::doInitialization(Module &M) {
// Only emit BTF when debuginfo available.
if (MAI->doesSupportDebugInformation() && !empty(M.debug_compile_units())) {
- Handlers.emplace_back(llvm::make_unique<BTFDebug>(this), "emit",
- "Debug Info Emission", "BTF", "BTF Emission");
+ BTF = new BTFDebug(this);
+ Handlers.push_back(HandlerInfo(std::unique_ptr<BTFDebug>(BTF), "emit",
+ "Debug Info Emission", "BTF",
+ "BTF Emission"));
}
return false;
@@ -133,11 +138,12 @@ bool BPFAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
}
void BPFAsmPrinter::EmitInstruction(const MachineInstr *MI) {
-
- BPFMCInstLower MCInstLowering(OutContext, *this);
-
MCInst TmpInst;
- MCInstLowering.Lower(MI, TmpInst);
+
+ if (!BTF || !BTF->InstLower(MI, TmpInst)) {
+ BPFMCInstLower MCInstLowering(OutContext, *this);
+ MCInstLowering.Lower(MI, TmpInst);
+ }
EmitToStreamer(*OutStreamer, TmpInst);
}
diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h
new file mode 100644
index 00000000000..e0950d95f8d
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFCORE.h
@@ -0,0 +1,24 @@
+//===- BPFCORE.h - Common info for Compile-Once Run-EveryWhere -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BPFCORE_H
+#define LLVM_LIB_TARGET_BPF_BPFCORE_H
+
+namespace llvm {
+
+class BPFCoreSharedInfo {
+public:
+ /// The attribute attached to globals representing a member offset
+ static const std::string AmaAttr;
+ /// The section name to identify a patchable external global
+ static const std::string PatchableExtSecName;
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp b/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp
new file mode 100644
index 00000000000..e9114d7187e
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp
@@ -0,0 +1,163 @@
+//===----- BPFMISimplifyPatchable.cpp - MI Simplify Patchable Insts -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass targets a subset of instructions like below
+// ld_imm64 r1, @global
+// ldd r2, r1, 0
+// add r3, struct_base_reg, r2
+//
+// Here @global should either present a AMA (abstruct member access) or
+// a patchable extern variable. And these two kinds of accesses
+// are subject to bpf load time patching. After this pass, the
+// code becomes
+// ld_imm64 r1, @global
+// add r3, struct_base_reg, r1
+//
+// Eventually, at BTF output stage, a relocation record will be generated
+// for ld_imm64 which should be replaced later by bpf loader:
+// r1 = <calculated offset> or <to_be_patched_extern_val>
+// add r3, struct_base_reg, r1
+// or
+// ld_imm64 r1, <to_be_patched_extern_val>
+// add r3, struct_base_reg, r1
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPF.h"
+#include "BPFCORE.h"
+#include "BPFInstrInfo.h"
+#include "BPFTargetMachine.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "bpf-mi-simplify-patchable"
+
+namespace {
+
+struct BPFMISimplifyPatchable : public MachineFunctionPass {
+
+ static char ID;
+ const BPFInstrInfo *TII;
+ MachineFunction *MF;
+
+ BPFMISimplifyPatchable() : MachineFunctionPass(ID) {
+ initializeBPFMISimplifyPatchablePass(*PassRegistry::getPassRegistry());
+ }
+
+private:
+ // Initialize class variables.
+ void initialize(MachineFunction &MFParm);
+
+ bool removeLD(void);
+
+public:
+ // Main entry point for this pass.
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ if (!skipFunction(MF.getFunction())) {
+ initialize(MF);
+ }
+ return removeLD();
+ }
+};
+
+// Initialize class variables.
+void BPFMISimplifyPatchable::initialize(MachineFunction &MFParm) {
+ MF = &MFParm;
+ TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
+ LLVM_DEBUG(dbgs() << "*** BPF simplify patchable insts pass ***\n\n");
+}
+
+/// Remove unneeded Load instructions.
+bool BPFMISimplifyPatchable::removeLD() {
+ MachineRegisterInfo *MRI = &MF->getRegInfo();
+ MachineInstr *ToErase = nullptr;
+ bool Changed = false;
+
+ for (MachineBasicBlock &MBB : *MF) {
+ for (MachineInstr &MI : MBB) {
+ if (ToErase) {
+ ToErase->eraseFromParent();
+ ToErase = nullptr;
+ }
+
+ // Ensure the register format is LOAD <reg>, <reg>, 0
+ if (MI.getOpcode() != BPF::LDD && MI.getOpcode() != BPF::LDW &&
+ MI.getOpcode() != BPF::LDH && MI.getOpcode() != BPF::LDB &&
+ MI.getOpcode() != BPF::LDW32 && MI.getOpcode() != BPF::LDH32 &&
+ MI.getOpcode() != BPF::LDB32)
+ continue;
+
+ if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg())
+ continue;
+
+ if (!MI.getOperand(2).isImm() || MI.getOperand(2).getImm())
+ continue;
+
+ unsigned DstReg = MI.getOperand(0).getReg();
+ unsigned SrcReg = MI.getOperand(1).getReg();
+ int64_t ImmVal = MI.getOperand(2).getImm();
+
+ MachineInstr *DefInst = MRI->getUniqueVRegDef(SrcReg);
+ if (!DefInst)
+ continue;
+
+ bool IsCandidate = false;
+ if (DefInst->getOpcode() == BPF::LD_imm64) {
+ const MachineOperand &MO = DefInst->getOperand(1);
+ if (MO.isGlobal()) {
+ const GlobalValue *GVal = MO.getGlobal();
+ auto *GVar = dyn_cast<GlobalVariable>(GVal);
+ if (GVar) {
+ // Global variables representing structure offset or
+ // patchable extern globals.
+ if (GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
+ assert(ImmVal == 0);
+ IsCandidate = true;
+ } else if (!GVar->hasInitializer() && GVar->hasExternalLinkage() &&
+ GVar->getSection() ==
+ BPFCoreSharedInfo::PatchableExtSecName) {
+ if (ImmVal == 0)
+ IsCandidate = true;
+ else
+ errs() << "WARNING: unhandled patchable extern "
+ << GVar->getName() << " with load offset " << ImmVal
+ << "\n";
+ }
+ }
+ }
+ }
+
+ if (!IsCandidate)
+ continue;
+
+ auto Begin = MRI->use_begin(DstReg), End = MRI->use_end();
+ decltype(End) NextI;
+ for (auto I = Begin; I != End; I = NextI) {
+ NextI = std::next(I);
+ I->setReg(SrcReg);
+ }
+
+ ToErase = &MI;
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
+
+} // namespace
+
+INITIALIZE_PASS(BPFMISimplifyPatchable, DEBUG_TYPE,
+ "BPF PreEmit SimplifyPatchable", false, false)
+
+char BPFMISimplifyPatchable::ID = 0;
+FunctionPass *llvm::createBPFMISimplifyPatchablePass() {
+ return new BPFMISimplifyPatchable();
+}
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index e5d2e72e451..24c0ff0f7f1 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -34,6 +34,7 @@ extern "C" void LLVMInitializeBPFTarget() {
RegisterTargetMachine<BPFTargetMachine> Z(getTheBPFTarget());
PassRegistry &PR = *PassRegistry::getPassRegistry();
+ initializeBPFAbstractMemberAccessPass(PR);
initializeBPFMIPeepholePass(PR);
}
@@ -68,6 +69,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
static_cast<BPFMCAsmInfo *>(const_cast<MCAsmInfo *>(AsmInfo.get()));
MAI->setDwarfUsesRelocationsAcrossSections(!Subtarget.getUseDwarfRIS());
}
+
namespace {
// BPF Code Generator Pass Configuration Options.
class BPFPassConfig : public TargetPassConfig {
@@ -79,6 +81,7 @@ public:
return getTM<BPFTargetMachine>();
}
+ void addIRPasses() override;
bool addInstSelector() override;
void addMachineSSAOptimization() override;
void addPreEmitPass() override;
@@ -89,6 +92,13 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) {
return new BPFPassConfig(*this, PM);
}
+void BPFPassConfig::addIRPasses() {
+
+ addPass(createBPFAbstractMemberAccess());
+
+ TargetPassConfig::addIRPasses();
+}
+
// Install an instruction selector pass using
// the ISelDag to gen BPF code.
bool BPFPassConfig::addInstSelector() {
@@ -98,6 +108,8 @@ bool BPFPassConfig::addInstSelector() {
}
void BPFPassConfig::addMachineSSAOptimization() {
+ addPass(createBPFMISimplifyPatchablePass());
+
// The default implementation must be called first as we want eBPF
// Peephole ran at last.
TargetPassConfig::addMachineSSAOptimization();
diff --git a/llvm/lib/Target/BPF/BTF.h b/llvm/lib/Target/BPF/BTF.h
index 23af0f07b4a..ad56716710a 100644
--- a/llvm/lib/Target/BPF/BTF.h
+++ b/llvm/lib/Target/BPF/BTF.h
@@ -17,7 +17,7 @@
///
/// The binary layout for .BTF.ext section:
/// struct ExtHeader
-/// FuncInfo and LineInfo subsections
+/// FuncInfo, LineInfo, OffsetReloc and ExternReloc subsections
/// The FuncInfo subsection is defined as below:
/// BTFFuncInfo Size
/// struct SecFuncInfo for ELF section #1
@@ -32,6 +32,20 @@
/// struct SecLineInfo for ELF section #2
/// A number of struct BPFLineInfo for ELF section #2
/// ...
+/// The OffsetReloc subsection is defined as below:
+/// BPFOffsetReloc Size
+/// struct SecOffsetReloc for ELF section #1
+/// A number of struct BPFOffsetReloc for ELF section #1
+/// struct SecOffsetReloc for ELF section #2
+/// A number of struct BPFOffsetReloc for ELF section #2
+/// ...
+/// The ExternReloc subsection is defined as below:
+/// BPFExternReloc Size
+/// struct SecExternReloc for ELF section #1
+/// A number of struct BPFExternReloc for ELF section #1
+/// struct SecExternReloc for ELF section #2
+/// A number of struct BPFExternReloc for ELF section #2
+/// ...
///
/// The section formats are also defined at
/// https://github.com/torvalds/linux/blob/master/include/uapi/linux/btf.h
@@ -49,7 +63,7 @@ enum : uint32_t { MAGIC = 0xeB9F, VERSION = 1 };
/// Sizes in bytes of various things in the BTF format.
enum {
HeaderSize = 24,
- ExtHeaderSize = 24,
+ ExtHeaderSize = 40,
CommonTypeSize = 12,
BTFArraySize = 12,
BTFEnumSize = 8,
@@ -58,8 +72,12 @@ enum {
BTFDataSecVarSize = 12,
SecFuncInfoSize = 8,
SecLineInfoSize = 8,
+ SecOffsetRelocSize = 8,
+ SecExternRelocSize = 8,
BPFFuncInfoSize = 8,
- BPFLineInfoSize = 16
+ BPFLineInfoSize = 16,
+ BPFOffsetRelocSize = 12,
+ BPFExternRelocSize = 8,
};
/// The .BTF section header definition.
@@ -191,10 +209,14 @@ struct ExtHeader {
uint8_t Flags;
uint32_t HdrLen;
- uint32_t FuncInfoOff; ///< Offset of func info section
- uint32_t FuncInfoLen; ///< Length of func info section
- uint32_t LineInfoOff; ///< Offset of line info section
- uint32_t LineInfoLen; ///< Length of line info section
+ uint32_t FuncInfoOff; ///< Offset of func info section
+ uint32_t FuncInfoLen; ///< Length of func info section
+ uint32_t LineInfoOff; ///< Offset of line info section
+ uint32_t LineInfoLen; ///< Length of line info section
+ uint32_t OffsetRelocOff; ///< Offset of offset reloc section
+ uint32_t OffsetRelocLen; ///< Length of offset reloc section
+ uint32_t ExternRelocOff; ///< Offset of extern reloc section
+ uint32_t ExternRelocLen; ///< Length of extern reloc section
};
/// Specifying one function info.
@@ -220,10 +242,35 @@ struct BPFLineInfo {
/// Specifying line info's in one section.
struct SecLineInfo {
- uint32_t SecNameOff; ///< Section name index in the .BTF string tble
+ uint32_t SecNameOff; ///< Section name index in the .BTF string table
uint32_t NumLineInfo; ///< Number of line info's in this section
};
+/// Specifying one offset relocation.
+struct BPFOffsetReloc {
+ uint32_t InsnOffset; ///< Byte offset in this section
+ uint32_t TypeID; ///< TypeID for the relocation
+ uint32_t OffsetNameOff; ///< The string to traverse types
+};
+
+/// Specifying offset relocation's in one section.
+struct SecOffsetReloc {
+ uint32_t SecNameOff; ///< Section name index in the .BTF string table
+ uint32_t NumOffsetReloc; ///< Number of offset reloc's in this section
+};
+
+/// Specifying one offset relocation.
+struct BPFExternReloc {
+ uint32_t InsnOffset; ///< Byte offset in this section
+ uint32_t ExternNameOff; ///< The string for external variable
+};
+
+/// Specifying extern relocation's in one section.
+struct SecExternReloc {
+ uint32_t SecNameOff; ///< Section name index in the .BTF string table
+ uint32_t NumExternReloc; ///< Number of extern reloc's in this section
+};
+
} // End namespace BTF.
} // End namespace llvm.
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index 1442dc3123e..fa35c6619e2 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -11,6 +11,9 @@
//===----------------------------------------------------------------------===//
#include "BTFDebug.h"
+#include "BPF.h"
+#include "BPFCORE.h"
+#include "MCTargetDesc/BPFMCTargetDesc.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
@@ -37,8 +40,9 @@ void BTFTypeBase::emitType(MCStreamer &OS) {
OS.EmitIntValue(BTFType.Size, 4);
}
-BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag)
- : DTy(DTy) {
+BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag,
+ bool NeedsFixup)
+ : DTy(DTy), NeedsFixup(NeedsFixup) {
switch (Tag) {
case dwarf::DW_TAG_pointer_type:
Kind = BTF::BTF_KIND_PTR;
@@ -62,8 +66,15 @@ BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag)
}
void BTFTypeDerived::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(DTy->getName());
+ if (NeedsFixup)
+ return;
+
// The base type for PTR/CONST/VOLATILE could be void.
const DIType *ResolvedType = DTy->getBaseType();
if (!ResolvedType) {
@@ -78,6 +89,10 @@ void BTFTypeDerived::completeType(BTFDebug &BDebug) {
void BTFTypeDerived::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); }
+void BTFTypeDerived::setPointeeType(uint32_t PointeeType) {
+ BTFType.Type = PointeeType;
+}
+
/// Represent a struct/union forward declaration.
BTFTypeFwd::BTFTypeFwd(StringRef Name, bool IsUnion) : Name(Name) {
Kind = BTF::BTF_KIND_FWD;
@@ -86,6 +101,10 @@ BTFTypeFwd::BTFTypeFwd(StringRef Name, bool IsUnion) : Name(Name) {
}
void BTFTypeFwd::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(Name);
}
@@ -119,6 +138,10 @@ BTFTypeInt::BTFTypeInt(uint32_t Encoding, uint32_t SizeInBits,
}
void BTFTypeInt::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(Name);
}
@@ -135,6 +158,10 @@ BTFTypeEnum::BTFTypeEnum(const DICompositeType *ETy, uint32_t VLen) : ETy(ETy) {
}
void BTFTypeEnum::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(ETy->getName());
DINodeArray Elements = ETy->getElements();
@@ -157,7 +184,9 @@ void BTFTypeEnum::emitType(MCStreamer &OS) {
}
}
-BTFTypeArray::BTFTypeArray(uint32_t ElemTypeId, uint32_t NumElems) {
+BTFTypeArray::BTFTypeArray(uint32_t ElemTypeId, uint32_t ElemSize,
+ uint32_t NumElems)
+ : ElemSize(ElemSize) {
Kind = BTF::BTF_KIND_ARRAY;
BTFType.NameOff = 0;
BTFType.Info = Kind << 24;
@@ -169,6 +198,9 @@ BTFTypeArray::BTFTypeArray(uint32_t ElemTypeId, uint32_t NumElems) {
/// Represent a BTF array.
void BTFTypeArray::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
// The IR does not really have a type for the index.
// A special type for array index should have been
@@ -184,6 +216,12 @@ void BTFTypeArray::emitType(MCStreamer &OS) {
OS.EmitIntValue(ArrayInfo.Nelems, 4);
}
+void BTFTypeArray::getLocInfo(uint32_t Loc, uint32_t &LocOffset,
+ uint32_t &ElementTypeId) {
+ ElementTypeId = ArrayInfo.ElemType;
+ LocOffset = Loc * ElemSize;
+}
+
/// Represent either a struct or a union.
BTFTypeStruct::BTFTypeStruct(const DICompositeType *STy, bool IsStruct,
bool HasBitField, uint32_t Vlen)
@@ -194,6 +232,10 @@ BTFTypeStruct::BTFTypeStruct(const DICompositeType *STy, bool IsStruct,
}
void BTFTypeStruct::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(STy->getName());
// Add struct/union members.
@@ -224,6 +266,17 @@ void BTFTypeStruct::emitType(MCStreamer &OS) {
}
}
+std::string BTFTypeStruct::getName() { return STy->getName(); }
+
+void BTFTypeStruct::getMemberInfo(uint32_t Loc, uint32_t &MemberOffset,
+ uint32_t &MemberType) {
+ MemberType = Members[Loc].Type;
+ MemberOffset =
+ HasBitField ? Members[Loc].Offset & 0xffffff : Members[Loc].Offset;
+}
+
+uint32_t BTFTypeStruct::getStructSize() { return STy->getSizeInBits() >> 3; }
+
/// The Func kind represents both subprogram and pointee of function
/// pointers. If the FuncName is empty, it represents a pointee of function
/// pointer. Otherwise, it represents a subprogram. The func arg names
@@ -238,6 +291,10 @@ BTFTypeFuncProto::BTFTypeFuncProto(
}
void BTFTypeFuncProto::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
DITypeRefArray Elements = STy->getTypeArray();
auto RetType = Elements[0];
BTFType.Type = RetType ? BDebug.getTypeId(RetType) : 0;
@@ -275,6 +332,10 @@ BTFTypeFunc::BTFTypeFunc(StringRef FuncName, uint32_t ProtoTypeId)
}
void BTFTypeFunc::completeType(BTFDebug &BDebug) {
+ if (IsCompleted)
+ return;
+ IsCompleted = true;
+
BTFType.NameOff = BDebug.addString(Name);
}
@@ -335,7 +396,8 @@ uint32_t BTFStringTable::addString(StringRef S) {
BTFDebug::BTFDebug(AsmPrinter *AP)
: DebugHandlerBase(AP), OS(*Asm->OutStreamer), SkipInstruction(false),
- LineInfoGenerated(false), SecNameOff(0), ArrayIndexTypeId(0) {
+ LineInfoGenerated(false), SecNameOff(0), ArrayIndexTypeId(0),
+ MapDefNotCollected(true) {
addString("\0");
}
@@ -417,6 +479,7 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct,
auto TypeEntry =
llvm::make_unique<BTFTypeStruct>(CTy, IsStruct, HasBitField, VLen);
+ StructTypes.push_back(TypeEntry.get());
TypeId = addType(std::move(TypeEntry), CTy);
// Visit all struct members.
@@ -426,11 +489,14 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct,
void BTFDebug::visitArrayType(const DICompositeType *CTy, uint32_t &TypeId) {
// Visit array element type.
- uint32_t ElemTypeId;
- visitTypeEntry(CTy->getBaseType(), ElemTypeId);
+ uint32_t ElemTypeId, ElemSize;
+ const DIType *ElemType = CTy->getBaseType();
+ visitTypeEntry(ElemType, ElemTypeId, false, false);
+ ElemSize = ElemType->getSizeInBits() >> 3;
if (!CTy->getSizeInBits()) {
- auto TypeEntry = llvm::make_unique<BTFTypeArray>(ElemTypeId, 0);
+ auto TypeEntry = llvm::make_unique<BTFTypeArray>(ElemTypeId, 0, 0);
+ ArrayTypes.push_back(TypeEntry.get());
ElemTypeId = addType(std::move(TypeEntry), CTy);
} else {
// Visit array dimensions.
@@ -442,11 +508,14 @@ void BTFDebug::visitArrayType(const DICompositeType *CTy, uint32_t &TypeId) {
auto *CI = SR->getCount().dyn_cast<ConstantInt *>();
int64_t Count = CI->getSExtValue();
- auto TypeEntry = llvm::make_unique<BTFTypeArray>(ElemTypeId, Count);
+ auto TypeEntry =
+ llvm::make_unique<BTFTypeArray>(ElemTypeId, ElemSize, Count);
+ ArrayTypes.push_back(TypeEntry.get());
if (I == 0)
ElemTypeId = addType(std::move(TypeEntry), CTy);
else
ElemTypeId = addType(std::move(TypeEntry));
+ ElemSize = ElemSize * Count;
}
}
}
@@ -498,13 +567,42 @@ void BTFDebug::visitCompositeType(const DICompositeType *CTy,
}
/// Handle pointer, typedef, const, volatile, restrict and member types.
-void BTFDebug::visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId) {
+void BTFDebug::visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId,
+ bool CheckPointer, bool SeenPointer) {
unsigned Tag = DTy->getTag();
+ /// Try to avoid chasing pointees, esp. structure pointees which may
+ /// unnecessary bring in a lot of types.
+ if (CheckPointer && !SeenPointer) {
+ SeenPointer = Tag == dwarf::DW_TAG_pointer_type;
+ }
+
+ if (CheckPointer && SeenPointer) {
+ const DIType *Base = DTy->getBaseType();
+ if (Base) {
+ if (const auto *CTy = dyn_cast<DICompositeType>(Base)) {
+ auto CTag = CTy->getTag();
+ if ((CTag == dwarf::DW_TAG_structure_type ||
+ CTag == dwarf::DW_TAG_union_type) &&
+ !CTy->isForwardDecl()) {
+ /// Find a candidate, generate a fixup. Later on the struct/union
+ /// pointee type will be replaced with either a real type or
+ /// a forward declaration.
+ auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag, true);
+ auto &Fixup = FixupDerivedTypes[CTy->getName()];
+ Fixup.first = CTag == dwarf::DW_TAG_union_type;
+ Fixup.second.push_back(TypeEntry.get());
+ TypeId = addType(std::move(TypeEntry), DTy);
+ return;
+ }
+ }
+ }
+ }
+
if (Tag == dwarf::DW_TAG_pointer_type || Tag == dwarf::DW_TAG_typedef ||
Tag == dwarf::DW_TAG_const_type || Tag == dwarf::DW_TAG_volatile_type ||
Tag == dwarf::DW_TAG_restrict_type) {
- auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag);
+ auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag, false);
TypeId = addType(std::move(TypeEntry), DTy);
} else if (Tag != dwarf::DW_TAG_member) {
return;
@@ -513,10 +611,14 @@ void BTFDebug::visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId) {
// Visit base type of pointer, typedef, const, volatile, restrict or
// struct/union member.
uint32_t TempTypeId = 0;
- visitTypeEntry(DTy->getBaseType(), TempTypeId);
+ if (Tag == dwarf::DW_TAG_member)
+ visitTypeEntry(DTy->getBaseType(), TempTypeId, true, false);
+ else
+ visitTypeEntry(DTy->getBaseType(), TempTypeId, CheckPointer, SeenPointer);
}
-void BTFDebug::visitTypeEntry(const DIType *Ty, uint32_t &TypeId) {
+void BTFDebug::visitTypeEntry(const DIType *Ty, uint32_t &TypeId,
+ bool CheckPointer, bool SeenPointer) {
if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end()) {
TypeId = DIToIdMap[Ty];
return;
@@ -530,14 +632,52 @@ void BTFDebug::visitTypeEntry(const DIType *Ty, uint32_t &TypeId) {
else if (const auto *CTy = dyn_cast<DICompositeType>(Ty))
visitCompositeType(CTy, TypeId);
else if (const auto *DTy = dyn_cast<DIDerivedType>(Ty))
- visitDerivedType(DTy, TypeId);
+ visitDerivedType(DTy, TypeId, CheckPointer, SeenPointer);
else
llvm_unreachable("Unknown DIType");
}
void BTFDebug::visitTypeEntry(const DIType *Ty) {
uint32_t TypeId;
- visitTypeEntry(Ty, TypeId);
+ visitTypeEntry(Ty, TypeId, false, false);
+}
+
+void BTFDebug::visitMapDefType(const DIType *Ty, uint32_t &TypeId) {
+ if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end()) {
+ TypeId = DIToIdMap[Ty];
+ return;
+ }
+
+ // MapDef type is a struct type
+ const auto *CTy = dyn_cast<DICompositeType>(Ty);
+ if (!CTy)
+ return;
+
+ auto Tag = CTy->getTag();
+ if (Tag != dwarf::DW_TAG_structure_type || CTy->isForwardDecl())
+ return;
+
+ // Record this type
+ const DINodeArray Elements = CTy->getElements();
+ bool HasBitField = false;
+ for (const auto *Element : Elements) {
+ auto E = cast<DIDerivedType>(Element);
+ if (E->isBitField()) {
+ HasBitField = true;
+ break;
+ }
+ }
+
+ auto TypeEntry =
+ llvm::make_unique<BTFTypeStruct>(CTy, true, HasBitField, Elements.size());
+ StructTypes.push_back(TypeEntry.get());
+ TypeId = addType(std::move(TypeEntry), CTy);
+
+ // Visit all struct members
+ for (const auto *Element : Elements) {
+ const auto *MemberType = cast<DIDerivedType>(Element);
+ visitTypeEntry(MemberType->getBaseType());
+ }
}
/// Read file contents from the actual file or from the source
@@ -635,7 +775,8 @@ void BTFDebug::emitBTFSection() {
void BTFDebug::emitBTFExtSection() {
// Do not emit section if empty FuncInfoTable and LineInfoTable.
- if (!FuncInfoTable.size() && !LineInfoTable.size())
+ if (!FuncInfoTable.size() && !LineInfoTable.size() &&
+ !OffsetRelocTable.size() && !ExternRelocTable.size())
return;
MCContext &Ctx = OS.getContext();
@@ -647,6 +788,8 @@ void BTFDebug::emitBTFExtSection() {
// Account for FuncInfo/LineInfo record size as well.
uint32_t FuncLen = 4, LineLen = 4;
+ // Do not account for optional OffsetReloc/ExternReloc.
+ uint32_t OffsetRelocLen = 0, ExternRelocLen = 0;
for (const auto &FuncSec : FuncInfoTable) {
FuncLen += BTF::SecFuncInfoSize;
FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize;
@@ -655,11 +798,28 @@ void BTFDebug::emitBTFExtSection() {
LineLen += BTF::SecLineInfoSize;
LineLen += LineSec.second.size() * BTF::BPFLineInfoSize;
}
+ for (const auto &OffsetRelocSec : OffsetRelocTable) {
+ OffsetRelocLen += BTF::SecOffsetRelocSize;
+ OffsetRelocLen += OffsetRelocSec.second.size() * BTF::BPFOffsetRelocSize;
+ }
+ for (const auto &ExternRelocSec : ExternRelocTable) {
+ ExternRelocLen += BTF::SecExternRelocSize;
+ ExternRelocLen += ExternRelocSec.second.size() * BTF::BPFExternRelocSize;
+ }
+
+ if (OffsetRelocLen)
+ OffsetRelocLen += 4;
+ if (ExternRelocLen)
+ ExternRelocLen += 4;
OS.EmitIntValue(0, 4);
OS.EmitIntValue(FuncLen, 4);
OS.EmitIntValue(FuncLen, 4);
OS.EmitIntValue(LineLen, 4);
+ OS.EmitIntValue(FuncLen + LineLen, 4);
+ OS.EmitIntValue(OffsetRelocLen, 4);
+ OS.EmitIntValue(FuncLen + LineLen + OffsetRelocLen, 4);
+ OS.EmitIntValue(ExternRelocLen, 4);
// Emit func_info table.
OS.AddComment("FuncInfo");
@@ -692,6 +852,39 @@ void BTFDebug::emitBTFExtSection() {
OS.EmitIntValue(LineInfo.LineNum << 10 | LineInfo.ColumnNum, 4);
}
}
+
+ // Emit offset reloc table.
+ if (OffsetRelocLen) {
+ OS.AddComment("OffsetReloc");
+ OS.EmitIntValue(BTF::BPFOffsetRelocSize, 4);
+ for (const auto &OffsetRelocSec : OffsetRelocTable) {
+ OS.AddComment("Offset reloc section string offset=" +
+ std::to_string(OffsetRelocSec.first));
+ OS.EmitIntValue(OffsetRelocSec.first, 4);
+ OS.EmitIntValue(OffsetRelocSec.second.size(), 4);
+ for (const auto &OffsetRelocInfo : OffsetRelocSec.second) {
+ Asm->EmitLabelReference(OffsetRelocInfo.Label, 4);
+ OS.EmitIntValue(OffsetRelocInfo.TypeID, 4);
+ OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4);
+ }
+ }
+ }
+
+ // Emit extern reloc table.
+ if (ExternRelocLen) {
+ OS.AddComment("ExternReloc");
+ OS.EmitIntValue(BTF::BPFExternRelocSize, 4);
+ for (const auto &ExternRelocSec : ExternRelocTable) {
+ OS.AddComment("Extern reloc section string offset=" +
+ std::to_string(ExternRelocSec.first));
+ OS.EmitIntValue(ExternRelocSec.first, 4);
+ OS.EmitIntValue(ExternRelocSec.second.size(), 4);
+ for (const auto &ExternRelocInfo : ExternRelocSec.second) {
+ Asm->EmitLabelReference(ExternRelocInfo.Label, 4);
+ OS.EmitIntValue(ExternRelocInfo.ExternNameOff, 4);
+ }
+ }
+ }
}
void BTFDebug::beginFunctionImpl(const MachineFunction *MF) {
@@ -704,6 +897,30 @@ void BTFDebug::beginFunctionImpl(const MachineFunction *MF) {
}
SkipInstruction = false;
+ // Collect MapDef types. Map definition needs to collect
+ // pointee types. Do it first. Otherwise, for the following
+ // case:
+ // struct m { ...};
+ // struct t {
+ // struct m *key;
+ // };
+ // foo(struct t *arg);
+ //
+ // struct mapdef {
+ // ...
+ // struct m *key;
+ // ...
+ // } __attribute__((section(".maps"))) hash_map;
+ //
+ // If subroutine foo is traversed first, a type chain
+ // "ptr->struct m(fwd)" will be created and later on
+ // when traversing mapdef, since "ptr->struct m" exists,
+ // the traversal of "struct m" will be omitted.
+ if (MapDefNotCollected) {
+ processGlobals(true);
+ MapDefNotCollected = false;
+ }
+
// Collect all types locally referenced in this function.
// Use RetainedNodes so we can collect all argument names
// even if the argument is not used.
@@ -728,6 +945,9 @@ void BTFDebug::beginFunctionImpl(const MachineFunction *MF) {
llvm::make_unique<BTFTypeFunc>(SP->getName(), ProtoTypeId);
uint32_t FuncTypeId = addType(std::move(FuncTypeEntry));
+ for (const auto &TypeEntry : TypeEntries)
+ TypeEntry->completeType(*this);
+
// Construct funcinfo and the first lineinfo for the function.
MCSymbol *FuncLabel = Asm->getFunctionBegin();
BTFFuncInfo FuncInfo;
@@ -750,6 +970,133 @@ void BTFDebug::endFunctionImpl(const MachineFunction *MF) {
SecNameOff = 0;
}
+/// On-demand populate struct types as requested from abstract member
+/// accessing.
+unsigned BTFDebug::populateStructType(const DIType *Ty) {
+ unsigned Id;
+ visitTypeEntry(Ty, Id, false, false);
+ for (const auto &TypeEntry : TypeEntries)
+ TypeEntry->completeType(*this);
+ return Id;
+}
+
+// Find struct/array debuginfo types given a type id.
+void BTFDebug::setTypeFromId(uint32_t TypeId, BTFTypeStruct **PrevStructType,
+ BTFTypeArray **PrevArrayType) {
+ for (const auto &StructType : StructTypes) {
+ if (StructType->getId() == TypeId) {
+ *PrevStructType = StructType;
+ return;
+ }
+ }
+ for (const auto &ArrayType : ArrayTypes) {
+ if (ArrayType->getId() == TypeId) {
+ *PrevArrayType = ArrayType;
+ return;
+ }
+ }
+}
+
+/// Generate a struct member offset relocation.
+void BTFDebug::generateOffsetReloc(const MachineInstr *MI,
+ const MCSymbol *ORSym, DIType *RootTy,
+ StringRef AccessPattern) {
+ BTFTypeStruct *PrevStructType = nullptr;
+ BTFTypeArray *PrevArrayType = nullptr;
+ unsigned RootId = populateStructType(RootTy);
+ setTypeFromId(RootId, &PrevStructType, &PrevArrayType);
+ unsigned RootTySize = PrevStructType->getStructSize();
+
+ BTFOffsetReloc OffsetReloc;
+ OffsetReloc.Label = ORSym;
+ OffsetReloc.OffsetNameOff = addString(AccessPattern.drop_back());
+ OffsetReloc.TypeID = RootId;
+
+ uint32_t Start = 0, End = 0, Offset = 0;
+ bool FirstAccess = true;
+ for (auto C : AccessPattern) {
+ if (C != ':') {
+ End++;
+ } else {
+ std::string SubStr = AccessPattern.substr(Start, End - Start);
+ int Loc = std::stoi(SubStr);
+
+ if (FirstAccess) {
+ Offset = Loc * RootTySize;
+ FirstAccess = false;
+ } else if (PrevStructType) {
+ uint32_t MemberOffset, MemberTypeId;
+ PrevStructType->getMemberInfo(Loc, MemberOffset, MemberTypeId);
+
+ Offset += MemberOffset >> 3;
+ PrevStructType = nullptr;
+ setTypeFromId(MemberTypeId, &PrevStructType, &PrevArrayType);
+ } else if (PrevArrayType) {
+ uint32_t LocOffset, ElementTypeId;
+ PrevArrayType->getLocInfo(Loc, LocOffset, ElementTypeId);
+
+ Offset += LocOffset;
+ PrevArrayType = nullptr;
+ setTypeFromId(ElementTypeId, &PrevStructType, &PrevArrayType);
+ }
+ Start = End + 1;
+ End = Start;
+ }
+ }
+ AccessOffsets[RootTy->getName().str() + ":" + AccessPattern.str()] = Offset;
+ OffsetRelocTable[SecNameOff].push_back(OffsetReloc);
+}
+
+void BTFDebug::processLDimm64(const MachineInstr *MI) {
+ // If the insn is an LD_imm64, the following two cases
+ // will generate an .BTF.ext record.
+ //
+ // If the insn is "r2 = LD_imm64 @__BTF_...",
+ // add this insn into the .BTF.ext OffsetReloc subsection.
+ // Relocation looks like:
+ // . SecName:
+ // . InstOffset
+ // . TypeID
+ // . OffSetNameOff
+ // Later, the insn is replaced with "r2 = <offset>"
+ // where "<offset>" equals to the offset based on current
+ // type definitions.
+ //
+ // If the insn is "r2 = LD_imm64 @VAR" and VAR is
+ // a patchable external global, add this insn into the .BTF.ext
+ // ExternReloc subsection.
+ // Relocation looks like:
+ // . SecName:
+ // . InstOffset
+ // . ExternNameOff
+ // Later, the insn is replaced with "r2 = <value>" or
+ // "LD_imm64 r2, <value>" where "<value>" = 0.
+
+ // check whether this is a candidate or not
+ const MachineOperand &MO = MI->getOperand(1);
+ if (MO.isGlobal()) {
+ const GlobalValue *GVal = MO.getGlobal();
+ auto *GVar = dyn_cast<GlobalVariable>(GVal);
+ if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
+ MCSymbol *ORSym = OS.getContext().createTempSymbol();
+ OS.EmitLabel(ORSym);
+
+ MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index);
+ DIType *Ty = dyn_cast<DIType>(MDN);
+ generateOffsetReloc(MI, ORSym, Ty, GVar->getName());
+ } else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() &&
+ GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) {
+ MCSymbol *ORSym = OS.getContext().createTempSymbol();
+ OS.EmitLabel(ORSym);
+
+ BTFExternReloc ExternReloc;
+ ExternReloc.Label = ORSym;
+ ExternReloc.ExternNameOff = addString(GVar->getName());
+ ExternRelocTable[SecNameOff].push_back(ExternReloc);
+ }
+ }
+}
+
void BTFDebug::beginInstruction(const MachineInstr *MI) {
DebugHandlerBase::beginInstruction(MI);
@@ -770,6 +1117,9 @@ void BTFDebug::beginInstruction(const MachineInstr *MI) {
return;
}
+ if (MI->getOpcode() == BPF::LD_imm64)
+ processLDimm64(MI);
+
// Skip this instruction if no DebugLoc or the DebugLoc
// is the same as the previous instruction.
const DebugLoc &DL = MI->getDebugLoc();
@@ -798,7 +1148,7 @@ void BTFDebug::beginInstruction(const MachineInstr *MI) {
PrevInstLoc = DL;
}
-void BTFDebug::processGlobals() {
+void BTFDebug::processGlobals(bool ProcessingMapDef) {
// Collect all types referenced by globals.
const Module *M = MMI->getModule();
for (const GlobalVariable &Global : M->globals()) {
@@ -806,11 +1156,29 @@ void BTFDebug::processGlobals() {
if (!Global.hasInitializer() && Global.hasExternalLinkage())
continue;
+ // Decide the section name.
+ StringRef SecName;
+ if (Global.hasSection()) {
+ SecName = Global.getSection();
+ } else {
+ // data, bss, or readonly sections
+ if (Global.isConstant())
+ SecName = ".rodata";
+ else
+ SecName = Global.getInitializer()->isZeroValue() ? ".bss" : ".data";
+ }
+
+ if (ProcessingMapDef != SecName.startswith(".maps"))
+ continue;
+
SmallVector<DIGlobalVariableExpression *, 1> GVs;
Global.getDebugInfo(GVs);
uint32_t GVTypeId = 0;
for (auto *GVE : GVs) {
- visitTypeEntry(GVE->getVariable()->getType(), GVTypeId);
+ if (SecName.startswith(".maps"))
+ visitMapDefType(GVE->getVariable()->getType(), GVTypeId);
+ else
+ visitTypeEntry(GVE->getVariable()->getType(), GVTypeId, false, false);
break;
}
@@ -835,18 +1203,6 @@ void BTFDebug::processGlobals() {
llvm::make_unique<BTFKindVar>(Global.getName(), GVTypeId, GVarInfo);
uint32_t VarId = addType(std::move(VarEntry));
- // Decide the section name.
- std::string SecName;
- if (Global.hasSection()) {
- SecName = Global.getSection().str();
- } else {
- // data, bss, or readonly sections
- if (Global.isConstant())
- SecName += ".rodata";
- else
- SecName += Global.getInitializer()->isZeroValue() ? ".bss" : ".data";
- }
-
// Find or create a DataSec
if (DataSecEntries.find(SecName) == DataSecEntries.end()) {
DataSecEntries[SecName] = llvm::make_unique<BTFKindDataSec>(Asm, SecName);
@@ -858,14 +1214,81 @@ void BTFDebug::processGlobals() {
DataSecEntries[SecName]->addVar(VarId, Asm->getSymbol(&Global), Size);
}
+}
+
+/// Emit proper patchable instructions.
+bool BTFDebug::InstLower(const MachineInstr *MI, MCInst &OutMI) {
+ if (MI->getOpcode() == BPF::LD_imm64) {
+ const MachineOperand &MO = MI->getOperand(1);
+ if (MO.isGlobal()) {
+ const GlobalValue *GVal = MO.getGlobal();
+ auto *GVar = dyn_cast<GlobalVariable>(GVal);
+ if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
+ MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index);
+ DIType *Ty = dyn_cast<DIType>(MDN);
+ std::string TypeName = Ty->getName();
+ int64_t Imm = AccessOffsets[TypeName + ":" + GVar->getName().str()];
+
+ // Emit "mov ri, <imm>" for abstract member accesses.
+ OutMI.setOpcode(BPF::MOV_ri);
+ OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ OutMI.addOperand(MCOperand::createImm(Imm));
+ return true;
+ } else if (GVar && !GVar->hasInitializer() &&
+ GVar->hasExternalLinkage() &&
+ GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) {
+ const IntegerType *IntTy = dyn_cast<IntegerType>(GVar->getValueType());
+ assert(IntTy);
+ // For patchable externals, emit "LD_imm64, ri, 0" if the external
+ // variable is 64bit width, emit "mov ri, 0" otherwise.
+ if (IntTy->getBitWidth() == 64)
+ OutMI.setOpcode(BPF::LD_imm64);
+ else
+ OutMI.setOpcode(BPF::MOV_ri);
+ OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ OutMI.addOperand(MCOperand::createImm(0));
+ return true;
+ }
+ }
+ }
+ return false;
+}
+void BTFDebug::endModule() {
+ // Collect MapDef globals if not collected yet.
+ if (MapDefNotCollected) {
+ processGlobals(true);
+ MapDefNotCollected = false;
+ }
+
+ // Collect global types/variables except MapDef globals.
+ processGlobals(false);
for (auto &DataSec : DataSecEntries)
addType(std::move(DataSec.second));
-}
-void BTFDebug::endModule() {
- // Collect all global types/variables.
- processGlobals();
+ // Fixups
+ for (auto &Fixup : FixupDerivedTypes) {
+ StringRef TypeName = Fixup.first;
+ bool IsUnion = Fixup.second.first;
+
+ // Search through struct types
+ uint32_t StructTypeId = 0;
+ for (const auto &StructType : StructTypes) {
+ if (StructType->getName() == TypeName) {
+ StructTypeId = StructType->getId();
+ break;
+ }
+ }
+
+ if (StructTypeId == 0) {
+ auto FwdTypeEntry = llvm::make_unique<BTFTypeFwd>(TypeName, IsUnion);
+ StructTypeId = addType(std::move(FwdTypeEntry));
+ }
+
+ for (auto &DType : Fixup.second.second) {
+ DType->setPointeeType(StructTypeId);
+ }
+ }
// Complete BTF type cross refereences.
for (const auto &TypeEntry : TypeEntries)
diff --git a/llvm/lib/Target/BPF/BTFDebug.h b/llvm/lib/Target/BPF/BTFDebug.h
index 7d7a1d9b707..6c0cdde17d9 100644
--- a/llvm/lib/Target/BPF/BTFDebug.h
+++ b/llvm/lib/Target/BPF/BTFDebug.h
@@ -32,10 +32,12 @@ class MachineFunction;
class BTFTypeBase {
protected:
uint8_t Kind;
+ bool IsCompleted;
uint32_t Id;
struct BTF::CommonType BTFType;
public:
+ BTFTypeBase() : IsCompleted(false) {}
virtual ~BTFTypeBase() = default;
void setId(uint32_t Id) { this->Id = Id; }
uint32_t getId() { return Id; }
@@ -54,11 +56,13 @@ public:
/// volatile, typedef and restrict.
class BTFTypeDerived : public BTFTypeBase {
const DIDerivedType *DTy;
+ bool NeedsFixup;
public:
- BTFTypeDerived(const DIDerivedType *Ty, unsigned Tag);
+ BTFTypeDerived(const DIDerivedType *Ty, unsigned Tag, bool NeedsFixup);
void completeType(BTFDebug &BDebug);
void emitType(MCStreamer &OS);
+ void setPointeeType(uint32_t PointeeType);
};
/// Handle struct or union forward declaration.
@@ -100,13 +104,15 @@ public:
/// Handle array type.
class BTFTypeArray : public BTFTypeBase {
+ uint32_t ElemSize;
struct BTF::BTFArray ArrayInfo;
public:
- BTFTypeArray(uint32_t ElemTypeId, uint32_t NumElems);
+ BTFTypeArray(uint32_t ElemTypeId, uint32_t ElemSize, uint32_t NumElems);
uint32_t getSize() { return BTFTypeBase::getSize() + BTF::BTFArraySize; }
void completeType(BTFDebug &BDebug);
void emitType(MCStreamer &OS);
+ void getLocInfo(uint32_t Loc, uint32_t &LocOffset, uint32_t &ElementTypeId);
};
/// Handle struct/union type.
@@ -123,6 +129,9 @@ public:
}
void completeType(BTFDebug &BDebug);
void emitType(MCStreamer &OS);
+ std::string getName();
+ void getMemberInfo(uint32_t Loc, uint32_t &Offset, uint32_t &MemberType);
+ uint32_t getStructSize();
};
/// Handle function pointer.
@@ -218,6 +227,19 @@ struct BTFLineInfo {
uint32_t ColumnNum; ///< the column number
};
+/// Represent one offset relocation.
+struct BTFOffsetReloc {
+ const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc
+ uint32_t TypeID; ///< Type ID
+ uint32_t OffsetNameOff; ///< The string to traverse types
+};
+
+/// Represent one extern relocation.
+struct BTFExternReloc {
+ const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc
+ uint32_t ExternNameOff; ///< The extern variable name
+};
+
/// Collect and emit BTF information.
class BTFDebug : public DebugHandlerBase {
MCStreamer &OS;
@@ -225,14 +247,21 @@ class BTFDebug : public DebugHandlerBase {
bool LineInfoGenerated;
uint32_t SecNameOff;
uint32_t ArrayIndexTypeId;
+ bool MapDefNotCollected;
BTFStringTable StringTable;
std::vector<std::unique_ptr<BTFTypeBase>> TypeEntries;
std::unordered_map<const DIType *, uint32_t> DIToIdMap;
std::map<uint32_t, std::vector<BTFFuncInfo>> FuncInfoTable;
std::map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable;
+ std::map<uint32_t, std::vector<BTFOffsetReloc>> OffsetRelocTable;
+ std::map<uint32_t, std::vector<BTFExternReloc>> ExternRelocTable;
StringMap<std::vector<std::string>> FileContent;
- std::map<std::string, std::unique_ptr<BTFKindDataSec>>
- DataSecEntries;
+ std::map<std::string, std::unique_ptr<BTFKindDataSec>> DataSecEntries;
+ std::vector<BTFTypeStruct *> StructTypes;
+ std::vector<BTFTypeArray *> ArrayTypes;
+ std::map<std::string, int64_t> AccessOffsets;
+ std::map<StringRef, std::pair<bool, std::vector<BTFTypeDerived *>>>
+ FixupDerivedTypes;
/// Add types to TypeEntries.
/// @{
@@ -245,7 +274,8 @@ class BTFDebug : public DebugHandlerBase {
/// IR type visiting functions.
/// @{
void visitTypeEntry(const DIType *Ty);
- void visitTypeEntry(const DIType *Ty, uint32_t &TypeId);
+ void visitTypeEntry(const DIType *Ty, uint32_t &TypeId, bool CheckPointer,
+ bool SeenPointer);
void visitBasicType(const DIBasicType *BTy, uint32_t &TypeId);
void visitSubroutineType(
const DISubroutineType *STy, bool ForSubprog,
@@ -258,7 +288,9 @@ class BTFDebug : public DebugHandlerBase {
uint32_t &TypeId);
void visitArrayType(const DICompositeType *ATy, uint32_t &TypeId);
void visitEnumType(const DICompositeType *ETy, uint32_t &TypeId);
- void visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId);
+ void visitDerivedType(const DIDerivedType *DTy, uint32_t &TypeId,
+ bool CheckPointer, bool SeenPointer);
+ void visitMapDefType(const DIType *Ty, uint32_t &TypeId);
/// @}
/// Get the file content for the subprogram. Certain lines of the file
@@ -270,7 +302,21 @@ class BTFDebug : public DebugHandlerBase {
uint32_t Column);
/// Generate types and variables for globals.
- void processGlobals(void);
+ void processGlobals(bool ProcessingMapDef);
+
+ /// Generate one offset relocation record.
+ void generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym,
+ DIType *RootTy, StringRef AccessPattern);
+
+ /// Set the to-be-traversed Struct/Array Type based on TypeId.
+ void setTypeFromId(uint32_t TypeId, BTFTypeStruct **PrevStructType,
+ BTFTypeArray **PrevArrayType);
+
+ /// Populating unprocessed struct type.
+ unsigned populateStructType(const DIType *Ty);
+
+ /// Process LD_imm64 instructions.
+ void processLDimm64(const MachineInstr *MI);
/// Emit common header of .BTF and .BTF.ext sections.
void emitCommonHeader();
@@ -291,6 +337,9 @@ protected:
public:
BTFDebug(AsmPrinter *AP);
+ ///
+ bool InstLower(const MachineInstr *MI, MCInst &OutMI);
+
/// Get the special array index type id.
uint32_t getArrayIndexTypeId() {
assert(ArrayIndexTypeId);
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index 37c03aa511e..42d7bf9ec8b 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -13,6 +13,7 @@ tablegen(LLVM BPFGenSubtargetInfo.inc -gen-subtarget)
add_public_tablegen_target(BPFCommonTableGen)
add_llvm_target(BPFCodeGen
+ BPFAbstrctMemberAccess.cpp
BPFAsmPrinter.cpp
BPFFrameLowering.cpp
BPFInstrInfo.cpp
@@ -25,6 +26,7 @@ add_llvm_target(BPFCodeGen
BPFTargetMachine.cpp
BPFMIPeephole.cpp
BPFMIChecking.cpp
+ BPFMISimplifyPatchable.cpp
BTFDebug.cpp
)
OpenPOWER on IntegriCloud