1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ProfileData/InstrProfCorrelator.h"
10 #include "llvm/DebugInfo/DIContext.h"
11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
17 #include "llvm/Object/MachO.h"
18 #include "llvm/Support/Debug.h"
19 #include <optional>
20
21 #define DEBUG_TYPE "correlator"
22
23 using namespace llvm;
24
25 /// Get the __llvm_prf_cnts section.
getCountersSection(const object::ObjectFile & Obj)26 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
27 for (auto &Section : Obj.sections())
28 if (auto SectionName = Section.getName())
29 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
30 return Section;
31 return make_error<InstrProfError>(
32 instrprof_error::unable_to_correlate_profile,
33 "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
34 }
35
36 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
37 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
38 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
39
40 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
get(std::unique_ptr<MemoryBuffer> Buffer,const object::ObjectFile & Obj)41 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
42 const object::ObjectFile &Obj) {
43 auto CountersSection = getCountersSection(Obj);
44 if (auto Err = CountersSection.takeError())
45 return std::move(Err);
46 auto C = std::make_unique<Context>();
47 C->Buffer = std::move(Buffer);
48 C->CountersSectionStart = CountersSection->getAddress();
49 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
50 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
51 return Expected<std::unique_ptr<Context>>(std::move(C));
52 }
53
54 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(StringRef DebugInfoFilename)55 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
56 auto DsymObjectsOrErr =
57 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
58 if (auto Err = DsymObjectsOrErr.takeError())
59 return std::move(Err);
60 if (!DsymObjectsOrErr->empty()) {
61 // TODO: Enable profile correlation when there are multiple objects in a
62 // dSYM bundle.
63 if (DsymObjectsOrErr->size() > 1)
64 return make_error<InstrProfError>(
65 instrprof_error::unable_to_correlate_profile,
66 "using multiple objects is not yet supported");
67 DebugInfoFilename = *DsymObjectsOrErr->begin();
68 }
69 auto BufferOrErr =
70 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
71 if (auto Err = BufferOrErr.takeError())
72 return std::move(Err);
73
74 return get(std::move(*BufferOrErr));
75 }
76
77 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(std::unique_ptr<MemoryBuffer> Buffer)78 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
79 auto BinOrErr = object::createBinary(*Buffer);
80 if (auto Err = BinOrErr.takeError())
81 return std::move(Err);
82
83 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
84 auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
85 if (auto Err = CtxOrErr.takeError())
86 return std::move(Err);
87 auto T = Obj->makeTriple();
88 if (T.isArch64Bit())
89 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
90 if (T.isArch32Bit())
91 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
92 }
93 return make_error<InstrProfError>(
94 instrprof_error::unable_to_correlate_profile, "not an object file");
95 }
96
getDataSize() const97 std::optional<size_t> InstrProfCorrelator::getDataSize() const {
98 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
99 return C->getDataSize();
100 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
101 return C->getDataSize();
102 }
103 return {};
104 }
105
106 namespace llvm {
107
108 template <>
InstrProfCorrelatorImpl(std::unique_ptr<InstrProfCorrelator::Context> Ctx)109 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
110 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
111 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
112 std::move(Ctx)) {}
113 template <>
InstrProfCorrelatorImpl(std::unique_ptr<InstrProfCorrelator::Context> Ctx)114 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
115 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
116 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
117 std::move(Ctx)) {}
118 template <>
classof(const InstrProfCorrelator * C)119 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
120 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
121 }
122 template <>
classof(const InstrProfCorrelator * C)123 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
124 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
125 }
126
127 } // end namespace llvm
128
129 template <class IntPtrT>
130 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,const object::ObjectFile & Obj)131 InstrProfCorrelatorImpl<IntPtrT>::get(
132 std::unique_ptr<InstrProfCorrelator::Context> Ctx,
133 const object::ObjectFile &Obj) {
134 if (Obj.isELF() || Obj.isMachO()) {
135 auto DICtx = DWARFContext::create(Obj);
136 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
137 std::move(Ctx));
138 }
139 return make_error<InstrProfError>(
140 instrprof_error::unable_to_correlate_profile,
141 "unsupported debug info format (only DWARF is supported)");
142 }
143
144 template <class IntPtrT>
correlateProfileData()145 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
146 assert(Data.empty() && Names.empty() && NamesVec.empty());
147 correlateProfileDataImpl();
148 if (Data.empty() || NamesVec.empty())
149 return make_error<InstrProfError>(
150 instrprof_error::unable_to_correlate_profile,
151 "could not find any profile metadata in debug info");
152 auto Result =
153 collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
154 CounterOffsets.clear();
155 NamesVec.clear();
156 return Result;
157 }
158
159 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
mappingyaml::MappingTraits160 static void mapping(yaml::IO &io,
161 InstrProfCorrelator::CorrelationData &Data) {
162 io.mapRequired("Probes", Data.Probes);
163 }
164 };
165
166 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
mappingyaml::MappingTraits167 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
168 io.mapRequired("Function Name", P.FunctionName);
169 io.mapOptional("Linkage Name", P.LinkageName);
170 io.mapRequired("CFG Hash", P.CFGHash);
171 io.mapRequired("Counter Offset", P.CounterOffset);
172 io.mapRequired("Num Counters", P.NumCounters);
173 io.mapOptional("File", P.FilePath);
174 io.mapOptional("Line", P.LineNumber);
175 }
176 };
177
178 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
179 static const bool flow = false;
180 };
181
182 template <class IntPtrT>
dumpYaml(raw_ostream & OS)183 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) {
184 InstrProfCorrelator::CorrelationData Data;
185 correlateProfileDataImpl(&Data);
186 if (Data.Probes.empty())
187 return make_error<InstrProfError>(
188 instrprof_error::unable_to_correlate_profile,
189 "could not find any profile metadata in debug info");
190 yaml::Output YamlOS(OS);
191 YamlOS << Data;
192 return Error::success();
193 }
194
195 template <class IntPtrT>
addProbe(StringRef FunctionName,uint64_t CFGHash,IntPtrT CounterOffset,IntPtrT FunctionPtr,uint32_t NumCounters)196 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
197 uint64_t CFGHash,
198 IntPtrT CounterOffset,
199 IntPtrT FunctionPtr,
200 uint32_t NumCounters) {
201 // Check if a probe was already added for this counter offset.
202 if (!CounterOffsets.insert(CounterOffset).second)
203 return;
204 Data.push_back({
205 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
206 maybeSwap<uint64_t>(CFGHash),
207 // In this mode, CounterPtr actually stores the section relative address
208 // of the counter.
209 maybeSwap<IntPtrT>(CounterOffset),
210 maybeSwap<IntPtrT>(FunctionPtr),
211 // TODO: Value profiling is not yet supported.
212 /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
213 maybeSwap<uint32_t>(NumCounters),
214 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
215 });
216 NamesVec.push_back(FunctionName.str());
217 }
218
219 template <class IntPtrT>
220 std::optional<uint64_t>
getLocation(const DWARFDie & Die) const221 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
222 auto Locations = Die.getLocations(dwarf::DW_AT_location);
223 if (!Locations) {
224 consumeError(Locations.takeError());
225 return {};
226 }
227 auto &DU = *Die.getDwarfUnit();
228 auto AddressSize = DU.getAddressByteSize();
229 for (auto &Location : *Locations) {
230 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
231 DWARFExpression Expr(Data, AddressSize);
232 for (auto &Op : Expr) {
233 if (Op.getCode() == dwarf::DW_OP_addr) {
234 return Op.getRawOperand(0);
235 } else if (Op.getCode() == dwarf::DW_OP_addrx) {
236 uint64_t Index = Op.getRawOperand(0);
237 if (auto SA = DU.getAddrOffsetSectionItem(Index))
238 return SA->Address;
239 }
240 }
241 }
242 return {};
243 }
244
245 template <class IntPtrT>
isDIEOfProbe(const DWARFDie & Die)246 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
247 const auto &ParentDie = Die.getParent();
248 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
249 return false;
250 if (Die.getTag() != dwarf::DW_TAG_variable)
251 return false;
252 if (!ParentDie.isSubprogramDIE())
253 return false;
254 if (!Die.hasChildren())
255 return false;
256 if (const char *Name = Die.getName(DINameKind::ShortName))
257 return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
258 return false;
259 }
260
261 template <class IntPtrT>
correlateProfileDataImpl(InstrProfCorrelator::CorrelationData * Data)262 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
263 InstrProfCorrelator::CorrelationData *Data) {
264 auto maybeAddProbe = [&](DWARFDie Die) {
265 if (!isDIEOfProbe(Die))
266 return;
267 std::optional<const char *> FunctionName;
268 std::optional<uint64_t> CFGHash;
269 std::optional<uint64_t> CounterPtr = getLocation(Die);
270 auto FnDie = Die.getParent();
271 auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
272 std::optional<uint64_t> NumCounters;
273 for (const DWARFDie &Child : Die.children()) {
274 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
275 continue;
276 auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
277 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
278 if (!AnnotationFormName || !AnnotationFormValue)
279 continue;
280 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
281 if (auto Err = AnnotationNameOrErr.takeError()) {
282 consumeError(std::move(Err));
283 continue;
284 }
285 StringRef AnnotationName = *AnnotationNameOrErr;
286 if (AnnotationName.compare(
287 InstrProfCorrelator::FunctionNameAttributeName) == 0) {
288 if (auto EC =
289 AnnotationFormValue->getAsCString().moveInto(FunctionName))
290 consumeError(std::move(EC));
291 } else if (AnnotationName.compare(
292 InstrProfCorrelator::CFGHashAttributeName) == 0) {
293 CFGHash = AnnotationFormValue->getAsUnsignedConstant();
294 } else if (AnnotationName.compare(
295 InstrProfCorrelator::NumCountersAttributeName) == 0) {
296 NumCounters = AnnotationFormValue->getAsUnsignedConstant();
297 }
298 }
299 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
300 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
301 << FunctionName << "\n\tCFGHash: " << CFGHash
302 << "\n\tCounterPtr: " << CounterPtr
303 << "\n\tNumCounters: " << NumCounters);
304 LLVM_DEBUG(Die.dump(dbgs()));
305 return;
306 }
307 uint64_t CountersStart = this->Ctx->CountersSectionStart;
308 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
309 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
310 LLVM_DEBUG(
311 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
312 << FunctionName << "\n\tExpected: [0x"
313 << Twine::utohexstr(CountersStart) << ", 0x"
314 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
315 << Twine::utohexstr(*CounterPtr));
316 LLVM_DEBUG(Die.dump(dbgs()));
317 return;
318 }
319 if (!FunctionPtr) {
320 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
321 << "\n");
322 LLVM_DEBUG(Die.dump(dbgs()));
323 }
324 IntPtrT CounterOffset = *CounterPtr - CountersStart;
325 if (Data) {
326 InstrProfCorrelator::Probe P;
327 P.FunctionName = *FunctionName;
328 if (auto Name = FnDie.getName(DINameKind::LinkageName))
329 P.LinkageName = Name;
330 P.CFGHash = *CFGHash;
331 P.CounterOffset = CounterOffset;
332 P.NumCounters = *NumCounters;
333 auto FilePath = FnDie.getDeclFile(
334 DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
335 if (!FilePath.empty())
336 P.FilePath = FilePath;
337 if (auto LineNumber = FnDie.getDeclLine())
338 P.LineNumber = LineNumber;
339 Data->Probes.push_back(P);
340 } else {
341 this->addProbe(*FunctionName, *CFGHash, CounterOffset,
342 FunctionPtr.value_or(0), *NumCounters);
343 }
344 };
345 for (auto &CU : DICtx->normal_units())
346 for (const auto &Entry : CU->dies())
347 maybeAddProbe(DWARFDie(CU.get(), &Entry));
348 for (auto &CU : DICtx->dwo_units())
349 for (const auto &Entry : CU->dies())
350 maybeAddProbe(DWARFDie(CU.get(), &Entry));
351 }
352