Viewing file:  InstrProf.h (48.96 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Instrumentation-based profiling data is generated by instrumented
// binaries through library functions in compiler-rt, and read by the clang
// frontend to feed PGO.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_PROFILEDATA_INSTRPROF_H
#define LLVM_PROFILEDATA_INSTRPROF_H
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/IntervalMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/BalancedPartitioning.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Host.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <list>
#include <memory>
#include <string>
#include <system_error>
#include <utility>
#include <vector>
namespace llvm {
class Function;
class GlobalVariable;
struct InstrProfRecord;
class InstrProfSymtab;
class Instruction;
class MDNode;
class Module;
enum InstrProfSectKind {
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
#include "llvm/ProfileData/InstrProfData.inc"
};
/// Return the max count value. We reserver a few large values for special use.
inline uint64_t getInstrMaxCountValue() {
return std::numeric_limits<uint64_t>::max() - 2;
}
/// Return the name of the profile section corresponding to \p IPSK.
///
/// The name of the section depends on the object format type \p OF. If
/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
/// be added to the section name (this is the default).
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
Triple::ObjectFormatType OF,
bool AddSegmentInfo = true);
/// Return the name profile runtime entry point to do value profiling
/// for a given site.
inline StringRef getInstrProfValueProfFuncName() {
return INSTR_PROF_VALUE_PROF_FUNC_STR;
}
/// Return the name profile runtime entry point to do memop size value
/// profiling.
inline StringRef getInstrProfValueProfMemOpFuncName() {
return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
}
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
/// Return the name prefix of variables containing virtual table profile data.
inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }
/// Return the name prefix of variables containing per-function control data.
inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
/// Return the name prefix of profile counter variables.
inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
/// Return the name prefix of profile bitmap variables.
inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; }
/// Return the name prefix of value profile variables.
inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
/// Return the name of value profile node array variables:
inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
/// Return the name of the variable holding the strings (possibly compressed)
/// of all function's PGO names.
inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; }
inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
/// Return the name of a covarage mapping variable (internal linkage)
/// for each instrumented source module. Such variables are allocated
/// in the __llvm_covmap section.
inline StringRef getCoverageMappingVarName() {
return "__llvm_coverage_mapping";
}
/// Return the name of the internal variable recording the array
/// of PGO name vars referenced by the coverage mapping. The owning
/// functions of those names are not emitted by FE (e.g, unused inline
/// functions.)
inline StringRef getCoverageUnusedNamesVarName() {
return "__llvm_coverage_names";
}
/// Return the name of function that registers all the per-function control
/// data at program startup time by calling __llvm_register_function. This
/// function has internal linkage and is called by __llvm_profile_init
/// runtime method. This function is not generated for these platforms:
/// Darwin, Linux, and FreeBSD.
inline StringRef getInstrProfRegFuncsName() {
return "__llvm_profile_register_functions";
}
/// Return the name of the runtime interface that registers per-function control
/// data for one instrumented function.
inline StringRef getInstrProfRegFuncName() {
return "__llvm_profile_register_function";
}
/// Return the name of the runtime interface that registers the PGO name
/// strings.
inline StringRef getInstrProfNamesRegFuncName() {
return "__llvm_profile_register_names_function";
}
/// Return the name of the runtime initialization method that is generated by
/// the compiler. The function calls __llvm_profile_register_functions and
/// __llvm_profile_override_default_filename functions if needed. This function
/// has internal linkage and invoked at startup time via init_array.
inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
/// Return the name of the hook variable defined in profile runtime library.
/// A reference to the variable causes the linker to link in the runtime
/// initialization module (which defines the hook variable).
inline StringRef getInstrProfRuntimeHookVarName() {
return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
}
/// Return the name of the compiler generated function that references the
/// runtime hook variable. The function is a weak global.
inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
return "__llvm_profile_runtime_user";
}
inline StringRef getInstrProfCounterBiasVarName() {
return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
}
/// Return the marker used to separate PGO names during serialization.
inline StringRef getInstrProfNameSeparator() { return "\01"; }
/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
/// for front-end (Clang, etc) instrumentation.
/// Return the modified name for function \c F suitable to be
/// used the key for profile lookup. Variable \c InLTO indicates if this
/// is called in LTO optimization passes.
std::string getPGOFuncName(const Function &F, bool InLTO = false,
uint64_t Version = INSTR_PROF_INDEX_VERSION);
/// Return the modified name for a function suitable to be
/// used the key for profile lookup. The function's original
/// name is \c RawFuncName and has linkage of type \c Linkage.
/// The function is defined in module \c FileName.
std::string getPGOFuncName(StringRef RawFuncName,
GlobalValue::LinkageTypes Linkage,
StringRef FileName,
uint64_t Version = INSTR_PROF_INDEX_VERSION);
/// \return the modified name for function \c F suitable to be
/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
/// called from LTO optimization passes.
std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
/// \return the filename and the function name parsed from the output of
/// \c getIRPGOFuncName()
std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName);
/// Return the name of the global variable used to store a function
/// name in PGO instrumentation. \c FuncName is the IRPGO function name
/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO
/// function name (returned by \c getPGOFuncName) for front-end instrumentation.
std::string getPGOFuncNameVarName(StringRef FuncName,
GlobalValue::LinkageTypes Linkage);
/// Create and return the global variable for function name used in PGO
/// instrumentation. \c FuncName is the IRPGO function name (returned by
/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
/// (returned by \c getPGOFuncName) for front-end instrumentation.
GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
/// Create and return the global variable for function name used in PGO
/// instrumentation. \c FuncName is the IRPGO function name (returned by
/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
/// (returned by \c getPGOFuncName) for front-end instrumentation.
GlobalVariable *createPGOFuncNameVar(Module &M,
GlobalValue::LinkageTypes Linkage,
StringRef PGOFuncName);
/// Return the initializer in string of the PGO name var \c NameVar.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
/// Given a PGO function name, remove the filename prefix and return
/// the original (static) function name.
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
StringRef FileName = "<unknown>");
/// Given a vector of strings (names of global objects like functions or,
/// virtual tables) \c NameStrs, the method generates a combined string \c
/// Result that is ready to be serialized. The \c Result string is comprised of
/// three fields: The first field is the length of the uncompressed strings, and
/// the the second field is the length of the zlib-compressed string. Both
/// fields are encoded in ULEB128. If \c doCompress is false, the
/// third field is the uncompressed strings; otherwise it is the
/// compressed string. When the string compression is off, the
/// second field will have value zero.
Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
bool doCompression, std::string &Result);
/// Produce \c Result string with the same format described above. The input
/// is vector of PGO function name variables that are referenced.
/// The global variable element in 'NameVars' is a string containing the pgo
/// name of a function. See `createPGOFuncNameVar` that creates these global
/// variables.
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
std::string &Result, bool doCompression = true);
Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
std::string &Result, bool doCompression);
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);
/// Check if we can safely rename this Comdat function. Instances of the same
/// comdat function may have different control flows thus can not share the
/// same counter variable.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
enum InstrProfValueKind : uint32_t {
#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
#include "llvm/ProfileData/InstrProfData.inc"
};
/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
/// and annotate the instruction \p Inst with the value profile meta data.
/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
void annotateValueSite(Module &M, Instruction &Inst,
const InstrProfRecord &InstrProfR,
InstrProfValueKind ValueKind, uint32_t SiteIndx,
uint32_t MaxMDCount = 3);
/// Same as the above interface but using an ArrayRef, as well as \p Sum.
/// This function will not annotate !prof metadata on the instruction if the
/// referenced array is empty.
void annotateValueSite(Module &M, Instruction &Inst,
ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
InstrProfValueKind ValueKind, uint32_t MaxMDCount);
// TODO: Unify metadata name 'PGOFuncName' and 'PGOName', by supporting read
// of this metadata for backward compatibility and generating 'PGOName' only.
/// Extract the value profile data from \p Inst and returns them if \p Inst is
/// annotated with value profile data. Returns an empty vector otherwise.
SmallVector<InstrProfValueData, 4>
getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
uint32_t MaxNumValueData, uint64_t &TotalC,
bool GetNoICPValue = false);
inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
inline StringRef getPGONameMetadataName() { return "PGOName"; }
/// Return the PGOFuncName meta data associated with a function.
MDNode *getPGOFuncNameMetadata(const Function &F);
std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
/// Create the PGOFuncName meta data if PGOFuncName is different from
/// function's raw name. This should only apply to internal linkage functions
/// declared by users only.
/// TODO: Update all callers to 'createPGONameMetadata' and deprecate this
/// function.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
/// Create the PGOName metadata if a global object's PGO name is different from
/// its mangled name. This should apply to local-linkage global objects only.
void createPGONameMetadata(GlobalObject &GO, StringRef PGOName);
/// Check if we can use Comdat for profile variables. This will eliminate
/// the duplicated profile variables for Comdat functions.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind {
Unknown = 0x0,
// A frontend clang profile, incompatible with other attrs.
FrontendInstrumentation = 0x1,
// An IR-level profile (default when -fprofile-generate is used).
IRInstrumentation = 0x2,
// A profile with entry basic block instrumentation.
FunctionEntryInstrumentation = 0x4,
// A context sensitive IR-level profile.
ContextSensitive = 0x8,
// Use single byte probes for coverage.
SingleByteCoverage = 0x10,
// Only instrument the function entry basic block.
FunctionEntryOnly = 0x20,
// A memory profile collected using -fprofile=memory.
MemProf = 0x40,
// A temporal profile.
TemporalProfile = 0x80,
LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/TemporalProfile)
};
const std::error_category &instrprof_category();
enum class instrprof_error {
success = 0,
eof,
unrecognized_format,
bad_magic,
bad_header,
unsupported_version,
unsupported_hash_type,
too_large,
truncated,
malformed,
missing_correlation_info,
unexpected_correlation_info,
unable_to_correlate_profile,
unknown_function,
invalid_prof,
hash_mismatch,
count_mismatch,
bitmap_mismatch,
counter_overflow,
value_site_count_mismatch,
compress_failed,
uncompress_failed,
empty_raw_profile,
zlib_unavailable,
raw_profile_version_mismatch,
counter_value_too_large,
};
/// An ordered list of functions identified by their NameRef found in
/// INSTR_PROF_DATA
struct TemporalProfTraceTy {
std::vector<uint64_t> FunctionNameRefs;
uint64_t Weight;
TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {},
uint64_t Weight = 1)
: FunctionNameRefs(Trace), Weight(Weight) {}
/// Use a set of temporal profile traces to create a list of balanced
/// partitioning function nodes used by BalancedPartitioning to generate a
/// function order that reduces page faults during startup
static void createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces,
std::vector<BPFunctionNode> &Nodes,
bool RemoveOutlierUNs = true);
};
inline std::error_code make_error_code(instrprof_error E) {
return std::error_code(static_cast<int>(E), instrprof_category());
}
class InstrProfError : public ErrorInfo<InstrProfError> {
public:
InstrProfError(instrprof_error Err, const Twine &ErrStr = Twine())
: Err(Err), Msg(ErrStr.str()) {
assert(Err != instrprof_error::success && "Not an error");
}
std::string message() const override;
void log(raw_ostream &OS) const override { OS << message(); }
std::error_code convertToErrorCode() const override {
return make_error_code(Err);
}
instrprof_error get() const { return Err; }
const std::string &getMessage() const { return Msg; }
/// Consume an Error and return the raw enum value contained within it, and
/// the optional error message. The Error must either be a success value, or
/// contain a single InstrProfError.
static std::pair<instrprof_error, std::string> take(Error E) {
auto Err = instrprof_error::success;
std::string Msg = "";
handleAllErrors(std::move(E), [&Err, &Msg](const InstrProfError &IPE) {
assert(Err == instrprof_error::success && "Multiple errors encountered");
Err = IPE.get();
Msg = IPE.getMessage();
});
return {Err, Msg};
}
static char ID;
private:
instrprof_error Err;
std::string Msg;
};
namespace object {
class SectionRef;
} // end namespace object
namespace IndexedInstrProf {
uint64_t ComputeHash(StringRef K);
} // end namespace IndexedInstrProf
/// A symbol table used for function [IR]PGO name look-up with keys
/// (such as pointers, md5hash values) to the function. A function's
/// [IR]PGO name or name's md5hash are used in retrieving the profile
/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName
/// methods for details how [IR]PGO name is formed.
class InstrProfSymtab {
public:
using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
private:
using AddrIntervalMap =
IntervalMap<uint64_t, uint64_t, 4, IntervalMapHalfOpenInfo<uint64_t>>;
StringRef Data;
uint64_t Address = 0;
// Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
// going to be sorted) has unique MD5 keys in the first place.
StringSet<> NameTab;
// Records the unique virtual table names. This is used by InstrProfWriter to
// write out an on-disk chained hash table of virtual table names.
// InstrProfWriter stores per function profile data (keyed by function names)
// so it doesn't use a StringSet for function names.
StringSet<> VTableNames;
// A map from MD5 keys to function name strings.
std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
// A map from MD5 keys to function define. We only populate this map
// when build the Symtab from a Module.
std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
// A map from MD5 to the global variable. This map is only populated when
// building the symtab from a module. Use separate container instances for
// `MD5FuncMap` and `MD5VTableMap`.
// TODO: Unify the container type and the lambda function 'mapName' inside
// add{Func,VTable}WithName.
DenseMap<uint64_t, GlobalVariable *> MD5VTableMap;
// A map from function runtime address to function name MD5 hash.
// This map is only populated and used by raw instr profile reader.
AddrHashMap AddrToMD5Map;
AddrIntervalMap::Allocator VTableAddrMapAllocator;
// This map is only populated and used by raw instr profile reader.
AddrIntervalMap VTableAddrMap;
bool Sorted = false;
static StringRef getExternalSymbol() { return "** External Symbol **"; }
// Returns the canonial name of the given PGOName. In a canonical name, all
// suffixes that begins with "." except ".__uniq." are stripped.
// FIXME: Unify this with `FunctionSamples::getCanonicalFnName`.
static StringRef getCanonicalName(StringRef PGOName);
// Add the function into the symbol table, by creating the following
// map entries:
// name-set = {PGOFuncName} union {getCanonicalName(PGOFuncName)}
// - In MD5NameMap: <MD5Hash(name), name> for name in name-set
// - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set
Error addFuncWithName(Function &F, StringRef PGOFuncName);
// Add the vtable into the symbol table, by creating the following
// map entries:
// name-set = {PGOName} union {getCanonicalName(PGOName)}
// - In MD5NameMap: <MD5Hash(name), name> for name in name-set
// - In MD5VTableMap: <MD5Hash(name), name> for name in name-set
Error addVTableWithName(GlobalVariable &V, StringRef PGOVTableName);
// If the symtab is created by a series of calls to \c addFuncName, \c
// finalizeSymtab needs to be called before looking up function names.
// This is required because the underlying map is a vector (for space
// efficiency) which needs to be sorted.
inline void finalizeSymtab();
public:
InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {}
// Not copyable or movable.
// Consider std::unique_ptr for move.
InstrProfSymtab(const InstrProfSymtab &) = delete;
InstrProfSymtab &operator=(const InstrProfSymtab &) = delete;
InstrProfSymtab(InstrProfSymtab &&) = delete;
InstrProfSymtab &operator=(InstrProfSymtab &&) = delete;
/// Create InstrProfSymtab from an object file section which
/// contains function PGO names. When section may contain raw
/// string data or string data in compressed form. This method
/// only initialize the symtab with reference to the data and
/// the section base address. The decompression will be delayed
/// until before it is used. See also \c create(StringRef) method.
Error create(object::SectionRef &Section);
/// \c NameStrings is a string composed of one of more sub-strings
/// encoded in the format described in \c collectPGOFuncNameStrings.
/// This method is a wrapper to \c readAndDecodeStrings method.
Error create(StringRef NameStrings);
/// Initialize symtab states with function names and vtable names. \c
/// FuncNameStrings is a string composed of one or more encoded function name
/// strings, and \c VTableNameStrings composes of one or more encoded vtable
/// names. This interface is solely used by raw profile reader.
Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
/// Initialize 'this' with the set of vtable names encoded in
/// \c CompressedVTableNames.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames);
/// This interface is used by reader of CoverageMapping test
/// format.
inline Error create(StringRef D, uint64_t BaseAddr);
/// A wrapper interface to populate the PGO symtab with functions
/// decls from module \c M. This interface is used by transformation
/// passes such as indirect function call promotion. Variable \c InLTO
/// indicates if this is called from LTO optimization passes.
Error create(Module &M, bool InLTO = false);
/// Create InstrProfSymtab from a set of names iteratable from
/// \p IterRange. This interface is used by IndexedProfReader.
template <typename NameIterRange>
Error create(const NameIterRange &IterRange);
/// Create InstrProfSymtab from a set of function names and vtable
/// names iteratable from \p IterRange. This interface is used by
/// IndexedProfReader.
template <typename FuncNameIterRange, typename VTableNameIterRange>
Error create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange);
// Map the MD5 of the symbol name to the name.
Error addSymbolName(StringRef SymbolName) {
if (SymbolName.empty())
return make_error<InstrProfError>(instrprof_error::malformed,
"symbol name is empty");
// Insert into NameTab so that MD5NameMap (a vector that will be sorted)
// won't have duplicated entries in the first place.
auto Ins = NameTab.insert(SymbolName);
if (Ins.second) {
MD5NameMap.push_back(std::make_pair(
IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey()));
Sorted = false;
}
return Error::success();
}
/// The method name is kept since there are many callers.
/// It just forwards to 'addSymbolName'.
Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
/// Adds VTableName as a known symbol, and inserts it to a map that
/// tracks all vtable names.
Error addVTableName(StringRef VTableName) {
if (Error E = addSymbolName(VTableName))
return E;
// Record VTableName. InstrProfWriter uses this set. The comment around
// class member explains why.
VTableNames.insert(VTableName);
return Error::success();
}
const StringSet<> &getVTableNames() const { return VTableNames; }
/// Map a function address to its name's MD5 hash. This interface
/// is only used by the raw profiler reader.
void mapAddress(uint64_t Addr, uint64_t MD5Val) {
AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
}
/// Map the address range (i.e., [start_address, end_address)) of a variable
/// to its names' MD5 hash. This interface is only used by the raw profile
/// reader.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
VTableAddrMap.insert(StartAddr, EndAddr, MD5Val);
}
/// Return a function's hash, or 0, if the function isn't in this SymTab.
uint64_t getFunctionHashFromAddress(uint64_t Address);
/// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
uint64_t getVTableHashFromAddress(uint64_t Address);
/// Return function's PGO name from the function name's symbol
/// address in the object file. If an error occurs, return
/// an empty string.
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
/// Return name of functions or global variables from the name's md5 hash
/// value. If not found, return an empty string.
inline StringRef getFuncOrVarName(uint64_t ValMD5Hash);
/// Just like getFuncOrVarName, except that it will return literal string
/// 'External Symbol' if the function or global variable is external to
/// this symbol table.
inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
/// True if Symbol is the value used to represent external symbols.
static bool isExternalSymbol(const StringRef &Symbol) {
return Symbol == InstrProfSymtab::getExternalSymbol();
}
/// Return function from the name's md5 hash. Return nullptr if not found.
inline Function *getFunction(uint64_t FuncMD5Hash);
/// Return the global variable corresponding to md5 hash. Return nullptr if
/// not found.
inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash);
/// Return the name section data.
inline StringRef getNameData() const { return Data; }
/// Dump the symbols in this table.
void dumpNames(raw_ostream &OS) const;
};
Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
Data = D;
Address = BaseAddr;
return Error::success();
}
template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
for (auto Name : IterRange)
if (Error E = addFuncName(Name))
return E;
finalizeSymtab();
return Error::success();
}
template <typename FuncNameIterRange, typename VTableNameIterRange>
Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
const VTableNameIterRange &VTableIterRange) {
// Iterate elements by StringRef rather than by const reference.
// StringRef is small enough, so the loop is efficient whether
// element in the range is std::string or StringRef.
for (StringRef Name : FuncIterRange)
if (Error E = addFuncName(Name))
return E;
for (StringRef VTableName : VTableIterRange)
if (Error E = addVTableName(VTableName))
return E;
finalizeSymtab();
return Error::success();
}
void InstrProfSymtab::finalizeSymtab() {
if (Sorted)
return;
llvm::sort(MD5NameMap, less_first());
llvm::sort(MD5FuncMap, less_first());
llvm::sort(AddrToMD5Map, less_first());
AddrToMD5Map.erase(llvm::unique(AddrToMD5Map), AddrToMD5Map.end());
Sorted = true;
}
StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) {
StringRef ret = getFuncOrVarName(MD5Hash);
if (ret.empty())
return InstrProfSymtab::getExternalSymbol();
return ret;
}
StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
finalizeSymtab();
auto Result = llvm::lower_bound(MD5NameMap, MD5Hash,
[](const std::pair<uint64_t, StringRef> &LHS,
uint64_t RHS) { return LHS.first < RHS; });
if (Result != MD5NameMap.end() && Result->first == MD5Hash)
return Result->second;
return StringRef();
}
Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
finalizeSymtab();
auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
[](const std::pair<uint64_t, Function *> &LHS,
uint64_t RHS) { return LHS.first < RHS; });
if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
return Result->second;
return nullptr;
}
GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) {
return MD5VTableMap.lookup(MD5Hash);
}
// To store the sums of profile count values, or the percentage of
// the sums of the total count values.
struct CountSumOrPercent {
uint64_t NumEntries = 0;
double CountSum = 0.0f;
std::array<double, IPVK_Last - IPVK_First + 1> ValueCounts = {};
CountSumOrPercent() = default;
void reset() {
NumEntries = 0;
CountSum = 0.0f;
ValueCounts.fill(0.0f);
}
};
// Function level or program level overlap information.
struct OverlapStats {
enum OverlapStatsLevel { ProgramLevel, FunctionLevel };
// Sum of the total count values for the base profile.
CountSumOrPercent Base;
// Sum of the total count values for the test profile.
CountSumOrPercent Test;
// Overlap lap score. Should be in range of [0.0f to 1.0f].
CountSumOrPercent Overlap;
CountSumOrPercent Mismatch;
CountSumOrPercent Unique;
OverlapStatsLevel Level;
const std::string *BaseFilename = nullptr;
const std::string *TestFilename = nullptr;
StringRef FuncName;
uint64_t FuncHash = 0;
bool Valid = false;
OverlapStats(OverlapStatsLevel L = ProgramLevel) : Level(L) {}
void dump(raw_fd_ostream &OS) const;
void setFuncInfo(StringRef Name, uint64_t Hash) {
FuncName = Name;
FuncHash = Hash;
}
Error accumulateCounts(const std::string &BaseFilename,
const std::string &TestFilename, bool IsCS);
void addOneMismatch(const CountSumOrPercent &MismatchFunc);
void addOneUnique(const CountSumOrPercent &UniqueFunc);
static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
double Sum2) {
if (Sum1 < 1.0f || Sum2 < 1.0f)
return 0.0f;
return std::min(Val1 / Sum1, Val2 / Sum2);
}
};
// This is used to filter the functions whose overlap information
// to be output.
struct OverlapFuncFilters {
uint64_t ValueCutoff;
const std::string NameFilter;
};
struct InstrProfValueSiteRecord {
/// Value profiling data pairs at a given value site.
std::vector<InstrProfValueData> ValueData;
InstrProfValueSiteRecord() = default;
InstrProfValueSiteRecord(std::vector<InstrProfValueData> &&VD)
: ValueData(VD) {}
/// Sort ValueData ascending by Value
void sortByTargetValues() {
llvm::sort(ValueData,
[](const InstrProfValueData &L, const InstrProfValueData &R) {
return L.Value < R.Value;
});
}
/// Sort ValueData Descending by Count
inline void sortByCount();
/// Merge data from another InstrProfValueSiteRecord
/// Optionally scale merged counts by \p Weight.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
function_ref<void(instrprof_error)> Warn);
/// Scale up value profile data counts by N (Numerator) / D (Denominator).
void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
/// Compute the overlap b/w this record and Input record.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
};
/// Profiling information for a single function.
struct InstrProfRecord {
std::vector<uint64_t> Counts;
std::vector<uint8_t> BitmapBytes;
InstrProfRecord() = default;
InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
InstrProfRecord(std::vector<uint64_t> Counts,
std::vector<uint8_t> BitmapBytes)
: Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {}
InstrProfRecord(InstrProfRecord &&) = default;
InstrProfRecord(const InstrProfRecord &RHS)
: Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes),
ValueData(RHS.ValueData
? std::make_unique<ValueProfData>(*RHS.ValueData)
: nullptr) {}
InstrProfRecord &operator=(InstrProfRecord &&) = default;
InstrProfRecord &operator=(const InstrProfRecord &RHS) {
Counts = RHS.Counts;
BitmapBytes = RHS.BitmapBytes;
if (!RHS.ValueData) {
ValueData = nullptr;
return *this;
}
if (!ValueData)
ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);
else
*ValueData = *RHS.ValueData;
return *this;
}
/// Return the number of value profile kinds with non-zero number
/// of profile sites.
inline uint32_t getNumValueKinds() const;
/// Return the number of instrumented sites for ValueKind.
inline uint32_t getNumValueSites(uint32_t ValueKind) const;
/// Return the total number of ValueData for ValueKind.
inline uint32_t getNumValueData(uint32_t ValueKind) const;
/// Return the array of profiled values at \p Site.
inline ArrayRef<InstrProfValueData> getValueArrayForSite(uint32_t ValueKind,
uint32_t Site) const;
/// Reserve space for NumValueSites sites.
inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
/// Add ValueData for ValueKind at value Site. We do not support adding sites
/// out of order. Site must go up from 0 one by one.
void addValueData(uint32_t ValueKind, uint32_t Site,
ArrayRef<InstrProfValueData> VData,
InstrProfSymtab *SymTab);
/// Merge the counts in \p Other into this one.
/// Optionally scale merged counts by \p Weight.
void merge(InstrProfRecord &Other, uint64_t Weight,
function_ref<void(instrprof_error)> Warn);
/// Scale up profile counts (including value profile data) by
/// a factor of (N / D).
void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn);
/// Sort value profile data (per site) by count.
void sortValueData() {
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
for (auto &SR : getValueSitesForKind(Kind))
SR.sortByCount();
}
/// Clear value data entries and edge counters.
void Clear() {
Counts.clear();
clearValueData();
}
/// Clear value data entries
void clearValueData() { ValueData = nullptr; }
/// Compute the sums of all counts and store in Sum.
void accumulateCounts(CountSumOrPercent &Sum) const;
/// Compute the overlap b/w this IntrprofRecord and Other.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
/// Compute the overlap of value profile counts.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
OverlapStats &Overlap,
OverlapStats &FuncLevelOverlap);
enum CountPseudoKind {
NotPseudo = 0,
PseudoHot,
PseudoWarm,
};
enum PseudoCountVal {
HotFunctionVal = -1,
WarmFunctionVal = -2,
};
CountPseudoKind getCountPseudoKind() const {
uint64_t FirstCount = Counts[0];
if (FirstCount == (uint64_t)HotFunctionVal)
return PseudoHot;
if (FirstCount == (uint64_t)WarmFunctionVal)
return PseudoWarm;
return NotPseudo;
}
void setPseudoCount(CountPseudoKind Kind) {
if (Kind == PseudoHot)
Counts[0] = (uint64_t)HotFunctionVal;
else if (Kind == PseudoWarm)
Counts[0] = (uint64_t)WarmFunctionVal;
}
private:
using ValueProfData = std::array<std::vector<InstrProfValueSiteRecord>,
IPVK_Last - IPVK_First + 1>;
std::unique_ptr<ValueProfData> ValueData;
MutableArrayRef<InstrProfValueSiteRecord>
getValueSitesForKind(uint32_t ValueKind) {
// Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
// implemented in LLVM) to call the const overload of this function, then
// cast away the constness from the result.
auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
ValueKind);
return MutableArrayRef(
const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
}
ArrayRef<InstrProfValueSiteRecord>
getValueSitesForKind(uint32_t ValueKind) const {
if (!ValueData)
return std::nullopt;
assert(IPVK_First <= ValueKind && ValueKind <= IPVK_Last &&
"Unknown value kind!");
return (*ValueData)[ValueKind - IPVK_First];
}
std::vector<InstrProfValueSiteRecord> &
getOrCreateValueSitesForKind(uint32_t ValueKind) {
if (!ValueData)
ValueData = std::make_unique<ValueProfData>();
assert(IPVK_First <= ValueKind && ValueKind <= IPVK_Last &&
"Unknown value kind!");
return (*ValueData)[ValueKind - IPVK_First];
}
// Map indirect call target name hash to name string.
uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
InstrProfSymtab *SymTab);
// Merge Value Profile data from Src record to this record for ValueKind.
// Scale merged value counts by \p Weight.
void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
uint64_t Weight,
function_ref<void(instrprof_error)> Warn);
// Scale up value profile data count by N (Numerator) / D (Denominator).
void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
function_ref<void(instrprof_error)> Warn);
};
struct NamedInstrProfRecord : InstrProfRecord {
StringRef Name;
uint64_t Hash;
// We reserve this bit as the flag for context sensitive profile record.
static const int CS_FLAG_IN_FUNC_HASH = 60;
NamedInstrProfRecord() = default;
NamedInstrProfRecord(StringRef Name, uint64_t Hash,
std::vector<uint64_t> Counts)
: InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
NamedInstrProfRecord(StringRef Name, uint64_t Hash,
std::vector<uint64_t> Counts,
std::vector<uint8_t> BitmapBytes)
: InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name),
Hash(Hash) {}
static bool hasCSFlagInHash(uint64_t FuncHash) {
return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
}
static void setCSFlagInHash(uint64_t &FuncHash) {
FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
}
};
uint32_t InstrProfRecord::getNumValueKinds() const {
uint32_t NumValueKinds = 0;
for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
NumValueKinds += !(getValueSitesForKind(Kind).empty());
return NumValueKinds;
}
uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
uint32_t N = 0;
for (const auto &SR : getValueSitesForKind(ValueKind))
N += SR.ValueData.size();
return N;
}
uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
return getValueSitesForKind(ValueKind).size();
}
ArrayRef<InstrProfValueData>
InstrProfRecord::getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const {
return getValueSitesForKind(ValueKind)[Site].ValueData;
}
void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
if (!NumValueSites)
return;
getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
}
// Include definitions for value profile data
#define INSTR_PROF_VALUE_PROF_DATA
#include "llvm/ProfileData/InstrProfData.inc"
void InstrProfValueSiteRecord::sortByCount() {
llvm::stable_sort(
ValueData, [](const InstrProfValueData &L, const InstrProfValueData &R) {
return L.Count > R.Count;
});
// Now truncate
size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
if (ValueData.size() > max_s)
ValueData.resize(max_s);
}
namespace IndexedInstrProf {
enum class HashT : uint32_t {
MD5,
Last = MD5
};
inline uint64_t ComputeHash(HashT Type, StringRef K) {
switch (Type) {
case HashT::MD5:
return MD5Hash(K);
}
llvm_unreachable("Unhandled hash type");
}
const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
enum ProfVersion {
// Version 1 is the first version. In this version, the value of
// a key/value pair can only include profile data of a single function.
// Due to this restriction, the number of block counters for a given
// function is not recorded but derived from the length of the value.
Version1 = 1,
// The version 2 format supports recording profile data of multiple
// functions which share the same key in one value field. To support this,
// the number block counters is recorded as an uint64_t field right after the
// function structural hash.
Version2 = 2,
// Version 3 supports value profile data. The value profile data is expected
// to follow the block counter profile data.
Version3 = 3,
// In this version, profile summary data \c IndexedInstrProf::Summary is
// stored after the profile header.
Version4 = 4,
// In this version, the frontend PGO stable hash algorithm defaults to V2.
Version5 = 5,
// In this version, the frontend PGO stable hash algorithm got fixed and
// may produce hashes different from Version5.
Version6 = 6,
// An additional counter is added around logical operators.
Version7 = 7,
// An additional (optional) memory profile type is added.
Version8 = 8,
// Binary ids are added.
Version9 = 9,
// An additional (optional) temporal profile traces section is added.
Version10 = 10,
// An additional field is used for bitmap bytes.
Version11 = 11,
// VTable profiling, decision record and bitmap are modified for mcdc.
Version12 = 12,
// The current version is 12.
CurrentVersion = INSTR_PROF_INDEX_VERSION
};
const uint64_t Version = ProfVersion::CurrentVersion;
const HashT HashType = HashT::MD5;
inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); }
// This structure defines the file header of the LLVM profile
// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst
// as appropriate when updating the indexed profile format.
struct Header {
uint64_t Magic = IndexedInstrProf::Magic;
// The lower 32 bits specify the version of the indexed profile.
// The most significant 32 bits are reserved to specify the variant types of
// the profile.
uint64_t Version = 0;
uint64_t Unused = 0; // Becomes unused since version 4
uint64_t HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
// This field records the offset of this hash table's metadata (i.e., the
// number of buckets and entries), which follows right after the payload of
// the entire hash table.
uint64_t HashOffset = 0;
uint64_t MemProfOffset = 0;
uint64_t BinaryIdOffset = 0;
uint64_t TemporalProfTracesOffset = 0;
uint64_t VTableNamesOffset = 0;
// New fields should only be added at the end to ensure that the size
// computation is correct. The methods below need to be updated to ensure that
// the new field is read correctly.
// Reads a header struct from the buffer. Header fields are in machine native
// endianness.
static Expected<Header> readFromBuffer(const unsigned char *Buffer);
// Returns the size of the header in bytes for all valid fields based on the
// version. I.e a older version header will return a smaller size.
size_t size() const;
// Return the indexed profile version, i.e., the least significant 32 bits
// in Header.Version.
uint64_t getIndexedProfileVersion() const;
};
// Profile summary data recorded in the profile data file in indexed
// format. It is introduced in version 4. The summary data follows
// right after the profile file header.
struct Summary {
struct Entry {
uint64_t Cutoff; ///< The required percentile of total execution count.
uint64_t
MinBlockCount; ///< The minimum execution count for this percentile.
uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
};
// The field kind enumerator to assigned value mapping should remain
// unchanged when a new kind is added or an old kind gets deleted in
// the future.
enum SummaryFieldKind {
/// The total number of functions instrumented.
TotalNumFunctions = 0,
/// Total number of instrumented blocks/edges.
TotalNumBlocks = 1,
/// The maximal execution count among all functions.
/// This field does not exist for profile data from IR based
/// instrumentation.
MaxFunctionCount = 2,
/// Max block count of the program.
MaxBlockCount = 3,
/// Max internal block count of the program (excluding entry blocks).
MaxInternalBlockCount = 4,
/// The sum of all instrumented block counts.
TotalBlockCount = 5,
NumKinds = TotalBlockCount + 1
};
// The number of summmary fields following the summary header.
uint64_t NumSummaryFields;
// The number of Cutoff Entries (Summary::Entry) following summary fields.
uint64_t NumCutoffEntries;
Summary() = delete;
Summary(uint32_t Size) { memset(this, 0, Size); }
void operator delete(void *ptr) { ::operator delete(ptr); }
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) {
return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
NumSumFields * sizeof(uint64_t);
}
const uint64_t *getSummaryDataBase() const {
return reinterpret_cast<const uint64_t *>(this + 1);
}
uint64_t *getSummaryDataBase() {
return reinterpret_cast<uint64_t *>(this + 1);
}
const Entry *getCutoffEntryBase() const {
return reinterpret_cast<const Entry *>(
&getSummaryDataBase()[NumSummaryFields]);
}
Entry *getCutoffEntryBase() {
return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
}
uint64_t get(SummaryFieldKind K) const {
return getSummaryDataBase()[K];
}
void set(SummaryFieldKind K, uint64_t V) {
getSummaryDataBase()[K] = V;
}
const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
void setEntry(uint32_t I, const ProfileSummaryEntry &E) {
Entry &ER = getCutoffEntryBase()[I];
ER.Cutoff = E.Cutoff;
ER.MinBlockCount = E.MinCount;
ER.NumBlocks = E.NumCounts;
}
};
inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
return std::unique_ptr<Summary>(new (::operator new(TotalSize))
Summary(TotalSize));
}
} // end namespace IndexedInstrProf
namespace RawInstrProf {
// Version 1: First version
// Version 2: Added value profile data section. Per-function control data
// struct has more fields to describe value profile information.
// Version 3: Compressed name section support. Function PGO name reference
// from control data struct is changed from raw pointer to Name's MD5 value.
// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
// raw header.
// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
// sensitive records.
// Version 6: Added binary id.
// Version 7: Reorder binary id and include version in signature.
// Version 8: Use relative counter pointer.
// Version 9: Added relative bitmap bytes pointer and count used by MC/DC.
// Version 10: Added vtable, a new type of value profile data.
const uint64_t Version = INSTR_PROF_RAW_VERSION;
template <class IntPtrT> inline uint64_t getMagic();
template <> inline uint64_t getMagic<uint64_t>() {
return INSTR_PROF_RAW_MAGIC_64;
}
template <> inline uint64_t getMagic<uint32_t>() {
return INSTR_PROF_RAW_MAGIC_32;
}
// Per-function profile data header/control structure.
// The definition should match the structure defined in
// compiler-rt/lib/profile/InstrProfiling.h.
// It should also match the synthesized type in
// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
template <class IntPtrT> struct alignas(8) ProfileData {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};
template <class IntPtrT> struct alignas(8) VTableProfileData {
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};
// File header structure of the LLVM profile data in raw format.
// The definition should match the header referenced in
// compiler-rt/lib/profile/InstrProfilingFile.c and
// InstrProfilingBuffer.c.
struct Header {
#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
#include "llvm/ProfileData/InstrProfData.inc"
};
} // end namespace RawInstrProf
// Create the variable for the profile file name.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
// Whether to compress function names in profile records, and filenames in
// code coverage mappings. Used by the Instrumentation library and unit tests.
extern cl::opt<bool> DoInstrProfNameCompression;
} // end namespace llvm
#endif // LLVM_PROFILEDATA_INSTRPROF_H
|