diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 3da63af5ba571..734f1398a7ddd 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -414,6 +414,8 @@ class AsmPrinter : public MachineFunctionPass { void emitBBAddrMapSection(const MachineFunction &MF); + void emitFuncMapSection(const MachineFunction &MF); + void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol); virtual void emitKCFITypeId(const MachineFunction &MF); diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 57ba40f7ac26f..6fc9eaafeb09e 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -177,6 +177,9 @@ class MCContext { /// LLVM_BB_ADDR_MAP version to emit. uint8_t BBAddrMapVersion = 2; + /// LLVM_FUNC_MAP version to emit. + uint8_t FuncMapVersion = 1; + /// The file name of the log file from the environment variable /// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique /// directive is used or it is an error. @@ -656,6 +659,8 @@ class MCContext { uint8_t getBBAddrMapVersion() const { return BBAddrMapVersion; } + uint8_t getFuncMapVersion() const { return FuncMapVersion; } + /// @} /// \name Dwarf Management diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h index fb575fe721015..88e7b1d895714 100644 --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -364,6 +364,9 @@ class MCObjectFileInfo { MCSection *getBBAddrMapSection(const MCSection &TextSec) const; + MCSection *getFuncMapSection(const MCSection &TextSec, + unsigned EntrySize) const; + MCSection *getKCFITrapSection(const MCSection &TextSec) const; MCSection *getPseudoProbeSection(const MCSection &TextSec) const; diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index 4ff4ec5f1b2c4..981d0dfd31c88 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -1033,6 +1033,11 @@ struct FuncMap { uint64_t DynamicInstCount = 0; // Dynamic instruction count for this function. uint64_t getFunctionAddress() const { return FunctionAddress; } + + static unsigned getEntrySize(unsigned PointSize) { + return 1 /* version */ + PointSize /* FunctionAddress */ + + 8 /* DynamicInstCount */; + } }; } // end namespace object. diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 44b10c3ef9972..ff2c594c43667 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -147,6 +147,7 @@ enum class PGOMapFeaturesEnum { BrProb, All, }; + static cl::bits PgoAnalysisMapFeatures( "pgo-analysis-map", cl::Hidden, cl::CommaSeparated, cl::values( @@ -161,6 +162,13 @@ static cl::bits PgoAnalysisMapFeatures( "Enable extended information within the SHT_LLVM_BB_ADDR_MAP that is " "extracted from PGO related analysis.")); +static cl::opt EmitFuncMap( + "func-map", + cl::desc( + "Emit features of function address map in SHT_LLVM_FUNC_MAP " + "section(Currently only support dynamic instruction count feature)."), + cl::Hidden, cl::init(false)); + static cl::opt BBAddrMapSkipEmitBBEntries( "basic-block-address-map-skip-bb-entries", cl::desc("Skip emitting basic block entries in the SHT_LLVM_BB_ADDR_MAP " @@ -1548,6 +1556,39 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { OutStreamer->popSection(); } +void AsmPrinter::emitFuncMapSection(const MachineFunction &MF) { + if (!EmitFuncMap) + return; + + MCSection *FuncMapSection = getObjFileLowering().getFuncMapSection( + *MF.getSection(), llvm::object::FuncMap::getEntrySize(getPointerSize())); + assert(FuncMapSection && ".llvm_func_map section is not initialized."); + const MCSymbol *FunctionSymbol = getFunctionBegin(); + OutStreamer->pushSection(); + OutStreamer->switchSection(FuncMapSection); + OutStreamer->AddComment("version"); + uint8_t FuncMapVersion = OutStreamer->getContext().getFuncMapVersion(); + OutStreamer->emitInt8(FuncMapVersion); + + OutStreamer->AddComment("function address"); + OutStreamer->emitSymbolValue(FunctionSymbol, getPointerSize()); + + const MachineBlockFrequencyInfo *MBFI = + &getAnalysis().getBFI(); + uint64_t DynInstCount = 0; + for (const MachineBasicBlock &MBB : MF) { + for (const MachineInstr &MI : MBB) { + if (MI.isDebugValue() || MI.isPseudoProbe()) + continue; + DynInstCount += MBFI->getBlockProfileCount(&MBB).value_or(0); + } + } + + OutStreamer->AddComment("dynamic instruction count"); + OutStreamer->emitInt64(DynInstCount); + OutStreamer->popSection(); +} + void AsmPrinter::emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol) { MCSection *Section = @@ -2119,6 +2160,7 @@ void AsmPrinter::emitFunctionBody() { MF->getContext().reportWarning( SMLoc(), "pgo-analysis-map is enabled for function " + MF->getName() + " but it does not have labels"); + emitFuncMapSection(*MF); } // Emit sections containing instruction and function PCs. @@ -2749,7 +2791,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { F.hasFnAttribute("xray-instruction-threshold") || needFuncLabels(MF, *this) || NeedsLocalForSize || MF.getTarget().Options.EmitStackSizeSection || - MF.getTarget().Options.BBAddrMap) { + MF.getTarget().Options.BBAddrMap || EmitFuncMap) { CurrentFnBegin = createTempSymbol("func_begin"); if (NeedsLocalForSize) CurrentFnSymForSize = CurrentFnBegin; diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index 150e38a94db6a..50cf8f27f9f04 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -1120,6 +1120,24 @@ MCObjectFileInfo::getBBAddrMapSection(const MCSection &TextSec) const { cast(TextSec.getBeginSymbol())); } +MCSection *MCObjectFileInfo::getFuncMapSection(const MCSection &TextSec, + unsigned EntrySize) const { + if (Ctx->getObjectFileType() != MCContext::IsELF) + return nullptr; + + const MCSectionELF &ElfSec = static_cast(TextSec); + unsigned Flags = ELF::SHF_LINK_ORDER | ELF::SHF_MERGE; + StringRef GroupName; + if (const MCSymbol *Group = ElfSec.getGroup()) { + GroupName = Group->getName(); + Flags |= ELF::SHF_GROUP; + } + + return Ctx->getELFSection(".llvm_func_map", ELF::SHT_LLVM_FUNC_MAP, Flags, + EntrySize, GroupName, true, ElfSec.getUniqueID(), + cast(TextSec.getBeginSymbol())); +} + MCSection * MCObjectFileInfo::getKCFITrapSection(const MCSection &TextSec) const { if (Ctx->getObjectFileType() != MCContext::IsELF) diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp index b58210b3c268e..c9df8a3a8e8e4 100644 --- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp +++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp @@ -679,6 +679,8 @@ bool ELFAsmParser::parseSectionArguments(bool IsPush, SMLoc loc) { Type = ELF::SHT_LLVM_LTO; else if (TypeName == "llvm_jt_sizes") Type = ELF::SHT_LLVM_JT_SIZES; + else if (TypeName == "llvm_func_map") + Type = ELF::SHT_LLVM_FUNC_MAP; else if (TypeName.getAsInteger(0, Type)) return TokError("unknown section type"); } diff --git a/llvm/lib/MC/MCSectionELF.cpp b/llvm/lib/MC/MCSectionELF.cpp index 25e62b70b5e2a..fa1a5eb107193 100644 --- a/llvm/lib/MC/MCSectionELF.cpp +++ b/llvm/lib/MC/MCSectionELF.cpp @@ -174,6 +174,8 @@ void MCSectionELF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T, OS << "llvm_lto"; else if (Type == ELF::SHT_LLVM_JT_SIZES) OS << "llvm_jt_sizes"; + else if (Type == ELF::SHT_LLVM_FUNC_MAP) + OS << "llvm_func_map"; else OS << "0x" << Twine::utohexstr(Type); diff --git a/llvm/test/CodeGen/X86/function-address-map-dyn-inst-count.ll b/llvm/test/CodeGen/X86/function-address-map-dyn-inst-count.ll new file mode 100644 index 0000000000000..f2a2dbf96fd3c --- /dev/null +++ b/llvm/test/CodeGen/X86/function-address-map-dyn-inst-count.ll @@ -0,0 +1,108 @@ +; Test emitting dynamic instruction count feature in .llvm_func_map section. +; RUN: llc < %s -mtriple=x86_64 -function-sections -func-map | FileCheck %s + +;; Check we add SHF_LINK_ORDER for .llvm_func_map and link it with the corresponding .text sections. +; CHECK: .section .text.foo,"ax",@progbits +; CHECK-LABEL: foo: +; CHECK-NEXT: [[FOO_BEGIN:.Lfunc_begin[0-9]+]]: +; CHECK: .section .llvm_func_map,"Mo",@llvm_func_map,17,.text.foo{{$}} +; CHECK-NEXT: .byte 1 # version +; CHECK-NEXT: .quad [[FOO_BEGIN]] # function address +; CHECK-NEXT: .quad 170 # dynamic instruction count + + +; CHECK: .section .text.main,"ax",@progbits +; CHECK-LABEL: main: +; CHECK-NEXT: [[MAIN_BEGIN:.Lfunc_begin[0-9]+]]: +; CHECK: .section .llvm_func_map,"Mo",@llvm_func_map,17,.text.main{{$}} +; CHECK-NEXT: .byte 1 # version +; CHECK-NEXT: .quad [[MAIN_BEGIN]] # function address +; CHECK-NEXT: .quad 437 # dynamic instruction count + + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define i32 @foo(i32 %x) !dbg !4 !prof !6 { +entry: + #dbg_value(i32 %x, !7, !DIExpression(), !9) + call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1) + %rem = mul i32 %x, 5 + %tobool.not = icmp eq i32 %rem, 0, !dbg !10 + br i1 %tobool.not, label %if.end, label %if.then, !prof !12 + +if.then: ; preds = %entry + %inc = add i32 0, 0 + call void @llvm.pseudoprobe(i64 6699318081062747564, i64 2, i32 0, i64 -1) + #dbg_value(i32 %inc, !7, !DIExpression(), !9) + br label %if.end + +if.end: ; preds = %if.then, %entry + %x.addr.0 = phi i32 [ 0, %if.then ], [ 1, %entry ] + #dbg_value(i32 %x.addr.0, !7, !DIExpression(), !9) + ret i32 %x.addr.0 +} + +define i32 @main() #0 !dbg !13 !prof !15 { +entry: + #dbg_value(i32 0, !16, !DIExpression(), !17) + br label %while.cond + +while.cond: ; preds = %if.then, %if.else, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %if.else ], [ %inc, %if.then ] + #dbg_value(i32 %i.0, !16, !DIExpression(), !17) + %inc = add i32 %i.0, 1 + #dbg_value(i32 %inc, !16, !DIExpression(), !17) + %cmp = icmp ult i32 %i.0, 1600000 + br i1 %cmp, label %while.body, label %while.end, !prof !18 + +while.body: ; preds = %while.cond + %rem = urem i32 %inc, 11 + %tobool.not = icmp eq i32 %rem, 0 + br i1 %tobool.not, label %if.else, label %if.then, !prof !19 + +if.then: ; preds = %while.body + %call = call i32 @foo(i32 0), !dbg !20 + %0 = load volatile i32, ptr null, align 4 + br label %while.cond + +if.else: ; preds = %while.body + store i32 0, ptr null, align 4 + br label %while.cond + +while.end: ; preds = %while.cond + ret i32 0 + +; uselistorder directives + uselistorder label %while.cond, { 1, 0, 2 } + uselistorder i32 %inc, { 2, 1, 0 } +} + +attributes #0 = { "target-cpu"="x86-64" } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 20.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/home", checksumkind: CSK_MD5, checksum: "920887ee2258042655d8340f78e732e9") +!2 = !{} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!5 = distinct !DISubroutineType(types: !2) +!6 = !{!"function_entry_count", i64 20} +!7 = !DILocalVariable(name: "x", arg: 1, scope: !4, file: !1, line: 3, type: !8) +!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!9 = !DILocation(line: 0, scope: !4) +!10 = !DILocation(line: 4, column: 9, scope: !11) +!11 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 7) +!12 = !{!"branch_weights", i32 15, i32 5} +!13 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 9, type: !14, scopeLine: 9, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!14 = !DISubroutineType(types: !2) +!15 = !{!"function_entry_count", i64 1} +!16 = !DILocalVariable(name: "i", scope: !13, file: !1, line: 10, type: !8) +!17 = !DILocation(line: 0, scope: !13) +!18 = !{!"branch_weights", i32 22, i32 1} +!19 = !{!"branch_weights", i32 2, i32 20} +!20 = !DILocation(line: 12, column: 22, scope: !21) +!21 = !DILexicalBlockFile(scope: !22, file: !1, discriminator: 455082031) +!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 12, column: 9) diff --git a/llvm/test/CodeGen/X86/function-address-map-function-sections.ll b/llvm/test/CodeGen/X86/function-address-map-function-sections.ll new file mode 100644 index 0000000000000..532072a8da460 --- /dev/null +++ b/llvm/test/CodeGen/X86/function-address-map-function-sections.ll @@ -0,0 +1,40 @@ +; RUN: llc < %s -mtriple=x86_64 -function-sections -func-map | FileCheck %s + +$_Z4fooTIiET_v = comdat any + +define dso_local i32 @_Z3barv() { + ret i32 0 +} + +;; Check we add SHF_LINK_ORDER for .llvm_func_map and link it with the corresponding .text sections. +; CHECK: .section .text._Z3barv,"ax",@progbits +; CHECK-LABEL: _Z3barv: +; CHECK-NEXT: [[BAR_BEGIN:.Lfunc_begin[0-9]+]]: +; CHECK: .section .llvm_func_map,"Mo",@llvm_func_map,17,.text._Z3barv{{$}} +; CHECK-NEXT: .byte 1 # version +; CHECK-NEXT: .quad [[BAR_BEGIN]] # function address + +define dso_local i32 @_Z3foov() { + %1 = call i32 @_Z4fooTIiET_v() + ret i32 %1 +} + +; CHECK: .section .text._Z3foov,"ax",@progbits +; CHECK-LABEL: _Z3foov: +; CHECK-NEXT: [[FOO_BEGIN:.Lfunc_begin[0-9]+]]: +; CHECK: .section .llvm_func_map,"Mo",@llvm_func_map,17,.text._Z3foov{{$}} +; CHECK-NEXT: .byte 1 # version +; CHECK-NEXT: .quad [[FOO_BEGIN]] # function address + + +define linkonce_odr dso_local i32 @_Z4fooTIiET_v() comdat { + ret i32 0 +} + +;; Check we add .llvm_func_map section to a COMDAT group with the corresponding .text section if such a COMDAT exists. +; CHECK: .section .text._Z4fooTIiET_v,"axG",@progbits,_Z4fooTIiET_v,comdat +; CHECK-LABEL: _Z4fooTIiET_v: +; CHECK-NEXT: [[FOOCOMDAT_BEGIN:.Lfunc_begin[0-9]+]]: +; CHECK: .section .llvm_func_map,"MoG",@llvm_func_map,17,.text._Z4fooTIiET_v,_Z4fooTIiET_v,comdat{{$}} +; CHECK-NEXT: .byte 1 # version +; CHECK-NEXT: .quad [[FOOCOMDAT_BEGIN]] # function address diff --git a/llvm/test/MC/AsmParser/llvm_section_types.s b/llvm/test/MC/AsmParser/llvm_section_types.s index 147b1499d2b88..f3f3150ac30f1 100644 --- a/llvm/test/MC/AsmParser/llvm_section_types.s +++ b/llvm/test/MC/AsmParser/llvm_section_types.s @@ -17,6 +17,8 @@ .byte 1 .section .section8,"",@llvm_lto .byte 1 +.section .section9,"",@llvm_func_map +.byte 1 # CHECK: Name: .section1 # CHECK-NEXT: Type: SHT_LLVM_BB_ADDR_MAP @@ -34,3 +36,5 @@ # CHECK-NEXT: Type: SHT_LLVM_OFFLOADING # CHECK: Name: .section8 # CHECK-NEXT: Type: SHT_LLVM_LTO +# CHECK: Name: .section9 +# CHECK-NEXT: Type: SHT_LLVM_FUNC_MAP