From af9e55068c2e38b1f7afc7f7374a5a53c59b2001 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Wed, 12 Jul 2023 17:07:34 -0400 Subject: [PATCH] debuginfo: add compiler option to allow compressed debuginfo sections LLVM already supports emitting compressed debuginfo. In debuginfo=full builds, the debug section is often a large amount of data, and it typically compresses very well (3x is not unreasonable.) We add a new knob to allow debuginfo to be compressed when the matching LLVM functionality is present. Like clang, if a known-but-disabled compression mechanism is requested, we disable compression and emit uncompressed debuginfo sections. The API is different enough on older LLVMs we just pretend the support is missing on LLVM older than 16. --- compiler/rustc_codegen_llvm/messages.ftl | 2 + compiler/rustc_codegen_llvm/src/back/write.rs | 23 ++++++++++- compiler/rustc_codegen_llvm/src/errors.rs | 6 +++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 +++ .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 11 +++++ .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 16 ++++++++ compiler/rustc_session/src/config.rs | 40 ++++++++++++++++--- compiler/rustc_session/src/options.rs | 19 ++++++++- tests/run-make/compressed-debuginfo/Makefile | 17 ++++++++ tests/run-make/compressed-debuginfo/foo.rs | 3 ++ 10 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 tests/run-make/compressed-debuginfo/Makefile create mode 100644 tests/run-make/compressed-debuginfo/foo.rs diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index aed4a8f3c85..ddaff36f24b 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -83,6 +83,8 @@ codegen_llvm_unknown_ctarget_feature_prefix = unknown feature specified for `-Ctarget-feature`: `{$feature}` .note = features must begin with a `+` to enable or `-` to disable it +codegen_llvm_unknown_debuginfo_compression = unknown debuginfo compression algorithm {$algorithm} - will fall back to uncompressed debuginfo + codegen_llvm_write_bytecode = failed to write bytecode to {$path}: {$err} codegen_llvm_write_ir = failed to write LLVM IR to {$path} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index de6541635cf..1f394a7335c 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,13 +5,17 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::errors::{ - CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode, + CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, + WithLlvmError, WriteBytecode, }; use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; use crate::ModuleLlvm; +use llvm::{ + LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, +}; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, @@ -234,6 +238,22 @@ pub fn target_machine_factory( args_cstr_buff }; + let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); + match sess.opts.debuginfo_compression { + rustc_session::config::DebugInfoCompression::Zlib => { + if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zlib" }); + } + } + rustc_session::config::DebugInfoCompression::Zstd => { + if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zstd" }); + } + } + rustc_session::config::DebugInfoCompression::None => {} + }; + let debuginfo_compression = SmallCStr::new(&debuginfo_compression); + Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; @@ -259,6 +279,7 @@ pub fn target_machine_factory( relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), + debuginfo_compression.as_ptr(), force_emulated_tls, args_cstr_buff.as_ptr() as *const c_char, args_cstr_buff.len(), diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index fced6d504d2..264c273ba30 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -226,3 +226,9 @@ pub(crate) struct WriteBytecode<'a> { pub(crate) struct CopyBitcode { pub err: std::io::Error, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_debuginfo_compression)] +pub struct UnknownCompression { + pub algorithm: &'static str, +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 8e96410deaf..2ebfdae39e8 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2131,6 +2131,7 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, + DebugInfoCompression: *const c_char, ForceEmulatedTls: bool, ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, @@ -2366,6 +2367,10 @@ extern "C" { pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool; + pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; + + pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + pub fn LLVMRustGetSymbols( buf_ptr: *const u8, buf_len: usize, diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 204f833de92..87ef74b9264 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -407,6 +407,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( bool RelaxELFRelocations, bool UseInitArray, const char *SplitDwarfFile, + const char *DebugInfoCompression, bool ForceEmulatedTls, const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { @@ -438,6 +439,16 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( if (SplitDwarfFile) { Options.MCOptions.SplitDwarfFile = SplitDwarfFile; } +#if LLVM_VERSION_GE(16, 0) + if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zlib; + } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zstd; + } else if (!strcmp("none", DebugInfoCompression)) { + Options.CompressDebugSections = DebugCompressionType::None; + } +#endif + Options.RelaxELFRelocations = RelaxELFRelocations; Options.UseInitArray = UseInitArray; diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 70cdf3d6d23..4390486b0de 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -2044,3 +2044,19 @@ extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { } return false; } + +extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zlib::isAvailable(); +#else + return false; +#endif +} + +extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zstd::isAvailable(); +#else + return false; +#endif +} diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 302e4e55d59..86a03124fe1 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -381,6 +381,24 @@ pub enum DebugInfo { Full, } +#[derive(Clone, Copy, Debug, PartialEq, Hash)] +pub enum DebugInfoCompression { + None, + Zlib, + Zstd, +} + +impl ToString for DebugInfoCompression { + fn to_string(&self) -> String { + match self { + DebugInfoCompression::None => "none", + DebugInfoCompression::Zlib => "zlib", + DebugInfoCompression::Zstd => "zstd", + } + .to_owned() + } +} + /// Split debug-information is enabled by `-C split-debuginfo`, this enum is only used if split /// debug-information is enabled (in either `Packed` or `Unpacked` modes), and the platform /// uses DWARF for debug-information. @@ -1015,6 +1033,7 @@ impl Default for Options { crate_types: Vec::new(), optimize: OptLevel::No, debuginfo: DebugInfo::None, + debuginfo_compression: DebugInfoCompression::None, lint_opts: Vec::new(), lint_cap: None, describe_lints: false, @@ -2277,6 +2296,13 @@ fn select_debuginfo(matches: &getopts::Matches, cg: &CodegenOptions) -> DebugInf if max_g > max_c { DebugInfo::Full } else { cg.debuginfo } } +fn select_debuginfo_compression( + _handler: &EarlyErrorHandler, + unstable_opts: &UnstableOptions, +) -> DebugInfoCompression { + unstable_opts.debuginfo_compression +} + pub(crate) fn parse_assert_incr_state( handler: &EarlyErrorHandler, opt_assertion: &Option, @@ -2752,6 +2778,8 @@ pub fn build_session_options( // for more details. let debug_assertions = cg.debug_assertions.unwrap_or(opt_level == OptLevel::No); let debuginfo = select_debuginfo(matches, &cg); + let debuginfo_compression: DebugInfoCompression = + select_debuginfo_compression(handler, &unstable_opts); let mut search_paths = vec![]; for s in &matches.opt_strs("L") { @@ -2828,6 +2856,7 @@ pub fn build_session_options( crate_types, optimize: opt_level, debuginfo, + debuginfo_compression, lint_opts, lint_cap, describe_lints, @@ -3113,11 +3142,11 @@ impl PpMode { /// how the hash should be calculated when adding a new command-line argument. pub(crate) mod dep_tracking { use super::{ - BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, ErrorOutputType, - InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, LocationDetail, LtoCli, - OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Passes, ResolveDocLinks, - SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, - TraitSolver, TrimmedDefPaths, + BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression, + ErrorOutputType, InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, + LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, + Passes, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, + SymbolManglingVersion, TraitSolver, TrimmedDefPaths, }; use crate::lint; use crate::options::WasiExecModel; @@ -3195,6 +3224,7 @@ pub(crate) mod dep_tracking { OptLevel, LtoCli, DebugInfo, + DebugInfoCompression, UnstableFeatures, NativeLib, NativeLibKind, diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 40099de707b..5ddefb2260e 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -139,6 +139,7 @@ top_level_options!( /// can influence whether overflow checks are done or not. debug_assertions: bool [TRACKED], debuginfo: DebugInfo [TRACKED], + debuginfo_compression: DebugInfoCompression [TRACKED], lint_opts: Vec<(String, lint::Level)> [TRACKED_NO_CRATE_HASH], lint_cap: Option [TRACKED_NO_CRATE_HASH], describe_lints: bool [UNTRACKED], @@ -376,6 +377,7 @@ mod desc { "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`"; pub const parse_cfprotection: &str = "`none`|`no`|`n` (default), `branch`, `return`, or `full`|`yes`|`y` (equivalent to `branch` and `return`)"; pub const parse_debuginfo: &str = "either an integer (0, 1, 2), `none`, `line-directives-only`, `line-tables-only`, `limited`, or `full`"; + pub const parse_debuginfo_compression: &str = "one of `none`, `zlib`, or `zstd`"; pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`"; pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of(); pub const parse_optimization_fuel: &str = "crate=integer"; @@ -782,6 +784,19 @@ mod parse { true } + pub(crate) fn parse_debuginfo_compression( + slot: &mut DebugInfoCompression, + v: Option<&str>, + ) -> bool { + match v { + Some("none") => *slot = DebugInfoCompression::None, + Some("zlib") => *slot = DebugInfoCompression::Zlib, + Some("zstd") => *slot = DebugInfoCompression::Zstd, + _ => return false, + }; + true + } + pub(crate) fn parse_linker_flavor(slot: &mut Option, v: Option<&str>) -> bool { match v.and_then(LinkerFlavorCli::from_str) { Some(lf) => *slot = Some(lf), @@ -1422,7 +1437,9 @@ options! { "inject the given attribute in the crate"), debug_info_for_profiling: bool = (false, parse_bool, [TRACKED], "emit discriminators and other data necessary for AutoFDO"), - debug_macros: bool = (false, parse_bool, [TRACKED], + debuginfo_compression: DebugInfoCompression = (DebugInfoCompression::None, parse_debuginfo_compression, [TRACKED], + "compress debug info sections (none, zlib, zstd, default: none)"), + debug_macros: bool = (false, parse_bool, [TRACKED], "emit line numbers debug info inside macros (default: no)"), deduplicate_diagnostics: bool = (true, parse_bool, [UNTRACKED], "deduplicate identical diagnostics (default: yes)"), diff --git a/tests/run-make/compressed-debuginfo/Makefile b/tests/run-make/compressed-debuginfo/Makefile new file mode 100644 index 00000000000..3c50d4df17a --- /dev/null +++ b/tests/run-make/compressed-debuginfo/Makefile @@ -0,0 +1,17 @@ +# ignore-cross-compile +include ../tools.mk + +# only-linux +# min-llvm-version: 16.0 +# +# This tests debuginfo-compression. + +all: zlib zstandard + +zlib: + $(RUSTC) --crate-name=foo --crate-type=lib --emit=obj -C debuginfo=full -Z debuginfo-compression=zlib foo.rs + test "`ld.lld --compress-debug-sections=zlib 2>&1 | sed 's/.*unknown.*zlib/missing/'`" = missing || readelf -t $(TMPDIR)/foo.o | grep -q ZLIB + +zstandard: + $(RUSTC) --crate-name=foo --crate-type=lib --emit=obj -C debuginfo=full -Z debuginfo-compression=zstd foo.rs + test "`ld.lld --compress-debug-sections=zstd 2>&1 | sed 's/.*unknown.*zstd/missing/'`" = missing || readelf -t $(TMPDIR)/foo.o | grep -q ZST diff --git a/tests/run-make/compressed-debuginfo/foo.rs b/tests/run-make/compressed-debuginfo/foo.rs new file mode 100644 index 00000000000..185ce22450c --- /dev/null +++ b/tests/run-make/compressed-debuginfo/foo.rs @@ -0,0 +1,3 @@ +pub fn foo() -> i32 { + 42 +}