diff --git a/Cargo.lock b/Cargo.lock index 51e21f6dc82..e4dcf13a84b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1965,9 +1965,9 @@ checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libdbus-sys" @@ -3990,6 +3990,7 @@ name = "rustc_metadata" version = "0.0.0" dependencies = [ "bitflags 2.6.0", + "libc", "libloading", "odht", "rustc_abi", diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index fd1126e8528..5149e3a12f2 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -777,6 +777,16 @@ fn link_natively( info!("preparing {:?} to {:?}", crate_type, out_filename); let (linker_path, flavor) = linker_and_flavor(sess); let self_contained_components = self_contained_components(sess, crate_type); + + // On AIX, we ship all libraries as .a big_af archive + // the expected format is lib.a(libname.so) for the actual + // dynamic library. So we link to a temporary .so file to be archived + // at the final out_filename location + let should_archive = crate_type != CrateType::Executable && sess.target.is_like_aix; + let archive_member = + should_archive.then(|| tmpdir.join(out_filename.file_name().unwrap()).with_extension("so")); + let temp_filename = archive_member.as_deref().unwrap_or(out_filename); + let mut cmd = linker_with_args( &linker_path, flavor, @@ -784,7 +794,7 @@ fn link_natively( archive_builder_builder, crate_type, tmpdir, - out_filename, + temp_filename, codegen_results, self_contained_components, )?; @@ -1158,6 +1168,12 @@ fn link_natively( } } + if should_archive { + let mut ab = archive_builder_builder.new_archive_builder(sess); + ab.add_file(temp_filename); + ab.build(out_filename); + } + Ok(()) } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index bdf7030f946..ba7b53321a8 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::io::Write; use std::path::Path; +use itertools::Itertools; use object::write::{self, StandardSegment, Symbol, SymbolSection}; use object::{ Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, ObjectSymbol, @@ -21,6 +22,7 @@ use rustc_middle::bug; use rustc_session::Session; use rustc_span::sym; use rustc_target::spec::{RelocModel, Target, ef_avr_arch}; +use tracing::debug; use super::apple; @@ -53,6 +55,7 @@ fn load_metadata_with( impl MetadataLoader for DefaultMetadataLoader { fn get_rlib_metadata(&self, target: &Target, path: &Path) -> Result { + debug!("getting rlib metadata for {}", path.display()); load_metadata_with(path, |data| { let archive = object::read::archive::ArchiveFile::parse(&*data) .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; @@ -77,8 +80,26 @@ impl MetadataLoader for DefaultMetadataLoader { } fn get_dylib_metadata(&self, target: &Target, path: &Path) -> Result { + debug!("getting dylib metadata for {}", path.display()); if target.is_like_aix { - load_metadata_with(path, |data| get_metadata_xcoff(path, data)) + load_metadata_with(path, |data| { + let archive = object::read::archive::ArchiveFile::parse(&*data).map_err(|e| { + format!("failed to parse aix dylib '{}': {}", path.display(), e) + })?; + + match archive.members().exactly_one() { + Ok(lib) => { + let lib = lib.map_err(|e| { + format!("failed to parse aix dylib '{}': {}", path.display(), e) + })?; + let data = lib.data(data).map_err(|e| { + format!("failed to parse aix dylib '{}': {}", path.display(), e) + })?; + get_metadata_xcoff(path, data) + } + Err(e) => Err(format!("failed to parse aix dylib '{}': {}", path.display(), e)), + } + }) } else { load_metadata_with(path, |data| search_for_section(path, data, ".rustc")) } diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index cece700b4dd..12519be9870 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] # tidy-alphabetical-start bitflags = "2.4.1" +libc = "0.2" libloading = "0.8.0" odht = { version = "0.3.1", features = ["nightly"] } rustc_abi = { path = "../rustc_abi" } diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index ca16a66763a..a611e8010be 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -540,6 +540,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { Some(cnum) } Err(err) => { + debug!("failed to resolve crate {} {:?}", name, dep_kind); let missing_core = self.maybe_resolve_crate(sym::core, CrateDepKind::Explicit, None).is_err(); err.report(self.sess, span, missing_core); @@ -588,6 +589,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { match self.load(&mut locator)? { Some(res) => (res, None), None => { + info!("falling back to loading proc_macro"); dep_kind = CrateDepKind::MacrosOnly; match self.load_proc_macro(&mut locator, path_kind, host_hash)? { Some(res) => res, @@ -599,6 +601,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { match result { (LoadResult::Previous(cnum), None) => { + info!("library for `{}` was loaded previously", name); // When `private_dep` is none, it indicates the directly dependent crate. If it is // not specified by `--extern` on command line parameters, it may be // `private-dependency` when `register_crate` is called for the first time. Then it must be updated to @@ -613,6 +616,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { Ok(cnum) } (LoadResult::Loaded(library), host_library) => { + info!("register newly loaded library for `{}`", name); self.register_crate(host_library, root, library, dep_kind, name, private_dep) } _ => panic!(), @@ -696,7 +700,25 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> { stable_crate_id: StableCrateId, ) -> Result<&'static [ProcMacro], CrateError> { let sym_name = self.sess.generate_proc_macro_decls_symbol(stable_crate_id); - Ok(unsafe { *load_symbol_from_dylib::<*const &[ProcMacro]>(path, &sym_name)? }) + debug!("trying to dlsym proc_macros {} for symbol `{}`", path.display(), sym_name); + + unsafe { + let result = load_symbol_from_dylib::<*const &[ProcMacro]>(path, &sym_name); + match result { + Ok(result) => { + debug!("loaded dlsym proc_macros {} for symbol `{}`", path.display(), sym_name); + Ok(*result) + } + Err(err) => { + debug!( + "failed to dlsym proc_macros {} for symbol `{}`", + path.display(), + sym_name + ); + Err(err.into()) + } + } + } } fn inject_panic_runtime(&mut self, krate: &ast::Crate) { @@ -1141,6 +1163,29 @@ fn format_dlopen_err(e: &(dyn std::error::Error + 'static)) -> String { e.sources().map(|e| format!(": {e}")).collect() } +fn attempt_load_dylib(path: &Path) -> Result { + #[cfg(target_os = "aix")] + if let Some(ext) = path.extension() + && ext.eq("a") + { + // On AIX, we ship all libraries as .a big_af archive + // the expected format is lib.a(libname.so) for the actual + // dynamic library + let library_name = path.file_stem().expect("expect a library name"); + let mut archive_member = OsString::from("a("); + archive_member.push(library_name); + archive_member.push(".so)"); + let new_path = path.with_extension(archive_member); + + // On AIX, we need RTLD_MEMBER to dlopen an archived shared + let flags = libc::RTLD_LAZY | libc::RTLD_LOCAL | libc::RTLD_MEMBER; + return unsafe { libloading::os::unix::Library::open(Some(&new_path), flags) } + .map(|lib| lib.into()); + } + + unsafe { libloading::Library::new(&path) } +} + // On Windows the compiler would sometimes intermittently fail to open the // proc-macro DLL with `Error::LoadLibraryExW`. It is suspected that something in the // system still holds a lock on the file, so we retry a few times before calling it @@ -1151,7 +1196,8 @@ fn load_dylib(path: &Path, max_attempts: usize) -> Result { if attempt > 0 { debug!( @@ -1165,6 +1211,7 @@ fn load_dylib(path: &Path, max_attempts: usize) -> Result { // Only try to recover from this specific error. if !matches!(err, libloading::Error::LoadLibraryExW { .. }) { + debug!("Failed to load proc-macro `{}`. Not retrying", path.display()); let err = format_dlopen_err(&err); // We include the path of the dylib in the error ourselves, so // if it's in the error, we strip it. diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index 0b53e5eeaa8..d59ec7af6ec 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -847,7 +847,10 @@ fn get_metadata_section<'p>( ))); }; match blob.check_compatibility(cfg_version) { - Ok(()) => Ok(blob), + Ok(()) => { + debug!("metadata blob read okay"); + Ok(blob) + } Err(None) => Err(MetadataError::LoadFailure(format!( "invalid metadata version found: {}", filename.display() diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 24be705f481..8e088682f92 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1457,7 +1457,7 @@ impl Step for CodegenBackend { } let mut files = files.into_iter().filter(|f| { let filename = f.file_name().unwrap().to_str().unwrap(); - is_dylib(filename) && filename.contains("rustc_codegen_") + is_dylib(f) && filename.contains("rustc_codegen_") }); let codegen_backend = match files.next() { Some(f) => f, @@ -1936,7 +1936,7 @@ impl Step for Assemble { let filename = f.file_name().into_string().unwrap(); let is_proc_macro = proc_macros.contains(&filename); - let is_dylib_or_debug = is_dylib(&filename) || is_debug_info(&filename); + let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename); // If we link statically to stdlib, do not copy the libstd dynamic library file // FIXME: Also do this for Windows once incremental post-optimization stage0 tests @@ -2089,7 +2089,7 @@ pub fn run_cargo( if filename.ends_with(".lib") || filename.ends_with(".a") || is_debug_info(&filename) - || is_dylib(&filename) + || is_dylib(Path::new(&*filename)) { // Always keep native libraries, rust dylibs and debuginfo keep = true; @@ -2189,7 +2189,7 @@ pub fn run_cargo( Some(triple) => triple.0.to_str().unwrap(), None => panic!("no output generated for {prefix:?} {extension:?}"), }; - if is_dylib(path_to_add) { + if is_dylib(Path::new(path_to_add)) { let candidate = format!("{path_to_add}.lib"); let candidate = PathBuf::from(candidate); if candidate.exists() { diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index c022285211f..0216ac81162 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -436,13 +436,10 @@ impl Step for Rustc { if libdir_relative.to_str() != Some("bin") { let libdir = builder.rustc_libdir(compiler); for entry in builder.read_dir(&libdir) { - let name = entry.file_name(); - if let Some(s) = name.to_str() { - if is_dylib(s) { - // Don't use custom libdir here because ^lib/ will be resolved again - // with installer - builder.install(&entry.path(), &image.join("lib"), 0o644); - } + if is_dylib(&entry.path()) { + // Don't use custom libdir here because ^lib/ will be resolved again + // with installer + builder.install(&entry.path(), &image.join("lib"), 0o644); } } } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 532c8f767eb..dcea9f5f7d1 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2949,8 +2949,7 @@ impl Step for RemoteCopyLibs { // Push all our dylibs to the emulator for f in t!(builder.sysroot_target_libdir(compiler, target).read_dir()) { let f = t!(f); - let name = f.file_name().into_string().unwrap(); - if helpers::is_dylib(&name) { + if helpers::is_dylib(&f.path()) { command(&tool).arg("push").arg(f.path()).run(builder); } } diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 3fb2330469a..9ca036a2afd 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -11,6 +11,7 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{env, fs, io, str}; use build_helper::util::fail; +use object::read::archive::ArchiveFile; use crate::LldMode; use crate::core::builder::Builder; @@ -53,8 +54,27 @@ pub fn exe(name: &str, target: TargetSelection) -> String { } /// Returns `true` if the file name given looks like a dynamic library. -pub fn is_dylib(name: &str) -> bool { - name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") +pub fn is_dylib(path: &Path) -> bool { + path.extension().and_then(|ext| ext.to_str()).map_or(false, |ext| { + ext == "dylib" || ext == "so" || ext == "dll" || (ext == "a" && is_aix_shared_archive(path)) + }) +} + +fn is_aix_shared_archive(path: &Path) -> bool { + // FIXME(#133268): reading the entire file as &[u8] into memory seems excessive + // look into either mmap it or use the ReadCache + let data = match fs::read(path) { + Ok(data) => data, + Err(_) => return false, + }; + let file = match ArchiveFile::parse(&*data) { + Ok(file) => file, + Err(_) => return false, + }; + + file.members() + .filter_map(Result::ok) + .any(|entry| String::from_utf8_lossy(entry.name()).contains(".so")) } /// Returns `true` if the file name given looks like a debug info file