mathieu/os-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #92260 - jyn514:less-python-logic, r=Mark-Simulacrum

Browse source 
Move some more bootstrap logic from python to rust

Same rationale as https://github.com/rust-lang/rust/pull/76544; it would be nice to make python entirely optional at some point.

This also removes $ROOT as an option for the build directory; I haven't been using it, and like Alex
said in https://github.com/rust-lang/rust/pull/76544#discussion_r488248930 it seems like a misfeature.

This allows running `cargo run` from src/bootstrap, although that still gives
lots of compile errors if you don't use the beta toolchain. It's not exactly the same as using `x.py`, since it won't have `BOOTSTRAP_DOWNLOAD_RUSTC` set, but it's pretty close. Doing this from the top-level directory requires https://github.com/rust-lang/cargo/issues/7290 to be fixed, or using `cargo run -p bootstrap`.

The next steps for making python optional are to move download-ci-llvm and download-rustc support into rustbuild, likely be shelling out as the python scripts do today.

It would also be nice (although not required) to move submodule support there, but that would require taking bootstrap out of the workspace to avoid errors from crates that haven't been cloned yet.

r? `@Mark-Simulacrum`
This commit is contained in:
bors 2022-03-08 17:04:45 +00:00
commit 64187b8374
10 changed files with 171 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/bootstrap/Cargo.toml
View file

@ -3,6 +3,7 @@ name = "bootstrap"
version = "0.0.0"
edition = "2021"
build = "build.rs"
default-run = "bootstrap"
[lib]
path = "lib.rs"

5
src/bootstrap/bootstrap.py
View file

@ -1267,7 +1267,7 @@ def bootstrap(help_triggered):
build.check_vendored_status()
build_dir = build.get_toml('build-dir', 'build') or 'build'
build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
build.build_dir = os.path.abspath(build_dir)
with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
data = json.load(f)
@ -1302,10 +1302,7 @@ def bootstrap(help_triggered):
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
env["BOOTSTRAP_PYTHON"] = sys.executable
env["BUILD_DIR"] = build.build_dir
env["RUSTC_BOOTSTRAP"] = '1'
if toml_path:
env["BOOTSTRAP_CONFIG"] = toml_path
if build.rustc_commit is not None:
env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
run(args, env=env, verbose=build.verbose, is_bootstrap=True)

6
src/bootstrap/builder.rs
View file

@ -883,7 +883,7 @@ impl<'a> Builder<'a> {
}
pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc"));
cmd.env("RUSTC_STAGE", compiler.stage.to_string())
.env("RUSTC_SYSROOT", self.sysroot(compiler))
// Note that this is *not* the sysroot_libdir because rustdoc must be linked
@ -1249,7 +1249,7 @@ impl<'a> Builder<'a> {
.env("RUSTC_STAGE", stage.to_string())
.env("RUSTC_SYSROOT", &sysroot)
.env("RUSTC_LIBDIR", &libdir)
.env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
.env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
.env(
"RUSTDOC_REAL",
if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
@ -1263,7 +1263,7 @@ impl<'a> Builder<'a> {
// Clippy support is a hack and uses the default `cargo-clippy` in path.
// Don't override RUSTC so that the `cargo-clippy` in path will be run.
if cmd != "clippy" {
cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc"));
cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
}
// Dealing with rpath here is a little special, so let's go into some

2
src/bootstrap/builder/tests.rs
View file

@ -8,10 +8,10 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
config.save_toolstates = None;
config.dry_run = true;
config.ninja_in_file = false;
// try to avoid spurious failures in dist where we create/delete each others file
config.out = PathBuf::from(env::var_os("BOOTSTRAP_OUTPUT_DIRECTORY").unwrap());
config.initial_rustc = PathBuf::from(env::var_os("RUSTC").unwrap());
config.initial_cargo = PathBuf::from(env::var_os("BOOTSTRAP_INITIAL_CARGO").unwrap());
// try to avoid spurious failures in dist where we create/delete each others file
let dir = config
.out
.join("tmp-rustbuild-tests")

51
src/bootstrap/config.rs
View file

@ -6,7 +6,6 @@
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::env;
use std::ffi::OsString;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
@ -392,7 +391,6 @@ derive_merge! {
build: Option<String>,
host: Option<Vec<String>>,
target: Option<Vec<String>>,
// This is ignored, the rust code always gets the build directory from the `BUILD_DIR` env variable
build_dir: Option<String>,
cargo: Option<String>,
rustc: Option<String>,
@ -588,18 +586,6 @@ derive_merge! {
}
impl Config {
fn path_from_python(var_key: &str) -> PathBuf {
match env::var_os(var_key) {
Some(var_val) => Self::normalize_python_path(var_val),
_ => panic!("expected '{}' to be set", var_key),
}
}
/// Normalizes paths from Python slightly. We don't trust paths from Python (#49785).
fn normalize_python_path(path: OsString) -> PathBuf {
Path::new(&path).components().collect()
}
pub fn default_opts() -> Config {
let mut config = Config::default();
config.llvm_optimize = true;
@ -625,7 +611,7 @@ impl Config {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
// Undo `src/bootstrap`
config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned();
config.out = Config::path_from_python("BUILD_DIR");
config.out = PathBuf::from("build");
config.initial_cargo = PathBuf::from(env!("CARGO"));
config.initial_rustc = PathBuf::from(env!("RUSTC"));
@ -655,12 +641,6 @@ impl Config {
config.llvm_profile_use = flags.llvm_profile_use;
config.llvm_profile_generate = flags.llvm_profile_generate;
if config.dry_run {
let dir = config.out.join("tmp-dry-run");
t!(fs::create_dir_all(&dir));
config.out = dir;
}
#[cfg(test)]
let get_toml = |_| TomlConfig::default();
#[cfg(not(test))]
@ -677,7 +657,15 @@ impl Config {
}
};
let mut toml = flags.config.as_deref().map(get_toml).unwrap_or_else(TomlConfig::default);
// check --config first, then `$RUST_BOOTSTRAP_CONFIG` first, then `config.toml`
let toml_path = flags
.config
.clone()
.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from))
.unwrap_or_else(|| PathBuf::from("config.toml"));
let mut toml =
if toml_path.exists() { get_toml(&toml_path) } else { TomlConfig::default() };
if let Some(include) = &toml.profile {
let mut include_path = config.src.clone();
include_path.push("src");
@ -689,12 +677,25 @@ impl Config {
}
config.changelog_seen = toml.changelog_seen;
if let Some(cfg) = flags.config {
config.config = cfg;
}
config.config = toml_path;
let build = toml.build.unwrap_or_default();
set(&mut config.initial_rustc, build.rustc.map(PathBuf::from));
set(&mut config.out, build.build_dir.map(PathBuf::from));
// NOTE: Bootstrap spawns various commands with different working directories.
// To avoid writing to random places on the file system, `config.out` needs to be an absolute path.
if !config.out.is_absolute() {
// `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead.
config.out = crate::util::absolute(&config.out);
}
if config.dry_run {
let dir = config.out.join("tmp-dry-run");
t!(fs::create_dir_all(&dir));
config.out = dir;
}
config.hosts = if let Some(arg_host) = flags.host {
arg_host
} else if let Some(file_host) = build.host {

4
src/bootstrap/flags.rs
View file

@ -3,7 +3,6 @@
//! This module implements the command-line parsing of the build system which
//! has various flags to configure how it's run.
use std::env;
use std::path::PathBuf;
use std::process;
@ -541,7 +540,6 @@ Arguments:
// Get any optional paths which occur after the subcommand
let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>();
let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from);
let verbose = matches.opt_present("verbose");
// User passed in -h/--help?
@ -671,7 +669,7 @@ Arguments:
} else {
None
},
config: cfg_file,
config: matches.opt_str("config").map(PathBuf::from),
jobs: matches.opt_str("jobs").map(|j| j.parse().expect("`jobs` should be a number")),
cmd,
incremental: matches.opt_present("incremental"),

18
src/bootstrap/lib.rs
View file

@ -261,6 +261,7 @@ pub struct Build {
// Properties derived from the above configuration
src: PathBuf,
out: PathBuf,
bootstrap_out: PathBuf,
rust_info: channel::GitInfo,
cargo_info: channel::GitInfo,
rls_info: channel::GitInfo,
@ -435,6 +436,20 @@ impl Build {
.expect("failed to read src/version");
let version = version.trim();
let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() {
out.join("bootstrap").join("debug")
} else {
let workspace_target_dir = std::env::var("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| src.join("target"));
let bootstrap_out = workspace_target_dir.join("debug");
if !bootstrap_out.join("rustc").exists() {
// this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented
panic!("run `cargo build --bins` before `cargo run`")
}
bootstrap_out
};
let mut build = Build {
initial_rustc: config.initial_rustc.clone(),
initial_cargo: config.initial_cargo.clone(),
@ -453,6 +468,7 @@ impl Build {
version: version.to_string(),
src,
out,
bootstrap_out,
rust_info,
cargo_info,
@ -629,7 +645,7 @@ impl Build {
}
if let Subcommand::Setup { profile } = &self.config.cmd {
return setup::setup(&self.config.src, *profile);
return setup::setup(&self.config, *profile);
}
{

20
src/bootstrap/setup.rs
View file

@ -1,5 +1,5 @@
use crate::TargetSelection;
use crate::{t, VERSION};
use crate::{Config, TargetSelection};
use std::env::consts::EXE_SUFFIX;
use std::fmt::Write as _;
use std::fs::File;
@ -81,24 +81,22 @@ impl fmt::Display for Profile {
}
}
pub fn setup(src_path: &Path, profile: Profile) {
let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from);
pub fn setup(config: &Config, profile: Profile) {
let path = &config.config;
if cfg_file.as_ref().map_or(false, |f| f.exists()) {
let file = cfg_file.unwrap();
if path.exists() {
println!(
"error: you asked `x.py` to setup a new config file, but one already exists at `{}`",
file.display()
path.display()
);
println!("help: try adding `profile = \"{}\"` at the top of {}", profile, file.display());
println!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display());
println!(
"note: this will use the configuration in {}",
profile.include_path(src_path).display()
profile.include_path(&config.src).display()
);
std::process::exit(1);
}
let path = cfg_file.unwrap_or_else(|| "config.toml".into());
let settings = format!(
"# Includes one of the default files in src/bootstrap/defaults\n\
profile = \"{}\"\n\
@ -107,7 +105,7 @@ pub fn setup(src_path: &Path, profile: Profile) {
);
t!(fs::write(path, settings));
let include_path = profile.include_path(src_path);
let include_path = profile.include_path(&config.src);
println!("`x.py` will now use the configuration at {}", include_path.display());
let build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
@ -138,7 +136,7 @@ pub fn setup(src_path: &Path, profile: Profile) {
println!();
t!(install_git_hook_maybe(src_path));
t!(install_git_hook_maybe(&config.src));
println!();

4
src/bootstrap/test.rs
View file

@ -730,7 +730,7 @@ impl Step for RustdocTheme {
}
fn run(self, builder: &Builder<'_>) {
let rustdoc = builder.out.join("bootstrap/debug/rustdoc");
let rustdoc = builder.bootstrap_out.join("rustdoc");
let mut cmd = builder.tool_cmd(Tool::RustdocTheme);
cmd.arg(rustdoc.to_str().unwrap())
.arg(builder.src.join("src/librustdoc/html/static/css/themes").to_str().unwrap())
@ -2346,6 +2346,8 @@ impl Step for Bootstrap {
.current_dir(builder.src.join("src/bootstrap"))
.env("RUSTFLAGS", "-Cdebuginfo=2")
.env("CARGO_TARGET_DIR", builder.out.join("bootstrap"))
// HACK: bootstrap's tests want to know the output directory, but there's no way to set
// it except through config.toml. Set it through an env variable instead.
.env("BOOTSTRAP_OUTPUT_DIRECTORY", &builder.config.out)
.env("BOOTSTRAP_INITIAL_CARGO", &builder.config.initial_cargo)
.env("RUSTC_BOOTSTRAP", "1")

109
src/bootstrap/util.rs
View file

@ -440,3 +440,112 @@ fn fail(s: &str) -> ! {
println!("\n\n{}\n\n", s);
std::process::exit(1);
}
/// Copied from `std::path::absolute` until it stabilizes.
///
/// FIXME: this shouldn't exist.
pub(crate) fn absolute(path: &Path) -> PathBuf {
if path.as_os_str().is_empty() {
panic!("can't make empty path absolute");
}
#[cfg(unix)]
{
t!(absolute_unix(path), format!("could not make path absolute: {}", path.display()))
}
#[cfg(windows)]
{
t!(absolute_windows(path), format!("could not make path absolute: {}", path.display()))
}
#[cfg(not(any(unix, windows)))]
{
println!("warning: bootstrap is not supported on non-unix platforms");
t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path)
}
}
#[cfg(unix)]
/// Make a POSIX path absolute without changing its semantics.
fn absolute_unix(path: &Path) -> io::Result<PathBuf> {
// This is mostly a wrapper around collecting `Path::components`, with
// exceptions made where this conflicts with the POSIX specification.
// See 4.13 Pathname Resolution, IEEE Std 1003.1-2017
// https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13
use std::os::unix::prelude::OsStrExt;
let mut components = path.components();
let path_os = path.as_os_str().as_bytes();
let mut normalized = if path.is_absolute() {
// "If a pathname begins with two successive <slash> characters, the
// first component following the leading <slash> characters may be
// interpreted in an implementation-defined manner, although more than
// two leading <slash> characters shall be treated as a single <slash>
// character."
if path_os.starts_with(b"//") && !path_os.starts_with(b"///") {
components.next();
PathBuf::from("//")
} else {
PathBuf::new()
}
} else {
env::current_dir()?
};
normalized.extend(components);
// "Interfaces using pathname resolution may specify additional constraints
// when a pathname that does not name an existing directory contains at
// least one non- <slash> character and contains one or more trailing
// <slash> characters".
// A trailing <slash> is also meaningful if "a symbolic link is
// encountered during pathname resolution".
if path_os.ends_with(b"/") {
normalized.push("");
}
Ok(normalized)
}
#[cfg(windows)]
fn absolute_windows(path: &std::path::Path) -> std::io::Result<std::path::PathBuf> {
use std::ffi::OsString;
use std::io::Error;
use std::os::windows::ffi::{OsStrExt, OsStringExt};
use std::ptr::null_mut;
#[link(name = "kernel32")]
extern "system" {
fn GetFullPathNameW(
lpFileName: *const u16,
nBufferLength: u32,
lpBuffer: *mut u16,
lpFilePart: *mut *const u16,
) -> u32;
}
unsafe {
// encode the path as UTF-16
let path: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect();
let mut buffer = Vec::new();
// Loop until either success or failure.
loop {
// Try to get the absolute path
let len = GetFullPathNameW(
path.as_ptr(),
buffer.len().try_into().unwrap(),
buffer.as_mut_ptr(),
null_mut(),
);
match len as usize {
// Failure
0 => return Err(Error::last_os_error()),
// Buffer is too small, resize.
len if len > buffer.len() => buffer.resize(len, 0),
// Success!
len => {
buffer.truncate(len);
return Ok(OsString::from_wide(&buffer).into());
}
}
}
}
}