From 282b1e476883fbfd8924535185512199b4eaff3c Mon Sep 17 00:00:00 2001 From: Tuna Date: Wed, 14 Sep 2022 18:17:05 -0500 Subject: [PATCH] Distribute bootstrap in CI artifacts - Add a new `bootstrap` component Originally, we planned to combine this with the `rust-dev` component. However, I realized that would force LLVM to be redownloaded whenever bootstrap is modified. LLVM is a much larger download, so split this to get better caching. - Build bootstrap for all tier 1 and 2 targets --- .github/workflows/ci.yml | 20 +++++------ src/bootstrap/builder.rs | 1 + src/bootstrap/dist.rs | 35 +++++++++++++++++++ .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 20 +++++------ 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec1ef041b20..5e458c5a987 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -291,7 +291,7 @@ jobs: os: ubuntu-20.04-xl - name: dist-x86_64-apple env: - SCRIPT: "./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin" + SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -302,7 +302,7 @@ jobs: os: macos-latest - name: dist-apple-various env: - SCRIPT: "./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" + SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -312,7 +312,7 @@ jobs: os: macos-latest - name: dist-x86_64-apple-alt env: - SCRIPT: "./x.py dist" + SCRIPT: "./x.py dist bootstrap --include-default-paths" RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -344,7 +344,7 @@ jobs: os: macos-latest - name: dist-aarch64-apple env: - SCRIPT: "./x.py dist --stage 2" + SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2" RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 USE_XCODE_CLANG: 1 @@ -418,19 +418,19 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 WINDOWS_SDK_20348_HACK: 1 os: windows-latest-xl @@ -438,13 +438,13 @@ jobs: env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain" NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain" NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 @@ -453,7 +453,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths os: windows-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 14e8ebd6876..8ec64657f47 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -723,6 +723,7 @@ impl<'a> Builder<'a> { dist::Miri, dist::LlvmTools, dist::RustDev, + dist::Bootstrap, dist::Extended, // It seems that PlainSourceTarball somehow changes how some of the tools // perceive their dependencies (see #93033) which would invalidate fingerprints diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 1a59b3958f1..679f017c770 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2050,6 +2050,41 @@ impl Step for RustDev { } } +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Bootstrap { + pub target: TargetSelection, +} + +impl Step for Bootstrap { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + let tarball = Tarball::new(builder, "bootstrap", &target.triple); + + let bootstrap_outdir = &builder.bootstrap_out; + for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] { + tarball.add_file(bootstrap_outdir.join(file), "bootstrap/bin", 0o755); + } + + Some(tarball.generate()) + } +} + /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the /// release process to avoid cloning the monorepo and building stuff. /// diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 973c43072bf..b960239807a 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -82,7 +82,7 @@ ENV RUST_CONFIGURE_ARGS \ ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ - build-manifest + build-manifest bootstrap ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 6e4b0b0c2c3..6056e722362 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -455,7 +455,7 @@ jobs: - name: dist-x86_64-apple env: - SCRIPT: ./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin + SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -467,7 +467,7 @@ jobs: - name: dist-apple-various env: - SCRIPT: ./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim + SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -478,7 +478,7 @@ jobs: - name: dist-x86_64-apple-alt env: - SCRIPT: ./x.py dist + SCRIPT: ./x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -508,7 +508,7 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist --stage 2 + SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 RUST_CONFIGURE_ARGS: >- --build=x86_64-apple-darwin --host=aarch64-apple-darwin @@ -652,7 +652,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl @@ -664,7 +664,7 @@ jobs: --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl @@ -675,7 +675,7 @@ jobs: --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 # Hack around this SDK version, because it doesn't work with clang. # See https://github.com/rust-lang/rust/issues/88796 @@ -692,14 +692,14 @@ jobs: # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools @@ -715,7 +715,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist + SCRIPT: python x.py dist bootstrap --include-default-paths <<: *job-windows-xl try: