mathieu/os-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove the expand-yaml-anchors tool

Browse source
This commit is contained in:
Jakub Beránek 2024-04-25 11:13:37 +02:00
parent 1ca92c0857
commit 0b1ecf1d8d
No known key found for this signature in database
GPG key ID: 909CD0D26483516B
11 changed files with 5 additions and 313 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
Cargo.lock
View file

@ -1254,14 +1254,6 @@ dependencies = [
"mdbook",
]
[[package]]
name = "expand-yaml-anchors"
version = "0.1.0"
dependencies = [
"yaml-merge-keys",
"yaml-rust",
]
[[package]]
name = "expect-test"
version = "1.5.0"
@ -2236,12 +2228,6 @@ dependencies = [
"regex",
]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "lint-docs"
version = "0.1.0"
@ -6547,26 +6533,6 @@ dependencies = [
"lzma-sys",
]
[[package]]
name = "yaml-merge-keys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd236a7dc9bb598f349fe4a8754f49181fee50284daa15cd1ba652d722280004"
dependencies = [
"lazy_static",
"thiserror",
"yaml-rust",
]
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "yansi-term"
version = "0.1.2"

1
Cargo.toml
View file

@ -31,7 +31,6 @@ members = [
"src/tools/miri/cargo-miri",
"src/tools/rustdoc-themes",
"src/tools/unicode-table-generator",
"src/tools/expand-yaml-anchors",
"src/tools/jsondocck",
"src/tools/jsondoclint",
"src/tools/llvm-bitcode-linker",

26
src/bootstrap/src/core/build_steps/run.rs
View file

@ -15,32 +15,6 @@ use crate::core::config::TargetSelection;
use crate::utils::helpers::output;
use crate::Mode;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ExpandYamlAnchors;
impl Step for ExpandYamlAnchors {
type Output = ();
/// Runs the `expand-yaml_anchors` tool.
///
/// This tool in `src/tools` reads the CI configuration files written in YAML and expands the
/// anchors in them, since GitHub Actions doesn't support them.
fn run(self, builder: &Builder<'_>) {
builder.info("Expanding YAML anchors in the GitHub Actions configuration");
builder.run_delaying_failure(
builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
);
}
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/expand-yaml-anchors")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(ExpandYamlAnchors);
}
}
#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
pub struct BuildManifest;

35
src/bootstrap/src/core/build_steps/test.rs
View file

@ -1146,8 +1146,6 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to
builder.info("tidy check");
builder.run_delaying_failure(&mut cmd);
builder.ensure(ExpandYamlAnchors);
builder.info("x.py completions check");
let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"]
.map(|filename| builder.src.join("src/etc/completions").join(filename));
@ -1175,39 +1173,6 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ExpandYamlAnchors;
impl Step for ExpandYamlAnchors {
type Output = ();
const ONLY_HOSTS: bool = true;
/// Ensure the `generate-ci-config` tool was run locally.
///
/// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
/// appropriate configuration for all our CI providers. This step ensures the tool was called
/// by the user before committing CI changes.
fn run(self, builder: &Builder<'_>) {
// NOTE: `.github/` is not included in dist-src tarballs
if !builder.src.join(".github/workflows/ci.yml").exists() {
builder.info("Skipping YAML anchors check: GitHub Actions config not found");
return;
}
builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
builder.run_delaying_failure(
builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
);
}
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/expand-yaml-anchors")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(ExpandYamlAnchors);
}
}
fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf {
builder.out.join(host.triple).join("test")
}

1
src/bootstrap/src/core/build_steps/tool.rs
View file

@ -302,7 +302,6 @@ bootstrap_tool!(
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RustInstaller, "src/tools/rust-installer", "rust-installer";
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
LintDocs, "src/tools/lint-docs", "lint-docs";
JsonDocCk, "src/tools/jsondocck", "jsondocck";
JsonDocLint, "src/tools/jsondoclint", "jsondoclint";

2
src/bootstrap/src/core/builder.rs
View file

@ -791,7 +791,6 @@ impl<'a> Builder<'a> {
),
Kind::Test => describe!(
crate::core::build_steps::toolstate::ToolStateCheck,
test::ExpandYamlAnchors,
test::Tidy,
test::Ui,
test::Crashes,
@ -933,7 +932,6 @@ impl<'a> Builder<'a> {
install::Src,
),
Kind::Run => describe!(
run::ExpandYamlAnchors,
run::BuildManifest,
run::BumpStage0,
run::ReplaceVersionPlaceholder,

2
src/bootstrap/src/core/config/flags.rs
View file

@ -420,7 +420,7 @@ pub enum Subcommand {
Arguments:
This subcommand accepts a number of paths to tools to build and run. For
example:
./x.py run src/tools/expand-yaml-anchors
./x.py run src/tools/bump-stage0
At least a tool needs to be called.")]
/// Run tools contained in this repository
Run {

9
src/ci/docker/host-x86_64/mingw-check/Dockerfile
View file

@ -42,11 +42,10 @@ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
# Check library crates on all tier 1 targets.
# We disable optimized compiler built-ins because that requires a C toolchain for the target.
# We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
# Check library crates on all tier 1 targets.
# We disable optimized compiler built-ins because that requires a C toolchain for the target.
# We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
python3 ../x.py clippy bootstrap -Dwarnings && \
python3 ../x.py clippy compiler library -Aclippy::all -Dclippy::correctness && \

2
src/ci/github-actions/jobs.yml
View file

@ -1,7 +1,5 @@
# This file contains definitions of CI job parameters that are loaded
# dynamically in CI from ci.yml.
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
# modify this file.
runners:
- &base-job
env: { }

8
src/tools/expand-yaml-anchors/Cargo.toml
View file

@ -1,8 +0,0 @@
[package]
name = "expand-yaml-anchors"
version = "0.1.0"
edition = "2021"
[dependencies]
yaml-rust = "0.4.3"
yaml-merge-keys = "0.4.0"

198
src/tools/expand-yaml-anchors/src/main.rs
View file

@ -1,198 +0,0 @@
use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
/// List of files to expand. The first tuple element is the source
/// file, while the second tuple element is the destination file.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"),
];
/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";
/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
# WARNING: automatically generated file, DO NOT CHANGE! #
#############################################################
# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
# {source}
#
# Once you make changes to that file you need to run:
#
# ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.
";
enum Mode {
Check,
Generate,
}
struct App {
mode: Mode,
base: PathBuf,
}
impl App {
fn from_args() -> Result<Self, Box<dyn Error>> {
// Parse CLI arguments
let args = std::env::args().skip(1).collect::<Vec<_>>();
let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
["check", ref base] => (Mode::Check, PathBuf::from(base)),
_ => {
eprintln!("usage: expand-yaml-anchors <generate|check> <base-dir>");
std::process::exit(1);
}
};
Ok(App { mode, base })
}
fn run(&self) -> Result<(), Box<dyn Error>> {
for (source, dest) in TO_EXPAND {
let source = self.base.join(source);
let dest_path = self.base.join(dest);
self.expand(&source, &dest_path).with_context(|| match self.mode {
Mode::Generate => format!(
"failed to expand {} into {}",
self.path(&source),
self.path(&dest_path)
),
Mode::Check => format!(
"{} is not up to date; please run \
`x.py run src/tools/expand-yaml-anchors`.",
self.path(&dest_path)
),
})?;
}
Ok(())
}
fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
let content = std::fs::read_to_string(source)
.with_context(|| format!("failed to read {}", self.path(source)))?;
let mut buf =
HEADER_MESSAGE.replace("{source}", &self.path(source).to_string().replace("\\", "/"));
let documents = YamlLoader::load_from_str(&content)
.with_context(|| format!("failed to parse {}", self.path(source)))?;
for mut document in documents.into_iter() {
document = yaml_merge_keys::merge_keys(document)
.with_context(|| format!("failed to expand {}", self.path(source)))?;
document = filter_document(document);
YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
context: "failed to serialize the expanded yaml".into(),
source: Box::new(err),
})?;
buf.push('\n');
}
match self.mode {
Mode::Check => {
let old = std::fs::read_to_string(dest)
.with_context(|| format!("failed to read {}", self.path(dest)))?;
if old != buf {
return Err(Box::new(StrError(format!(
"{} and {} are different",
self.path(source),
self.path(dest),
))));
}
}
Mode::Generate => {
std::fs::write(dest, buf.as_bytes())
.with_context(|| format!("failed to write to {}", self.path(dest)))?;
}
}
Ok(())
}
fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
path.strip_prefix(&self.base).unwrap_or(path).display()
}
}
fn filter_document(document: Yaml) -> Yaml {
match document {
Yaml::Hash(map) => Yaml::Hash(
map.into_iter()
.filter(|(key, _)| {
if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
})
.map(|(key, value)| (filter_document(key), filter_document(value)))
.collect(),
),
Yaml::Array(vec) => Yaml::Array(vec.into_iter().map(filter_document).collect()),
other => other,
}
}
fn main() {
if let Err(err) = App::from_args().and_then(|app| app.run()) {
eprintln!("error: {}", err);
let mut source = err.as_ref() as &dyn Error;
while let Some(err) = source.source() {
eprintln!("caused by: {}", err);
source = err;
}
std::process::exit(1);
}
}
#[derive(Debug)]
struct StrError(String);
impl Error for StrError {}
impl std::fmt::Display for StrError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.0, f)
}
}
#[derive(Debug)]
struct WithContext {
context: String,
source: Box<dyn Error>,
}
impl std::fmt::Display for WithContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.context)
}
}
impl Error for WithContext {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(self.source.as_ref())
}
}
pub(crate) trait ResultExt<T> {
fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}
impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
match self {
Ok(ok) => Ok(ok),
Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
}
}
}