add relnotes-api-list tool

Author: pietroalbini

Cargo.lock

@@ -3255,6 +3255,17 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "relnotes-api-list"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "rustdoc-json-types",
+ "serde",
+ "serde_json",
+ "tempfile",
+]
+
 [[package]]
 name = "remote-test-client"
 version = "0.1.0"

Cargo.toml

@@ -30,6 +30,7 @@ members = [
   "src/tools/miri/cargo-miri",
   "src/tools/miropt-test-tools",
   "src/tools/opt-dist",
+  "src/tools/relnotes-api-list",
   "src/tools/remote-test-client",
   "src/tools/remote-test-server",
   "src/tools/replace-version-placeholder",

src/bootstrap/src/core/build_steps/dist.rs

@@ -2643,3 +2643,62 @@ impl Step for Gcc {
         tarball.generate()
     }
 }
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct RelnotesApiList {
+    pub host: TargetSelection,
+}
+
+impl Step for RelnotesApiList {
+    type Output = ();
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let default = run.builder.config.docs;
+        run.alias("relnotes-api-list").default_condition(default)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RelnotesApiList { host: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let host = self.host;
+        let dest = builder.out.join("dist").join(format!("relnotes-api-list-{host}.json"));
+        builder.create_dir(dest.parent().unwrap());
+
+        // The HTML documentation for the standard library is needed to check all links generated by
+        // the tool are not broken.
+        builder.ensure(crate::core::build_steps::doc::Std::new(
+            builder.top_stage,
+            host,
+            DocumentationFormat::Html,
+        ));
+
+        if std::env::var_os("EMILY_SKIP_DOC").is_none() {
+            // TODO: remove the condition
+            builder.ensure(
+                crate::core::build_steps::doc::Std::new(
+                    builder.top_stage,
+                    host,
+                    DocumentationFormat::Json,
+                )
+                // Crates containing symbols exported by any std crate:
+                .add_extra_crate("rustc-literal-escaper")
+                .add_extra_crate("std_detect"),
+            );
+        }
+
+        let linkchecker = builder.tool_exe(Tool::Linkchecker);
+
+        builder.info("Generating the API list for the release notes");
+        builder
+            .tool_cmd(Tool::RelnotesApiList)
+            .arg(builder.json_doc_out(host))
+            .arg(&dest)
+            .env("LINKCHECKER_PATH", linkchecker)
+            .env("STD_HTML_DOCS", builder.doc_out(self.host))
+            .run(builder);
+        builder.info(&format!("API list for the release notes available at {}", dest.display()));
+    }
+}

src/bootstrap/src/core/build_steps/doc.rs

@@ -560,11 +560,17 @@ pub struct Std {
     pub target: TargetSelection,
     pub format: DocumentationFormat,
     crates: Vec<String>,
+    extra_crates: Vec<String>,
 }
 
 impl Std {
     pub(crate) fn new(stage: u32, target: TargetSelection, format: DocumentationFormat) -> Self {
-        Std { stage, target, format, crates: vec![] }
+        Std { stage, target, format, crates: vec![], extra_crates: vec![] }
+    }
+
+    pub(crate) fn add_extra_crate(mut self, krate: &str) -> Self {
+        self.extra_crates.push(krate.to_string());
+        self
     }
 }
 
@@ -592,6 +598,7 @@ impl Step for Std {
                 DocumentationFormat::Html
             },
             crates,
+            extra_crates: vec![],
         });
     }
 
@@ -602,7 +609,7 @@ impl Step for Std {
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let target = self.target;
-        let crates = if self.crates.is_empty() {
+        let mut crates = if self.crates.is_empty() {
             builder
                 .in_tree_crates("sysroot", Some(target))
                 .iter()
@@ -611,6 +618,7 @@ impl Step for Std {
         } else {
             self.crates
         };
+        crates.extend(self.extra_crates.iter().cloned());
 
         let out = match self.format {
             DocumentationFormat::Html => builder.doc_out(target),

src/bootstrap/src/core/build_steps/tool.rs

@@ -568,6 +568,7 @@ bootstrap_tool!(
     FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump";
     OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"];
     RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library;
+    RelnotesApiList, "src/tools/relnotes-api-list", "relnotes-api-list";
 );
 
 /// These are the submodules that are required for rustbook to work due to

src/bootstrap/src/core/builder/mod.rs

@@ -986,7 +986,8 @@ impl<'a> Builder<'a> {
                 tool::LlvmBitcodeLinker,
                 tool::RustcPerf,
                 tool::WasmComponentLd,
-                tool::LldWrapper
+                tool::LldWrapper,
+                tool::RelnotesApiList,
             ),
             Kind::Clippy => describe!(
                 clippy::Std,
@@ -1163,7 +1164,8 @@ impl<'a> Builder<'a> {
                 dist::PlainSourceTarball,
                 dist::BuildManifest,
                 dist::ReproducibleArtifacts,
-                dist::Gcc
+                dist::Gcc,
+                dist::RelnotesApiList,
             ),
             Kind::Install => describe!(
                 install::Docs,

src/tools/relnotes-api-list/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "relnotes-api-list"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+rustdoc-json-types = { path = "../../rustdoc-json-types" }
+tempfile = "3.20.0"

src/tools/relnotes-api-list/README.md

@@ -0,0 +1,17 @@
+# API list generator for the release notes
+
+Rust's [release notes] include a "Stabilized APIs" section for each
+release, listing all APIs that became stable each release. This tool supports
+the creation of that section by generating a JSON file containing a concise
+representation of the standard library API. The [relnotes tool] will then
+compare the JSON files of two releases to generate the section.
+
+The tool is executed by CI and produces the `relnotes-api-list-$target.json`
+dist artifact. You can also run the tool locally with:
+
+```
+./x dist relnotes-api-list
+```
+
+[release notes]: https://doc.rust-lang.org/stable/releases.html
+[relnotes tool]: https://github.com/rust-lang/relnotes

src/tools/relnotes-api-list/src/check_urls.rs

@@ -0,0 +1,65 @@
+//! We generate a lot of URLs in the resulting JSON, and we need all those URLs to be correct. While
+//! some URLs are fairly trivial to generate, others are quite tricky (especially `impl` blocks).
+//!
+//! To ensure we always generate good URLs, we prepare a temporary HTML file containing `<a>` tags for
+//! every itme we collected, and we run it through linkchecker. If this fails, it means the code
+//! generating URLs has a bug.
+
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::Command;
+
+use anyhow::{Error, anyhow, bail};
+use tempfile::tempdir;
+
+use crate::schema::{Schema, SchemaItem};
+
+pub(crate) fn check_urls(schema: &Schema) -> Result<(), Error> {
+    let mut urls = Vec::new();
+    collect_urls(&mut urls, &schema.items);
+
+    let html_dir = tempdir()?;
+
+    let mut file = File::create(html_dir.path().join("urls.html"))?;
+    file.write_all(render_html(&urls).as_bytes())?;
+
+    eprintln!("checking that all generated URLs are valid...");
+    let result = Command::new(require_env("LINKCHECKER_PATH")?)
+        .arg(html_dir.path())
+        .arg("--link-targets-dir")
+        .arg(require_env("STD_HTML_DOCS")?)
+        .status()?;
+
+    if !result.success() {
+        bail!("some URLs are broken, the relnotes-api-list tool has a bug");
+    }
+
+    dbg!(require_env("STD_HTML_DOCS")?);
+
+    Ok(())
+}
+
+fn collect_urls<'a>(result: &mut Vec<&'a str>, items: &'a [SchemaItem]) {
+    for item in items {
+        if let Some(url) = &item.url {
+            result.push(url);
+        }
+        collect_urls(result, &item.children);
+    }
+}
+
+fn render_html(urls: &[&str]) -> String {
+    let mut content = "<!DOCTYPE html>\n".to_string();
+    for url in urls {
+        content.push_str(&format!("<a href=\"{url}\"></a>\n"));
+    }
+    content
+}
+
+fn require_env(name: &str) -> Result<PathBuf, Error> {
+    match std::env::var_os(name) {
+        Some(value) => Ok(value.into()),
+        None => Err(anyhow!("missing environment variable {name}")),
+    }
+}