Rollup merge of rust-lang#97507 - jyn514:download-rustfmt, r=Mark-Simulacrum

Move rustfmt downloads from bootstrap.py to rustbuild

- Allow verifying CI downloads using src/stage0.json
- Change download functions not to hard-code `ci-artifacts.rust-lang.org`
- Change `format::format` to take a `Builder` so it has access to `download_component`. I think we may want to reconsider the distinction between Build and Builder at some point; I don't think it's particularly useful.
- Move rustfmt downloads out of bootstrap.py

Fixes rust-lang#95136. Helps with rust-lang#94829. This is based on rust-lang#96687 for simplicity.

Author: matthiaskrgr

Cargo.lock

@@ -216,13 +216,15 @@ dependencies = [
  "cmake",
  "filetime",
  "getopts",
+ "hex 0.4.2",
  "ignore",
  "libc",
  "once_cell",
  "opener",
  "pretty_assertions 0.7.2",
  "serde",
  "serde_json",
+ "sha2",
  "sysinfo",
  "tar",
  "toml",

src/bootstrap/Cargo.toml

@@ -40,8 +40,10 @@ filetime = "0.2"
 getopts = "0.2.19"
 cc = "1.0.69"
 libc = "0.2"
+hex = "0.4"
 serde = { version = "1.0.8", features = ["derive"] }
 serde_json = "1.0.2"
+sha2 = "0.10"
 tar = "0.4"
 toml = "0.5"
 ignore = "0.4.10"

src/bootstrap/bootstrap.py

@@ -63,31 +63,30 @@ def support_xz():
     except tarfile.CompressionError:
         return False
 
-def get(base, url, path, checksums, verbose=False, do_verify=True):
+def get(base, url, path, checksums, verbose=False):
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
         temp_path = temp_file.name
 
     try:
-        if do_verify:
-            if url not in checksums:
-                raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
-                                    "Pre-built artifacts might not available for this "
-                                    "target at this time, see https://doc.rust-lang.org/nightly"
-                                    "/rustc/platform-support.html for more information.")
-                                   .format(url))
-            sha256 = checksums[url]
-            if os.path.exists(path):
-                if verify(path, sha256, False):
-                    if verbose:
-                        print("using already-download file", path)
-                    return
-                else:
-                    if verbose:
-                        print("ignoring already-download file",
-                            path, "due to failed verification")
-                    os.unlink(path)
+        if url not in checksums:
+            raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
+                                "Pre-built artifacts might not be available for this "
+                                "target at this time, see https://doc.rust-lang.org/nightly"
+                                "/rustc/platform-support.html for more information.")
+                                .format(url))
+        sha256 = checksums[url]
+        if os.path.exists(path):
+            if verify(path, sha256, False):
+                if verbose:
+                    print("using already-download file", path)
+                return
+            else:
+                if verbose:
+                    print("ignoring already-download file",
+                        path, "due to failed verification")
+                os.unlink(path)
         download(temp_path, "{}/{}".format(base, url), True, verbose)
-        if do_verify and not verify(temp_path, sha256, verbose):
+        if not verify(temp_path, sha256, verbose):
             raise RuntimeError("failed verification")
         if verbose:
             print("moving {} to {}".format(temp_path, path))
@@ -430,7 +429,6 @@ class RustBuild(object):
     def __init__(self):
         self.checksums_sha256 = {}
         self.stage0_compiler = None
-        self.stage0_rustfmt = None
         self._download_url = ''
         self.build = ''
         self.build_dir = ''
@@ -484,31 +482,10 @@ def download_toolchain(self):
             with output(self.rustc_stamp()) as rust_stamp:
                 rust_stamp.write(key)
 
-        if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
-            not os.path.exists(self.rustfmt())
-            or self.program_out_of_date(
-                self.rustfmt_stamp(),
-                "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
-            )
-        ):
-            if self.stage0_rustfmt is not None:
-                tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
-                filename = "rustfmt-{}-{}{}".format(
-                    self.stage0_rustfmt.version, self.build, tarball_suffix,
-                )
-                self._download_component_helper(
-                    filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
-                )
-                self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
-                self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
-                with output(self.rustfmt_stamp()) as rustfmt_stamp:
-                    rustfmt_stamp.write(self.stage0_rustfmt.channel())
-
     def _download_component_helper(
-        self, filename, pattern, tarball_suffix, key=None
+        self, filename, pattern, tarball_suffix,
     ):
-        if key is None:
-            key = self.stage0_compiler.date
+        key = self.stage0_compiler.date
         cache_dst = os.path.join(self.build_dir, "cache")
         rustc_cache = os.path.join(cache_dst, key)
         if not os.path.exists(rustc_cache):
@@ -524,7 +501,6 @@ def _download_component_helper(
                 tarball,
                 self.checksums_sha256,
                 verbose=self.verbose,
-                do_verify=True,
             )
         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 
@@ -634,16 +610,6 @@ def rustc_stamp(self):
         """
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
-    def rustfmt_stamp(self):
-        """Return the path for .rustfmt-stamp
-
-        >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
-        True
-        """
-        return os.path.join(self.bin_root(), '.rustfmt-stamp')
-
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -717,12 +683,6 @@ def rustc(self):
         """Return config path for rustc"""
         return self.program_config('rustc')
 
-    def rustfmt(self):
-        """Return config path for rustfmt"""
-        if self.stage0_rustfmt is None:
-            return None
-        return self.program_config('rustfmt')
-
     def program_config(self, program):
         """Return config path for the given program at the given stage
 
@@ -1082,8 +1042,6 @@ def bootstrap(help_triggered):
         data = json.load(f)
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
-    if data.get("rustfmt") is not None:
-        build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
 
     build.set_dist_environment(data["dist_server"])
 

src/bootstrap/builder.rs

@@ -728,7 +728,8 @@ impl<'a> Builder<'a> {
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
             Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
-            Subcommand::Format { .. } | Subcommand::Clean { .. } | Subcommand::Setup { .. } => {
+            Subcommand::Format { .. } => (Kind::Format, &[][..]),
+            Subcommand::Clean { .. } | Subcommand::Setup { .. } => {
                 panic!()
             }
         };
@@ -878,7 +879,6 @@ impl<'a> Builder<'a> {
     ) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        // FIXME: support `do_verify` (only really needed for nightly rustfmt)
         self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
         t!(std::fs::rename(&tempfile, dest_path));
     }
@@ -970,6 +970,28 @@ impl<'a> Builder<'a> {
         t!(fs::remove_dir_all(dst.join(directory_prefix)));
     }
 
+    /// Returns whether the SHA256 checksum of `path` matches `expected`.
+    pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
+        use sha2::Digest;
+
+        self.verbose(&format!("verifying {}", path.display()));
+        let mut hasher = sha2::Sha256::new();
+        // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+        // Consider using streaming IO instead?
+        let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) };
+        hasher.update(&contents);
+        let found = hex::encode(hasher.finalize().as_slice());
+        let verified = found == expected;
+        if !verified && !self.config.dry_run {
+            println!(
+                "invalid checksum: \n\
+                found:    {found}\n\
+                expected: {expected}",
+            );
+        }
+        return verified;
+    }
+
     /// Obtain a compiler at a given stage and for a given host. Explicitly does
     /// not take `Compiler` since all `Compiler` instances are meant to be
     /// obtained through this function, since it ensures that they are valid
@@ -1192,6 +1214,10 @@ impl<'a> Builder<'a> {
         Config::download_rustc(self)
     }
 
+    pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> {
+        Config::initial_rustfmt(self)
+    }
+
     /// Prepares an invocation of `cargo` to be run.
     ///
     /// This will create a `Command` that represents a pending execution of