Redeploy 'configurable index location' (#1148)

* Revert "Revert "Support custom indexes""

This reverts commit 5085d57.

* Revert "Revert "Update Cargo.toml""

This reverts commit 02d3cfd.

* Revert "Revert "Use registry from config, instead of cli argument""

This reverts commit 1edaace.

* Revert "Revert "Use 0.11 rustwide""

This reverts commit 0128d2d.

* Revert "Revert "Favor registry for queued crates""

This reverts commit 7d81b24.

* Revert "Revert "Don`t check for deleted CRATE_VERSION""

This reverts commit c276522.

Author: jyn514

Cargo.lock

@@ -20,7 +20,7 @@ log = "0.4"
 regex = "1"
 structopt = "0.3"
 crates-index = "0.15.1"
-crates-index-diff = "7"
+crates-index-diff = "7.1.1"
 reqwest = { version = "0.10.6", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
 semver = { version = "0.9", features = ["serde"] }
 slug = "=0.1.1"
@@ -40,7 +40,7 @@ schemamama = "0.3"
 schemamama_postgres = "0.3"
 systemstat = "0.1.4"
 prometheus = { version = "0.10.0", default-features = false }
-rustwide = "0.10.0"
+rustwide = "0.11"
 mime_guess = "2"
 dotenv = "0.15"
 zstd = "0.5"

Cargo.toml

@@ -6,7 +6,8 @@ use std::sync::Arc;
 use docs_rs::db::{self, add_path_into_database, Pool, PoolClient};
 use docs_rs::utils::{remove_crate_priority, set_crate_priority};
 use docs_rs::{
-    BuildQueue, Config, Context, DocBuilder, Index, Metrics, RustwideBuilder, Server, Storage,
+    BuildQueue, Config, Context, DocBuilder, Index, Metrics, PackageKind, RustwideBuilder, Server,
+    Storage,
 };
 use failure::{err_msg, Error, ResultExt};
 use once_cell::sync::OnceCell;
@@ -169,9 +170,12 @@ impl QueueSubcommand {
                 crate_name,
                 crate_version,
                 build_priority,
-            } => ctx
-                .build_queue()?
-                .add_crate(&crate_name, &crate_version, build_priority)?,
+            } => ctx.build_queue()?.add_crate(
+                &crate_name,
+                &crate_version,
+                build_priority,
+                ctx.config()?.registry_url.as_deref(),
+            )?,
 
             Self::DefaultPriority { subcommand } => subcommand.handle_args(ctx)?,
         }
@@ -308,12 +312,16 @@ impl BuildSubcommand {
                         .build_local_package(&path)
                         .context("Building documentation failed")?;
                 } else {
+                    let registry_url = ctx.config()?.registry_url.clone();
                     builder
                         .build_package(
                             &crate_name.ok_or_else(|| err_msg("must specify name if not local"))?,
                             &crate_version
                                 .ok_or_else(|| err_msg("must specify version if not local"))?,
-                            None,
+                            registry_url
+                                .as_ref()
+                                .map(|s| PackageKind::Registry(s.as_str()))
+                                .unwrap_or(PackageKind::CratesIo),
                         )
                         .context("Building documentation failed")?;
                 }
@@ -593,7 +601,16 @@ impl Context for BinContext {
     fn index(&self) -> Result<Arc<Index>, Error> {
         Ok(self
             .index
-            .get_or_try_init::<_, Error>(|| Ok(Arc::new(Index::new(&*self.config()?)?)))?
+            .get_or_try_init::<_, Error>(|| {
+                let config = self.config()?;
+                Ok(Arc::new(
+                    if let Some(registry_url) = config.registry_url.clone() {
+                        Index::from_url(config.registry_index_path.clone(), registry_url)
+                    } else {
+                        Index::new(config.registry_index_path.clone())
+                    }?,
+                ))
+            })?
             .clone())
     }
 }

src/bin/cratesfyi.rs

@@ -11,6 +11,7 @@ pub(crate) struct QueuedCrate {
     pub(crate) name: String,
     pub(crate) version: String,
     pub(crate) priority: i32,
+    pub(crate) registry: Option<String>,
 }
 
 #[derive(Debug)]
@@ -29,10 +30,16 @@ impl BuildQueue {
         }
     }
 
-    pub fn add_crate(&self, name: &str, version: &str, priority: i32) -> Result<()> {
+    pub fn add_crate(
+        &self,
+        name: &str,
+        version: &str,
+        priority: i32,
+        registry: Option<&str>,
+    ) -> Result<()> {
         self.db.get()?.execute(
-            "INSERT INTO queue (name, version, priority) VALUES ($1, $2, $3);",
-            &[&name, &version, &priority],
+            "INSERT INTO queue (name, version, priority, registry) VALUES ($1, $2, $3, $4);",
+            &[&name, &version, &priority, &registry],
         )?;
         Ok(())
     }
@@ -63,7 +70,7 @@ impl BuildQueue {
 
     pub(crate) fn queued_crates(&self) -> Result<Vec<QueuedCrate>> {
         let query = self.db.get()?.query(
-            "SELECT id, name, version, priority
+            "SELECT id, name, version, priority, registry
              FROM queue
              WHERE attempt < $1
              ORDER BY priority ASC, attempt ASC, id ASC",
@@ -77,6 +84,7 @@ impl BuildQueue {
                 name: row.get("name"),
                 version: row.get("version"),
                 priority: row.get("priority"),
+                registry: row.get("registry"),
             })
             .collect())
     }
@@ -149,7 +157,7 @@ mod tests {
                 ("high-priority-baz", "1.0.0", -1000),
             ];
             for krate in &test_crates {
-                queue.add_crate(krate.0, krate.1, krate.2)?;
+                queue.add_crate(krate.0, krate.1, krate.2, None)?;
             }
 
             let assert_next = |name| -> Result<()> {
@@ -214,9 +222,9 @@ mod tests {
             let queue = env.build_queue();
 
             assert_eq!(queue.pending_count()?, 0);
-            queue.add_crate("foo", "1.0.0", 0)?;
+            queue.add_crate("foo", "1.0.0", 0, None)?;
             assert_eq!(queue.pending_count()?, 1);
-            queue.add_crate("bar", "1.0.0", 0)?;
+            queue.add_crate("bar", "1.0.0", 0, None)?;
             assert_eq!(queue.pending_count()?, 2);
 
             queue.process_next_crate(|krate| {
@@ -235,11 +243,11 @@ mod tests {
             let queue = env.build_queue();
 
             assert_eq!(queue.prioritized_count()?, 0);
-            queue.add_crate("foo", "1.0.0", 0)?;
+            queue.add_crate("foo", "1.0.0", 0, None)?;
             assert_eq!(queue.prioritized_count()?, 1);
-            queue.add_crate("bar", "1.0.0", -100)?;
+            queue.add_crate("bar", "1.0.0", -100, None)?;
             assert_eq!(queue.prioritized_count()?, 2);
-            queue.add_crate("baz", "1.0.0", 100)?;
+            queue.add_crate("baz", "1.0.0", 100, None)?;
             assert_eq!(queue.prioritized_count()?, 2);
 
             queue.process_next_crate(|krate| {
@@ -262,9 +270,9 @@ mod tests {
             let queue = env.build_queue();
 
             assert_eq!(queue.failed_count()?, 0);
-            queue.add_crate("foo", "1.0.0", -100)?;
+            queue.add_crate("foo", "1.0.0", -100, None)?;
             assert_eq!(queue.failed_count()?, 0);
-            queue.add_crate("bar", "1.0.0", 0)?;
+            queue.add_crate("bar", "1.0.0", 0, None)?;
 
             for _ in 0..MAX_ATTEMPTS {
                 assert_eq!(queue.failed_count()?, 0);
@@ -296,7 +304,7 @@ mod tests {
                 ("baz", "1.0.0", 10),
             ];
             for krate in &test_crates {
-                queue.add_crate(krate.0, krate.1, krate.2)?;
+                queue.add_crate(krate.0, krate.1, krate.2, None)?;
             }
 
             assert_eq!(

src/build_queue.rs

@@ -12,6 +12,7 @@ pub struct Config {
 
     pub prefix: PathBuf,
     pub registry_index_path: PathBuf,
+    pub registry_url: Option<String>,
 
     // Database connection params
     pub(crate) database_url: String,
@@ -56,6 +57,7 @@ impl Config {
 
             prefix: prefix.clone(),
             registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?,
+            registry_url: maybe_env("REGISTRY_URL")?,
 
             database_url: require_env("CRATESFYI_DATABASE_URL")?,
             max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?,

src/config.rs

@@ -475,6 +475,20 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
                 DROP TYPE feature;                         
             "
         ),
+        migration!(
+            context,
+            20,
+            // description
+            "Support alternative registries",
+            // upgrade query
+            "
+                ALTER TABLE queue ADD COLUMN registry TEXT DEFAULT NULL;
+            ",
+            // downgrade query
+            "
+                ALTER TABLE queue DROP COLUMN registry;
+            "
+        )
     ];
 
     for migration in migrations {

src/db/migrate.rs

@@ -4,8 +4,8 @@ mod queue;
 mod rustwide_builder;
 
 pub(crate) use self::limits::Limits;
-pub use self::rustwide_builder::RustwideBuilder;
 pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage};
+pub use self::rustwide_builder::{PackageKind, RustwideBuilder};
 
 use crate::db::Pool;
 use crate::error::Result;

src/docbuilder/mod.rs

@@ -1,6 +1,6 @@
 //! Updates registry index and builds new packages
 
-use super::{DocBuilder, RustwideBuilder};
+use super::{DocBuilder, PackageKind, RustwideBuilder};
 use crate::error::Result;
 use crate::utils::get_crate_priority;
 use crate::Index;
@@ -45,10 +45,12 @@ impl DocBuilder {
                 ChangeKind::Added => {
                     let priority = get_crate_priority(&mut conn, &krate.name)?;
 
-                    match self
-                        .build_queue
-                        .add_crate(&krate.name, &krate.version, priority)
-                    {
+                    match self.build_queue.add_crate(
+                        &krate.name,
+                        &krate.version,
+                        priority,
+                        index.repository_url(),
+                    ) {
                         Ok(()) => {
                             debug!("{}-{} added into build queue", krate.name, krate.version);
                             crates_added += 1;
@@ -79,7 +81,13 @@ impl DocBuilder {
         queue.process_next_crate(|krate| {
             processed = true;
 
-            builder.build_package(&krate.name, &krate.version, None)?;
+            let kind = krate
+                .registry
+                .as_ref()
+                .map(|r| PackageKind::Registry(r.as_str()))
+                .unwrap_or(PackageKind::CratesIo);
+
+            builder.build_package(&krate.name, &krate.version, kind)?;
             Ok(())
         })?;