Use new crates format

Bump to 0.3.0 for breaking change
Fail when old crate format is detected
Add new use_new_crates_format flag in mirror.toml
Add warning when flag is set, but new format is detected
Update nginx sample config
Update rewrite function

Author: k3d3

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "panamax"
-version = "0.2.3"
+version = "0.3.0"
 authors = ["k3d3 <[email protected]>"]
 description = "Mirror rustup and crates.io repositories, for offline Rust and Cargo usage."
 license = "MIT/Apache-2.0"

Cargo.toml

@@ -24,5 +24,11 @@ server {
 		fastcgi_param SCRIPT_FILENAME     /usr/lib/git-core/git-http-backend;
 		fastcgi_param GIT_HTTP_EXPORT_ALL "";
 		fastcgi_param PATH_INFO           $1;
-        }
+    }
+
+	# Rewrite the download URLs to match the proper crates location.
+	rewrite "^/crates/([^/])/([^/]+)$"                     "/crates/1/$1/$2"         last;
+	rewrite "^/crates/([^/]{2})/([^/]+)$"                  "/crates/2/$1/$2"         last;
+	rewrite "^/crates/([^/])([^/]{2})/([^/]+)$"            "/crates/3/$1/$1$2/$3"    last;
+	rewrite "^/crates/([^/]{2})([^/]{2})([^/]*)/([^/]+)$"  "/crates/$1/$2/$1$2$3/$4" last;
 }

nginx.sample.conf

@@ -5,7 +5,8 @@ use git2::Repository;
 use reqwest::header::HeaderValue;
 use scoped_threadpool::Pool;
 use serde::{Deserialize, Serialize};
-use std::path::Path;
+use std::fs::read_dir;
+use std::path::{Path, PathBuf};
 use std::{
     fs,
     io::{self, BufRead, Cursor},
@@ -56,11 +57,38 @@ pub fn sync_one_crate_entry(
         )
     };
 
+    // let file_path = path
+    //     .join("crates")
+    //     .join(&crate_entry.name)
+    //     .join(&crate_entry.vers)
+    //     .join("download");
+
+    let crate_name = format!("{}-{}.crate", &crate_entry.name, &crate_entry.vers);
+
+    let crate_path = match crate_entry.name.len() {
+        1 => PathBuf::from("1"),
+        2 => PathBuf::from("2"),
+        3 => PathBuf::from("3"),
+        n if n >= 4 => {
+            let first_two = &crate_entry
+                .name
+                .get(0..2)
+                .expect("crate name len >= 4 but couldn't get first 2 chars");
+            let second_two = &crate_entry
+                .name
+                .get(2..4)
+                .expect("crate name len >= 4 but couldn't get second 2 chars");
+            [first_two, second_two].iter().collect()
+        }
+        _ => return Err(DownloadError::BadCrate("Empty crate name".into())),
+    };
+
     let file_path = path
         .join("crates")
+        .join(crate_path)
         .join(&crate_entry.name)
         .join(&crate_entry.vers)
-        .join("download");
+        .join(crate_name);
 
     download(
         &url[..],
@@ -185,7 +213,8 @@ pub fn sync_crates_files(
                 // Get the data for this crate file
                 let oid = df.id();
                 if oid.is_zero() {
-                    // The crate was removed, continue to next crate
+                    // The crate was removed, continue to next crate.
+                    // Note that this does not include yanked crates.
                     removed_crates.push(p.to_path_buf());
                     return true;
                 }
@@ -250,3 +279,39 @@ pub fn sync_crates_files(
 
     Ok(())
 }
+
+/// Detect if the crates directory is using the old format.
+pub fn is_new_crates_format(path: &Path) -> Result<bool, io::Error> {
+    for crate_dir in read_dir(path)? {
+        let crate_dir = crate_dir?;
+        if !crate_dir.file_type()?.is_dir() {
+            // Ignore any files in the directory. Only look at other directories.
+            continue;
+        }
+
+        let dir_name = crate_dir
+            .file_name()
+            .into_string()
+            .map_err(|_| io::ErrorKind::Other)?;
+        match dir_name.as_str() {
+            "1" | "2" | "3" => continue, // 1-letter crate names cannot be numbers, so this must be new format.
+            x if x.len() == 2 => {
+                // Verify that this isn't a 2-char crate, by checking for 2 char crates inside.
+                // In the old format, this would contain version numbers, which are almost never 2 chars long.
+                for inner_dir in read_dir(path.join(crate_dir.file_name()))? {
+                    let inner_dir = inner_dir?;
+                    if inner_dir.file_name().len() != 2 {
+                        // Not the new crates format.
+                        return Ok(false);
+                    }
+                }
+            }
+            _ => {
+                // Unrecognized directory found, might be crate in old format.
+                return Ok(false);
+            }
+        };
+    }
+
+    Ok(true)
+}

src/crates.rs

@@ -147,9 +147,11 @@ pub fn rewrite_config_json(repo_path: &Path, base_url: &str) -> Result<(), Index
 
     let mut index = repo.index()?;
 
+    let crate_path = format!("{}/{}", base_url, "{crate}/{crate}-{version}.crate");
+
     // Create the new config.json.
     let config_json = ConfigJson {
-        dl: base_url.to_string(),
+        dl: crate_path,
         api: base_url.to_string(),
     };
     let contents = serde_json::to_vec_pretty(&config_json)?;

src/crates_index.rs

@@ -13,6 +13,8 @@ pub enum DownloadError {
     Io(#[from] io::Error),
     #[error("HTTP download error: {0}")]
     Download(#[from] reqwest::Error),
+    #[error("Got bad crate: {0}")]
+    BadCrate(String),
     #[error("Mismatched hash - expected '{expected}', got '{actual}'")]
     MismatchedHash { expected: String, actual: String },
     #[error("HTTP not found. Status: {status}, URL: {url}, data: {data}")]

src/download.rs

@@ -164,6 +164,9 @@ source = "https://crates.io/api/v1/crates"
 # Where to clone the crates.io-index repository from.
 source_index = "https://github.com/rust-lang/crates.io-index"
 
+# Set to true if the 0.3 crates directory format is being used.
+use_new_crates_format = true
+
 # URL where this mirror's crates directory can be accessed from.
 # Used for rewriting crates.io-index's config.json.
 # Remove this parameter to perform no rewriting.

src/mirror.default.toml

@@ -6,6 +6,7 @@ use reqwest::header::HeaderValue;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
+use crate::crates::is_new_crates_format;
 use crate::crates_index::rewrite_config_json;
 
 #[derive(Error, Debug)]
@@ -43,6 +44,7 @@ pub struct ConfigCrates {
     pub download_threads: usize,
     pub source: String,
     pub source_index: String,
+    pub use_new_crates_format: Option<bool>,
     pub base_url: Option<String>,
 }
 
@@ -121,6 +123,28 @@ pub fn sync(path: &Path) -> Result<(), MirrorError> {
     }
     let mirror = load_mirror_toml(path)?;
 
+    // Fail if use_new_crates_format is not true, and old format is detected.
+    // If use_new_crates_format is true and new format is detected, warn the user.
+    // If use_new_crates_format is true, ignore the format and assume it's new.
+    if let Some(crates) = &mirror.crates {
+        if crates.sync {
+            if crates.use_new_crates_format != Some(true) {
+                if is_new_crates_format(&path.join("crates"))? {
+                    eprintln!("Your crates/ directory is using the new 0.3 format, however");
+                    eprintln!("use_new_crates_format is not set in mirror.toml. To remove this warning,");
+                    eprintln!("Please add 'use_new_crates_format = true' to mirror.toml's [crates] section.");
+                    eprintln!();
+                } else {
+                    eprintln!("Your crates directory is using the old 0.2 format, however");
+                    eprintln!("Panamax 0.3 has deprecated this format for a new one.");
+                    eprintln!("Please delete crates/ and crates.io-index/ from your mirror to continue,");
+                    eprintln!("and add 'use_new_crates_format = true' to mirror.toml's [crates] section.");
+                    return Ok(());
+                }
+            }
+        }
+    }
+    
     // Handle the contact information
 
     let user_agent_str = if let Some(ref contact) = mirror.mirror.contact {