Add panamax serve command (panamax-rs#35)

* Initial implementation of panamax serve

* Fix up css, finish up backend stuff.

Everything works!

* Fix up some error handling

* Format everything

* Clippy everything

* One single space.

* Minor correction in structopt docstring

* Remove some unwraps

* Remove old TODO

* A few documentation things

* Multiple changes

Remove console.log
Use TlsConfig instead of tuple
Use Platform instead of tuple
Add some extra commenting

* Combine two git handlers into one

* Format

* Deduplicate crate path handling

* Formatting

* Clippy

Author: k3d3

Cargo.lock

@@ -25,10 +25,19 @@ glob = "0.3.0"
 git2 = "0.13.18"
 serde_json = "1.0.64"
 thiserror = "1.0.24"
+tokio = { version = "1.7.1", features = ["full"] }
+warp = { version = "0.3.1", features = ["tls"] }
+askama = "0.10.5"
+askama_warp = "0.11.0"
+include_dir = "0.6.0"
+bytes = "1.0.1"
+tokio-stream = "0.1.6"
+tokio-util = "0.6.7"
+futures-util = "0.3.15"
 
 [features]
 default = []
 
 # For development purposes, reduce the crate download count
 # and only download crates that start with z
-dev_reduced_crates = []
+dev_reduced_crates = []

Cargo.toml

@@ -24,6 +24,8 @@ Panamax is available as a docker image, so you can run:
 $ docker run -it -v /path/to/mirror/:/mirror k3d3/panamax init /mirror
 (Modify /path/to/mirror/mirror.toml as needed)
 $ docker run -it -v /path/to/mirror/:/mirror k3d3/panamax sync /mirror
+(Once synced, serve the mirror)
+$ docker run -it -v /path/to/mirror/:/mirror -p8080:8080 k3d3/panamax serve /mirror
 ```
 
 Alternatively, you can run panamax in a bare-metal environment like below.
@@ -79,9 +81,18 @@ Additionally, this mirror can continually by synchronized in the future - one re
 
 ## Server
 
-Panamax grabs the files needed to make a full mirror, however once the mirror directory is at its destination, it needs to be hosted as a server. Panamax doesn't provide this, however it should be fairly simple to host a mirror - everything can be accessed via HTTP, with the exception of the `crates.io-index` which uses git.
+Panamax provides a warp-based HTTP(S) server that can handle serving a Rust mirror fast and at scale. This is the recommended way to serve the mirror.
 
-A sample `nginx` configuration file, `nginx.sample.conf` has been provided in the repository which will handle hosting a mirror server. Use this in the `sites-available` nginx directory, or copy it into `nginx.conf`.
+```
+$ panamax serve my-mirror
+Running HTTP on [::]:8080
+```
+
+The server's index page provides all the instructions needed on how to set up a Rust client that uses this mirror.
+
+If you would prefer having these instructions elsewhere, the rest of this README will describe the setup process in more detail.
+
+Additionally, if you would prefer hosting a server with nginx, there is a sample nginx configuration in the repository, at `nginx.sample.conf`.
 
 ## Configuring `rustup` and `cargo`
 

README.md

@@ -57,32 +57,8 @@ pub fn sync_one_crate_entry(
         )
     };
 
-    let crate_name = format!("{}-{}.crate", &crate_entry.name, &crate_entry.vers);
-
-    let crate_path = match crate_entry.name.len() {
-        1 => PathBuf::from("1"),
-        2 => PathBuf::from("2"),
-        3 => PathBuf::from("3"),
-        n if n >= 4 => {
-            let first_two = &crate_entry
-                .name
-                .get(0..2)
-                .expect("crate name len >= 4 but couldn't get first 2 chars");
-            let second_two = &crate_entry
-                .name
-                .get(2..4)
-                .expect("crate name len >= 4 but couldn't get second 2 chars");
-            [first_two, second_two].iter().collect()
-        }
-        _ => return Err(DownloadError::BadCrate("Empty crate name".into())),
-    };
-
-    let file_path = path
-        .join("crates")
-        .join(crate_path)
-        .join(&crate_entry.name)
-        .join(&crate_entry.vers)
-        .join(crate_name);
+    let file_path = get_crate_path(path, &crate_entry.name, &crate_entry.vers)
+        .ok_or_else(|| DownloadError::BadCrate(crate_entry.name.clone()))?;
 
     download(
         &url[..],
@@ -306,3 +282,30 @@ pub fn is_new_crates_format(path: &Path) -> Result<bool, io::Error> {
 
     Ok(true)
 }
+
+pub fn get_crate_path(
+    mirror_path: &Path,
+    crate_name: &str,
+    crate_version: &str,
+) -> Option<PathBuf> {
+    let crate_path = match crate_name.len() {
+        1 => PathBuf::from("1"),
+        2 => PathBuf::from("2"),
+        3 => PathBuf::from("3"),
+        n if n >= 4 => {
+            let first_two = crate_name.get(0..2)?;
+            let second_two = crate_name.get(2..4)?;
+            [first_two, second_two].iter().collect()
+        }
+        _ => return None,
+    };
+
+    Some(
+        mirror_path
+            .join("crates")
+            .join(crate_path)
+            .join(crate_name)
+            .join(crate_version)
+            .join(format!("{}-{}.crate", crate_name, crate_version)),
+    )
+}

src/crates.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::{net::IpAddr, path::PathBuf};
 use structopt::StructOpt;
 
 mod crates;
@@ -7,6 +7,7 @@ mod download;
 mod mirror;
 mod progress_bar;
 mod rustup;
+mod serve;
 
 /// Mirror rustup and crates.io repositories, for offline Rust and cargo usage.
 #[derive(Debug, StructOpt)]
@@ -41,6 +42,33 @@ enum Panamax {
         #[structopt(short, long)]
         base_url: Option<String>,
     },
+
+    /// Serve a mirror directory.
+    #[structopt(name = "serve")]
+    Serve {
+        /// Mirror directory.
+        #[structopt(parse(from_os_str))]
+        path: PathBuf,
+
+        /// IP address to listen on. Defaults to listening on everything.
+        #[structopt(short, long)]
+        listen: Option<IpAddr>,
+
+        /// Port to listen on.
+        /// Defaults to 8080, or 8443 if TLS certificate provided.
+        #[structopt(short, long)]
+        port: Option<u16>,
+
+        /// Path to a TLS certificate file. This enables TLS.
+        /// Also requires key_path.
+        #[structopt(long)]
+        cert_path: Option<PathBuf>,
+
+        /// Path to a TLS key file.
+        /// Also requires cert_path.
+        #[structopt(long)]
+        key_path: Option<PathBuf>,
+    },
 }
 
 fn main() {
@@ -50,6 +78,13 @@ fn main() {
         Panamax::Init { path } => mirror::init(&path),
         Panamax::Sync { path } => mirror::sync(&path),
         Panamax::Rewrite { path, base_url } => mirror::rewrite(&path, base_url),
+        Panamax::Serve {
+            path,
+            listen,
+            port,
+            cert_path,
+            key_path,
+        } => mirror::serve(path, listen, port, cert_path, key_path),
     }
     .unwrap_or_else(|e| eprintln!("Panamax command failed! {}", e));
 }

src/main.rs

@@ -1,4 +1,5 @@
-use std::path::Path;
+use std::net::{IpAddr, SocketAddr};
+use std::path::{Path, PathBuf};
 use std::{fs, io};
 
 use console::style;
@@ -8,6 +9,7 @@ use thiserror::Error;
 
 use crate::crates::is_new_crates_format;
 use crate::crates_index::rewrite_config_json;
+use crate::serve::TlsConfig;
 
 #[derive(Error, Debug)]
 pub enum MirrorError {
@@ -17,6 +19,8 @@ pub enum MirrorError {
     Parse(#[from] toml::de::Error),
     #[error("Config file error: {0}")]
     Config(String),
+    #[error("Command line error: {0}")]
+    CmdLine(String),
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -244,3 +248,44 @@ pub fn sync_crates(
 
     eprintln!("{}", style("Syncing Crates repositories complete!").bold());
 }
+
+pub fn serve(
+    path: PathBuf,
+    listen: Option<IpAddr>,
+    port: Option<u16>,
+    cert_path: Option<PathBuf>,
+    key_path: Option<PathBuf>,
+) -> Result<(), MirrorError> {
+    let listen = listen.unwrap_or_else(|| {
+        "::".parse()
+            .expect(":: IPv6 address should never fail to parse")
+    });
+    let port = port.unwrap_or_else(|| if cert_path.is_some() { 8443 } else { 8080 });
+    let socket_addr = SocketAddr::new(listen, port);
+
+    let rt = tokio::runtime::Runtime::new()?;
+
+    match (cert_path, key_path) {
+        (Some(cert_path), Some(key_path)) => rt.block_on(crate::serve::serve(
+            path,
+            socket_addr,
+            Some(TlsConfig {
+                cert_path,
+                key_path,
+            }),
+        )),
+        (None, None) => rt.block_on(crate::serve::serve(path, socket_addr, None)),
+        (Some(_), None) => {
+            return Err(MirrorError::CmdLine(
+                "cert_path set but key_path not set.".to_string(),
+            ))
+        }
+        (None, Some(_)) => {
+            return Err(MirrorError::CmdLine(
+                "key_path set but cert_path not set.".to_string(),
+            ))
+        }
+    };
+
+    Ok(())
+}