Add different verbosity levels

README.md

@@ -223,8 +223,11 @@ Options:
 
           [default: ./lychee.toml]
 
-  -v, --verbose
-          Verbose program output
+  -v, --verbose...
+          Set verbosity level; more output per occurrence (e.g. `-v` or `-vv`)
+
+  -q, --quiet...
+          Less output per occurrence (e.g. `-q` or `-qq`)
 
   -n, --no-progress
           Do not show progress bar.

fixtures/TEST_DATA_URIS.html

@@ -0,0 +1,8 @@
+<a href="http://www.w3.org/2000/svg">False positive link</a>
+<a href="data:,Hello%2C%20World%21">Hello World</a>
+<a href="data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==">Plaintext</a>
+<a
+  href="data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 147 40'"
+  >data URI</a
+>
+<a href="http://localhost/assets/img/bg-water.webp">normal link</a>

lychee-bin/Cargo.toml

@@ -48,6 +48,7 @@ humantime-serde = "1.1.1"
 secrecy = { version = "0.8.0", features = ["serde"] }
 supports-color = "1.3.0"
 log = "0.4.17"
+env_logger = "0.9.3"
 
 [dependencies.clap]
 version = "4.0.27"

lychee-bin/src/commands/check.rs

@@ -12,6 +12,7 @@ use tokio_stream::wrappers::ReceiverStream;
 use tokio_stream::StreamExt;
 
 use crate::formatters::response::ResponseFormatter;
+use crate::verbosity::{Verbosity, WarnLevel};
 use crate::{cache::Cache, stats::ResponseStats, ExitCode};
 use lychee_lib::{Client, Request, Response};
 
@@ -71,7 +72,7 @@ where
         let verbose = params.cfg.verbose;
         async move {
             while let Some(response) = recv_resp.recv().await {
-                show_progress(&mut io::stdout(), &pb, &response, &formatter, verbose)?;
+                show_progress(&mut io::stdout(), &pb, &response, &formatter, &verbose)?;
                 stats.add(response);
             }
             Ok((pb, stats))
@@ -162,23 +163,26 @@ fn show_progress(
     progress_bar: &Option<ProgressBar>,
     response: &Response,
     formatter: &Arc<Box<dyn ResponseFormatter>>,
-    verbose: bool,
+    verbose: &Verbosity<WarnLevel>,
 ) -> Result<()> {
     let out = formatter.write_response(response)?;
     if let Some(pb) = progress_bar {
         pb.inc(1);
         pb.set_message(out.clone());
-        if verbose {
+        if verbose.is_verbose() {
             pb.println(out);
         }
-    } else if verbose || (!response.status().is_success() && !response.status().is_excluded()) {
+    } else if verbose.is_verbose()
+        || (!response.status().is_success() && !response.status().is_excluded())
+    {
         writeln!(output, "{}", out)?;
     }
     Ok(())
 }
 
 #[cfg(test)]
 mod tests {
+    use log::info;
     use lychee_lib::{CacheStatus, InputSource, ResponseBody, Uri};
 
     use crate::formatters;
@@ -197,9 +201,16 @@ mod tests {
         );
         let formatter: Arc<Box<dyn ResponseFormatter>> =
             Arc::new(Box::new(formatters::response::Raw::new()));
-        show_progress(&mut buf, &None, &response, &formatter, false).unwrap();
+        show_progress(
+            &mut buf,
+            &None,
+            &response,
+            &formatter,
+            &Verbosity::new(0, 0),
+        )
+        .unwrap();
 
-        println!("{:?}", String::from_utf8_lossy(&buf));
+        info!("{:?}", String::from_utf8_lossy(&buf));
         assert!(buf.is_empty());
     }
 }

lychee-bin/src/commands/dump.rs

@@ -1,10 +1,13 @@
+use log::error;
 use lychee_lib::Request;
 use lychee_lib::Result;
 use std::fs;
 use std::io::{self, Write};
 use std::path::PathBuf;
 use tokio_stream::StreamExt;
 
+use crate::verbosity::Verbosity;
+use crate::verbosity::WarnLevel;
 use crate::ExitCode;
 
 use super::CommandParams;
@@ -53,14 +56,13 @@ where
         // to another program like `grep`.
 
         let excluded = params.client.is_excluded(&request.uri);
-        let verbose = params.cfg.verbose;
 
-        if excluded && !verbose {
+        if excluded && !params.cfg.verbose.is_verbose() {
             continue;
         }
-        if let Err(e) = write(&mut writer, &request, verbose, excluded) {
+        if let Err(e) = write(&mut writer, &request, &params.cfg.verbose, excluded) {
             if e.kind() != io::ErrorKind::BrokenPipe {
-                eprintln!("{e}");
+                error!("{e}");
                 return Ok(ExitCode::UnexpectedFailure);
             }
         }
@@ -73,10 +75,15 @@ where
 fn write(
     writer: &mut Box<dyn Write>,
     request: &Request,
-    verbose: bool,
+    verbosity: &Verbosity<WarnLevel>,
     excluded: bool,
 ) -> io::Result<()> {
-    let request = if verbose {
+    // Only print `data:` URIs if verbose mode is enabled
+    if request.uri.is_data() && !verbosity.is_verbose() {
+        return Ok(());
+    }
+
+    let request = if verbosity.is_verbose() {
         // Only print source in verbose mode. This way the normal link output
         // can be fed into another tool without data mangling.
         request.to_string()

lychee-bin/src/main.rs

@@ -67,8 +67,9 @@ use clap::Parser;
 use color::YELLOW;
 use commands::CommandParams;
 use formatters::response::ResponseFormatter;
-use log::warn;
-use openssl_sys as _; // required for vendored-openssl feature
+use log::{info, warn};
+use openssl_sys as _;
+// required for vendored-openssl feature
 use ring as _; // required for apple silicon
 
 use lychee_lib::Collector;
@@ -82,6 +83,7 @@ mod options;
 mod parse;
 mod stats;
 mod time;
+mod verbosity;
 
 use crate::{
     cache::{Cache, StoreExt},
@@ -132,6 +134,16 @@ fn read_lines(file: &File) -> Result<Vec<String>> {
 fn load_config() -> Result<LycheeOptions> {
     let mut opts = LycheeOptions::parse();
 
+    env_logger::Builder::new()
+        // super basic formatting; no timestamps, no module path, no target
+        .format_timestamp(None)
+        .format_indent(Some(0))
+        .format_module_path(false)
+        .format_target(false)
+        .filter_module("lychee", opts.config.verbose.log_level_filter())
+        .filter_module("lychee_lib", opts.config.verbose.log_level_filter())
+        .init();
+
     // Load a potentially existing config file and merge it into the config from
     // the CLI
     if let Some(c) = Config::load_from_file(&opts.config_file)? {
@@ -177,21 +189,26 @@ fn load_cache(cfg: &Config) -> Option<Cache> {
             let modified = metadata.modified().ok()?;
             let elapsed = modified.elapsed().ok()?;
             if elapsed > cfg.max_cache_age {
-                eprintln!(
-                    "Cache is too old (age: {}, max age: {}). Discarding and recreating",
+                warn!(
+                    "Cache is too old (age: {}, max age: {}). Discarding and recreating.",
                     humantime::format_duration(elapsed),
                     humantime::format_duration(cfg.max_cache_age)
                 );
                 return None;
             }
+            info!(
+                "Cache is recent (age: {}, max age: {}). Using.",
+                humantime::format_duration(elapsed),
+                humantime::format_duration(cfg.max_cache_age)
+            );
         }
     }
 
     let cache = Cache::load(LYCHEE_CACHE_FILE, cfg.max_cache_age.as_secs());
     match cache {
         Ok(cache) => Some(cache),
         Err(e) => {
-            eprintln!("Error while loading cache: {e}. Continuing without.");
+            warn!("Error while loading cache: {e}. Continuing without.");
             None
         }
     }
@@ -285,7 +302,7 @@ async fn run(opts: &LycheeOptions) -> Result<i32> {
             if let Some(output) = &opts.config.output {
                 fs::write(output, formatted).context("Cannot write status output to file")?;
             } else {
-                if opts.config.verbose && !is_empty {
+                if opts.config.verbose.log_level() == Some(log::Level::Debug) && !is_empty {
                     // separate summary from the verbose list of links above
                     // with a newline
                     writeln!(io::stdout())?;

lychee-bin/src/options.rs

@@ -1,4 +1,5 @@
 use crate::parse::{parse_base, parse_statuscodes};
+use crate::verbosity::{Verbosity, WarnLevel};
 use anyhow::{anyhow, Context, Error, Result};
 use clap::{arg, Parser};
 use const_format::{concatcp, formatcp};
@@ -113,7 +114,7 @@ pub(crate) struct LycheeOptions {
     #[arg(short, long = "config", default_value = "./lychee.toml")]
     pub(crate) config_file: String,
 
-    #[command(flatten)]
+    #[clap(flatten)]
     pub(crate) config: Config,
 }
 
@@ -140,9 +141,8 @@ impl LycheeOptions {
 #[derive(Parser, Debug, Deserialize, Clone)]
 pub(crate) struct Config {
     /// Verbose program output
-    #[arg(short, long)]
-    #[serde(default)]
-    pub(crate) verbose: bool,
+    #[clap(flatten)]
+    pub(crate) verbose: Verbosity<WarnLevel>,
 
     /// Do not show progress bar.
     /// This is recommended for non-interactive shells (e.g. for continuous integration)
@@ -363,7 +363,7 @@ impl Config {
             self, toml;
 
             // Keys with defaults to assign
-            verbose: false;
+            verbose: Verbosity::new(0, 0);
             cache: false;
             no_progress: false;
             max_redirects: DEFAULT_MAX_REDIRECTS;