fix test and more clippy

src/main.rs

@@ -30,6 +30,12 @@ struct Args {
     #[arg(help = "Starting URL to scrape")]
     start_url: String,
 
+    #[arg(
+        long,
+        help = "Also crawl subdomains of the start URL's host (e.g. git.flo.fo when scraping flo.fo)"
+    )]
+    subdomains: bool,
+
     #[arg(
         long,
         help = "Only scrape the single page at START_URL, do not crawl for links"
@@ -79,6 +85,11 @@ async fn main() -> Result<()> {
 
     let url = Url::parse(&start_url)?;
 
+    let base_host = url
+        .host_str()
+        .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?
+        .to_string();
+
     let base_domain = url_utils::derive_base_domain(&url)
         .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?;
 
@@ -124,7 +135,11 @@ async fn main() -> Result<()> {
     };
 
     log::info!("Scraping: {start_url}");
-    log::info!("Base domain: {base_domain}");
+    log::info!("Base host: {base_host}");
+    log::info!(
+        "Subdomain crawling: {}",
+        if args.subdomains { "enabled" } else { "disabled" }
+    );
     log::info!("Output dir: {}", output_dir.display());
     log::info!("Logs dir: {}", logs_dir.display());
     log::info!(
@@ -160,7 +175,7 @@ async fn main() -> Result<()> {
     let mut scraper = Scraper::new(
         output_dir,
         log_path.clone(),
-        base_domain,
+        base_host,
         robots,
         convert_docs,
         fetcher,
@@ -168,6 +183,7 @@ async fn main() -> Result<()> {
         exclude_types,
         scope_path,
         args.single,
+        args.subdomains,
     );
     let (count, errors) = scraper.run(&url).await;