fix test and more clippy

src/scraper.rs

@@ -45,7 +45,7 @@ pub struct Scraper {
     seen: HashSet<String>,
     output_dir: std::path::PathBuf,
     log_path: std::path::PathBuf,
-    base_domain: String,
+    base_host: String,
     robots: RobotsRule,
     doc_stats: DocStats,
     convert_docs: bool,
@@ -53,6 +53,7 @@ pub struct Scraper {
     exclude_types: Option<HashSet<String>>,
     scope_path: Option<String>,
     single_page: bool,
+    allow_subdomains: bool,
 }
 
 impl Scraper {
@@ -60,7 +61,7 @@ impl Scraper {
     pub fn new(
         output_dir: std::path::PathBuf,
         log_path: std::path::PathBuf,
-        base_domain: String,
+        base_host: String,
         robots: RobotsRule,
         convert_docs: bool,
         fetcher: Fetcher,
@@ -68,13 +69,14 @@ impl Scraper {
         exclude_types: Option<HashSet<String>>,
         scope_path: Option<String>,
         single_page: bool,
+        allow_subdomains: bool,
     ) -> Self {
         Self {
             fetcher,
             seen: HashSet::new(),
             output_dir,
             log_path,
-            base_domain,
+            base_host,
             robots,
             doc_stats: DocStats::default(),
             convert_docs,
@@ -82,6 +84,7 @@ impl Scraper {
             exclude_types,
             scope_path,
             single_page,
+            allow_subdomains,
         }
     }
 
@@ -111,7 +114,7 @@ impl Scraper {
         html: &str,
         final_url: &Url,
     ) -> (usize, Vec<Url>) {
-        let links = extract_links(html, final_url, &self.base_domain);
+        let links = extract_links(html, final_url, &self.base_host, self.allow_subdomains);
         let new_count = links.len();
         let new_links: Vec<Url> = links
             .into_iter()
@@ -296,7 +299,7 @@ mod tests {
             seen: HashSet::new(),
             output_dir: std::path::PathBuf::new(),
             log_path: std::path::PathBuf::new(),
-            base_domain: "example.com".to_string(),
+            base_host: "example.com".to_string(),
             robots: RobotsRule {
                 allowed: Vec::new(),
                 disallowed: Vec::new(),
@@ -307,6 +310,7 @@ mod tests {
             exclude_types,
             scope_path: None,
             single_page: false,
+            allow_subdomains: false,
         }
     }