scrape ms flag

src/scraper.rs

@@ -54,6 +54,7 @@ pub struct Scraper {
     scope_path: Option<String>,
     single_page: bool,
     allow_subdomains: bool,
+    delay_ms: u64,
 }
 
 impl Scraper {
@@ -70,6 +71,7 @@ impl Scraper {
         scope_path: Option<String>,
         single_page: bool,
         allow_subdomains: bool,
+        delay_ms: u64,
     ) -> Self {
         Self {
             fetcher,
@@ -85,6 +87,7 @@ impl Scraper {
             scope_path,
             single_page,
             allow_subdomains,
+            delay_ms,
         }
     }
 
@@ -213,7 +216,7 @@ impl Scraper {
             }
 
             count += 1;
-            tokio::time::sleep(Duration::from_millis(crate::config::RATE_LIMIT_MS)).await;
+            tokio::time::sleep(Duration::from_millis(self.delay_ms)).await;
         }
 
         if self.doc_stats.converted > 0 || self.doc_stats.raw > 0 || self.doc_stats.errors > 0 {
@@ -300,6 +303,8 @@ mod tests {
             output_dir: std::path::PathBuf::new(),
             log_path: std::path::PathBuf::new(),
             base_host: "example.com".to_string(),
+            allow_subdomains: false,
+            delay_ms: 1000,
             robots: RobotsRule {
                 allowed: Vec::new(),
                 disallowed: Vec::new(),
@@ -310,7 +315,6 @@ mod tests {
             exclude_types,
             scope_path: None,
             single_page: false,
-            allow_subdomains: false,
         }
     }