scrape ms flag

src/main.rs

@@ -30,6 +30,13 @@ struct Args {
     #[arg(help = "Starting URL to scrape")]
     start_url: String,
 
+    #[arg(
+        long,
+        default_value = "1000",
+        help = "Delay in milliseconds between requests (default: 1000)"
+    )]
+    delay_ms: u64,
+
     #[arg(
         long,
         help = "Also crawl subdomains of the start URL's host (e.g. git.flo.fo when scraping flo.fo)"
@@ -82,6 +89,7 @@ async fn main() -> Result<()> {
 
     let start_url = args.start_url;
     let convert_docs = !args.no_doc_conversion;
+    let delay_ms = args.delay_ms;
 
     let url = Url::parse(&start_url)?;
 
@@ -135,6 +143,7 @@ async fn main() -> Result<()> {
     };
 
     log::info!("Scraping: {start_url}");
+    log::info!("Request delay: {delay_ms} ms");
     log::info!("Base host: {base_host}");
     log::info!(
         "Subdomain crawling: {}",
@@ -184,6 +193,7 @@ async fn main() -> Result<()> {
         scope_path,
         args.single,
         args.subdomains,
+        delay_ms,
     );
     let (count, errors) = scraper.run(&url).await;