diff --git a/Cargo.lock b/Cargo.lock index f473db1..4eb152d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2601,7 +2601,7 @@ dependencies = [ [[package]] name = "web-scraper" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anydoc", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 8228616..d4a8683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "web-scraper" -version = "0.1.1" +version = "0.1.2" edition = "2024" [dependencies] diff --git a/src/main.rs b/src/main.rs index 4de36d3..71e4691 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,17 +26,11 @@ use url::Url; #[command(author = "FLÓ")] #[command(version = "0.2.0")] #[command(about = "Web scraper for Faroese public sector sites")] +#[allow(clippy::struct_excessive_bools)] struct Args { #[arg(help = "Starting URL to scrape")] start_url: String, - #[arg( - long, - default_value = "1000", - help = "Delay in milliseconds between requests (default: 1000)" - )] - delay_ms: u64, - #[arg( long, help = "Also crawl subdomains of the start URL's host (e.g. git.flo.fo when scraping flo.fo)" @@ -49,9 +43,19 @@ struct Args { )] single: bool, + #[arg(long, help = "Disable path scoping — crawl any path on the same host")] + no_scope: bool, + #[arg(long, help = "Save documents as-is, skip Markdown conversion")] no_doc_conversion: bool, + #[arg( + long, + default_value = "1000", + help = "Delay in milliseconds between requests (default: 1000)" + )] + delay_ms: u64, + #[arg( long, help = "Comma-separated extensions to include (e.g. pdf,docx,html). \ @@ -81,6 +85,50 @@ fn parse_ext_list(s: &str) -> HashSet { .collect() } +#[allow(clippy::too_many_arguments)] +fn print_config( + start_url: &str, + base_host: &str, + subdomains: bool, + output_dir: &std::path::Path, + logs_dir: &std::path::Path, + convert_docs: bool, + single: bool, + delay_ms: u64, + include_types: Option<&HashSet>, + exclude_types: Option<&HashSet>, + scope_path: Option<&String>, +) { + log::info!("Scraping: {start_url}"); + log::info!("Base host: {base_host}"); + log::info!( + "Subdomain crawling: {}", + if subdomains { "enabled" } else { "disabled" } + ); + log::info!("Output dir: {}", output_dir.display()); + log::info!("Logs dir: {}", logs_dir.display()); + log::info!( + "Document conversion: {}", + if convert_docs { "enabled" } else { "disabled" } + ); + log::info!( + "Single page mode: {}", + if single { "enabled" } else { "disabled" } + ); + log::info!("Request delay: {delay_ms} ms"); + + match (include_types, exclude_types) { + (Some(t), _) => log::info!("Type filter: include {t:?}"), + (_, Some(t)) => log::info!("Type filter: exclude {t:?}"), + _ => log::info!("Type filter: none (all types)"), + } + + match scope_path { + Some(s) => log::info!("Path scope: {s} (only this path and deeper)"), + None => log::info!("Path scope: none (full site)"), + } +} + #[tokio::main] async fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); @@ -134,7 +182,9 @@ async fn main() -> Result<()> { (None, None) => (None, None), }; - let scope_path = { + let scope_path = if args.no_scope { + None + } else { let path = url.path().trim_end_matches('/'); if path.is_empty() { None @@ -143,38 +193,19 @@ async fn main() -> Result<()> { } }; - log::info!("Scraping: {start_url}"); - log::info!("Request delay: {delay_ms} ms"); - log::info!("Base host: {base_host}"); - log::info!( - "Subdomain crawling: {}", - if args.subdomains { - "enabled" - } else { - "disabled" - } + print_config( + &start_url, + &base_host, + args.subdomains, + &output_dir, + &logs_dir, + convert_docs, + args.single, + delay_ms, + include_types.as_ref(), + exclude_types.as_ref(), + scope_path.as_ref(), ); - log::info!("Output dir: {}", output_dir.display()); - log::info!("Logs dir: {}", logs_dir.display()); - log::info!( - "Document conversion: {}", - if convert_docs { "enabled" } else { "disabled" } - ); - log::info!( - "Single page mode: {}", - if args.single { "enabled" } else { "disabled" } - ); - - match (&include_types, &exclude_types) { - (Some(t), _) => log::info!("Type filter: include {t:?}"), - (_, Some(t)) => log::info!("Type filter: exclude {t:?}"), - _ => log::info!("Type filter: none (all types)"), - } - - match &scope_path { - Some(s) => log::info!("Path scope: {s} (only this path and deeper)"), - None => log::info!("Path scope: none (full site)"), - } let client = Client::builder() .timeout(Duration::from_secs(config::TIMEOUT_SECS))