fixings

src/main.rs

@@ -1,13 +1,16 @@
-// src/main.rs
-
 mod config;
+mod converter;
 mod error_logger;
 mod extractor;
 mod fetcher;
+mod pdf_processor;
+mod robots;
 mod scraper;
 mod url_utils;
 
-use scraper::Scraper;
+use crate::config::USER_AGENT;
+use crate::robots::fetch_robots;
+use crate::scraper::Scraper;
 use std::fs;
 use std::path::PathBuf;
 use url::Url;
@@ -16,14 +19,44 @@ use url::Url;
 async fn main() -> anyhow::Result<()> {
     let args: Vec<String> = std::env::args().collect();
 
-    if args.len() < 2 {
-        eprintln!("Usage: site-scraper <start-url>");
-        eprintln!("Example: site-scraper https://www.logting.fo");
-        std::process::exit(1);
+    // Parse flags
+    let mut convert_pdfs = true;
+    let mut url_arg: Option<String> = None;
+
+    for arg in args.iter().skip(1) {
+        match arg.as_str() {
+            "--no-pdf-conversion" => convert_pdfs = false,
+            "--help" | "-h" => {
+                println!("Usage: site-scraper [OPTIONS] <start-url>");
+                println!();
+                println!("Options:");
+                println!("  --no-pdf-conversion   Save PDFs as-is, skip text extraction");
+                println!("  -h, --help            Show this help message");
+                println!();
+                println!("Example:");
+                println!("  site-scraper https://www.logting.fo");
+                println!("  site-scraper --no-pdf-conversion https://taks.fo");
+                std::process::exit(0);
+            }
+            _ => {
+                if url_arg.is_none() {
+                    url_arg = Some(arg.clone());
+                }
+            }
+        }
     }
 
-    let start_url = &args[1];
-    let url = Url::parse(start_url)?;
+    let start_url = match url_arg {
+        Some(u) => u,
+        None => {
+            eprintln!("Usage: site-scraper [OPTIONS] <start-url>");
+            eprintln!("Example: site-scraper https://www.logting.fo");
+            eprintln!("Run with --help for options.");
+            std::process::exit(1);
+        }
+    };
+
+    let url = Url::parse(&start_url)?;
 
     let base_domain = url_utils::derive_base_domain(&url)
         .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?;
@@ -40,9 +73,22 @@ async fn main() -> anyhow::Result<()> {
     println!("Scraping: {}", start_url);
     println!("Base domain: {}", base_domain);
     println!("Output dir: {}", output_dir.display());
+    println!(
+        "PDF conversion: {}",
+        if convert_pdfs { "enabled" } else { "disabled" }
+    );
+
+    // Fetch robots.txt
+    let client = reqwest::Client::builder()
+        .timeout(std::time::Duration::from_secs(30))
+        .user_agent(USER_AGENT)
+        .build()?;
+
+    println!("Fetching robots.txt...");
+    let robots = fetch_robots(&client, &url, "web-scraper").await;
     println!();
 
-    let scraper = Scraper::new(output_dir, log_path, base_domain);
+    let scraper = Scraper::new(output_dir, log_path, base_domain, robots, convert_pdfs);
     let (count, errors) = scraper.run(&url).await;
 
     println!("\nDone. Fetched {} URLs. Errors: {}.", count, errors);