yeah works

src/scraper.rs

@@ -0,0 +1,135 @@
+// src/scraper.rs
+
+use crate::error_logger::log_error;
+use crate::extractor::extract_links;
+use crate::fetcher::{FetchResult, Fetcher};
+use crate::url_utils::{normalize_url, url_to_filename};
+use anyhow::Result;
+use std::collections::HashSet;
+use std::collections::VecDeque;
+use std::fs;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Mutex;
+use url::Url;
+
+pub struct Scraper {
+    fetcher: Fetcher,
+    seen: Arc<Mutex<HashSet<String>>>,
+    output_dir: std::path::PathBuf,
+    log_path: std::path::PathBuf,
+    base_domain: String,
+}
+
+impl Scraper {
+    pub fn new(
+        output_dir: std::path::PathBuf,
+        log_path: std::path::PathBuf,
+        base_domain: String,
+    ) -> Self {
+        Self {
+            fetcher: Fetcher::new(),
+            seen: Arc::new(Mutex::new(HashSet::new())),
+            output_dir,
+            log_path,
+            base_domain,
+        }
+    }
+
+    pub async fn run(&self, start_url: &Url) -> (usize, usize) {
+        let mut queue: VecDeque<Url> = VecDeque::new();
+        queue.push_back(start_url.clone());
+
+        let mut count = 0usize;
+        let mut error_count = 0usize;
+
+        while let Some(raw_url) = queue.pop_front() {
+            let url = match normalize_url(raw_url.as_str()) {
+                Some(u) => u,
+                None => continue,
+            };
+
+            let url_key = url.as_str().to_string();
+
+            {
+                let mut seen = self.seen.lock().await;
+                if seen.contains(&url_key) {
+                    continue;
+                }
+                seen.insert(url_key.clone());
+            }
+
+            println!("[{}] fetching: {}", count, url);
+
+            match self.fetcher.fetch_with_retry(&url).await {
+                Ok(result) => {
+                    let is_html = result.content_type.contains("text/html");
+
+                    match self.save(&result).await {
+                        Ok(()) => {
+                            if is_html {
+                                if let Ok(html) = std::str::from_utf8(&result.bytes) {
+                                    let links =
+                                        extract_links(html, &result.final_url, &self.base_domain);
+
+                                    // Only queue links not already seen — do NOT insert into seen here.
+                                    // They get inserted when actually fetched, matching the Python version.
+                                    let mut new_links = Vec::new();
+
+                                    {
+                                        let seen = self.seen.lock().await;
+                                        for link in &links {
+                                            if !seen.contains(link.as_str()) {
+                                                new_links.push(link.clone());
+                                            }
+                                        }
+                                    }
+
+                                    let new_count = new_links.len();
+
+                                    for link in new_links {
+                                        queue.push_back(link);
+                                    }
+
+                                    println!("  found {} links ({} new)", links.len(), new_count);
+                                }
+                            } else {
+                                println!("  binary: {}", result.content_type);
+                            }
+                        }
+                        Err(e) => {
+                            log_error(
+                                &self.log_path,
+                                &result.final_url,
+                                "save_error",
+                                &e.to_string(),
+                                None,
+                            );
+                            error_count += 1;
+                        }
+                    }
+                }
+                Err(e) => {
+                    log_error(&self.log_path, &url, "fetch_error", &e.to_string(), None);
+                    error_count += 1;
+                }
+            }
+
+            count += 1;
+            tokio::time::sleep(Duration::from_millis(crate::config::DELAY_MS)).await;
+        }
+
+        (count, error_count)
+    }
+
+    async fn save(&self, result: &FetchResult) -> Result<()> {
+        let filename = url_to_filename(&result.final_url, &result.content_type);
+        let filepath = self.output_dir.join(&filename);
+        fs::write(&filepath, &result.bytes)?;
+        println!(
+            "  saved -> {}",
+            filepath.file_name().unwrap_or_default().to_string_lossy()
+        );
+        Ok(())
+    }
+}