99 lines
3.0 KiB
Rust
99 lines
3.0 KiB
Rust
mod config;
|
|
mod converter;
|
|
mod error_logger;
|
|
mod extractor;
|
|
mod fetcher;
|
|
mod pdf_processor;
|
|
mod robots;
|
|
mod scraper;
|
|
mod url_utils;
|
|
|
|
use crate::config::USER_AGENT;
|
|
use crate::robots::fetch_robots;
|
|
use crate::scraper::Scraper;
|
|
use std::fs;
|
|
use std::path::PathBuf;
|
|
use url::Url;
|
|
|
|
#[tokio::main]
|
|
async fn main() -> anyhow::Result<()> {
|
|
let args: Vec<String> = std::env::args().collect();
|
|
|
|
// Parse flags
|
|
let mut convert_pdfs = true;
|
|
let mut url_arg: Option<String> = None;
|
|
|
|
for arg in args.iter().skip(1) {
|
|
match arg.as_str() {
|
|
"--no-pdf-conversion" => convert_pdfs = false,
|
|
"--help" | "-h" => {
|
|
println!("Usage: site-scraper [OPTIONS] <start-url>");
|
|
println!();
|
|
println!("Options:");
|
|
println!(" --no-pdf-conversion Save PDFs as-is, skip text extraction");
|
|
println!(" -h, --help Show this help message");
|
|
println!();
|
|
println!("Example:");
|
|
println!(" site-scraper https://www.logting.fo");
|
|
println!(" site-scraper --no-pdf-conversion https://taks.fo");
|
|
std::process::exit(0);
|
|
}
|
|
_ => {
|
|
if url_arg.is_none() {
|
|
url_arg = Some(arg.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let start_url = match url_arg {
|
|
Some(u) => u,
|
|
None => {
|
|
eprintln!("Usage: site-scraper [OPTIONS] <start-url>");
|
|
eprintln!("Example: site-scraper https://www.logting.fo");
|
|
eprintln!("Run with --help for options.");
|
|
std::process::exit(1);
|
|
}
|
|
};
|
|
|
|
let url = Url::parse(&start_url)?;
|
|
|
|
let base_domain = url_utils::derive_base_domain(&url)
|
|
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?;
|
|
|
|
let output_dir_name = format!("scraped_{}", base_domain.replace('.', "_"));
|
|
let output_dir = PathBuf::from(&output_dir_name);
|
|
let logs_dir = output_dir.join("logs");
|
|
|
|
fs::create_dir_all(&logs_dir)?;
|
|
|
|
let log_path = logs_dir.join("scrape_errors.jsonl");
|
|
fs::remove_file(&log_path).ok();
|
|
|
|
println!("Scraping: {}", start_url);
|
|
println!("Base domain: {}", base_domain);
|
|
println!("Output dir: {}", output_dir.display());
|
|
println!(
|
|
"PDF conversion: {}",
|
|
if convert_pdfs { "enabled" } else { "disabled" }
|
|
);
|
|
|
|
// Fetch robots.txt
|
|
let client = reqwest::Client::builder()
|
|
.timeout(std::time::Duration::from_secs(30))
|
|
.user_agent(USER_AGENT)
|
|
.build()?;
|
|
|
|
println!("Fetching robots.txt...");
|
|
let robots = fetch_robots(&client, &url, "web-scraper").await;
|
|
println!();
|
|
|
|
let scraper = Scraper::new(output_dir, log_path, base_domain, robots, convert_pdfs);
|
|
let (count, errors) = scraper.run(&url).await;
|
|
|
|
println!("\nDone. Fetched {} URLs. Errors: {}.", count, errors);
|
|
println!("Error log: {}/logs/scrape_errors.jsonl", output_dir_name);
|
|
|
|
Ok(())
|
|
}
|