88 lines
2.3 KiB
Rust
88 lines
2.3 KiB
Rust
mod config;
|
|
mod converter;
|
|
mod doc_processor;
|
|
mod error_logger;
|
|
mod extractor;
|
|
mod fetcher;
|
|
mod robots;
|
|
mod scraper;
|
|
mod url_utils;
|
|
|
|
use crate::robots::fetch_robots;
|
|
use crate::scraper::Scraper;
|
|
use anyhow::Result;
|
|
use clap::Parser;
|
|
use std::fs;
|
|
use std::path::PathBuf;
|
|
use url::Url;
|
|
|
|
#[derive(Parser, Debug)]
|
|
#[command(name = "rs-scraper")]
|
|
#[command(author = "FLÓ")]
|
|
#[command(version = "0.1.0")]
|
|
#[command(about = "Web scraper for Faroese public sector sites")]
|
|
struct Args {
|
|
#[arg(help = "Starting URL to scrape")]
|
|
start_url: String,
|
|
|
|
#[arg(long, help = "Save documents as-is, skip Markdown conversion")]
|
|
no_doc_conversion: bool,
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
|
|
|
|
let args = Args::parse();
|
|
|
|
let start_url = args.start_url;
|
|
let convert_docs = !args.no_doc_conversion;
|
|
|
|
let url = Url::parse(&start_url)?;
|
|
|
|
let base_domain = url_utils::derive_base_domain(&url)
|
|
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?;
|
|
|
|
let output_dir_name = format!("scraped_{}", base_domain.replace('.', "_"));
|
|
let output_dir = PathBuf::from(&output_dir_name);
|
|
let logs_dir = output_dir.join("logs");
|
|
|
|
fs::create_dir_all(&logs_dir)?;
|
|
|
|
let log_path = logs_dir.join("scrape_errors.jsonl");
|
|
fs::remove_file(&log_path).ok();
|
|
|
|
log::info!("Scraping: {}", start_url);
|
|
log::info!("Base domain: {}", base_domain);
|
|
log::info!("Output dir: {}", output_dir.display());
|
|
log::info!(
|
|
"Document conversion: {}",
|
|
if convert_docs { "enabled" } else { "disabled" }
|
|
);
|
|
|
|
let client = reqwest::Client::builder()
|
|
.timeout(std::time::Duration::from_secs(30))
|
|
.user_agent(config::USER_AGENT)
|
|
.build()?;
|
|
|
|
log::info!("Fetching robots.txt...");
|
|
let robots = fetch_robots(&client, &url, config::USER_AGENT).await;
|
|
log::info!("");
|
|
|
|
let fetcher = fetcher::Fetcher::new()?;
|
|
let scraper = Scraper::new(
|
|
output_dir,
|
|
log_path,
|
|
base_domain,
|
|
robots,
|
|
convert_docs,
|
|
fetcher,
|
|
);
|
|
let (count, errors) = scraper.run(&url).await;
|
|
|
|
log::info!("\nDone. Fetched {} URLs. Errors: {}.", count, errors);
|
|
log::info!("Error log: {}/logs/scrape_errors.jsonl", output_dir_name);
|
|
|
|
Ok(())
|
|
}
|