mod config;
mod converter;
mod doc_processor;
mod error_logger;
mod extractor;
mod fetcher;
mod robots;
mod scraper;
mod url_utils;

use crate::robots::fetch_robots;
use crate::scraper::Scraper;
use anyhow::Result;
use clap::Parser;
use std::fs;
use std::path::PathBuf;
use url::Url;

#[derive(Parser, Debug)]
#[command(name = "rs-scraper")]
#[command(author = "FLÓ")]
#[command(version = "0.1.0")]
#[command(about = "Web scraper for Faroese public sector sites")]
struct Args {
    #[arg(help = "Starting URL to scrape")]
    start_url: String,

    #[arg(long, help = "Save documents as-is, skip Markdown conversion")]
    no_doc_conversion: bool,
}

#[tokio::main]
async fn main() -> Result<()> {
    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();

    let args = Args::parse();

    let start_url = args.start_url;
    let convert_docs = !args.no_doc_conversion;

    let url = Url::parse(&start_url)?;

    let base_domain = url_utils::derive_base_domain(&url)
        .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?;

    let output_dir_name = format!("scraped_{}", base_domain.replace('.', "_"));
    let output_dir = PathBuf::from(&output_dir_name);
    let logs_dir = output_dir.join("logs");

    fs::create_dir_all(&logs_dir)?;

    let log_path = logs_dir.join("scrape_errors.jsonl");
    fs::remove_file(&log_path).ok();

    log::info!("Scraping: {}", start_url);
    log::info!("Base domain: {}", base_domain);
    log::info!("Output dir: {}", output_dir.display());
    log::info!(
        "Document conversion: {}",
        if convert_docs { "enabled" } else { "disabled" }
    );

    let client = reqwest::Client::builder()
        .timeout(std::time::Duration::from_secs(30))
        .user_agent(config::USER_AGENT)
        .build()?;

    log::info!("Fetching robots.txt...");
    let robots = fetch_robots(&client, &url, config::USER_AGENT).await;
    log::info!("");

    let fetcher = fetcher::Fetcher::new()?;
    let scraper = Scraper::new(
        output_dir,
        log_path,
        base_domain,
        robots,
        convert_docs,
        fetcher,
    );
    let (count, errors) = scraper.run(&url).await;

    log::info!("\nDone. Fetched {} URLs. Errors: {}.", count, errors);
    log::info!("Error log: {}/logs/scrape_errors.jsonl", output_dir_name);

    Ok(())
}