start on ratatui. fixed clippy

Cargo.toml

@@ -8,9 +8,11 @@ anydoc = "0.1"
 anyhow = "1"
 chrono = { version = "0.4", features = ["serde"] }
 clap = { version = "4", features = ["derive"] }
+crossterm = "0.29"
 hex = "0.4"
 htmd = "0.1"
 log = "0.4"
+ratatui = "0.30"
 env_logger = "0.11"
 md-5 = "0.10"
 reqwest = "0.12"

src/main.rs

@@ -8,16 +8,18 @@ mod extractor;
 mod fetcher;
 mod robots;
 mod scraper;
+mod tui;
 mod url_utils;
 
 use crate::robots::fetch_robots;
 use crate::scraper::Scraper;
+use crate::tui::TuiConfig;
 use anyhow::{Result, bail};
 use clap::Parser;
 use reqwest::Client;
 use std::collections::HashSet;
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 use url::Url;
 
@@ -29,7 +31,14 @@ use url::Url;
 #[allow(clippy::struct_excessive_bools)]
 struct Args {
     #[arg(help = "Starting URL to scrape")]
-    start_url: String,
+    start_url: Option<String>,
+
+    #[arg(
+        long = "interactive",
+        short = 'i',
+        help = "Launch interactive TUI configuration instead of CLI"
+    )]
+    interactive: bool,
 
     #[arg(
         long,
@@ -90,8 +99,8 @@ fn print_config(
     start_url: &str,
     base_host: &str,
     subdomains: bool,
-    output_dir: &std::path::Path,
+    output_dir: &Path,
-    logs_dir: &std::path::Path,
+    logs_dir: &Path,
     convert_docs: bool,
     single: bool,
     delay_ms: u64,
@@ -101,20 +110,11 @@ fn print_config(
 ) {
     log::info!("Scraping: {start_url}");
     log::info!("Base host: {base_host}");
-    log::info!(
+    log::info!("Subdomain crawling: {}", if subdomains { "enabled" } else { "disabled" });
-        "Subdomain crawling: {}",
-        if subdomains { "enabled" } else { "disabled" }
-    );
     log::info!("Output dir: {}", output_dir.display());
     log::info!("Logs dir: {}", logs_dir.display());
-    log::info!(
+    log::info!("Document conversion: {}", if convert_docs { "enabled" } else { "disabled" });
-        "Document conversion: {}",
+    log::info!("Single page mode: {}", if single { "enabled" } else { "disabled" });
-        if convert_docs { "enabled" } else { "disabled" }
-    );
-    log::info!(
-        "Single page mode: {}",
-        if single { "enabled" } else { "disabled" }
-    );
     log::info!("Request delay: {delay_ms} ms");
 
     match (include_types, exclude_types) {
@@ -129,31 +129,50 @@ fn print_config(
     }
 }
 
-#[tokio::main]
+fn build_config_from_args(args: &Args) -> Result<(String, TuiConfig)> {
-async fn main() -> Result<()> {
+    let url_str = args.start_url.as_ref()
-    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+        .ok_or_else(|| anyhow::anyhow!("START_URL is required when not using --interactive"))?;
 
-    let args = Args::parse();
+    let tui_config = TuiConfig {
+        start_url: url_str.clone(),
+        output_dir: args.output.clone().unwrap_or_default(),
+        subdomains: args.subdomains,
+        single: args.single,
+        no_scope: args.no_scope,
+        no_doc_conversion: args.no_doc_conversion,
+        delay_ms: args.delay_ms,
+        types: args.types.clone(),
+        exclude_types: args.exclude_types.clone(),
+    };
 
-    let start_url = args.start_url;
+    Ok((url_str.clone(), tui_config))
-    let convert_docs = !args.no_doc_conversion;
+}
-    let delay_ms = args.delay_ms;
 
-    let url = Url::parse(&start_url)?;
+#[allow(clippy::too_many_arguments)]
+#[allow(clippy::fn_params_excessive_bools)]
+async fn run_scraper(
+    start_url: &str,
+    subdomains: bool,
+    single: bool,
+    no_scope: bool,
+    no_doc_conversion: bool,
+    delay_ms: u64,
+    types: Option<&str>,
+    exclude_types: Option<&str>,
+    output: Option<&str>,
+) -> Result<()> {
+    let url = Url::parse(start_url)?;
 
-    let base_host = url
+    let base_host = url.host_str()
-        .host_str()
+        .ok_or_else(|| anyhow::anyhow!("Could not parse hostname"))?
-        .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?
         .to_string();
 
     let base_domain = url_utils::derive_base_domain(&url)
-        .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?;
+        .ok_or_else(|| anyhow::anyhow!("Could not parse base domain"))?;
 
-    let output_dir = if let Some(p) = &args.output {
+    let output_dir = match output {
-        PathBuf::from(p)
+        Some(p) if !p.is_empty() => PathBuf::from(p),
-    } else {
+        _ => PathBuf::from(format!("{}_scraped", base_domain.replace('.', "_"))),
-        let default_name = format!("{}_scraped", base_domain.replace('.', "_"));
-        PathBuf::from(default_name)
     };
 
     let logs_dir = {
@@ -162,8 +181,7 @@ async fn main() -> Result<()> {
             |n| n.to_string_lossy().into_owned(),
         );
         let logs_name = format!("{output_name}_logs");
-        output_dir
+        output_dir.parent()
-            .parent()
             .map_or_else(|| PathBuf::from(&logs_name), |p| p.join(&logs_name))
     };
 
@@ -173,34 +191,28 @@ async fn main() -> Result<()> {
     let log_path = logs_dir.join("scrape_errors.jsonl");
     fs::remove_file(&log_path).ok();
 
-    let (include_types, exclude_types) = match (&args.types, &args.exclude_types) {
+    let (include_types, exclude_types) = match (types, exclude_types) {
-        (Some(_), Some(_)) => {
+        (Some(_), Some(_)) => bail!("--types and --exclude-types are mutually exclusive"),
-            bail!("--types and --exclude-types are mutually exclusive");
-        }
         (Some(t), None) => (Some(parse_ext_list(t)), None),
         (None, Some(t)) => (None, Some(parse_ext_list(t))),
         (None, None) => (None, None),
     };
 
-    let scope_path = if args.no_scope {
+    let scope_path = if no_scope {
         None
     } else {
         let path = url.path().trim_end_matches('/');
-        if path.is_empty() {
+        if path.is_empty() { None } else { Some(path.to_string()) }
-            None
-        } else {
-            Some(path.to_string())
-        }
     };
 
     print_config(
-        &start_url,
+        start_url,
         &base_host,
-        args.subdomains,
+        subdomains,
         &output_dir,
         &logs_dir,
-        convert_docs,
+        !no_doc_conversion,
-        args.single,
+        single,
         delay_ms,
         include_types.as_ref(),
         exclude_types.as_ref(),
@@ -222,19 +234,72 @@ async fn main() -> Result<()> {
         log_path.clone(),
         base_host,
         robots,
-        convert_docs,
+        !no_doc_conversion,
         fetcher,
         include_types,
         exclude_types,
         scope_path,
-        args.single,
+        single,
-        args.subdomains,
+        subdomains,
         delay_ms,
     );
-    let (count, errors) = scraper.run(&url).await;
 
+    let (count, errors) = scraper.run(&url).await;
     log::info!("\nDone. Fetched {count} URLs. Errors: {errors}.");
     log::info!("Error log: {}", log_path.display());
 
     Ok(())
 }
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    let args = Args::parse();
+
+    let cli_command;
+
+    if args.interactive {
+        log::info!("Launching interactive TUI...");
+
+        if let Some(config) = tui::run_tui()? {
+            cli_command = config.build_command();
+            log::info!("Running: {cli_command}");
+
+            run_scraper(
+                &config.start_url,
+                config.subdomains,
+                config.single,
+                config.no_scope,
+                config.no_doc_conversion,
+                config.delay_ms,
+                config.types.as_deref(),
+                config.exclude_types.as_deref(),
+                if config.output_dir.is_empty() { None } else { Some(&config.output_dir) },
+            ).await?;
+        } else {
+            log::info!("Cancelled by user.");
+            return Ok(());
+        }
+    } else {
+        let (start_url, tui_config) = build_config_from_args(&args)?;
+        cli_command = tui_config.build_command();
+        log::info!("Running: {cli_command}");
+
+        run_scraper(
+            &start_url,
+            args.subdomains,
+            args.single,
+            args.no_scope,
+            args.no_doc_conversion,
+            args.delay_ms,
+            args.types.as_deref(),
+            args.exclude_types.as_deref(),
+            args.output.as_deref(),
+        ).await?;
+    }
+
+    eprintln!("Command used: {cli_command}");
+
+    Ok(())
+}