start on ratatui. fixed clippy
This commit is contained in:
+118
-53
@@ -8,16 +8,18 @@ mod extractor;
|
||||
mod fetcher;
|
||||
mod robots;
|
||||
mod scraper;
|
||||
mod tui;
|
||||
mod url_utils;
|
||||
|
||||
use crate::robots::fetch_robots;
|
||||
use crate::scraper::Scraper;
|
||||
use crate::tui::TuiConfig;
|
||||
use anyhow::{Result, bail};
|
||||
use clap::Parser;
|
||||
use reqwest::Client;
|
||||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
@@ -29,7 +31,14 @@ use url::Url;
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
struct Args {
|
||||
#[arg(help = "Starting URL to scrape")]
|
||||
start_url: String,
|
||||
start_url: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long = "interactive",
|
||||
short = 'i',
|
||||
help = "Launch interactive TUI configuration instead of CLI"
|
||||
)]
|
||||
interactive: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
@@ -90,8 +99,8 @@ fn print_config(
|
||||
start_url: &str,
|
||||
base_host: &str,
|
||||
subdomains: bool,
|
||||
output_dir: &std::path::Path,
|
||||
logs_dir: &std::path::Path,
|
||||
output_dir: &Path,
|
||||
logs_dir: &Path,
|
||||
convert_docs: bool,
|
||||
single: bool,
|
||||
delay_ms: u64,
|
||||
@@ -101,20 +110,11 @@ fn print_config(
|
||||
) {
|
||||
log::info!("Scraping: {start_url}");
|
||||
log::info!("Base host: {base_host}");
|
||||
log::info!(
|
||||
"Subdomain crawling: {}",
|
||||
if subdomains { "enabled" } else { "disabled" }
|
||||
);
|
||||
log::info!("Subdomain crawling: {}", if subdomains { "enabled" } else { "disabled" });
|
||||
log::info!("Output dir: {}", output_dir.display());
|
||||
log::info!("Logs dir: {}", logs_dir.display());
|
||||
log::info!(
|
||||
"Document conversion: {}",
|
||||
if convert_docs { "enabled" } else { "disabled" }
|
||||
);
|
||||
log::info!(
|
||||
"Single page mode: {}",
|
||||
if single { "enabled" } else { "disabled" }
|
||||
);
|
||||
log::info!("Document conversion: {}", if convert_docs { "enabled" } else { "disabled" });
|
||||
log::info!("Single page mode: {}", if single { "enabled" } else { "disabled" });
|
||||
log::info!("Request delay: {delay_ms} ms");
|
||||
|
||||
match (include_types, exclude_types) {
|
||||
@@ -129,31 +129,50 @@ fn print_config(
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
|
||||
fn build_config_from_args(args: &Args) -> Result<(String, TuiConfig)> {
|
||||
let url_str = args.start_url.as_ref()
|
||||
.ok_or_else(|| anyhow::anyhow!("START_URL is required when not using --interactive"))?;
|
||||
|
||||
let args = Args::parse();
|
||||
let tui_config = TuiConfig {
|
||||
start_url: url_str.clone(),
|
||||
output_dir: args.output.clone().unwrap_or_default(),
|
||||
subdomains: args.subdomains,
|
||||
single: args.single,
|
||||
no_scope: args.no_scope,
|
||||
no_doc_conversion: args.no_doc_conversion,
|
||||
delay_ms: args.delay_ms,
|
||||
types: args.types.clone(),
|
||||
exclude_types: args.exclude_types.clone(),
|
||||
};
|
||||
|
||||
let start_url = args.start_url;
|
||||
let convert_docs = !args.no_doc_conversion;
|
||||
let delay_ms = args.delay_ms;
|
||||
Ok((url_str.clone(), tui_config))
|
||||
}
|
||||
|
||||
let url = Url::parse(&start_url)?;
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[allow(clippy::fn_params_excessive_bools)]
|
||||
async fn run_scraper(
|
||||
start_url: &str,
|
||||
subdomains: bool,
|
||||
single: bool,
|
||||
no_scope: bool,
|
||||
no_doc_conversion: bool,
|
||||
delay_ms: u64,
|
||||
types: Option<&str>,
|
||||
exclude_types: Option<&str>,
|
||||
output: Option<&str>,
|
||||
) -> Result<()> {
|
||||
let url = Url::parse(start_url)?;
|
||||
|
||||
let base_host = url
|
||||
.host_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?
|
||||
let base_host = url.host_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname"))?
|
||||
.to_string();
|
||||
|
||||
let base_domain = url_utils::derive_base_domain(&url)
|
||||
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?;
|
||||
.ok_or_else(|| anyhow::anyhow!("Could not parse base domain"))?;
|
||||
|
||||
let output_dir = if let Some(p) = &args.output {
|
||||
PathBuf::from(p)
|
||||
} else {
|
||||
let default_name = format!("{}_scraped", base_domain.replace('.', "_"));
|
||||
PathBuf::from(default_name)
|
||||
let output_dir = match output {
|
||||
Some(p) if !p.is_empty() => PathBuf::from(p),
|
||||
_ => PathBuf::from(format!("{}_scraped", base_domain.replace('.', "_"))),
|
||||
};
|
||||
|
||||
let logs_dir = {
|
||||
@@ -162,8 +181,7 @@ async fn main() -> Result<()> {
|
||||
|n| n.to_string_lossy().into_owned(),
|
||||
);
|
||||
let logs_name = format!("{output_name}_logs");
|
||||
output_dir
|
||||
.parent()
|
||||
output_dir.parent()
|
||||
.map_or_else(|| PathBuf::from(&logs_name), |p| p.join(&logs_name))
|
||||
};
|
||||
|
||||
@@ -173,34 +191,28 @@ async fn main() -> Result<()> {
|
||||
let log_path = logs_dir.join("scrape_errors.jsonl");
|
||||
fs::remove_file(&log_path).ok();
|
||||
|
||||
let (include_types, exclude_types) = match (&args.types, &args.exclude_types) {
|
||||
(Some(_), Some(_)) => {
|
||||
bail!("--types and --exclude-types are mutually exclusive");
|
||||
}
|
||||
let (include_types, exclude_types) = match (types, exclude_types) {
|
||||
(Some(_), Some(_)) => bail!("--types and --exclude-types are mutually exclusive"),
|
||||
(Some(t), None) => (Some(parse_ext_list(t)), None),
|
||||
(None, Some(t)) => (None, Some(parse_ext_list(t))),
|
||||
(None, None) => (None, None),
|
||||
};
|
||||
|
||||
let scope_path = if args.no_scope {
|
||||
let scope_path = if no_scope {
|
||||
None
|
||||
} else {
|
||||
let path = url.path().trim_end_matches('/');
|
||||
if path.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(path.to_string())
|
||||
}
|
||||
if path.is_empty() { None } else { Some(path.to_string()) }
|
||||
};
|
||||
|
||||
print_config(
|
||||
&start_url,
|
||||
start_url,
|
||||
&base_host,
|
||||
args.subdomains,
|
||||
subdomains,
|
||||
&output_dir,
|
||||
&logs_dir,
|
||||
convert_docs,
|
||||
args.single,
|
||||
!no_doc_conversion,
|
||||
single,
|
||||
delay_ms,
|
||||
include_types.as_ref(),
|
||||
exclude_types.as_ref(),
|
||||
@@ -222,19 +234,72 @@ async fn main() -> Result<()> {
|
||||
log_path.clone(),
|
||||
base_host,
|
||||
robots,
|
||||
convert_docs,
|
||||
!no_doc_conversion,
|
||||
fetcher,
|
||||
include_types,
|
||||
exclude_types,
|
||||
scope_path,
|
||||
args.single,
|
||||
args.subdomains,
|
||||
single,
|
||||
subdomains,
|
||||
delay_ms,
|
||||
);
|
||||
let (count, errors) = scraper.run(&url).await;
|
||||
|
||||
let (count, errors) = scraper.run(&url).await;
|
||||
log::info!("\nDone. Fetched {count} URLs. Errors: {errors}.");
|
||||
log::info!("Error log: {}", log_path.display());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
let cli_command;
|
||||
|
||||
if args.interactive {
|
||||
log::info!("Launching interactive TUI...");
|
||||
|
||||
if let Some(config) = tui::run_tui()? {
|
||||
cli_command = config.build_command();
|
||||
log::info!("Running: {cli_command}");
|
||||
|
||||
run_scraper(
|
||||
&config.start_url,
|
||||
config.subdomains,
|
||||
config.single,
|
||||
config.no_scope,
|
||||
config.no_doc_conversion,
|
||||
config.delay_ms,
|
||||
config.types.as_deref(),
|
||||
config.exclude_types.as_deref(),
|
||||
if config.output_dir.is_empty() { None } else { Some(&config.output_dir) },
|
||||
).await?;
|
||||
} else {
|
||||
log::info!("Cancelled by user.");
|
||||
return Ok(());
|
||||
}
|
||||
} else {
|
||||
let (start_url, tui_config) = build_config_from_args(&args)?;
|
||||
cli_command = tui_config.build_command();
|
||||
log::info!("Running: {cli_command}");
|
||||
|
||||
run_scraper(
|
||||
&start_url,
|
||||
args.subdomains,
|
||||
args.single,
|
||||
args.no_scope,
|
||||
args.no_doc_conversion,
|
||||
args.delay_ms,
|
||||
args.types.as_deref(),
|
||||
args.exclude_types.as_deref(),
|
||||
args.output.as_deref(),
|
||||
).await?;
|
||||
}
|
||||
|
||||
eprintln!("Command used: {cli_command}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user