FLO/web-scraper-rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix clippy warnigns

Browse Source
This commit is contained in:
bartal-lsn
2026-08-25 12:01:07 +01:00
parent da8f7302c1
commit fe4b5ed18a
11 changed files with 106 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.rs
+16 -19
View File
@@ -80,26 +80,23 @@ async fn main() -> Result<()> {
let url = Url::parse(&start_url)?;
let base_domain = url_utils::derive_base_domain(&url)
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?;
.ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {start_url}"))?;
let output_dir = match &args.output {
Some(p) => PathBuf::from(p),
None => {
let default_name = format!("{}_scraped", base_domain.replace('.', "_"));
PathBuf::from(default_name)
}
let output_dir = if let Some(p) = &args.output {
PathBuf::from(p)
} else {
let default_name = format!("{}_scraped", base_domain.replace('.', "_"));
PathBuf::from(default_name)
};
let logs_dir = {
let output_name = output_dir
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "output".to_string());
let logs_name = format!("{}_logs", output_name);
.map_or_else(|| "output".to_string(), |n| n.to_string_lossy().into_owned());
let logs_name = format!("{output_name}_logs");
output_dir
.parent()
.map(|p| p.join(&logs_name))
.unwrap_or_else(|| PathBuf::from(&logs_name))
.map_or_else(|| PathBuf::from(&logs_name), |p| p.join(&logs_name))
};
fs::create_dir_all(&output_dir)?;
@@ -126,8 +123,8 @@ async fn main() -> Result<()> {
}
};
log::info!("Scraping: {}", start_url);
log::info!("Base domain: {}", base_domain);
log::info!("Scraping: {start_url}");
log::info!("Base domain: {base_domain}");
log::info!("Output dir: {}", output_dir.display());
log::info!("Logs dir: {}", logs_dir.display());
log::info!(
@@ -140,13 +137,13 @@ async fn main() -> Result<()> {
);
match (&include_types, &exclude_types) {
(Some(t), _) => log::info!("Type filter: include {:?}", t),
(_, Some(t)) => log::info!("Type filter: exclude {:?}", t),
(Some(t), _) => log::info!("Type filter: include {t:?}"),
(_, Some(t)) => log::info!("Type filter: exclude {t:?}"),
_ => log::info!("Type filter: none (all types)"),
}
match &scope_path {
Some(s) => log::info!("Path scope: {} (only this path and deeper)", s),
Some(s) => log::info!("Path scope: {s} (only this path and deeper)"),
None => log::info!("Path scope: none (full site)"),
}
@@ -159,7 +156,7 @@ async fn main() -> Result<()> {
let robots = fetch_robots(&client, &url, config::USER_AGENT).await;
log::info!("");
let fetcher = fetcher::Fetcher::new(client)?;
let fetcher = fetcher::Fetcher::new(client);
let mut scraper = Scraper::new(
output_dir,
log_path.clone(),
@@ -174,7 +171,7 @@ async fn main() -> Result<()> {
);
let (count, errors) = scraper.run(&url).await;
log::info!("\nDone. Fetched {} URLs. Errors: {}.", count, errors);
log::info!("\nDone. Fetched {count} URLs. Errors: {errors}.");
log::info!("Error log: {}", log_path.display());
Ok(())