clippy fixes
This commit is contained in:
+1
-3
@@ -32,9 +32,7 @@ impl Fetcher {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
if attempt < MAX_RETRIES {
|
||||
log::warn!(
|
||||
"retry {attempt}/{MAX_RETRIES}: body read failed ({e})"
|
||||
);
|
||||
log::warn!("retry {attempt}/{MAX_RETRIES}: body read failed ({e})");
|
||||
tokio::time::sleep(Duration::from_secs(
|
||||
RETRY_BACKOFF_SECS * u64::from(attempt),
|
||||
))
|
||||
|
||||
+9
-4
@@ -109,9 +109,10 @@ async fn main() -> Result<()> {
|
||||
};
|
||||
|
||||
let logs_dir = {
|
||||
let output_name = output_dir
|
||||
.file_name()
|
||||
.map_or_else(|| "output".to_string(), |n| n.to_string_lossy().into_owned());
|
||||
let output_name = output_dir.file_name().map_or_else(
|
||||
|| "output".to_string(),
|
||||
|n| n.to_string_lossy().into_owned(),
|
||||
);
|
||||
let logs_name = format!("{output_name}_logs");
|
||||
output_dir
|
||||
.parent()
|
||||
@@ -147,7 +148,11 @@ async fn main() -> Result<()> {
|
||||
log::info!("Base host: {base_host}");
|
||||
log::info!(
|
||||
"Subdomain crawling: {}",
|
||||
if args.subdomains { "enabled" } else { "disabled" }
|
||||
if args.subdomains {
|
||||
"enabled"
|
||||
} else {
|
||||
"disabled"
|
||||
}
|
||||
);
|
||||
log::info!("Output dir: {}", output_dir.display());
|
||||
log::info!("Logs dir: {}", logs_dir.display());
|
||||
|
||||
+1
-5
@@ -112,11 +112,7 @@ impl Scraper {
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_new_links(
|
||||
&self,
|
||||
html: &str,
|
||||
final_url: &Url,
|
||||
) -> (usize, Vec<Url>) {
|
||||
fn extract_new_links(&self, html: &str, final_url: &Url) -> (usize, Vec<Url>) {
|
||||
let links = extract_links(html, final_url, &self.base_host, self.allow_subdomains);
|
||||
let new_count = links.len();
|
||||
let new_links: Vec<Url> = links
|
||||
|
||||
+1
-4
@@ -111,10 +111,7 @@ pub fn url_to_filename(url: &Url, content_type: &str) -> String {
|
||||
return format!("{}{}", &hash[..16], ext);
|
||||
}
|
||||
|
||||
let host = url
|
||||
.host_str()
|
||||
.unwrap_or("unknown")
|
||||
.replace(['.', ':'], "_");
|
||||
let host = url.host_str().unwrap_or("unknown").replace(['.', ':'], "_");
|
||||
let path = url.path().trim_start_matches('/');
|
||||
let path = if path.is_empty() { "index" } else { path };
|
||||
let path = path.trim_end_matches('/');
|
||||
|
||||
Reference in New Issue
Block a user