clippy fixes
This commit is contained in:
+1
-3
@@ -32,9 +32,7 @@ impl Fetcher {
|
|||||||
Ok(b) => b,
|
Ok(b) => b,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if attempt < MAX_RETRIES {
|
if attempt < MAX_RETRIES {
|
||||||
log::warn!(
|
log::warn!("retry {attempt}/{MAX_RETRIES}: body read failed ({e})");
|
||||||
"retry {attempt}/{MAX_RETRIES}: body read failed ({e})"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_secs(
|
tokio::time::sleep(Duration::from_secs(
|
||||||
RETRY_BACKOFF_SECS * u64::from(attempt),
|
RETRY_BACKOFF_SECS * u64::from(attempt),
|
||||||
))
|
))
|
||||||
|
|||||||
+9
-4
@@ -109,9 +109,10 @@ async fn main() -> Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let logs_dir = {
|
let logs_dir = {
|
||||||
let output_name = output_dir
|
let output_name = output_dir.file_name().map_or_else(
|
||||||
.file_name()
|
|| "output".to_string(),
|
||||||
.map_or_else(|| "output".to_string(), |n| n.to_string_lossy().into_owned());
|
|n| n.to_string_lossy().into_owned(),
|
||||||
|
);
|
||||||
let logs_name = format!("{output_name}_logs");
|
let logs_name = format!("{output_name}_logs");
|
||||||
output_dir
|
output_dir
|
||||||
.parent()
|
.parent()
|
||||||
@@ -147,7 +148,11 @@ async fn main() -> Result<()> {
|
|||||||
log::info!("Base host: {base_host}");
|
log::info!("Base host: {base_host}");
|
||||||
log::info!(
|
log::info!(
|
||||||
"Subdomain crawling: {}",
|
"Subdomain crawling: {}",
|
||||||
if args.subdomains { "enabled" } else { "disabled" }
|
if args.subdomains {
|
||||||
|
"enabled"
|
||||||
|
} else {
|
||||||
|
"disabled"
|
||||||
|
}
|
||||||
);
|
);
|
||||||
log::info!("Output dir: {}", output_dir.display());
|
log::info!("Output dir: {}", output_dir.display());
|
||||||
log::info!("Logs dir: {}", logs_dir.display());
|
log::info!("Logs dir: {}", logs_dir.display());
|
||||||
|
|||||||
+1
-5
@@ -112,11 +112,7 @@ impl Scraper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_new_links(
|
fn extract_new_links(&self, html: &str, final_url: &Url) -> (usize, Vec<Url>) {
|
||||||
&self,
|
|
||||||
html: &str,
|
|
||||||
final_url: &Url,
|
|
||||||
) -> (usize, Vec<Url>) {
|
|
||||||
let links = extract_links(html, final_url, &self.base_host, self.allow_subdomains);
|
let links = extract_links(html, final_url, &self.base_host, self.allow_subdomains);
|
||||||
let new_count = links.len();
|
let new_count = links.len();
|
||||||
let new_links: Vec<Url> = links
|
let new_links: Vec<Url> = links
|
||||||
|
|||||||
+1
-4
@@ -111,10 +111,7 @@ pub fn url_to_filename(url: &Url, content_type: &str) -> String {
|
|||||||
return format!("{}{}", &hash[..16], ext);
|
return format!("{}{}", &hash[..16], ext);
|
||||||
}
|
}
|
||||||
|
|
||||||
let host = url
|
let host = url.host_str().unwrap_or("unknown").replace(['.', ':'], "_");
|
||||||
.host_str()
|
|
||||||
.unwrap_or("unknown")
|
|
||||||
.replace(['.', ':'], "_");
|
|
||||||
let path = url.path().trim_start_matches('/');
|
let path = url.path().trim_start_matches('/');
|
||||||
let path = if path.is_empty() { "index" } else { path };
|
let path = if path.is_empty() { "index" } else { path };
|
||||||
let path = path.trim_end_matches('/');
|
let path = path.trim_end_matches('/');
|
||||||
|
|||||||
Reference in New Issue
Block a user