fix clippy warnigns

This commit is contained in:
2026-08-25 12:01:07 +01:00
parent da8f7302c1
commit fe4b5ed18a
11 changed files with 106 additions and 155 deletions
+2 -9
View File
@@ -93,10 +93,7 @@ pub async fn fetch_robots(client: &Client, base_url: &Url, user_agent: &str) ->
);
}
Err(e) => {
log::warn!(
"Failed to fetch robots.txt ({}): assuming no restrictions",
e
);
log::warn!("Failed to fetch robots.txt ({e}): assuming no restrictions");
}
}
@@ -133,11 +130,7 @@ fn parse_robots_txt(text: &str, target_agent: &str) -> RobotsRule {
let agent = agent.trim().to_lowercase();
current_agents.push(agent.clone());
if agent == "*" || agent == target_lower || agent == "web-scraper" {
collecting = true;
} else {
collecting = false;
}
collecting = agent == "*" || agent == target_lower || agent == "web-scraper";
} else if collecting {
if let Some(path) = line.strip_prefix("Disallow:") {
let path = path.trim().to_string();