fix clippy warnigns
This commit is contained in:
+2
-9
@@ -93,10 +93,7 @@ pub async fn fetch_robots(client: &Client, base_url: &Url, user_agent: &str) ->
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Failed to fetch robots.txt ({}): assuming no restrictions",
|
||||
e
|
||||
);
|
||||
log::warn!("Failed to fetch robots.txt ({e}): assuming no restrictions");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,11 +130,7 @@ fn parse_robots_txt(text: &str, target_agent: &str) -> RobotsRule {
|
||||
let agent = agent.trim().to_lowercase();
|
||||
current_agents.push(agent.clone());
|
||||
|
||||
if agent == "*" || agent == target_lower || agent == "web-scraper" {
|
||||
collecting = true;
|
||||
} else {
|
||||
collecting = false;
|
||||
}
|
||||
collecting = agent == "*" || agent == target_lower || agent == "web-scraper";
|
||||
} else if collecting {
|
||||
if let Some(path) = line.strip_prefix("Disallow:") {
|
||||
let path = path.trim().to_string();
|
||||
|
||||
Reference in New Issue
Block a user