FLO/web-scraper-rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

add single page html to md

Browse Source
This commit is contained in:
bartal-lsn
2026-08-18 11:18:03 +01:00
parent 7f01f1a341
commit 9932b81627
2 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scraper.rs
+19 -11
View File
@@ -41,6 +41,7 @@ pub struct Scraper {
include_types: Option<HashSet<String>>,
exclude_types: Option<HashSet<String>>,
scope_path: Option<String>,
single_page: bool,
}
impl Scraper {
@@ -54,6 +55,7 @@ impl Scraper {
include_types: Option<HashSet<String>>,
exclude_types: Option<HashSet<String>>,
scope_path: Option<String>,
single_page: bool,
) -> Self {
Self {
fetcher,
@@ -67,6 +69,7 @@ impl Scraper {
include_types,
exclude_types,
scope_path,
single_page,
}
}
@@ -164,20 +167,24 @@ impl Scraper {
}
if is_html {
if let Ok(html) = std::str::from_utf8(&result.bytes) {
let links = extract_links(html, &final_url, &self.base_domain);
let new_count = links.len();
if !self.single_page {
if let Ok(html) = std::str::from_utf8(&result.bytes) {
let links = extract_links(html, &final_url, &self.base_domain);
let new_count = links.len();
let new_links: Vec<Url> = links
.into_iter()
.filter(|link| !self.seen.contains(link.as_str()))
.collect();
let new_links: Vec<Url> = links
.into_iter()
.filter(|link| !self.seen.contains(link.as_str()))
.collect();
for link in &new_links {
queue.push_back(link.clone());
for link in &new_links {
queue.push_back(link.clone());
}
log::info!(" found {} links ({} new)", new_count, new_links.len());
}
log::info!(" found {} links ({} new)", new_count, new_links.len());
} else {
log::info!(" [single-page mode] not crawling for links");
}
}
}
@@ -303,6 +310,7 @@ mod tests {
include_types,
exclude_types,
scope_path: None,
single_page: false,
}
}