diff --git a/Cargo.lock b/Cargo.lock index e55e292..28d98e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,32 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -11,6 +37,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -35,6 +111,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -50,6 +132,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -68,6 +159,15 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.2" @@ -84,6 +184,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -98,6 +209,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "core-foundation" version = "0.9.4" @@ -133,6 +260,49 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -166,6 +336,43 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_more" version = "0.99.20" @@ -213,12 +420,27 @@ dependencies = [ "dtoa", ] +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "ego-tree" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -228,6 +450,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -256,6 +501,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -381,8 +636,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -416,6 +674,30 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "htmd" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1642def6e8e4dc182941f35454f7d2af917787f91f3f5133300030b41006d0" +dependencies = [ + "html5ever 0.27.0", + "markup5ever_rcdom", +] + +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "html5ever" version = "0.29.1" @@ -424,7 +706,7 @@ checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.14.1", "match_token", ] @@ -671,6 +953,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -681,18 +982,87 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "js-sys" version = "0.3.104" @@ -737,12 +1107,57 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "rand 0.10.2", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + [[package]] name = "mac" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "markup5ever" version = "0.14.1" @@ -757,6 +1172,18 @@ dependencies = [ "tendril", ] +[[package]] +name = "markup5ever_rcdom" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18" +dependencies = [ + "html5ever 0.27.0", + "markup5ever 0.12.1", + "tendril", + "xml5ever", +] + [[package]] name = "match_token" version = "0.1.0" @@ -790,6 +1217,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -824,6 +1261,21 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -839,13 +1291,19 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl" version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -905,6 +1363,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pdf-inspector" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f323996b5c71c5f48860be70c265965135c9bedb39a33271dfc95d5f53dbdb34" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -938,7 +1414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.7", ] [[package]] @@ -975,6 +1451,21 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -984,6 +1475,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "precomputed-hash" version = "0.1.1" @@ -1020,7 +1517,18 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1029,15 +1537,76 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.1", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "reqwest" version = "0.12.28" @@ -1099,7 +1668,9 @@ dependencies = [ "anyhow", "chrono", "hex", + "htmd", "md-5", + "pdf-inspector", "reqwest", "scraper", "serde", @@ -1115,7 +1686,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -1191,7 +1762,7 @@ dependencies = [ "cssparser", "ego-tree", "getopts", - "html5ever", + "html5ever 0.29.1", "precomputed-hash", "selectors", "tendril", @@ -1203,7 +1774,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -1226,7 +1797,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cssparser", "derive_more", "fxhash", @@ -1310,7 +1881,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1330,6 +1901,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.3" @@ -1389,6 +1966,17 @@ dependencies = [ "quote", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1443,7 +2031,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -1482,6 +2070,56 @@ dependencies = [ "utf-8", ] +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1492,6 +2130,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -1575,7 +2228,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -1624,18 +2277,45 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-width" version = "0.2.2" @@ -1672,6 +2352,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1764,6 +2450,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "windows-core" version = "0.62.2" @@ -1922,6 +2614,17 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xml5ever" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index a4ba20c..c2964c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,9 @@ edition = "2024" anyhow = "1" chrono = { version = "0.4", features = ["serde"] } hex = "0.4" +htmd = "0.1" md-5 = "0.10" +pdf-inspector = "1" reqwest = "0.12" scraper = "0.22" serde = { version = "1", features = ["derive"] } diff --git a/src/config.rs b/src/config.rs index 05795a3..791281b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -pub const DELAY_MS: u64 = 100; +pub const DELAY_MS: u64 = 0; pub const TIMEOUT_SECS: u64 = 60; pub const MAX_RETRIES: u32 = 3; pub const RETRY_BACKOFF_SECS: u64 = 5; diff --git a/src/converter.rs b/src/converter.rs new file mode 100644 index 0000000..5b723da --- /dev/null +++ b/src/converter.rs @@ -0,0 +1,15 @@ +use htmd::HtmlToMarkdown; + +pub fn html_to_markdown(html: &str) -> String { + let converter = HtmlToMarkdown::new(); + match converter.convert(html) { + Ok(md) => md, + Err(e) => { + eprintln!( + " WARN: HTML-to-MD conversion failed ({}), saving raw HTML", + e + ); + html.to_string() + } + } +} diff --git a/src/fetcher.rs b/src/fetcher.rs index 5c8852e..51f13ce 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -1,3 +1,5 @@ +// src/fetcher.rs + use crate::config::{MAX_RETRIES, RETRY_BACKOFF_SECS, TIMEOUT_SECS, USER_AGENT}; use anyhow::{Result, anyhow}; use reqwest::Client; @@ -60,6 +62,7 @@ impl Fetcher { bytes, content_type, final_url, + _status_code: status.as_u16(), }); } Err(e) => { @@ -96,4 +99,5 @@ pub struct FetchResult { pub bytes: Vec, pub content_type: String, pub final_url: Url, + pub _status_code: u16, } diff --git a/src/main.rs b/src/main.rs index 87b6436..ee3529a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,16 @@ -// src/main.rs - mod config; +mod converter; mod error_logger; mod extractor; mod fetcher; +mod pdf_processor; +mod robots; mod scraper; mod url_utils; -use scraper::Scraper; +use crate::config::USER_AGENT; +use crate::robots::fetch_robots; +use crate::scraper::Scraper; use std::fs; use std::path::PathBuf; use url::Url; @@ -16,14 +19,44 @@ use url::Url; async fn main() -> anyhow::Result<()> { let args: Vec = std::env::args().collect(); - if args.len() < 2 { - eprintln!("Usage: site-scraper "); - eprintln!("Example: site-scraper https://www.logting.fo"); - std::process::exit(1); + // Parse flags + let mut convert_pdfs = true; + let mut url_arg: Option = None; + + for arg in args.iter().skip(1) { + match arg.as_str() { + "--no-pdf-conversion" => convert_pdfs = false, + "--help" | "-h" => { + println!("Usage: site-scraper [OPTIONS] "); + println!(); + println!("Options:"); + println!(" --no-pdf-conversion Save PDFs as-is, skip text extraction"); + println!(" -h, --help Show this help message"); + println!(); + println!("Example:"); + println!(" site-scraper https://www.logting.fo"); + println!(" site-scraper --no-pdf-conversion https://taks.fo"); + std::process::exit(0); + } + _ => { + if url_arg.is_none() { + url_arg = Some(arg.clone()); + } + } + } } - let start_url = &args[1]; - let url = Url::parse(start_url)?; + let start_url = match url_arg { + Some(u) => u, + None => { + eprintln!("Usage: site-scraper [OPTIONS] "); + eprintln!("Example: site-scraper https://www.logting.fo"); + eprintln!("Run with --help for options."); + std::process::exit(1); + } + }; + + let url = Url::parse(&start_url)?; let base_domain = url_utils::derive_base_domain(&url) .ok_or_else(|| anyhow::anyhow!("Could not parse hostname from {}", start_url))?; @@ -40,9 +73,22 @@ async fn main() -> anyhow::Result<()> { println!("Scraping: {}", start_url); println!("Base domain: {}", base_domain); println!("Output dir: {}", output_dir.display()); + println!( + "PDF conversion: {}", + if convert_pdfs { "enabled" } else { "disabled" } + ); + + // Fetch robots.txt + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .user_agent(USER_AGENT) + .build()?; + + println!("Fetching robots.txt..."); + let robots = fetch_robots(&client, &url, "web-scraper").await; println!(); - let scraper = Scraper::new(output_dir, log_path, base_domain); + let scraper = Scraper::new(output_dir, log_path, base_domain, robots, convert_pdfs); let (count, errors) = scraper.run(&url).await; println!("\nDone. Fetched {} URLs. Errors: {}.", count, errors); diff --git a/src/pdf_processor.rs b/src/pdf_processor.rs new file mode 100644 index 0000000..e8a1f33 --- /dev/null +++ b/src/pdf_processor.rs @@ -0,0 +1,71 @@ +// src/pdf_processor.rs + +use anyhow::Result; +use std::path::Path; + +pub enum PdfProcessResult { + Markdown(String), + Scanned, +} + +pub fn process_pdf(pdf_path: &Path) -> Result { + use pdf_inspector::process_pdf; + + let result = process_pdf(pdf_path)?; + + match result.pdf_type { + pdf_inspector::PdfType::TextBased | pdf_inspector::PdfType::Mixed => { + match &result.markdown { + Some(md) if !md.trim().is_empty() => Ok(PdfProcessResult::Markdown(md.clone())), + _ => Ok(PdfProcessResult::Scanned), + } + } + pdf_inspector::PdfType::Scanned | pdf_inspector::PdfType::ImageBased => { + Ok(PdfProcessResult::Scanned) + } + } +} + +pub fn batch_convert_pdfs(dir: &Path) -> Result<(usize, usize, usize)> { + let mut text_based = 0usize; + let mut scanned = 0usize; + let mut errors = 0usize; + + let entries: Vec<_> = std::fs::read_dir(dir)? + .filter_map(|e| e.ok()) + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pdf")) + }) + .collect(); + + let total = entries.len(); + println!("Found {} PDFs to process", total); + + for (i, entry) in entries.iter().enumerate() { + let path = entry.path(); + let filename = path.file_name().unwrap_or_default().to_string_lossy(); + println!("[{}/{}] processing: {}", i + 1, total, filename); + + match process_pdf(&path) { + Ok(PdfProcessResult::Markdown(md)) => { + let md_path = path.with_extension("md"); + std::fs::write(&md_path, md)?; + std::fs::remove_file(&path)?; + println!(" text-based -> {}", md_path.display()); + text_based += 1; + } + Ok(PdfProcessResult::Scanned) => { + println!(" scanned — keeping as PDF"); + scanned += 1; + } + Err(e) => { + eprintln!(" error: {}", e); + errors += 1; + } + } + } + + Ok((text_based, scanned, errors)) +} diff --git a/src/robots.rs b/src/robots.rs new file mode 100644 index 0000000..9d13ee0 --- /dev/null +++ b/src/robots.rs @@ -0,0 +1,160 @@ +use reqwest::Client; +use std::time::Duration; +use url::Url; + +pub struct RobotsRule { + pub allowed: Vec, + pub disallowed: Vec, +} + +impl RobotsRule { + pub fn is_allowed(&self, path: &str) -> bool { + let mut matched_disallow_len = 0; + let mut matched_allow_len = 0; + + for rule in &self.disallowed { + if path_matches(rule, path) && rule.len() > matched_disallow_len { + matched_disallow_len = rule.len(); + } + } + + for rule in &self.allowed { + if path_matches(rule, path) && rule.len() > matched_allow_len { + matched_allow_len = rule.len(); + } + } + + matched_allow_len >= matched_disallow_len + } +} + +fn path_matches(pattern: &str, path: &str) -> bool { + if pattern.is_empty() { + return false; + } + + if !pattern.contains('*') { + return path.starts_with(pattern) || path == pattern; + } + + let parts: Vec<&str> = pattern.split('*').collect(); + let mut pos = 0; + + for (i, part) in parts.iter().enumerate() { + if i == 0 { + if !path.starts_with(part) { + return false; + } + pos = part.len(); + } else { + match path[pos..].find(part) { + Some(idx) => pos += idx + part.len(), + None => return false, + } + } + } + + true +} + +pub async fn fetch_robots(client: &Client, base_url: &Url, user_agent: &str) -> RobotsRule { + let robots_url = format!( + "{}://{}/robots.txt", + base_url.scheme(), + base_url.host_str().unwrap_or("") + ); + + let mut rule = RobotsRule { + allowed: Vec::new(), + disallowed: Vec::new(), + }; + + match client + .get(&robots_url) + .header("User-Agent", user_agent) + .timeout(Duration::from_secs(30)) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + if let Ok(text) = resp.text().await { + rule = parse_robots_txt(&text, user_agent); + } + } + Ok(resp) => { + println!( + "robots.txt returned HTTP {} — assuming no restrictions", + resp.status() + ); + } + Err(e) => { + println!( + "Failed to fetch robots.txt ({}): assuming no restrictions", + e + ); + } + } + + rule +} + +fn parse_robots_txt(text: &str, target_agent: &str) -> RobotsRule { + let target_lower = target_agent.to_lowercase(); + let mut rule = RobotsRule { + allowed: Vec::new(), + disallowed: Vec::new(), + }; + + let mut current_agents: Vec = Vec::new(); + let mut collecting = false; + + for line in text.lines() { + let line = line.trim(); + + if line.is_empty() { + current_agents.clear(); + collecting = false; + continue; + } + + if line.starts_with('#') { + continue; + } + + if let Some(agent) = line + .strip_prefix("User-agent:") + .or_else(|| line.strip_prefix("User-Agent:")) + { + let agent = agent.trim().to_lowercase(); + current_agents.push(agent.clone()); + + if agent == "*" || agent == target_lower || agent == "web-scraper" { + collecting = true; + } else { + collecting = false; + } + } else if collecting { + if let Some(path) = line.strip_prefix("Disallow:") { + let path = path.trim().to_string(); + if !path.is_empty() { + rule.disallowed.push(path); + } + } else if let Some(path) = line.strip_prefix("Allow:") { + let path = path.trim().to_string(); + if !path.is_empty() { + rule.allowed.push(path); + } + } + } + } + + if !rule.disallowed.is_empty() || !rule.allowed.is_empty() { + println!( + "robots.txt: {} disallow rules, {} allow rules", + rule.disallowed.len(), + rule.allowed.len() + ); + } + + rule +} diff --git a/src/scraper.rs b/src/scraper.rs index fff4933..9d5257d 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -1,24 +1,43 @@ -// src/scraper.rs - +use crate::converter::html_to_markdown; use crate::error_logger::log_error; use crate::extractor::extract_links; use crate::fetcher::{FetchResult, Fetcher}; +use crate::pdf_processor::{PdfProcessResult, process_pdf}; +use crate::robots::RobotsRule; use crate::url_utils::{normalize_url, url_to_filename}; use anyhow::Result; -use std::collections::HashSet; -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::fs; use std::sync::Arc; use std::time::Duration; use tokio::sync::Mutex; use url::Url; +pub struct PdfStats { + pub text_based: usize, + pub scanned: usize, + pub errors: usize, +} + +impl Default for PdfStats { + fn default() -> Self { + Self { + text_based: 0, + scanned: 0, + errors: 0, + } + } +} + pub struct Scraper { fetcher: Fetcher, seen: Arc>>, output_dir: std::path::PathBuf, log_path: std::path::PathBuf, base_domain: String, + robots: RobotsRule, + pdf_stats: Arc>, + convert_pdfs: bool, } impl Scraper { @@ -26,6 +45,8 @@ impl Scraper { output_dir: std::path::PathBuf, log_path: std::path::PathBuf, base_domain: String, + robots: RobotsRule, + convert_pdfs: bool, ) -> Self { Self { fetcher: Fetcher::new(), @@ -33,6 +54,9 @@ impl Scraper { output_dir, log_path, base_domain, + robots, + pdf_stats: Arc::new(Mutex::new(PdfStats::default())), + convert_pdfs, } } @@ -42,6 +66,7 @@ impl Scraper { let mut count = 0usize; let mut error_count = 0usize; + let mut skipped_robots = 0usize; while let Some(raw_url) = queue.pop_front() { let url = match normalize_url(raw_url.as_str()) { @@ -59,21 +84,26 @@ impl Scraper { seen.insert(url_key.clone()); } + if !self.robots.is_allowed(url.path()) { + println!("[skip] robots.txt disallows: {}", url); + skipped_robots += 1; + continue; + } + println!("[{}] fetching: {}", count, url); match self.fetcher.fetch_with_retry(&url).await { Ok(result) => { let is_html = result.content_type.contains("text/html"); + let is_pdf = result.content_type.contains("application/pdf"); - match self.save(&result).await { + match self.save(&result, is_html, is_pdf).await { Ok(()) => { if is_html { if let Ok(html) = std::str::from_utf8(&result.bytes) { let links = extract_links(html, &result.final_url, &self.base_domain); - // Only queue links not already seen — do NOT insert into seen here. - // They get inserted when actually fetched, matching the Python version. let mut new_links = Vec::new(); { @@ -106,6 +136,11 @@ impl Scraper { None, ); error_count += 1; + + if is_pdf { + let mut stats = self.pdf_stats.lock().await; + stats.errors += 1; + } } } } @@ -119,13 +154,76 @@ impl Scraper { tokio::time::sleep(Duration::from_millis(crate::config::DELAY_MS)).await; } + { + let stats = self.pdf_stats.lock().await; + if stats.text_based > 0 || stats.scanned > 0 || stats.errors > 0 { + println!("\nPDF Statistics:"); + println!(" Text-based (converted): {}", stats.text_based); + println!(" Scanned (kept as PDF): {}", stats.scanned); + println!(" Errors: {}", stats.errors); + } + } + + if skipped_robots > 0 { + println!("Skipped {} URLs due to robots.txt", skipped_robots); + } + (count, error_count) } - async fn save(&self, result: &FetchResult) -> Result<()> { - let filename = url_to_filename(&result.final_url, &result.content_type); + async fn save(&self, result: &FetchResult, is_html: bool, is_pdf: bool) -> Result<()> { + let mut filename = url_to_filename(&result.final_url, &result.content_type); + + let content: Vec; + + if is_html { + let html = std::str::from_utf8(&result.bytes)?; + let md = html_to_markdown(html); + if filename.ends_with(".html") { + filename = filename.replace(".html", ".md"); + } + content = md.into_bytes(); + } else if is_pdf && self.convert_pdfs { + // Save PDF to temp file for pdf-inspector processing + let temp_path = self.output_dir.join("temp_pdf.tmp"); + fs::write(&temp_path, &result.bytes)?; + + match process_pdf(&temp_path) { + Ok(PdfProcessResult::Markdown(md)) => { + if filename.ends_with(".pdf") { + filename = filename.replace(".pdf", ".md"); + } + content = md.into_bytes(); + fs::remove_file(&temp_path)?; + + println!(" [PDF] text-based — converted to Markdown"); + let mut stats = self.pdf_stats.lock().await; + stats.text_based += 1; + } + Ok(PdfProcessResult::Scanned) => { + println!(" [PDF] scanned document — saved as-is"); + let mut stats = self.pdf_stats.lock().await; + stats.scanned += 1; + + fs::rename(&temp_path, self.output_dir.join(&filename))?; + return Ok(()); + } + Err(e) => { + println!(" [PDF] processing error ({}), saved as-is", e); + let mut stats = self.pdf_stats.lock().await; + stats.errors += 1; + + fs::rename(&temp_path, self.output_dir.join(&filename))?; + return Ok(()); + } + } + } else { + // Non-HTML, non-PDF, or PDF conversion disabled — save as-is + content = result.bytes.clone(); + } + let filepath = self.output_dir.join(&filename); - fs::write(&filepath, &result.bytes)?; + fs::write(&filepath, &content)?; println!( " saved -> {}", filepath.file_name().unwrap_or_default().to_string_lossy() diff --git a/src/url_utils.rs b/src/url_utils.rs index ebbbff7..eb33ecf 100644 --- a/src/url_utils.rs +++ b/src/url_utils.rs @@ -1,5 +1,3 @@ -// src/url_utils.rs - use md5::Md5; use sha2::{Digest, Sha256}; use url::Url; @@ -90,6 +88,10 @@ pub fn url_to_filename(url: &Url, content_type: &str) -> String { let path = url.path().trim_start_matches('/'); let path = if path.is_empty() { "index" } else { path }; + // FIX: strip trailing slash so /support/ -> "support" not "" + let path = path.trim_end_matches('/'); + let path = if path.is_empty() { "index" } else { path }; + let last_segment = path.rsplit('/').next().unwrap_or(path); let (stem, ext_from_path) = match last_segment.rfind('.') {