rays/ipodderx-rs
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Say what a site sent when it is not XML at all

Browse Source 
doghouse answers 200 with "Unable to establish a DB connection", and parse()
reported two parser errors about end of input that buried it. A body that does
not start with < now reports its first line, and explain_failure flags it as
the publisher's problem. Malformed XML keeps the parsers' errors.

Fixes #2

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SZbKERNSt4vQfyGV8rvkqp
This commit is contained in:
Ray Slakinski
2026-09-14 21:33:13 +00:00
parent af38583b53
commit afbe6367cc
2 changed files with 52 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/feed.rs
View File

@@ -101,7 +101,7 @@ pub struct Failure {
/// text already shown once a feed is open.
///
/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP
/// {status}")`, and the "got a web page" message above) plus the substrings a DNS failure
/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure
/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing
/// nothing extra, so a miss costs a clearer message, not a wrong one.
pub fn explain_failure(msg: &str) -> Option<Failure> {
@@ -112,6 +112,9 @@ pub fn explain_failure(msg: &str) -> Option<Failure> {
.map(str::to_owned);
return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
}
if msg.contains("the site sent ") {
return Some(Failure { reason: "The site sent a message instead of the feed; the publisher has to fix it.", new_url: None });
}
let low = msg.to_ascii_lowercase();
if low.contains("http 404") {
return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None });
@@ -265,6 +268,9 @@ pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
Ok(feed) => Ok(from_atom(feed)),
Err(atom_err) => {
if let Some(said) = plain_text(bytes) {
return Err(anyhow!("the site sent {said} instead of a feed"));
}
// Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
// "&amp;". Strict XML parsers refuse it; browsers don't. Retry once with
// every offending "&" escaped rather than fail outright.
@@ -297,6 +303,26 @@ fn looks_like_html(bytes: &[u8]) -> bool {
head.contains("<!doctype html") || head.contains("<html")
}
/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with
/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried
/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors.
fn plain_text(bytes: &[u8]) -> Option<String> {
let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]);
let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}');
if text.starts_with('<') {
return None;
}
let line = text.lines().next().unwrap_or("").trim_end();
if line.is_empty() {
return Some("an empty reply".into());
}
let mut said: String = line.chars().take(80).collect();
if said.len() < line.len() {
said.push('…');
}
Some(format!("\"{said}\""))
}
/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml"
/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now,
/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos.
@@ -748,7 +774,14 @@ mod tests {
let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
for transient in ["HTTP 500 Internal Server Error", "HTTP 429 Too Many Requests", "operation timed out"] {
let down = explain_failure("the site sent \"Unable to establish a DB connection\" instead of a feed").unwrap();
assert_eq!(down.reason, "The site sent a message instead of the feed; the publisher has to fix it.");
for transient in [
"HTTP 500 Internal Server Error",
"HTTP 429 Too Many Requests",
"operation timed out",
"not RSS (reached end of input without finding a complete channel) and not Atom (unexpected end of input)",
] {
assert!(explain_failure(transient).is_none(), "{transient} must not be flagged");
}
}
@@ -769,11 +802,20 @@ mod tests {
}
#[test]
fn garbage_that_is_not_html_gets_the_original_parser_errors() {
let err = parse(b"not xml at all").unwrap_err().to_string();
fn malformed_xml_gets_the_original_parser_errors() {
let err = parse(b"<rss><channel><title>cut off").unwrap_err().to_string();
assert!(err.starts_with("not RSS ("), "{err}");
}
#[test]
fn a_body_with_no_markup_says_what_the_site_sent() {
let err = parse(b"\xef\xbb\xbf\r\n Unable to establish a DB connection\nmore").unwrap_err().to_string();
assert_eq!(err, "the site sent \"Unable to establish a DB connection\" instead of a feed");
let long = parse("x".repeat(200).as_bytes()).unwrap_err().to_string();
assert_eq!(long, format!("the site sent \"{}…\" instead of a feed", "x".repeat(80)));
assert_eq!(parse(b" \n").unwrap_err().to_string(), "the site sent an empty reply instead of a feed");
}
#[test]
fn parses_atom_enclosure_links() {
let bytes = include_bytes!("../tests/data/atom.xml");