ipx/crates/feed/src/parse.rs

//! RSS and Atom, read into `ParsedFeed`: titles as text, notes, enclosures, artwork, dates and durations.

use crate::{discover, wordpress};
use anyhow::{Result, anyhow};
use chrono::DateTime;
use std::collections::HashSet;

/// A feed as `parse` reads it, RSS or Atom alike: the channel's own details and its items.
#[derive(Debug, Default)]
pub struct ParsedFeed {
    pub title: Option<String>,
    /// RSS's `<ttl>`: how long the publisher asks to be left alone between reads.
    pub ttl_mins: Option<u64>,
    /// The show's artwork, `<itunes:image>` before the channel's `<image>`.
    pub image: Option<String>,
    /// The site the feed belongs to, where a favicon can stand in for missing artwork.
    pub site: Option<String>,
    /// The channel's first `<itunes:category>`, for the Directory's categories.
    pub category: Option<String>,
    /// What the feed says it is, for its page in the Directory (#130): RSS's `<description>`,
    /// or iTunes' summary where that is empty, or Atom's subtitle. HTML as often as not.
    pub description: Option<String>,
    pub entries: Vec<Entry>,
}

/// One item of a feed.
#[derive(Debug, Default)]
pub struct Entry {
    /// What tells this item from the feed's others: its `<guid>`, or failing that its link, its
    /// enclosure's address or its title (`pick_guid`).
    pub guid: String,
    pub title: Option<String>,
    pub link: Option<String>,
    /// Unix seconds.
    pub published: Option<i64>,
    pub description: Option<String>,
    /// The item's Markdown, when the feed sends it beside HTML of its own (`<source:markdown>`):
    /// the page can show either (#285). With no HTML it is the description instead (#284).
    pub markdown: Option<String>,
    pub categories: Vec<String>,
    /// `<itunes:explicit>` said yes.
    pub explicit: bool,
    /// Episode artwork; falls back to the feed's in the UI.
    pub image: Option<String>,
    /// Seconds.
    pub duration: Option<i64>,
    pub episode: Option<i64>,
    pub season: Option<i64>,
    pub enclosures: Vec<Enclosure>,
}

/// A file an item carries, an episode as a rule.
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Enclosure {
    pub url: String,
    /// The type the feed declares, which may be missing or wrong.
    pub mime: Option<String>,
    /// The size the feed declares, in bytes.
    pub length: Option<i64>,
}

/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
#[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
    if bytes.trim_ascii_start().starts_with(b"[")
        && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
    {
        return Ok(wordpress::from_wp_posts(&posts));
    }
    match rss::Channel::read_from(bytes) {
        Ok(ch) => Ok(from_rss(ch, bytes)),
        Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
            Ok(feed) => Ok(from_atom(feed)),
            Err(atom_err) => {
                if let Some(said) = plain_text(bytes) {
                    return Err(anyhow!("the site sent {said} instead of a feed"));
                }
                // Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
                // "&amp;". Strict XML parsers refuse it; browsers don't. Retry once with
                // every offending "&" escaped rather than fail outright.
                let escaped = escape_bare_ampersands(bytes);
                if escaped != bytes {
                    if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) {
                        return Ok(from_rss(ch, &escaped));
                    }
                    if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) {
                        return Ok(from_atom(feed));
                    }
                }
                Err(match discover::alternate_feed_link(bytes) {
                    Some(href) if looks_like_html(bytes) => {
                        anyhow!("got a web page, not a feed; it links {href} as its feed")
                    }
                    None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"),
                    _ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"),
                })
            }
        },
    }
}

/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed
/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a
/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case.
pub(crate) fn looks_like_html(bytes: &[u8]) -> bool {
    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase();
    head.contains("<!doctype html") || head.contains("<html")
}

/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with
/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried
/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors.
pub(crate) fn plain_text(bytes: &[u8]) -> Option<String> {
    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]);
    let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}');
    if text.starts_with('<') {
        return None;
    }
    let line = text.lines().next().unwrap_or("").trim_end();
    if line.is_empty() {
        return Some("an empty reply".into());
    }
    let mut said: String = line.chars().take(80).collect();
    if said.len() < line.len() {
        said.push('…');
    }
    Some(format!("\"{said}\""))
}

/// Escapes every `&` that does not already start a recognized XML entity
/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&apos;`, or a numeric reference like `&#39;`).
pub(crate) fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> {
    fn is_entity_start(rest: &[u8]) -> bool {
        for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] {
            if rest.starts_with(named) {
                return true;
            }
        }
        let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") {
            &rest[2..]
        } else if rest.starts_with(b"#") {
            &rest[1..]
        } else {
            return false;
        };
        let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count();
        len > 0 && digits.get(len) == Some(&b';')
    }

    let mut out = Vec::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) {
            out.extend_from_slice(b"&amp;");
        } else {
            out.push(bytes[i]);
        }
        i += 1;
    }
    out
}

/// Every `<enclosure>` of every `<item>`, in document order.
///
/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
/// says -- and when a feed carries several it keeps only the *last*, silently losing the
/// rest. Feeds do ship several, so read them from the XML directly.
pub(crate) fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
    use quick_xml::events::Event;

    let mut reader = quick_xml::Reader::from_reader(bytes);
    reader.config_mut().trim_text(true);
    let mut buf = Vec::new();
    let mut out: Vec<Vec<Enclosure>> = Vec::new();
    let mut current: Option<Vec<Enclosure>> = None;

    let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> {
        let (mut url, mut mime, mut length) = (String::new(), None, None);
        for attr in e.attributes().flatten() {
            // Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
            let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default();
            match attr.key.local_name().as_ref() {
                "url" => url = val,
                "type" => mime = Some(val).filter(|v| !v.is_empty()),
                "length" => length = val.parse().ok(),
                _ => {}
            }
        }
        (!url.is_empty()).then_some(Enclosure { url, mime, length })
    };

    loop {
        match reader.read_event_into(&mut buf) {
            Ok(Event::Start(e)) => match e.name().local_name().as_ref() {
                "item" => current = Some(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::Empty(e)) => match e.name().local_name().as_ref() {
                // <item/> with no children still counts, so the indexes stay aligned.
                "item" => out.push(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::End(e)) => {
                if e.name().local_name().as_ref() == "item"
                    && let Some(list) = current.take()
                {
                    out.push(list);
                }
            }
            Ok(Event::Eof) | Err(_) => break,
            _ => {}
        }
        buf.clear();
    }
    if let Some(list) = current.take() {
        out.push(list);
    }
    out
}

pub(crate) fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
    let per_item = enclosures_by_item(bytes);
    let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);

    let entries = ch
        .items()
        .iter()
        .enumerate()
        .filter_map(|(idx, item)| {
            // Straight from the XML, so an item with several keeps all of them. Falls
            // back to the parsed one if the scan and the parser disagree on item count.
            let mut enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| {
                item.enclosure()
                    .into_iter()
                    .map(|e| Enclosure {
                        url: e.url().trim().to_owned(),
                        mime: non_empty(Some(e.mime_type())),
                        length: e.length().parse().ok(),
                    })
                    .filter(|e| !e.url.is_empty())
                    .collect()
            });
            drop_player_repeats(&mut enclosures);

            let guid = pick_guid(
                item.guid().map(|g| g.value()),
                item.link(),
                enclosures.first().map(|e| e.url.as_str()),
                item.title(),
            )?;

            let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
            let it = item.itunes_ext();

            Some(Entry {
                guid,
                title: title_text(item.title()),
                link: non_empty(item.link()),
                published: item.pub_date().and_then(parse_date),
                // Content wins over description, as __getEntries preferred entry.content. The
                // source namespace's Markdown copy only when there is neither (#284).
                description: notes(item.content(), item.description(), source_markdown(item)),
                markdown: non_empty(source_markdown(item))
                    .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()),
                categories: item
                    .categories()
                    .iter()
                    .map(|c| c.name().to_owned())
                    .filter(|c| !c.is_empty() && !c.starts_with("http"))
                    .collect(),
                explicit: explicit || entry_explicit,
                image: item_image(item, &enclosures),
                duration: it.and_then(|i| i.duration()).and_then(parse_duration),
                episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
                season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: title_text(Some(ch.title())),
        ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
        // itunes:image is the square artwork; <image><url> is the older, often smaller one.
        image: ch
            .itunes_ext()
            .and_then(|i| i.image())
            .map(str::to_owned)
            .or_else(|| ch.image().map(|i| i.url().to_owned())),
        site: non_empty(Some(ch.link().trim())),
        // Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
        // would fill the Directory with one-off tags. The subcategory where there is one: Apple
        // files every tabletop and gaming show under Leisure, which says little; Games says it.
        description: non_empty(Some(ch.description())).or_else(|| non_empty(ch.itunes_ext().and_then(|i| i.summary()))),
        category: ch
            .itunes_ext()
            .and_then(|i| i.categories().first())
            .map(|c| c.subcategory().filter(|s| !s.text().trim().is_empty()).unwrap_or(c))
            .and_then(|c| non_empty(Some(c.text().trim()))),
        entries,
    }
}

pub(crate) fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
    let entries = feed
        .entries()
        .iter()
        .filter_map(|e| {
            // Atom carries enclosures as <link rel="enclosure">.
            let mut enclosures: Vec<Enclosure> = e
                .links()
                .iter()
                .filter(|l| l.rel() == "enclosure")
                .map(|l| Enclosure {
                    url: l.href().trim().to_owned(),
                    mime: non_empty(l.mime_type()),
                    length: l.length().and_then(|s| s.parse().ok()),
                })
                .filter(|e| !e.url.is_empty())
                .collect();
            drop_player_repeats(&mut enclosures);

            let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href());

            let guid =
                pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?;

            Some(Entry {
                guid,
                title: title_text(Some(e.title().as_str())),
                link: alt.map(str::to_owned),
                published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
                description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None),
                markdown: None,
                categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
                explicit: false,
                image: None,
                duration: None,
                episode: None,
                season: None,
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: title_text(Some(feed.title().as_str())),
        ttl_mins: None,
        image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
        site: feed
            .links()
            .iter()
            .find(|l| l.rel() == "alternate")
            .map(|l| l.href().trim().to_owned())
            .filter(|h| !h.is_empty()),
        category: None,
        description: non_empty(feed.subtitle().map(|t| t.as_str())),
        entries,
    }
}

/// The original fell back to hashing the title or description. A guid, permalink or
/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
/// none of them has nothing to download and is dropped.
pub(crate) fn pick_guid(
    guid: Option<&str>,
    link: Option<&str>,
    enclosure: Option<&str>,
    title: Option<&str>,
) -> Option<String> {
    [guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned)
}

pub(crate) fn is_yes(s: &str) -> bool {
    matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
}

pub(crate) fn non_empty(s: Option<&str>) -> Option<String> {
    s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
}

/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post
/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion
/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays.
pub(crate) fn drop_player_repeats(encs: &mut Vec<Enclosure>) {
    let mut seen = HashSet::new();
    encs.retain(|e| seen.insert(same_file_key(&e.url)));
}

/// An enclosure URL without WordPress's player number, for telling repeats of one file apart
/// from different files.
#[must_use]
pub fn same_file_key(url: &str) -> String {
    let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() };
    let kept: Vec<(String, String)> = u
        .query_pairs()
        .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit())))
        .map(|(k, v)| (k.into_owned(), v.into_owned()))
        .collect();
    if kept.is_empty() {
        u.set_query(None);
    } else {
        u.query_pairs_mut().clear().extend_pairs(kept);
    }
    u.to_string()
}

/// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through
/// the XML parser with its HTML entities intact: The Verge's "Meta&#8217;s" reached the page as
/// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is
/// instead of failing the whole title.
pub(crate) fn title_text(s: Option<&str>) -> Option<String> {
    let s = non_empty(s)?;
    let mut out = String::with_capacity(s.len());
    let mut rest = s.as_str();
    while let Some(at) = rest.find('&') {
        out.push_str(&rest[..at]);
        rest = &rest[at..];
        let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1);
        let decoded = rest[len..]
            .starts_with(';')
            .then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok())
            .flatten();
        match decoded {
            Some(v) => {
                out.push_str(&v);
                rest = &rest[len + 1..];
            }
            None => {
                out.push('&');
                rest = &rest[1..];
            }
        }
    }
    out.push_str(rest);
    non_empty(Some(&out))
}

/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in
/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML.
pub(crate) fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option<String> {
    match body(content, description) {
        Some(b) if is_markdown(&b) => Some(markdown_html(&b)),
        Some(b) => Some(b),
        None => non_empty(markdown).map(|m| markdown_html(&m)),
    }
}

/// `<source:markdown>`, from https://source.scripting.com/: the item's text in Markdown.
///
/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding
/// it to another prefix is not read, until one turns up.
pub(crate) fn source_markdown(item: &rss::Item) -> Option<&str> {
    item.extensions().get("source")?.get("markdown")?.first()?.value()
}

/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold.
/// Plain text without any is left as it was, since rendering it would change nothing but
/// could still read a line starting "1984." as a list.
pub(crate) fn is_markdown(s: &str) -> bool {
    let tag = s
        .match_indices('<')
        .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!'));
    let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m));
    !tag && (s.contains("](") || s.contains("**") || s.lines().any(block))
}

/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML.
#[must_use]
pub fn markdown_html(md: &str) -> String {
    use pulldown_cmark::{Options, Parser, html};
    let mut out = String::new();
    html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH));
    out
}

/// An item's show notes: its full body when that is whole, else its description.
///
/// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name
/// pasted from a web app (`[&:has([data-writing-block])>*]:pointer-events-auto`), so the body
/// began halfway through a tag and the page showed the rest of the tag as text. The same item's
/// `description` was whole. With no description to fall back on, a damaged body beats none.
pub(crate) fn body(content: Option<&str>, description: Option<&str>) -> Option<String> {
    match non_empty(content).filter(|c| !starts_mid_tag(c)) {
        Some(c) => Some(match subtitle(&c, description) {
            Some(s) => format!("<p><em>{}</em></p>{c}", quick_xml::escape::escape(s.as_str())),
            None => c,
        }),
        None => non_empty(description).or_else(|| non_empty(content)),
    }
}

/// A description that is a subtitle rather than a second copy of the notes: Substack puts the
/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it.
/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a
/// description found in the body is not shown twice.
///
/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its
/// notes passes too and shows above them, which reads fine; a real subtitle field would need an
/// `entries` column.
pub(crate) fn subtitle(body: &str, description: Option<&str>) -> Option<String> {
    let d = title_text(description)?;
    if d.contains('<') || d.chars().count() > 300 {
        return None;
    }
    // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut
    // description ends in "…" or "[...]".
    let words = |s: &str| {
        s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::<Vec<_>>().join(" ").to_lowercase()
    };
    let want = words(&d);
    let text = title_text(Some(&text_of(body))).unwrap_or_default();
    (!want.is_empty() && !words(&text).contains(&want)).then_some(d)
}

/// HTML with its tags taken out, each replaced by a space so words either side stay apart.
pub(crate) fn text_of(html: &str) -> String {
    let mut out = String::with_capacity(html.len());
    let mut in_tag = false;
    for c in html.chars() {
        match c {
            '<' => in_tag = true,
            '>' if in_tag => {
                in_tag = false;
                out.push(' ');
            }
            _ if !in_tag => out.push(c),
            _ => {}
        }
    }
    out
}

/// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose
/// start was cut off.
pub(crate) fn starts_mid_tag(html: &str) -> bool {
    html[..html.find('<').unwrap_or(html.len())].contains("\">")
}

/// The picture to show beside an item, in order of how deliberate it is:
/// `itunes:image`, then Media RSS `media:thumbnail`, then a `media:content` that is an
/// image, and finally an image enclosure -- which is how a blog's article picture arrives
/// (Substack puts it there), so those entries get artwork rather than a blank square.
pub(crate) fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> {
    if let Some(url) = item.itunes_ext().and_then(|i| i.image()) {
        return non_empty(Some(url));
    }

    let media = item.extensions().get("media");
    let attr = |name: &str, want_image: bool| -> Option<String> {
        media?.get(name)?.iter().find_map(|e| {
            if want_image {
                // media:content carries anything; only take it when it says it is a picture.
                let is_image = e.attrs.get("type").is_some_and(|t| t.starts_with("image/"))
                    || e.attrs.get("medium").is_some_and(|m| m == "image");
                if !is_image {
                    return None;
                }
            }
            non_empty(e.attrs.get("url").map(String::as_str))
        })
    };
    attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| {
        enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone())
    })
}

/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
pub(crate) fn parse_duration(s: &str) -> Option<i64> {
    let s = s.trim();
    if s.is_empty() {
        return None;
    }
    if !s.contains(':') {
        return s.parse().ok().filter(|n| *n > 0);
    }
    let mut total: i64 = 0;
    for part in s.split(':') {
        total = total * 60 + part.trim().parse::<i64>().ok()?;
    }
    Some(total).filter(|n| *n > 0)
}

/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
pub(crate) fn parse_date(s: &str) -> Option<i64> {
    let s = s.trim();
    DateTime::parse_from_rfc2822(s).or_else(|_| DateTime::parse_from_rfc3339(s)).ok().map(|d| d.timestamp())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::failure;

    #[test]
    fn parses_rss_with_itunes_extensions() {
        let bytes = include_bytes!("../../../tests/data/rss2.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Test Cast"));
        assert_eq!(feed.ttl_mins, Some(45));
        assert_eq!(feed.category.as_deref(), Some("Podcasting"), "the first, by its subcategory");
        assert_eq!(feed.description.as_deref(), Some("A synthetic feed used by the parser tests."));
        assert_eq!(feed.entries.len(), 3);

        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "https://example.com/ep/1");
        assert_eq!(ep.title.as_deref(), Some("Episode One"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(ep.categories, vec!["Tech"]);
        assert!(!ep.explicit);
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.com/ep1.mp3".into(),
                mime: Some("audio/mpeg".into()),
                length: Some(12_345_678),
            }]
        );

        assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
        assert_eq!(feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent"));
    }

    #[test]
    fn a_feed_says_what_it_is_in_itunes_summary_or_atom_subtitle() {
        // An empty <description> gives way to iTunes' summary (#130).
        let rss = br#"<?xml version="1.0"?><rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>R</title><description> </description><itunes:summary>Said here.</itunes:summary></channel></rss>"#;
        assert_eq!(parse(rss).unwrap().description.as_deref(), Some("Said here."));
        let atom = br#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>A</title>
        <subtitle>Or here.</subtitle><id>urn:a</id><updated>2026-01-01T00:00:00Z</updated></feed>"#;
        assert_eq!(parse(atom).unwrap().description.as_deref(), Some("Or here."));
    }

    #[test]
    fn markdown_notes_are_made_html_and_html_is_left_alone() {
        let feed = |items: &str| {
            let xml = format!(
                r#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>{items}</channel></rss>"#
            );
            parse(xml.as_bytes()).unwrap().entries.into_iter().map(|e| e.description).collect::<Vec<_>>()
        };
        let got = feed(
            r#"<item><guid>a</guid><description># Hi

Some **bold** and [a link](https://x.example/).</description></item>
            <item><guid>b</guid><source:markdown>Only *Markdown* here.</source:markdown></item>
            <item><guid>c</guid><description>&lt;p&gt;HTML **as sent**&lt;/p&gt;</description><source:markdown>ignored</source:markdown></item>
            <item><guid>d</guid><description>1984. Plain text, left as it is.</description></item>"#,
        );
        assert_eq!(
            got[0].as_deref(),
            Some("<h1>Hi</h1>\n<p>Some <strong>bold</strong> and <a href=\"https://x.example/\">a link</a>.</p>\n")
        );
        assert_eq!(got[1].as_deref(), Some("<p>Only <em>Markdown</em> here.</p>\n"), "the Markdown copy, alone");
        assert_eq!(got[2].as_deref(), Some("<p>HTML **as sent**</p>"), "HTML wins, untouched");
        assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is."));
    }

    #[test]
    fn markdown_sent_beside_html_is_kept_for_the_page_to_switch_to() {
        let xml = br#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>
        <item><guid>both</guid><description>&lt;p&gt;Hi&lt;/p&gt;</description><source:markdown>*Hi*</source:markdown></item>
        <item><guid>alone</guid><source:markdown>*Hi*</source:markdown></item></channel></rss>"#;
        let got: Vec<_> = parse(xml).unwrap().entries.into_iter().map(|e| e.markdown).collect();
        assert_eq!(got, [Some("*Hi*".to_owned()), None], "only beside HTML; alone, it is the description (#284)");
    }

    #[test]
    fn a_file_wordpress_lists_twice_is_one_enclosure() {
        let xml = br#"<?xml version="1.0"?><rss version="2.0"><channel><title>R</title>
        <item><title>The Promotion Paradox</title><guid>p</guid>
        <enclosure url="https://x/ep.mp3" length="1" type="audio/mpeg"/>
        <enclosure url="https://x/ep.mp3?_=2" length="1" type="audio/mpeg"/>
        <enclosure url="https://x/other.mp3?_=3&amp;key=k" length="1" type="audio/mpeg"/>
        </item></channel></rss>"#;
        let urls: Vec<String> = parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
        assert_eq!(
            urls,
            ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"],
            "the repeat goes, a different file stays"
        );
        assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k"));
        assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number");
    }

    #[test]
    fn titles_are_read_as_text_not_html() {
        // The Verge: an Atom title of type="html", its entity inside CDATA.
        let xml = br#"<?xml version="1.0"?>
        <feed xmlns="http://www.w3.org/2005/Atom"><title type="text">V</title><id>v</id>
        <updated>2026-09-15T00:00:00Z</updated>
        <entry><title type="html"><![CDATA[Meta&#8217;s new One]]></title><id>e1</id>
        <updated>2026-09-15T00:00:00Z</updated></entry></feed>"#;
        assert_eq!(parse(xml).unwrap().entries[0].title.as_deref(), Some("Meta\u{2019}s new One"));
        // HTML names as well as numbers; a bare `&` and an unknown name are left as they are.
        assert_eq!(
            title_text(Some("Pe&ntilde;a &amp; &#x201C;Q&A&#8221; &bogus; AT&T;")).as_deref(),
            Some("Pe\u{f1}a & \u{201c}Q&A\u{201d} &bogus; AT&T;")
        );
    }

    #[test]
    fn a_body_cut_off_mid_tag_gives_way_to_the_description() {
        // How libsyn served Daily Meditation Podcast #3477: content:encoded began inside a tag.
        let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant"> <p>What if</p>"#;
        let whole = r#"<div class="[&:has([data-writing-block])>*]:pointer-events-auto"><p>What if</p></div>"#;
        assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole));
        assert_eq!(
            body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(),
            Some("<p>Notes</p>"),
            "a whole body wins"
        );
        assert_eq!(
            body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(),
            Some("Plain notes, no tags.")
        );
        assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none");
        assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary"));
    }

    #[test]
    fn a_subtitle_missing_from_the_body_is_kept_above_it() {
        // Substack: the subtitle is the description, and content:encoded does not repeat it.
        assert_eq!(
            body(Some("<p>The post.</p>"), Some("Why the <b> tag & I fell out")).as_deref(),
            Some("<p>The post.</p>"),
            "a description with markup in it is notes, not a subtitle",
        );
        assert_eq!(
            body(Some("<p>The post.</p>"), Some("Why Q&amp;A threads go wrong")).as_deref(),
            Some("<p><em>Why Q&amp;A threads go wrong</em></p><p>The post.</p>"),
        );
        // A podcast repeating its notes, whole, cut short, or differently spaced: shown once.
        let notes = "<p>This week we talk about <a href=\"x\">tape</a>, drums and a very long list.</p>";
        for d in [
            "This week we talk about tape, drums and a very long list.",
            "This week we talk about tape, drums…",
            "This week we  talk about\ntape [...]",
        ] {
            assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body");
        }
        let long = "word ".repeat(80);
        assert_eq!(
            body(Some("<p>The post.</p>"), Some(&long)).as_deref(),
            Some("<p>The post.</p>"),
            "a long description is notes, not a subtitle"
        );
    }

    #[test]
    fn feed_level_explicit_overrides_entries() {
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>X</title><link>https://x</link><description>d</description>
        <itunes:explicit>yes</itunes:explicit>
        <item><title>a</title><guid>g1</guid>
        <enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();
        assert!(feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit");
    }

    #[test]
    fn failures_are_named_by_kind_for_the_log() {
        let k = |m: &str| failure::failure_kind(m);
        assert_eq!(k("HTTP 404 Not Found"), ("404".into(), Some(404)));
        assert_eq!(k("HTTP 503 Service Unavailable"), ("503".into(), Some(503)));
        assert_eq!(
            k("connecting: error following redirect for url (https://www.toddstashwick.com/): too many redirects").0,
            "redirect_loop"
        );
        assert_eq!(k("connecting: dns error: failed to lookup address information").0, "dns");
        assert_eq!(k("operation timed out").0, "timeout");
        assert_eq!(k("something new").0, "other");
    }

    #[test]
    fn a_web_page_says_so_and_names_the_feed_it_links() {
        let html = br#"<!doctype html><html><head>
            <link rel="alternate" type="application/rss+xml" href="https://x.example/feed">
            </head><body>not a feed</body></html>"#;
        let err = parse(html).unwrap_err().to_string();
        assert_eq!(err, "got a web page, not a feed; it links https://x.example/feed as its feed");
    }

    #[test]
    fn a_web_page_with_no_feed_link_still_says_so() {
        let html = b"<!doctype html><html><body>moved</body></html>";
        assert_eq!(parse(html).unwrap_err().to_string(), "got a web page, not a feed");
    }

    #[test]
    fn malformed_xml_gets_the_original_parser_errors() {
        let err = parse(b"<rss><channel><title>cut off").unwrap_err().to_string();
        assert!(err.starts_with("not RSS ("), "{err}");
    }

    #[test]
    fn a_body_with_no_markup_says_what_the_site_sent() {
        let err = parse(b"\xef\xbb\xbf\r\n  Unable to establish a DB connection\nmore").unwrap_err().to_string();
        assert_eq!(err, "the site sent \"Unable to establish a DB connection\" instead of a feed");
        let long = parse("x".repeat(200).as_bytes()).unwrap_err().to_string();
        assert_eq!(long, format!("the site sent \"{}…\" instead of a feed", "x".repeat(80)));
        assert_eq!(parse(b" \n").unwrap_err().to_string(), "the site sent an empty reply instead of a feed");
    }

    #[test]
    fn parses_atom_enclosure_links() {
        let bytes = include_bytes!("../../../tests/data/atom.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
        assert_eq!(feed.entries.len(), 1);
        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
        assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.org/ep1.m4a".into(),
                mime: Some("audio/mp4".into()),
                length: Some(9_876_543),
            }],
            "rel=enclosure only; the alternate link must not become an enclosure"
        );
    }

    #[test]
    fn a_bare_ampersand_in_a_link_is_repaired_and_parsed() {
        // kcpw.org: <link>https://kcpw.org/?post_type=post&p=125715</link> -- a bare "&"
        // that strict XML rejects but browsers accept.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0"><channel><title>X</title><link>https://x</link><description>d</description>
        <item><title>a</title><guid>g1</guid>
        <link>https://kcpw.org/?post_type=post&p=125715</link>
        <enclosure url="https://x/a.mp3?a=1&b=2" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();
        assert_eq!(feed.entries[0].link.as_deref(), Some("https://kcpw.org/?post_type=post&p=125715"));
        assert_eq!(feed.entries[0].enclosures[0].url, "https://x/a.mp3?a=1&b=2");
    }

    #[test]
    fn escape_bare_ampersands_leaves_real_entities_alone() {
        let out = escape_bare_ampersands(b"a&amp;b &lt;x&gt; &#39; &#x2F; c&d");
        assert_eq!(out, b"a&amp;b &lt;x&gt; &#39; &#x2F; c&amp;d");
    }

    #[test]
    fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
        // Some feeds set a different itunes:title. The displayed title is always the RSS
        // <title>, verbatim -- separators and all -- and season/episode are stored
        // alongside it rather than folded into it.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>Show</title><link>https://x</link><description>d</description>
        <item>
          <title>Music from a Darkened Room | Session Zero</title>
          <itunes:title>Session Zero</itunes:title>
          <guid>sz</guid>
          <itunes:season>8</itunes:season>
          <itunes:duration>6720</itunes:duration>
          <enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
        </item>
        <item>
          <title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
          <guid>p1</guid>
          <itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
          <enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
        </item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();

        let sz = &feed.entries[0];
        assert_eq!(
            sz.title.as_deref(),
            Some("Music from a Darkened Room | Session Zero"),
            "itunes:title must not override the RSS title"
        );
        assert_eq!(sz.season, Some(8));
        assert_eq!(sz.episode, None, "a missing episode number stays missing");
        assert_eq!(sz.duration, Some(6720));

        let p1 = &feed.entries[1];
        assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
        assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
    }

    #[test]
    fn an_item_may_carry_several_enclosures() {
        // The rss crate keeps only one per item -- the last -- so these come from the XML.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0"><channel><title>M</title><link>https://x</link><description>d</description>
        <item><title>Two files</title><guid>m1</guid>
          <enclosure url="https://x/a.mp3?v=1&amp;t=2" length="111" type="audio/mpeg"/>
          <enclosure url="https://x/b.mp4" length="222" type="video/mp4"/>
        </item>
        <item><title>One file</title><guid>m2</guid>
          <enclosure url="https://x/c.mp3" length="333" type="audio/mpeg"/></item>
        <item><title>None</title><guid>m3</guid></item>
        </channel></rss>"#;
        let f = parse(xml).unwrap();
        assert_eq!(f.entries.len(), 3);

        let two = &f.entries[0].enclosures;
        assert_eq!(two.len(), 2, "both enclosures survive");
        assert_eq!(two[0].url, "https://x/a.mp3?v=1&t=2", "document order, and the escaped ampersand is decoded");
        assert_eq!(two[0].length, Some(111));
        assert_eq!(two[1].url, "https://x/b.mp4");
        assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));

        assert_eq!(f.entries[1].enclosures.len(), 1);
        assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
    }

    #[test]
    fn an_items_picture_comes_from_the_most_deliberate_source() {
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
             xmlns:media="http://search.yahoo.com/mrss/">
        <channel><title>P</title><link>https://x</link><description>d</description>
        <item><title>Has itunes</title><guid>a</guid>
          <itunes:image href="https://x/itunes.jpg"/>
          <media:thumbnail url="https://x/thumb.jpg"/>
          <enclosure url="https://x/a.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Has thumbnail</title><guid>b</guid>
          <media:thumbnail url="https://x/thumb.jpg"/>
          <enclosure url="https://x/b.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Has media content</title><guid>c</guid>
          <media:content url="https://x/pic.jpg" type="image/jpeg"/>
          <media:content url="https://x/clip.mp4" type="video/mp4"/></item>
        <item><title>Only an image enclosure</title><guid>d</guid>
          <enclosure url="https://x/d.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Audio only</title><guid>e</guid>
          <enclosure url="https://x/e.mp3" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let f = parse(xml).unwrap();
        let img = |i: usize| f.entries[i].image.as_deref();
        assert_eq!(img(0), Some("https://x/itunes.jpg"), "itunes:image wins");
        assert_eq!(img(1), Some("https://x/thumb.jpg"), "then media:thumbnail");
        assert_eq!(img(2), Some("https://x/pic.jpg"), "media:content, and only the image one");
        assert_eq!(img(3), Some("https://x/d.jpg"), "a blog's article picture arrives as an enclosure");
        assert_eq!(img(4), None, "audio is not a picture");
    }

    #[test]
    fn durations_parse_from_seconds_or_a_clock() {
        assert_eq!(parse_duration("5649"), Some(5649));
        assert_eq!(parse_duration("23:45"), Some(1425));
        assert_eq!(parse_duration("1:34:09"), Some(5649));
        assert_eq!(parse_duration("0"), None, "zero is not a duration");
        assert_eq!(parse_duration(""), None);
        assert_eq!(parse_duration("garbage"), None);
    }

    #[test]
    fn rejects_html_masquerading_as_a_feed() {
        assert!(parse(b"<html><body>nope</body></html>").is_err());
    }

    #[test]
    fn guid_falls_back_through_link_then_enclosure() {
        assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
        assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
        assert_eq!(pick_guid(None, None, None, None), None);
    }
}