ipx-feed's lib.rs splits into leaf modules (#356)

lib.rs held 2,000 lines: fetching, parsing, OPML, Patreon, discovery,
artwork, categories, Jev and redaction, and their tests. It is now the
module list and the re-exports, so ipx and the directory call
feed::fetch, feed::parse and the rest as before:

  fetch.rs      fetch and its limits    parse.rs     RSS and Atom
  outlines.rs   OPML (not opml.rs, which would shadow the opml crate)
  patreon.rs    a creator's shows       discover.rs  finding a feed
  wordpress.rs  WordPress.com's API     failure.rs   failures in words
  privacy.rs    redact, looks_private   artwork.rs   https art, icons
  category.rs   Apple's categories      jev.rs       Jev's category

Moved verbatim, private items pub(crate) where a sibling calls them,
calls across modules module::function. Every definition and doc comment
line of the old lib.rs is in exactly one of the new files; each test
went with what it tests, and all 40 run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/feed/src/artwork.rs

@@ -0,0 +1,117 @@
+//! Artwork: on https when its host serves it there, and a site icon where a feed has none.
+
+use crate::{discover, fetch};
+use std::collections::HashMap;
+use std::time::Duration;
+
+/// Artwork for a feed that has none: the icon its site's page names, or else the site's
+/// `/favicon.ico`. None if neither is there.
+#[tracing::instrument(skip_all, fields(site = site))]
+pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option<String> {
+    let timeout = Duration::from_secs(20);
+    let resp = client.get(site).timeout(timeout).send().await.ok()?;
+    // Relative to where the page ended up, not where it was asked for: a site that redirects
+    // to /en/ would otherwise have its icon looked for in the wrong place.
+    let base = resp.url().clone();
+    // The icon a page names can be gone: antirez.com names /images/favicon.png, a 404, while its
+    // /favicon.ico is there. Stored unchecked, it was a broken image that was never looked up again.
+    if resp.status().is_success()
+        && let Ok(page) = fetch::read_capped(resp).await
+        && let Some(href) = page_icon(&page)
+        && let Ok(url) = base.join(&href)
+        && is_image(client, url.as_str()).await
+    {
+        return Some(url.into());
+    }
+    let ico = base.join("/favicon.ico").ok()?;
+    is_image(client, ico.as_str()).await.then(|| ico.into())
+}
+
+/// How many hosts one scan of a feed asks whether its artwork is on https.
+pub(crate) const MAX_ART_HOSTS: usize = 8;
+
+/// Artwork's address on https when its host serves it there, else as it was. The page is https
+/// and must not load http; the host is asked once per `known` (one feed's read), and an http
+/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five
+/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN
+/// presents another name's certificate (#110).
+pub async fn prefer_https(client: &reqwest::Client, url: &str, known: &mut HashMap<String, bool>) -> String {
+    let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() };
+    let host = rest.split('/').next().unwrap_or("").to_owned();
+    let secure = format!("https://{rest}");
+    let ok = match known.get(&host) {
+        Some(ok) => *ok,
+        // Each new host is a probe of up to 20 s, one after another on the scan, and the feed's
+        // author decides how many hosts there are: ports and user names make as many as they like
+        // of one server (#339). Real feeds put their artwork on a few.
+        None if known.len() >= MAX_ART_HOSTS => false,
+        None => {
+            let ok = is_image(client, &secure).await;
+            known.insert(host, ok);
+            ok
+        }
+    };
+    if ok { secure } else { url.to_owned() }
+}
+
+/// Whether `url` answers with an image. A site with no favicon often answers 200 with its home
+/// page, which is not an icon.
+pub async fn is_image(client: &reqwest::Client, url: &str) -> bool {
+    let timeout = Duration::from_secs(20);
+    let Ok(resp) = client.get(url).timeout(timeout).send().await else { return false };
+    resp.status().is_success()
+        && resp
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|v| v.to_str().ok())
+            .is_some_and(|t| t.starts_with("image/"))
+}
+
+/// The icon a web page names in its `<link>` tags, the larger apple-touch-icon first: a plain
+/// `icon` is often 16 pixels, which blurs at the size the list draws artwork.
+pub(crate) fn page_icon(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    // ASCII only, so byte offsets in the lowered copy stay valid in the original.
+    let lower = text.to_ascii_lowercase();
+    let (mut touch, mut icon) = (None, None);
+    let mut pos = 0;
+    while let Some(at) = lower[pos..].find("<link") {
+        let start = pos + at;
+        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
+        pos = end + 1;
+        let tag = &text[start..end];
+        let Some(rel) = discover::tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue };
+        let rels: Vec<&str> = rel.split_whitespace().collect();
+        if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) {
+            touch = discover::tag_attr(tag, "href");
+        } else if icon.is_none() && rels.contains(&"icon") {
+            icon = discover::tag_attr(tag, "href");
+        }
+    }
+    touch.or(icon).filter(|h| !h.is_empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn artwork_on_more_hosts_than_the_limit_is_not_asked_about() {
+        let mut known: std::collections::HashMap<String, bool> =
+            (0..MAX_ART_HOSTS).map(|n| (format!("h{n}:80"), false)).collect();
+        // No server answers here: were it asked, this would wait on a connection, not return.
+        let url = "http://192.0.2.1:9/a.png";
+        assert_eq!(prefer_https(&reqwest::Client::new(), url, &mut known).await, url);
+        assert_eq!(known.len(), MAX_ART_HOSTS);
+    }
+
+    #[test]
+    fn page_icon_prefers_the_touch_icon() {
+        let page = br#"<head><link rel="stylesheet" href="/a.css">
+            <link rel="shortcut icon" href="/fav.ico">
+            <LINK REL="apple-touch-icon-precomposed" sizes="180x180" href='/touch.png'></head>"#;
+        assert_eq!(page_icon(page).as_deref(), Some("/touch.png"));
+        assert_eq!(page_icon(br#"<link rel="icon" href="i.svg">"#).as_deref(), Some("i.svg"));
+        assert_eq!(page_icon(br#"<link rel="stylesheet" href="/a.css">"#), None);
+    }
+}

crates/feed/src/category.rs

@@ -0,0 +1,125 @@
+//! Apple Podcasts' categories, and where a feed's category sits among them (#118).
+
+/// Apple's podcast categories, each with its subcategories
+/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own
+/// `<itunes:category>` comes from, and a fixed one, so the Directory's categories stay few.
+pub(crate) const CATEGORIES: &[(&str, &[&str])] = &[
+    ("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]),
+    ("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]),
+    ("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]),
+    ("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]),
+    ("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]),
+    ("Government", &[]),
+    ("History", &[]),
+    ("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]),
+    ("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]),
+    (
+        "Leisure",
+        &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"],
+    ),
+    ("Music", &["Music Commentary", "Music History", "Music Interviews"]),
+    (
+        "News",
+        &[
+            "Business News",
+            "Daily News",
+            "Entertainment News",
+            "News Commentary",
+            "Politics",
+            "Sports News",
+            "Tech News",
+        ],
+    ),
+    (
+        "Religion & Spirituality",
+        &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"],
+    ),
+    (
+        "Science",
+        &[
+            "Astronomy",
+            "Chemistry",
+            "Earth Sciences",
+            "Life Sciences",
+            "Mathematics",
+            "Natural Sciences",
+            "Nature",
+            "Physics",
+            "Social Sciences",
+        ],
+    ),
+    ("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]),
+    (
+        "Sports",
+        &[
+            "Baseball",
+            "Basketball",
+            "Cricket",
+            "Fantasy Sports",
+            "Football",
+            "Golf",
+            "Hockey",
+            "Rugby",
+            "Running",
+            "Soccer",
+            "Swimming",
+            "Tennis",
+            "Volleyball",
+            "Wilderness",
+            "Wrestling",
+        ],
+    ),
+    ("Technology", &[]),
+    ("True Crime", &[]),
+    ("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]),
+];
+
+/// Names that are not in Apple's list but mean one that is (#125). Jev picked from this list when
+/// it spelt Football and Soccer its own way, and stored those answers; Software How-To is Apple's
+/// Technology subcategory from before its 2019 list, which older feeds still name.
+pub(crate) const ALIASES: &[(&str, &str)] =
+    &[("American Football", "Football"), ("Football (Soccer)", "Soccer"), ("Software How-To", "Technology")];
+
+/// Apple's categories as ipx has them, for the directory to check against Apple's own (#252).
+#[must_use]
+pub fn categories() -> &'static [(&'static str, &'static [&'static str])] {
+    CATEGORIES
+}
+
+/// Where a category sits in Apple's list, as (category, subcategory), so the Directory browses a
+/// category and then its subcategories as Apple's does (#118). A podcast's own is stored as its
+/// subcategory, and one an admin typed that is not Apple's stands as a category of its own.
+#[must_use]
+pub fn category_path(name: &str) -> (String, Option<String>) {
+    // A feed's own text, which can end in a stray comma: "Soccer," was a category of its own.
+    let name = name.trim().trim_end_matches([',', ';', '.']).trim_end();
+    let name = ALIASES.iter().find(|(old, _)| old.eq_ignore_ascii_case(name)).map_or(name, |(_, now)| now);
+    for (category, subs) in CATEGORIES {
+        if category.eq_ignore_ascii_case(name) {
+            return (category.to_string(), None);
+        }
+        if let Some(sub) = subs.iter().find(|s| s.eq_ignore_ascii_case(name)) {
+            return (category.to_string(), Some(sub.to_string()));
+        }
+    }
+    (name.to_string(), None)
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn category_path_finds_a_subcategory_s_category() {
+        use super::category_path;
+        assert_eq!(category_path("Tech News"), ("News".into(), Some("Tech News".into())));
+        assert_eq!(category_path("video games"), ("Leisure".into(), Some("Video Games".into())), "Apple's spelling");
+        assert_eq!(category_path("Technology"), ("Technology".into(), None));
+        assert_eq!(category_path("Homebrew"), ("Homebrew".into(), None), "not Apple's: its own category");
+        // Apple's spelling, an old answer of Jev's, a stray comma and Apple's old list (#125).
+        assert_eq!(category_path("Soccer"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Football (Soccer)"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Soccer,"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Football"), ("Sports".into(), Some("Football".into())));
+        assert_eq!(category_path("Software How-To"), ("Technology".into(), None));
+    }
+}

crates/feed/src/discover.rs

@@ -0,0 +1,107 @@
+//! Finding a feed from what someone typed: a feed address, a site, or a page linking one.
+
+use crate::{fetch, outlines, parse};
+use anyhow::{Context, Result};
+use std::time::Duration;
+
+/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's
+/// feed, since the token alone says whose it is. An address with no scheme is https: 'cnn.com'
+/// was stored as typed and every check failed with "relative URL without a base" (#101).
+#[must_use]
+pub fn expand_input(input: &str) -> String {
+    let s = input.trim();
+    let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
+    if token {
+        format!("https://www.patreon.com/rss?auth={s}")
+    } else if let Some(rest) = s.strip_prefix("//") {
+        format!("https://{rest}")
+    } else if !s.contains("://") {
+        format!("https://{s}")
+    } else {
+        s.to_owned()
+    }
+}
+
+/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml"
+/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now,
+/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos.
+pub(crate) fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    // ASCII only: to_lowercase changes some characters' length (U+0130 grows a byte), and the
+    // offsets found in the lowered copy then sliced the original off a char boundary.
+    let lower = text.to_ascii_lowercase();
+    let mut pos = 0;
+    while let Some(rel) = lower[pos..].find("<link") {
+        let start = pos + rel;
+        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
+        pos = end + 1;
+        let tag = &text[start..end];
+        let tag_lower = &lower[start..end];
+        let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'");
+        let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml");
+        if is_alternate
+            && is_feed_type
+            && let Some(href) = tag_attr(tag, "href")
+        {
+            return Some(href);
+        }
+    }
+    None
+}
+
+/// What someone asked to add, as a feed: the address itself when it is a feed or an OPML list,
+/// otherwise the feed its web page links as its own, otherwise an error and nothing is added.
+/// A page that linked no feed used to be added as it was and failed on every check, called a
+/// feed that moved (#102); cnn.com is one.
+pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> {
+    let read = |u: String| async move {
+        let resp = client.get(&u).timeout(Duration::from_secs(30)).send().await.context("connecting")?;
+        anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status());
+        let base = resp.url().clone();
+        anyhow::Ok((base, fetch::read_capped(resp).await.context("reading")?))
+    };
+    let is_feed = |b: &[u8]| outlines::is_opml(b) || parse::parse(b).is_ok();
+    let (base, body) = read(url.to_owned()).await.with_context(|| format!("could not read {url}"))?;
+    if is_feed(&body) {
+        return Ok(url.to_owned());
+    }
+    if !parse::looks_like_html(&body) {
+        let why = parse::parse(&body).err().map(|e| format!("{e:#}")).unwrap_or_default();
+        anyhow::bail!("{url} is not a feed: {why}");
+    }
+    let Some(href) = alternate_feed_link(&body) else {
+        anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to");
+    };
+    let linked =
+        base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?;
+    let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?;
+    anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either");
+    Ok(linked.into())
+}
+
+/// The value of one attribute in an HTML/XML start tag, however it is quoted.
+pub(crate) fn tag_attr(tag: &str, name: &str) -> Option<String> {
+    let key = format!("{name}=");
+    let idx = tag.to_lowercase().find(&key)?;
+    let after = &tag[idx + key.len()..];
+    let quote = after.chars().next()?;
+    if quote != '"' && quote != '\'' {
+        return None;
+    }
+    let rest = &after[1..];
+    let close = rest.find(quote)?;
+    Some(rest[..close].trim().to_owned())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_feed_link_after_non_ascii_text_is_found() {
+        // U+0130 lowercases to three bytes from two, which once shifted every offset after it.
+        let page = "<html><title>\u{130}stanbul \u{130}\u{130}</title>\
+            <link rel=\"alternate\" type=\"application/rss+xml\" href=\"/feed.xml\">";
+        assert_eq!(alternate_feed_link(page.as_bytes()).as_deref(), Some("/feed.xml"));
+    }
+}

crates/feed/src/failure.rs

@@ -0,0 +1,145 @@
+//! A failed read, in words for whoever subscribes, and as a kind for the log.
+
+/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
+/// it is, and whether there is a new address to switch to.
+pub struct Failure {
+    /// What to tell them, in a sentence.
+    pub reason: &'static str,
+    /// Where the feed went, when the error says.
+    pub new_url: Option<String>,
+}
+
+/// A failure's kind, for the log's `error.type` (#91): the HTTP status where there is one, as
+/// OpenTelemetry names an HTTP error, and otherwise a word for what went wrong. Matches the
+/// same wording as `explain_failure`; a message it does not know is "other", never a wrong kind.
+#[must_use]
+pub fn failure_kind(msg: &str) -> (String, Option<u16>) {
+    let low = msg.to_ascii_lowercase();
+    let code = low.split("http ").skip(1).find_map(|r| r.get(..3)?.parse::<u16>().ok());
+    if let Some(c) = code.filter(|c| (100..600).contains(c)) {
+        return (c.to_string(), Some(c));
+    }
+    let kind = if low.contains("dns error")
+        || low.contains("failed to lookup address")
+        || low.contains("no address associated")
+    {
+        "dns"
+    } else if low.contains("too many redirects") {
+        "redirect_loop"
+    } else if low.contains("timed out") || low.contains("timeout") {
+        "timeout"
+    } else if low.contains("certificate") || low.contains("tls") {
+        "tls"
+    } else if low.contains("got a web page") {
+        "not_a_feed"
+    } else if low.contains("the site sent ") {
+        "site_message"
+    } else if low.contains("connect") {
+        "connect"
+    } else if low.contains("pars") {
+        "parse"
+    } else {
+        "other"
+    };
+    (kind.into(), None)
+}
+
+/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the
+/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling
+/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled --
+/// comes back `None`: transient by nature, or with nothing more useful to say than the raw
+/// text already shown once a feed is open.
+///
+/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP
+/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure
+/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing
+/// nothing extra, so a miss costs a clearer message, not a wrong one.
+pub fn explain_failure(msg: &str) -> Option<Failure> {
+    if let Some(rest) = msg.strip_prefix("got a web page, not a feed") {
+        let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned);
+        return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
+    }
+    if msg.contains("the site sent ") {
+        return Some(Failure {
+            reason: "The site sent a message instead of the feed; the publisher has to fix it.",
+            new_url: None,
+        });
+    }
+    let low = msg.to_ascii_lowercase();
+    if low.contains("http 404") {
+        return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None });
+    }
+    if low.contains("http 401") || low.contains("http 403") {
+        return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None });
+    }
+    if low.contains("answered with nothing") {
+        return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None });
+    }
+    if low.contains("http 402") {
+        return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None });
+    }
+    // Only a name that does not exist. A lookup that failed for the moment says nothing about the
+    // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143).
+    if !is_temporary_lookup_failure(msg)
+        && (low.contains("dns error")
+            || low.contains("failed to lookup address")
+            || low.contains("no address associated"))
+    {
+        return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None });
+    }
+    None
+}
+
+/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's
+/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does
+/// not exist ("Name or service not known"). It says the network was not answering, not that the
+/// feed is gone (#143).
+#[must_use]
+pub fn is_temporary_lookup_failure(msg: &str) -> bool {
+    let low = msg.to_ascii_lowercase();
+    low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn explain_failure_translates_the_errors_the_ui_should_flag() {
+        assert_eq!(
+            explain_failure("HTTP 404 Not Found").unwrap().reason,
+            "The publisher took this feed down, or moved it."
+        );
+        assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests.");
+        assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests.");
+        assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan.");
+        assert_eq!(
+            explain_failure("the feed's address answered with nothing at all").unwrap().reason,
+            "This address answers with nothing; the site may be gone."
+        );
+        let dns =
+            explain_failure("connecting: dns error: failed to lookup address information: Name or service not known")
+                .unwrap();
+        assert_eq!(dns.reason, "This address no longer resolves; the site may be gone.");
+        // A lookup that failed for the moment is not a site gone (#143).
+        let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution";
+        assert!(is_temporary_lookup_failure(blip));
+        assert!(explain_failure(blip).is_none());
+        assert!(!is_temporary_lookup_failure(
+            "connecting: dns error: failed to lookup address information: Name or service not known"
+        ));
+        let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
+        assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
+        assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
+        let down = explain_failure("the site sent \"Unable to establish a DB connection\" instead of a feed").unwrap();
+        assert_eq!(down.reason, "The site sent a message instead of the feed; the publisher has to fix it.");
+        for transient in [
+            "HTTP 500 Internal Server Error",
+            "HTTP 429 Too Many Requests",
+            "operation timed out",
+            "not RSS (reached end of input without finding a complete channel) and not Atom (unexpected end of input)",
+        ] {
+            assert!(explain_failure(transient).is_none(), "{transient} must not be flagged");
+        }
+    }
+}

crates/feed/src/fetch.rs

@@ -0,0 +1,189 @@
+//! Reading a feed over HTTP: conditional requests, redirects followed one by one, bounded bodies.
+
+use crate::{privacy, wordpress};
+use anyhow::{Context, Result, anyhow};
+use reqwest::StatusCode;
+use reqwest::header::HeaderName;
+use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};
+use std::sync::atomic::AtomicU64;
+use std::time::Duration;
+
+/// What `fetch` got.
+pub enum Fetched {
+    /// Server said 304, or returned a body we already have.
+    NotModified,
+    /// A body to read, with the validators to send next time.
+    Body { bytes: Vec<u8>, etag: Option<String>, last_modified: Option<String> },
+}
+
+/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and
+/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504
+/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small;
+/// downloads, which are not, have no such limit.
+pub const FEED_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// The longest a feed's server may send nothing, for the clients that fetch feeds to set: one
+/// that took the connection and never answered ran the whole 30 s, every scan it was in (#230).
+/// No feed that answered took over 8 s in a day of traces; one that keeps sending, however
+/// slowly, still has until FEED_TIMEOUT.
+pub const FEED_SILENCE: Duration = Duration::from_secs(15);
+
+/// Feeds fetched and the bytes of their bodies, since the process started, for /metrics to set
+/// beside memory (#224): a body is read whole, so a night of large feeds is a night of large
+/// allocations.
+pub static FETCHES: AtomicU64 = AtomicU64::new(0);
+
+/// See `FETCHES`.
+pub static FETCHED_BYTES: AtomicU64 = AtomicU64::new(0);
+
+/// The most of a feed or page read. The largest feeds here are a few MB; without a limit an
+/// episode's MP3 given as a feed was read whole into memory, at every check (#295).
+pub(crate) const MAX_BODY: usize = 50 << 20;
+
+/// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read.
+pub(crate) async fn read_capped(mut resp: reqwest::Response) -> Result<Vec<u8>> {
+    if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) {
+        return Err(anyhow!("larger than {} MB, too large to be a feed", MAX_BODY >> 20));
+    }
+    let mut out = Vec::new();
+    while let Some(c) = resp.chunk().await? {
+        out.extend_from_slice(&c);
+        anyhow::ensure!(out.len() <= MAX_BODY, "larger than {} MB, too large to be a feed", MAX_BODY >> 20);
+    }
+    Ok(out)
+}
+
+/// Reads a feed, asking only for what changed since `etag` and `last_modified`. Redirects are
+/// followed here, up to ten, so `auth` goes to the feed's own host alone. Also returns the new
+/// address when every hop said the feed moved for good, for the caller to keep instead.
+#[tracing::instrument(skip_all, fields(url = %privacy::redact(feed_url), http.response.body.size))]
+pub async fn fetch(
+    client: &reqwest::Client,
+    feed_url: &str,
+    auth: Option<(&str, Option<String>)>,
+    etag: Option<&str>,
+    last_modified: Option<&str>,
+) -> Result<(Fetched, Option<String>)> {
+    let start = reqwest::Url::parse(feed_url).context("the feed's address")?;
+    let mut url = start.clone();
+    let mut permanent = true;
+    for _ in 0..10 {
+        let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT);
+        if let Some(tag) = etag {
+            req = req.header(IF_NONE_MATCH, tag);
+        }
+        if let Some(lm) = last_modified {
+            req = req.header(IF_MODIFIED_SINCE, lm);
+        }
+        // The feed's own host only: a redirect elsewhere must not be handed the password.
+        if let Some((user, password)) = &auth
+            && url.host_str() == start.host_str()
+        {
+            req = req.basic_auth(user, password.as_deref());
+        }
+
+        let resp = req.send().await.context("connecting")?;
+        let status = resp.status();
+        if status.is_redirection() && status != StatusCode::NOT_MODIFIED {
+            let to = resp
+                .headers()
+                .get(reqwest::header::LOCATION)
+                .and_then(|v| v.to_str().ok())
+                .ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?;
+            url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?;
+            permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT);
+            continue;
+        }
+        let moved = (permanent && url != start).then(|| url.to_string());
+        FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        if status == StatusCode::NOT_MODIFIED {
+            return Ok((Fetched::NotModified, moved));
+        }
+        if !status.is_success() {
+            // WordPress.com answers some readers' feed requests with a browser check that only
+            // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the
+            // same posts and no check (#317).
+            if status == StatusCode::FORBIDDEN
+                && let Some(host) = url.host_str().map(str::to_owned)
+                && read_capped(resp).await.is_ok_and(|b| wordpress::is_wpcom_challenge(&b))
+            {
+                return Ok((wordpress::wpcom_posts(client, &host).await?, moved));
+            }
+            // The original surfaced 401/407 specially; the code is enough for a UI to switch on.
+            return Err(anyhow!("HTTP {status}"));
+        }
+        let header = |h: HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned);
+        let etag = header(ETAG);
+        let last_modified = header(LAST_MODIFIED);
+        let bytes = read_capped(resp).await.context("reading body")?;
+        tracing::Span::current().record("http.response.body.size", bytes.len());
+        FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
+        return Ok((Fetched::Body { bytes, etag, last_modified }, moved));
+    }
+    // Worded as reqwest worded it, which failure_kind reads as a redirect loop.
+    Err(anyhow!("error following redirect for url ({url}): too many redirects"))
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::failure;
+
+    /// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
+    /// to /tmp, /new is the feed.
+    async fn redirecting_server() -> String {
+        use tokio::io::{AsyncReadExt, AsyncWriteExt};
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        tokio::spawn(async move {
+            loop {
+                let Ok((mut sock, _)) = listener.accept().await else { return };
+                tokio::spawn(async move {
+                    let mut buf = [0u8; 2048];
+                    let n = sock.read(&mut buf).await.unwrap_or(0);
+                    let req = String::from_utf8_lossy(&buf[..n]);
+                    let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned();
+                    let body = "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title></channel></rss>";
+                    let resp = match path.as_str() {
+                        "/old" => {
+                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        "/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
+                        "/chain" => {
+                            "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        // An episode's file given as a feed: its size said, its body never sent.
+                        "/huge" => "HTTP/1.1 200 OK\r\nContent-Length: 600000000\r\n\r\n".to_owned(),
+                        "/loop" => {
+                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        _ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()),
+                    };
+                    let _ = sock.write_all(resp.as_bytes()).await;
+                });
+            }
+        });
+        format!("http://{addr}")
+    }
+
+    #[tokio::test]
+    async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() {
+        let base = redirecting_server().await;
+        let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap();
+        let get = |path: &str| {
+            let url = format!("{base}{path}");
+            let client = client.clone();
+            async move { super::fetch(&client, &url, None, None, None).await }
+        };
+        let (got, moved) = get("/old").await.unwrap();
+        assert!(matches!(got, super::Fetched::Body { .. }));
+        assert_eq!(moved, Some(format!("{base}/new")));
+        assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
+        let huge = get("/huge").await.err().map(|e| format!("{e:#}")).unwrap_or_default();
+        assert!(huge.contains("too large to be a feed"), "{huge}");
+        assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
+        assert_eq!(get("/new").await.unwrap().1, None); // never moved
+        let looped = get("/loop").await.err().unwrap().to_string();
+        assert_eq!(failure::failure_kind(&looped).0, "redirect_loop");
+    }
+}

crates/feed/src/jev.rs

@@ -0,0 +1,67 @@
+//! Asking TypeSafe's Jev which Apple category a feed belongs to.
+
+use crate::parse::ParsedFeed;
+use crate::{category, fetch};
+use anyhow::{Result, anyhow};
+
+/// TypeSafe's System One endpoint, where Jev answers questions about a piece of state.
+pub const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone";
+
+/// A Directory category for a feed that names none of its own, which most blogs do not (#117):
+/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and
+/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest
+/// is taken however unsure, since an admin can change it and asking again would cost a call a scan.
+pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result<String> {
+    let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect();
+    // A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous.
+    let mut criteria = serde_json::Map::new();
+    for (category, subs) in category::CATEGORIES {
+        criteria.insert(category.to_string(), serde_json::Value::Null);
+        for sub in *subs {
+            criteria.insert(sub.to_string(), format!("Within {category}").into());
+        }
+    }
+    let body = serde_json::json!({
+        "model": "jev-latest",
+        "state": { "title": feed.title, "recent_items": items },
+        "questions": { "category": {
+            "type": "choice",
+            "instructions": "Which category best describes what this feed publishes?",
+            "criteria": criteria,
+        }},
+    });
+    let answer: serde_json::Value = client
+        .post(JEV_URL)
+        .bearer_auth(key)
+        .json(&body)
+        .timeout(fetch::FEED_TIMEOUT)
+        .send()
+        .await?
+        .error_for_status()?
+        .json()
+        .await?;
+    picked(&answer)
+}
+
+pub(crate) fn picked(answer: &serde_json::Value) -> Result<String> {
+    let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default();
+    category::CATEGORIES
+        .iter()
+        .flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter()))
+        .find(|c| **c == choice)
+        .map(|c| c.to_string())
+        .ok_or_else(|| anyhow!("no category in the answer: {answer}"))
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn picked_takes_only_an_apple_category() {
+        let answer = |c: &str| serde_json::json!({"answers": {"category": {"type": "choice", "choice": c}}});
+        assert_eq!(super::picked(&answer("Technology")).unwrap(), "Technology");
+        assert_eq!(super::picked(&answer("Tech News")).unwrap(), "Tech News", "a subcategory");
+        assert!(super::picked(&answer("Blogs")).is_err(), "not one of Apple's");
+        assert!(super::picked(&serde_json::json!({"error": "rate limited"})).is_err());
+    }
+}

crates/feed/src/outlines.rs

@@ -0,0 +1,111 @@
+//! OPML documents: recognising one, reading it safely, and the feeds it lists.
+
+use anyhow::{Result, anyhow};
+
+/// True when a body is an OPML document rather than a feed.
+///
+/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
+/// OPML served from a URL without that extension. Sniffing the body catches both.
+#[must_use]
+pub fn is_opml(bytes: &[u8]) -> bool {
+    let head = &bytes[..bytes.len().min(1024)];
+    let text = String::from_utf8_lossy(head).to_lowercase();
+    text.contains("<opml")
+}
+
+/// How deep an OPML document's elements may nest. Real ones are a folder or two deep.
+pub(crate) const MAX_OPML_DEPTH: usize = 100;
+
+/// Parses an OPML document, refusing one nested deeper than `MAX_OPML_DEPTH`. The opml crate's
+/// reader recurses once per level with no limit, and a few MB of `<outline>` overflowed the
+/// stack, which aborts the whole daemon, and again on each restart for a subscribed feed (#322).
+pub fn read_opml(text: &str) -> Result<opml::OPML> {
+    use quick_xml::events::Event;
+    let mut reader = quick_xml::Reader::from_str(text);
+    let mut depth = 0usize;
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(_)) => {
+                depth += 1;
+                if depth > MAX_OPML_DEPTH {
+                    anyhow::bail!("that OPML nests more than {MAX_OPML_DEPTH} levels deep");
+                }
+            }
+            Ok(Event::End(_)) => depth = depth.saturating_sub(1),
+            // Malformed XML is the opml crate's to describe.
+            Ok(Event::Eof) | Err(_) => break,
+            Ok(_) => {}
+        }
+    }
+    opml::OPML::from_str(text).map_err(|e| anyhow!("{e}"))
+}
+
+/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
+pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
+    let text = String::from_utf8_lossy(bytes);
+    let doc = read_opml(&text).map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
+    let mut out = vec![];
+    collect_outlines(&doc.body.outlines, &mut out);
+    Ok(out)
+}
+
+/// OPML nests feeds inside folder outlines, so this walks the whole tree.
+pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, String)>) {
+    for o in outlines {
+        if let Some(url) = &o.xml_url {
+            let title = o.title.clone().unwrap_or_else(|| o.text.clone());
+            out.push((title, url.clone()));
+        }
+        collect_outlines(&o.outlines, out);
+    }
+}
+
+/// The <head><title> of an OPML document.
+#[must_use]
+pub fn opml_title(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    let doc = read_opml(&text).ok()?;
+    doc.head.and_then(|h| h.title).map(|t| t.trim().to_owned()).filter(|t| !t.is_empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deeply_nested_opml_is_refused_before_it_is_parsed() {
+        let nest = |n: usize| {
+            format!(
+                r#"<opml version="2.0"><head><title>T</title></head><body>{}<outline text="f" xmlUrl="https://e.com/f"/>{}</body></opml>"#,
+                r#"<outline text="a">"#.repeat(n),
+                "</outline>".repeat(n)
+            )
+        };
+        assert_eq!(parse_opml(nest(5).as_bytes()).unwrap().len(), 1, "a few folders deep is fine");
+        // Deep enough to overflow a 2 MiB stack in the opml crate's reader without the check.
+        let deep = nest(100_000);
+        assert!(parse_opml(deep.as_bytes()).unwrap_err().to_string().contains("levels deep"));
+        assert_eq!(opml_title(deep.as_bytes()), None);
+    }
+
+    #[test]
+    fn opml_is_recognised_and_its_feeds_listed() {
+        let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
+          <outline text="Folder">
+            <outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
+            <outline type="rss" text="Beta"  xmlUrl="https://b.example/rss"/>
+          </outline>
+          <outline text="Not a feed"/>
+        </body></opml>"#;
+        assert!(is_opml(xml));
+        assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));
+
+        let feeds = parse_opml(xml).unwrap();
+        assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
+        assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));
+
+        // A feed must never be mistaken for a subscription list.
+        assert!(!is_opml(include_bytes!("../../../tests/data/rss2.xml")));
+        assert!(!is_opml(include_bytes!("../../../tests/data/atom.xml")));
+    }
+}

crates/feed/src/parse.rs

@@ -0,0 +1,980 @@
+//! RSS and Atom, read into `ParsedFeed`: titles as text, notes, enclosures, artwork, dates and durations.
+
+use crate::{discover, wordpress};
+use anyhow::{Result, anyhow};
+use chrono::DateTime;
+use std::collections::HashSet;
+
+/// A feed as `parse` reads it, RSS or Atom alike: the channel's own details and its items.
+#[derive(Debug, Default)]
+pub struct ParsedFeed {
+    pub title: Option<String>,
+    /// RSS's `<ttl>`: how long the publisher asks to be left alone between reads.
+    pub ttl_mins: Option<u64>,
+    /// The show's artwork, `<itunes:image>` before the channel's `<image>`.
+    pub image: Option<String>,
+    /// The site the feed belongs to, where a favicon can stand in for missing artwork.
+    pub site: Option<String>,
+    /// The channel's first `<itunes:category>`, for the Directory's categories.
+    pub category: Option<String>,
+    /// What the feed says it is, for its page in the Directory (#130): RSS's `<description>`,
+    /// or iTunes' summary where that is empty, or Atom's subtitle. HTML as often as not.
+    pub description: Option<String>,
+    pub entries: Vec<Entry>,
+}
+
+/// One item of a feed.
+#[derive(Debug, Default)]
+pub struct Entry {
+    /// What tells this item from the feed's others: its `<guid>`, or failing that its link, its
+    /// enclosure's address or its title (`pick_guid`).
+    pub guid: String,
+    pub title: Option<String>,
+    pub link: Option<String>,
+    /// Unix seconds.
+    pub published: Option<i64>,
+    pub description: Option<String>,
+    /// The item's Markdown, when the feed sends it beside HTML of its own (`<source:markdown>`):
+    /// the page can show either (#285). With no HTML it is the description instead (#284).
+    pub markdown: Option<String>,
+    pub categories: Vec<String>,
+    /// `<itunes:explicit>` said yes.
+    pub explicit: bool,
+    /// Episode artwork; falls back to the feed's in the UI.
+    pub image: Option<String>,
+    /// Seconds.
+    pub duration: Option<i64>,
+    pub episode: Option<i64>,
+    pub season: Option<i64>,
+    pub enclosures: Vec<Enclosure>,
+}
+
+/// A file an item carries, an episode as a rule.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct Enclosure {
+    pub url: String,
+    /// The type the feed declares, which may be missing or wrong.
+    pub mime: Option<String>,
+    /// The size the feed declares, in bytes.
+    pub length: Option<i64>,
+}
+
+/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
+#[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
+pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
+    if bytes.trim_ascii_start().starts_with(b"[")
+        && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
+    {
+        return Ok(wordpress::from_wp_posts(&posts));
+    }
+    match rss::Channel::read_from(bytes) {
+        Ok(ch) => Ok(from_rss(ch, bytes)),
+        Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
+            Ok(feed) => Ok(from_atom(feed)),
+            Err(atom_err) => {
+                if let Some(said) = plain_text(bytes) {
+                    return Err(anyhow!("the site sent {said} instead of a feed"));
+                }
+                // Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
+                // "&amp;". Strict XML parsers refuse it; browsers don't. Retry once with
+                // every offending "&" escaped rather than fail outright.
+                let escaped = escape_bare_ampersands(bytes);
+                if escaped != bytes {
+                    if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) {
+                        return Ok(from_rss(ch, &escaped));
+                    }
+                    if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) {
+                        return Ok(from_atom(feed));
+                    }
+                }
+                Err(match discover::alternate_feed_link(bytes) {
+                    Some(href) if looks_like_html(bytes) => {
+                        anyhow!("got a web page, not a feed; it links {href} as its feed")
+                    }
+                    None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"),
+                    _ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"),
+                })
+            }
+        },
+    }
+}
+
+/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed
+/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a
+/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case.
+pub(crate) fn looks_like_html(bytes: &[u8]) -> bool {
+    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase();
+    head.contains("<!doctype html") || head.contains("<html")
+}
+
+/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with
+/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried
+/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors.
+pub(crate) fn plain_text(bytes: &[u8]) -> Option<String> {
+    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]);
+    let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}');
+    if text.starts_with('<') {
+        return None;
+    }
+    let line = text.lines().next().unwrap_or("").trim_end();
+    if line.is_empty() {
+        return Some("an empty reply".into());
+    }
+    let mut said: String = line.chars().take(80).collect();
+    if said.len() < line.len() {
+        said.push('…');
+    }
+    Some(format!("\"{said}\""))
+}
+
+/// Escapes every `&` that does not already start a recognized XML entity
+/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&apos;`, or a numeric reference like `&#39;`).
+pub(crate) fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> {
+    fn is_entity_start(rest: &[u8]) -> bool {
+        for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] {
+            if rest.starts_with(named) {
+                return true;
+            }
+        }
+        let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") {
+            &rest[2..]
+        } else if rest.starts_with(b"#") {
+            &rest[1..]
+        } else {
+            return false;
+        };
+        let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count();
+        len > 0 && digits.get(len) == Some(&b';')
+    }
+
+    let mut out = Vec::with_capacity(bytes.len());
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) {
+            out.extend_from_slice(b"&amp;");
+        } else {
+            out.push(bytes[i]);
+        }
+        i += 1;
+    }
+    out
+}
+
+/// Every `<enclosure>` of every `<item>`, in document order.
+///
+/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
+/// says -- and when a feed carries several it keeps only the *last*, silently losing the
+/// rest. Feeds do ship several, so read them from the XML directly.
+pub(crate) fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
+    use quick_xml::events::Event;
+
+    let mut reader = quick_xml::Reader::from_reader(bytes);
+    reader.config_mut().trim_text(true);
+    let mut buf = Vec::new();
+    let mut out: Vec<Vec<Enclosure>> = Vec::new();
+    let mut current: Option<Vec<Enclosure>> = None;
+
+    let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> {
+        let (mut url, mut mime, mut length) = (String::new(), None, None);
+        for attr in e.attributes().flatten() {
+            // Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
+            let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default();
+            match attr.key.local_name().as_ref() {
+                "url" => url = val,
+                "type" => mime = Some(val).filter(|v| !v.is_empty()),
+                "length" => length = val.parse().ok(),
+                _ => {}
+            }
+        }
+        (!url.is_empty()).then_some(Enclosure { url, mime, length })
+    };
+
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(e)) => match e.name().local_name().as_ref() {
+                "item" => current = Some(Vec::new()),
+                "enclosure" => {
+                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
+                        list.push(enc);
+                    }
+                }
+                _ => {}
+            },
+            Ok(Event::Empty(e)) => match e.name().local_name().as_ref() {
+                // <item/> with no children still counts, so the indexes stay aligned.
+                "item" => out.push(Vec::new()),
+                "enclosure" => {
+                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
+                        list.push(enc);
+                    }
+                }
+                _ => {}
+            },
+            Ok(Event::End(e)) => {
+                if e.name().local_name().as_ref() == "item"
+                    && let Some(list) = current.take()
+                {
+                    out.push(list);
+                }
+            }
+            Ok(Event::Eof) | Err(_) => break,
+            _ => {}
+        }
+        buf.clear();
+    }
+    if let Some(list) = current.take() {
+        out.push(list);
+    }
+    out
+}
+
+pub(crate) fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
+    let per_item = enclosures_by_item(bytes);
+    let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
+
+    let entries = ch
+        .items()
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, item)| {
+            // Straight from the XML, so an item with several keeps all of them. Falls
+            // back to the parsed one if the scan and the parser disagree on item count.
+            let mut enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| {
+                item.enclosure()
+                    .into_iter()
+                    .map(|e| Enclosure {
+                        url: e.url().trim().to_owned(),
+                        mime: non_empty(Some(e.mime_type())),
+                        length: e.length().parse().ok(),
+                    })
+                    .filter(|e| !e.url.is_empty())
+                    .collect()
+            });
+            drop_player_repeats(&mut enclosures);
+
+            let guid = pick_guid(
+                item.guid().map(|g| g.value()),
+                item.link(),
+                enclosures.first().map(|e| e.url.as_str()),
+                item.title(),
+            )?;
+
+            let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
+            let it = item.itunes_ext();
+
+            Some(Entry {
+                guid,
+                title: title_text(item.title()),
+                link: non_empty(item.link()),
+                published: item.pub_date().and_then(parse_date),
+                // Content wins over description, as __getEntries preferred entry.content. The
+                // source namespace's Markdown copy only when there is neither (#284).
+                description: notes(item.content(), item.description(), source_markdown(item)),
+                markdown: non_empty(source_markdown(item))
+                    .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()),
+                categories: item
+                    .categories()
+                    .iter()
+                    .map(|c| c.name().to_owned())
+                    .filter(|c| !c.is_empty() && !c.starts_with("http"))
+                    .collect(),
+                explicit: explicit || entry_explicit,
+                image: item_image(item, &enclosures),
+                duration: it.and_then(|i| i.duration()).and_then(parse_duration),
+                episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
+                season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
+                enclosures,
+            })
+        })
+        .collect();
+
+    ParsedFeed {
+        title: title_text(Some(ch.title())),
+        ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
+        // itunes:image is the square artwork; <image><url> is the older, often smaller one.
+        image: ch
+            .itunes_ext()
+            .and_then(|i| i.image())
+            .map(str::to_owned)
+            .or_else(|| ch.image().map(|i| i.url().to_owned())),
+        site: non_empty(Some(ch.link().trim())),
+        // Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
+        // would fill the Directory with one-off tags. The subcategory where there is one: Apple
+        // files every tabletop and gaming show under Leisure, which says little; Games says it.
+        description: non_empty(Some(ch.description())).or_else(|| non_empty(ch.itunes_ext().and_then(|i| i.summary()))),
+        category: ch
+            .itunes_ext()
+            .and_then(|i| i.categories().first())
+            .map(|c| c.subcategory().filter(|s| !s.text().trim().is_empty()).unwrap_or(c))
+            .and_then(|c| non_empty(Some(c.text().trim()))),
+        entries,
+    }
+}
+
+pub(crate) fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
+    let entries = feed
+        .entries()
+        .iter()
+        .filter_map(|e| {
+            // Atom carries enclosures as <link rel="enclosure">.
+            let mut enclosures: Vec<Enclosure> = e
+                .links()
+                .iter()
+                .filter(|l| l.rel() == "enclosure")
+                .map(|l| Enclosure {
+                    url: l.href().trim().to_owned(),
+                    mime: non_empty(l.mime_type()),
+                    length: l.length().and_then(|s| s.parse().ok()),
+                })
+                .filter(|e| !e.url.is_empty())
+                .collect();
+            drop_player_repeats(&mut enclosures);
+
+            let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href());
+
+            let guid =
+                pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?;
+
+            Some(Entry {
+                guid,
+                title: title_text(Some(e.title().as_str())),
+                link: alt.map(str::to_owned),
+                published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
+                description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None),
+                markdown: None,
+                categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
+                explicit: false,
+                image: None,
+                duration: None,
+                episode: None,
+                season: None,
+                enclosures,
+            })
+        })
+        .collect();
+
+    ParsedFeed {
+        title: title_text(Some(feed.title().as_str())),
+        ttl_mins: None,
+        image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
+        site: feed
+            .links()
+            .iter()
+            .find(|l| l.rel() == "alternate")
+            .map(|l| l.href().trim().to_owned())
+            .filter(|h| !h.is_empty()),
+        category: None,
+        description: non_empty(feed.subtitle().map(|t| t.as_str())),
+        entries,
+    }
+}
+
+/// The original fell back to hashing the title or description. A guid, permalink or
+/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
+/// none of them has nothing to download and is dropped.
+pub(crate) fn pick_guid(
+    guid: Option<&str>,
+    link: Option<&str>,
+    enclosure: Option<&str>,
+    title: Option<&str>,
+) -> Option<String> {
+    [guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned)
+}
+
+pub(crate) fn is_yes(s: &str) -> bool {
+    matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
+}
+
+pub(crate) fn non_empty(s: Option<&str>) -> Option<String> {
+    s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
+}
+
+/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post
+/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion
+/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays.
+pub(crate) fn drop_player_repeats(encs: &mut Vec<Enclosure>) {
+    let mut seen = HashSet::new();
+    encs.retain(|e| seen.insert(same_file_key(&e.url)));
+}
+
+/// An enclosure URL without WordPress's player number, for telling repeats of one file apart
+/// from different files.
+#[must_use]
+pub fn same_file_key(url: &str) -> String {
+    let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() };
+    let kept: Vec<(String, String)> = u
+        .query_pairs()
+        .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit())))
+        .map(|(k, v)| (k.into_owned(), v.into_owned()))
+        .collect();
+    if kept.is_empty() {
+        u.set_query(None);
+    } else {
+        u.query_pairs_mut().clear().extend_pairs(kept);
+    }
+    u.to_string()
+}
+
+/// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through
+/// the XML parser with its HTML entities intact: The Verge's "Meta&#8217;s" reached the page as
+/// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is
+/// instead of failing the whole title.
+pub(crate) fn title_text(s: Option<&str>) -> Option<String> {
+    let s = non_empty(s)?;
+    let mut out = String::with_capacity(s.len());
+    let mut rest = s.as_str();
+    while let Some(at) = rest.find('&') {
+        out.push_str(&rest[..at]);
+        rest = &rest[at..];
+        let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1);
+        let decoded = rest[len..]
+            .starts_with(';')
+            .then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok())
+            .flatten();
+        match decoded {
+            Some(v) => {
+                out.push_str(&v);
+                rest = &rest[len + 1..];
+            }
+            None => {
+                out.push('&');
+                rest = &rest[1..];
+            }
+        }
+    }
+    out.push_str(rest);
+    non_empty(Some(&out))
+}
+
+/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in
+/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML.
+pub(crate) fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option<String> {
+    match body(content, description) {
+        Some(b) if is_markdown(&b) => Some(markdown_html(&b)),
+        Some(b) => Some(b),
+        None => non_empty(markdown).map(|m| markdown_html(&m)),
+    }
+}
+
+/// `<source:markdown>`, from https://source.scripting.com/: the item's text in Markdown.
+///
+/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding
+/// it to another prefix is not read, until one turns up.
+pub(crate) fn source_markdown(item: &rss::Item) -> Option<&str> {
+    item.extensions().get("source")?.get("markdown")?.first()?.value()
+}
+
+/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold.
+/// Plain text without any is left as it was, since rendering it would change nothing but
+/// could still read a line starting "1984." as a list.
+pub(crate) fn is_markdown(s: &str) -> bool {
+    let tag = s
+        .match_indices('<')
+        .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!'));
+    let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m));
+    !tag && (s.contains("](") || s.contains("**") || s.lines().any(block))
+}
+
+/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML.
+#[must_use]
+pub fn markdown_html(md: &str) -> String {
+    use pulldown_cmark::{Options, Parser, html};
+    let mut out = String::new();
+    html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH));
+    out
+}
+
+/// An item's show notes: its full body when that is whole, else its description.
+///
+/// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name
+/// pasted from a web app (`[&:has([data-writing-block])>*]:pointer-events-auto`), so the body
+/// began halfway through a tag and the page showed the rest of the tag as text. The same item's
+/// `description` was whole. With no description to fall back on, a damaged body beats none.
+pub(crate) fn body(content: Option<&str>, description: Option<&str>) -> Option<String> {
+    match non_empty(content).filter(|c| !starts_mid_tag(c)) {
+        Some(c) => Some(match subtitle(&c, description) {
+            Some(s) => format!("<p><em>{}</em></p>{c}", quick_xml::escape::escape(s.as_str())),
+            None => c,
+        }),
+        None => non_empty(description).or_else(|| non_empty(content)),
+    }
+}
+
+/// A description that is a subtitle rather than a second copy of the notes: Substack puts the
+/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it.
+/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a
+/// description found in the body is not shown twice.
+///
+/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its
+/// notes passes too and shows above them, which reads fine; a real subtitle field would need an
+/// `entries` column.
+pub(crate) fn subtitle(body: &str, description: Option<&str>) -> Option<String> {
+    let d = title_text(description)?;
+    if d.contains('<') || d.chars().count() > 300 {
+        return None;
+    }
+    // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut
+    // description ends in "…" or "[...]".
+    let words = |s: &str| {
+        s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::<Vec<_>>().join(" ").to_lowercase()
+    };
+    let want = words(&d);
+    let text = title_text(Some(&text_of(body))).unwrap_or_default();
+    (!want.is_empty() && !words(&text).contains(&want)).then_some(d)
+}
+
+/// HTML with its tags taken out, each replaced by a space so words either side stay apart.
+pub(crate) fn text_of(html: &str) -> String {
+    let mut out = String::with_capacity(html.len());
+    let mut in_tag = false;
+    for c in html.chars() {
+        match c {
+            '<' => in_tag = true,
+            '>' if in_tag => {
+                in_tag = false;
+                out.push(' ');
+            }
+            _ if !in_tag => out.push(c),
+            _ => {}
+        }
+    }
+    out
+}
+
+/// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose
+/// start was cut off.
+pub(crate) fn starts_mid_tag(html: &str) -> bool {
+    html[..html.find('<').unwrap_or(html.len())].contains("\">")
+}
+
+/// The picture to show beside an item, in order of how deliberate it is:
+/// `itunes:image`, then Media RSS `media:thumbnail`, then a `media:content` that is an
+/// image, and finally an image enclosure -- which is how a blog's article picture arrives
+/// (Substack puts it there), so those entries get artwork rather than a blank square.
+pub(crate) fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> {
+    if let Some(url) = item.itunes_ext().and_then(|i| i.image()) {
+        return non_empty(Some(url));
+    }
+
+    let media = item.extensions().get("media");
+    let attr = |name: &str, want_image: bool| -> Option<String> {
+        media?.get(name)?.iter().find_map(|e| {
+            if want_image {
+                // media:content carries anything; only take it when it says it is a picture.
+                let is_image = e.attrs.get("type").is_some_and(|t| t.starts_with("image/"))
+                    || e.attrs.get("medium").is_some_and(|m| m == "image");
+                if !is_image {
+                    return None;
+                }
+            }
+            non_empty(e.attrs.get("url").map(String::as_str))
+        })
+    };
+    attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| {
+        enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone())
+    })
+}
+
+/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
+pub(crate) fn parse_duration(s: &str) -> Option<i64> {
+    let s = s.trim();
+    if s.is_empty() {
+        return None;
+    }
+    if !s.contains(':') {
+        return s.parse().ok().filter(|n| *n > 0);
+    }
+    let mut total: i64 = 0;
+    for part in s.split(':') {
+        total = total * 60 + part.trim().parse::<i64>().ok()?;
+    }
+    Some(total).filter(|n| *n > 0)
+}
+
+/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
+pub(crate) fn parse_date(s: &str) -> Option<i64> {
+    let s = s.trim();
+    DateTime::parse_from_rfc2822(s).or_else(|_| DateTime::parse_from_rfc3339(s)).ok().map(|d| d.timestamp())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::failure;
+
+    #[test]
+    fn parses_rss_with_itunes_extensions() {
+        let bytes = include_bytes!("../../../tests/data/rss2.xml");
+        let feed = parse(bytes).unwrap();
+
+        assert_eq!(feed.title.as_deref(), Some("Test Cast"));
+        assert_eq!(feed.ttl_mins, Some(45));
+        assert_eq!(feed.category.as_deref(), Some("Podcasting"), "the first, by its subcategory");
+        assert_eq!(feed.description.as_deref(), Some("A synthetic feed used by the parser tests."));
+        assert_eq!(feed.entries.len(), 3);
+
+        let ep = &feed.entries[0];
+        assert_eq!(ep.guid, "https://example.com/ep/1");
+        assert_eq!(ep.title.as_deref(), Some("Episode One"));
+        assert_eq!(ep.published, Some(1_078_016_400));
+        assert_eq!(ep.categories, vec!["Tech"]);
+        assert!(!ep.explicit);
+        assert_eq!(
+            ep.enclosures,
+            vec![Enclosure {
+                url: "https://example.com/ep1.mp3".into(),
+                mime: Some("audio/mpeg".into()),
+                length: Some(12_345_678),
+            }]
+        );
+
+        assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
+        assert_eq!(feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent"));
+    }
+
+    #[test]
+    fn a_feed_says_what_it_is_in_itunes_summary_or_atom_subtitle() {
+        // An empty <description> gives way to iTunes' summary (#130).
+        let rss = br#"<?xml version="1.0"?><rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>R</title><description> </description><itunes:summary>Said here.</itunes:summary></channel></rss>"#;
+        assert_eq!(parse(rss).unwrap().description.as_deref(), Some("Said here."));
+        let atom = br#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>A</title>
+        <subtitle>Or here.</subtitle><id>urn:a</id><updated>2026-01-01T00:00:00Z</updated></feed>"#;
+        assert_eq!(parse(atom).unwrap().description.as_deref(), Some("Or here."));
+    }
+
+    #[test]
+    fn markdown_notes_are_made_html_and_html_is_left_alone() {
+        let feed = |items: &str| {
+            let xml = format!(
+                r#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>{items}</channel></rss>"#
+            );
+            parse(xml.as_bytes()).unwrap().entries.into_iter().map(|e| e.description).collect::<Vec<_>>()
+        };
+        let got = feed(
+            r#"<item><guid>a</guid><description># Hi
+
+Some **bold** and [a link](https://x.example/).</description></item>
+            <item><guid>b</guid><source:markdown>Only *Markdown* here.</source:markdown></item>
+            <item><guid>c</guid><description>&lt;p&gt;HTML **as sent**&lt;/p&gt;</description><source:markdown>ignored</source:markdown></item>
+            <item><guid>d</guid><description>1984. Plain text, left as it is.</description></item>"#,
+        );
+        assert_eq!(
+            got[0].as_deref(),
+            Some("<h1>Hi</h1>\n<p>Some <strong>bold</strong> and <a href=\"https://x.example/\">a link</a>.</p>\n")
+        );
+        assert_eq!(got[1].as_deref(), Some("<p>Only <em>Markdown</em> here.</p>\n"), "the Markdown copy, alone");
+        assert_eq!(got[2].as_deref(), Some("<p>HTML **as sent**</p>"), "HTML wins, untouched");
+        assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is."));
+    }
+
+    #[test]
+    fn markdown_sent_beside_html_is_kept_for_the_page_to_switch_to() {
+        let xml = br#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>
+        <item><guid>both</guid><description>&lt;p&gt;Hi&lt;/p&gt;</description><source:markdown>*Hi*</source:markdown></item>
+        <item><guid>alone</guid><source:markdown>*Hi*</source:markdown></item></channel></rss>"#;
+        let got: Vec<_> = parse(xml).unwrap().entries.into_iter().map(|e| e.markdown).collect();
+        assert_eq!(got, [Some("*Hi*".to_owned()), None], "only beside HTML; alone, it is the description (#284)");
+    }
+
+    #[test]
+    fn a_file_wordpress_lists_twice_is_one_enclosure() {
+        let xml = br#"<?xml version="1.0"?><rss version="2.0"><channel><title>R</title>
+        <item><title>The Promotion Paradox</title><guid>p</guid>
+        <enclosure url="https://x/ep.mp3" length="1" type="audio/mpeg"/>
+        <enclosure url="https://x/ep.mp3?_=2" length="1" type="audio/mpeg"/>
+        <enclosure url="https://x/other.mp3?_=3&amp;key=k" length="1" type="audio/mpeg"/>
+        </item></channel></rss>"#;
+        let urls: Vec<String> = parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
+        assert_eq!(
+            urls,
+            ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"],
+            "the repeat goes, a different file stays"
+        );
+        assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k"));
+        assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number");
+    }
+
+    #[test]
+    fn titles_are_read_as_text_not_html() {
+        // The Verge: an Atom title of type="html", its entity inside CDATA.
+        let xml = br#"<?xml version="1.0"?>
+        <feed xmlns="http://www.w3.org/2005/Atom"><title type="text">V</title><id>v</id>
+        <updated>2026-09-15T00:00:00Z</updated>
+        <entry><title type="html"><![CDATA[Meta&#8217;s new One]]></title><id>e1</id>
+        <updated>2026-09-15T00:00:00Z</updated></entry></feed>"#;
+        assert_eq!(parse(xml).unwrap().entries[0].title.as_deref(), Some("Meta\u{2019}s new One"));
+        // HTML names as well as numbers; a bare `&` and an unknown name are left as they are.
+        assert_eq!(
+            title_text(Some("Pe&ntilde;a &amp; &#x201C;Q&A&#8221; &bogus; AT&T;")).as_deref(),
+            Some("Pe\u{f1}a & \u{201c}Q&A\u{201d} &bogus; AT&T;")
+        );
+    }
+
+    #[test]
+    fn a_body_cut_off_mid_tag_gives_way_to_the_description() {
+        // How libsyn served Daily Meditation Podcast #3477: content:encoded began inside a tag.
+        let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant"> <p>What if</p>"#;
+        let whole = r#"<div class="[&:has([data-writing-block])>*]:pointer-events-auto"><p>What if</p></div>"#;
+        assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole));
+        assert_eq!(
+            body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(),
+            Some("<p>Notes</p>"),
+            "a whole body wins"
+        );
+        assert_eq!(
+            body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(),
+            Some("Plain notes, no tags.")
+        );
+        assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none");
+        assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary"));
+    }
+
+    #[test]
+    fn a_subtitle_missing_from_the_body_is_kept_above_it() {
+        // Substack: the subtitle is the description, and content:encoded does not repeat it.
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some("Why the <b> tag & I fell out")).as_deref(),
+            Some("<p>The post.</p>"),
+            "a description with markup in it is notes, not a subtitle",
+        );
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some("Why Q&amp;A threads go wrong")).as_deref(),
+            Some("<p><em>Why Q&amp;A threads go wrong</em></p><p>The post.</p>"),
+        );
+        // A podcast repeating its notes, whole, cut short, or differently spaced: shown once.
+        let notes = "<p>This week we talk about <a href=\"x\">tape</a>, drums and a very long list.</p>";
+        for d in [
+            "This week we talk about tape, drums and a very long list.",
+            "This week we talk about tape, drums…",
+            "This week we  talk about\ntape [...]",
+        ] {
+            assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body");
+        }
+        let long = "word ".repeat(80);
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some(&long)).as_deref(),
+            Some("<p>The post.</p>"),
+            "a long description is notes, not a subtitle"
+        );
+    }
+
+    #[test]
+    fn feed_level_explicit_overrides_entries() {
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>X</title><link>https://x</link><description>d</description>
+        <itunes:explicit>yes</itunes:explicit>
+        <item><title>a</title><guid>g1</guid>
+        <enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+        assert!(feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit");
+    }
+
+    #[test]
+    fn failures_are_named_by_kind_for_the_log() {
+        let k = |m: &str| failure::failure_kind(m);
+        assert_eq!(k("HTTP 404 Not Found"), ("404".into(), Some(404)));
+        assert_eq!(k("HTTP 503 Service Unavailable"), ("503".into(), Some(503)));
+        assert_eq!(
+            k("connecting: error following redirect for url (https://www.toddstashwick.com/): too many redirects").0,
+            "redirect_loop"
+        );
+        assert_eq!(k("connecting: dns error: failed to lookup address information").0, "dns");
+        assert_eq!(k("operation timed out").0, "timeout");
+        assert_eq!(k("something new").0, "other");
+    }
+
+    #[test]
+    fn a_web_page_says_so_and_names_the_feed_it_links() {
+        let html = br#"<!doctype html><html><head>
+            <link rel="alternate" type="application/rss+xml" href="https://x.example/feed">
+            </head><body>not a feed</body></html>"#;
+        let err = parse(html).unwrap_err().to_string();
+        assert_eq!(err, "got a web page, not a feed; it links https://x.example/feed as its feed");
+    }
+
+    #[test]
+    fn a_web_page_with_no_feed_link_still_says_so() {
+        let html = b"<!doctype html><html><body>moved</body></html>";
+        assert_eq!(parse(html).unwrap_err().to_string(), "got a web page, not a feed");
+    }
+
+    #[test]
+    fn malformed_xml_gets_the_original_parser_errors() {
+        let err = parse(b"<rss><channel><title>cut off").unwrap_err().to_string();
+        assert!(err.starts_with("not RSS ("), "{err}");
+    }
+
+    #[test]
+    fn a_body_with_no_markup_says_what_the_site_sent() {
+        let err = parse(b"\xef\xbb\xbf\r\n  Unable to establish a DB connection\nmore").unwrap_err().to_string();
+        assert_eq!(err, "the site sent \"Unable to establish a DB connection\" instead of a feed");
+        let long = parse("x".repeat(200).as_bytes()).unwrap_err().to_string();
+        assert_eq!(long, format!("the site sent \"{}…\" instead of a feed", "x".repeat(80)));
+        assert_eq!(parse(b" \n").unwrap_err().to_string(), "the site sent an empty reply instead of a feed");
+    }
+
+    #[test]
+    fn parses_atom_enclosure_links() {
+        let bytes = include_bytes!("../../../tests/data/atom.xml");
+        let feed = parse(bytes).unwrap();
+
+        assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
+        assert_eq!(feed.entries.len(), 1);
+        let ep = &feed.entries[0];
+        assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
+        assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
+        assert_eq!(ep.published, Some(1_078_016_400));
+        assert_eq!(
+            ep.enclosures,
+            vec![Enclosure {
+                url: "https://example.org/ep1.m4a".into(),
+                mime: Some("audio/mp4".into()),
+                length: Some(9_876_543),
+            }],
+            "rel=enclosure only; the alternate link must not become an enclosure"
+        );
+    }
+
+    #[test]
+    fn a_bare_ampersand_in_a_link_is_repaired_and_parsed() {
+        // kcpw.org: <link>https://kcpw.org/?post_type=post&p=125715</link> -- a bare "&"
+        // that strict XML rejects but browsers accept.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0"><channel><title>X</title><link>https://x</link><description>d</description>
+        <item><title>a</title><guid>g1</guid>
+        <link>https://kcpw.org/?post_type=post&p=125715</link>
+        <enclosure url="https://x/a.mp3?a=1&b=2" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+        assert_eq!(feed.entries[0].link.as_deref(), Some("https://kcpw.org/?post_type=post&p=125715"));
+        assert_eq!(feed.entries[0].enclosures[0].url, "https://x/a.mp3?a=1&b=2");
+    }
+
+    #[test]
+    fn escape_bare_ampersands_leaves_real_entities_alone() {
+        let out = escape_bare_ampersands(b"a&amp;b &lt;x&gt; &#39; &#x2F; c&d");
+        assert_eq!(out, b"a&amp;b &lt;x&gt; &#39; &#x2F; c&amp;d");
+    }
+
+    #[test]
+    fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
+        // Some feeds set a different itunes:title. The displayed title is always the RSS
+        // <title>, verbatim -- separators and all -- and season/episode are stored
+        // alongside it rather than folded into it.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>Show</title><link>https://x</link><description>d</description>
+        <item>
+          <title>Music from a Darkened Room | Session Zero</title>
+          <itunes:title>Session Zero</itunes:title>
+          <guid>sz</guid>
+          <itunes:season>8</itunes:season>
+          <itunes:duration>6720</itunes:duration>
+          <enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
+        </item>
+        <item>
+          <title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
+          <guid>p1</guid>
+          <itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
+          <enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
+        </item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+
+        let sz = &feed.entries[0];
+        assert_eq!(
+            sz.title.as_deref(),
+            Some("Music from a Darkened Room | Session Zero"),
+            "itunes:title must not override the RSS title"
+        );
+        assert_eq!(sz.season, Some(8));
+        assert_eq!(sz.episode, None, "a missing episode number stays missing");
+        assert_eq!(sz.duration, Some(6720));
+
+        let p1 = &feed.entries[1];
+        assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
+        assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
+    }
+
+    #[test]
+    fn an_item_may_carry_several_enclosures() {
+        // The rss crate keeps only one per item -- the last -- so these come from the XML.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0"><channel><title>M</title><link>https://x</link><description>d</description>
+        <item><title>Two files</title><guid>m1</guid>
+          <enclosure url="https://x/a.mp3?v=1&amp;t=2" length="111" type="audio/mpeg"/>
+          <enclosure url="https://x/b.mp4" length="222" type="video/mp4"/>
+        </item>
+        <item><title>One file</title><guid>m2</guid>
+          <enclosure url="https://x/c.mp3" length="333" type="audio/mpeg"/></item>
+        <item><title>None</title><guid>m3</guid></item>
+        </channel></rss>"#;
+        let f = parse(xml).unwrap();
+        assert_eq!(f.entries.len(), 3);
+
+        let two = &f.entries[0].enclosures;
+        assert_eq!(two.len(), 2, "both enclosures survive");
+        assert_eq!(two[0].url, "https://x/a.mp3?v=1&t=2", "document order, and the escaped ampersand is decoded");
+        assert_eq!(two[0].length, Some(111));
+        assert_eq!(two[1].url, "https://x/b.mp4");
+        assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));
+
+        assert_eq!(f.entries[1].enclosures.len(), 1);
+        assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
+    }
+
+    #[test]
+    fn an_items_picture_comes_from_the_most_deliberate_source() {
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
+             xmlns:media="http://search.yahoo.com/mrss/">
+        <channel><title>P</title><link>https://x</link><description>d</description>
+        <item><title>Has itunes</title><guid>a</guid>
+          <itunes:image href="https://x/itunes.jpg"/>
+          <media:thumbnail url="https://x/thumb.jpg"/>
+          <enclosure url="https://x/a.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Has thumbnail</title><guid>b</guid>
+          <media:thumbnail url="https://x/thumb.jpg"/>
+          <enclosure url="https://x/b.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Has media content</title><guid>c</guid>
+          <media:content url="https://x/pic.jpg" type="image/jpeg"/>
+          <media:content url="https://x/clip.mp4" type="video/mp4"/></item>
+        <item><title>Only an image enclosure</title><guid>d</guid>
+          <enclosure url="https://x/d.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Audio only</title><guid>e</guid>
+          <enclosure url="https://x/e.mp3" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let f = parse(xml).unwrap();
+        let img = |i: usize| f.entries[i].image.as_deref();
+        assert_eq!(img(0), Some("https://x/itunes.jpg"), "itunes:image wins");
+        assert_eq!(img(1), Some("https://x/thumb.jpg"), "then media:thumbnail");
+        assert_eq!(img(2), Some("https://x/pic.jpg"), "media:content, and only the image one");
+        assert_eq!(img(3), Some("https://x/d.jpg"), "a blog's article picture arrives as an enclosure");
+        assert_eq!(img(4), None, "audio is not a picture");
+    }
+
+    #[test]
+    fn durations_parse_from_seconds_or_a_clock() {
+        assert_eq!(parse_duration("5649"), Some(5649));
+        assert_eq!(parse_duration("23:45"), Some(1425));
+        assert_eq!(parse_duration("1:34:09"), Some(5649));
+        assert_eq!(parse_duration("0"), None, "zero is not a duration");
+        assert_eq!(parse_duration(""), None);
+        assert_eq!(parse_duration("garbage"), None);
+    }
+
+    #[test]
+    fn rejects_html_masquerading_as_a_feed() {
+        assert!(parse(b"<html><body>nope</body></html>").is_err());
+    }
+
+    #[test]
+    fn guid_falls_back_through_link_then_enclosure() {
+        assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
+        assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
+        assert_eq!(pick_guid(None, None, None, None), None);
+    }
+}

crates/feed/src/patreon.rs

@@ -0,0 +1,147 @@
+//! Patreon: a creator's feed, its shows, and when two Patreon addresses are the same feed.
+
+use crate::fetch;
+use anyhow::{Context, Result, anyhow};
+
+/// The token and show of a Patreon feed link, or None for any other URL.
+///
+/// Patreon gives each patron one token per creator. With no show it stands for the creator,
+/// whose feed carries every show at once.
+pub(crate) fn patreon_parts(url: &str) -> Option<(String, Option<String>)> {
+    let u = url::Url::parse(url).ok()?;
+    if !matches!(u.host_str()?, "patreon.com" | "www.patreon.com") || !u.path().starts_with("/rss") {
+        return None;
+    }
+    let param = |name: &str| u.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v.into_owned());
+    Some((param("auth")?, param("show")))
+}
+
+/// A Patreon link naming a creator but no show.
+#[must_use]
+pub fn is_patreon_creator(url: &str) -> bool {
+    matches!(patreon_parts(url), Some((_, None)))
+}
+
+/// Whether two URLs are the same feed. One Patreon show has several spellings -- by the
+/// creator's name, by number, or with no creator at all -- and the token and show are what
+/// identify it.
+#[must_use]
+pub fn same_feed(a: &str, b: &str) -> bool {
+    // Parsing both is the slow part, and only Patreon's addresses can differ and still match.
+    a == b
+        || (a.contains("patreon.com")
+            && b.contains("patreon.com")
+            && patreon_parts(a).is_some_and(|p| Some(p) == patreon_parts(b)))
+}
+
+/// A Patreon creator's name and shows, each show as (title, feed URL).
+///
+/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes,
+/// finding shows stops and the show feeds already found keep working. The documented API
+/// needs an OAuth client per install and does not list shows.
+pub async fn patreon_shows(client: &reqwest::Client, url: &str) -> Result<(Option<String>, Vec<(String, String)>)> {
+    // The creator feed names its campaign by number in its self link, a few hundred bytes in.
+    // The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up.
+    let mut resp = client.get(url).timeout(fetch::FEED_TIMEOUT).send().await.context("connecting")?;
+    if !resp.status().is_success() {
+        return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status()));
+    }
+    let mut head = Vec::new();
+    while patreon_campaign(&head).is_none() && head.len() < 64 * 1024 {
+        let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break };
+        head.extend_from_slice(&chunk);
+    }
+    let campaign = patreon_campaign(&head).ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;
+
+    let api = format!(
+        "https://www.patreon.com/api/campaigns/{campaign}\
+         ?include=shows&fields%5Bcampaign%5D=name&fields%5Bcollection%5D=title"
+    );
+    let resp = client.get(api).send().await.context("asking Patreon for the shows")?;
+    if !resp.status().is_success() {
+        return Err(anyhow!("Patreon would not list the shows: HTTP {}", resp.status()));
+    }
+    let (name, shows) = parse_patreon_shows(&resp.bytes().await.context("reading the shows")?)?;
+    Ok((name, shows.into_iter().map(|(id, title)| (title, format!("{url}&show={id}"))).collect()))
+}
+
+/// The campaign number in the start of a Patreon feed.
+pub(crate) fn patreon_campaign(head: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(head);
+    text.match_indices("patreon.com/rss/").find_map(|(i, m)| {
+        let id: String = text[i + m.len()..].chars().take_while(char::is_ascii_digit).collect();
+        (!id.is_empty()).then_some(id)
+    })
+}
+
+/// A campaign's name, and its shows as (id, title).
+pub(crate) type PatreonShows = (Option<String>, Vec<(String, String)>);
+
+/// A campaign's name and its shows as (id, title), from Patreon's JSON:API answer.
+pub(crate) fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> {
+    let v: serde_json::Value = serde_json::from_slice(json).context("Patreon's answer is not JSON")?;
+    // Missing is not the same as none. Read as no shows, the creator feed would be scanned as
+    // a plain feed, claim every show's files, and leave the shows empty once the list returned.
+    let ids = v["data"]["relationships"]["shows"]["data"]
+        .as_array()
+        .ok_or_else(|| anyhow!("Patreon's answer does not list the shows"))?;
+    let title = |id: &str| -> Option<String> {
+        let show = v["included"].as_array()?.iter().find(|x| x["type"] == "collection" && x["id"] == id)?;
+        show["attributes"]["title"].as_str().map(|t| t.trim().to_owned())
+    };
+    let shows = ids
+        .iter()
+        .filter_map(|s| s["id"].as_str())
+        .map(|id| (id.to_owned(), title(id).unwrap_or_else(|| format!("Show {id}"))))
+        .collect();
+    Ok((v["data"]["attributes"]["name"].as_str().map(str::to_owned), shows))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::discover;
+
+    #[test]
+    fn a_patreon_creator_is_a_list_of_its_shows() {
+        let tok = "AbCdEfGhIjKlMnOpQrStUvWxYz012_-9";
+        assert_eq!(discover::expand_input(&format!(" {tok} ")), format!("https://www.patreon.com/rss?auth={tok}"));
+        assert_eq!(discover::expand_input("https://example.com/rss"), "https://example.com/rss");
+        assert_eq!(discover::expand_input("http://example.com/rss"), "http://example.com/rss");
+        assert_eq!(discover::expand_input(" cnn.com "), "https://cnn.com");
+        assert_eq!(discover::expand_input("example.com/feed.xml"), "https://example.com/feed.xml");
+        assert_eq!(discover::expand_input("//example.com/rss"), "https://example.com/rss");
+
+        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss/glasscannon?auth={tok}")));
+        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss?auth={tok}")));
+        assert!(!is_patreon_creator(&format!("https://www.patreon.com/rss/x?auth={tok}&show=1")), "one show is a feed");
+        assert!(!is_patreon_creator(&format!("https://example.com/rss?auth={tok}")));
+
+        // The show you already have by name is the one a bare token would add by number.
+        assert!(same_feed(
+            &format!("https://www.patreon.com/rss/glasscannon?auth={tok}&show=2073588"),
+            &format!("https://www.patreon.com/rss?auth={tok}&show=2073588"),
+        ));
+        assert!(!same_feed(
+            &format!("https://www.patreon.com/rss?auth={tok}&show=1"),
+            &format!("https://www.patreon.com/rss?auth={tok}&show=2"),
+        ));
+
+        // The self link carries the campaign by number, whichever spelling was asked for.
+        let head = br#"<rss><channel><link>https://www.patreon.com/glasscannon</link>
+            <atom:link href="https://www.patreon.com/rss/369921?auth=t" rel="self"/>"#;
+        assert_eq!(patreon_campaign(head).as_deref(), Some("369921"));
+        assert_eq!(patreon_campaign(b"<rss><channel><title>T"), None);
+
+        let json = br#"{"data":{"id":"369921","type":"campaign","attributes":{"name":"The Glass Cannon Network"},
+            "relationships":{"shows":{"data":[{"id":"2073588","type":"collection"},{"id":"2073636","type":"collection"}]}}},
+            "included":[{"id":"2073588","type":"collection","attributes":{"title":"Get in the Trunk "}},
+                        {"id":"2073636","type":"collection","attributes":{"title":"Shadowdark"}}]}"#;
+        let (name, shows) = parse_patreon_shows(json).unwrap();
+        assert_eq!(name.as_deref(), Some("The Glass Cannon Network"));
+        assert_eq!(shows, [("2073588".into(), "Get in the Trunk".into()), ("2073636".into(), "Shadowdark".into())]);
+
+        // An answer that stops naming the shows is an error, never "this creator has none".
+        assert!(parse_patreon_shows(br#"{"data":{"attributes":{"name":"X"}}}"#).is_err());
+    }
+}

crates/feed/src/privacy.rs

@@ -0,0 +1,105 @@
+//! Addresses that carry someone's access: taking it out of what is logged, and keeping such feeds private.
+
+/// Conditional GET, following redirects itself: `client` must follow none (`feed_client`), so
+/// each hop is seen. The second value is where the feed now is when every hop said so for good
+/// (301 or 308): a publisher that moved its feed, which the catalogue should follow rather
+/// than be redirected on every read. A temporary redirect (302, 307) moves nothing.
+/// `system-proxy` reads http_proxy/https_proxy.
+/// `auth` is a username and password, sent only to the feed's own host.
+/// An address as a trace may keep it: without its query or user and password, where a private
+/// feed carries its credential (Patreon's `?auth=`), which Tempo kept for anyone reading it (#225).
+/// shortcut: a token in the path (`/feed/<token>.xml`) stays; strip paths too if a feed does that.
+#[must_use]
+pub fn redact(url: &str) -> String {
+    match url::Url::parse(url) {
+        Ok(mut u) => {
+            u.set_query(None);
+            u.set_fragment(None);
+            let _ = u.set_username("");
+            let _ = u.set_password(None);
+            u.to_string()
+        }
+        // Not an address ipx could fetch; keep what comes before anything that could be a query.
+        Err(_) => url.split(['?', '#', '@']).next().unwrap_or_default().to_owned(),
+    }
+}
+
+/// Text with each address in it redacted, for what is logged and sent to every open page: an
+/// error from reqwest names the whole address it fetched, a private feed's key with it (#238).
+#[must_use]
+pub fn redact_text(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut rest = text;
+    while let Some(i) = rest.find("http://").into_iter().chain(rest.find("https://")).min() {
+        out.push_str(&rest[..i]);
+        let end = rest[i..].find(|c: char| c.is_whitespace() || "()<>\"'".contains(c)).map_or(rest.len(), |n| i + n);
+        out.push_str(&redact(&rest[i..end]));
+        rest = &rest[end..];
+    }
+    out.push_str(rest);
+    out
+}
+
+/// An address that carries a subscriber's own access: a paid-feed host, credentials in the URL,
+/// or a key in the query. ipx leaves such a feed out of anything others see, and the directory
+/// refuses one sent to it (#202).
+///
+/// ponytail: a heuristic. A key hidden in the path of a host not in `PAID_HOSTS` gets
+/// through; a per-feed private flag is the upgrade (#205).
+#[must_use]
+pub fn url_looks_private(feed_url: &str) -> bool {
+    let Ok(u) = url::Url::parse(feed_url) else { return true };
+    // Paid-feed services put the subscriber's key in the path as often as in the query, and a
+    // long path segment alone proves nothing (acast's public show ids look the same). So a
+    // feed from one of them is private whatever its URL looks like. Supercast is why this
+    // exists: `feeds.supercast.com/feeds/<key>` reached the directory before it did.
+    let host = u.host_str().unwrap_or("");
+    PAID_HOSTS.iter().any(|h| host == *h || host.ends_with(&format!(".{h}")))
+        || !u.username().is_empty()
+        || u.password().is_some()
+        || u.query_pairs().any(|(k, _)| {
+            let k = k.to_ascii_lowercase();
+            ["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"].iter().any(|w| k.contains(w))
+        })
+}
+
+/// Services whose feeds are always one subscriber's own.
+pub(crate) const PAID_HOSTS: &[&str] =
+    &["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"];
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn redact_drops_what_carries_a_credential() {
+        assert_eq!(super::redact("https://www.patreon.com/rss/x?auth=SECRET&show=1"), "https://www.patreon.com/rss/x");
+        assert_eq!(super::redact("https://me:pw@example.com/feed.xml#top"), "https://example.com/feed.xml");
+        assert_eq!(super::redact("https://example.com/feed.xml"), "https://example.com/feed.xml");
+        assert_eq!(super::redact("not a url?token=SECRET"), "not a url");
+        // As reqwest words a failure, and with two addresses and none.
+        assert_eq!(
+            super::redact_text(
+                "connecting: error sending request for url (https://www.patreon.com/rss/x?auth=SECRET&show=1): operation timed out"
+            ),
+            "connecting: error sending request for url (https://www.patreon.com/rss/x): operation timed out"
+        );
+        assert_eq!(super::redact_text("from http://a/f?k=1 to https://u:p@b/g"), "from http://a/f to https://b/g");
+        assert_eq!(super::redact_text("HTTP 404 Not Found"), "HTTP 404 Not Found");
+    }
+
+    #[test]
+    fn an_address_carrying_someones_access_looks_private() {
+        let private = super::url_looks_private;
+        assert!(!private("https://feeds.twit.tv/twit.xml"));
+        // Patreon's shape: the key is a query parameter.
+        assert!(private("https://www.patreon.com/rss/x?auth=abc123&show=2073588"));
+        assert!(private("https://example.com/rss?api_key=abc"));
+        // Paid-feed services put the key in the path; the host gives them away.
+        assert!(private("https://feeds.supercast.com/feeds/abcdefghijklmnopqrstuvwx"));
+        assert!(private("https://someshow.supportingcast.fm/content/abc123.rss"));
+        // A long path segment alone is not a key: acast's public show ids look the same.
+        assert!(!private("https://feeds.acast.com/public/shows/0123456789abcdef01234567"));
+        assert!(private("https://ray:hunter2@example.com/rss"));
+        assert!(private("not a url"), "unparseable is not safe to list");
+    }
+}

crates/feed/src/wordpress.rs

@@ -0,0 +1,86 @@
+//! WordPress.com sites that answer their feed with a browser check, read from WordPress.com's API instead (#317).
+
+use crate::fetch::Fetched;
+use crate::parse::{Entry, ParsedFeed};
+use crate::{fetch, parse};
+use anyhow::{Context, Result};
+use chrono::NaiveDateTime;
+
+/// WordPress.com's browser check, in place of the feed it was asked for.
+pub(crate) fn is_wpcom_challenge(body: &[u8]) -> bool {
+    String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/")
+}
+
+/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed.
+/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed
+/// set to carry more gets ten here.
+pub(crate) async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result<Fetched> {
+    let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts");
+    let resp = client
+        .get(&api)
+        .timeout(fetch::FEED_TIMEOUT)
+        .send()
+        .await
+        .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?;
+    anyhow::ensure!(
+        resp.status().is_success(),
+        "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}",
+        resp.status()
+    );
+    let bytes = fetch::read_capped(resp).await.context("reading WordPress.com's API")?;
+    fetch::FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    fetch::FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
+    Ok(Fetched::Body { bytes, etag: None, last_modified: None })
+}
+
+/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's
+/// own feed gives each post, so they are the items already stored. No title for the feed: the
+/// stored one stays.
+pub(crate) fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed {
+    let entries = posts
+        .iter()
+        .filter_map(|p| {
+            Some(Entry {
+                guid: parse::non_empty(p["guid"]["rendered"].as_str())
+                    .or_else(|| parse::non_empty(p["link"].as_str()))?,
+                title: parse::title_text(p["title"]["rendered"].as_str()),
+                link: parse::non_empty(p["link"].as_str()),
+                published: p["date_gmt"]
+                    .as_str()
+                    .and_then(|d| NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok())
+                    .map(|d| d.and_utc().timestamp()),
+                description: parse::non_empty(p["content"]["rendered"].as_str()),
+                image: parse::non_empty(p["jetpack_featured_media_url"].as_str()),
+                ..Default::default()
+            })
+        })
+        .collect();
+    ParsedFeed { entries, ..Default::default() }
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn reads_wordpress_posts_as_a_feed() {
+        // As WordPress.com's API sends them, trimmed (#317).
+        let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00",
+        "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"},
+        "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/",
+        "title":{"rendered":"Baby&#8217;s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"},
+        "jetpack_featured_media_url":""}]"#;
+        let feed = super::parse(json).expect("WordPress posts parse as a feed");
+        assert_eq!(feed.title, None);
+        let e = &feed.entries[0];
+        assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260");
+        assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky"));
+        assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/"));
+        assert_eq!(e.published, Some(1_791_552_840));
+        assert_eq!(e.description.as_deref(), Some("<p>Notes</p>"));
+        assert_eq!(e.image, None);
+        assert!(super::is_wpcom_challenge(
+            b"<!DOCTYPE html><title>Checking your browser...</title><script src=\"/__challenge/probes.js\">"
+        ));
+        assert!(!super::is_wpcom_challenge(b"<html>Forbidden</html>"));
+    }
+}