//! Feed fetching and parsing, shared by ipx and the directory (#202). Replaces
//! FeedData.__getFeed / __getEntries.

use anyhow::{Context, Result, anyhow};
use reqwest::StatusCode;
use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};

#[derive(Debug, Default)]
pub struct ParsedFeed {
    pub title: Option<String>,
    pub ttl_mins: Option<u64>,
    pub image: Option<String>,
    /// The site the feed belongs to, where a favicon can stand in for missing artwork.
    pub site: Option<String>,
    /// The channel's first `<itunes:category>`, for the Directory's categories.
    pub category: Option<String>,
    /// What the feed says it is, for its page in the Directory (#130): RSS's `<description>`,
    /// or iTunes' summary where that is empty, or Atom's subtitle. HTML as often as not.
    pub description: Option<String>,
    pub entries: Vec<Entry>,
}

#[derive(Debug, Default)]
pub struct Entry {
    pub guid: String,
    pub title: Option<String>,
    pub link: Option<String>,
    pub published: Option<i64>,
    pub description: Option<String>,
    /// The item's Markdown, when the feed sends it beside HTML of its own (`<source:markdown>`):
    /// the page can show either (#285). With no HTML it is the description instead (#284).
    pub markdown: Option<String>,
    pub categories: Vec<String>,
    pub explicit: bool,
    /// Episode artwork; falls back to the feed's in the UI.
    pub image: Option<String>,
    /// Seconds.
    pub duration: Option<i64>,
    pub episode: Option<i64>,
    pub season: Option<i64>,
    pub enclosures: Vec<Enclosure>,
}

#[derive(Debug, Default, Clone, PartialEq)]
pub struct Enclosure {
    pub url: String,
    pub mime: Option<String>,
    pub length: Option<i64>,
}

pub enum Fetched {
    /// Server said 304, or returned a body we already have.
    NotModified,
    Body {
        bytes: Vec<u8>,
        etag: Option<String>,
        last_modified: Option<String>,
    },
}

/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and
/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504
/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small;
/// downloads, which are not, have no such limit.
pub const FEED_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);

/// The longest a feed's server may send nothing, for the clients that fetch feeds to set: one
/// that took the connection and never answered ran the whole 30 s, every scan it was in (#230).
/// No feed that answered took over 8 s in a day of traces; one that keeps sending, however
/// slowly, still has until FEED_TIMEOUT.
pub const FEED_SILENCE: std::time::Duration = std::time::Duration::from_secs(15);

/// Feeds fetched and the bytes of their bodies, since the process started, for /metrics to set
/// beside memory (#224): a body is read whole, so a night of large feeds is a night of large
/// allocations.
pub static FETCHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
pub static FETCHED_BYTES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);

/// Conditional GET, following redirects itself: `client` must follow none (`feed_client`), so
/// each hop is seen. The second value is where the feed now is when every hop said so for good
/// (301 or 308): a publisher that moved its feed, which the catalogue should follow rather
/// than be redirected on every read. A temporary redirect (302, 307) moves nothing.
/// `system-proxy` reads http_proxy/https_proxy.
/// `auth` is a username and password, sent only to the feed's own host.
/// An address as a trace may keep it: without its query or user and password, where a private
/// feed carries its credential (Patreon's `?auth=`), which Tempo kept for anyone reading it (#225).
/// shortcut: a token in the path (`/feed/<token>.xml`) stays; strip paths too if a feed does that.
pub fn redact(url: &str) -> String {
    match url::Url::parse(url) {
        Ok(mut u) => {
            u.set_query(None);
            u.set_fragment(None);
            let _ = u.set_username("");
            let _ = u.set_password(None);
            u.to_string()
        }
        // Not an address ipx could fetch; keep what comes before anything that could be a query.
        Err(_) => url.split(['?', '#', '@']).next().unwrap_or_default().to_owned(),
    }
}

/// Text with each address in it redacted, for what is logged and sent to every open page: an
/// error from reqwest names the whole address it fetched, a private feed's key with it (#238).
pub fn redact_text(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut rest = text;
    while let Some(i) = rest.find("http://").into_iter().chain(rest.find("https://")).min() {
        out.push_str(&rest[..i]);
        let end = rest[i..].find(|c: char| c.is_whitespace() || "()<>\"'".contains(c)).map_or(rest.len(), |n| i + n);
        out.push_str(&redact(&rest[i..end]));
        rest = &rest[end..];
    }
    out.push_str(rest);
    out
}

/// The most of a feed or page read. The largest feeds here are a few MB; without a limit an
/// episode's MP3 given as a feed was read whole into memory, at every check (#295).
const MAX_BODY: usize = 50 << 20;

/// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read.
async fn read_capped(mut resp: reqwest::Response) -> Result<Vec<u8>> {
    if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) {
        return Err(anyhow!("larger than {} MB, too large to be a feed", MAX_BODY >> 20));
    }
    let mut out = Vec::new();
    while let Some(c) = resp.chunk().await? {
        out.extend_from_slice(&c);
        anyhow::ensure!(out.len() <= MAX_BODY, "larger than {} MB, too large to be a feed", MAX_BODY >> 20);
    }
    Ok(out)
}

#[tracing::instrument(skip_all, fields(url = %redact(feed_url), http.response.body.size))]
pub async fn fetch(
    client: &reqwest::Client,
    feed_url: &str,
    auth: Option<(&str, Option<String>)>,
    etag: Option<&str>,
    last_modified: Option<&str>,
) -> Result<(Fetched, Option<String>)> {
    let start = reqwest::Url::parse(feed_url).context("the feed's address")?;
    let mut url = start.clone();
    let mut permanent = true;
    for _ in 0..10 {
        let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT);
        if let Some(tag) = etag {
            req = req.header(IF_NONE_MATCH, tag);
        }
        if let Some(lm) = last_modified {
            req = req.header(IF_MODIFIED_SINCE, lm);
        }
        // The feed's own host only: a redirect elsewhere must not be handed the password.
        if let Some((user, password)) = &auth
            && url.host_str() == start.host_str()
        {
            req = req.basic_auth(user, password.as_deref());
        }

        let resp = req.send().await.context("connecting")?;
        let status = resp.status();
        if status.is_redirection() && status != StatusCode::NOT_MODIFIED {
            let to = resp
                .headers()
                .get(reqwest::header::LOCATION)
                .and_then(|v| v.to_str().ok())
                .ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?;
            url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?;
            permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT);
            continue;
        }
        let moved = (permanent && url != start).then(|| url.to_string());
        FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        if status == StatusCode::NOT_MODIFIED {
            return Ok((Fetched::NotModified, moved));
        }
        if !status.is_success() {
            // WordPress.com answers some readers' feed requests with a browser check that only
            // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the
            // same posts and no check (#317).
            if status == StatusCode::FORBIDDEN
                && let Some(host) = url.host_str().map(str::to_owned)
                && read_capped(resp).await.is_ok_and(|b| is_wpcom_challenge(&b))
            {
                return Ok((wpcom_posts(client, &host).await?, moved));
            }
            // The original surfaced 401/407 specially; the code is enough for a UI to switch on.
            return Err(anyhow!("HTTP {status}"));
        }
        let header =
            |h: reqwest::header::HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned);
        let etag = header(ETAG);
        let last_modified = header(LAST_MODIFIED);
        let bytes = read_capped(resp).await.context("reading body")?;
        tracing::Span::current().record("http.response.body.size", bytes.len());
        FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
        return Ok((Fetched::Body { bytes, etag, last_modified }, moved));
    }
    // Worded as reqwest worded it, which failure_kind reads as a redirect loop.
    Err(anyhow!("error following redirect for url ({url}): too many redirects"))
}

/// WordPress.com's browser check, in place of the feed it was asked for.
fn is_wpcom_challenge(body: &[u8]) -> bool {
    String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/")
}

/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed.
/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed
/// set to carry more gets ten here.
async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result<Fetched> {
    let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts");
    let resp = client
        .get(&api)
        .timeout(FEED_TIMEOUT)
        .send()
        .await
        .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?;
    anyhow::ensure!(
        resp.status().is_success(),
        "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}",
        resp.status()
    );
    let bytes = read_capped(resp).await.context("reading WordPress.com's API")?;
    FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
    Ok(Fetched::Body { bytes, etag: None, last_modified: None })
}

/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's
/// own feed gives each post, so they are the items already stored. No title for the feed: the
/// stored one stays.
fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed {
    let entries = posts
        .iter()
        .filter_map(|p| {
            Some(Entry {
                guid: non_empty(p["guid"]["rendered"].as_str()).or_else(|| non_empty(p["link"].as_str()))?,
                title: title_text(p["title"]["rendered"].as_str()),
                link: non_empty(p["link"].as_str()),
                published: p["date_gmt"]
                    .as_str()
                    .and_then(|d| chrono::NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok())
                    .map(|d| d.and_utc().timestamp()),
                description: non_empty(p["content"]["rendered"].as_str()),
                image: non_empty(p["jetpack_featured_media_url"].as_str()),
                ..Default::default()
            })
        })
        .collect();
    ParsedFeed { entries, ..Default::default() }
}

/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
/// it is, and whether there is a new address to switch to.
pub struct Failure {
    pub reason: &'static str,
    pub new_url: Option<String>,
}

/// A failure's kind, for the log's `error.type` (#91): the HTTP status where there is one, as
/// OpenTelemetry names an HTTP error, and otherwise a word for what went wrong. Matches the
/// same wording as `explain_failure`; a message it does not know is "other", never a wrong kind.
pub fn failure_kind(msg: &str) -> (String, Option<u16>) {
    let low = msg.to_ascii_lowercase();
    let code = low.split("http ").skip(1).find_map(|r| r.get(..3)?.parse::<u16>().ok());
    if let Some(c) = code.filter(|c| (100..600).contains(c)) {
        return (c.to_string(), Some(c));
    }
    let kind = if low.contains("dns error")
        || low.contains("failed to lookup address")
        || low.contains("no address associated")
    {
        "dns"
    } else if low.contains("too many redirects") {
        "redirect_loop"
    } else if low.contains("timed out") || low.contains("timeout") {
        "timeout"
    } else if low.contains("certificate") || low.contains("tls") {
        "tls"
    } else if low.contains("got a web page") {
        "not_a_feed"
    } else if low.contains("the site sent ") {
        "site_message"
    } else if low.contains("connect") {
        "connect"
    } else if low.contains("pars") {
        "parse"
    } else {
        "other"
    };
    (kind.into(), None)
}

/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the
/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling
/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled --
/// comes back `None`: transient by nature, or with nothing more useful to say than the raw
/// text already shown once a feed is open.
///
/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP
/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure
/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing
/// nothing extra, so a miss costs a clearer message, not a wrong one.
pub fn explain_failure(msg: &str) -> Option<Failure> {
    if let Some(rest) = msg.strip_prefix("got a web page, not a feed") {
        let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned);
        return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
    }
    if msg.contains("the site sent ") {
        return Some(Failure {
            reason: "The site sent a message instead of the feed; the publisher has to fix it.",
            new_url: None,
        });
    }
    let low = msg.to_ascii_lowercase();
    if low.contains("http 404") {
        return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None });
    }
    if low.contains("http 401") || low.contains("http 403") {
        return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None });
    }
    if low.contains("answered with nothing") {
        return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None });
    }
    if low.contains("http 402") {
        return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None });
    }
    // Only a name that does not exist. A lookup that failed for the moment says nothing about the
    // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143).
    if !is_temporary_lookup_failure(msg)
        && (low.contains("dns error")
            || low.contains("failed to lookup address")
            || low.contains("no address associated"))
    {
        return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None });
    }
    None
}

/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's
/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does
/// not exist ("Name or service not known"). It says the network was not answering, not that the
/// feed is gone (#143).
pub fn is_temporary_lookup_failure(msg: &str) -> bool {
    let low = msg.to_ascii_lowercase();
    low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again"))
}

/// True when a body is an OPML document rather than a feed.
///
/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
/// OPML served from a URL without that extension. Sniffing the body catches both.
pub fn is_opml(bytes: &[u8]) -> bool {
    let head = &bytes[..bytes.len().min(1024)];
    let text = String::from_utf8_lossy(head).to_lowercase();
    text.contains("<opml")
}

/// How deep an OPML document's elements may nest. Real ones are a folder or two deep.
const MAX_OPML_DEPTH: usize = 100;

/// Parses an OPML document, refusing one nested deeper than `MAX_OPML_DEPTH`. The opml crate's
/// reader recurses once per level with no limit, and a few MB of `<outline>` overflowed the
/// stack, which aborts the whole daemon, and again on each restart for a subscribed feed (#322).
pub fn read_opml(text: &str) -> Result<opml::OPML> {
    use quick_xml::events::Event;
    let mut reader = quick_xml::Reader::from_str(text);
    let mut depth = 0usize;
    loop {
        match reader.read_event() {
            Ok(Event::Start(_)) => {
                depth += 1;
                if depth > MAX_OPML_DEPTH {
                    anyhow::bail!("that OPML nests more than {MAX_OPML_DEPTH} levels deep");
                }
            }
            Ok(Event::End(_)) => depth = depth.saturating_sub(1),
            // Malformed XML is the opml crate's to describe.
            Ok(Event::Eof) | Err(_) => break,
            Ok(_) => {}
        }
    }
    opml::OPML::from_str(text).map_err(|e| anyhow!("{e}"))
}

/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
    let text = String::from_utf8_lossy(bytes);
    let doc = read_opml(&text).map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
    let mut out = vec![];
    collect_outlines(&doc.body.outlines, &mut out);
    Ok(out)
}

/// OPML nests feeds inside folder outlines, so this walks the whole tree.
pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, String)>) {
    for o in outlines {
        if let Some(url) = &o.xml_url {
            let title = o.title.clone().unwrap_or_else(|| o.text.clone());
            out.push((title, url.clone()));
        }
        collect_outlines(&o.outlines, out);
    }
}

/// The <head><title> of an OPML document.
pub fn opml_title(bytes: &[u8]) -> Option<String> {
    let text = String::from_utf8_lossy(bytes);
    let doc = read_opml(&text).ok()?;
    doc.head.and_then(|h| h.title).map(|t| t.trim().to_owned()).filter(|t| !t.is_empty())
}

/// The token and show of a Patreon feed link, or None for any other URL.
///
/// Patreon gives each patron one token per creator. With no show it stands for the creator,
/// whose feed carries every show at once.
fn patreon_parts(url: &str) -> Option<(String, Option<String>)> {
    let u = url::Url::parse(url).ok()?;
    if !matches!(u.host_str()?, "patreon.com" | "www.patreon.com") || !u.path().starts_with("/rss") {
        return None;
    }
    let param = |name: &str| u.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v.into_owned());
    Some((param("auth")?, param("show")))
}

/// A Patreon link naming a creator but no show.
pub fn is_patreon_creator(url: &str) -> bool {
    matches!(patreon_parts(url), Some((_, None)))
}

/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's
/// feed, since the token alone says whose it is. An address with no scheme is https: 'cnn.com'
/// was stored as typed and every check failed with "relative URL without a base" (#101).
pub fn expand_input(input: &str) -> String {
    let s = input.trim();
    let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
    if token {
        format!("https://www.patreon.com/rss?auth={s}")
    } else if let Some(rest) = s.strip_prefix("//") {
        format!("https://{rest}")
    } else if !s.contains("://") {
        format!("https://{s}")
    } else {
        s.to_owned()
    }
}

/// Whether two URLs are the same feed. One Patreon show has several spellings -- by the
/// creator's name, by number, or with no creator at all -- and the token and show are what
/// identify it.
pub fn same_feed(a: &str, b: &str) -> bool {
    a == b || patreon_parts(a).is_some_and(|p| Some(p) == patreon_parts(b))
}

/// A Patreon creator's name and shows, each show as (title, feed URL).
///
/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes,
/// finding shows stops and the show feeds already found keep working. The documented API
/// needs an OAuth client per install and does not list shows.
pub async fn patreon_shows(client: &reqwest::Client, url: &str) -> Result<(Option<String>, Vec<(String, String)>)> {
    // The creator feed names its campaign by number in its self link, a few hundred bytes in.
    // The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up.
    let mut resp = client.get(url).timeout(FEED_TIMEOUT).send().await.context("connecting")?;
    if !resp.status().is_success() {
        return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status()));
    }
    let mut head = Vec::new();
    while patreon_campaign(&head).is_none() && head.len() < 64 * 1024 {
        let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break };
        head.extend_from_slice(&chunk);
    }
    let campaign = patreon_campaign(&head).ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;

    let api = format!(
        "https://www.patreon.com/api/campaigns/{campaign}\
         ?include=shows&fields%5Bcampaign%5D=name&fields%5Bcollection%5D=title"
    );
    let resp = client.get(api).send().await.context("asking Patreon for the shows")?;
    if !resp.status().is_success() {
        return Err(anyhow!("Patreon would not list the shows: HTTP {}", resp.status()));
    }
    let (name, shows) = parse_patreon_shows(&resp.bytes().await.context("reading the shows")?)?;
    Ok((name, shows.into_iter().map(|(id, title)| (title, format!("{url}&show={id}"))).collect()))
}

/// The campaign number in the start of a Patreon feed.
fn patreon_campaign(head: &[u8]) -> Option<String> {
    let text = String::from_utf8_lossy(head);
    text.match_indices("patreon.com/rss/").find_map(|(i, m)| {
        let id: String = text[i + m.len()..].chars().take_while(char::is_ascii_digit).collect();
        (!id.is_empty()).then_some(id)
    })
}

/// A campaign's name, and its shows as (id, title).
type PatreonShows = (Option<String>, Vec<(String, String)>);

/// A campaign's name and its shows as (id, title), from Patreon's JSON:API answer.
fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> {
    let v: serde_json::Value = serde_json::from_slice(json).context("Patreon's answer is not JSON")?;
    // Missing is not the same as none. Read as no shows, the creator feed would be scanned as
    // a plain feed, claim every show's files, and leave the shows empty once the list returned.
    let ids = v["data"]["relationships"]["shows"]["data"]
        .as_array()
        .ok_or_else(|| anyhow!("Patreon's answer does not list the shows"))?;
    let title = |id: &str| -> Option<String> {
        let show = v["included"].as_array()?.iter().find(|x| x["type"] == "collection" && x["id"] == id)?;
        show["attributes"]["title"].as_str().map(|t| t.trim().to_owned())
    };
    let shows = ids
        .iter()
        .filter_map(|s| s["id"].as_str())
        .map(|id| (id.to_owned(), title(id).unwrap_or_else(|| format!("Show {id}"))))
        .collect();
    Ok((v["data"]["attributes"]["name"].as_str().map(str::to_owned), shows))
}

/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
#[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
    if bytes.trim_ascii_start().starts_with(b"[")
        && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
    {
        return Ok(from_wp_posts(&posts));
    }
    match rss::Channel::read_from(bytes) {
        Ok(ch) => Ok(from_rss(ch, bytes)),
        Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
            Ok(feed) => Ok(from_atom(feed)),
            Err(atom_err) => {
                if let Some(said) = plain_text(bytes) {
                    return Err(anyhow!("the site sent {said} instead of a feed"));
                }
                // Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
                // "&amp;". Strict XML parsers refuse it; browsers don't. Retry once with
                // every offending "&" escaped rather than fail outright.
                let escaped = escape_bare_ampersands(bytes);
                if escaped != bytes {
                    if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) {
                        return Ok(from_rss(ch, &escaped));
                    }
                    if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) {
                        return Ok(from_atom(feed));
                    }
                }
                Err(match alternate_feed_link(bytes) {
                    Some(href) if looks_like_html(bytes) => {
                        anyhow!("got a web page, not a feed; it links {href} as its feed")
                    }
                    None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"),
                    _ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"),
                })
            }
        },
    }
}

/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed
/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a
/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case.
fn looks_like_html(bytes: &[u8]) -> bool {
    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase();
    head.contains("<!doctype html") || head.contains("<html")
}

/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with
/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried
/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors.
fn plain_text(bytes: &[u8]) -> Option<String> {
    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]);
    let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}');
    if text.starts_with('<') {
        return None;
    }
    let line = text.lines().next().unwrap_or("").trim_end();
    if line.is_empty() {
        return Some("an empty reply".into());
    }
    let mut said: String = line.chars().take(80).collect();
    if said.len() < line.len() {
        said.push('…');
    }
    Some(format!("\"{said}\""))
}

/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml"
/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now,
/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos.
fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
    let text = String::from_utf8_lossy(bytes);
    // ASCII only: to_lowercase changes some characters' length (U+0130 grows a byte), and the
    // offsets found in the lowered copy then sliced the original off a char boundary.
    let lower = text.to_ascii_lowercase();
    let mut pos = 0;
    while let Some(rel) = lower[pos..].find("<link") {
        let start = pos + rel;
        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
        pos = end + 1;
        let tag = &text[start..end];
        let tag_lower = &lower[start..end];
        let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'");
        let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml");
        if is_alternate
            && is_feed_type
            && let Some(href) = tag_attr(tag, "href")
        {
            return Some(href);
        }
    }
    None
}

/// What someone asked to add, as a feed: the address itself when it is a feed or an OPML list,
/// otherwise the feed its web page links as its own, otherwise an error and nothing is added.
/// A page that linked no feed used to be added as it was and failed on every check, called a
/// feed that moved (#102); cnn.com is one.
pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> {
    let read = |u: String| async move {
        let resp = client.get(&u).timeout(std::time::Duration::from_secs(30)).send().await.context("connecting")?;
        anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status());
        let base = resp.url().clone();
        anyhow::Ok((base, read_capped(resp).await.context("reading")?))
    };
    let is_feed = |b: &[u8]| is_opml(b) || parse(b).is_ok();
    let (base, body) = read(url.to_owned()).await.with_context(|| format!("could not read {url}"))?;
    if is_feed(&body) {
        return Ok(url.to_owned());
    }
    if !looks_like_html(&body) {
        let why = parse(&body).err().map(|e| format!("{e:#}")).unwrap_or_default();
        anyhow::bail!("{url} is not a feed: {why}");
    }
    let Some(href) = alternate_feed_link(&body) else {
        anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to");
    };
    let linked =
        base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?;
    let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?;
    anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either");
    Ok(linked.into())
}

/// Artwork for a feed that has none: the icon its site's page names, or else the site's
/// `/favicon.ico`. None if neither is there.
#[tracing::instrument(skip_all, fields(site = site))]
pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option<String> {
    let timeout = std::time::Duration::from_secs(20);
    let resp = client.get(site).timeout(timeout).send().await.ok()?;
    // Relative to where the page ended up, not where it was asked for: a site that redirects
    // to /en/ would otherwise have its icon looked for in the wrong place.
    let base = resp.url().clone();
    // The icon a page names can be gone: antirez.com names /images/favicon.png, a 404, while its
    // /favicon.ico is there. Stored unchecked, it was a broken image that was never looked up again.
    if resp.status().is_success()
        && let Ok(page) = read_capped(resp).await
        && let Some(href) = page_icon(&page)
        && let Ok(url) = base.join(&href)
        && is_image(client, url.as_str()).await
    {
        return Some(url.into());
    }
    let ico = base.join("/favicon.ico").ok()?;
    is_image(client, ico.as_str()).await.then(|| ico.into())
}

/// Artwork's address on https when its host serves it there, else as it was. The page is https
/// and must not load http; the host is asked once per `known` (one feed's read), and an http
/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five
/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN
/// presents another name's certificate (#110).
pub async fn prefer_https(
    client: &reqwest::Client,
    url: &str,
    known: &mut std::collections::HashMap<String, bool>,
) -> String {
    let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() };
    let host = rest.split('/').next().unwrap_or("").to_owned();
    let secure = format!("https://{rest}");
    let ok = match known.get(&host) {
        Some(ok) => *ok,
        None => {
            let ok = is_image(client, &secure).await;
            known.insert(host, ok);
            ok
        }
    };
    if ok { secure } else { url.to_owned() }
}

/// Whether `url` answers with an image. A site with no favicon often answers 200 with its home
/// page, which is not an icon.
pub async fn is_image(client: &reqwest::Client, url: &str) -> bool {
    let timeout = std::time::Duration::from_secs(20);
    let Ok(resp) = client.get(url).timeout(timeout).send().await else { return false };
    resp.status().is_success()
        && resp
            .headers()
            .get(reqwest::header::CONTENT_TYPE)
            .and_then(|v| v.to_str().ok())
            .is_some_and(|t| t.starts_with("image/"))
}

/// The icon a web page names in its `<link>` tags, the larger apple-touch-icon first: a plain
/// `icon` is often 16 pixels, which blurs at the size the list draws artwork.
fn page_icon(bytes: &[u8]) -> Option<String> {
    let text = String::from_utf8_lossy(bytes);
    // ASCII only, so byte offsets in the lowered copy stay valid in the original.
    let lower = text.to_ascii_lowercase();
    let (mut touch, mut icon) = (None, None);
    let mut pos = 0;
    while let Some(at) = lower[pos..].find("<link") {
        let start = pos + at;
        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
        pos = end + 1;
        let tag = &text[start..end];
        let Some(rel) = tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue };
        let rels: Vec<&str> = rel.split_whitespace().collect();
        if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) {
            touch = tag_attr(tag, "href");
        } else if icon.is_none() && rels.contains(&"icon") {
            icon = tag_attr(tag, "href");
        }
    }
    touch.or(icon).filter(|h| !h.is_empty())
}

/// The value of one attribute in an HTML/XML start tag, however it is quoted.
fn tag_attr(tag: &str, name: &str) -> Option<String> {
    let key = format!("{name}=");
    let idx = tag.to_lowercase().find(&key)?;
    let after = &tag[idx + key.len()..];
    let quote = after.chars().next()?;
    if quote != '"' && quote != '\'' {
        return None;
    }
    let rest = &after[1..];
    let close = rest.find(quote)?;
    Some(rest[..close].trim().to_owned())
}

/// Escapes every `&` that does not already start a recognized XML entity
/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&apos;`, or a numeric reference like `&#39;`).
fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> {
    fn is_entity_start(rest: &[u8]) -> bool {
        for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] {
            if rest.starts_with(named) {
                return true;
            }
        }
        let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") {
            &rest[2..]
        } else if rest.starts_with(b"#") {
            &rest[1..]
        } else {
            return false;
        };
        let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count();
        len > 0 && digits.get(len) == Some(&b';')
    }

    let mut out = Vec::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) {
            out.extend_from_slice(b"&amp;");
        } else {
            out.push(bytes[i]);
        }
        i += 1;
    }
    out
}

/// Every `<enclosure>` of every `<item>`, in document order.
///
/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
/// says -- and when a feed carries several it keeps only the *last*, silently losing the
/// rest. Feeds do ship several, so read them from the XML directly.
fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
    use quick_xml::events::Event;

    let mut reader = quick_xml::Reader::from_reader(bytes);
    reader.config_mut().trim_text(true);
    let mut buf = Vec::new();
    let mut out: Vec<Vec<Enclosure>> = Vec::new();
    let mut current: Option<Vec<Enclosure>> = None;

    let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> {
        let (mut url, mut mime, mut length) = (String::new(), None, None);
        for attr in e.attributes().flatten() {
            // Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
            let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default();
            match attr.key.local_name().as_ref() {
                "url" => url = val,
                "type" => mime = Some(val).filter(|v| !v.is_empty()),
                "length" => length = val.parse().ok(),
                _ => {}
            }
        }
        (!url.is_empty()).then_some(Enclosure { url, mime, length })
    };

    loop {
        match reader.read_event_into(&mut buf) {
            Ok(Event::Start(e)) => match e.name().local_name().as_ref() {
                "item" => current = Some(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::Empty(e)) => match e.name().local_name().as_ref() {
                // <item/> with no children still counts, so the indexes stay aligned.
                "item" => out.push(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::End(e)) => {
                if e.name().local_name().as_ref() == "item"
                    && let Some(list) = current.take()
                {
                    out.push(list);
                }
            }
            Ok(Event::Eof) | Err(_) => break,
            _ => {}
        }
        buf.clear();
    }
    if let Some(list) = current.take() {
        out.push(list);
    }
    out
}

fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
    let per_item = enclosures_by_item(bytes);
    let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);

    let entries = ch
        .items()
        .iter()
        .enumerate()
        .filter_map(|(idx, item)| {
            // Straight from the XML, so an item with several keeps all of them. Falls
            // back to the parsed one if the scan and the parser disagree on item count.
            let mut enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| {
                item.enclosure()
                    .into_iter()
                    .map(|e| Enclosure {
                        url: e.url().trim().to_owned(),
                        mime: non_empty(Some(e.mime_type())),
                        length: e.length().parse().ok(),
                    })
                    .filter(|e| !e.url.is_empty())
                    .collect()
            });
            drop_player_repeats(&mut enclosures);

            let guid = pick_guid(
                item.guid().map(|g| g.value()),
                item.link(),
                enclosures.first().map(|e| e.url.as_str()),
                item.title(),
            )?;

            let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
            let it = item.itunes_ext();

            Some(Entry {
                guid,
                title: title_text(item.title()),
                link: non_empty(item.link()),
                published: item.pub_date().and_then(parse_date),
                // Content wins over description, as __getEntries preferred entry.content. The
                // source namespace's Markdown copy only when there is neither (#284).
                description: notes(item.content(), item.description(), source_markdown(item)),
                markdown: non_empty(source_markdown(item))
                    .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()),
                categories: item
                    .categories()
                    .iter()
                    .map(|c| c.name().to_owned())
                    .filter(|c| !c.is_empty() && !c.starts_with("http"))
                    .collect(),
                explicit: explicit || entry_explicit,
                image: item_image(item, &enclosures),
                duration: it.and_then(|i| i.duration()).and_then(parse_duration),
                episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
                season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: title_text(Some(ch.title())),
        ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
        // itunes:image is the square artwork; <image><url> is the older, often smaller one.
        image: ch
            .itunes_ext()
            .and_then(|i| i.image())
            .map(str::to_owned)
            .or_else(|| ch.image().map(|i| i.url().to_owned())),
        site: non_empty(Some(ch.link().trim())),
        // Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
        // would fill the Directory with one-off tags. The subcategory where there is one: Apple
        // files every tabletop and gaming show under Leisure, which says little; Games says it.
        description: non_empty(Some(ch.description())).or_else(|| non_empty(ch.itunes_ext().and_then(|i| i.summary()))),
        category: ch
            .itunes_ext()
            .and_then(|i| i.categories().first())
            .map(|c| c.subcategory().filter(|s| !s.text().trim().is_empty()).unwrap_or(c))
            .and_then(|c| non_empty(Some(c.text().trim()))),
        entries,
    }
}

fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
    let entries = feed
        .entries()
        .iter()
        .filter_map(|e| {
            // Atom carries enclosures as <link rel="enclosure">.
            let mut enclosures: Vec<Enclosure> = e
                .links()
                .iter()
                .filter(|l| l.rel() == "enclosure")
                .map(|l| Enclosure {
                    url: l.href().trim().to_owned(),
                    mime: non_empty(l.mime_type()),
                    length: l.length().and_then(|s| s.parse().ok()),
                })
                .filter(|e| !e.url.is_empty())
                .collect();
            drop_player_repeats(&mut enclosures);

            let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href());

            let guid =
                pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?;

            Some(Entry {
                guid,
                title: title_text(Some(e.title().as_str())),
                link: alt.map(str::to_owned),
                published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
                description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None),
                markdown: None,
                categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
                explicit: false,
                image: None,
                duration: None,
                episode: None,
                season: None,
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: title_text(Some(feed.title().as_str())),
        ttl_mins: None,
        image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
        site: feed
            .links()
            .iter()
            .find(|l| l.rel() == "alternate")
            .map(|l| l.href().trim().to_owned())
            .filter(|h| !h.is_empty()),
        category: None,
        description: non_empty(feed.subtitle().map(|t| t.as_str())),
        entries,
    }
}

/// The original fell back to hashing the title or description. A guid, permalink or
/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
/// none of them has nothing to download and is dropped.
fn pick_guid(guid: Option<&str>, link: Option<&str>, enclosure: Option<&str>, title: Option<&str>) -> Option<String> {
    [guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned)
}

fn is_yes(s: &str) -> bool {
    matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
}

fn non_empty(s: Option<&str>) -> Option<String> {
    s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
}

/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post
/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion
/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays.
fn drop_player_repeats(encs: &mut Vec<Enclosure>) {
    let mut seen = std::collections::HashSet::new();
    encs.retain(|e| seen.insert(same_file_key(&e.url)));
}

/// An enclosure URL without WordPress's player number, for telling repeats of one file apart
/// from different files.
pub fn same_file_key(url: &str) -> String {
    let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() };
    let kept: Vec<(String, String)> = u
        .query_pairs()
        .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit())))
        .map(|(k, v)| (k.into_owned(), v.into_owned()))
        .collect();
    if kept.is_empty() {
        u.set_query(None);
    } else {
        u.query_pairs_mut().clear().extend_pairs(kept);
    }
    u.to_string()
}

/// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through
/// the XML parser with its HTML entities intact: The Verge's "Meta&#8217;s" reached the page as
/// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is
/// instead of failing the whole title.
fn title_text(s: Option<&str>) -> Option<String> {
    let s = non_empty(s)?;
    let mut out = String::with_capacity(s.len());
    let mut rest = s.as_str();
    while let Some(at) = rest.find('&') {
        out.push_str(&rest[..at]);
        rest = &rest[at..];
        let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1);
        let decoded = rest[len..]
            .starts_with(';')
            .then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok())
            .flatten();
        match decoded {
            Some(v) => {
                out.push_str(&v);
                rest = &rest[len + 1..];
            }
            None => {
                out.push('&');
                rest = &rest[1..];
            }
        }
    }
    out.push_str(rest);
    non_empty(Some(&out))
}

/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in
/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML.
fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option<String> {
    match body(content, description) {
        Some(b) if is_markdown(&b) => Some(markdown_html(&b)),
        Some(b) => Some(b),
        None => non_empty(markdown).map(|m| markdown_html(&m)),
    }
}

/// `<source:markdown>`, from https://source.scripting.com/: the item's text in Markdown.
///
/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding
/// it to another prefix is not read, until one turns up.
fn source_markdown(item: &rss::Item) -> Option<&str> {
    item.extensions().get("source")?.get("markdown")?.first()?.value()
}

/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold.
/// Plain text without any is left as it was, since rendering it would change nothing but
/// could still read a line starting "1984." as a list.
fn is_markdown(s: &str) -> bool {
    let tag = s
        .match_indices('<')
        .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!'));
    let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m));
    !tag && (s.contains("](") || s.contains("**") || s.lines().any(block))
}

/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML.
pub fn markdown_html(md: &str) -> String {
    use pulldown_cmark::{Options, Parser, html};
    let mut out = String::new();
    html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH));
    out
}

/// An item's show notes: its full body when that is whole, else its description.
///
/// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name
/// pasted from a web app (`[&:has([data-writing-block])>*]:pointer-events-auto`), so the body
/// began halfway through a tag and the page showed the rest of the tag as text. The same item's
/// `description` was whole. With no description to fall back on, a damaged body beats none.
fn body(content: Option<&str>, description: Option<&str>) -> Option<String> {
    match non_empty(content).filter(|c| !starts_mid_tag(c)) {
        Some(c) => Some(match subtitle(&c, description) {
            Some(s) => format!("<p><em>{}</em></p>{c}", quick_xml::escape::escape(s.as_str())),
            None => c,
        }),
        None => non_empty(description).or_else(|| non_empty(content)),
    }
}

/// A description that is a subtitle rather than a second copy of the notes: Substack puts the
/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it.
/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a
/// description found in the body is not shown twice.
///
/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its
/// notes passes too and shows above them, which reads fine; a real subtitle field would need an
/// `entries` column.
fn subtitle(body: &str, description: Option<&str>) -> Option<String> {
    let d = title_text(description)?;
    if d.contains('<') || d.chars().count() > 300 {
        return None;
    }
    // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut
    // description ends in "…" or "[...]".
    let words = |s: &str| {
        s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::<Vec<_>>().join(" ").to_lowercase()
    };
    let want = words(&d);
    let text = title_text(Some(&text_of(body))).unwrap_or_default();
    (!want.is_empty() && !words(&text).contains(&want)).then_some(d)
}

/// HTML with its tags taken out, each replaced by a space so words either side stay apart.
fn text_of(html: &str) -> String {
    let mut out = String::with_capacity(html.len());
    let mut in_tag = false;
    for c in html.chars() {
        match c {
            '<' => in_tag = true,
            '>' if in_tag => {
                in_tag = false;
                out.push(' ');
            }
            _ if !in_tag => out.push(c),
            _ => {}
        }
    }
    out
}

/// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose
/// start was cut off.
fn starts_mid_tag(html: &str) -> bool {
    html[..html.find('<').unwrap_or(html.len())].contains("\">")
}

/// The picture to show beside an item, in order of how deliberate it is:
/// `itunes:image`, then Media RSS `media:thumbnail`, then a `media:content` that is an
/// image, and finally an image enclosure -- which is how a blog's article picture arrives
/// (Substack puts it there), so those entries get artwork rather than a blank square.
fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> {
    if let Some(url) = item.itunes_ext().and_then(|i| i.image()) {
        return non_empty(Some(url));
    }

    let media = item.extensions().get("media");
    let attr = |name: &str, want_image: bool| -> Option<String> {
        media?.get(name)?.iter().find_map(|e| {
            if want_image {
                // media:content carries anything; only take it when it says it is a picture.
                let is_image = e.attrs.get("type").is_some_and(|t| t.starts_with("image/"))
                    || e.attrs.get("medium").is_some_and(|m| m == "image");
                if !is_image {
                    return None;
                }
            }
            non_empty(e.attrs.get("url").map(String::as_str))
        })
    };
    attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| {
        enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone())
    })
}

/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
fn parse_duration(s: &str) -> Option<i64> {
    let s = s.trim();
    if s.is_empty() {
        return None;
    }
    if !s.contains(':') {
        return s.parse().ok().filter(|n| *n > 0);
    }
    let mut total: i64 = 0;
    for part in s.split(':') {
        total = total * 60 + part.trim().parse::<i64>().ok()?;
    }
    Some(total).filter(|n| *n > 0)
}

/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
fn parse_date(s: &str) -> Option<i64> {
    let s = s.trim();
    chrono::DateTime::parse_from_rfc2822(s)
        .or_else(|_| chrono::DateTime::parse_from_rfc3339(s))
        .ok()
        .map(|d| d.timestamp())
}

/// Apple's podcast categories, each with its subcategories
/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own
/// `<itunes:category>` comes from, and a fixed one, so the Directory's categories stay few.
const CATEGORIES: &[(&str, &[&str])] = &[
    ("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]),
    ("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]),
    ("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]),
    ("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]),
    ("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]),
    ("Government", &[]),
    ("History", &[]),
    ("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]),
    ("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]),
    (
        "Leisure",
        &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"],
    ),
    ("Music", &["Music Commentary", "Music History", "Music Interviews"]),
    (
        "News",
        &[
            "Business News",
            "Daily News",
            "Entertainment News",
            "News Commentary",
            "Politics",
            "Sports News",
            "Tech News",
        ],
    ),
    (
        "Religion & Spirituality",
        &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"],
    ),
    (
        "Science",
        &[
            "Astronomy",
            "Chemistry",
            "Earth Sciences",
            "Life Sciences",
            "Mathematics",
            "Natural Sciences",
            "Nature",
            "Physics",
            "Social Sciences",
        ],
    ),
    ("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]),
    (
        "Sports",
        &[
            "Baseball",
            "Basketball",
            "Cricket",
            "Fantasy Sports",
            "Football",
            "Golf",
            "Hockey",
            "Rugby",
            "Running",
            "Soccer",
            "Swimming",
            "Tennis",
            "Volleyball",
            "Wilderness",
            "Wrestling",
        ],
    ),
    ("Technology", &[]),
    ("True Crime", &[]),
    ("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]),
];

/// Names that are not in Apple's list but mean one that is (#125). Jev picked from this list when
/// it spelt Football and Soccer its own way, and stored those answers; Software How-To is Apple's
/// Technology subcategory from before its 2019 list, which older feeds still name.
const ALIASES: &[(&str, &str)] =
    &[("American Football", "Football"), ("Football (Soccer)", "Soccer"), ("Software How-To", "Technology")];

/// Where a category sits in Apple's list, as (category, subcategory), so the Directory browses a
/// category and then its subcategories as Apple's does (#118). A podcast's own is stored as its
/// subcategory, and one an admin typed that is not Apple's stands as a category of its own.
/// Apple's categories as ipx has them, for the directory to check against Apple's own (#252).
pub fn categories() -> &'static [(&'static str, &'static [&'static str])] {
    CATEGORIES
}

pub fn category_path(name: &str) -> (String, Option<String>) {
    // A feed's own text, which can end in a stray comma: "Soccer," was a category of its own.
    let name = name.trim().trim_end_matches([',', ';', '.']).trim_end();
    let name = ALIASES.iter().find(|(old, _)| old.eq_ignore_ascii_case(name)).map_or(name, |(_, now)| now);
    for (category, subs) in CATEGORIES {
        if category.eq_ignore_ascii_case(name) {
            return (category.to_string(), None);
        }
        if let Some(sub) = subs.iter().find(|s| s.eq_ignore_ascii_case(name)) {
            return (category.to_string(), Some(sub.to_string()));
        }
    }
    (name.to_string(), None)
}

/// TypeSafe's System One endpoint, where Jev answers questions about a piece of state.
pub const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone";

/// A Directory category for a feed that names none of its own, which most blogs do not (#117):
/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and
/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest
/// is taken however unsure, since an admin can change it and asking again would cost a call a scan.
pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result<String> {
    let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect();
    // A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous.
    let mut criteria = serde_json::Map::new();
    for (category, subs) in CATEGORIES {
        criteria.insert(category.to_string(), serde_json::Value::Null);
        for sub in *subs {
            criteria.insert(sub.to_string(), format!("Within {category}").into());
        }
    }
    let body = serde_json::json!({
        "model": "jev-latest",
        "state": { "title": feed.title, "recent_items": items },
        "questions": { "category": {
            "type": "choice",
            "instructions": "Which category best describes what this feed publishes?",
            "criteria": criteria,
        }},
    });
    let answer: serde_json::Value = client
        .post(JEV_URL)
        .bearer_auth(key)
        .json(&body)
        .timeout(FEED_TIMEOUT)
        .send()
        .await?
        .error_for_status()?
        .json()
        .await?;
    picked(&answer)
}

fn picked(answer: &serde_json::Value) -> Result<String> {
    let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default();
    CATEGORIES
        .iter()
        .flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter()))
        .find(|c| **c == choice)
        .map(|c| c.to_string())
        .ok_or_else(|| anyhow!("no category in the answer: {answer}"))
}

/// An address that carries a subscriber's own access: a paid-feed host, credentials in the URL,
/// or a key in the query. ipx leaves such a feed out of anything others see, and the directory
/// refuses one sent to it (#202).
///
/// ponytail: a heuristic. A key hidden in the path of a host not in `PAID_HOSTS` gets
/// through; a per-feed private flag is the upgrade (#205).
pub fn url_looks_private(feed_url: &str) -> bool {
    let Ok(u) = url::Url::parse(feed_url) else { return true };
    // Paid-feed services put the subscriber's key in the path as often as in the query, and a
    // long path segment alone proves nothing (acast's public show ids look the same). So a
    // feed from one of them is private whatever its URL looks like. Supercast is why this
    // exists: `feeds.supercast.com/feeds/<key>` reached the directory before it did.
    let host = u.host_str().unwrap_or("");
    PAID_HOSTS.iter().any(|h| host == *h || host.ends_with(&format!(".{h}")))
        || !u.username().is_empty()
        || u.password().is_some()
        || u.query_pairs().any(|(k, _)| {
            let k = k.to_ascii_lowercase();
            ["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"].iter().any(|w| k.contains(w))
        })
}

/// Services whose feeds are always one subscriber's own.
const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"];

#[cfg(test)]
mod tests {
    #[test]
    fn reads_wordpress_posts_as_a_feed() {
        // As WordPress.com's API sends them, trimmed (#317).
        let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00",
        "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"},
        "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/",
        "title":{"rendered":"Baby&#8217;s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"},
        "jetpack_featured_media_url":""}]"#;
        let feed = super::parse(json).expect("WordPress posts parse as a feed");
        assert_eq!(feed.title, None);
        let e = &feed.entries[0];
        assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260");
        assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky"));
        assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/"));
        assert_eq!(e.published, Some(1_791_552_840));
        assert_eq!(e.description.as_deref(), Some("<p>Notes</p>"));
        assert_eq!(e.image, None);
        assert!(super::is_wpcom_challenge(
            b"<!DOCTYPE html><title>Checking your browser...</title><script src=\"/__challenge/probes.js\">"
        ));
        assert!(!super::is_wpcom_challenge(b"<html>Forbidden</html>"));
    }

    #[test]
    fn redact_drops_what_carries_a_credential() {
        assert_eq!(super::redact("https://www.patreon.com/rss/x?auth=SECRET&show=1"), "https://www.patreon.com/rss/x");
        assert_eq!(super::redact("https://me:pw@example.com/feed.xml#top"), "https://example.com/feed.xml");
        assert_eq!(super::redact("https://example.com/feed.xml"), "https://example.com/feed.xml");
        assert_eq!(super::redact("not a url?token=SECRET"), "not a url");
        // As reqwest words a failure, and with two addresses and none.
        assert_eq!(
            super::redact_text(
                "connecting: error sending request for url (https://www.patreon.com/rss/x?auth=SECRET&show=1): operation timed out"
            ),
            "connecting: error sending request for url (https://www.patreon.com/rss/x): operation timed out"
        );
        assert_eq!(super::redact_text("from http://a/f?k=1 to https://u:p@b/g"), "from http://a/f to https://b/g");
        assert_eq!(super::redact_text("HTTP 404 Not Found"), "HTTP 404 Not Found");
    }

    #[test]
    fn an_address_carrying_someones_access_looks_private() {
        let private = super::url_looks_private;
        assert!(!private("https://feeds.twit.tv/twit.xml"));
        // Patreon's shape: the key is a query parameter.
        assert!(private("https://www.patreon.com/rss/x?auth=abc123&show=2073588"));
        assert!(private("https://example.com/rss?api_key=abc"));
        // Paid-feed services put the key in the path; the host gives them away.
        assert!(private("https://feeds.supercast.com/feeds/abcdefghijklmnopqrstuvwx"));
        assert!(private("https://someshow.supportingcast.fm/content/abc123.rss"));
        // A long path segment alone is not a key: acast's public show ids look the same.
        assert!(!private("https://feeds.acast.com/public/shows/0123456789abcdef01234567"));
        assert!(private("https://ray:hunter2@example.com/rss"));
        assert!(private("not a url"), "unparseable is not safe to list");
    }

    /// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
    /// to /tmp, /new is the feed.
    async fn redirecting_server() -> String {
        use tokio::io::{AsyncReadExt, AsyncWriteExt};
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        tokio::spawn(async move {
            loop {
                let Ok((mut sock, _)) = listener.accept().await else { return };
                tokio::spawn(async move {
                    let mut buf = [0u8; 2048];
                    let n = sock.read(&mut buf).await.unwrap_or(0);
                    let req = String::from_utf8_lossy(&buf[..n]);
                    let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned();
                    let body = "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title></channel></rss>";
                    let resp = match path.as_str() {
                        "/old" => {
                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned()
                        }
                        "/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
                        "/chain" => {
                            "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned()
                        }
                        // An episode's file given as a feed: its size said, its body never sent.
                        "/huge" => "HTTP/1.1 200 OK\r\nContent-Length: 600000000\r\n\r\n".to_owned(),
                        "/loop" => {
                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned()
                        }
                        _ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()),
                    };
                    let _ = sock.write_all(resp.as_bytes()).await;
                });
            }
        });
        format!("http://{addr}")
    }

    #[tokio::test]
    async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() {
        let base = redirecting_server().await;
        let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap();
        let get = |path: &str| {
            let url = format!("{base}{path}");
            let client = client.clone();
            async move { super::fetch(&client, &url, None, None, None).await }
        };
        let (got, moved) = get("/old").await.unwrap();
        assert!(matches!(got, super::Fetched::Body { .. }));
        assert_eq!(moved, Some(format!("{base}/new")));
        assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
        let huge = get("/huge").await.err().map(|e| format!("{e:#}")).unwrap_or_default();
        assert!(huge.contains("too large to be a feed"), "{huge}");
        assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
        assert_eq!(get("/new").await.unwrap().1, None); // never moved
        let looped = get("/loop").await.err().unwrap().to_string();
        assert_eq!(super::failure_kind(&looped).0, "redirect_loop");
    }

    use super::*;

    #[test]
    fn parses_rss_with_itunes_extensions() {
        let bytes = include_bytes!("../../../tests/data/rss2.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Test Cast"));
        assert_eq!(feed.ttl_mins, Some(45));
        assert_eq!(feed.category.as_deref(), Some("Podcasting"), "the first, by its subcategory");
        assert_eq!(feed.description.as_deref(), Some("A synthetic feed used by the parser tests."));
        assert_eq!(feed.entries.len(), 3);

        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "https://example.com/ep/1");
        assert_eq!(ep.title.as_deref(), Some("Episode One"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(ep.categories, vec!["Tech"]);
        assert!(!ep.explicit);
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.com/ep1.mp3".into(),
                mime: Some("audio/mpeg".into()),
                length: Some(12_345_678),
            }]
        );

        assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
        assert_eq!(feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent"));
    }

    #[test]
    fn a_feed_says_what_it_is_in_itunes_summary_or_atom_subtitle() {
        // An empty <description> gives way to iTunes' summary (#130).
        let rss = br#"<?xml version="1.0"?><rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>R</title><description> </description><itunes:summary>Said here.</itunes:summary></channel></rss>"#;
        assert_eq!(parse(rss).unwrap().description.as_deref(), Some("Said here."));
        let atom = br#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>A</title>
        <subtitle>Or here.</subtitle><id>urn:a</id><updated>2026-01-01T00:00:00Z</updated></feed>"#;
        assert_eq!(parse(atom).unwrap().description.as_deref(), Some("Or here."));
    }

    #[test]
    fn markdown_notes_are_made_html_and_html_is_left_alone() {
        let feed = |items: &str| {
            let xml = format!(
                r#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>{items}</channel></rss>"#
            );
            parse(xml.as_bytes()).unwrap().entries.into_iter().map(|e| e.description).collect::<Vec<_>>()
        };
        let got = feed(
            r#"<item><guid>a</guid><description># Hi

Some **bold** and [a link](https://x.example/).</description></item>
            <item><guid>b</guid><source:markdown>Only *Markdown* here.</source:markdown></item>
            <item><guid>c</guid><description>&lt;p&gt;HTML **as sent**&lt;/p&gt;</description><source:markdown>ignored</source:markdown></item>
            <item><guid>d</guid><description>1984. Plain text, left as it is.</description></item>"#,
        );
        assert_eq!(
            got[0].as_deref(),
            Some("<h1>Hi</h1>\n<p>Some <strong>bold</strong> and <a href=\"https://x.example/\">a link</a>.</p>\n")
        );
        assert_eq!(got[1].as_deref(), Some("<p>Only <em>Markdown</em> here.</p>\n"), "the Markdown copy, alone");
        assert_eq!(got[2].as_deref(), Some("<p>HTML **as sent**</p>"), "HTML wins, untouched");
        assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is."));
    }

    #[test]
    fn markdown_sent_beside_html_is_kept_for_the_page_to_switch_to() {
        let xml = br#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>
        <item><guid>both</guid><description>&lt;p&gt;Hi&lt;/p&gt;</description><source:markdown>*Hi*</source:markdown></item>
        <item><guid>alone</guid><source:markdown>*Hi*</source:markdown></item></channel></rss>"#;
        let got: Vec<_> = parse(xml).unwrap().entries.into_iter().map(|e| e.markdown).collect();
        assert_eq!(got, [Some("*Hi*".to_owned()), None], "only beside HTML; alone, it is the description (#284)");
    }

    #[test]
    fn a_file_wordpress_lists_twice_is_one_enclosure() {
        let xml = br#"<?xml version="1.0"?><rss version="2.0"><channel><title>R</title>
        <item><title>The Promotion Paradox</title><guid>p</guid>
        <enclosure url="https://x/ep.mp3" length="1" type="audio/mpeg"/>
        <enclosure url="https://x/ep.mp3?_=2" length="1" type="audio/mpeg"/>
        <enclosure url="https://x/other.mp3?_=3&amp;key=k" length="1" type="audio/mpeg"/>
        </item></channel></rss>"#;
        let urls: Vec<String> = parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
        assert_eq!(
            urls,
            ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"],
            "the repeat goes, a different file stays"
        );
        assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k"));
        assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number");
    }

    #[test]
    fn titles_are_read_as_text_not_html() {
        // The Verge: an Atom title of type="html", its entity inside CDATA.
        let xml = br#"<?xml version="1.0"?>
        <feed xmlns="http://www.w3.org/2005/Atom"><title type="text">V</title><id>v</id>
        <updated>2026-09-15T00:00:00Z</updated>
        <entry><title type="html"><![CDATA[Meta&#8217;s new One]]></title><id>e1</id>
        <updated>2026-09-15T00:00:00Z</updated></entry></feed>"#;
        assert_eq!(parse(xml).unwrap().entries[0].title.as_deref(), Some("Meta\u{2019}s new One"));
        // HTML names as well as numbers; a bare `&` and an unknown name are left as they are.
        assert_eq!(
            title_text(Some("Pe&ntilde;a &amp; &#x201C;Q&A&#8221; &bogus; AT&T;")).as_deref(),
            Some("Pe\u{f1}a & \u{201c}Q&A\u{201d} &bogus; AT&T;")
        );
    }

    #[test]
    fn a_body_cut_off_mid_tag_gives_way_to_the_description() {
        // How libsyn served Daily Meditation Podcast #3477: content:encoded began inside a tag.
        let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant"> <p>What if</p>"#;
        let whole = r#"<div class="[&:has([data-writing-block])>*]:pointer-events-auto"><p>What if</p></div>"#;
        assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole));
        assert_eq!(
            body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(),
            Some("<p>Notes</p>"),
            "a whole body wins"
        );
        assert_eq!(
            body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(),
            Some("Plain notes, no tags.")
        );
        assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none");
        assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary"));
    }

    #[test]
    fn a_subtitle_missing_from_the_body_is_kept_above_it() {
        // Substack: the subtitle is the description, and content:encoded does not repeat it.
        assert_eq!(
            body(Some("<p>The post.</p>"), Some("Why the <b> tag & I fell out")).as_deref(),
            Some("<p>The post.</p>"),
            "a description with markup in it is notes, not a subtitle",
        );
        assert_eq!(
            body(Some("<p>The post.</p>"), Some("Why Q&amp;A threads go wrong")).as_deref(),
            Some("<p><em>Why Q&amp;A threads go wrong</em></p><p>The post.</p>"),
        );
        // A podcast repeating its notes, whole, cut short, or differently spaced: shown once.
        let notes = "<p>This week we talk about <a href=\"x\">tape</a>, drums and a very long list.</p>";
        for d in [
            "This week we talk about tape, drums and a very long list.",
            "This week we talk about tape, drums…",
            "This week we  talk about\ntape [...]",
        ] {
            assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body");
        }
        let long = "word ".repeat(80);
        assert_eq!(
            body(Some("<p>The post.</p>"), Some(&long)).as_deref(),
            Some("<p>The post.</p>"),
            "a long description is notes, not a subtitle"
        );
    }

    #[test]
    fn feed_level_explicit_overrides_entries() {
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>X</title><link>https://x</link><description>d</description>
        <itunes:explicit>yes</itunes:explicit>
        <item><title>a</title><guid>g1</guid>
        <enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();
        assert!(feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit");
    }

    #[test]
    fn failures_are_named_by_kind_for_the_log() {
        let k = |m: &str| failure_kind(m);
        assert_eq!(k("HTTP 404 Not Found"), ("404".into(), Some(404)));
        assert_eq!(k("HTTP 503 Service Unavailable"), ("503".into(), Some(503)));
        assert_eq!(
            k("connecting: error following redirect for url (https://www.toddstashwick.com/): too many redirects").0,
            "redirect_loop"
        );
        assert_eq!(k("connecting: dns error: failed to lookup address information").0, "dns");
        assert_eq!(k("operation timed out").0, "timeout");
        assert_eq!(k("something new").0, "other");
    }

    #[test]
    fn explain_failure_translates_the_errors_the_ui_should_flag() {
        assert_eq!(
            explain_failure("HTTP 404 Not Found").unwrap().reason,
            "The publisher took this feed down, or moved it."
        );
        assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests.");
        assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests.");
        assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan.");
        assert_eq!(
            explain_failure("the feed's address answered with nothing at all").unwrap().reason,
            "This address answers with nothing; the site may be gone."
        );
        let dns =
            explain_failure("connecting: dns error: failed to lookup address information: Name or service not known")
                .unwrap();
        assert_eq!(dns.reason, "This address no longer resolves; the site may be gone.");
        // A lookup that failed for the moment is not a site gone (#143).
        let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution";
        assert!(is_temporary_lookup_failure(blip));
        assert!(explain_failure(blip).is_none());
        assert!(!is_temporary_lookup_failure(
            "connecting: dns error: failed to lookup address information: Name or service not known"
        ));
        let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
        assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
        assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
        let down = explain_failure("the site sent \"Unable to establish a DB connection\" instead of a feed").unwrap();
        assert_eq!(down.reason, "The site sent a message instead of the feed; the publisher has to fix it.");
        for transient in [
            "HTTP 500 Internal Server Error",
            "HTTP 429 Too Many Requests",
            "operation timed out",
            "not RSS (reached end of input without finding a complete channel) and not Atom (unexpected end of input)",
        ] {
            assert!(explain_failure(transient).is_none(), "{transient} must not be flagged");
        }
    }

    #[test]
    fn a_web_page_says_so_and_names_the_feed_it_links() {
        let html = br#"<!doctype html><html><head>
            <link rel="alternate" type="application/rss+xml" href="https://x.example/feed">
            </head><body>not a feed</body></html>"#;
        let err = parse(html).unwrap_err().to_string();
        assert_eq!(err, "got a web page, not a feed; it links https://x.example/feed as its feed");
    }

    #[test]
    fn a_web_page_with_no_feed_link_still_says_so() {
        let html = b"<!doctype html><html><body>moved</body></html>";
        assert_eq!(parse(html).unwrap_err().to_string(), "got a web page, not a feed");
    }

    #[test]
    fn malformed_xml_gets_the_original_parser_errors() {
        let err = parse(b"<rss><channel><title>cut off").unwrap_err().to_string();
        assert!(err.starts_with("not RSS ("), "{err}");
    }

    #[test]
    fn a_body_with_no_markup_says_what_the_site_sent() {
        let err = parse(b"\xef\xbb\xbf\r\n  Unable to establish a DB connection\nmore").unwrap_err().to_string();
        assert_eq!(err, "the site sent \"Unable to establish a DB connection\" instead of a feed");
        let long = parse("x".repeat(200).as_bytes()).unwrap_err().to_string();
        assert_eq!(long, format!("the site sent \"{}…\" instead of a feed", "x".repeat(80)));
        assert_eq!(parse(b" \n").unwrap_err().to_string(), "the site sent an empty reply instead of a feed");
    }

    #[test]
    fn parses_atom_enclosure_links() {
        let bytes = include_bytes!("../../../tests/data/atom.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
        assert_eq!(feed.entries.len(), 1);
        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
        assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.org/ep1.m4a".into(),
                mime: Some("audio/mp4".into()),
                length: Some(9_876_543),
            }],
            "rel=enclosure only; the alternate link must not become an enclosure"
        );
    }

    #[test]
    fn a_bare_ampersand_in_a_link_is_repaired_and_parsed() {
        // kcpw.org: <link>https://kcpw.org/?post_type=post&p=125715</link> -- a bare "&"
        // that strict XML rejects but browsers accept.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0"><channel><title>X</title><link>https://x</link><description>d</description>
        <item><title>a</title><guid>g1</guid>
        <link>https://kcpw.org/?post_type=post&p=125715</link>
        <enclosure url="https://x/a.mp3?a=1&b=2" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();
        assert_eq!(feed.entries[0].link.as_deref(), Some("https://kcpw.org/?post_type=post&p=125715"));
        assert_eq!(feed.entries[0].enclosures[0].url, "https://x/a.mp3?a=1&b=2");
    }

    #[test]
    fn escape_bare_ampersands_leaves_real_entities_alone() {
        let out = escape_bare_ampersands(b"a&amp;b &lt;x&gt; &#39; &#x2F; c&d");
        assert_eq!(out, b"a&amp;b &lt;x&gt; &#39; &#x2F; c&amp;d");
    }

    #[test]
    fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
        // Some feeds set a different itunes:title. The displayed title is always the RSS
        // <title>, verbatim -- separators and all -- and season/episode are stored
        // alongside it rather than folded into it.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>Show</title><link>https://x</link><description>d</description>
        <item>
          <title>Music from a Darkened Room | Session Zero</title>
          <itunes:title>Session Zero</itunes:title>
          <guid>sz</guid>
          <itunes:season>8</itunes:season>
          <itunes:duration>6720</itunes:duration>
          <enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
        </item>
        <item>
          <title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
          <guid>p1</guid>
          <itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
          <enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
        </item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();

        let sz = &feed.entries[0];
        assert_eq!(
            sz.title.as_deref(),
            Some("Music from a Darkened Room | Session Zero"),
            "itunes:title must not override the RSS title"
        );
        assert_eq!(sz.season, Some(8));
        assert_eq!(sz.episode, None, "a missing episode number stays missing");
        assert_eq!(sz.duration, Some(6720));

        let p1 = &feed.entries[1];
        assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
        assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
    }

    #[test]
    fn deeply_nested_opml_is_refused_before_it_is_parsed() {
        let nest = |n: usize| {
            format!(
                r#"<opml version="2.0"><head><title>T</title></head><body>{}<outline text="f" xmlUrl="https://e.com/f"/>{}</body></opml>"#,
                r#"<outline text="a">"#.repeat(n),
                "</outline>".repeat(n)
            )
        };
        assert_eq!(parse_opml(nest(5).as_bytes()).unwrap().len(), 1, "a few folders deep is fine");
        // Deep enough to overflow a 2 MiB stack in the opml crate's reader without the check.
        let deep = nest(100_000);
        assert!(parse_opml(deep.as_bytes()).unwrap_err().to_string().contains("levels deep"));
        assert_eq!(opml_title(deep.as_bytes()), None);
    }

    #[test]
    fn opml_is_recognised_and_its_feeds_listed() {
        let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
          <outline text="Folder">
            <outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
            <outline type="rss" text="Beta"  xmlUrl="https://b.example/rss"/>
          </outline>
          <outline text="Not a feed"/>
        </body></opml>"#;
        assert!(is_opml(xml));
        assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));

        let feeds = parse_opml(xml).unwrap();
        assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
        assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));

        // A feed must never be mistaken for a subscription list.
        assert!(!is_opml(include_bytes!("../../../tests/data/rss2.xml")));
        assert!(!is_opml(include_bytes!("../../../tests/data/atom.xml")));
    }

    #[test]
    fn a_patreon_creator_is_a_list_of_its_shows() {
        let tok = "AbCdEfGhIjKlMnOpQrStUvWxYz012_-9";
        assert_eq!(expand_input(&format!(" {tok} ")), format!("https://www.patreon.com/rss?auth={tok}"));
        assert_eq!(expand_input("https://example.com/rss"), "https://example.com/rss");
        assert_eq!(expand_input("http://example.com/rss"), "http://example.com/rss");
        assert_eq!(expand_input(" cnn.com "), "https://cnn.com");
        assert_eq!(expand_input("example.com/feed.xml"), "https://example.com/feed.xml");
        assert_eq!(expand_input("//example.com/rss"), "https://example.com/rss");

        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss/glasscannon?auth={tok}")));
        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss?auth={tok}")));
        assert!(!is_patreon_creator(&format!("https://www.patreon.com/rss/x?auth={tok}&show=1")), "one show is a feed");
        assert!(!is_patreon_creator(&format!("https://example.com/rss?auth={tok}")));

        // The show you already have by name is the one a bare token would add by number.
        assert!(same_feed(
            &format!("https://www.patreon.com/rss/glasscannon?auth={tok}&show=2073588"),
            &format!("https://www.patreon.com/rss?auth={tok}&show=2073588"),
        ));
        assert!(!same_feed(
            &format!("https://www.patreon.com/rss?auth={tok}&show=1"),
            &format!("https://www.patreon.com/rss?auth={tok}&show=2"),
        ));

        // The self link carries the campaign by number, whichever spelling was asked for.
        let head = br#"<rss><channel><link>https://www.patreon.com/glasscannon</link>
            <atom:link href="https://www.patreon.com/rss/369921?auth=t" rel="self"/>"#;
        assert_eq!(patreon_campaign(head).as_deref(), Some("369921"));
        assert_eq!(patreon_campaign(b"<rss><channel><title>T"), None);

        let json = br#"{"data":{"id":"369921","type":"campaign","attributes":{"name":"The Glass Cannon Network"},
            "relationships":{"shows":{"data":[{"id":"2073588","type":"collection"},{"id":"2073636","type":"collection"}]}}},
            "included":[{"id":"2073588","type":"collection","attributes":{"title":"Get in the Trunk "}},
                        {"id":"2073636","type":"collection","attributes":{"title":"Shadowdark"}}]}"#;
        let (name, shows) = parse_patreon_shows(json).unwrap();
        assert_eq!(name.as_deref(), Some("The Glass Cannon Network"));
        assert_eq!(shows, [("2073588".into(), "Get in the Trunk".into()), ("2073636".into(), "Shadowdark".into())]);

        // An answer that stops naming the shows is an error, never "this creator has none".
        assert!(parse_patreon_shows(br#"{"data":{"attributes":{"name":"X"}}}"#).is_err());
    }

    #[test]
    fn an_item_may_carry_several_enclosures() {
        // The rss crate keeps only one per item -- the last -- so these come from the XML.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0"><channel><title>M</title><link>https://x</link><description>d</description>
        <item><title>Two files</title><guid>m1</guid>
          <enclosure url="https://x/a.mp3?v=1&amp;t=2" length="111" type="audio/mpeg"/>
          <enclosure url="https://x/b.mp4" length="222" type="video/mp4"/>
        </item>
        <item><title>One file</title><guid>m2</guid>
          <enclosure url="https://x/c.mp3" length="333" type="audio/mpeg"/></item>
        <item><title>None</title><guid>m3</guid></item>
        </channel></rss>"#;
        let f = parse(xml).unwrap();
        assert_eq!(f.entries.len(), 3);

        let two = &f.entries[0].enclosures;
        assert_eq!(two.len(), 2, "both enclosures survive");
        assert_eq!(two[0].url, "https://x/a.mp3?v=1&t=2", "document order, and the escaped ampersand is decoded");
        assert_eq!(two[0].length, Some(111));
        assert_eq!(two[1].url, "https://x/b.mp4");
        assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));

        assert_eq!(f.entries[1].enclosures.len(), 1);
        assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
    }

    #[test]
    fn a_feed_link_after_non_ascii_text_is_found() {
        // U+0130 lowercases to three bytes from two, which once shifted every offset after it.
        let page = "<html><title>\u{130}stanbul \u{130}\u{130}</title>\
            <link rel=\"alternate\" type=\"application/rss+xml\" href=\"/feed.xml\">";
        assert_eq!(alternate_feed_link(page.as_bytes()).as_deref(), Some("/feed.xml"));
    }

    #[test]
    fn page_icon_prefers_the_touch_icon() {
        let page = br#"<head><link rel="stylesheet" href="/a.css">
            <link rel="shortcut icon" href="/fav.ico">
            <LINK REL="apple-touch-icon-precomposed" sizes="180x180" href='/touch.png'></head>"#;
        assert_eq!(page_icon(page).as_deref(), Some("/touch.png"));
        assert_eq!(page_icon(br#"<link rel="icon" href="i.svg">"#).as_deref(), Some("i.svg"));
        assert_eq!(page_icon(br#"<link rel="stylesheet" href="/a.css">"#), None);
    }

    #[test]
    fn an_items_picture_comes_from_the_most_deliberate_source() {
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
             xmlns:media="http://search.yahoo.com/mrss/">
        <channel><title>P</title><link>https://x</link><description>d</description>
        <item><title>Has itunes</title><guid>a</guid>
          <itunes:image href="https://x/itunes.jpg"/>
          <media:thumbnail url="https://x/thumb.jpg"/>
          <enclosure url="https://x/a.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Has thumbnail</title><guid>b</guid>
          <media:thumbnail url="https://x/thumb.jpg"/>
          <enclosure url="https://x/b.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Has media content</title><guid>c</guid>
          <media:content url="https://x/pic.jpg" type="image/jpeg"/>
          <media:content url="https://x/clip.mp4" type="video/mp4"/></item>
        <item><title>Only an image enclosure</title><guid>d</guid>
          <enclosure url="https://x/d.jpg" length="1" type="image/jpeg"/></item>
        <item><title>Audio only</title><guid>e</guid>
          <enclosure url="https://x/e.mp3" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let f = parse(xml).unwrap();
        let img = |i: usize| f.entries[i].image.as_deref();
        assert_eq!(img(0), Some("https://x/itunes.jpg"), "itunes:image wins");
        assert_eq!(img(1), Some("https://x/thumb.jpg"), "then media:thumbnail");
        assert_eq!(img(2), Some("https://x/pic.jpg"), "media:content, and only the image one");
        assert_eq!(img(3), Some("https://x/d.jpg"), "a blog's article picture arrives as an enclosure");
        assert_eq!(img(4), None, "audio is not a picture");
    }

    #[test]
    fn durations_parse_from_seconds_or_a_clock() {
        assert_eq!(parse_duration("5649"), Some(5649));
        assert_eq!(parse_duration("23:45"), Some(1425));
        assert_eq!(parse_duration("1:34:09"), Some(5649));
        assert_eq!(parse_duration("0"), None, "zero is not a duration");
        assert_eq!(parse_duration(""), None);
        assert_eq!(parse_duration("garbage"), None);
    }

    #[test]
    fn rejects_html_masquerading_as_a_feed() {
        assert!(parse(b"<html><body>nope</body></html>").is_err());
    }

    #[test]
    fn guid_falls_back_through_link_then_enclosure() {
        assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
        assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
        assert_eq!(pick_guid(None, None, None, None), None);
    }

    #[test]
    fn picked_takes_only_an_apple_category() {
        let answer = |c: &str| serde_json::json!({"answers": {"category": {"type": "choice", "choice": c}}});
        assert_eq!(super::picked(&answer("Technology")).unwrap(), "Technology");
        assert_eq!(super::picked(&answer("Tech News")).unwrap(), "Tech News", "a subcategory");
        assert!(super::picked(&answer("Blogs")).is_err(), "not one of Apple's");
        assert!(super::picked(&serde_json::json!({"error": "rate limited"})).is_err());
    }

    #[test]
    fn category_path_finds_a_subcategory_s_category() {
        use super::category_path;
        assert_eq!(category_path("Tech News"), ("News".into(), Some("Tech News".into())));
        assert_eq!(category_path("video games"), ("Leisure".into(), Some("Video Games".into())), "Apple's spelling");
        assert_eq!(category_path("Technology"), ("Technology".into(), None));
        assert_eq!(category_path("Homebrew"), ("Homebrew".into(), None), "not Apple's: its own category");
        // Apple's spelling, an old answer of Jev's, a stray comma and Apple's old list (#125).
        assert_eq!(category_path("Soccer"), ("Sports".into(), Some("Soccer".into())));
        assert_eq!(category_path("Football (Soccer)"), ("Sports".into(), Some("Soccer".into())));
        assert_eq!(category_path("Soccer,"), ("Sports".into(), Some("Soccer".into())));
        assert_eq!(category_path("Football"), ("Sports".into(), Some("Football".into())));
        assert_eq!(category_path("Software How-To"), ("Technology".into(), None));
    }
}