ipx/src/feed.rs

//! Feed fetching and parsing. Replaces FeedData.__getFeed / __getEntries.

use anyhow::{Context, Result, anyhow};
use reqwest::StatusCode;
use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};

use crate::config::Feed as FeedCfg;

#[derive(Debug, Default)]
pub struct ParsedFeed {
    pub title: Option<String>,
    pub ttl_mins: Option<u64>,
    pub image: Option<String>,
    pub entries: Vec<Entry>,
}

#[derive(Debug, Default)]
pub struct Entry {
    pub guid: String,
    pub title: Option<String>,
    pub link: Option<String>,
    pub published: Option<i64>,
    pub description: Option<String>,
    pub categories: Vec<String>,
    pub explicit: bool,
    /// Episode artwork; falls back to the feed's in the UI.
    pub image: Option<String>,
    /// Seconds.
    pub duration: Option<i64>,
    pub episode: Option<i64>,
    pub season: Option<i64>,
    pub enclosures: Vec<Enclosure>,
}

#[derive(Debug, Default, Clone, PartialEq)]
pub struct Enclosure {
    pub url: String,
    pub mime: Option<String>,
    pub length: Option<i64>,
}

pub enum Fetched {
    /// Server said 304, or returned a body we already have.
    NotModified,
    Body {
        bytes: Vec<u8>,
        etag: Option<String>,
        last_modified: Option<String>,
    },
}

/// Conditional GET. reqwest handles gzip and redirects; the original's hand-rolled
/// CONNECT/socket.ssl proxy path is gone -- `system-proxy` reads http_proxy/https_proxy.
pub async fn fetch(
    client: &reqwest::Client,
    cfg: &FeedCfg,
    etag: Option<&str>,
    last_modified: Option<&str>,
) -> Result<Fetched> {
    let mut req = client.get(&cfg.url);
    if let Some(tag) = etag {
        req = req.header(IF_NONE_MATCH, tag);
    }
    if let Some(lm) = last_modified {
        req = req.header(IF_MODIFIED_SINCE, lm);
    }
    if let Some(user) = &cfg.username {
        req = req.basic_auth(user, cfg.password());
    }

    let resp = req.send().await.context("connecting")?;
    if resp.status() == StatusCode::NOT_MODIFIED {
        return Ok(Fetched::NotModified);
    }
    let status = resp.status();
    if !status.is_success() {
        // The original surfaced 401/407 specially; the code is enough for a UI to switch on.
        return Err(anyhow!("HTTP {status}"));
    }

    let header = |h: reqwest::header::HeaderName| {
        resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned)
    };
    let etag = header(ETAG);
    let last_modified = header(LAST_MODIFIED);
    let bytes = resp.bytes().await.context("reading body")?.to_vec();
    Ok(Fetched::Body { bytes, etag, last_modified })
}

/// True when a body is an OPML document rather than a feed.
///
/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
/// OPML served from a URL without that extension. Sniffing the body catches both.
pub fn is_opml(bytes: &[u8]) -> bool {
    let head = &bytes[..bytes.len().min(1024)];
    let text = String::from_utf8_lossy(head).to_lowercase();
    text.contains("<opml")
}

/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
    let text = String::from_utf8_lossy(bytes);
    let doc = opml::OPML::from_str(&text)
        .map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
    let mut out = vec![];
    crate::collect_outlines(&doc.body.outlines, &mut out);
    Ok(out)
}

/// The <head><title> of an OPML document.
pub fn opml_title(bytes: &[u8]) -> Option<String> {
    let text = String::from_utf8_lossy(bytes);
    let doc = opml::OPML::from_str(&text).ok()?;
    doc.head
        .and_then(|h| h.title)
        .map(|t| t.trim().to_owned())
        .filter(|t| !t.is_empty())
}

/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
    match rss::Channel::read_from(bytes) {
        Ok(ch) => Ok(from_rss(ch, bytes)),
        Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
            Ok(feed) => Ok(from_atom(feed)),
            Err(atom_err) => Err(anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})")),
        },
    }
}

/// Every `<enclosure>` of every `<item>`, in document order.
///
/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
/// says -- and when a feed carries several it keeps only the *last*, silently losing the
/// rest. Feeds do ship several, so read them from the XML directly.
fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
    use quick_xml::events::Event;

    let mut reader = quick_xml::Reader::from_reader(bytes);
    reader.config_mut().trim_text(true);
    let mut buf = Vec::new();
    let mut out: Vec<Vec<Enclosure>> = Vec::new();
    let mut current: Option<Vec<Enclosure>> = None;

    let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> {
        let (mut url, mut mime, mut length) = (String::new(), None, None);
        for attr in e.attributes().flatten() {
            // Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
            let val = quick_xml::escape::unescape(&attr.value)
                .map(|v| v.trim().to_string())
                .unwrap_or_default();
            match attr.key.local_name().as_ref() {
                "url" => url = val,
                "type" => mime = Some(val).filter(|v| !v.is_empty()),
                "length" => length = val.parse().ok(),
                _ => {}
            }
        }
        (!url.is_empty()).then_some(Enclosure { url, mime, length })
    };

    loop {
        match reader.read_event_into(&mut buf) {
            Ok(Event::Start(e)) => match e.name().local_name().as_ref() {
                "item" => current = Some(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::Empty(e)) => match e.name().local_name().as_ref() {
                // <item/> with no children still counts, so the indexes stay aligned.
                "item" => out.push(Vec::new()),
                "enclosure" => {
                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
                        list.push(enc);
                    }
                }
                _ => {}
            },
            Ok(Event::End(e)) => {
                if e.name().local_name().as_ref() == "item"
                    && let Some(list) = current.take()
                {
                    out.push(list);
                }
            }
            Ok(Event::Eof) | Err(_) => break,
            _ => {}
        }
        buf.clear();
    }
    if let Some(list) = current.take() {
        out.push(list);
    }
    out
}

fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
    let per_item = enclosures_by_item(bytes);
    let explicit = ch
        .itunes_ext()
        .and_then(|it| it.explicit())
        .is_some_and(is_yes);

    let entries = ch
        .items()
        .iter()
        .enumerate()
        .filter_map(|(idx, item)| {
            // Straight from the XML, so an item with several keeps all of them. Falls
            // back to the parsed one if the scan and the parser disagree on item count.
            let enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| {
                item.enclosure()
                    .into_iter()
                    .map(|e| Enclosure {
                        url: e.url().trim().to_owned(),
                        mime: non_empty(Some(e.mime_type())),
                        length: e.length().parse().ok(),
                    })
                    .filter(|e| !e.url.is_empty())
                    .collect()
            });

            let guid = pick_guid(
                item.guid().map(|g| g.value()),
                item.link(),
                enclosures.first().map(|e| e.url.as_str()),
                item.title(),
            )?;

            let entry_explicit = item
                .itunes_ext()
                .and_then(|it| it.explicit())
                .is_some_and(is_yes);
            let it = item.itunes_ext();

            Some(Entry {
                guid,
                title: non_empty(item.title()),
                link: non_empty(item.link()),
                published: item.pub_date().and_then(parse_date),
                // Content wins over description, as __getEntries preferred entry.content.
                description: non_empty(item.content()).or_else(|| non_empty(item.description())),
                categories: item
                    .categories()
                    .iter()
                    .map(|c| c.name().to_owned())
                    .filter(|c| !c.is_empty() && !c.starts_with("http"))
                    .collect(),
                explicit: explicit || entry_explicit,
                image: it.and_then(|i| i.image()).map(str::to_owned),
                duration: it.and_then(|i| i.duration()).and_then(parse_duration),
                episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
                season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: non_empty(Some(ch.title())),
        ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
        // itunes:image is the square artwork; <image><url> is the older, often smaller one.
        image: ch
            .itunes_ext()
            .and_then(|i| i.image())
            .map(str::to_owned)
            .or_else(|| ch.image().map(|i| i.url().to_owned())),
        entries,
    }
}

fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
    let entries = feed
        .entries()
        .iter()
        .filter_map(|e| {
            // Atom carries enclosures as <link rel="enclosure">.
            let enclosures: Vec<Enclosure> = e
                .links()
                .iter()
                .filter(|l| l.rel() == "enclosure")
                .map(|l| Enclosure {
                    url: l.href().trim().to_owned(),
                    mime: non_empty(l.mime_type()),
                    length: l.length().and_then(|s| s.parse().ok()),
                })
                .filter(|e| !e.url.is_empty())
                .collect();

            let alt = e
                .links()
                .iter()
                .find(|l| l.rel() == "alternate" || l.rel().is_empty())
                .map(|l| l.href());

            let guid = pick_guid(
                Some(e.id()),
                alt,
                enclosures.first().map(|x| x.url.as_str()),
                Some(e.title().as_str()),
            )?;

            Some(Entry {
                guid,
                title: non_empty(Some(e.title().as_str())),
                link: alt.map(str::to_owned),
                published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
                description: e
                    .content()
                    .and_then(|c| c.value())
                    .or_else(|| e.summary().map(|s| s.as_str()))
                    .map(str::to_owned),
                categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
                explicit: false,
                image: None,
                duration: None,
                episode: None,
                season: None,
                enclosures,
            })
        })
        .collect();

    ParsedFeed {
        title: non_empty(Some(feed.title().as_str())),
        ttl_mins: None,
        image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
        entries,
    }
}

/// The original fell back to hashing the title or description. A guid, permalink or
/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
/// none of them has nothing to download and is dropped.
fn pick_guid(
    guid: Option<&str>,
    link: Option<&str>,
    enclosure: Option<&str>,
    title: Option<&str>,
) -> Option<String> {
    [guid, link, enclosure, title]
        .into_iter()
        .flatten()
        .map(str::trim)
        .find(|s| !s.is_empty())
        .map(str::to_owned)
}

fn is_yes(s: &str) -> bool {
    matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
}

fn non_empty(s: Option<&str>) -> Option<String> {
    s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
}

/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
fn parse_duration(s: &str) -> Option<i64> {
    let s = s.trim();
    if s.is_empty() {
        return None;
    }
    if !s.contains(':') {
        return s.parse().ok().filter(|n| *n > 0);
    }
    let mut total: i64 = 0;
    for part in s.split(':') {
        total = total * 60 + part.trim().parse::<i64>().ok()?;
    }
    Some(total).filter(|n| *n > 0)
}

/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
fn parse_date(s: &str) -> Option<i64> {
    let s = s.trim();
    chrono::DateTime::parse_from_rfc2822(s)
        .or_else(|_| chrono::DateTime::parse_from_rfc3339(s))
        .ok()
        .map(|d| d.timestamp())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parses_rss_with_itunes_extensions() {
        let bytes = include_bytes!("../tests/data/rss2.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Test Cast"));
        assert_eq!(feed.ttl_mins, Some(45));
        assert_eq!(feed.entries.len(), 3);

        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "https://example.com/ep/1");
        assert_eq!(ep.title.as_deref(), Some("Episode One"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(ep.categories, vec!["Tech"]);
        assert!(!ep.explicit);
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.com/ep1.mp3".into(),
                mime: Some("audio/mpeg".into()),
                length: Some(12_345_678),
            }]
        );

        assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
        assert_eq!(
            feed.entries[2].enclosures[0].mime.as_deref(),
            Some("application/x-bittorrent")
        );
    }

    #[test]
    fn feed_level_explicit_overrides_entries() {
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>X</title><link>https://x</link><description>d</description>
        <itunes:explicit>yes</itunes:explicit>
        <item><title>a</title><guid>g1</guid>
        <enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();
        assert!(
            feed.entries[0].explicit,
            "the entry says nothing; the feed-level flag must still mark it explicit"
        );
    }

    #[test]
    fn parses_atom_enclosure_links() {
        let bytes = include_bytes!("../tests/data/atom.xml");
        let feed = parse(bytes).unwrap();

        assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
        assert_eq!(feed.entries.len(), 1);
        let ep = &feed.entries[0];
        assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
        assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
        assert_eq!(ep.published, Some(1_078_016_400));
        assert_eq!(
            ep.enclosures,
            vec![Enclosure {
                url: "https://example.org/ep1.m4a".into(),
                mime: Some("audio/mp4".into()),
                length: Some(9_876_543),
            }],
            "rel=enclosure only; the alternate link must not become an enclosure"
        );
    }

    #[test]
    fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
        // Some feeds set a different itunes:title. The displayed title is always the RSS
        // <title>, verbatim -- separators and all -- and season/episode are stored
        // alongside it rather than folded into it.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
        <channel><title>Show</title><link>https://x</link><description>d</description>
        <item>
          <title>Music from a Darkened Room | Session Zero</title>
          <itunes:title>Session Zero</itunes:title>
          <guid>sz</guid>
          <itunes:season>8</itunes:season>
          <itunes:duration>6720</itunes:duration>
          <enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
        </item>
        <item>
          <title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
          <guid>p1</guid>
          <itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
          <enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
        </item>
        </channel></rss>"#;
        let feed = parse(xml).unwrap();

        let sz = &feed.entries[0];
        assert_eq!(
            sz.title.as_deref(),
            Some("Music from a Darkened Room | Session Zero"),
            "itunes:title must not override the RSS title"
        );
        assert_eq!(sz.season, Some(8));
        assert_eq!(sz.episode, None, "a missing episode number stays missing");
        assert_eq!(sz.duration, Some(6720));

        let p1 = &feed.entries[1];
        assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
        assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
    }

    #[test]
    fn opml_is_recognised_and_its_feeds_listed() {
        let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
          <outline text="Folder">
            <outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
            <outline type="rss" text="Beta"  xmlUrl="https://b.example/rss"/>
          </outline>
          <outline text="Not a feed"/>
        </body></opml>"#;
        assert!(is_opml(xml));
        assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));

        let feeds = parse_opml(xml).unwrap();
        assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
        assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));

        // A feed must never be mistaken for a subscription list.
        assert!(!is_opml(include_bytes!("../tests/data/rss2.xml")));
        assert!(!is_opml(include_bytes!("../tests/data/atom.xml")));
    }

    #[test]
    fn an_item_may_carry_several_enclosures() {
        // The rss crate keeps only one per item -- the last -- so these come from the XML.
        let xml = br#"<?xml version="1.0"?>
        <rss version="2.0"><channel><title>M</title><link>https://x</link><description>d</description>
        <item><title>Two files</title><guid>m1</guid>
          <enclosure url="https://x/a.mp3?v=1&amp;t=2" length="111" type="audio/mpeg"/>
          <enclosure url="https://x/b.mp4" length="222" type="video/mp4"/>
        </item>
        <item><title>One file</title><guid>m2</guid>
          <enclosure url="https://x/c.mp3" length="333" type="audio/mpeg"/></item>
        <item><title>None</title><guid>m3</guid></item>
        </channel></rss>"#;
        let f = parse(xml).unwrap();
        assert_eq!(f.entries.len(), 3);

        let two = &f.entries[0].enclosures;
        assert_eq!(two.len(), 2, "both enclosures survive");
        assert_eq!(
            two[0].url, "https://x/a.mp3?v=1&t=2",
            "document order, and the escaped ampersand is decoded"
        );
        assert_eq!(two[0].length, Some(111));
        assert_eq!(two[1].url, "https://x/b.mp4");
        assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));

        assert_eq!(f.entries[1].enclosures.len(), 1);
        assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
    }

    #[test]
    fn durations_parse_from_seconds_or_a_clock() {
        assert_eq!(parse_duration("5649"), Some(5649));
        assert_eq!(parse_duration("23:45"), Some(1425));
        assert_eq!(parse_duration("1:34:09"), Some(5649));
        assert_eq!(parse_duration("0"), None, "zero is not a duration");
        assert_eq!(parse_duration(""), None);
        assert_eq!(parse_duration("garbage"), None);
    }

    #[test]
    fn rejects_html_masquerading_as_a_feed() {
        assert!(parse(b"<html><body>nope</body></html>").is_err());
    }

    #[test]
    fn guid_falls_back_through_link_then_enclosure() {
        assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
        assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
        assert_eq!(pick_guid(None, None, None, None), None);
    }
}