//! Feed fetching and parsing. Replaces FeedData.__getFeed / __getEntries. use anyhow::{Context, Result, anyhow}; use reqwest::StatusCode; use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; use crate::config::Feed as FeedCfg; #[derive(Debug, Default)] pub struct ParsedFeed { pub title: Option, pub ttl_mins: Option, pub image: Option, pub entries: Vec, } #[derive(Debug, Default)] pub struct Entry { pub guid: String, pub title: Option, pub link: Option, pub published: Option, pub description: Option, pub categories: Vec, pub explicit: bool, /// Episode artwork; falls back to the feed's in the UI. pub image: Option, /// Seconds. pub duration: Option, pub episode: Option, pub season: Option, pub enclosures: Vec, } #[derive(Debug, Default, PartialEq)] pub struct Enclosure { pub url: String, pub mime: Option, pub length: Option, } pub enum Fetched { /// Server said 304, or returned a body we already have. NotModified, Body { bytes: Vec, etag: Option, last_modified: Option, }, } /// Conditional GET. reqwest handles gzip and redirects; the original's hand-rolled /// CONNECT/socket.ssl proxy path is gone -- `system-proxy` reads http_proxy/https_proxy. pub async fn fetch( client: &reqwest::Client, cfg: &FeedCfg, etag: Option<&str>, last_modified: Option<&str>, ) -> Result { let mut req = client.get(&cfg.url); if let Some(tag) = etag { req = req.header(IF_NONE_MATCH, tag); } if let Some(lm) = last_modified { req = req.header(IF_MODIFIED_SINCE, lm); } if let Some(user) = &cfg.username { req = req.basic_auth(user, cfg.password()); } let resp = req.send().await.context("connecting")?; if resp.status() == StatusCode::NOT_MODIFIED { return Ok(Fetched::NotModified); } let status = resp.status(); if !status.is_success() { // The original surfaced 401/407 specially; the code is enough for a UI to switch on. return Err(anyhow!("HTTP {status}")); } let header = |h: reqwest::header::HeaderName| { resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned) }; let etag = header(ETAG); let last_modified = header(LAST_MODIFIED); let bytes = resp.bytes().await.context("reading body")?.to_vec(); Ok(Fetched::Body { bytes, etag, last_modified }) } /// True when a body is an OPML document rather than a feed. /// /// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an /// OPML served from a URL without that extension. Sniffing the body catches both. pub fn is_opml(bytes: &[u8]) -> bool { let head = &bytes[..bytes.len().min(1024)]; let text = String::from_utf8_lossy(head).to_lowercase(); text.contains(" Result> { let text = String::from_utf8_lossy(bytes); let doc = opml::OPML::from_str(&text) .map_err(|e| anyhow!("that does not parse as OPML: {e}"))?; let mut out = vec![]; crate::collect_outlines(&doc.body.outlines, &mut out); Ok(out) } /// The of an OPML document. pub fn opml_title(bytes: &[u8]) -> Option<String> { let text = String::from_utf8_lossy(bytes); let doc = opml::OPML::from_str(&text).ok()?; doc.head .and_then(|h| h.title) .map(|t| t.trim().to_owned()) .filter(|t| !t.is_empty()) } /// RSS first, then Atom -- the same split the original made on `parsedFeed.version`. pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> { match rss::Channel::read_from(bytes) { Ok(ch) => Ok(from_rss(ch)), Err(rss_err) => match atom_syndication::Feed::read_from(bytes) { Ok(feed) => Ok(from_atom(feed)), Err(atom_err) => Err(anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})")), }, } } fn from_rss(ch: rss::Channel) -> ParsedFeed { let explicit = ch .itunes_ext() .and_then(|it| it.explicit()) .is_some_and(is_yes); let entries = ch .items() .iter() .filter_map(|item| { let enclosures: Vec<Enclosure> = item .enclosure() .into_iter() .map(|e| Enclosure { url: e.url().trim().to_owned(), mime: non_empty(Some(e.mime_type())), length: e.length().parse().ok(), }) .filter(|e| !e.url.is_empty()) .collect(); let guid = pick_guid( item.guid().map(|g| g.value()), item.link(), enclosures.first().map(|e| e.url.as_str()), item.title(), )?; let entry_explicit = item .itunes_ext() .and_then(|it| it.explicit()) .is_some_and(is_yes); let it = item.itunes_ext(); Some(Entry { guid, title: non_empty(item.title()), link: non_empty(item.link()), published: item.pub_date().and_then(parse_date), // Content wins over description, as __getEntries preferred entry.content. description: non_empty(item.content()).or_else(|| non_empty(item.description())), categories: item .categories() .iter() .map(|c| c.name().to_owned()) .filter(|c| !c.is_empty() && !c.starts_with("http")) .collect(), explicit: explicit || entry_explicit, image: it.and_then(|i| i.image()).map(str::to_owned), duration: it.and_then(|i| i.duration()).and_then(parse_duration), episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()), season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()), enclosures, }) }) .collect(); ParsedFeed { title: non_empty(Some(ch.title())), ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()), // itunes:image is the square artwork; <image><url> is the older, often smaller one. image: ch .itunes_ext() .and_then(|i| i.image()) .map(str::to_owned) .or_else(|| ch.image().map(|i| i.url().to_owned())), entries, } } fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { let entries = feed .entries() .iter() .filter_map(|e| { // Atom carries enclosures as <link rel="enclosure">. let enclosures: Vec<Enclosure> = e .links() .iter() .filter(|l| l.rel() == "enclosure") .map(|l| Enclosure { url: l.href().trim().to_owned(), mime: non_empty(l.mime_type()), length: l.length().and_then(|s| s.parse().ok()), }) .filter(|e| !e.url.is_empty()) .collect(); let alt = e .links() .iter() .find(|l| l.rel() == "alternate" || l.rel().is_empty()) .map(|l| l.href()); let guid = pick_guid( Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()), )?; Some(Entry { guid, title: non_empty(Some(e.title().as_str())), link: alt.map(str::to_owned), published: e.published().or(Some(e.updated())).map(|d| d.timestamp()), description: e .content() .and_then(|c| c.value()) .or_else(|| e.summary().map(|s| s.as_str())) .map(str::to_owned), categories: e.categories().iter().map(|c| c.term().to_owned()).collect(), explicit: false, image: None, duration: None, episode: None, season: None, enclosures, }) }) .collect(); ParsedFeed { title: non_empty(Some(feed.title().as_str())), ttl_mins: None, image: feed.logo().or_else(|| feed.icon()).map(str::to_owned), entries, } } /// The original fell back to hashing the title or description. A guid, permalink or /// enclosure URL is a stable identifier already, so no hashing is needed; an entry with /// none of them has nothing to download and is dropped. fn pick_guid( guid: Option<&str>, link: Option<&str>, enclosure: Option<&str>, title: Option<&str>, ) -> Option<String> { [guid, link, enclosure, title] .into_iter() .flatten() .map(str::trim) .find(|s| !s.is_empty()) .map(str::to_owned) } fn is_yes(s: &str) -> bool { matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit") } fn non_empty(s: Option<&str>) -> Option<String> { s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned) } /// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45"). fn parse_duration(s: &str) -> Option<i64> { let s = s.trim(); if s.is_empty() { return None; } if !s.contains(':') { return s.parse().ok().filter(|n| *n > 0); } let mut total: i64 = 0; for part in s.split(':') { total = total * 60 + part.trim().parse::<i64>().ok()?; } Some(total).filter(|n| *n > 0) } /// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead. fn parse_date(s: &str) -> Option<i64> { let s = s.trim(); chrono::DateTime::parse_from_rfc2822(s) .or_else(|_| chrono::DateTime::parse_from_rfc3339(s)) .ok() .map(|d| d.timestamp()) } #[cfg(test)] mod tests { use super::*; #[test] fn parses_rss_with_itunes_extensions() { let bytes = include_bytes!("../tests/data/rss2.xml"); let feed = parse(bytes).unwrap(); assert_eq!(feed.title.as_deref(), Some("Test Cast")); assert_eq!(feed.ttl_mins, Some(45)); assert_eq!(feed.entries.len(), 3); let ep = &feed.entries[0]; assert_eq!(ep.guid, "https://example.com/ep/1"); assert_eq!(ep.title.as_deref(), Some("Episode One")); assert_eq!(ep.published, Some(1_078_016_400)); assert_eq!(ep.categories, vec!["Tech"]); assert!(!ep.explicit); assert_eq!( ep.enclosures, vec![Enclosure { url: "https://example.com/ep1.mp3".into(), mime: Some("audio/mpeg".into()), length: Some(12_345_678), }] ); assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes"); assert_eq!( feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent") ); } #[test] fn feed_level_explicit_overrides_entries() { let xml = br#"<?xml version="1.0"?> <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"> <channel><title>Xhttps://xd yes ag1 "#; let feed = parse(xml).unwrap(); assert!( feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit" ); } #[test] fn parses_atom_enclosure_links() { let bytes = include_bytes!("../tests/data/atom.xml"); let feed = parse(bytes).unwrap(); assert_eq!(feed.title.as_deref(), Some("Atom Cast")); assert_eq!(feed.entries.len(), 1); let ep = &feed.entries[0]; assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a"); assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1")); assert_eq!(ep.published, Some(1_078_016_400)); assert_eq!( ep.enclosures, vec![Enclosure { url: "https://example.org/ep1.m4a".into(), mime: Some("audio/mp4".into()), length: Some(9_876_543), }], "rel=enclosure only; the alternate link must not become an enclosure" ); } #[test] fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() { // Some feeds set a different itunes:title. The displayed title is always the RSS // , verbatim -- separators and all -- and season/episode are stored // alongside it rather than folded into it. let xml = br#"<?xml version="1.0"?> <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"> <channel><title>Showhttps://xd Music from a Darkened Room | Session Zero Session Zero sz 8 6720 Music from a Darkened Room Part 1 | Murphy's Drawer p1 81 "#; let feed = parse(xml).unwrap(); let sz = &feed.entries[0]; assert_eq!( sz.title.as_deref(), Some("Music from a Darkened Room | Session Zero"), "itunes:title must not override the RSS title" ); assert_eq!(sz.season, Some(8)); assert_eq!(sz.episode, None, "a missing episode number stays missing"); assert_eq!(sz.duration, Some(6720)); let p1 = &feed.entries[1]; assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer")); assert_eq!((p1.season, p1.episode), (Some(8), Some(1))); } #[test] fn opml_is_recognised_and_its_feeds_listed() { let xml = br#"My Subscriptions "#; assert!(is_opml(xml)); assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions")); let feeds = parse_opml(xml).unwrap(); assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped"); assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into())); // A feed must never be mistaken for a subscription list. assert!(!is_opml(include_bytes!("../tests/data/rss2.xml"))); assert!(!is_opml(include_bytes!("../tests/data/atom.xml"))); } #[test] fn durations_parse_from_seconds_or_a_clock() { assert_eq!(parse_duration("5649"), Some(5649)); assert_eq!(parse_duration("23:45"), Some(1425)); assert_eq!(parse_duration("1:34:09"), Some(5649)); assert_eq!(parse_duration("0"), None, "zero is not a duration"); assert_eq!(parse_duration(""), None); assert_eq!(parse_duration("garbage"), None); } #[test] fn rejects_html_masquerading_as_a_feed() { assert!(parse(b"nope").is_err()); } #[test] fn guid_falls_back_through_link_then_enclosure() { assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l")); assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e")); assert_eq!(pick_guid(None, None, None, None), None); } }