rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c86d6983637a7d048f1d9af3c67484aa017ff0c5
ipodderx-rs/src/feed.rs
rays c86d698363 Subscribe to an OPML, not just import one 
A feed whose body sniffs as OPML is treated as a subscription list and
re-read on every scan, as iPodderX did. Listed feeds become real config
entries grouped under it, inherit its settings, land in one nested folder,
and are scanned in the same run.

When a feed leaves the OPML: removed if nothing was downloaded, kept and
flagged otherwise, so a downloaded file is never orphaned.

folder_for sanitized the whole folder string and would have flattened the
nesting; each segment is sanitized separately now, and a traversal still
cannot escape the download directory. Db::memory() also runs migrate(),
which it did not, so a migration-only column passed tests while missing in
production.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
2026-09-10 15:01:03 +00:00

467 lines
16 KiB
Rust
Raw Blame History

//! Feed fetching and parsing. Replaces FeedData.__getFeed / __getEntries.
use anyhow::{Context, Result, anyhow};
use reqwest::StatusCode;
use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};
use crate::config::Feed as FeedCfg;
#[derive(Debug, Default)]
pub struct ParsedFeed {
pub title: Option<String>,
pub ttl_mins: Option<u64>,
pub image: Option<String>,
pub entries: Vec<Entry>,
}
#[derive(Debug, Default)]
pub struct Entry {
pub guid: String,
pub title: Option<String>,
pub link: Option<String>,
pub published: Option<i64>,
pub description: Option<String>,
pub categories: Vec<String>,
pub explicit: bool,
/// Episode artwork; falls back to the feed's in the UI.
pub image: Option<String>,
/// Seconds.
pub duration: Option<i64>,
pub episode: Option<i64>,
pub season: Option<i64>,
pub enclosures: Vec<Enclosure>,
}
#[derive(Debug, Default, PartialEq)]
pub struct Enclosure {
pub url: String,
pub mime: Option<String>,
pub length: Option<i64>,
}
pub enum Fetched {
/// Server said 304, or returned a body we already have.
NotModified,
Body {
bytes: Vec<u8>,
etag: Option<String>,
last_modified: Option<String>,
},
}
/// Conditional GET. reqwest handles gzip and redirects; the original's hand-rolled
/// CONNECT/socket.ssl proxy path is gone -- `system-proxy` reads http_proxy/https_proxy.
pub async fn fetch(
client: &reqwest::Client,
cfg: &FeedCfg,
etag: Option<&str>,
last_modified: Option<&str>,
) -> Result<Fetched> {
let mut req = client.get(&cfg.url);
if let Some(tag) = etag {
req = req.header(IF_NONE_MATCH, tag);
}
if let Some(lm) = last_modified {
req = req.header(IF_MODIFIED_SINCE, lm);
}
if let Some(user) = &cfg.username {
req = req.basic_auth(user, cfg.password());
}
let resp = req.send().await.context("connecting")?;
if resp.status() == StatusCode::NOT_MODIFIED {
return Ok(Fetched::NotModified);
}
let status = resp.status();
if !status.is_success() {
// The original surfaced 401/407 specially; the code is enough for a UI to switch on.
return Err(anyhow!("HTTP {status}"));
}
let header = |h: reqwest::header::HeaderName| {
resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned)
};
let etag = header(ETAG);
let last_modified = header(LAST_MODIFIED);
let bytes = resp.bytes().await.context("reading body")?.to_vec();
Ok(Fetched::Body { bytes, etag, last_modified })
}
/// True when a body is an OPML document rather than a feed.
///
/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
/// OPML served from a URL without that extension. Sniffing the body catches both.
pub fn is_opml(bytes: &[u8]) -> bool {
let head = &bytes[..bytes.len().min(1024)];
let text = String::from_utf8_lossy(head).to_lowercase();
text.contains("<opml")
}
/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text)
.map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
let mut out = vec![];
crate::collect_outlines(&doc.body.outlines, &mut out);
Ok(out)
}
/// The <head><title> of an OPML document.
pub fn opml_title(bytes: &[u8]) -> Option<String> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text).ok()?;
doc.head
.and_then(|h| h.title)
.map(|t| t.trim().to_owned())
.filter(|t| !t.is_empty())
}
/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
match rss::Channel::read_from(bytes) {
Ok(ch) => Ok(from_rss(ch)),
Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
Ok(feed) => Ok(from_atom(feed)),
Err(atom_err) => Err(anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})")),
},
}
}
fn from_rss(ch: rss::Channel) -> ParsedFeed {
let explicit = ch
.itunes_ext()
.and_then(|it| it.explicit())
.is_some_and(is_yes);
let entries = ch
.items()
.iter()
.filter_map(|item| {
let enclosures: Vec<Enclosure> = item
.enclosure()
.into_iter()
.map(|e| Enclosure {
url: e.url().trim().to_owned(),
mime: non_empty(Some(e.mime_type())),
length: e.length().parse().ok(),
})
.filter(|e| !e.url.is_empty())
.collect();
let guid = pick_guid(
item.guid().map(|g| g.value()),
item.link(),
enclosures.first().map(|e| e.url.as_str()),
item.title(),
)?;
let entry_explicit = item
.itunes_ext()
.and_then(|it| it.explicit())
.is_some_and(is_yes);
let it = item.itunes_ext();
Some(Entry {
guid,
title: non_empty(item.title()),
link: non_empty(item.link()),
published: item.pub_date().and_then(parse_date),
// Content wins over description, as __getEntries preferred entry.content.
description: non_empty(item.content()).or_else(|| non_empty(item.description())),
categories: item
.categories()
.iter()
.map(|c| c.name().to_owned())
.filter(|c| !c.is_empty() && !c.starts_with("http"))
.collect(),
explicit: explicit || entry_explicit,
image: it.and_then(|i| i.image()).map(str::to_owned),
duration: it.and_then(|i| i.duration()).and_then(parse_duration),
episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
enclosures,
})
})
.collect();
ParsedFeed {
title: non_empty(Some(ch.title())),
ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
// itunes:image is the square artwork; <image><url> is the older, often smaller one.
image: ch
.itunes_ext()
.and_then(|i| i.image())
.map(str::to_owned)
.or_else(|| ch.image().map(|i| i.url().to_owned())),
entries,
}
}
fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
let entries = feed
.entries()
.iter()
.filter_map(|e| {
// Atom carries enclosures as <link rel="enclosure">.
let enclosures: Vec<Enclosure> = e
.links()
.iter()
.filter(|l| l.rel() == "enclosure")
.map(|l| Enclosure {
url: l.href().trim().to_owned(),
mime: non_empty(l.mime_type()),
length: l.length().and_then(|s| s.parse().ok()),
})
.filter(|e| !e.url.is_empty())
.collect();
let alt = e
.links()
.iter()
.find(|l| l.rel() == "alternate" || l.rel().is_empty())
.map(|l| l.href());
let guid = pick_guid(
Some(e.id()),
alt,
enclosures.first().map(|x| x.url.as_str()),
Some(e.title().as_str()),
)?;
Some(Entry {
guid,
title: non_empty(Some(e.title().as_str())),
link: alt.map(str::to_owned),
published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
description: e
.content()
.and_then(|c| c.value())
.or_else(|| e.summary().map(|s| s.as_str()))
.map(str::to_owned),
categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
explicit: false,
image: None,
duration: None,
episode: None,
season: None,
enclosures,
})
})
.collect();
ParsedFeed {
title: non_empty(Some(feed.title().as_str())),
ttl_mins: None,
image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
entries,
}
}
/// The original fell back to hashing the title or description. A guid, permalink or
/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
/// none of them has nothing to download and is dropped.
fn pick_guid(
guid: Option<&str>,
link: Option<&str>,
enclosure: Option<&str>,
title: Option<&str>,
) -> Option<String> {
[guid, link, enclosure, title]
.into_iter()
.flatten()
.map(str::trim)
.find(|s| !s.is_empty())
.map(str::to_owned)
}
fn is_yes(s: &str) -> bool {
matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
}
fn non_empty(s: Option<&str>) -> Option<String> {
s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
}
/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
fn parse_duration(s: &str) -> Option<i64> {
let s = s.trim();
if s.is_empty() {
return None;
}
if !s.contains(':') {
return s.parse().ok().filter(|n| *n > 0);
}
let mut total: i64 = 0;
for part in s.split(':') {
total = total * 60 + part.trim().parse::<i64>().ok()?;
}
Some(total).filter(|n| *n > 0)
}
/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
fn parse_date(s: &str) -> Option<i64> {
let s = s.trim();
chrono::DateTime::parse_from_rfc2822(s)
.or_else(|_| chrono::DateTime::parse_from_rfc3339(s))
.ok()
.map(|d| d.timestamp())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_rss_with_itunes_extensions() {
let bytes = include_bytes!("../tests/data/rss2.xml");
let feed = parse(bytes).unwrap();
assert_eq!(feed.title.as_deref(), Some("Test Cast"));
assert_eq!(feed.ttl_mins, Some(45));
assert_eq!(feed.entries.len(), 3);
let ep = &feed.entries[0];
assert_eq!(ep.guid, "https://example.com/ep/1");
assert_eq!(ep.title.as_deref(), Some("Episode One"));
assert_eq!(ep.published, Some(1_078_016_400));
assert_eq!(ep.categories, vec!["Tech"]);
assert!(!ep.explicit);
assert_eq!(
ep.enclosures,
vec![Enclosure {
url: "https://example.com/ep1.mp3".into(),
mime: Some("audio/mpeg".into()),
length: Some(12_345_678),
}]
);
assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
assert_eq!(
feed.entries[2].enclosures[0].mime.as_deref(),
Some("application/x-bittorrent")
);
}
#[test]
fn feed_level_explicit_overrides_entries() {
let xml = br#"<?xml version="1.0"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel><title>X</title><link>https://x</link><description>d</description>
<itunes:explicit>yes</itunes:explicit>
<item><title>a</title><guid>g1</guid>
<enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
</channel></rss>"#;
let feed = parse(xml).unwrap();
assert!(
feed.entries[0].explicit,
"the entry says nothing; the feed-level flag must still mark it explicit"
);
}
#[test]
fn parses_atom_enclosure_links() {
let bytes = include_bytes!("../tests/data/atom.xml");
let feed = parse(bytes).unwrap();
assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
assert_eq!(feed.entries.len(), 1);
let ep = &feed.entries[0];
assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
assert_eq!(ep.published, Some(1_078_016_400));
assert_eq!(
ep.enclosures,
vec![Enclosure {
url: "https://example.org/ep1.m4a".into(),
mime: Some("audio/mp4".into()),
length: Some(9_876_543),
}],
"rel=enclosure only; the alternate link must not become an enclosure"
);
}
#[test]
fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
// Some feeds set a different itunes:title. The displayed title is always the RSS
// <title>, verbatim -- separators and all -- and season/episode are stored
// alongside it rather than folded into it.
let xml = br#"<?xml version="1.0"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel><title>Show</title><link>https://x</link><description>d</description>
<item>
<title>Music from a Darkened Room | Session Zero</title>
<itunes:title>Session Zero</itunes:title>
<guid>sz</guid>
<itunes:season>8</itunes:season>
<itunes:duration>6720</itunes:duration>
<enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
</item>
<item>
<title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
<guid>p1</guid>
<itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
<enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
</item>
</channel></rss>"#;
let feed = parse(xml).unwrap();
let sz = &feed.entries[0];
assert_eq!(
sz.title.as_deref(),
Some("Music from a Darkened Room | Session Zero"),
"itunes:title must not override the RSS title"
);
assert_eq!(sz.season, Some(8));
assert_eq!(sz.episode, None, "a missing episode number stays missing");
assert_eq!(sz.duration, Some(6720));
let p1 = &feed.entries[1];
assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
}
#[test]
fn opml_is_recognised_and_its_feeds_listed() {
let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
<outline text="Folder">
<outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
<outline type="rss" text="Beta" xmlUrl="https://b.example/rss"/>
</outline>
<outline text="Not a feed"/>
</body></opml>"#;
assert!(is_opml(xml));
assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));
let feeds = parse_opml(xml).unwrap();
assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));
// A feed must never be mistaken for a subscription list.
assert!(!is_opml(include_bytes!("../tests/data/rss2.xml")));
assert!(!is_opml(include_bytes!("../tests/data/atom.xml")));
}
#[test]
fn durations_parse_from_seconds_or_a_clock() {
assert_eq!(parse_duration("5649"), Some(5649));
assert_eq!(parse_duration("23:45"), Some(1425));
assert_eq!(parse_duration("1:34:09"), Some(5649));
assert_eq!(parse_duration("0"), None, "zero is not a duration");
assert_eq!(parse_duration(""), None);
assert_eq!(parse_duration("garbage"), None);
}
#[test]
fn rejects_html_masquerading_as_a_feed() {
assert!(parse(b"<html><body>nope</body></html>").is_err());
}
#[test]
fn guid_falls_back_through_link_then_enclosure() {
assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
assert_eq!(pick_guid(None, None, None, None), None);
}
}
Reference in New Issue View Git Blame Copy Permalink