From ff5d786602c5a2ec4940cea18463d44bbe87a819 Mon Sep 17 00:00:00 2001 From: rays Date: Sat, 24 Oct 2026 01:41:14 +0000 Subject: [PATCH] ipx-feed's lib.rs splits into leaf modules (#356) lib.rs held 2,000 lines: fetching, parsing, OPML, Patreon, discovery, artwork, categories, Jev and redaction, and their tests. It is now the module list and the re-exports, so ipx and the directory call feed::fetch, feed::parse and the rest as before: fetch.rs fetch and its limits parse.rs RSS and Atom outlines.rs OPML (not opml.rs, which would shadow the opml crate) patreon.rs a creator's shows discover.rs finding a feed wordpress.rs WordPress.com's API failure.rs failures in words privacy.rs redact, looks_private artwork.rs https art, icons category.rs Apple's categories jev.rs Jev's category Moved verbatim, private items pub(crate) where a sibling calls them, calls across modules module::function. Every definition and doc comment line of the old lib.rs is in exactly one of the new files; each test went with what it tests, and all 40 run. Co-Authored-By: Claude Opus 5.5 --- crates/feed/src/artwork.rs | 117 ++ crates/feed/src/category.rs | 125 ++ crates/feed/src/discover.rs | 107 ++ crates/feed/src/failure.rs | 145 +++ crates/feed/src/fetch.rs | 189 +++ crates/feed/src/jev.rs | 67 ++ crates/feed/src/lib.rs | 2106 +--------------------------------- crates/feed/src/outlines.rs | 111 ++ crates/feed/src/parse.rs | 980 ++++++++++++++++ crates/feed/src/patreon.rs | 147 +++ crates/feed/src/privacy.rs | 105 ++ crates/feed/src/wordpress.rs | 86 ++ docs/architecture.md | 14 +- 13 files changed, 2214 insertions(+), 2085 deletions(-) create mode 100644 crates/feed/src/artwork.rs create mode 100644 crates/feed/src/category.rs create mode 100644 crates/feed/src/discover.rs create mode 100644 crates/feed/src/failure.rs create mode 100644 crates/feed/src/fetch.rs create mode 100644 crates/feed/src/jev.rs create mode 100644 crates/feed/src/outlines.rs create mode 100644 crates/feed/src/parse.rs create mode 100644 crates/feed/src/patreon.rs create mode 100644 crates/feed/src/privacy.rs create mode 100644 crates/feed/src/wordpress.rs diff --git a/crates/feed/src/artwork.rs b/crates/feed/src/artwork.rs new file mode 100644 index 0000000..651ce52 --- /dev/null +++ b/crates/feed/src/artwork.rs @@ -0,0 +1,117 @@ +//! Artwork: on https when its host serves it there, and a site icon where a feed has none. + +use crate::{discover, fetch}; +use std::collections::HashMap; +use std::time::Duration; + +/// Artwork for a feed that has none: the icon its site's page names, or else the site's +/// `/favicon.ico`. None if neither is there. +#[tracing::instrument(skip_all, fields(site = site))] +pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option { + let timeout = Duration::from_secs(20); + let resp = client.get(site).timeout(timeout).send().await.ok()?; + // Relative to where the page ended up, not where it was asked for: a site that redirects + // to /en/ would otherwise have its icon looked for in the wrong place. + let base = resp.url().clone(); + // The icon a page names can be gone: antirez.com names /images/favicon.png, a 404, while its + // /favicon.ico is there. Stored unchecked, it was a broken image that was never looked up again. + if resp.status().is_success() + && let Ok(page) = fetch::read_capped(resp).await + && let Some(href) = page_icon(&page) + && let Ok(url) = base.join(&href) + && is_image(client, url.as_str()).await + { + return Some(url.into()); + } + let ico = base.join("/favicon.ico").ok()?; + is_image(client, ico.as_str()).await.then(|| ico.into()) +} + +/// How many hosts one scan of a feed asks whether its artwork is on https. +pub(crate) const MAX_ART_HOSTS: usize = 8; + +/// Artwork's address on https when its host serves it there, else as it was. The page is https +/// and must not load http; the host is asked once per `known` (one feed's read), and an http +/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five +/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN +/// presents another name's certificate (#110). +pub async fn prefer_https(client: &reqwest::Client, url: &str, known: &mut HashMap) -> String { + let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() }; + let host = rest.split('/').next().unwrap_or("").to_owned(); + let secure = format!("https://{rest}"); + let ok = match known.get(&host) { + Some(ok) => *ok, + // Each new host is a probe of up to 20 s, one after another on the scan, and the feed's + // author decides how many hosts there are: ports and user names make as many as they like + // of one server (#339). Real feeds put their artwork on a few. + None if known.len() >= MAX_ART_HOSTS => false, + None => { + let ok = is_image(client, &secure).await; + known.insert(host, ok); + ok + } + }; + if ok { secure } else { url.to_owned() } +} + +/// Whether `url` answers with an image. A site with no favicon often answers 200 with its home +/// page, which is not an icon. +pub async fn is_image(client: &reqwest::Client, url: &str) -> bool { + let timeout = Duration::from_secs(20); + let Ok(resp) = client.get(url).timeout(timeout).send().await else { return false }; + resp.status().is_success() + && resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .is_some_and(|t| t.starts_with("image/")) +} + +/// The icon a web page names in its `` tags, the larger apple-touch-icon first: a plain +/// `icon` is often 16 pixels, which blurs at the size the list draws artwork. +pub(crate) fn page_icon(bytes: &[u8]) -> Option { + let text = String::from_utf8_lossy(bytes); + // ASCII only, so byte offsets in the lowered copy stay valid in the original. + let lower = text.to_ascii_lowercase(); + let (mut touch, mut icon) = (None, None); + let mut pos = 0; + while let Some(at) = lower[pos..].find("').map(|e| start + e) else { break }; + pos = end + 1; + let tag = &text[start..end]; + let Some(rel) = discover::tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue }; + let rels: Vec<&str> = rel.split_whitespace().collect(); + if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) { + touch = discover::tag_attr(tag, "href"); + } else if icon.is_none() && rels.contains(&"icon") { + icon = discover::tag_attr(tag, "href"); + } + } + touch.or(icon).filter(|h| !h.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn artwork_on_more_hosts_than_the_limit_is_not_asked_about() { + let mut known: std::collections::HashMap = + (0..MAX_ART_HOSTS).map(|n| (format!("h{n}:80"), false)).collect(); + // No server answers here: were it asked, this would wait on a connection, not return. + let url = "http://192.0.2.1:9/a.png"; + assert_eq!(prefer_https(&reqwest::Client::new(), url, &mut known).await, url); + assert_eq!(known.len(), MAX_ART_HOSTS); + } + + #[test] + fn page_icon_prefers_the_touch_icon() { + let page = br#" + + "#; + assert_eq!(page_icon(page).as_deref(), Some("/touch.png")); + assert_eq!(page_icon(br#""#).as_deref(), Some("i.svg")); + assert_eq!(page_icon(br#""#), None); + } +} diff --git a/crates/feed/src/category.rs b/crates/feed/src/category.rs new file mode 100644 index 0000000..bf00fbd --- /dev/null +++ b/crates/feed/src/category.rs @@ -0,0 +1,125 @@ +//! Apple Podcasts' categories, and where a feed's category sits among them (#118). + +/// Apple's podcast categories, each with its subcategories +/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own +/// `` comes from, and a fixed one, so the Directory's categories stay few. +pub(crate) const CATEGORIES: &[(&str, &[&str])] = &[ + ("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]), + ("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]), + ("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]), + ("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]), + ("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]), + ("Government", &[]), + ("History", &[]), + ("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]), + ("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]), + ( + "Leisure", + &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"], + ), + ("Music", &["Music Commentary", "Music History", "Music Interviews"]), + ( + "News", + &[ + "Business News", + "Daily News", + "Entertainment News", + "News Commentary", + "Politics", + "Sports News", + "Tech News", + ], + ), + ( + "Religion & Spirituality", + &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"], + ), + ( + "Science", + &[ + "Astronomy", + "Chemistry", + "Earth Sciences", + "Life Sciences", + "Mathematics", + "Natural Sciences", + "Nature", + "Physics", + "Social Sciences", + ], + ), + ("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]), + ( + "Sports", + &[ + "Baseball", + "Basketball", + "Cricket", + "Fantasy Sports", + "Football", + "Golf", + "Hockey", + "Rugby", + "Running", + "Soccer", + "Swimming", + "Tennis", + "Volleyball", + "Wilderness", + "Wrestling", + ], + ), + ("Technology", &[]), + ("True Crime", &[]), + ("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]), +]; + +/// Names that are not in Apple's list but mean one that is (#125). Jev picked from this list when +/// it spelt Football and Soccer its own way, and stored those answers; Software How-To is Apple's +/// Technology subcategory from before its 2019 list, which older feeds still name. +pub(crate) const ALIASES: &[(&str, &str)] = + &[("American Football", "Football"), ("Football (Soccer)", "Soccer"), ("Software How-To", "Technology")]; + +/// Apple's categories as ipx has them, for the directory to check against Apple's own (#252). +#[must_use] +pub fn categories() -> &'static [(&'static str, &'static [&'static str])] { + CATEGORIES +} + +/// Where a category sits in Apple's list, as (category, subcategory), so the Directory browses a +/// category and then its subcategories as Apple's does (#118). A podcast's own is stored as its +/// subcategory, and one an admin typed that is not Apple's stands as a category of its own. +#[must_use] +pub fn category_path(name: &str) -> (String, Option) { + // A feed's own text, which can end in a stray comma: "Soccer," was a category of its own. + let name = name.trim().trim_end_matches([',', ';', '.']).trim_end(); + let name = ALIASES.iter().find(|(old, _)| old.eq_ignore_ascii_case(name)).map_or(name, |(_, now)| now); + for (category, subs) in CATEGORIES { + if category.eq_ignore_ascii_case(name) { + return (category.to_string(), None); + } + if let Some(sub) = subs.iter().find(|s| s.eq_ignore_ascii_case(name)) { + return (category.to_string(), Some(sub.to_string())); + } + } + (name.to_string(), None) +} + +#[cfg(test)] +mod tests { + + #[test] + fn category_path_finds_a_subcategory_s_category() { + use super::category_path; + assert_eq!(category_path("Tech News"), ("News".into(), Some("Tech News".into()))); + assert_eq!(category_path("video games"), ("Leisure".into(), Some("Video Games".into())), "Apple's spelling"); + assert_eq!(category_path("Technology"), ("Technology".into(), None)); + assert_eq!(category_path("Homebrew"), ("Homebrew".into(), None), "not Apple's: its own category"); + // Apple's spelling, an old answer of Jev's, a stray comma and Apple's old list (#125). + assert_eq!(category_path("Soccer"), ("Sports".into(), Some("Soccer".into()))); + assert_eq!(category_path("Football (Soccer)"), ("Sports".into(), Some("Soccer".into()))); + assert_eq!(category_path("Soccer,"), ("Sports".into(), Some("Soccer".into()))); + assert_eq!(category_path("Football"), ("Sports".into(), Some("Football".into()))); + assert_eq!(category_path("Software How-To"), ("Technology".into(), None)); + } +} diff --git a/crates/feed/src/discover.rs b/crates/feed/src/discover.rs new file mode 100644 index 0000000..aecd876 --- /dev/null +++ b/crates/feed/src/discover.rs @@ -0,0 +1,107 @@ +//! Finding a feed from what someone typed: a feed address, a site, or a page linking one. + +use crate::{fetch, outlines, parse}; +use anyhow::{Context, Result}; +use std::time::Duration; + +/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's +/// feed, since the token alone says whose it is. An address with no scheme is https: 'cnn.com' +/// was stored as typed and every check failed with "relative URL without a base" (#101). +#[must_use] +pub fn expand_input(input: &str) -> String { + let s = input.trim(); + let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if token { + format!("https://www.patreon.com/rss?auth={s}") + } else if let Some(rest) = s.strip_prefix("//") { + format!("https://{rest}") + } else if !s.contains("://") { + format!("https://{s}") + } else { + s.to_owned() + } +} + +/// The feed a web page names as its own via `` (or the Atom equivalent) -- how the new address was found for om.co, ms.now, +/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos. +pub(crate) fn alternate_feed_link(bytes: &[u8]) -> Option { + let text = String::from_utf8_lossy(bytes); + // ASCII only: to_lowercase changes some characters' length (U+0130 grows a byte), and the + // offsets found in the lowered copy then sliced the original off a char boundary. + let lower = text.to_ascii_lowercase(); + let mut pos = 0; + while let Some(rel) = lower[pos..].find("').map(|e| start + e) else { break }; + pos = end + 1; + let tag = &text[start..end]; + let tag_lower = &lower[start..end]; + let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'"); + let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml"); + if is_alternate + && is_feed_type + && let Some(href) = tag_attr(tag, "href") + { + return Some(href); + } + } + None +} + +/// What someone asked to add, as a feed: the address itself when it is a feed or an OPML list, +/// otherwise the feed its web page links as its own, otherwise an error and nothing is added. +/// A page that linked no feed used to be added as it was and failed on every check, called a +/// feed that moved (#102); cnn.com is one. +pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result { + let read = |u: String| async move { + let resp = client.get(&u).timeout(Duration::from_secs(30)).send().await.context("connecting")?; + anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status()); + let base = resp.url().clone(); + anyhow::Ok((base, fetch::read_capped(resp).await.context("reading")?)) + }; + let is_feed = |b: &[u8]| outlines::is_opml(b) || parse::parse(b).is_ok(); + let (base, body) = read(url.to_owned()).await.with_context(|| format!("could not read {url}"))?; + if is_feed(&body) { + return Ok(url.to_owned()); + } + if !parse::looks_like_html(&body) { + let why = parse::parse(&body).err().map(|e| format!("{e:#}")).unwrap_or_default(); + anyhow::bail!("{url} is not a feed: {why}"); + } + let Some(href) = alternate_feed_link(&body) else { + anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to"); + }; + let linked = + base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?; + let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?; + anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either"); + Ok(linked.into()) +} + +/// The value of one attribute in an HTML/XML start tag, however it is quoted. +pub(crate) fn tag_attr(tag: &str, name: &str) -> Option { + let key = format!("{name}="); + let idx = tag.to_lowercase().find(&key)?; + let after = &tag[idx + key.len()..]; + let quote = after.chars().next()?; + if quote != '"' && quote != '\'' { + return None; + } + let rest = &after[1..]; + let close = rest.find(quote)?; + Some(rest[..close].trim().to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_feed_link_after_non_ascii_text_is_found() { + // U+0130 lowercases to three bytes from two, which once shifted every offset after it. + let page = "\u{130}stanbul \u{130}\u{130}\ + "; + assert_eq!(alternate_feed_link(page.as_bytes()).as_deref(), Some("/feed.xml")); + } +} diff --git a/crates/feed/src/failure.rs b/crates/feed/src/failure.rs new file mode 100644 index 0000000..7ddde4d --- /dev/null +++ b/crates/feed/src/failure.rs @@ -0,0 +1,145 @@ +//! A failed read, in words for whoever subscribes, and as a kind for the log. + +/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem +/// it is, and whether there is a new address to switch to. +pub struct Failure { + /// What to tell them, in a sentence. + pub reason: &'static str, + /// Where the feed went, when the error says. + pub new_url: Option, +} + +/// A failure's kind, for the log's `error.type` (#91): the HTTP status where there is one, as +/// OpenTelemetry names an HTTP error, and otherwise a word for what went wrong. Matches the +/// same wording as `explain_failure`; a message it does not know is "other", never a wrong kind. +#[must_use] +pub fn failure_kind(msg: &str) -> (String, Option) { + let low = msg.to_ascii_lowercase(); + let code = low.split("http ").skip(1).find_map(|r| r.get(..3)?.parse::().ok()); + if let Some(c) = code.filter(|c| (100..600).contains(c)) { + return (c.to_string(), Some(c)); + } + let kind = if low.contains("dns error") + || low.contains("failed to lookup address") + || low.contains("no address associated") + { + "dns" + } else if low.contains("too many redirects") { + "redirect_loop" + } else if low.contains("timed out") || low.contains("timeout") { + "timeout" + } else if low.contains("certificate") || low.contains("tls") { + "tls" + } else if low.contains("got a web page") { + "not_a_feed" + } else if low.contains("the site sent ") { + "site_message" + } else if low.contains("connect") { + "connect" + } else if low.contains("pars") { + "parse" + } else { + "other" + }; + (kind.into(), None) +} + +/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the +/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling +/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled -- +/// comes back `None`: transient by nature, or with nothing more useful to say than the raw +/// text already shown once a feed is open. +/// +/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP +/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure +/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing +/// nothing extra, so a miss costs a clearer message, not a wrong one. +pub fn explain_failure(msg: &str) -> Option { + if let Some(rest) = msg.strip_prefix("got a web page, not a feed") { + let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned); + return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url }); + } + if msg.contains("the site sent ") { + return Some(Failure { + reason: "The site sent a message instead of the feed; the publisher has to fix it.", + new_url: None, + }); + } + let low = msg.to_ascii_lowercase(); + if low.contains("http 404") { + return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None }); + } + if low.contains("http 401") || low.contains("http 403") { + return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None }); + } + if low.contains("answered with nothing") { + return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None }); + } + if low.contains("http 402") { + return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None }); + } + // Only a name that does not exist. A lookup that failed for the moment says nothing about the + // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143). + if !is_temporary_lookup_failure(msg) + && (low.contains("dns error") + || low.contains("failed to lookup address") + || low.contains("no address associated")) + { + return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None }); + } + None +} + +/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's +/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does +/// not exist ("Name or service not known"). It says the network was not answering, not that the +/// feed is gone (#143). +#[must_use] +pub fn is_temporary_lookup_failure(msg: &str) -> bool { + let low = msg.to_ascii_lowercase(); + low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn explain_failure_translates_the_errors_the_ui_should_flag() { + assert_eq!( + explain_failure("HTTP 404 Not Found").unwrap().reason, + "The publisher took this feed down, or moved it." + ); + assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests."); + assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests."); + assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan."); + assert_eq!( + explain_failure("the feed's address answered with nothing at all").unwrap().reason, + "This address answers with nothing; the site may be gone." + ); + let dns = + explain_failure("connecting: dns error: failed to lookup address information: Name or service not known") + .unwrap(); + assert_eq!(dns.reason, "This address no longer resolves; the site may be gone."); + // A lookup that failed for the moment is not a site gone (#143). + let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution"; + assert!(is_temporary_lookup_failure(blip)); + assert!(explain_failure(blip).is_none()); + assert!(!is_temporary_lookup_failure( + "connecting: dns error: failed to lookup address information: Name or service not known" + )); + let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap(); + assert_eq!(moved.new_url.as_deref(), Some("https://x/feed")); + assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none()); + let down = explain_failure("the site sent \"Unable to establish a DB connection\" instead of a feed").unwrap(); + assert_eq!(down.reason, "The site sent a message instead of the feed; the publisher has to fix it."); + for transient in [ + "HTTP 500 Internal Server Error", + "HTTP 429 Too Many Requests", + "operation timed out", + "not RSS (reached end of input without finding a complete channel) and not Atom (unexpected end of input)", + ] { + assert!(explain_failure(transient).is_none(), "{transient} must not be flagged"); + } + } +} diff --git a/crates/feed/src/fetch.rs b/crates/feed/src/fetch.rs new file mode 100644 index 0000000..8ed50de --- /dev/null +++ b/crates/feed/src/fetch.rs @@ -0,0 +1,189 @@ +//! Reading a feed over HTTP: conditional requests, redirects followed one by one, bounded bodies. + +use crate::{privacy, wordpress}; +use anyhow::{Context, Result, anyhow}; +use reqwest::StatusCode; +use reqwest::header::HeaderName; +use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; +use std::sync::atomic::AtomicU64; +use std::time::Duration; + +/// What `fetch` got. +pub enum Fetched { + /// Server said 304, or returned a body we already have. + NotModified, + /// A body to read, with the validators to send next time. + Body { bytes: Vec, etag: Option, last_modified: Option }, +} + +/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and +/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504 +/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small; +/// downloads, which are not, have no such limit. +pub const FEED_TIMEOUT: Duration = Duration::from_secs(30); + +/// The longest a feed's server may send nothing, for the clients that fetch feeds to set: one +/// that took the connection and never answered ran the whole 30 s, every scan it was in (#230). +/// No feed that answered took over 8 s in a day of traces; one that keeps sending, however +/// slowly, still has until FEED_TIMEOUT. +pub const FEED_SILENCE: Duration = Duration::from_secs(15); + +/// Feeds fetched and the bytes of their bodies, since the process started, for /metrics to set +/// beside memory (#224): a body is read whole, so a night of large feeds is a night of large +/// allocations. +pub static FETCHES: AtomicU64 = AtomicU64::new(0); + +/// See `FETCHES`. +pub static FETCHED_BYTES: AtomicU64 = AtomicU64::new(0); + +/// The most of a feed or page read. The largest feeds here are a few MB; without a limit an +/// episode's MP3 given as a feed was read whole into memory, at every check (#295). +pub(crate) const MAX_BODY: usize = 50 << 20; + +/// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read. +pub(crate) async fn read_capped(mut resp: reqwest::Response) -> Result> { + if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) { + return Err(anyhow!("larger than {} MB, too large to be a feed", MAX_BODY >> 20)); + } + let mut out = Vec::new(); + while let Some(c) = resp.chunk().await? { + out.extend_from_slice(&c); + anyhow::ensure!(out.len() <= MAX_BODY, "larger than {} MB, too large to be a feed", MAX_BODY >> 20); + } + Ok(out) +} + +/// Reads a feed, asking only for what changed since `etag` and `last_modified`. Redirects are +/// followed here, up to ten, so `auth` goes to the feed's own host alone. Also returns the new +/// address when every hop said the feed moved for good, for the caller to keep instead. +#[tracing::instrument(skip_all, fields(url = %privacy::redact(feed_url), http.response.body.size))] +pub async fn fetch( + client: &reqwest::Client, + feed_url: &str, + auth: Option<(&str, Option)>, + etag: Option<&str>, + last_modified: Option<&str>, +) -> Result<(Fetched, Option)> { + let start = reqwest::Url::parse(feed_url).context("the feed's address")?; + let mut url = start.clone(); + let mut permanent = true; + for _ in 0..10 { + let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT); + if let Some(tag) = etag { + req = req.header(IF_NONE_MATCH, tag); + } + if let Some(lm) = last_modified { + req = req.header(IF_MODIFIED_SINCE, lm); + } + // The feed's own host only: a redirect elsewhere must not be handed the password. + if let Some((user, password)) = &auth + && url.host_str() == start.host_str() + { + req = req.basic_auth(user, password.as_deref()); + } + + let resp = req.send().await.context("connecting")?; + let status = resp.status(); + if status.is_redirection() && status != StatusCode::NOT_MODIFIED { + let to = resp + .headers() + .get(reqwest::header::LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?; + url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?; + permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT); + continue; + } + let moved = (permanent && url != start).then(|| url.to_string()); + FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if status == StatusCode::NOT_MODIFIED { + return Ok((Fetched::NotModified, moved)); + } + if !status.is_success() { + // WordPress.com answers some readers' feed requests with a browser check that only + // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the + // same posts and no check (#317). + if status == StatusCode::FORBIDDEN + && let Some(host) = url.host_str().map(str::to_owned) + && read_capped(resp).await.is_ok_and(|b| wordpress::is_wpcom_challenge(&b)) + { + return Ok((wordpress::wpcom_posts(client, &host).await?, moved)); + } + // The original surfaced 401/407 specially; the code is enough for a UI to switch on. + return Err(anyhow!("HTTP {status}")); + } + let header = |h: HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned); + let etag = header(ETAG); + let last_modified = header(LAST_MODIFIED); + let bytes = read_capped(resp).await.context("reading body")?; + tracing::Span::current().record("http.response.body.size", bytes.len()); + FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed); + return Ok((Fetched::Body { bytes, etag, last_modified }, moved)); + } + // Worded as reqwest worded it, which failure_kind reads as a redirect loop. + Err(anyhow!("error following redirect for url ({url}): too many redirects")) +} + +#[cfg(test)] +mod tests { + + use crate::failure; + + /// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good + /// to /tmp, /new is the feed. + async fn redirecting_server() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { return }; + tokio::spawn(async move { + let mut buf = [0u8; 2048]; + let n = sock.read(&mut buf).await.unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..n]); + let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned(); + let body = "T"; + let resp = match path.as_str() { + "/old" => { + "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned() + } + "/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(), + "/chain" => { + "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned() + } + // An episode's file given as a feed: its size said, its body never sent. + "/huge" => "HTTP/1.1 200 OK\r\nContent-Length: 600000000\r\n\r\n".to_owned(), + "/loop" => { + "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned() + } + _ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()), + }; + let _ = sock.write_all(resp.as_bytes()).await; + }); + } + }); + format!("http://{addr}") + } + + #[tokio::test] + async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() { + let base = redirecting_server().await; + let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap(); + let get = |path: &str| { + let url = format!("{base}{path}"); + let client = client.clone(); + async move { super::fetch(&client, &url, None, None, None).await } + }; + let (got, moved) = get("/old").await.unwrap(); + assert!(matches!(got, super::Fetched::Body { .. })); + assert_eq!(moved, Some(format!("{base}/new"))); + assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now + let huge = get("/huge").await.err().map(|e| format!("{e:#}")).unwrap_or_default(); + assert!(huge.contains("too large to be a feed"), "{huge}"); + assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good + assert_eq!(get("/new").await.unwrap().1, None); // never moved + let looped = get("/loop").await.err().unwrap().to_string(); + assert_eq!(failure::failure_kind(&looped).0, "redirect_loop"); + } +} diff --git a/crates/feed/src/jev.rs b/crates/feed/src/jev.rs new file mode 100644 index 0000000..e121c23 --- /dev/null +++ b/crates/feed/src/jev.rs @@ -0,0 +1,67 @@ +//! Asking TypeSafe's Jev which Apple category a feed belongs to. + +use crate::parse::ParsedFeed; +use crate::{category, fetch}; +use anyhow::{Result, anyhow}; + +/// TypeSafe's System One endpoint, where Jev answers questions about a piece of state. +pub const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone"; + +/// A Directory category for a feed that names none of its own, which most blogs do not (#117): +/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and +/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest +/// is taken however unsure, since an admin can change it and asking again would cost a call a scan. +pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result { + let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect(); + // A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous. + let mut criteria = serde_json::Map::new(); + for (category, subs) in category::CATEGORIES { + criteria.insert(category.to_string(), serde_json::Value::Null); + for sub in *subs { + criteria.insert(sub.to_string(), format!("Within {category}").into()); + } + } + let body = serde_json::json!({ + "model": "jev-latest", + "state": { "title": feed.title, "recent_items": items }, + "questions": { "category": { + "type": "choice", + "instructions": "Which category best describes what this feed publishes?", + "criteria": criteria, + }}, + }); + let answer: serde_json::Value = client + .post(JEV_URL) + .bearer_auth(key) + .json(&body) + .timeout(fetch::FEED_TIMEOUT) + .send() + .await? + .error_for_status()? + .json() + .await?; + picked(&answer) +} + +pub(crate) fn picked(answer: &serde_json::Value) -> Result { + let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default(); + category::CATEGORIES + .iter() + .flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter())) + .find(|c| **c == choice) + .map(|c| c.to_string()) + .ok_or_else(|| anyhow!("no category in the answer: {answer}")) +} + +#[cfg(test)] +mod tests { + + #[test] + fn picked_takes_only_an_apple_category() { + let answer = |c: &str| serde_json::json!({"answers": {"category": {"type": "choice", "choice": c}}}); + assert_eq!(super::picked(&answer("Technology")).unwrap(), "Technology"); + assert_eq!(super::picked(&answer("Tech News")).unwrap(), "Tech News", "a subcategory"); + assert!(super::picked(&answer("Blogs")).is_err(), "not one of Apple's"); + assert!(super::picked(&serde_json::json!({"error": "rate limited"})).is_err()); + } +} diff --git a/crates/feed/src/lib.rs b/crates/feed/src/lib.rs index 711ba34..5299bf4 100644 --- a/crates/feed/src/lib.rs +++ b/crates/feed/src/lib.rs @@ -1,2088 +1,26 @@ //! Feed fetching and parsing, shared by ipx and the directory (#202). Replaces //! FeedData.__getFeed / __getEntries. +mod artwork; +mod category; +mod discover; +mod failure; +mod fetch; pub mod guard; - -use anyhow::{Context, Result, anyhow}; -use chrono::{DateTime, NaiveDateTime}; -use reqwest::StatusCode; -use reqwest::header::HeaderName; -use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; -use std::collections::{HashMap, HashSet}; -use std::sync::atomic::AtomicU64; -use std::time::Duration; - -/// A feed as `parse` reads it, RSS or Atom alike: the channel's own details and its items. -#[derive(Debug, Default)] -pub struct ParsedFeed { - pub title: Option, - /// RSS's ``: how long the publisher asks to be left alone between reads. - pub ttl_mins: Option, - /// The show's artwork, `` before the channel's ``. - pub image: Option, - /// The site the feed belongs to, where a favicon can stand in for missing artwork. - pub site: Option, - /// The channel's first ``, for the Directory's categories. - pub category: Option, - /// What the feed says it is, for its page in the Directory (#130): RSS's ``, - /// or iTunes' summary where that is empty, or Atom's subtitle. HTML as often as not. - pub description: Option, - pub entries: Vec, -} - -/// One item of a feed. -#[derive(Debug, Default)] -pub struct Entry { - /// What tells this item from the feed's others: its ``, or failing that its link, its - /// enclosure's address or its title (`pick_guid`). - pub guid: String, - pub title: Option, - pub link: Option, - /// Unix seconds. - pub published: Option, - pub description: Option, - /// The item's Markdown, when the feed sends it beside HTML of its own (``): - /// the page can show either (#285). With no HTML it is the description instead (#284). - pub markdown: Option, - pub categories: Vec, - /// `` said yes. - pub explicit: bool, - /// Episode artwork; falls back to the feed's in the UI. - pub image: Option, - /// Seconds. - pub duration: Option, - pub episode: Option, - pub season: Option, - pub enclosures: Vec, -} - -/// A file an item carries, an episode as a rule. -#[derive(Debug, Default, Clone, PartialEq)] -pub struct Enclosure { - pub url: String, - /// The type the feed declares, which may be missing or wrong. - pub mime: Option, - /// The size the feed declares, in bytes. - pub length: Option, -} - -/// What `fetch` got. -pub enum Fetched { - /// Server said 304, or returned a body we already have. - NotModified, - /// A body to read, with the validators to send next time. - Body { bytes: Vec, etag: Option, last_modified: Option }, -} - -/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and -/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504 -/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small; -/// downloads, which are not, have no such limit. -pub const FEED_TIMEOUT: Duration = Duration::from_secs(30); - -/// The longest a feed's server may send nothing, for the clients that fetch feeds to set: one -/// that took the connection and never answered ran the whole 30 s, every scan it was in (#230). -/// No feed that answered took over 8 s in a day of traces; one that keeps sending, however -/// slowly, still has until FEED_TIMEOUT. -pub const FEED_SILENCE: Duration = Duration::from_secs(15); - -/// Feeds fetched and the bytes of their bodies, since the process started, for /metrics to set -/// beside memory (#224): a body is read whole, so a night of large feeds is a night of large -/// allocations. -pub static FETCHES: AtomicU64 = AtomicU64::new(0); -/// See `FETCHES`. -pub static FETCHED_BYTES: AtomicU64 = AtomicU64::new(0); - -/// Conditional GET, following redirects itself: `client` must follow none (`feed_client`), so -/// each hop is seen. The second value is where the feed now is when every hop said so for good -/// (301 or 308): a publisher that moved its feed, which the catalogue should follow rather -/// than be redirected on every read. A temporary redirect (302, 307) moves nothing. -/// `system-proxy` reads http_proxy/https_proxy. -/// `auth` is a username and password, sent only to the feed's own host. -/// An address as a trace may keep it: without its query or user and password, where a private -/// feed carries its credential (Patreon's `?auth=`), which Tempo kept for anyone reading it (#225). -/// shortcut: a token in the path (`/feed/.xml`) stays; strip paths too if a feed does that. -#[must_use] -pub fn redact(url: &str) -> String { - match url::Url::parse(url) { - Ok(mut u) => { - u.set_query(None); - u.set_fragment(None); - let _ = u.set_username(""); - let _ = u.set_password(None); - u.to_string() - } - // Not an address ipx could fetch; keep what comes before anything that could be a query. - Err(_) => url.split(['?', '#', '@']).next().unwrap_or_default().to_owned(), - } -} - -/// Text with each address in it redacted, for what is logged and sent to every open page: an -/// error from reqwest names the whole address it fetched, a private feed's key with it (#238). -#[must_use] -pub fn redact_text(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - while let Some(i) = rest.find("http://").into_iter().chain(rest.find("https://")).min() { - out.push_str(&rest[..i]); - let end = rest[i..].find(|c: char| c.is_whitespace() || "()<>\"'".contains(c)).map_or(rest.len(), |n| i + n); - out.push_str(&redact(&rest[i..end])); - rest = &rest[end..]; - } - out.push_str(rest); - out -} - -/// The most of a feed or page read. The largest feeds here are a few MB; without a limit an -/// episode's MP3 given as a feed was read whole into memory, at every check (#295). -const MAX_BODY: usize = 50 << 20; - -/// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read. -async fn read_capped(mut resp: reqwest::Response) -> Result> { - if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) { - return Err(anyhow!("larger than {} MB, too large to be a feed", MAX_BODY >> 20)); - } - let mut out = Vec::new(); - while let Some(c) = resp.chunk().await? { - out.extend_from_slice(&c); - anyhow::ensure!(out.len() <= MAX_BODY, "larger than {} MB, too large to be a feed", MAX_BODY >> 20); - } - Ok(out) -} - -/// Reads a feed, asking only for what changed since `etag` and `last_modified`. Redirects are -/// followed here, up to ten, so `auth` goes to the feed's own host alone. Also returns the new -/// address when every hop said the feed moved for good, for the caller to keep instead. -#[tracing::instrument(skip_all, fields(url = %redact(feed_url), http.response.body.size))] -pub async fn fetch( - client: &reqwest::Client, - feed_url: &str, - auth: Option<(&str, Option)>, - etag: Option<&str>, - last_modified: Option<&str>, -) -> Result<(Fetched, Option)> { - let start = reqwest::Url::parse(feed_url).context("the feed's address")?; - let mut url = start.clone(); - let mut permanent = true; - for _ in 0..10 { - let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT); - if let Some(tag) = etag { - req = req.header(IF_NONE_MATCH, tag); - } - if let Some(lm) = last_modified { - req = req.header(IF_MODIFIED_SINCE, lm); - } - // The feed's own host only: a redirect elsewhere must not be handed the password. - if let Some((user, password)) = &auth - && url.host_str() == start.host_str() - { - req = req.basic_auth(user, password.as_deref()); - } - - let resp = req.send().await.context("connecting")?; - let status = resp.status(); - if status.is_redirection() && status != StatusCode::NOT_MODIFIED { - let to = resp - .headers() - .get(reqwest::header::LOCATION) - .and_then(|v| v.to_str().ok()) - .ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?; - url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?; - permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT); - continue; - } - let moved = (permanent && url != start).then(|| url.to_string()); - FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if status == StatusCode::NOT_MODIFIED { - return Ok((Fetched::NotModified, moved)); - } - if !status.is_success() { - // WordPress.com answers some readers' feed requests with a browser check that only - // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the - // same posts and no check (#317). - if status == StatusCode::FORBIDDEN - && let Some(host) = url.host_str().map(str::to_owned) - && read_capped(resp).await.is_ok_and(|b| is_wpcom_challenge(&b)) - { - return Ok((wpcom_posts(client, &host).await?, moved)); - } - // The original surfaced 401/407 specially; the code is enough for a UI to switch on. - return Err(anyhow!("HTTP {status}")); - } - let header = |h: HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned); - let etag = header(ETAG); - let last_modified = header(LAST_MODIFIED); - let bytes = read_capped(resp).await.context("reading body")?; - tracing::Span::current().record("http.response.body.size", bytes.len()); - FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed); - return Ok((Fetched::Body { bytes, etag, last_modified }, moved)); - } - // Worded as reqwest worded it, which failure_kind reads as a redirect loop. - Err(anyhow!("error following redirect for url ({url}): too many redirects")) -} - -/// WordPress.com's browser check, in place of the feed it was asked for. -fn is_wpcom_challenge(body: &[u8]) -> bool { - String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/") -} - -/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed. -/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed -/// set to carry more gets ten here. -async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result { - let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts"); - let resp = client - .get(&api) - .timeout(FEED_TIMEOUT) - .send() - .await - .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?; - anyhow::ensure!( - resp.status().is_success(), - "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}", - resp.status() - ); - let bytes = read_capped(resp).await.context("reading WordPress.com's API")?; - FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed); - Ok(Fetched::Body { bytes, etag: None, last_modified: None }) -} - -/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's -/// own feed gives each post, so they are the items already stored. No title for the feed: the -/// stored one stays. -fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed { - let entries = posts - .iter() - .filter_map(|p| { - Some(Entry { - guid: non_empty(p["guid"]["rendered"].as_str()).or_else(|| non_empty(p["link"].as_str()))?, - title: title_text(p["title"]["rendered"].as_str()), - link: non_empty(p["link"].as_str()), - published: p["date_gmt"] - .as_str() - .and_then(|d| NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok()) - .map(|d| d.and_utc().timestamp()), - description: non_empty(p["content"]["rendered"].as_str()), - image: non_empty(p["jetpack_featured_media_url"].as_str()), - ..Default::default() - }) - }) - .collect(); - ParsedFeed { entries, ..Default::default() } -} - -/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem -/// it is, and whether there is a new address to switch to. -pub struct Failure { - /// What to tell them, in a sentence. - pub reason: &'static str, - /// Where the feed went, when the error says. - pub new_url: Option, -} - -/// A failure's kind, for the log's `error.type` (#91): the HTTP status where there is one, as -/// OpenTelemetry names an HTTP error, and otherwise a word for what went wrong. Matches the -/// same wording as `explain_failure`; a message it does not know is "other", never a wrong kind. -#[must_use] -pub fn failure_kind(msg: &str) -> (String, Option) { - let low = msg.to_ascii_lowercase(); - let code = low.split("http ").skip(1).find_map(|r| r.get(..3)?.parse::().ok()); - if let Some(c) = code.filter(|c| (100..600).contains(c)) { - return (c.to_string(), Some(c)); - } - let kind = if low.contains("dns error") - || low.contains("failed to lookup address") - || low.contains("no address associated") - { - "dns" - } else if low.contains("too many redirects") { - "redirect_loop" - } else if low.contains("timed out") || low.contains("timeout") { - "timeout" - } else if low.contains("certificate") || low.contains("tls") { - "tls" - } else if low.contains("got a web page") { - "not_a_feed" - } else if low.contains("the site sent ") { - "site_message" - } else if low.contains("connect") { - "connect" - } else if low.contains("pars") { - "parse" - } else { - "other" - }; - (kind.into(), None) -} - -/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the -/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling -/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled -- -/// comes back `None`: transient by nature, or with nothing more useful to say than the raw -/// text already shown once a feed is open. -/// -/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP -/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure -/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing -/// nothing extra, so a miss costs a clearer message, not a wrong one. -pub fn explain_failure(msg: &str) -> Option { - if let Some(rest) = msg.strip_prefix("got a web page, not a feed") { - let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned); - return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url }); - } - if msg.contains("the site sent ") { - return Some(Failure { - reason: "The site sent a message instead of the feed; the publisher has to fix it.", - new_url: None, - }); - } - let low = msg.to_ascii_lowercase(); - if low.contains("http 404") { - return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None }); - } - if low.contains("http 401") || low.contains("http 403") { - return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None }); - } - if low.contains("answered with nothing") { - return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None }); - } - if low.contains("http 402") { - return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None }); - } - // Only a name that does not exist. A lookup that failed for the moment says nothing about the - // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143). - if !is_temporary_lookup_failure(msg) - && (low.contains("dns error") - || low.contains("failed to lookup address") - || low.contains("no address associated")) - { - return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None }); - } - None -} - -/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's -/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does -/// not exist ("Name or service not known"). It says the network was not answering, not that the -/// feed is gone (#143). -#[must_use] -pub fn is_temporary_lookup_failure(msg: &str) -> bool { - let low = msg.to_ascii_lowercase(); - low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again")) -} - -/// True when a body is an OPML document rather than a feed. -/// -/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an -/// OPML served from a URL without that extension. Sniffing the body catches both. -#[must_use] -pub fn is_opml(bytes: &[u8]) -> bool { - let head = &bytes[..bytes.len().min(1024)]; - let text = String::from_utf8_lossy(head).to_lowercase(); - text.contains("` overflowed the -/// stack, which aborts the whole daemon, and again on each restart for a subscribed feed (#322). -pub fn read_opml(text: &str) -> Result { - use quick_xml::events::Event; - let mut reader = quick_xml::Reader::from_str(text); - let mut depth = 0usize; - loop { - match reader.read_event() { - Ok(Event::Start(_)) => { - depth += 1; - if depth > MAX_OPML_DEPTH { - anyhow::bail!("that OPML nests more than {MAX_OPML_DEPTH} levels deep"); - } - } - Ok(Event::End(_)) => depth = depth.saturating_sub(1), - // Malformed XML is the opml crate's to describe. - Ok(Event::Eof) | Err(_) => break, - Ok(_) => {} - } - } - opml::OPML::from_str(text).map_err(|e| anyhow!("{e}")) -} - -/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders. -pub fn parse_opml(bytes: &[u8]) -> Result> { - let text = String::from_utf8_lossy(bytes); - let doc = read_opml(&text).map_err(|e| anyhow!("that does not parse as OPML: {e}"))?; - let mut out = vec![]; - collect_outlines(&doc.body.outlines, &mut out); - Ok(out) -} - -/// OPML nests feeds inside folder outlines, so this walks the whole tree. -pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, String)>) { - for o in outlines { - if let Some(url) = &o.xml_url { - let title = o.title.clone().unwrap_or_else(|| o.text.clone()); - out.push((title, url.clone())); - } - collect_outlines(&o.outlines, out); - } -} - -/// The of an OPML document. -#[must_use] -pub fn opml_title(bytes: &[u8]) -> Option<String> { - let text = String::from_utf8_lossy(bytes); - let doc = read_opml(&text).ok()?; - doc.head.and_then(|h| h.title).map(|t| t.trim().to_owned()).filter(|t| !t.is_empty()) -} - -/// The token and show of a Patreon feed link, or None for any other URL. -/// -/// Patreon gives each patron one token per creator. With no show it stands for the creator, -/// whose feed carries every show at once. -fn patreon_parts(url: &str) -> Option<(String, Option<String>)> { - let u = url::Url::parse(url).ok()?; - if !matches!(u.host_str()?, "patreon.com" | "www.patreon.com") || !u.path().starts_with("/rss") { - return None; - } - let param = |name: &str| u.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v.into_owned()); - Some((param("auth")?, param("show"))) -} - -/// A Patreon link naming a creator but no show. -#[must_use] -pub fn is_patreon_creator(url: &str) -> bool { - matches!(patreon_parts(url), Some((_, None))) -} - -/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's -/// feed, since the token alone says whose it is. An address with no scheme is https: 'cnn.com' -/// was stored as typed and every check failed with "relative URL without a base" (#101). -#[must_use] -pub fn expand_input(input: &str) -> String { - let s = input.trim(); - let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); - if token { - format!("https://www.patreon.com/rss?auth={s}") - } else if let Some(rest) = s.strip_prefix("//") { - format!("https://{rest}") - } else if !s.contains("://") { - format!("https://{s}") - } else { - s.to_owned() - } -} - -/// Whether two URLs are the same feed. One Patreon show has several spellings -- by the -/// creator's name, by number, or with no creator at all -- and the token and show are what -/// identify it. -#[must_use] -pub fn same_feed(a: &str, b: &str) -> bool { - // Parsing both is the slow part, and only Patreon's addresses can differ and still match. - a == b - || (a.contains("patreon.com") - && b.contains("patreon.com") - && patreon_parts(a).is_some_and(|p| Some(p) == patreon_parts(b))) -} - -/// A Patreon creator's name and shows, each show as (title, feed URL). -/// -/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes, -/// finding shows stops and the show feeds already found keep working. The documented API -/// needs an OAuth client per install and does not list shows. -pub async fn patreon_shows(client: &reqwest::Client, url: &str) -> Result<(Option<String>, Vec<(String, String)>)> { - // The creator feed names its campaign by number in its self link, a few hundred bytes in. - // The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up. - let mut resp = client.get(url).timeout(FEED_TIMEOUT).send().await.context("connecting")?; - if !resp.status().is_success() { - return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status())); - } - let mut head = Vec::new(); - while patreon_campaign(&head).is_none() && head.len() < 64 * 1024 { - let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break }; - head.extend_from_slice(&chunk); - } - let campaign = patreon_campaign(&head).ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?; - - let api = format!( - "https://www.patreon.com/api/campaigns/{campaign}\ - ?include=shows&fields%5Bcampaign%5D=name&fields%5Bcollection%5D=title" - ); - let resp = client.get(api).send().await.context("asking Patreon for the shows")?; - if !resp.status().is_success() { - return Err(anyhow!("Patreon would not list the shows: HTTP {}", resp.status())); - } - let (name, shows) = parse_patreon_shows(&resp.bytes().await.context("reading the shows")?)?; - Ok((name, shows.into_iter().map(|(id, title)| (title, format!("{url}&show={id}"))).collect())) -} - -/// The campaign number in the start of a Patreon feed. -fn patreon_campaign(head: &[u8]) -> Option<String> { - let text = String::from_utf8_lossy(head); - text.match_indices("patreon.com/rss/").find_map(|(i, m)| { - let id: String = text[i + m.len()..].chars().take_while(char::is_ascii_digit).collect(); - (!id.is_empty()).then_some(id) - }) -} - -/// A campaign's name, and its shows as (id, title). -type PatreonShows = (Option<String>, Vec<(String, String)>); - -/// A campaign's name and its shows as (id, title), from Patreon's JSON:API answer. -fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> { - let v: serde_json::Value = serde_json::from_slice(json).context("Patreon's answer is not JSON")?; - // Missing is not the same as none. Read as no shows, the creator feed would be scanned as - // a plain feed, claim every show's files, and leave the shows empty once the list returned. - let ids = v["data"]["relationships"]["shows"]["data"] - .as_array() - .ok_or_else(|| anyhow!("Patreon's answer does not list the shows"))?; - let title = |id: &str| -> Option<String> { - let show = v["included"].as_array()?.iter().find(|x| x["type"] == "collection" && x["id"] == id)?; - show["attributes"]["title"].as_str().map(|t| t.trim().to_owned()) - }; - let shows = ids - .iter() - .filter_map(|s| s["id"].as_str()) - .map(|id| (id.to_owned(), title(id).unwrap_or_else(|| format!("Show {id}")))) - .collect(); - Ok((v["data"]["attributes"]["name"].as_str().map(str::to_owned), shows)) -} - -/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`. -#[tracing::instrument(skip_all, fields(bytes = bytes.len()))] -pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> { - if bytes.trim_ascii_start().starts_with(b"[") - && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes) - { - return Ok(from_wp_posts(&posts)); - } - match rss::Channel::read_from(bytes) { - Ok(ch) => Ok(from_rss(ch, bytes)), - Err(rss_err) => match atom_syndication::Feed::read_from(bytes) { - Ok(feed) => Ok(from_atom(feed)), - Err(atom_err) => { - if let Some(said) = plain_text(bytes) { - return Err(anyhow!("the site sent {said} instead of a feed")); - } - // Some publishers (kcpw, feedland) write a bare "&" in a URL instead of - // "&". Strict XML parsers refuse it; browsers don't. Retry once with - // every offending "&" escaped rather than fail outright. - let escaped = escape_bare_ampersands(bytes); - if escaped != bytes { - if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) { - return Ok(from_rss(ch, &escaped)); - } - if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) { - return Ok(from_atom(feed)); - } - } - Err(match alternate_feed_link(bytes) { - Some(href) if looks_like_html(bytes) => { - anyhow!("got a web page, not a feed; it links {href} as its feed") - } - None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"), - _ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"), - }) - } - }, - } -} - -/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed -/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a -/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case. -fn looks_like_html(bytes: &[u8]) -> bool { - let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase(); - head.contains("<!doctype html") || head.contains("<html") -} - -/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with -/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried -/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors. -fn plain_text(bytes: &[u8]) -> Option<String> { - let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]); - let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}'); - if text.starts_with('<') { - return None; - } - let line = text.lines().next().unwrap_or("").trim_end(); - if line.is_empty() { - return Some("an empty reply".into()); - } - let mut said: String = line.chars().take(80).collect(); - if said.len() < line.len() { - said.push('…'); - } - Some(format!("\"{said}\"")) -} - -/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml" -/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now, -/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos. -fn alternate_feed_link(bytes: &[u8]) -> Option<String> { - let text = String::from_utf8_lossy(bytes); - // ASCII only: to_lowercase changes some characters' length (U+0130 grows a byte), and the - // offsets found in the lowered copy then sliced the original off a char boundary. - let lower = text.to_ascii_lowercase(); - let mut pos = 0; - while let Some(rel) = lower[pos..].find("<link") { - let start = pos + rel; - let Some(end) = lower[start..].find('>').map(|e| start + e) else { break }; - pos = end + 1; - let tag = &text[start..end]; - let tag_lower = &lower[start..end]; - let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'"); - let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml"); - if is_alternate - && is_feed_type - && let Some(href) = tag_attr(tag, "href") - { - return Some(href); - } - } - None -} - -/// What someone asked to add, as a feed: the address itself when it is a feed or an OPML list, -/// otherwise the feed its web page links as its own, otherwise an error and nothing is added. -/// A page that linked no feed used to be added as it was and failed on every check, called a -/// feed that moved (#102); cnn.com is one. -pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> { - let read = |u: String| async move { - let resp = client.get(&u).timeout(Duration::from_secs(30)).send().await.context("connecting")?; - anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status()); - let base = resp.url().clone(); - anyhow::Ok((base, read_capped(resp).await.context("reading")?)) - }; - let is_feed = |b: &[u8]| is_opml(b) || parse(b).is_ok(); - let (base, body) = read(url.to_owned()).await.with_context(|| format!("could not read {url}"))?; - if is_feed(&body) { - return Ok(url.to_owned()); - } - if !looks_like_html(&body) { - let why = parse(&body).err().map(|e| format!("{e:#}")).unwrap_or_default(); - anyhow::bail!("{url} is not a feed: {why}"); - } - let Some(href) = alternate_feed_link(&body) else { - anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to"); - }; - let linked = - base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?; - let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?; - anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either"); - Ok(linked.into()) -} - -/// Artwork for a feed that has none: the icon its site's page names, or else the site's -/// `/favicon.ico`. None if neither is there. -#[tracing::instrument(skip_all, fields(site = site))] -pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option<String> { - let timeout = Duration::from_secs(20); - let resp = client.get(site).timeout(timeout).send().await.ok()?; - // Relative to where the page ended up, not where it was asked for: a site that redirects - // to /en/ would otherwise have its icon looked for in the wrong place. - let base = resp.url().clone(); - // The icon a page names can be gone: antirez.com names /images/favicon.png, a 404, while its - // /favicon.ico is there. Stored unchecked, it was a broken image that was never looked up again. - if resp.status().is_success() - && let Ok(page) = read_capped(resp).await - && let Some(href) = page_icon(&page) - && let Ok(url) = base.join(&href) - && is_image(client, url.as_str()).await - { - return Some(url.into()); - } - let ico = base.join("/favicon.ico").ok()?; - is_image(client, ico.as_str()).await.then(|| ico.into()) -} - -/// How many hosts one scan of a feed asks whether its artwork is on https. -const MAX_ART_HOSTS: usize = 8; - -/// Artwork's address on https when its host serves it there, else as it was. The page is https -/// and must not load http; the host is asked once per `known` (one feed's read), and an http -/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five -/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN -/// presents another name's certificate (#110). -pub async fn prefer_https(client: &reqwest::Client, url: &str, known: &mut HashMap<String, bool>) -> String { - let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() }; - let host = rest.split('/').next().unwrap_or("").to_owned(); - let secure = format!("https://{rest}"); - let ok = match known.get(&host) { - Some(ok) => *ok, - // Each new host is a probe of up to 20 s, one after another on the scan, and the feed's - // author decides how many hosts there are: ports and user names make as many as they like - // of one server (#339). Real feeds put their artwork on a few. - None if known.len() >= MAX_ART_HOSTS => false, - None => { - let ok = is_image(client, &secure).await; - known.insert(host, ok); - ok - } - }; - if ok { secure } else { url.to_owned() } -} - -/// Whether `url` answers with an image. A site with no favicon often answers 200 with its home -/// page, which is not an icon. -pub async fn is_image(client: &reqwest::Client, url: &str) -> bool { - let timeout = Duration::from_secs(20); - let Ok(resp) = client.get(url).timeout(timeout).send().await else { return false }; - resp.status().is_success() - && resp - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .is_some_and(|t| t.starts_with("image/")) -} - -/// The icon a web page names in its `<link>` tags, the larger apple-touch-icon first: a plain -/// `icon` is often 16 pixels, which blurs at the size the list draws artwork. -fn page_icon(bytes: &[u8]) -> Option<String> { - let text = String::from_utf8_lossy(bytes); - // ASCII only, so byte offsets in the lowered copy stay valid in the original. - let lower = text.to_ascii_lowercase(); - let (mut touch, mut icon) = (None, None); - let mut pos = 0; - while let Some(at) = lower[pos..].find("<link") { - let start = pos + at; - let Some(end) = lower[start..].find('>').map(|e| start + e) else { break }; - pos = end + 1; - let tag = &text[start..end]; - let Some(rel) = tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue }; - let rels: Vec<&str> = rel.split_whitespace().collect(); - if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) { - touch = tag_attr(tag, "href"); - } else if icon.is_none() && rels.contains(&"icon") { - icon = tag_attr(tag, "href"); - } - } - touch.or(icon).filter(|h| !h.is_empty()) -} - -/// The value of one attribute in an HTML/XML start tag, however it is quoted. -fn tag_attr(tag: &str, name: &str) -> Option<String> { - let key = format!("{name}="); - let idx = tag.to_lowercase().find(&key)?; - let after = &tag[idx + key.len()..]; - let quote = after.chars().next()?; - if quote != '"' && quote != '\'' { - return None; - } - let rest = &after[1..]; - let close = rest.find(quote)?; - Some(rest[..close].trim().to_owned()) -} - -/// Escapes every `&` that does not already start a recognized XML entity -/// (`&`, `<`, `>`, `"`, `'`, or a numeric reference like `'`). -fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> { - fn is_entity_start(rest: &[u8]) -> bool { - for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] { - if rest.starts_with(named) { - return true; - } - } - let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") { - &rest[2..] - } else if rest.starts_with(b"#") { - &rest[1..] - } else { - return false; - }; - let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count(); - len > 0 && digits.get(len) == Some(&b';') - } - - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) { - out.extend_from_slice(b"&"); - } else { - out.push(bytes[i]); - } - i += 1; - } - out -} - -/// Every `<enclosure>` of every `<item>`, in document order. -/// -/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0 -/// says -- and when a feed carries several it keeps only the *last*, silently losing the -/// rest. Feeds do ship several, so read them from the XML directly. -fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> { - use quick_xml::events::Event; - - let mut reader = quick_xml::Reader::from_reader(bytes); - reader.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut out: Vec<Vec<Enclosure>> = Vec::new(); - let mut current: Option<Vec<Enclosure>> = None; - - let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> { - let (mut url, mut mime, mut length) = (String::new(), None, None); - for attr in e.attributes().flatten() { - // Values arrive escaped: a feed URL's "&" is "&" in the document. - let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default(); - match attr.key.local_name().as_ref() { - "url" => url = val, - "type" => mime = Some(val).filter(|v| !v.is_empty()), - "length" => length = val.parse().ok(), - _ => {} - } - } - (!url.is_empty()).then_some(Enclosure { url, mime, length }) - }; - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(e)) => match e.name().local_name().as_ref() { - "item" => current = Some(Vec::new()), - "enclosure" => { - if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) { - list.push(enc); - } - } - _ => {} - }, - Ok(Event::Empty(e)) => match e.name().local_name().as_ref() { - // <item/> with no children still counts, so the indexes stay aligned. - "item" => out.push(Vec::new()), - "enclosure" => { - if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) { - list.push(enc); - } - } - _ => {} - }, - Ok(Event::End(e)) => { - if e.name().local_name().as_ref() == "item" - && let Some(list) = current.take() - { - out.push(list); - } - } - Ok(Event::Eof) | Err(_) => break, - _ => {} - } - buf.clear(); - } - if let Some(list) = current.take() { - out.push(list); - } - out -} - -fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed { - let per_item = enclosures_by_item(bytes); - let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes); - - let entries = ch - .items() - .iter() - .enumerate() - .filter_map(|(idx, item)| { - // Straight from the XML, so an item with several keeps all of them. Falls - // back to the parsed one if the scan and the parser disagree on item count. - let mut enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| { - item.enclosure() - .into_iter() - .map(|e| Enclosure { - url: e.url().trim().to_owned(), - mime: non_empty(Some(e.mime_type())), - length: e.length().parse().ok(), - }) - .filter(|e| !e.url.is_empty()) - .collect() - }); - drop_player_repeats(&mut enclosures); - - let guid = pick_guid( - item.guid().map(|g| g.value()), - item.link(), - enclosures.first().map(|e| e.url.as_str()), - item.title(), - )?; - - let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes); - let it = item.itunes_ext(); - - Some(Entry { - guid, - title: title_text(item.title()), - link: non_empty(item.link()), - published: item.pub_date().and_then(parse_date), - // Content wins over description, as __getEntries preferred entry.content. The - // source namespace's Markdown copy only when there is neither (#284). - description: notes(item.content(), item.description(), source_markdown(item)), - markdown: non_empty(source_markdown(item)) - .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()), - categories: item - .categories() - .iter() - .map(|c| c.name().to_owned()) - .filter(|c| !c.is_empty() && !c.starts_with("http")) - .collect(), - explicit: explicit || entry_explicit, - image: item_image(item, &enclosures), - duration: it.and_then(|i| i.duration()).and_then(parse_duration), - episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()), - season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()), - enclosures, - }) - }) - .collect(); - - ParsedFeed { - title: title_text(Some(ch.title())), - ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()), - // itunes:image is the square artwork; <image><url> is the older, often smaller one. - image: ch - .itunes_ext() - .and_then(|i| i.image()) - .map(str::to_owned) - .or_else(|| ch.image().map(|i| i.url().to_owned())), - site: non_empty(Some(ch.link().trim())), - // Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and - // would fill the Directory with one-off tags. The subcategory where there is one: Apple - // files every tabletop and gaming show under Leisure, which says little; Games says it. - description: non_empty(Some(ch.description())).or_else(|| non_empty(ch.itunes_ext().and_then(|i| i.summary()))), - category: ch - .itunes_ext() - .and_then(|i| i.categories().first()) - .map(|c| c.subcategory().filter(|s| !s.text().trim().is_empty()).unwrap_or(c)) - .and_then(|c| non_empty(Some(c.text().trim()))), - entries, - } -} - -fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { - let entries = feed - .entries() - .iter() - .filter_map(|e| { - // Atom carries enclosures as <link rel="enclosure">. - let mut enclosures: Vec<Enclosure> = e - .links() - .iter() - .filter(|l| l.rel() == "enclosure") - .map(|l| Enclosure { - url: l.href().trim().to_owned(), - mime: non_empty(l.mime_type()), - length: l.length().and_then(|s| s.parse().ok()), - }) - .filter(|e| !e.url.is_empty()) - .collect(); - drop_player_repeats(&mut enclosures); - - let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href()); - - let guid = - pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?; - - Some(Entry { - guid, - title: title_text(Some(e.title().as_str())), - link: alt.map(str::to_owned), - published: e.published().or(Some(e.updated())).map(|d| d.timestamp()), - description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None), - markdown: None, - categories: e.categories().iter().map(|c| c.term().to_owned()).collect(), - explicit: false, - image: None, - duration: None, - episode: None, - season: None, - enclosures, - }) - }) - .collect(); - - ParsedFeed { - title: title_text(Some(feed.title().as_str())), - ttl_mins: None, - image: feed.logo().or_else(|| feed.icon()).map(str::to_owned), - site: feed - .links() - .iter() - .find(|l| l.rel() == "alternate") - .map(|l| l.href().trim().to_owned()) - .filter(|h| !h.is_empty()), - category: None, - description: non_empty(feed.subtitle().map(|t| t.as_str())), - entries, - } -} - -/// The original fell back to hashing the title or description. A guid, permalink or -/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with -/// none of them has nothing to download and is dropped. -fn pick_guid(guid: Option<&str>, link: Option<&str>, enclosure: Option<&str>, title: Option<&str>) -> Option<String> { - [guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned) -} - -fn is_yes(s: &str) -> bool { - matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit") -} - -fn non_empty(s: Option<&str>) -> Option<String> { - s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned) -} - -/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post -/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion -/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays. -fn drop_player_repeats(encs: &mut Vec<Enclosure>) { - let mut seen = HashSet::new(); - encs.retain(|e| seen.insert(same_file_key(&e.url))); -} - -/// An enclosure URL without WordPress's player number, for telling repeats of one file apart -/// from different files. -#[must_use] -pub fn same_file_key(url: &str) -> String { - let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() }; - let kept: Vec<(String, String)> = u - .query_pairs() - .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit()))) - .map(|(k, v)| (k.into_owned(), v.into_owned())) - .collect(); - if kept.is_empty() { - u.set_query(None); - } else { - u.query_pairs_mut().clear().extend_pairs(kept); - } - u.to_string() -} - -/// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through -/// the XML parser with its HTML entities intact: The Verge's "Meta’s" reached the page as -/// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is -/// instead of failing the whole title. -fn title_text(s: Option<&str>) -> Option<String> { - let s = non_empty(s)?; - let mut out = String::with_capacity(s.len()); - let mut rest = s.as_str(); - while let Some(at) = rest.find('&') { - out.push_str(&rest[..at]); - rest = &rest[at..]; - let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1); - let decoded = rest[len..] - .starts_with(';') - .then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok()) - .flatten(); - match decoded { - Some(v) => { - out.push_str(&v); - rest = &rest[len + 1..]; - } - None => { - out.push('&'); - rest = &rest[1..]; - } - } - } - out.push_str(rest); - non_empty(Some(&out)) -} - -/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in -/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML. -fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option<String> { - match body(content, description) { - Some(b) if is_markdown(&b) => Some(markdown_html(&b)), - Some(b) => Some(b), - None => non_empty(markdown).map(|m| markdown_html(&m)), - } -} - -/// `<source:markdown>`, from https://source.scripting.com/: the item's text in Markdown. -/// -/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding -/// it to another prefix is not read, until one turns up. -fn source_markdown(item: &rss::Item) -> Option<&str> { - item.extensions().get("source")?.get("markdown")?.first()?.value() -} - -/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold. -/// Plain text without any is left as it was, since rendering it would change nothing but -/// could still read a line starting "1984." as a list. -fn is_markdown(s: &str) -> bool { - let tag = s - .match_indices('<') - .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!')); - let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m)); - !tag && (s.contains("](") || s.contains("**") || s.lines().any(block)) -} - -/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML. -#[must_use] -pub fn markdown_html(md: &str) -> String { - use pulldown_cmark::{Options, Parser, html}; - let mut out = String::new(); - html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH)); - out -} - -/// An item's show notes: its full body when that is whole, else its description. -/// -/// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name -/// pasted from a web app (`[&:has([data-writing-block])>*]:pointer-events-auto`), so the body -/// began halfway through a tag and the page showed the rest of the tag as text. The same item's -/// `description` was whole. With no description to fall back on, a damaged body beats none. -fn body(content: Option<&str>, description: Option<&str>) -> Option<String> { - match non_empty(content).filter(|c| !starts_mid_tag(c)) { - Some(c) => Some(match subtitle(&c, description) { - Some(s) => format!("<p><em>{}</em></p>{c}", quick_xml::escape::escape(s.as_str())), - None => c, - }), - None => non_empty(description).or_else(|| non_empty(content)), - } -} - -/// A description that is a subtitle rather than a second copy of the notes: Substack puts the -/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it. -/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a -/// description found in the body is not shown twice. -/// -/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its -/// notes passes too and shows above them, which reads fine; a real subtitle field would need an -/// `entries` column. -fn subtitle(body: &str, description: Option<&str>) -> Option<String> { - let d = title_text(description)?; - if d.contains('<') || d.chars().count() > 300 { - return None; - } - // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut - // description ends in "…" or "[...]". - let words = |s: &str| { - s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::<Vec<_>>().join(" ").to_lowercase() - }; - let want = words(&d); - let text = title_text(Some(&text_of(body))).unwrap_or_default(); - (!want.is_empty() && !words(&text).contains(&want)).then_some(d) -} - -/// HTML with its tags taken out, each replaced by a space so words either side stay apart. -fn text_of(html: &str) -> String { - let mut out = String::with_capacity(html.len()); - let mut in_tag = false; - for c in html.chars() { - match c { - '<' => in_tag = true, - '>' if in_tag => { - in_tag = false; - out.push(' '); - } - _ if !in_tag => out.push(c), - _ => {} - } - } - out -} - -/// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose -/// start was cut off. -fn starts_mid_tag(html: &str) -> bool { - html[..html.find('<').unwrap_or(html.len())].contains("\">") -} - -/// The picture to show beside an item, in order of how deliberate it is: -/// `itunes:image`, then Media RSS `media:thumbnail`, then a `media:content` that is an -/// image, and finally an image enclosure -- which is how a blog's article picture arrives -/// (Substack puts it there), so those entries get artwork rather than a blank square. -fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> { - if let Some(url) = item.itunes_ext().and_then(|i| i.image()) { - return non_empty(Some(url)); - } - - let media = item.extensions().get("media"); - let attr = |name: &str, want_image: bool| -> Option<String> { - media?.get(name)?.iter().find_map(|e| { - if want_image { - // media:content carries anything; only take it when it says it is a picture. - let is_image = e.attrs.get("type").is_some_and(|t| t.starts_with("image/")) - || e.attrs.get("medium").is_some_and(|m| m == "image"); - if !is_image { - return None; - } - } - non_empty(e.attrs.get("url").map(String::as_str)) - }) - }; - attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| { - enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone()) - }) -} - -/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45"). -fn parse_duration(s: &str) -> Option<i64> { - let s = s.trim(); - if s.is_empty() { - return None; - } - if !s.contains(':') { - return s.parse().ok().filter(|n| *n > 0); - } - let mut total: i64 = 0; - for part in s.split(':') { - total = total * 60 + part.trim().parse::<i64>().ok()?; - } - Some(total).filter(|n| *n > 0) -} - -/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead. -fn parse_date(s: &str) -> Option<i64> { - let s = s.trim(); - DateTime::parse_from_rfc2822(s).or_else(|_| DateTime::parse_from_rfc3339(s)).ok().map(|d| d.timestamp()) -} - -/// Apple's podcast categories, each with its subcategories -/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own -/// `<itunes:category>` comes from, and a fixed one, so the Directory's categories stay few. -const CATEGORIES: &[(&str, &[&str])] = &[ - ("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]), - ("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]), - ("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]), - ("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]), - ("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]), - ("Government", &[]), - ("History", &[]), - ("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]), - ("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]), - ( - "Leisure", - &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"], - ), - ("Music", &["Music Commentary", "Music History", "Music Interviews"]), - ( - "News", - &[ - "Business News", - "Daily News", - "Entertainment News", - "News Commentary", - "Politics", - "Sports News", - "Tech News", - ], - ), - ( - "Religion & Spirituality", - &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"], - ), - ( - "Science", - &[ - "Astronomy", - "Chemistry", - "Earth Sciences", - "Life Sciences", - "Mathematics", - "Natural Sciences", - "Nature", - "Physics", - "Social Sciences", - ], - ), - ("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]), - ( - "Sports", - &[ - "Baseball", - "Basketball", - "Cricket", - "Fantasy Sports", - "Football", - "Golf", - "Hockey", - "Rugby", - "Running", - "Soccer", - "Swimming", - "Tennis", - "Volleyball", - "Wilderness", - "Wrestling", - ], - ), - ("Technology", &[]), - ("True Crime", &[]), - ("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]), -]; - -/// Names that are not in Apple's list but mean one that is (#125). Jev picked from this list when -/// it spelt Football and Soccer its own way, and stored those answers; Software How-To is Apple's -/// Technology subcategory from before its 2019 list, which older feeds still name. -const ALIASES: &[(&str, &str)] = - &[("American Football", "Football"), ("Football (Soccer)", "Soccer"), ("Software How-To", "Technology")]; - -/// Apple's categories as ipx has them, for the directory to check against Apple's own (#252). -#[must_use] -pub fn categories() -> &'static [(&'static str, &'static [&'static str])] { - CATEGORIES -} - -/// Where a category sits in Apple's list, as (category, subcategory), so the Directory browses a -/// category and then its subcategories as Apple's does (#118). A podcast's own is stored as its -/// subcategory, and one an admin typed that is not Apple's stands as a category of its own. -#[must_use] -pub fn category_path(name: &str) -> (String, Option<String>) { - // A feed's own text, which can end in a stray comma: "Soccer," was a category of its own. - let name = name.trim().trim_end_matches([',', ';', '.']).trim_end(); - let name = ALIASES.iter().find(|(old, _)| old.eq_ignore_ascii_case(name)).map_or(name, |(_, now)| now); - for (category, subs) in CATEGORIES { - if category.eq_ignore_ascii_case(name) { - return (category.to_string(), None); - } - if let Some(sub) = subs.iter().find(|s| s.eq_ignore_ascii_case(name)) { - return (category.to_string(), Some(sub.to_string())); - } - } - (name.to_string(), None) -} - -/// TypeSafe's System One endpoint, where Jev answers questions about a piece of state. -pub const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone"; - -/// A Directory category for a feed that names none of its own, which most blogs do not (#117): -/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and -/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest -/// is taken however unsure, since an admin can change it and asking again would cost a call a scan. -pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result<String> { - let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect(); - // A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous. - let mut criteria = serde_json::Map::new(); - for (category, subs) in CATEGORIES { - criteria.insert(category.to_string(), serde_json::Value::Null); - for sub in *subs { - criteria.insert(sub.to_string(), format!("Within {category}").into()); - } - } - let body = serde_json::json!({ - "model": "jev-latest", - "state": { "title": feed.title, "recent_items": items }, - "questions": { "category": { - "type": "choice", - "instructions": "Which category best describes what this feed publishes?", - "criteria": criteria, - }}, - }); - let answer: serde_json::Value = client - .post(JEV_URL) - .bearer_auth(key) - .json(&body) - .timeout(FEED_TIMEOUT) - .send() - .await? - .error_for_status()? - .json() - .await?; - picked(&answer) -} - -fn picked(answer: &serde_json::Value) -> Result<String> { - let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default(); - CATEGORIES - .iter() - .flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter())) - .find(|c| **c == choice) - .map(|c| c.to_string()) - .ok_or_else(|| anyhow!("no category in the answer: {answer}")) -} - -/// An address that carries a subscriber's own access: a paid-feed host, credentials in the URL, -/// or a key in the query. ipx leaves such a feed out of anything others see, and the directory -/// refuses one sent to it (#202). -/// -/// ponytail: a heuristic. A key hidden in the path of a host not in `PAID_HOSTS` gets -/// through; a per-feed private flag is the upgrade (#205). -#[must_use] -pub fn url_looks_private(feed_url: &str) -> bool { - let Ok(u) = url::Url::parse(feed_url) else { return true }; - // Paid-feed services put the subscriber's key in the path as often as in the query, and a - // long path segment alone proves nothing (acast's public show ids look the same). So a - // feed from one of them is private whatever its URL looks like. Supercast is why this - // exists: `feeds.supercast.com/feeds/<key>` reached the directory before it did. - let host = u.host_str().unwrap_or(""); - PAID_HOSTS.iter().any(|h| host == *h || host.ends_with(&format!(".{h}"))) - || !u.username().is_empty() - || u.password().is_some() - || u.query_pairs().any(|(k, _)| { - let k = k.to_ascii_lowercase(); - ["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"].iter().any(|w| k.contains(w)) - }) -} - -/// Services whose feeds are always one subscriber's own. -const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"]; - -#[cfg(test)] -mod tests { - #[test] - fn reads_wordpress_posts_as_a_feed() { - // As WordPress.com's API sends them, trimmed (#317). - let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00", - "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"}, - "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/", - "title":{"rendered":"Baby’s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"}, - "jetpack_featured_media_url":""}]"#; - let feed = super::parse(json).expect("WordPress posts parse as a feed"); - assert_eq!(feed.title, None); - let e = &feed.entries[0]; - assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260"); - assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky")); - assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/")); - assert_eq!(e.published, Some(1_791_552_840)); - assert_eq!(e.description.as_deref(), Some("<p>Notes</p>")); - assert_eq!(e.image, None); - assert!(super::is_wpcom_challenge( - b"<!DOCTYPE html><title>Checking your browser...