diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index 6021df8..3a2eb24 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- A WordPress.com site that answers its feed with a browser check (HTTP 403) is read from WordPress.com's API instead of failing (#317). + ## [1.3.0] - 2026-10-19 ### Added diff --git a/crates/feed/src/lib.rs b/crates/feed/src/lib.rs index 27f82a9..a1117f6 100644 --- a/crates/feed/src/lib.rs +++ b/crates/feed/src/lib.rs @@ -175,6 +175,15 @@ pub async fn fetch( return Ok((Fetched::NotModified, moved)); } if !status.is_success() { + // WordPress.com answers some readers' feed requests with a browser check that only + // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the + // same posts and no check (#317). + if status == StatusCode::FORBIDDEN + && let Some(host) = url.host_str().map(str::to_owned) + && read_capped(resp).await.is_ok_and(|b| is_wpcom_challenge(&b)) + { + return Ok((wpcom_posts(client, &host).await?, moved)); + } // The original surfaced 401/407 specially; the code is enough for a UI to switch on. return Err(anyhow!("HTTP {status}")); } @@ -191,6 +200,57 @@ pub async fn fetch( Err(anyhow!("error following redirect for url ({url}): too many redirects")) } +/// WordPress.com's browser check, in place of the feed it was asked for. +fn is_wpcom_challenge(body: &[u8]) -> bool { + String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/") +} + +/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed. +/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed +/// set to carry more gets ten here. +async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result { + let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts"); + let resp = client + .get(&api) + .timeout(FEED_TIMEOUT) + .send() + .await + .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?; + anyhow::ensure!( + resp.status().is_success(), + "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}", + resp.status() + ); + let bytes = read_capped(resp).await.context("reading WordPress.com's API")?; + FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed); + Ok(Fetched::Body { bytes, etag: None, last_modified: None }) +} + +/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's +/// own feed gives each post, so they are the items already stored. No title for the feed: the +/// stored one stays. +fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed { + let entries = posts + .iter() + .filter_map(|p| { + Some(Entry { + guid: non_empty(p["guid"]["rendered"].as_str()).or_else(|| non_empty(p["link"].as_str()))?, + title: title_text(p["title"]["rendered"].as_str()), + link: non_empty(p["link"].as_str()), + published: p["date_gmt"] + .as_str() + .and_then(|d| chrono::NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok()) + .map(|d| d.and_utc().timestamp()), + description: non_empty(p["content"]["rendered"].as_str()), + image: non_empty(p["jetpack_featured_media_url"].as_str()), + ..Default::default() + }) + }) + .collect(); + ParsedFeed { entries, ..Default::default() } +} + /// A stored `last_error`, translated into plain words for whoever subscribes: whose problem /// it is, and whether there is a new address to switch to. pub struct Failure { @@ -432,6 +492,11 @@ fn parse_patreon_shows(json: &[u8]) -> Result { /// RSS first, then Atom -- the same split the original made on `parsedFeed.version`. #[tracing::instrument(skip_all, fields(bytes = bytes.len()))] pub fn parse(bytes: &[u8]) -> Result { + if bytes.trim_ascii_start().starts_with(b"[") + && let Ok(posts) = serde_json::from_slice::>(bytes) + { + return Ok(from_wp_posts(&posts)); + } match rss::Channel::read_from(bytes) { Ok(ch) => Ok(from_rss(ch, bytes)), Err(rss_err) => match atom_syndication::Feed::read_from(bytes) { @@ -1292,6 +1357,29 @@ const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm #[cfg(test)] mod tests { + #[test] + fn reads_wordpress_posts_as_a_feed() { + // As WordPress.com's API sends them, trimmed (#317). + let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00", + "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"}, + "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/", + "title":{"rendered":"Baby’s First Whisky"},"content":{"rendered":"

Notes<\/p>"}, + "jetpack_featured_media_url":""}]"#; + let feed = super::parse(json).expect("WordPress posts parse as a feed"); + assert_eq!(feed.title, None); + let e = &feed.entries[0]; + assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260"); + assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky")); + assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/")); + assert_eq!(e.published, Some(1_791_552_840)); + assert_eq!(e.description.as_deref(), Some("

Notes

")); + assert_eq!(e.image, None); + assert!(super::is_wpcom_challenge( + b"Checking your browser...