Read a WordPress.com site's posts from its API when the feed is behind a browser check (#317)

pelgranepress.com is on WordPress.com, whose CDN answers /feed/, /feed/atom/ and ?feed=rss2
with HTTP 403 and "Checking your browser...", a JavaScript challenge, whatever the User-Agent.
The checks that hit it failed for hours (8 in the day before this), and the publisher has
not changed anything.

The API at public-api.wordpress.com/wp/v2/sites/<host>/posts has the same posts with no check,
and each post's guid is the feed's own (https://host/?p=<id>), so the items read from it are
the ones already stored. fetch recognises the challenge by its /__challenge/ script, asks the
API instead, and parse reads WordPress's posts JSON as a feed. It sends no title or image for
the feed, which record_feed keeps as they were.

shortcut: the API's default ten posts, as many as WordPress puts in a feed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/directory/CHANGELOG.md

@@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- A WordPress.com site that answers its feed with a browser check (HTTP 403) is read from WordPress.com's API instead of failing (#317).
+
 ## [1.3.0] - 2026-10-19
 
 ### Added

crates/feed/src/lib.rs

@@ -175,6 +175,15 @@ pub async fn fetch(
             return Ok((Fetched::NotModified, moved));
         }
         if !status.is_success() {
+            // WordPress.com answers some readers' feed requests with a browser check that only
+            // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the
+            // same posts and no check (#317).
+            if status == StatusCode::FORBIDDEN
+                && let Some(host) = url.host_str().map(str::to_owned)
+                && read_capped(resp).await.is_ok_and(|b| is_wpcom_challenge(&b))
+            {
+                return Ok((wpcom_posts(client, &host).await?, moved));
+            }
             // The original surfaced 401/407 specially; the code is enough for a UI to switch on.
             return Err(anyhow!("HTTP {status}"));
         }
@@ -191,6 +200,57 @@ pub async fn fetch(
     Err(anyhow!("error following redirect for url ({url}): too many redirects"))
 }
 
+/// WordPress.com's browser check, in place of the feed it was asked for.
+fn is_wpcom_challenge(body: &[u8]) -> bool {
+    String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/")
+}
+
+/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed.
+/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed
+/// set to carry more gets ten here.
+async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result<Fetched> {
+    let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts");
+    let resp = client
+        .get(&api)
+        .timeout(FEED_TIMEOUT)
+        .send()
+        .await
+        .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?;
+    anyhow::ensure!(
+        resp.status().is_success(),
+        "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}",
+        resp.status()
+    );
+    let bytes = read_capped(resp).await.context("reading WordPress.com's API")?;
+    FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
+    Ok(Fetched::Body { bytes, etag: None, last_modified: None })
+}
+
+/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's
+/// own feed gives each post, so they are the items already stored. No title for the feed: the
+/// stored one stays.
+fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed {
+    let entries = posts
+        .iter()
+        .filter_map(|p| {
+            Some(Entry {
+                guid: non_empty(p["guid"]["rendered"].as_str()).or_else(|| non_empty(p["link"].as_str()))?,
+                title: title_text(p["title"]["rendered"].as_str()),
+                link: non_empty(p["link"].as_str()),
+                published: p["date_gmt"]
+                    .as_str()
+                    .and_then(|d| chrono::NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok())
+                    .map(|d| d.and_utc().timestamp()),
+                description: non_empty(p["content"]["rendered"].as_str()),
+                image: non_empty(p["jetpack_featured_media_url"].as_str()),
+                ..Default::default()
+            })
+        })
+        .collect();
+    ParsedFeed { entries, ..Default::default() }
+}
+
 /// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
 /// it is, and whether there is a new address to switch to.
 pub struct Failure {
@@ -432,6 +492,11 @@ fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> {
 /// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
 #[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
 pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
+    if bytes.trim_ascii_start().starts_with(b"[")
+        && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
+    {
+        return Ok(from_wp_posts(&posts));
+    }
     match rss::Channel::read_from(bytes) {
         Ok(ch) => Ok(from_rss(ch, bytes)),
         Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
@@ -1292,6 +1357,29 @@ const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn reads_wordpress_posts_as_a_feed() {
+        // As WordPress.com's API sends them, trimmed (#317).
+        let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00",
+        "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"},
+        "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/",
+        "title":{"rendered":"Baby&#8217;s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"},
+        "jetpack_featured_media_url":""}]"#;
+        let feed = super::parse(json).expect("WordPress posts parse as a feed");
+        assert_eq!(feed.title, None);
+        let e = &feed.entries[0];
+        assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260");
+        assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky"));
+        assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/"));
+        assert_eq!(e.published, Some(1_791_552_840));
+        assert_eq!(e.description.as_deref(), Some("<p>Notes</p>"));
+        assert_eq!(e.image, None);
+        assert!(super::is_wpcom_challenge(
+            b"<!DOCTYPE html><title>Checking your browser...</title><script src=\"/__challenge/probes.js\">"
+        ));
+        assert!(!super::is_wpcom_challenge(b"<html>Forbidden</html>"));
+    }
+
     #[test]
     fn redact_drops_what_carries_a_credential() {
         assert_eq!(super::redact("https://www.patreon.com/rss/x?auth=SECRET&show=1"), "https://www.patreon.com/rss/x");

crates/ipx/CHANGELOG.md

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- A WordPress.com site that answers its feed with a browser check (HTTP 403, "Checking your browser...") is read from WordPress.com's API instead, so Pelgrane Press's feed no longer fails for hours at a time. The items are the ones the feed gave: same ids (#317).
+
 ## [1.4.0] - 2026-10-19
 
 ### Added