rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

Read a WordPress.com site's posts from its API when the feed is behind a browser check (#317)

Browse Source 
pelgranepress.com is on WordPress.com, whose CDN answers /feed/, /feed/atom/ and ?feed=rss2
with HTTP 403 and "Checking your browser...", a JavaScript challenge, whatever the User-Agent.
The checks that hit it failed for hours (8 in the day before this), and the publisher has
not changed anything.

The API at public-api.wordpress.com/wp/v2/sites/<host>/posts has the same posts with no check,
and each post's guid is the feed's own (https://host/?p=<id>), so the items read from it are
the ones already stored. fetch recognises the challenge by its /__challenge/ script, asks the
API instead, and parse reads WordPress's posts JSON as a feed. It sends no title or image for
the feed, which record_feed keeps as they were.

shortcut: the API's default ten posts, as many as WordPress puts in a feed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-23 12:06:22 +00:00
parent 84b4203daf
commit 4af12363bc
3 changed files with 96 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
crates/feed/src/lib.rs
View File

@@ -175,6 +175,15 @@ pub async fn fetch(
return Ok((Fetched::NotModified, moved));
}
if !status.is_success() {
// WordPress.com answers some readers' feed requests with a browser check that only
// JavaScript passes, for hours at a time and whatever the User-Agent; its API has the
// same posts and no check (#317).
if status == StatusCode::FORBIDDEN
&& let Some(host) = url.host_str().map(str::to_owned)
&& read_capped(resp).await.is_ok_and(|b| is_wpcom_challenge(&b))
{
return Ok((wpcom_posts(client, &host).await?, moved));
}
// The original surfaced 401/407 specially; the code is enough for a UI to switch on.
return Err(anyhow!("HTTP {status}"));
}
@@ -191,6 +200,57 @@ pub async fn fetch(
Err(anyhow!("error following redirect for url ({url}): too many redirects"))
}
/// WordPress.com's browser check, in place of the feed it was asked for.
fn is_wpcom_challenge(body: &[u8]) -> bool {
String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/")
}
/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed.
/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed
/// set to carry more gets ten here.
async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result<Fetched> {
let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts");
let resp = client
.get(&api)
.timeout(FEED_TIMEOUT)
.send()
.await
.context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?;
anyhow::ensure!(
resp.status().is_success(),
"HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}",
resp.status()
);
let bytes = read_capped(resp).await.context("reading WordPress.com's API")?;
FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
Ok(Fetched::Body { bytes, etag: None, last_modified: None })
}
/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's
/// own feed gives each post, so they are the items already stored. No title for the feed: the
/// stored one stays.
fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed {
let entries = posts
.iter()
.filter_map(|p| {
Some(Entry {
guid: non_empty(p["guid"]["rendered"].as_str()).or_else(|| non_empty(p["link"].as_str()))?,
title: title_text(p["title"]["rendered"].as_str()),
link: non_empty(p["link"].as_str()),
published: p["date_gmt"]
.as_str()
.and_then(|d| chrono::NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok())
.map(|d| d.and_utc().timestamp()),
description: non_empty(p["content"]["rendered"].as_str()),
image: non_empty(p["jetpack_featured_media_url"].as_str()),
..Default::default()
})
})
.collect();
ParsedFeed { entries, ..Default::default() }
}
/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
/// it is, and whether there is a new address to switch to.
pub struct Failure {
@@ -432,6 +492,11 @@ fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> {
/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
#[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
if bytes.trim_ascii_start().starts_with(b"[")
&& let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
{
return Ok(from_wp_posts(&posts));
}
match rss::Channel::read_from(bytes) {
Ok(ch) => Ok(from_rss(ch, bytes)),
Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
@@ -1292,6 +1357,29 @@ const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm
#[cfg(test)]
mod tests {
#[test]
fn reads_wordpress_posts_as_a_feed() {
// As WordPress.com's API sends them, trimmed (#317).
let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00",
"guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"},
"link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/",
"title":{"rendered":"Baby&#8217;s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"},
"jetpack_featured_media_url":""}]"#;
let feed = super::parse(json).expect("WordPress posts parse as a feed");
assert_eq!(feed.title, None);
let e = &feed.entries[0];
assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260");
assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky"));
assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/"));
assert_eq!(e.published, Some(1_791_552_840));
assert_eq!(e.description.as_deref(), Some("<p>Notes</p>"));
assert_eq!(e.image, None);
assert!(super::is_wpcom_challenge(
b"<!DOCTYPE html><title>Checking your browser...</title><script src=\"/__challenge/probes.js\">"
));
assert!(!super::is_wpcom_challenge(b"<html>Forbidden</html>"));
}
#[test]
fn redact_drops_what_carries_a_credential() {
assert_eq!(super::redact("https://www.patreon.com/rss/x?auth=SECRET&show=1"), "https://www.patreon.com/rss/x");