rays/ipx
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Follow a feed that has moved for good to its new address (#115)

Browse Source 
A feed whose address answered with a permanent redirect was read through it on every check,
and the catalogue kept the old address: 28 of 147 feeds in production, most http to https, some
to a new path or domain.

Feeds are now fetched with a client of their own that follows no redirects (Ctx::feed_client),
and feed::fetch follows them itself, up to 10 hops, so it sees each one. When every hop was
permanent (301 or 308) it says where the feed ended up, and the scan moves the feed there in the
catalogue (follow_move). A temporary hop (302, 307) anywhere moves nothing. A feed an OPML lists
is left alone, as the OPML would put the old address back, and so is a move onto an address
another feed has. A password goes only to the feed's own host, never to a redirect elsewhere;
reqwest's own following dropped it the same way. Ten hops is a loop, worded as reqwest worded
it so it still reads as redirect_loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-02 20:05:43 +00:00
parent ec8fd5dd86
commit f7f4b466dc
3 changed files with 137 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
src/feed.rs
View File

@@ -59,43 +59,66 @@ pub enum Fetched {
/// downloads, which are not, have no such limit.
pub const FEED_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
/// Conditional GET. reqwest handles gzip and redirects; the original's hand-rolled
/// CONNECT/socket.ssl proxy path is gone -- `system-proxy` reads http_proxy/https_proxy.
/// Conditional GET, following redirects itself: `client` must follow none (`feed_client`), so
/// each hop is seen. The second value is where the feed now is when every hop said so for good
/// (301 or 308): a publisher that moved its feed, which the catalogue should follow rather
/// than be redirected on every read. A temporary redirect (302, 307) moves nothing.
/// `system-proxy` reads http_proxy/https_proxy.
#[tracing::instrument(skip_all, fields(url = %cfg.url))]
pub async fn fetch(
client: &reqwest::Client,
cfg: &FeedCfg,
etag: Option<&str>,
last_modified: Option<&str>,
) -> Result<Fetched> {
let mut req = client.get(&cfg.url).timeout(FEED_TIMEOUT);
if let Some(tag) = etag {
req = req.header(IF_NONE_MATCH, tag);
}
if let Some(lm) = last_modified {
req = req.header(IF_MODIFIED_SINCE, lm);
}
if let Some(user) = &cfg.username {
req = req.basic_auth(user, cfg.password());
}
) -> Result<(Fetched, Option<String>)> {
let start = reqwest::Url::parse(&cfg.url).context("the feed's address")?;
let mut url = start.clone();
let mut permanent = true;
for _ in 0..10 {
let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT);
if let Some(tag) = etag {
req = req.header(IF_NONE_MATCH, tag);
}
if let Some(lm) = last_modified {
req = req.header(IF_MODIFIED_SINCE, lm);
}
// The feed's own host only: a redirect elsewhere must not be handed the password.
if let Some(user) = &cfg.username
&& url.host_str() == start.host_str()
{
req = req.basic_auth(user, cfg.password());
}
let resp = req.send().await.context("connecting")?;
if resp.status() == StatusCode::NOT_MODIFIED {
return Ok(Fetched::NotModified);
let resp = req.send().await.context("connecting")?;
let status = resp.status();
if status.is_redirection() && status != StatusCode::NOT_MODIFIED {
let to = resp
.headers()
.get(reqwest::header::LOCATION)
.and_then(|v| v.to_str().ok())
.ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?;
url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?;
permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT);
continue;
}
let moved = (permanent && url != start).then(|| url.to_string());
if status == StatusCode::NOT_MODIFIED {
return Ok((Fetched::NotModified, moved));
}
if !status.is_success() {
// The original surfaced 401/407 specially; the code is enough for a UI to switch on.
return Err(anyhow!("HTTP {status}"));
}
let header = |h: reqwest::header::HeaderName| {
resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned)
};
let etag = header(ETAG);
let last_modified = header(LAST_MODIFIED);
let bytes = resp.bytes().await.context("reading body")?.to_vec();
return Ok((Fetched::Body { bytes, etag, last_modified }, moved));
}
let status = resp.status();
if !status.is_success() {
// The original surfaced 401/407 specially; the code is enough for a UI to switch on.
return Err(anyhow!("HTTP {status}"));
}
let header = |h: reqwest::header::HeaderName| {
resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned)
};
let etag = header(ETAG);
let last_modified = header(LAST_MODIFIED);
let bytes = resp.bytes().await.context("reading body")?.to_vec();
Ok(Fetched::Body { bytes, etag, last_modified })
// Worded as reqwest worded it, which failure_kind reads as a redirect loop.
Err(anyhow!("error following redirect for url ({url}): too many redirects"))
}
/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
@@ -987,6 +1010,54 @@ fn parse_date(s: &str) -> Option<i64> {
#[cfg(test)]
mod tests {
/// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
/// to /tmp, /new is the feed.
async fn redirecting_server() -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else { return };
tokio::spawn(async move {
let mut buf = [0u8; 2048];
let n = sock.read(&mut buf).await.unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]);
let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned();
let body = "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title></channel></rss>";
let resp = match path.as_str() {
"/old" => "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/chain" => "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/loop" => "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned(),
_ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()),
};
let _ = sock.write_all(resp.as_bytes()).await;
});
}
});
format!("http://{addr}")
}
#[tokio::test]
async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() {
let base = redirecting_server().await;
let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap();
let get = |path: &str| {
let cfg: crate::config::Feed = serde_json::from_value(serde_json::json!({ "url": format!("{base}{path}") })).unwrap();
let client = client.clone();
async move { super::fetch(&client, &cfg, None, None).await }
};
let (got, moved) = get("/old").await.unwrap();
assert!(matches!(got, super::Fetched::Body { .. }));
assert_eq!(moved, Some(format!("{base}/new")));
assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
assert_eq!(get("/new").await.unwrap().1, None); // never moved
let looped = get("/loop").await.err().unwrap().to_string();
assert_eq!(super::failure_kind(&looped).0, "redirect_loop");
}
use super::*;
#[test]