Follow a feed that has moved for good to its new address (#115)

A feed whose address answered with a permanent redirect was read through it on every check,
and the catalogue kept the old address: 28 of 147 feeds in production, most http to https, some
to a new path or domain.

Feeds are now fetched with a client of their own that follows no redirects (Ctx::feed_client),
and feed::fetch follows them itself, up to 10 hops, so it sees each one. When every hop was
permanent (301 or 308) it says where the feed ended up, and the scan moves the feed there in the
catalogue (follow_move). A temporary hop (302, 307) anywhere moves nothing. A feed an OPML lists
is left alone, as the OPML would put the old address back, and so is a move onto an address
another feed has. A password goes only to the feed's own host, never to a redirect elsewhere;
reqwest's own following dropped it the same way. Ten hops is a loop, worded as reqwest worded
it so it still reads as redirect_loop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main.rs

@@ -125,6 +125,9 @@ pub struct Ctx {
     pub cfg: std::sync::RwLock<std::sync::Arc<config::Config>>,
     pub db: db::Db,
     pub client: reqwest::Client,
+    /// For feeds only: follows no redirects, so `feed::fetch` sees each hop and can tell a feed
+    /// that moved for good from one sent elsewhere for now.
+    pub feed_client: reqwest::Client,
     pub out: Emitter,
     /// Started on first use: a BitTorrent session binds ports and starts a DHT, which is
     /// rude to do for a config that has never seen a torrent.
@@ -258,6 +261,11 @@ async fn main() -> Result<()> {
             // Connecting only, so it bounds a download's start, not a long download (#108).
             .connect_timeout(std::time::Duration::from_secs(10))
             .build()?,
+        feed_client: reqwest::Client::builder()
+            .user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
+            .connect_timeout(std::time::Duration::from_secs(10))
+            .redirect(reqwest::redirect::Policy::none())
+            .build()?,
         out: if is_daemon { Emitter::socket(events.clone(), false) } else { Emitter::terminal() },
         torrents: tokio::sync::OnceCell::new(),
         torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),
@@ -741,7 +749,7 @@ pub async fn add_one(
         listed: false,
     };
 
-    let title = match feed::fetch(&ctx.client, &probe, None, None).await {
+    let title = match feed::fetch(&ctx.feed_client, &probe, None, None).await.map(|(got, _)| got) {
         // An OPML subscription is named from its own <head><title>, not by trying to
         // parse it as a feed and falling back to the hostname.
         Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
@@ -1077,7 +1085,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
     const AHEAD: usize = 6;
     let mut bodies = futures_util::StreamExt::buffered(
         futures_util::stream::iter(due.iter().map(|(sub, state)| {
-            let (client, feed_cfg) = (ctx.client.clone(), sub.cfg.clone());
+            let (client, feed_cfg) = (ctx.feed_client.clone(), sub.cfg.clone());
             let (etag, modified) = (state.etag.clone(), state.last_modified.clone());
             let early = !feed::is_patreon_creator(&feed_cfg.url);
             tokio::spawn(tracing::Instrument::instrument(
@@ -1361,6 +1369,23 @@ enum Outcome {
     Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
 }
 
+/// A feed that answered from a new address after permanent redirects moves there in the
+/// catalogue, so it is read from there and no longer redirected every time. One an OPML lists
+/// is left alone, since the OPML would only put the old address back; so is a move onto an
+/// address another feed already has.
+async fn follow_move(ctx: &Ctx, id: &str, to: &str) -> Result<()> {
+    if subscriptions(ctx).await?.iter().any(|s| s.id != id && feed::same_feed(&s.cfg.url, to)) {
+        tracing::info!(feed = id, to, "the feed moved to an address another feed already has; leaving it");
+        return Ok(());
+    }
+    let mut cfg = (*ctx.cfg()).clone();
+    let Some(f) = cfg.feeds.get_mut(id) else { return Ok(()) };
+    let from = std::mem::replace(&mut f.url, to.to_owned());
+    ctx.store_cfg(cfg).await?;
+    tracing::info!(feed = id, from, to, "the feed moved for good; following it to its new address");
+    Ok(())
+}
+
 #[tracing::instrument(name = "feed", skip_all, fields(feed = id))]
 async fn scan_one(
     ctx: &Arc<Ctx>,
@@ -1368,8 +1393,8 @@ async fn scan_one(
     feed_cfg: &config::Feed,
     state: &db::HttpState,
     force: bool,
-    // The body the scan already fetched for it, if it did (#104).
-    prefetched: Option<Result<feed::Fetched>>,
+    // The body the scan already fetched for it, if it did (#104), and where it moved, if it did.
+    prefetched: Option<Result<(feed::Fetched, Option<String>)>>,
 ) -> Result<Outcome> {
     // The queue follows the feed's settings each time it is due, changed or not, so files a
     // limit or auto-download no longer reaches stop counting as waiting (#113).
@@ -1400,10 +1425,13 @@ async fn scan_one(
         }
     }
 
-    let mut fetched = match prefetched {
+    let (mut fetched, moved) = match prefetched {
         Some(got) => got?,
-        None => feed::fetch(&ctx.client, feed_cfg, state.etag.as_deref(), state.last_modified.as_deref()).await?,
+        None => feed::fetch(&ctx.feed_client, feed_cfg, state.etag.as_deref(), state.last_modified.as_deref()).await?,
     };
+    if let Some(to) = moved {
+        follow_move(ctx, id, &to).await?;
+    }
 
     // A 304 while nothing is stored means the validator has outlived the data -- a restore
     // from backup, a manual edit, a cleanup that removed entries. Believe the database over
@@ -1414,7 +1442,7 @@ async fn scan_one(
     if matches!(fetched, feed::Fetched::NotModified) && (stored.entries == 0 || stored.image.is_none()) {
         tracing::info!(feed = id, "not modified, but nothing stored; refetching without the validator");
         ctx.db.clear_validators(id).await?;
-        fetched = feed::fetch(&ctx.client, feed_cfg, None, None).await?;
+        fetched = feed::fetch(&ctx.feed_client, feed_cfg, None, None).await?.0;
     }
 
     let (bytes, etag, last_modified) = match fetched {
@@ -2187,6 +2215,7 @@ mod tests {
             cfg: std::sync::RwLock::new(Arc::new(cfg)),
             db: db::Db::memory().await.unwrap(),
             client: reqwest::Client::new(),
+            feed_client: reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap(),
             out: Emitter::terminal(),
             torrents: tokio::sync::OnceCell::new(),
             torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),