From 245fa79fbf9304e2cac9e8029e808c45b2879147 Mon Sep 17 00:00:00 2001 From: rays Date: Wed, 4 Nov 2026 16:46:14 +0000 Subject: [PATCH] A YouTube feed's spurious 404 or 500 is asked again (#382) YouTube answers some requests for a channel's feed with a 404 or 500 in tens of milliseconds, while the next request for the same feed, or another channel's 200 ms later, reads. Checked once a day, the 39 YouTube channels in the catalogue missed most days: 178 failures in the week to 2026-11-04, against 74 in the ten days before, and all 39 read from Tower when asked by hand. feed::fetch now asks a youtube.com/feeds/videos.xml address again, up to twice, a second apart, when it answers 404 or 5xx. Other hosts' 404s are real and are not retried. tokio moves from ipx-feed's dev dependencies to its own, for the sleep; both services build it already. Co-Authored-By: Claude Opus 5.5 --- crates/directory/CHANGELOG.md | 4 ++++ crates/feed/Cargo.toml | 4 +--- crates/feed/src/fetch.rs | 35 ++++++++++++++++++++++++++++++++++- crates/ipx/CHANGELOG.md | 1 + 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index bb39e57..0b3d4db 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Its JSON log lines no longer carry their span, once on its own and again in a list, which said nothing the line's own fields did not (#363). +### Fixed + +- A YouTube channel's check no longer fails on the 404 or 500 YouTube answers some feed requests with and does not mean: it is asked again, twice, a second apart (#382). + ## [1.4.1] - 2026-10-24 ### Fixed diff --git a/crates/feed/Cargo.toml b/crates/feed/Cargo.toml index 7115df9..986a507 100644 --- a/crates/feed/Cargo.toml +++ b/crates/feed/Cargo.toml @@ -19,9 +19,7 @@ quick-xml.workspace = true reqwest.workspace = true rss.workspace = true serde_json.workspace = true +tokio.workspace = true tower.workspace = true tracing.workspace = true url.workspace = true - -[dev-dependencies] -tokio.workspace = true diff --git a/crates/feed/src/fetch.rs b/crates/feed/src/fetch.rs index b0289bd..ce21457 100644 --- a/crates/feed/src/fetch.rs +++ b/crates/feed/src/fetch.rs @@ -41,6 +41,19 @@ pub static FETCHED_BYTES: AtomicU64 = AtomicU64::new(0); /// catalogue was 62 MB, and 50 MB here stopped it updating (#357). pub(crate) const MAX_BODY: usize = 200 << 20; +/// How many times a YouTube feed's spurious failure is asked again, and how long between. +const YOUTUBE_RETRIES: u32 = 2; +const YOUTUBE_RETRY_AFTER: Duration = Duration::from_secs(1); + +/// YouTube answers some feed requests with a 404 or 500 it does not mean, in tens of +/// milliseconds, while the next request for the same feed reads; checked once a day, a channel +/// then missed most days (#382). +fn youtube_spurious(url: &reqwest::Url, status: StatusCode) -> bool { + url.host_str() == Some("www.youtube.com") + && url.path() == "/feeds/videos.xml" + && (status == StatusCode::NOT_FOUND || status.is_server_error()) +} + /// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read. pub(crate) async fn read_capped(mut resp: reqwest::Response) -> Result> { if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) { @@ -68,7 +81,9 @@ pub async fn fetch( let start = reqwest::Url::parse(feed_url).context("the feed's address")?; let mut url = start.clone(); let mut permanent = true; - for _ in 0..10 { + let mut retries = YOUTUBE_RETRIES; + // Ten redirects, and the retries beside them. + for _ in 0..10 + YOUTUBE_RETRIES { let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT); if let Some(tag) = etag { req = req.header(IF_NONE_MATCH, tag); @@ -100,6 +115,11 @@ pub async fn fetch( if status == StatusCode::NOT_MODIFIED { return Ok((Fetched::NotModified, moved)); } + if retries > 0 && youtube_spurious(&url, status) { + retries -= 1; + tokio::time::sleep(YOUTUBE_RETRY_AFTER).await; + continue; + } if !status.is_success() { // WordPress.com answers some readers' feed requests with a browser check that only // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the @@ -167,6 +187,19 @@ mod tests { format!("http://{addr}") } + #[test] + fn only_a_youtube_feeds_404_or_5xx_is_asked_again() { + use reqwest::StatusCode; + let yt = reqwest::Url::parse("https://www.youtube.com/feeds/videos.xml?user=x").unwrap(); + let other = reqwest::Url::parse("https://example.com/feeds/videos.xml").unwrap(); + let page = reqwest::Url::parse("https://www.youtube.com/watch?v=x").unwrap(); + assert!(super::youtube_spurious(&yt, StatusCode::NOT_FOUND)); + assert!(super::youtube_spurious(&yt, StatusCode::INTERNAL_SERVER_ERROR)); + assert!(!super::youtube_spurious(&yt, StatusCode::FORBIDDEN)); + assert!(!super::youtube_spurious(&other, StatusCode::NOT_FOUND)); + assert!(!super::youtube_spurious(&page, StatusCode::NOT_FOUND)); + } + #[tokio::test] async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() { let base = redirecting_server().await; diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index 2b52c94..d8b97e3 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A YouTube channel no longer misses most days' videos: YouTube answers some feed requests with a 404 or 500 it does not mean, and ipx now asks again, twice, a second apart, before calling the feed failing (#382). - Syncing an OPML or Patreon list reads who subscribes to it once, not once for every feed it lists: a list of 922 was thousands of the same queries a sync (#377). - A feed you already have, added again by the same address over http rather than https or the other way round, is the one you have, not a second subscription with every item twice. Changing a feed's address to another feed's over the other scheme is refused the same way (#362). - A podcast that was once in an OPML and is now a feed of its own is no longer hidden from All Subscriptions when that OPML is, so its unread items are listed there as the feed list counts them (#358).