diff --git a/crates/feed/src/fetch.rs b/crates/feed/src/fetch.rs index ce21457..1fc534b 100644 --- a/crates/feed/src/fetch.rs +++ b/crates/feed/src/fetch.rs @@ -49,9 +49,14 @@ const YOUTUBE_RETRY_AFTER: Duration = Duration::from_secs(1); /// milliseconds, while the next request for the same feed reads; checked once a day, a channel /// then missed most days (#382). fn youtube_spurious(url: &reqwest::Url, status: StatusCode) -> bool { - url.host_str() == Some("www.youtube.com") - && url.path() == "/feeds/videos.xml" - && (status == StatusCode::NOT_FOUND || status.is_server_error()) + is_youtube_feed(url.as_str()) && (status == StatusCode::NOT_FOUND || status.is_server_error()) +} + +/// A YouTube channel's or playlist's feed, which fails most reads in streaks that outlast +/// `fetch`'s retries, so a scheduler asks it again sooner than other feeds (#386). +#[must_use] +pub fn is_youtube_feed(url: &str) -> bool { + reqwest::Url::parse(url).is_ok_and(|u| u.host_str() == Some("www.youtube.com") && u.path() == "/feeds/videos.xml") } /// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read. diff --git a/crates/feed/src/lib.rs b/crates/feed/src/lib.rs index 5299bf4..e685b9d 100644 --- a/crates/feed/src/lib.rs +++ b/crates/feed/src/lib.rs @@ -18,7 +18,7 @@ pub use artwork::{is_image, prefer_https, site_icon}; pub use category::{categories, category_path}; pub use discover::{expand_input, find_feed}; pub use failure::{Failure, explain_failure, failure_kind, is_temporary_lookup_failure}; -pub use fetch::{FEED_SILENCE, FEED_TIMEOUT, FETCHED_BYTES, FETCHES, Fetched, fetch}; +pub use fetch::{FEED_SILENCE, FEED_TIMEOUT, FETCHED_BYTES, FETCHES, Fetched, fetch, is_youtube_feed}; pub use jev::{JEV_URL, classify}; pub use outlines::{collect_outlines, is_opml, opml_title, parse_opml, read_opml}; pub use parse::{Enclosure, Entry, ParsedFeed, markdown_html, parse, same_file_key}; diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index 3ea5fd4..c3f9777 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- A YouTube channel no longer misses most days' videos: YouTube answers some feed requests with a 404 or 500 it does not mean, and ipx now asks again, twice, a second apart, before calling the feed failing (#382). +- A YouTube channel no longer misses most days' videos: YouTube answers some feed requests with a 404 or 500 it does not mean, and ipx now asks again, twice, a second apart, before calling the feed failing (#382), and then every hour for a day where it would have waited a day, since YouTube's failures run for hours (#386). - Syncing an OPML or Patreon list reads who subscribes to it once, not once for every feed it lists: a list of 922 was thousands of the same queries a sync (#377). - A feed you already have, added again by the same address over http rather than https or the other way round, is the one you have, not a second subscription with every item twice. Changing a feed's address to another feed's over the other scheme is refused the same way (#362). - A podcast that was once in an OPML and is now a feed of its own is no longer hidden from All Subscriptions when that OPML is, so its unread items are listed there as the feed list counts them (#358). diff --git a/crates/ipx/src/scan.rs b/crates/ipx/src/scan.rs index b7b6bf6..5f7d472 100644 --- a/crates/ipx/src/scan.rs +++ b/crates/ipx/src/scan.rs @@ -171,7 +171,13 @@ pub(crate) async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope pub fn due_at(cfg: &config::Config, feed: &config::Feed, state: &db::HttpState, subscribed: bool) -> Option { let last = state.last_checked?; let floor = if subscribed { 0 } else { 86_400 }; - Some(last + due_after(cfg, feed, state.ttl_mins, state.error_since, state.same_since, last).max(floor) as i64) + let wait = due_after(cfg, feed, state.ttl_mins, state.error_since, state.same_since, last).max(floor); + // YouTube fails about 70% of reads from Tower, in streaks longer than fetch's retries; read + // once a day, its 39 catalogue channels missed 5 days in 7 (#386). Hourly for the first day of + // a run of failures finds a moment it answers, and a channel gone for good costs 24 requests. + let youtube_again = + state.error_since.is_some_and(|since| last - since < 86_400) && feed::is_youtube_feed(&feed.url); + Some(last + if youtube_again { wait.min(3600) } else { wait } as i64) } /// A per-feed schedule is an explicit instruction and wins outright. Without one, the @@ -907,6 +913,20 @@ mod tests { assert_eq!(at(Some(1000), false), Some(1000 + 86_400)); // listed, nobody subscribed } + #[test] + fn a_failing_youtube_feed_is_asked_again_hourly_for_a_day() { + let cfg = config::Config::default(); + let yt = config::Feed { url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCx".into(), ..feed() }; + let at = |f: &config::Feed, since: Option, last: i64| { + let state = db::HttpState { last_checked: Some(last), error_since: since, ..Default::default() }; + due_at(&cfg, f, &state, false).map(|t| t - last) + }; + assert_eq!(at(&yt, None, 1000), Some(86_400), "well, it waits its day like any listed feed"); + assert_eq!(at(&yt, Some(1000), 1000 + 7200), Some(3600), "failing two hours: in an hour"); + assert_eq!(at(&yt, Some(1000), 1000 + 90_000), Some(86_400), "failing over a day: back off"); + assert_eq!(at(&feed(), Some(1000), 1000 + 7200), Some(86_400), "another host's failure is not hurried"); + } + #[test] fn a_feed_long_the_same_is_checked_less_and_a_schedule_set_wins() { let (hour, day) = (3600u64, 86_400u64);