diff --git a/CHANGELOG.md b/CHANGELOG.md index 462a93b..27b79d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- A feed's page in the Directory says how many people here subscribe only when someone does, without "Nobody here subscribes yet". + +### Fixed + +- A moment's DNS failure no longer marks a feed as failing for a day or calls it gone: iPX tries it again in ten minutes. One failed lookup one morning left 205 podcasts unread, uncategorised and labelled gone until the next day. A name that does not resolve is said to be maybe gone, and only once it has been failing a day, on its Directory page as on its own. + ## [0.10.2] - 2026-10-05 ### Fixed diff --git a/docs/configuration.md b/docs/configuration.md index b250c38..15b8291 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -54,7 +54,8 @@ database as described above; `download_dir`, `socket` and `organize` stay in con * **`schedule`** — how often feeds are re-checked. A feed's own `` still wins when it asks to be polled *less* often, and a per-feed `schedule` overrides both. A feed that keeps failing waits as long as it has been failing before the next try, up to a day, and is back on schedule - after its first success. Admin-only from the UI. + after its first success. A moment's failure to look its host up does not count: the network, not + the feed, failed, and it is tried again in ten minutes. Admin-only from the UI. * **`organize`** — `feed` files downloads under the feed's folder; `date` under `YYYY-MM-DD`. * **`max_total_gb`** — the reaper deletes to get back under this, oldest first, keeping a 50 MB pad. Kept items are never deleted, and a file only counts as read once every subscriber has diff --git a/src/feed.rs b/src/feed.rs index 6d4e285..5b14200 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -196,15 +196,25 @@ pub fn explain_failure(msg: &str) -> Option { if low.contains("http 402") { return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None }); } - if low.contains("dns error") - || low.contains("failed to lookup address") - || low.contains("no address associated") + // Only a name that does not exist. A lookup that failed for the moment says nothing about the + // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143). + if !is_temporary_lookup_failure(msg) + && (low.contains("dns error") || low.contains("failed to lookup address") || low.contains("no address associated")) { - return Some(Failure { reason: "This address no longer resolves; the site is gone.", new_url: None }); + return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None }); } None } +/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's +/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does +/// not exist ("Name or service not known"). It says the network was not answering, not that the +/// feed is gone (#143). +pub fn is_temporary_lookup_failure(msg: &str) -> bool { + let low = msg.to_ascii_lowercase(); + low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again")) +} + /// True when a body is an OPML document rather than a feed. /// /// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an @@ -1317,8 +1327,13 @@ mod tests { explain_failure("the feed's address answered with nothing at all").unwrap().reason, "This address answers with nothing; the site may be gone." ); - let dns = explain_failure("connecting: dns error: failed to lookup address information").unwrap(); - assert_eq!(dns.reason, "This address no longer resolves; the site is gone."); + let dns = explain_failure("connecting: dns error: failed to lookup address information: Name or service not known").unwrap(); + assert_eq!(dns.reason, "This address no longer resolves; the site may be gone."); + // A lookup that failed for the moment is not a site gone (#143). + let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution"; + assert!(is_temporary_lookup_failure(blip)); + assert!(explain_failure(blip).is_none()); + assert!(!is_temporary_lookup_failure("connecting: dns error: failed to lookup address information: Name or service not known")); let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap(); assert_eq!(moved.new_url.as_deref(), Some("https://x/feed")); assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none()); diff --git a/src/main.rs b/src/main.rs index 2f45d6a..ad47b6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -141,9 +141,38 @@ pub struct Ctx { /// stops every feed scan, every HTTP download and every status command behind it. /// A one-shot CLI run keeps them inline, or the process would exit mid-download. pub detach_torrents: bool, + /// Feeds whose host could not be looked up for the moment, and when to try them again: a + /// failure that says nothing about the feed, kept off it and out of the database (#143). + /// Forgotten on a restart, which only makes the retry sooner. + pub retry: std::sync::Mutex>, } +/// How soon a feed whose host could not be looked up is tried again (#143). +const RETRY_LOOKUP_SECS: i64 = 600; + impl Ctx { + /// When a feed is next due, taking a lookup retry into account: not before it. + fn due(&self, cfg: &config::Config, feed: &config::Feed, id: &str, state: &db::HttpState, subscribed: bool) -> Option { + let at = due_at(cfg, feed, state, subscribed); + match self.retry.lock().unwrap().get(id) { + Some(&retry) => Some(at.unwrap_or(0).max(retry)), + None => at, + } + } + + /// A scan's failure, recorded on the feed, unless it was only a moment's failure to look its + /// host up: on 2026-10-05 a DNS outage on Tower failed 204 feeds in a minute with + /// "Temporary failure in name resolution", and each, read once a day and backed off as a + /// failing feed, sat a day failing, unread and labelled gone (#143). Tried again in minutes + /// instead, with nothing on the feed to say it failed. + async fn feed_failed(&self, id: &str, url: &str, msg: &str) -> Result<()> { + if feed::is_temporary_lookup_failure(msg) { + self.retry.lock().unwrap().insert(id.to_owned(), db::now() + RETRY_LOOKUP_SECS); + return Ok(()); + } + self.db.set_feed_error(id, url, msg).await + } + pub fn cfg(&self) -> std::sync::Arc { self.cfg.read().unwrap().clone() } @@ -275,6 +304,7 @@ async fn main() -> Result<()> { torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)), config_path: config_path.clone(), detach_torrents: is_daemon, + retry: Default::default(), }); let result = match cli.command { @@ -1093,7 +1123,7 @@ async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope: &[String] state.last_modified = None; } - if !force && let Some(at) = due_at(&cfg, &sub.cfg, &state, subscribed.contains_key(id)) { + if !force && let Some(at) = ctx.due(&cfg, &sub.cfg, id, &state, subscribed.contains_key(id)) { if at > db::now() { ctx.out.emit(Event::FeedSkip { feed: id.clone(), @@ -1165,7 +1195,7 @@ async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope: &[String] // One bad feed must not end the scan. let msg = format!("{e:#}"); ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() }); - ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?; + ctx.feed_failed(id, &feed_cfg.url, &msg).await?; } } } @@ -1191,7 +1221,7 @@ async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope: &[String] Err(e) => { let msg = format!("{e:#}"); ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() }); - ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?; + ctx.feed_failed(id, &feed_cfg.url, &msg).await?; } } } @@ -1329,7 +1359,9 @@ async fn until_next_scan(ctx: &Ctx) -> std::time::Duration { .iter() // A feed with no row yet has never been checked: due now. .map(|s| { - states.get(&s.id).and_then(|st| due_at(&cfg, &s.cfg, st, subscribed.contains_key(&s.id))).unwrap_or(0) + let never = db::HttpState::default(); + let state = states.get(&s.id).unwrap_or(&never); + ctx.due(&cfg, &s.cfg, &s.id, state, subscribed.contains_key(&s.id)).unwrap_or(0) }) .min(), ) @@ -2283,9 +2315,27 @@ mod tests { torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)), config_path: PathBuf::new(), detach_torrents: false, + retry: Default::default(), } } + #[tokio::test] + async fn a_moments_lookup_failure_is_tried_again_soon_and_not_held_against_the_feed() { + let ctx = test_ctx(config::Config::default()).await; + let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution"; + ctx.feed_failed("npr", "https://feeds.npr.org/x", blip).await.unwrap(); + // Nothing on the feed: no error, no run of failures to back off from or call it gone by. + let s = ctx.db.feed_summary("npr").await.unwrap(); + assert!(s.last_error.is_none() && s.error_since.is_none(), "{:?}", s.last_error); + // Due again in minutes, though a feed nobody subscribes to is otherwise read once a day. + let state = db::HttpState { last_checked: Some(db::now() - 2 * 86_400), ..Default::default() }; + let due = ctx.due(&config::Config::default(), &feed(), "npr", &state, false).unwrap(); + assert!((due - db::now() - RETRY_LOOKUP_SECS).abs() <= 2, "{due}"); + // A real failure is the feed's. + ctx.feed_failed("gone", "https://x/feed", "HTTP 404 Not Found").await.unwrap(); + assert_eq!(ctx.db.feed_summary("gone").await.unwrap().last_error.as_deref(), Some("HTTP 404 Not Found")); + } + #[tokio::test] async fn a_reload_takes_the_catalogue_another_ipx_wrote() { let with = |id: &str| { diff --git a/src/web.rs b/src/web.rs index 43ec937..01ef6bf 100644 --- a/src/web.rs +++ b/src/web.rs @@ -1029,10 +1029,15 @@ struct Listed { items: Vec, } -/// A feed's last error as its Directory page says it: explained, or that the check failed, never -/// the error as it stands, which can name the feed's address, and the Directory names none. -fn failure_words(err: &str) -> String { - crate::feed::explain_failure(err).map_or_else(|| "Its last check failed.".to_owned(), |f| f.reason.to_owned()) +/// A feed's last error as its Directory page says it: explained once it has been failing a day, as +/// the feed's own page does, or else that the check failed; never the error as it stands, which can +/// name the feed's address, and the Directory names none. A day, so that a site down for an hour, +/// or a resolver for a minute, is not called gone (#143). +fn failure_words(err: &str, failing_for: i64) -> String { + Some(err) + .filter(|_| failing_for >= FLAG_AFTER_SECS) + .and_then(crate::feed::explain_failure) + .map_or_else(|| "Its last check failed.".to_owned(), |f| f.reason.to_owned()) } /// A listed feed's description and latest twenty items. Only a feed the Directory lists, so a @@ -1049,7 +1054,9 @@ async fn get_listed( let rows = state.ctx.db.entries_in(user.id, Some(&id), crate::db::Filter::parse("all"), None, 0, 20, &order).await?; let mut sanitizer = feed_sanitizer(); let description = state.ctx.db.feed_description(&id).await?.map(|d| clean_description(&mut sanitizer, &d, None)); - let failing = state.ctx.db.feed_summary(&id).await?.last_error.as_deref().map(failure_words); + let summary = state.ctx.db.feed_summary(&id).await?; + let failing_for = summary.error_since.map_or(0, |since| crate::db::now() - since); + let failing = summary.last_error.as_deref().map(|e| failure_words(e, failing_for)); let items = rows .into_iter() .map(|e| ListedItem { @@ -1157,10 +1164,14 @@ mod tests { #[test] fn a_failing_feed_says_why_in_the_directory_without_its_address() { - assert_eq!(failure_words("the feed's address answered with nothing at all"), "This address answers with nothing; the site may be gone."); + let day = FLAG_AFTER_SECS; + let nothing = "the feed's address answered with nothing at all"; + assert_eq!(failure_words(nothing, day), "This address answers with nothing; the site may be gone."); + // Not before it has been failing a day, as on the feed's own page (#143). + assert_eq!(failure_words(nothing, 3600), "Its last check failed."); // reqwest names the URL; the Directory names none. let raw = "error sending request for url (http://daily-quests.com/comic/?feed=atom): connection closed"; - assert_eq!(failure_words(raw), "Its last check failed."); + assert_eq!(failure_words(raw, day), "Its last check failed."); } #[test] diff --git a/web/src/dialogs.ts b/web/src/dialogs.ts index 449291f..4a1614e 100644 --- a/web/src/dialogs.ts +++ b/web/src/dialogs.ts @@ -137,7 +137,7 @@ function drawDirectory(box=$('#directory'), focus?: string){
${artHTML(show.image,NAME(show))}

${esc(NAME(show))}

${ // Its category's page is one click away from here. show.category?``:''} -

${n?`${n} ${n===1?'person here subscribes':'people here subscribe'}`:'Nobody here subscribes yet'}

+ ${n?`

${n} ${n===1?'person here subscribes':'people here subscribe'}

`:''}
`