diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b7098..77a9f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A feed whose address answers with nothing, as a lapsed domain does behind a DNS filter's block page, shows as failing, with what to do about it, instead of as a show that has not posted yet. - On Postgres, a write no longer waits for the database server's disk. A scan read about one feed a second and now reads 1,500 in 20 seconds. - Checking every feed no longer makes every icon in the feed list flash while it runs: the list keeps the icons it has already drawn. diff --git a/src/feed.rs b/src/feed.rs index a356288..6d4e285 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -190,6 +190,9 @@ pub fn explain_failure(msg: &str) -> Option { if low.contains("http 401") || low.contains("http 403") { return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None }); } + if low.contains("answered with nothing") { + return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None }); + } if low.contains("http 402") { return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None }); } @@ -1310,6 +1313,10 @@ mod tests { assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests."); assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests."); assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan."); + assert_eq!( + explain_failure("the feed's address answered with nothing at all").unwrap().reason, + "This address answers with nothing; the site may be gone." + ); let dns = explain_failure("connecting: dns error: failed to lookup address information").unwrap(); assert_eq!(dns.reason, "This address no longer resolves; the site is gone."); let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap(); diff --git a/src/main.rs b/src/main.rs index bdff3e9..3701f00 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1455,6 +1455,13 @@ async fn scan_one( }; if bytes.iter().all(u8::is_ascii_whitespace) { + // From a feed that has published, nothing new, said badly (see Outcome::Empty). From one + // that has never stored an item, a feed that is not there: a lapsed domain behind a DNS + // filter's block page answers 200 and nothing, and looked like a show that had not + // posted yet, with no error to act on (#119). The next real read clears it. + if stored.entries == 0 { + anyhow::bail!("the feed's address answered with nothing at all"); + } ctx.db.touch_feed(id, &feed_cfg.url).await?; return Ok(Outcome::Empty); } @@ -2248,6 +2255,43 @@ mod tests { } } + /// Answers /empty with a 200 and nothing, anything else with a feed of one item. + async fn empty_or_feed_server() -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { return }; + tokio::spawn(async move { + let mut buf = [0u8; 2048]; + let n = sock.read(&mut buf).await.unwrap_or(0); + let empty = String::from_utf8_lossy(&buf[..n]).starts_with("GET /empty"); + let body = if empty { "\n" } else { + "TOneg1" + }; + let resp = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()); + let _ = sock.write_all(resp.as_bytes()).await; + }); + } + }); + format!("http://{addr}") + } + + #[tokio::test] + async fn nothing_from_a_feed_that_never_posted_is_an_error_and_from_one_that_has_is_not() { + let base = empty_or_feed_server().await; + let ctx = Arc::new(test_ctx(config::Config::default()).await); + let at = |path: &str| config::Feed { url: format!("{base}{path}"), ..feed() }; + let none = db::HttpState::default(); + // A lapsed domain behind a block page: never an item, and nothing (#119). + let err = scan_one(&ctx, "gone", &at("/empty"), &none, false, None).await.err().expect("an error"); + assert!(format!("{err:#}").contains("answered with nothing"), "{err:#}"); + // A feed that has posted answering nothing has nothing new, as the Antarctic Survey does. + assert!(matches!(scan_one(&ctx, "quiet", &at("/feed"), &none, false, None).await, Ok(Outcome::Feed(_)))); + assert!(matches!(scan_one(&ctx, "quiet", &at("/empty"), &none, false, None).await, Ok(Outcome::Empty))); + } + #[tokio::test] async fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() { // davewiner: the OPML subscription left config.toml, but its 922 derived rows