From 54d827f6558b6fe5e1319852d4cebf31a39b502e Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 2 Oct 2026 13:31:50 +0000 Subject: [PATCH] Time out a hung feed, serve the precomposed touch icon, and store http artwork on https (#108, #109, #110) #108: the HTTP client had no timeout, and scans handle feeds in order, so a hung server held every scan. Dreamwidth answered 504 after 60-67 s for a day and each scan took 70-80 s instead of 15. A feed fetch, and a Patreon creator's show list, now gets 30 s from connecting to the last byte (feed::FEED_TIMEOUT); the client gets a 10 s connect timeout, which bounds a download's start but not a long download. #109: iOS asks for /apple-touch-icon-precomposed.png first when the site is added to a home screen; it was a 404 and the only non-feed warning in the log. It serves the same icon. #110: the page is https and loads no http. Artwork on http came through /api/art (#90) even when its host serves https too. A scan now tries each http artwork host on https once per feed (feed::prefer_https) and stores the https address where the host answers with an image, rewriting that feed's stored items from the same host (Db::secure_images). 4 of the 5 hosts in production do; cdn.thesecretcabal.com presents another name's certificate and stays on /api/art. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ src/db.rs | 32 ++++++++++++++++++++++++++++++++ src/feed.rs | 34 ++++++++++++++++++++++++++++++++-- src/main.rs | 23 +++++++++++++++++++++++ src/web.rs | 2 ++ tests/ui/app.spec.js | 2 +- 6 files changed, 96 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b60dcd..c99747c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 listed feed downloads nothing until someone subscribes. - The Directory loads faster: it asked the database three questions per feed. +### Fixed + +- Artwork published on http is stored on https when its host serves it there, so the page loads it directly; the rest still comes through iPX. +- An iPhone adding the site to its home screen finds the icon at the first address it tries. +- A feed whose server hangs no longer holds up every scan: a feed gets 30 seconds, and connecting anywhere 10. + ## [0.9.1] - 2026-09-29 ### Added diff --git a/src/db.rs b/src/db.rs index b01629a..884ce81 100644 --- a/src/db.rs +++ b/src/db.rs @@ -791,6 +791,21 @@ impl Db { Ok(()) } + /// Rewrites a feed's stored artwork on `host` from http to https, once the host is known to + /// serve it there (`feed::prefer_https`), for the items a scan does not write again. + pub async fn secure_images(&self, feed_id: &str, host: &str) -> Result<()> { + let like = format!("http://{host}/%"); + for table in ["entries", "feeds"] { + let col = if table == "feeds" { "id" } else { "feed_id" }; + self.exec( + &format!("UPDATE {table} SET image = 'https://' || substr(image, 8) WHERE {col} = $1 AND image LIKE $2"), + vec![feed_id.into(), like.clone().into()], + ) + .await?; + } + Ok(()) + } + /// Forgets a feed and everything stored about it. pub async fn forget_feed(&self, id: &str) -> Result<()> { self.forget_rows(id).await?; @@ -2032,6 +2047,23 @@ mod tests { assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's } + #[tokio::test] + async fn a_hosts_artwork_moves_to_https_for_one_feed() { + let db = Db::memory().await.unwrap(); + db.exec_for_test( + "INSERT INTO feeds (id, url, image) VALUES ('f','u','http://a.example/logo.png'),('g','v','http://a.example/g.png'); + INSERT INTO entries (feed_id, guid, first_seen, image) VALUES + ('f','1',0,'http://a.example/1.jpg'),('f','2',0,'http://b.example/2.jpg'),('g','3',0,'http://a.example/3.jpg');", + ).await + .unwrap(); + db.secure_images("f", "a.example").await.unwrap(); + assert_eq!(db.feed_summary("f").await.unwrap().image.as_deref(), Some("https://a.example/logo.png")); + assert!(db.names_image("https://a.example/1.jpg").await.unwrap()); + assert!(db.names_image("http://b.example/2.jpg").await.unwrap()); // another host + assert!(db.names_image("http://a.example/3.jpg").await.unwrap()); // another feed + assert_eq!(db.feed_summary("g").await.unwrap().image.as_deref(), Some("http://a.example/g.png")); + } + #[tokio::test] async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() { let db = Db::memory().await.unwrap(); diff --git a/src/feed.rs b/src/feed.rs index 84e4257..9771d6e 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -53,6 +53,12 @@ pub enum Fetched { }, } +/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and +/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504 +/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small; +/// downloads, which are not, have no such limit. +pub const FEED_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + /// Conditional GET. reqwest handles gzip and redirects; the original's hand-rolled /// CONNECT/socket.ssl proxy path is gone -- `system-proxy` reads http_proxy/https_proxy. #[tracing::instrument(skip_all, fields(url = %cfg.url))] @@ -62,7 +68,7 @@ pub async fn fetch( etag: Option<&str>, last_modified: Option<&str>, ) -> Result { - let mut req = client.get(&cfg.url); + let mut req = client.get(&cfg.url).timeout(FEED_TIMEOUT); if let Some(tag) = etag { req = req.header(IF_NONE_MATCH, tag); } @@ -253,7 +259,7 @@ pub async fn patreon_shows( ) -> Result<(Option, Vec<(String, String)>)> { // The creator feed names its campaign by number in its self link, a few hundred bytes in. // The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up. - let mut resp = client.get(url).send().await.context("connecting")?; + let mut resp = client.get(url).timeout(FEED_TIMEOUT).send().await.context("connecting")?; if !resp.status().is_success() { return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status())); } @@ -451,6 +457,30 @@ pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option { is_image(client, ico.as_str()).await.then(|| ico.into()) } +/// Artwork's address on https when its host serves it there, else as it was. The page is https +/// and must not load http; the host is asked once per `known` (one feed's read), and an http +/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five +/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN +/// presents another name's certificate (#110). +pub async fn prefer_https( + client: &reqwest::Client, + url: &str, + known: &mut std::collections::HashMap, +) -> String { + let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() }; + let host = rest.split('/').next().unwrap_or("").to_owned(); + let secure = format!("https://{rest}"); + let ok = match known.get(&host) { + Some(ok) => *ok, + None => { + let ok = is_image(client, &secure).await; + known.insert(host, ok); + ok + } + }; + if ok { secure } else { url.to_owned() } +} + /// Whether `url` answers with an image. A site with no favicon often answers 200 with its home /// page, which is not an icon. pub async fn is_image(client: &reqwest::Client, url: &str) -> bool { diff --git a/src/main.rs b/src/main.rs index 4844ea7..0e5422f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -254,6 +254,8 @@ async fn main() -> Result<()> { db, client: reqwest::Client::builder() .user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION"))) + // Connecting only, so it bounds a download's start, not a long download (#108). + .connect_timeout(std::time::Duration::from_secs(10)) .build()?, out: if is_daemon { Emitter::socket(events.clone(), false) } else { Emitter::terminal() }, torrents: tokio::sync::OnceCell::new(), @@ -1389,6 +1391,17 @@ async fn scan_one( } let mut parsed = feed::parse(&bytes)?; + // Artwork on http moves to https where its host serves it (#110), before it is compared + // with what is stored, which is then the https address too. + let mut secure = std::collections::HashMap::new(); + if let Some(art) = &parsed.image { + parsed.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await); + } + for entry in parsed.entries.iter_mut() { + if let Some(art) = &entry.image { + entry.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await); + } + } // Artwork is looked at when it may have changed: the feed names different artwork from what // is stored, or someone asked for a refresh, so an icon the site changes or fixes still // follows it (#80). Looked at on every full read, a feed without validators asked its site @@ -1413,6 +1426,16 @@ async fn scan_one( }); } }, art_span).await; + // A site's icon comes back on whatever the site is on. + if let Some(art) = &parsed.image { + parsed.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await); + } + // Items stored before, which a scan does not write again, move with their host. + for (host, ok) in &secure { + if *ok { + ctx.db.secure_images(id, host).await?; + } + } ctx.db.record_feed( id, &feed_cfg.url, diff --git a/src/web.rs b/src/web.rs index 3ad895a..df1d817 100644 --- a/src/web.rs +++ b/src/web.rs @@ -76,6 +76,8 @@ pub fn router(state: WebState) -> Router { .route("/favicon.png", get(favicon)) .route("/favicon-dark.png", get(favicon_dark)) .route("/apple-touch-icon.png", get(touch_icon)) + // iOS asks for this one first when the site is added to a home screen (#109). + .route("/apple-touch-icon-precomposed.png", get(touch_icon)) .route("/app.js", get(app_js)) .route("/app.css", get(app_css)) .route("/login.js", get(login_js)) diff --git a/tests/ui/app.spec.js b/tests/ui/app.spec.js index 8f178e1..ad3adda 100644 --- a/tests/ui/app.spec.js +++ b/tests/ui/app.spec.js @@ -1233,7 +1233,7 @@ test('the favicon is the logo, square, from both pages', async ({ page }) => { await page.evaluate(() => setTheme('modern', 'dark')); await expect(page.locator('#favicon')).toHaveAttribute('href', /^\/favicon-dark\.png\?v=[0-9a-f]{12}$/); // A browser asks for /favicon.ico on its own, signed in or not. - for (const path of ['/favicon.ico', '/favicon.png', '/favicon-dark.png', '/apple-touch-icon.png']) { + for (const path of ['/favicon.ico', '/favicon.png', '/favicon-dark.png', '/apple-touch-icon.png', '/apple-touch-icon-precomposed.png']) { const r = await page.request.get(path, { headers: { cookie: '' } }); expect(r.status(), path).toBe(200); expect(r.headers()['content-type'], path).toBe('image/png');