Time out a hung feed, serve the precomposed touch icon, and store http artwork on https (#108, #109, #110)
#108: the HTTP client had no timeout, and scans handle feeds in order, so a hung server held every scan. Dreamwidth answered 504 after 60-67 s for a day and each scan took 70-80 s instead of 15. A feed fetch, and a Patreon creator's show list, now gets 30 s from connecting to the last byte (feed::FEED_TIMEOUT); the client gets a 10 s connect timeout, which bounds a download's start but not a long download. #109: iOS asks for /apple-touch-icon-precomposed.png first when the site is added to a home screen; it was a 404 and the only non-feed warning in the log. It serves the same icon. #110: the page is https and loads no http. Artwork on http came through /api/art (#90) even when its host serves https too. A scan now tries each http artwork host on https once per feed (feed::prefer_https) and stores the https address where the host answers with an image, rewriting that feed's stored items from the same host (Db::secure_images). 4 of the 5 hosts in production do; cdn.thesecretcabal.com presents another name's certificate and stays on /api/art. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
23
src/main.rs
23
src/main.rs
@@ -254,6 +254,8 @@ async fn main() -> Result<()> {
|
||||
db,
|
||||
client: reqwest::Client::builder()
|
||||
.user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
|
||||
// Connecting only, so it bounds a download's start, not a long download (#108).
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.build()?,
|
||||
out: if is_daemon { Emitter::socket(events.clone(), false) } else { Emitter::terminal() },
|
||||
torrents: tokio::sync::OnceCell::new(),
|
||||
@@ -1389,6 +1391,17 @@ async fn scan_one(
|
||||
}
|
||||
|
||||
let mut parsed = feed::parse(&bytes)?;
|
||||
// Artwork on http moves to https where its host serves it (#110), before it is compared
|
||||
// with what is stored, which is then the https address too.
|
||||
let mut secure = std::collections::HashMap::new();
|
||||
if let Some(art) = &parsed.image {
|
||||
parsed.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await);
|
||||
}
|
||||
for entry in parsed.entries.iter_mut() {
|
||||
if let Some(art) = &entry.image {
|
||||
entry.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await);
|
||||
}
|
||||
}
|
||||
// Artwork is looked at when it may have changed: the feed names different artwork from what
|
||||
// is stored, or someone asked for a refresh, so an icon the site changes or fixes still
|
||||
// follows it (#80). Looked at on every full read, a feed without validators asked its site
|
||||
@@ -1413,6 +1426,16 @@ async fn scan_one(
|
||||
});
|
||||
}
|
||||
}, art_span).await;
|
||||
// A site's icon comes back on whatever the site is on.
|
||||
if let Some(art) = &parsed.image {
|
||||
parsed.image = Some(feed::prefer_https(&ctx.client, art, &mut secure).await);
|
||||
}
|
||||
// Items stored before, which a scan does not write again, move with their host.
|
||||
for (host, ok) in &secure {
|
||||
if *ok {
|
||||
ctx.db.secure_images(id, host).await?;
|
||||
}
|
||||
}
|
||||
ctx.db.record_feed(
|
||||
id,
|
||||
&feed_cfg.url,
|
||||
|
||||
Reference in New Issue
Block a user