A feed with no artwork takes its site's icon (#70)
When a feed names no image and none is stored, the scan fetches the channel's site link (RSS <link>, Atom rel=alternate) and uses the apple-touch-icon or icon it names, falling back to /favicon.ico when that answers with an image. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- A feed that has no artwork of its own shows its website's icon instead.
|
||||||
- `/api/status` gives the number of feeds, items waiting to download and files downloaded, and
|
- `/api/status` gives the number of feeds, items waiting to download and files downloaded, and
|
||||||
the version, for a dashboard such as Homepage.
|
the version, for a dashboard such as Homepage.
|
||||||
|
|
||||||
|
|||||||
69
src/feed.rs
69
src/feed.rs
@@ -11,6 +11,8 @@ pub struct ParsedFeed {
|
|||||||
pub title: Option<String>,
|
pub title: Option<String>,
|
||||||
pub ttl_mins: Option<u64>,
|
pub ttl_mins: Option<u64>,
|
||||||
pub image: Option<String>,
|
pub image: Option<String>,
|
||||||
|
/// The site the feed belongs to, where a favicon can stand in for missing artwork.
|
||||||
|
pub site: Option<String>,
|
||||||
/// The channel's first `<itunes:category>`, for the Directory's chips.
|
/// The channel's first `<itunes:category>`, for the Directory's chips.
|
||||||
pub category: Option<String>,
|
pub category: Option<String>,
|
||||||
pub entries: Vec<Entry>,
|
pub entries: Vec<Entry>,
|
||||||
@@ -349,6 +351,56 @@ fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Artwork for a feed that has none: the icon its site's page names, or else the site's
|
||||||
|
/// `/favicon.ico`. None if neither is there.
|
||||||
|
pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option<String> {
|
||||||
|
let timeout = std::time::Duration::from_secs(20);
|
||||||
|
let resp = client.get(site).timeout(timeout).send().await.ok()?;
|
||||||
|
// Relative to where the page ended up, not where it was asked for: a site that redirects
|
||||||
|
// to /en/ would otherwise have its icon looked for in the wrong place.
|
||||||
|
let base = resp.url().clone();
|
||||||
|
if resp.status().is_success()
|
||||||
|
&& let Ok(page) = resp.bytes().await
|
||||||
|
&& let Some(href) = page_icon(&page)
|
||||||
|
&& let Ok(url) = base.join(&href)
|
||||||
|
{
|
||||||
|
return Some(url.into());
|
||||||
|
}
|
||||||
|
let ico = base.join("/favicon.ico").ok()?;
|
||||||
|
let resp = client.get(ico.clone()).timeout(timeout).send().await.ok()?;
|
||||||
|
let is_image = resp
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::CONTENT_TYPE)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.is_some_and(|t| t.starts_with("image/"));
|
||||||
|
// A site with no favicon often answers 200 with its home page, which is not an icon.
|
||||||
|
(resp.status().is_success() && is_image).then(|| ico.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The icon a web page names in its `<link>` tags, the larger apple-touch-icon first: a plain
|
||||||
|
/// `icon` is often 16 pixels, which blurs at the size the list draws artwork.
|
||||||
|
fn page_icon(bytes: &[u8]) -> Option<String> {
|
||||||
|
let text = String::from_utf8_lossy(bytes);
|
||||||
|
// ASCII only, so byte offsets in the lowered copy stay valid in the original.
|
||||||
|
let lower = text.to_ascii_lowercase();
|
||||||
|
let (mut touch, mut icon) = (None, None);
|
||||||
|
let mut pos = 0;
|
||||||
|
while let Some(at) = lower[pos..].find("<link") {
|
||||||
|
let start = pos + at;
|
||||||
|
let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
|
||||||
|
pos = end + 1;
|
||||||
|
let tag = &text[start..end];
|
||||||
|
let Some(rel) = tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue };
|
||||||
|
let rels: Vec<&str> = rel.split_whitespace().collect();
|
||||||
|
if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) {
|
||||||
|
touch = tag_attr(tag, "href");
|
||||||
|
} else if icon.is_none() && rels.contains(&"icon") {
|
||||||
|
icon = tag_attr(tag, "href");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
touch.or(icon).filter(|h| !h.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
/// The value of one attribute in an HTML/XML start tag, however it is quoted.
|
/// The value of one attribute in an HTML/XML start tag, however it is quoted.
|
||||||
fn tag_attr(tag: &str, name: &str) -> Option<String> {
|
fn tag_attr(tag: &str, name: &str) -> Option<String> {
|
||||||
let key = format!("{name}=");
|
let key = format!("{name}=");
|
||||||
@@ -538,6 +590,7 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
|
|||||||
.and_then(|i| i.image())
|
.and_then(|i| i.image())
|
||||||
.map(str::to_owned)
|
.map(str::to_owned)
|
||||||
.or_else(|| ch.image().map(|i| i.url().to_owned())),
|
.or_else(|| ch.image().map(|i| i.url().to_owned())),
|
||||||
|
site: non_empty(Some(ch.link().trim())),
|
||||||
// Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
|
// Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
|
||||||
// would fill the Directory with one-off tags. The subcategory where there is one: Apple
|
// would fill the Directory with one-off tags. The subcategory where there is one: Apple
|
||||||
// files every tabletop and gaming show under Leisure, which says little; Games says it.
|
// files every tabletop and gaming show under Leisure, which says little; Games says it.
|
||||||
@@ -603,6 +656,12 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
|
|||||||
title: title_text(Some(feed.title().as_str())),
|
title: title_text(Some(feed.title().as_str())),
|
||||||
ttl_mins: None,
|
ttl_mins: None,
|
||||||
image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
|
image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
|
||||||
|
site: feed
|
||||||
|
.links()
|
||||||
|
.iter()
|
||||||
|
.find(|l| l.rel() == "alternate")
|
||||||
|
.map(|l| l.href().trim().to_owned())
|
||||||
|
.filter(|h| !h.is_empty()),
|
||||||
category: None,
|
category: None,
|
||||||
entries,
|
entries,
|
||||||
}
|
}
|
||||||
@@ -1157,6 +1216,16 @@ mod tests {
|
|||||||
assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
|
assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn page_icon_prefers_the_touch_icon() {
|
||||||
|
let page = br#"<head><link rel="stylesheet" href="/a.css">
|
||||||
|
<link rel="shortcut icon" href="/fav.ico">
|
||||||
|
<LINK REL="apple-touch-icon-precomposed" sizes="180x180" href='/touch.png'></head>"#;
|
||||||
|
assert_eq!(page_icon(page).as_deref(), Some("/touch.png"));
|
||||||
|
assert_eq!(page_icon(br#"<link rel="icon" href="i.svg">"#).as_deref(), Some("i.svg"));
|
||||||
|
assert_eq!(page_icon(br#"<link rel="stylesheet" href="/a.css">"#), None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_items_picture_comes_from_the_most_deliberate_source() {
|
fn an_items_picture_comes_from_the_most_deliberate_source() {
|
||||||
let xml = br#"<?xml version="1.0"?>
|
let xml = br#"<?xml version="1.0"?>
|
||||||
|
|||||||
10
src/main.rs
10
src/main.rs
@@ -1205,7 +1205,15 @@ async fn scan_one(
|
|||||||
return sync_opml(ctx, id, feed_cfg, &bytes).await;
|
return sync_opml(ctx, id, feed_cfg, &bytes).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let parsed = feed::parse(&bytes)?;
|
let mut parsed = feed::parse(&bytes)?;
|
||||||
|
// ponytail: a site with no icon at all is asked again on every scan that finds the feed
|
||||||
|
// changed; remember the miss in the feeds table if that turns out to cost.
|
||||||
|
if parsed.image.is_none()
|
||||||
|
&& let Some(site) = &parsed.site
|
||||||
|
&& ctx.db.feed_summary(id).await?.image.is_none()
|
||||||
|
{
|
||||||
|
parsed.image = feed::site_icon(&ctx.client, site).await;
|
||||||
|
}
|
||||||
ctx.db.record_feed(
|
ctx.db.record_feed(
|
||||||
id,
|
id,
|
||||||
&feed_cfg.url,
|
&feed_cfg.url,
|
||||||
|
|||||||
Reference in New Issue
Block a user