rays/ipx
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Look at a feed's artwork only when it may have changed, and trace a scan's database work (#95, #96)

Browse Source 
A feed read in full checked its own artwork and, without one, asked its website for an icon,
every time; a feed without validators is read in full every scan, so looking-for-group spent
2 s of every scan loading lfg.co's home page. Now the check runs when the feed names different
artwork from what is stored, or the scan was asked for, which keeps #80's point: a refresh
still picks up an icon the site changes or fixes.

Feed spans ran seconds past their fetch with nothing to say where (#96). The artwork lookup,
the loop that stores each item, and the per-feed database calls (feed_summary, record_feed,
subscribers, adopt, skipped_by_filter, rehide, pending) now have spans of their own.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-29 17:27:31 +00:00
parent e8fd3fe9ed
commit 2799704d30
3 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/main.rs
View File

@@ -993,7 +993,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
match scan_one(ctx, id, feed_cfg, &state).await {
match scan_one(ctx, id, feed_cfg, &state, force).await {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
@@ -1039,7 +1039,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
let state = ctx.db.http_state(id).await?;
match scan_one(ctx, id, feed_cfg, &state).await {
match scan_one(ctx, id, feed_cfg, &state, force).await {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
@@ -1199,6 +1199,7 @@ async fn scan_one(
id: &str,
feed_cfg: &config::Feed,
state: &db::HttpState,
force: bool,
) -> Result<Outcome> {
// A Patreon creator with more than one show is a list of feeds, like an OPML.
if feed::is_patreon_creator(&feed_cfg.url) {
@@ -1266,25 +1267,30 @@ async fn scan_one(
}
let mut parsed = feed::parse(&bytes)?;
// Looked for again whenever the feed is read in full, which is when it has changed or
// someone asked for a refresh, so an icon the site changes or fixes follows it (#80). A miss
// is stored as "", which the page draws as no art, and which stops the refetch above.
// ponytail: a feed without validators is read in full every scan and asks its site each
// time too; keep a checked-at time per feed if that shows up in anyone's logs.
// Artwork is looked at when it may have changed: the feed names different artwork from what
// is stored, or someone asked for a refresh, so an icon the site changes or fixes still
// follows it (#80). Looked at on every full read, a feed without validators asked its site
// on every scan: 2 s a scan for lfg.co (#95). A miss is stored as "", which the page draws
// as no art, and which stops the refetch above.
// A feed's own artwork has to be there too: Ken and Robin's names a 404, and stored unasked
// it stood in the way of the site's icon, which works (#89).
if let Some(art) = &parsed.image
&& !feed::is_image(&ctx.client, art).await
{
tracing::info!(feed = id, art = %art, "the feed's artwork is not an image; trying its site's icon");
parsed.image = None;
}
if parsed.image.is_none() {
parsed.image = Some(match &parsed.site {
Some(site) => feed::site_icon(&ctx.client, site).await.unwrap_or_default(),
None => String::new(),
});
}
let art_span = tracing::info_span!("artwork");
tracing::Instrument::instrument(async {
if let Some(art) = &parsed.image
&& stored.image.as_deref() != Some(art.as_str())
&& !feed::is_image(&ctx.client, art).await
{
tracing::info!(feed = id, art = %art, "the feed's artwork is not an image; trying its site's icon");
parsed.image = None;
}
if parsed.image.is_none() {
parsed.image = Some(match (&stored.image, &parsed.site) {
(Some(known), _) if !force => known.clone(),
(_, Some(site)) => feed::site_icon(&ctx.client, site).await.unwrap_or_default(),
(_, None) => String::new(),
});
}
}, art_span).await;
ctx.db.record_feed(
id,
&feed_cfg.url,
@@ -1311,28 +1317,32 @@ async fn scan_one(
// changed nothing however often the feed was scanned.
let skipped = ctx.db.skipped_by_filter(id).await?;
let mut scan = Scan::default();
for entry in &parsed.entries {
if ctx.db.record_entry(id, entry).await? {
scan.new_entries += 1;
}
for enc in &entry.enclosures {
let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())
} else {
continue; // Settled: queued, downloaded, reaped, or another feed's file.
};
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
if now != was {
match now {
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?,
// Its own span: the time a feed spends after its fetch was untraced (#96).
let store = tracing::info_span!("store", items = parsed.entries.len());
tracing::Instrument::instrument(async {
for entry in &parsed.entries {
if ctx.db.record_entry(id, entry).await? {
scan.new_entries += 1;
}
for enc in &entry.enclosures {
let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())
} else {
continue; // Settled: queued, downloaded, reaped, or another feed's file.
};
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
if now != was {
match now {
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?,
}
}
}
}
}
anyhow::Ok(())
}, store).await?;
if scan.new_entries > 0 {
ctx.db.rehide(id).await?; // what is new may hold someone's blocked words
}