rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write again a stored item the feed now tells differently (#141)

Browse Source 
"APPOFENIACS - A Technology Movie about Human Behavior" in Ain't It Cool News showed 66,170
characters: the feed's own HTTP headers and then the whole feed, tags stripped, and every other
item of the feed carried the text of the one before it. The publisher's feed had served it that
way, as only its XML could make a shift like that, and has since been fixed; ipx had read the
fixed feed and kept the broken text. Since 57ab419 (#96) a scan called record_entry only for a
guid it had not stored, so its UPDATE of title, description, image and duration was never
reached, and no correction, retitled episode or fixed length arrived.

Db::changed_items reads back what is stored for the items the feed lists, not the archive, and
names those it now gives a different title, text, artwork, length or number, a field it leaves
out counting as unchanged; the scan writes those again too. A corrected item keeps its read
state. A test serves a feed broken and then fixed from a socket of its own.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-05 23:50:15 +00:00
parent 50c3b9db64
commit 5684a02e79
3 changed files with 116 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/db.rs
View File

@@ -2059,6 +2059,64 @@ impl Db {
))
}
/// Of these items, the stored ones the feed now tells differently: a title, text, artwork,
/// length or number that is not what is stored, a field the feed leaves out counting as
/// unchanged, as record_entry writes them. A scan inserted only items it had not stored
/// (#96), and with that alone a publisher's correction never arrived: Ain't It Cool News's
/// items kept a broken copy, each with the text of the one before it, after the feed was
/// fixed (#141). Only what the feed lists is read back, so a scan reads as much as it was
/// sent, not the archive.
#[tracing::instrument(skip_all)]
pub async fn changed_items(
&self,
feed_id: &str,
entries: &[crate::feed::Entry],
) -> Result<std::collections::HashSet<String>> {
if entries.is_empty() {
return Ok(Default::default());
}
let mut a = Args::default();
let feed = a.p(feed_id);
let guids: Vec<String> = entries.iter().map(|e| a.p(e.guid.clone())).collect();
let sql = format!(
"SELECT guid, title, description, image, duration, episode, season FROM entries
WHERE feed_id = {feed} AND guid IN ({})",
guids.join(", ")
);
type Stored = (Option<String>, Option<String>, Option<String>, Option<i64>, Option<i64>, Option<i64>);
let mut stored = std::collections::HashMap::<String, Stored>::new();
for r in self.rows(&sql, a.0).await? {
stored.insert(
r.try_get("", "guid")?,
(
r.try_get("", "title")?,
r.try_get("", "description")?,
r.try_get("", "image")?,
r.try_get("", "duration")?,
r.try_get("", "episode")?,
r.try_get("", "season")?,
),
);
}
fn differs<T: PartialEq>(new: &Option<T>, old: &Option<T>) -> bool {
new.is_some() && new != old
}
Ok(entries
.iter()
.filter(|e| {
stored.get(&e.guid).is_some_and(|s| {
differs(&e.title, &s.0)
|| differs(&e.description, &s.1)
|| differs(&e.image, &s.2)
|| differs(&e.duration, &s.3)
|| differs(&e.episode, &s.4)
|| differs(&e.season, &s.5)
})
})
.map(|e| e.guid.clone())
.collect())
}
#[tracing::instrument(skip_all)]
pub async fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> {
self.rows(
@@ -2396,6 +2454,31 @@ mod tests {
assert_eq!(files, ["u1"].map(String::from).into()); // u2 is g's: left to the insert to find
}
#[tokio::test]
async fn a_scan_finds_the_stored_items_the_feed_now_tells_differently() {
let db = Db::memory().await.unwrap();
let item = |guid: &str, title: &str, text: Option<&str>| crate::feed::Entry {
guid: guid.into(),
title: Some(title.into()),
description: text.map(str::to_owned),
..Default::default()
};
db.record_entry("f", &item("a", "A", Some("the review"))).await.unwrap();
db.record_entry("f", &item("b", "B", Some("the next review"))).await.unwrap();
db.record_entry("f", &item("c", "C", Some("as it was"))).await.unwrap();
let changed = db
.changed_items("f", &[
item("a", "A", Some("the review, fixed")), // the text corrected
item("b", "B renamed", Some("the next review")), // retitled
item("c", "C", None), // says nothing of its text: what is stored stands
item("d", "D", Some("new")), // not stored: the insert's, not this
])
.await
.unwrap();
assert_eq!(changed, ["a", "b"].map(String::from).into());
assert!(db.changed_items("f", &[]).await.unwrap().is_empty());
}
#[tokio::test]
async fn only_artwork_a_feed_names_is_fetched_for_the_page() {
let db = Db::memory().await.unwrap();

35
src/main.rs
View File

@@ -1598,13 +1598,17 @@ async fn scan_one(
// changed nothing however often the feed was scanned.
let skipped = ctx.db.skipped_by_filter(id).await?;
let (known_items, known_files) = ctx.db.stored_items(id).await?;
let changed = ctx.db.changed_items(id, &parsed.entries).await?;
let mut scan = Scan::default();
// Its own span: the time a feed spends after its fetch was untraced (#96).
let store = tracing::info_span!("store", items = parsed.entries.len());
tracing::Instrument::instrument(async {
for entry in &parsed.entries {
// Only what is not stored yet is inserted; the insert would find the rest and do nothing.
if !known_items.contains(&entry.guid) && ctx.db.record_entry(id, entry).await? {
// What is not stored yet is inserted, and what the feed has changed since is written
// again (#141); the rest is left alone. A correction keeps the item's read state.
if (!known_items.contains(&entry.guid) || changed.contains(&entry.guid))
&& ctx.db.record_entry(id, entry).await?
{
scan.new_entries += 1;
}
for enc in &entry.enclosures {
@@ -2299,7 +2303,8 @@ mod tests {
assert_eq!(urls, vec!["http://x/added-by-the-cli.xml".to_string()]);
}
/// Answers /empty with a 200 and nothing, anything else with a feed of one item.
/// Answers /empty with a 200 and nothing, /fixed with a feed of one item as it should read,
/// anything else with the same item's text broken.
async fn empty_or_feed_server() -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2310,9 +2315,13 @@ mod tests {
tokio::spawn(async move {
let mut buf = [0u8; 2048];
let n = sock.read(&mut buf).await.unwrap_or(0);
let empty = String::from_utf8_lossy(&buf[..n]).starts_with("GET /empty");
let body = if empty { "\n" } else {
"<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title><item><title>One</title><guid>g1</guid></item></channel></rss>"
let req = String::from_utf8_lossy(&buf[..n]).into_owned();
let body = if req.starts_with("GET /empty") {
"\n"
} else if req.starts_with("GET /fixed") {
"<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title><item><title>One</title><guid>g1</guid><description>As it should read.</description></item></channel></rss>"
} else {
"<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title><item><title>One</title><guid>g1</guid><description>Somebody else's text.</description></item></channel></rss>"
};
let resp = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len());
let _ = sock.write_all(resp.as_bytes()).await;
@@ -2322,6 +2331,20 @@ mod tests {
format!("http://{addr}")
}
#[tokio::test]
async fn a_publishers_correction_reaches_an_item_already_stored() {
let base = empty_or_feed_server().await;
let ctx = Arc::new(test_ctx(config::Config::default()).await);
let at = |path: &str| config::Feed { url: format!("{base}{path}"), ..feed() };
let none = db::HttpState::default();
let text = || ctx.db.strings_for_test("SELECT description FROM entries WHERE feed_id = 'aicn'");
scan_one(&ctx, "aicn", &at("/feed"), &none, false, None).await.unwrap();
assert_eq!(text().await, ["Somebody else's text."]);
// The feed fixed, the item already stored: it was left as it was (#141).
scan_one(&ctx, "aicn", &at("/fixed"), &none, false, None).await.unwrap();
assert_eq!(text().await, ["As it should read."]);
}
#[tokio::test]
async fn nothing_from_a_feed_that_never_posted_is_an_error_and_from_one_that_has_is_not() {
let base = empty_or_feed_server().await;