rays/ipx
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Drop the start-up pass that folds files WordPress listed twice (#83)

Browse Source 
Before 0.6.0 the parser took WordPress's numbered player URLs (?_=2) for
separate files and downloaded some episodes twice. Since then it drops
the repeats while reading (same_file_key), and merge_repeated_enclosures
cleaned up what was already stored. Production has run it; on every
start since it has only cost a query that finds nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-29 13:12:10 +00:00
parent 469467bf04
commit 8c5eddd783
3 changed files with 5 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -30,6 +30,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- The logo is in the top bar beside the add-feed button, in place of the name at the top of the
feed list. Hovering it shows iPX's version.
### Removed
- Starting up no longer looks for files downloaded twice before 0.6.0 read WordPress's player
links correctly; that clean-up has run.
### Fixed
- A feed whose website names an icon that is missing shows the site's `/favicon.ico` instead of

100
src/db.rs
View File

@@ -1696,76 +1696,6 @@ impl Db {
Ok(())
}
/// Folds enclosures of one item that `key` says are the same file into the first of them, for
/// WordPress's numbered player URLs (`feed::same_file_key`). The first is the one the parser
/// keeps, so it keeps its row, taking a repeat's file if it has none of its own; the repeats'
/// rows go. Returns how many went and the copies left spare, for the caller to delete.
pub async fn merge_repeated_enclosures(&self, key: impl Fn(&str) -> String) -> Result<(usize, Vec<String>)> {
use sea_orm::TransactionTrait;
use std::collections::hash_map::Entry;
let backend = self.orm.get_database_backend();
let tx = self.orm.begin().await?;
// Items with a URL carrying WordPress's `_=` parameter. A LIKE, with the underscore
// escaped, where SQLite had GLOB '*[?&]_=[0-9]*', which Postgres lacks. It lets through
// `_=` without a number too, which is harmless: `key` only folds `_=` and digits.
let rows = tx
.query_all_raw(Statement::from_string(
backend,
r"SELECT id, feed_id, guid, url, path FROM enclosures
WHERE (feed_id, guid) IN
(SELECT feed_id, guid FROM enclosures
WHERE url LIKE '%?\_=%' ESCAPE '\' OR url LIKE '%&\_=%' ESCAPE '\')
ORDER BY id",
))
.await?;
// The first row of each file, and whether it has the file on disk yet.
let mut first: std::collections::HashMap<(String, String, String), (i64, bool)> = Default::default();
let (mut gone, mut spare) = (0, vec![]);
for r in rows {
let (id, feed, guid, url, path): (i64, String, String, String, Option<String>) = (
r.try_get("", "id")?,
r.try_get("", "feed_id")?,
r.try_get("", "guid")?,
r.try_get("", "url")?,
r.try_get("", "path")?,
);
match first.entry((feed, guid, key(&url))) {
Entry::Vacant(v) => {
v.insert((id, path.is_some()));
}
Entry::Occupied(mut o) => {
let (keep, has) = o.get_mut();
if let Some(p) = path {
if *has {
spare.push(p);
} else {
// The only copy is the repeat's: Rands' episode 97 was downloaded
// under its ?_=2 URL alone.
tx.execute_raw(Statement::from_sql_and_values(
backend,
"UPDATE enclosures SET (path, state, bytes_done, downloaded_at) =
(SELECT path, state, bytes_done, downloaded_at FROM enclosures WHERE id = $2)
WHERE id = $1",
vec![(*keep).into(), id.into()],
))
.await?;
*has = true;
}
}
tx.execute_raw(Statement::from_sql_and_values(
backend,
"DELETE FROM enclosures WHERE id = $1",
vec![id.into()],
))
.await?;
gone += 1;
}
}
}
tx.commit().await?;
Ok((gone, spare))
}
/// Stops treating a feed as derived, because it now has its own config entry.
pub async fn unmanage(&self, id: &str) -> Result<()> {
self.exec("UPDATE feeds SET managed = false WHERE id = $1", vec![id.into()]).await?;
@@ -1892,36 +1822,6 @@ pub fn now() -> i64 {
mod tests {
use super::*;
#[tokio::test]
async fn a_file_wordpress_listed_twice_is_folded_into_one() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','a','https://x/a.mp3','/d/a-2.mp3','done'),
(2,'f','a','https://x/a.mp3?_=2','/d/a.mp3','done'),
(3,'f','b','https://x/b.mp3',NULL,'reaped'),
(4,'f','b','https://x/b.mp3?_=2','/d/b.mp3','done'),
(5,'f','c','https://x/c.mp3?_=1','/d/c.mp3','done'),
(6,'f','d','https://x/d1.mp3?_=1','/d/d1.mp3','done'),
(7,'f','d','https://x/d2.mp3?_=2','/d/d2.mp3','done');",
).await
.unwrap();
let key = crate::feed::same_file_key;
assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (2, vec!["/d/a.mp3".to_string()]));
assert_eq!(
db.i64s_for_test("SELECT id FROM enclosures ORDER BY id").await,
[1, 3, 5, 6, 7],
"a lone ?_=1 and two different files stay"
);
assert_eq!(
db.strings_for_test("SELECT path FROM enclosures WHERE id = 3 UNION ALL SELECT state FROM enclosures WHERE id = 3")
.await,
["/d/b.mp3", "done"],
"the only copy moves, not deleted"
);
assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (0, vec![]), "and only once");
}
#[tokio::test]
async fn every_sort_column_runs_and_orders_both_ways() {
let db = Db::memory().await.unwrap();

15
src/main.rs
View File

@@ -422,21 +422,6 @@ async fn daemon(
Err(e) => tracing::warn!(error = ?e, "could not retire feeds whose OPML is no longer in config"),
}
// Before the parser knew WordPress's numbered player URLs, a file it listed twice was
// downloaded twice. The repeats fold into the first, and their spare copies are deleted.
match ctx.db.merge_repeated_enclosures(feed::same_file_key).await {
Ok((0, _)) => {}
Ok((n, spare)) => {
for path in &spare {
if let Err(e) = std::fs::remove_file(path) {
tracing::warn!(path, error = %e, "could not delete a spare copy");
}
}
tracing::info!(enclosures = n, files = spare.len(), "folded files WordPress listed twice");
}
Err(e) => tracing::warn!(error = ?e, "could not fold files WordPress listed twice"),
}
let (tx_cmd, mut rx_cmd) = mpsc::channel::<Cmd>(64);
let web = start_web(&ctx, &config_path, web_addr, &tx_cmd, &events).await?;