Drop the start-up pass that folds files WordPress listed twice (#83)
Before 0.6.0 the parser took WordPress's numbered player URLs (?_=2) for separate files and downloaded some episodes twice. Since then it drops the repeats while reading (same_file_key), and merge_repeated_enclosures cleaned up what was already stored. Production has run it; on every start since it has only cost a query that finds nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
100
src/db.rs
100
src/db.rs
@@ -1696,76 +1696,6 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Folds enclosures of one item that `key` says are the same file into the first of them, for
|
||||
/// WordPress's numbered player URLs (`feed::same_file_key`). The first is the one the parser
|
||||
/// keeps, so it keeps its row, taking a repeat's file if it has none of its own; the repeats'
|
||||
/// rows go. Returns how many went and the copies left spare, for the caller to delete.
|
||||
pub async fn merge_repeated_enclosures(&self, key: impl Fn(&str) -> String) -> Result<(usize, Vec<String>)> {
|
||||
use sea_orm::TransactionTrait;
|
||||
use std::collections::hash_map::Entry;
|
||||
let backend = self.orm.get_database_backend();
|
||||
let tx = self.orm.begin().await?;
|
||||
// Items with a URL carrying WordPress's `_=` parameter. A LIKE, with the underscore
|
||||
// escaped, where SQLite had GLOB '*[?&]_=[0-9]*', which Postgres lacks. It lets through
|
||||
// `_=` without a number too, which is harmless: `key` only folds `_=` and digits.
|
||||
let rows = tx
|
||||
.query_all_raw(Statement::from_string(
|
||||
backend,
|
||||
r"SELECT id, feed_id, guid, url, path FROM enclosures
|
||||
WHERE (feed_id, guid) IN
|
||||
(SELECT feed_id, guid FROM enclosures
|
||||
WHERE url LIKE '%?\_=%' ESCAPE '\' OR url LIKE '%&\_=%' ESCAPE '\')
|
||||
ORDER BY id",
|
||||
))
|
||||
.await?;
|
||||
// The first row of each file, and whether it has the file on disk yet.
|
||||
let mut first: std::collections::HashMap<(String, String, String), (i64, bool)> = Default::default();
|
||||
let (mut gone, mut spare) = (0, vec![]);
|
||||
for r in rows {
|
||||
let (id, feed, guid, url, path): (i64, String, String, String, Option<String>) = (
|
||||
r.try_get("", "id")?,
|
||||
r.try_get("", "feed_id")?,
|
||||
r.try_get("", "guid")?,
|
||||
r.try_get("", "url")?,
|
||||
r.try_get("", "path")?,
|
||||
);
|
||||
match first.entry((feed, guid, key(&url))) {
|
||||
Entry::Vacant(v) => {
|
||||
v.insert((id, path.is_some()));
|
||||
}
|
||||
Entry::Occupied(mut o) => {
|
||||
let (keep, has) = o.get_mut();
|
||||
if let Some(p) = path {
|
||||
if *has {
|
||||
spare.push(p);
|
||||
} else {
|
||||
// The only copy is the repeat's: Rands' episode 97 was downloaded
|
||||
// under its ?_=2 URL alone.
|
||||
tx.execute_raw(Statement::from_sql_and_values(
|
||||
backend,
|
||||
"UPDATE enclosures SET (path, state, bytes_done, downloaded_at) =
|
||||
(SELECT path, state, bytes_done, downloaded_at FROM enclosures WHERE id = $2)
|
||||
WHERE id = $1",
|
||||
vec![(*keep).into(), id.into()],
|
||||
))
|
||||
.await?;
|
||||
*has = true;
|
||||
}
|
||||
}
|
||||
tx.execute_raw(Statement::from_sql_and_values(
|
||||
backend,
|
||||
"DELETE FROM enclosures WHERE id = $1",
|
||||
vec![id.into()],
|
||||
))
|
||||
.await?;
|
||||
gone += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok((gone, spare))
|
||||
}
|
||||
|
||||
/// Stops treating a feed as derived, because it now has its own config entry.
|
||||
pub async fn unmanage(&self, id: &str) -> Result<()> {
|
||||
self.exec("UPDATE feeds SET managed = false WHERE id = $1", vec![id.into()]).await?;
|
||||
@@ -1892,36 +1822,6 @@ pub fn now() -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_file_wordpress_listed_twice_is_folded_into_one() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
|
||||
(1,'f','a','https://x/a.mp3','/d/a-2.mp3','done'),
|
||||
(2,'f','a','https://x/a.mp3?_=2','/d/a.mp3','done'),
|
||||
(3,'f','b','https://x/b.mp3',NULL,'reaped'),
|
||||
(4,'f','b','https://x/b.mp3?_=2','/d/b.mp3','done'),
|
||||
(5,'f','c','https://x/c.mp3?_=1','/d/c.mp3','done'),
|
||||
(6,'f','d','https://x/d1.mp3?_=1','/d/d1.mp3','done'),
|
||||
(7,'f','d','https://x/d2.mp3?_=2','/d/d2.mp3','done');",
|
||||
).await
|
||||
.unwrap();
|
||||
let key = crate::feed::same_file_key;
|
||||
assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (2, vec!["/d/a.mp3".to_string()]));
|
||||
assert_eq!(
|
||||
db.i64s_for_test("SELECT id FROM enclosures ORDER BY id").await,
|
||||
[1, 3, 5, 6, 7],
|
||||
"a lone ?_=1 and two different files stay"
|
||||
);
|
||||
assert_eq!(
|
||||
db.strings_for_test("SELECT path FROM enclosures WHERE id = 3 UNION ALL SELECT state FROM enclosures WHERE id = 3")
|
||||
.await,
|
||||
["/d/b.mp3", "done"],
|
||||
"the only copy moves, not deleted"
|
||||
);
|
||||
assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (0, vec![]), "and only once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_sort_column_runs_and_orders_both_ways() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user