rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trim the state database; Popular lists feeds the way Directory does

Browse Source 
Drops the created columns on users, subscriptions and sessions, which were
written by every insert and read by nothing, and migrate()'s add list, whose
columns all predate 0.3.0. Removes Db::subscribed_feed_ids (no callers),
Db::subscriber_count (one caller wanting > 0) and Managed.orphaned (never
read). The old-database test now builds the tables with foreign keys on.

Popular now lists the feeds inside an OPML or a Patreon creator, never the
collection, as Directory does.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TAC7sLVqfKmY6rsTLXzNgk
This commit is contained in:
Ray Slakinski
2026-09-12 13:15:55 +00:00
parent 457a58dcc5
commit a958f7cb37
10 changed files with 159 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
src/db.rs
View File

@@ -72,8 +72,7 @@ CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE COLLATE NOCASE,
pass_hash TEXT,
is_admin INTEGER NOT NULL DEFAULT 0,
created INTEGER NOT NULL
is_admin INTEGER NOT NULL DEFAULT 0
);
-- What one person wants from a feed. The feed, its items and its files are shared; this
@@ -85,7 +84,6 @@ CREATE TABLE IF NOT EXISTS subscriptions (
auto_download INTEGER,
allow_explicit INTEGER,
max_new_per_check INTEGER,
created INTEGER NOT NULL,
PRIMARY KEY (user_id, feed_id)
);
@@ -104,7 +102,6 @@ CREATE TABLE IF NOT EXISTS entry_state (
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created INTEGER NOT NULL,
seen INTEGER NOT NULL
);
";
@@ -136,40 +133,30 @@ pub struct Managed {
pub url: String,
pub title: Option<String>,
pub group_id: String,
pub orphaned: bool,
}
/// Adds the columns later versions introduced and drops the ones they retired. CREATE TABLE IF
/// NOT EXISTS does nothing to a table that already exists, so an installed database needs both.
/// Drops the columns later versions retired. CREATE TABLE IF NOT EXISTS leaves a table that
/// already exists alone, so an installed database needs this done explicitly. A new column on an
/// existing table would need an ALTER TABLE ADD COLUMN here too; none does yet, since every one
/// added so far predates 0.3.0, the oldest version an upgrade may start from.
fn migrate(conn: &Connection) -> Result<()> {
let wanted: &[(&str, &str, &str)] = &[
("feeds", "image", "TEXT"),
("feeds", "orphaned", "INTEGER NOT NULL DEFAULT 0"),
("feeds", "group_id", "TEXT"),
("feeds", "managed", "INTEGER NOT NULL DEFAULT 0"),
("entries", "image", "TEXT"),
("entries", "duration", "INTEGER"),
("entries", "episode", "INTEGER"),
("entries", "season", "INTEGER"),
let retired: &[(&str, &str)] = &[
// Read state from before accounts, long since moved to entry_state. Two bugs came from
// queries still reading these after they stopped meaning anything.
("entries", "read"),
("entries", "flagged"),
("entries", "position"),
// Written by every insert and read by nothing.
("users", "created"),
("subscriptions", "created"),
("sessions", "created"),
];
// Read state from before accounts, long since moved to entry_state. Two bugs came from
// queries still reading these after they stopped meaning anything, so they go.
let retired: &[(&str, &str)] = &[("entries", "read"), ("entries", "flagged"), ("entries", "position")];
let has = |table: &str, column: &str| -> Result<bool> {
let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?;
let names = stmt
.query_map([], |r| r.get::<_, String>(1))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(names.iter().any(|c| c == column))
};
for (table, column, ty) in wanted {
if !has(table, column)? {
tracing::info!(table, column, "adding column");
conn.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {column} {ty}"))?;
}
}
for (table, column) in retired {
if has(table, column)? {
let names: Vec<String> = conn
.prepare(&format!("PRAGMA table_info({table})"))?
.query_map([], |r| r.get(1))?
.collect::<rusqlite::Result<_>>()?;
if names.iter().any(|c| c == column) {
tracing::info!(table, column, "dropping column");
conn.execute_batch(&format!("ALTER TABLE {table} DROP COLUMN {column}"))?;
}
@@ -829,15 +816,14 @@ impl Db {
}
for id in catalogue {
conn.execute(
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id, created) VALUES (?1, ?2, ?3)",
params![user_id, id, now()],
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id) VALUES (?1, ?2)",
params![user_id, id],
)?;
}
// Feeds that exist only in the database (OPML children) count too.
conn.execute(
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id, created)
SELECT ?1, id, ?2 FROM feeds",
params![user_id, now()],
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id) SELECT ?1, id FROM feeds",
params![user_id],
)?;
Ok(catalogue.len())
}
@@ -954,8 +940,8 @@ impl Db {
pub fn subscribe(&self, user_id: i64, feed_id: &str) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id, created) VALUES (?1, ?2, ?3)",
params![user_id, feed_id, now()],
"INSERT OR IGNORE INTO subscriptions (user_id, feed_id) VALUES (?1, ?2)",
params![user_id, feed_id],
)?;
Ok(())
}
@@ -969,16 +955,6 @@ impl Db {
Ok(())
}
/// How many people want this feed. Nobody means it stops being scanned.
pub fn subscriber_count(&self, feed_id: &str) -> Result<i64> {
let conn = self.conn.lock().unwrap();
Ok(conn.query_row(
"SELECT count(*) FROM subscriptions WHERE feed_id = ?1",
[feed_id],
|r| r.get(0),
)?)
}
/// Overwrites one person's settings for a feed. A None field means: follow the feed.
pub fn set_subscription(&self, user_id: i64, sub: &Sub) -> Result<()> {
let conn = self.conn.lock().unwrap();
@@ -989,8 +965,8 @@ impl Db {
.transpose()?;
conn.execute(
"INSERT INTO subscriptions
(user_id, feed_id, keywords, auto_download, allow_explicit, max_new_per_check, created)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
(user_id, feed_id, keywords, auto_download, allow_explicit, max_new_per_check)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
ON CONFLICT(user_id, feed_id) DO UPDATE SET
keywords = excluded.keywords,
auto_download = excluded.auto_download,
@@ -1003,29 +979,18 @@ impl Db {
sub.auto_download.map(|v| v as i64),
sub.allow_explicit.map(|v| v as i64),
sub.max_new_per_check,
now()
],
)?;
Ok(())
}
/// Feeds with at least one subscriber. What the scanner walks.
pub fn subscribed_feed_ids(&self) -> Result<Vec<String>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare("SELECT DISTINCT feed_id FROM subscriptions")?;
let out = stmt
.query_map([], |r| r.get::<_, String>(0))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(out)
}
// ---- users and sessions ----
pub fn create_user(&self, name: &str, pass_hash: Option<&str>, admin: bool) -> Result<i64> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO users (name, pass_hash, is_admin, created) VALUES (?1, ?2, ?3, ?4)",
params![name, pass_hash, admin as i64, now()],
"INSERT INTO users (name, pass_hash, is_admin) VALUES (?1, ?2, ?3)",
params![name, pass_hash, admin as i64],
)?;
Ok(conn.last_insert_rowid())
}
@@ -1093,7 +1058,7 @@ impl Db {
pub fn create_session(&self, user_id: i64, token: &str) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO sessions (token, user_id, created, seen) VALUES (?1, ?2, ?3, ?3)",
"INSERT INTO sessions (token, user_id, seen) VALUES (?1, ?2, ?3)",
params![token, user_id, now()],
)?;
Ok(())
@@ -1195,7 +1160,7 @@ impl Db {
.optional()?)
}
/// Read and starred, per person. The row is created on first touch.
/// Read and kept, per person. The row is created on first touch.
pub fn set_entry_flag(
&self,
user_id: i64,
@@ -1270,7 +1235,7 @@ impl Db {
pub fn managed_feeds(&self) -> Result<Vec<Managed>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, url, title, group_id, orphaned FROM feeds
"SELECT id, url, title, group_id FROM feeds
WHERE managed = 1 AND group_id IS NOT NULL ORDER BY coalesce(title, id)",
)?;
Ok(stmt
@@ -1280,7 +1245,6 @@ impl Db {
url: r.get(1)?,
title: r.get(2)?,
group_id: r.get(3)?,
orphaned: r.get::<_, i64>(4)? != 0,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?)
@@ -1423,8 +1387,8 @@ mod tests {
fn every_sort_column_runs_and_orders_both_ways() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0);
INSERT INTO subscriptions (user_id, feed_id, created) VALUES (1,'f',0),(1,'g',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g');
INSERT INTO feeds (id, url, title) VALUES ('f','u','Zebra'),('g','v','Aardvark');
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','banana',100),('g','b','Apple',200),('f','c','cherry',300);
@@ -1457,8 +1421,8 @@ mod tests {
fn deleting_a_shared_file_asks_about_everyone_else() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0),(2,'sam',0,0),(3,'kit',0,0);
INSERT INTO subscriptions (user_id, feed_id, created) VALUES (1,'f',0),(2,'f',0),(3,'f',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0),(3,'kit',0);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'),(3,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','a','u1','/tmp/a','done');",
@@ -1483,7 +1447,7 @@ mod tests {
fn read_state_belongs_to_one_person() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0),(2,'sam',0,0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0);
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','One',100),('f','b','Two',200);",
)
@@ -1526,27 +1490,52 @@ mod tests {
}
#[test]
fn an_old_database_loses_the_retired_read_columns() {
fn an_old_database_loses_its_retired_columns() {
let conn = Connection::open_in_memory().unwrap();
// As open() has it: a DROP COLUMN on a table that references another is the part worth
// proving, and it has to work with the foreign keys switched on.
conn.pragma_update(None, "foreign_keys", "ON").unwrap();
conn.execute_batch(
"CREATE TABLE entries (feed_id TEXT NOT NULL, guid TEXT NOT NULL,
first_seen INTEGER NOT NULL, read INTEGER NOT NULL DEFAULT 0,
flagged INTEGER NOT NULL DEFAULT 0, position INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (feed_id, guid));",
PRIMARY KEY (feed_id, guid));
CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE,
pass_hash TEXT, is_admin INTEGER NOT NULL DEFAULT 0, created INTEGER NOT NULL);
CREATE TABLE subscriptions (
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
feed_id TEXT NOT NULL, created INTEGER NOT NULL, PRIMARY KEY (user_id, feed_id));
CREATE TABLE sessions (token TEXT PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created INTEGER NOT NULL, seen INTEGER NOT NULL);
INSERT INTO users VALUES (1, 'ray', NULL, 1, 0);
INSERT INTO subscriptions VALUES (1, 'f', 0);
INSERT INTO sessions VALUES ('t', 1, 0, 0);",
)
.unwrap();
// The same order as open(): the schema leaves the old table alone, migrate() fixes it.
// The same order as open(): the schema leaves the old tables alone, migrate() fixes them.
conn.execute_batch(SCHEMA).unwrap();
migrate(&conn).unwrap();
let cols: Vec<String> = conn
.prepare("PRAGMA table_info(entries)")
.unwrap()
.query_map([], |r| r.get(1))
.unwrap()
.collect::<rusqlite::Result<_>>()
for (table, gone) in [
("entries", &["read", "flagged", "position"][..]),
("users", &["created"][..]),
("subscriptions", &["created"][..]),
("sessions", &["created"][..]),
] {
let cols: Vec<String> = conn
.prepare(&format!("PRAGMA table_info({table})"))
.unwrap()
.query_map([], |r| r.get(1))
.unwrap()
.collect::<rusqlite::Result<_>>()
.unwrap();
assert!(!cols.iter().any(|c| gone.contains(&c.as_str())), "{table}: {cols:?}");
}
// And the rows come through it.
let kept: i64 = conn
.query_row("SELECT count(*) FROM subscriptions JOIN sessions USING (user_id)", [], |r| r.get(0))
.unwrap();
assert!(!cols.iter().any(|c| ["read", "flagged", "position"].contains(&c.as_str())), "{cols:?}");
assert!(cols.iter().any(|c| c == "image"), "and it still gains the newer ones");
assert_eq!(kept, 1);
}
#[test]
@@ -1554,7 +1543,7 @@ mod tests {
// Cutting this along with the dead read columns left the browser suite's admin with an
// empty sidebar: it is how a fresh install's first account gets config.toml's feeds.
let db = Db::memory().unwrap();
db.exec_for_test("INSERT INTO users (id, name, is_admin, created) VALUES (1,'admin',1,0);")
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);")
.unwrap();
let subs = || -> i64 {
db.conn.lock().unwrap().query_row("SELECT count(*) FROM subscriptions", [], |r| r.get(0)).unwrap()
@@ -1582,7 +1571,7 @@ mod tests {
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','b','u1','/tmp/b','done');
-- Read and starred belong to a person now, so say which one.
INSERT INTO users (id, name, is_admin, created) VALUES (7,'reader',1,0);
INSERT INTO users (id, name, is_admin) VALUES (7,'reader',1);
INSERT INTO entry_state (user_id, feed_id, guid, read, flagged) VALUES
(7,'f','b',1,0),
(7,'f','c',1,1);",
@@ -1652,7 +1641,7 @@ mod tests {
fn a_show_takes_over_what_its_creator_held() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1);
INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES
(1,'creator','a','u1','done','/x/a.mp3',NULL),
(2,'creator','b','u2','skipped',NULL,'explicit'),
@@ -1688,9 +1677,9 @@ mod tests {
fn a_feed_in_a_group_follows_your_settings_on_the_group() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0),(2,'sam',0,0);
INSERT INTO subscriptions (user_id, feed_id, allow_explicit, created) VALUES
(1,'group',1,0),(1,'show',NULL,0),(2,'group',1,0),(2,'show',0,0);",
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0);
INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES
(1,'group',1),(1,'show',NULL),(2,'group',1),(2,'show',0);",
)
.unwrap();
let explicit = |group| -> Vec<Option<bool>> {

6
src/retention.rs
View File

@@ -154,8 +154,8 @@ mod tests {
// One file serves both subscribers, so it takes both of them to release it.
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0),(2,'sam',0,0);
INSERT INTO subscriptions (user_id, feed_id, created) VALUES (1,'f',0),(2,'f',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES
('f', 'keep', 0),
('f', 'half', 0),
@@ -189,7 +189,7 @@ mod tests {
fn prune_keeps_entries_that_still_have_a_file() {
let db = Db::memory().unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1);
INSERT INTO entry_state (user_id, feed_id, guid, flagged) VALUES (1,'f','flagged',1);
INSERT INTO entries (feed_id, guid, first_seen) VALUES
('f', 'has-file', 100),

31
src/web.rs
View File

@@ -579,14 +579,10 @@ struct PopularRow {
}
/// Every feed that may be listed, with everyone counted, you included, most subscribers
/// first. Popular is the top of one list and the directory is all of the other, and between
/// them they are all that `subscribe_popular` will subscribe you to.
///
/// `folders` says how an OPML appears. Popular lists the OPML once and not the feeds inside:
/// everyone subscribed to it counts for every one of them, and eighty of those would bury
/// everything anyone chose on purpose. The directory, which is for finding a show, lists the
/// feeds inside and never the OPML.
fn popular(state: &WebState, user_id: i64, folders: bool) -> Result<Vec<PopularRow>> {
/// first. Popular is the top of it, the directory is all of it, and it is all that
/// `subscribe_popular` will subscribe you to. An OPML or a Patreon creator is listed as the
/// feeds inside it and never itself: both lists are for finding a show.
fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> {
let db = &state.ctx.db;
let mine: std::collections::HashSet<String> =
db.subscriptions_for(user_id)?.into_iter().map(|s| s.feed_id).collect();
@@ -599,11 +595,13 @@ fn popular(state: &WebState, user_id: i64, folders: bool) -> Result<Vec<PopularR
let mut out = vec![];
for s in &catalogue {
let n = counts.get(&s.id).copied().unwrap_or(0);
let inside = s.managed || s.cfg.group.is_some();
let hidden = if folders { inside } else { is_folder.contains(s.id.as_str()) };
// A feed inside an OPML that looks private is as private as the OPML.
let folder = s.cfg.group.as_deref().and_then(|g| by_id.get(g));
if n == 0 || hidden || looks_private(&s.cfg) || folder.is_some_and(|f| looks_private(f)) {
if n == 0
|| is_folder.contains(s.id.as_str())
|| looks_private(&s.cfg)
|| folder.is_some_and(|f| looks_private(f))
{
continue;
}
let sum = db.feed_summary(&s.id)?;
@@ -618,7 +616,7 @@ async fn get_popular(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<PopularRow>>, ApiError> {
let mut rows = popular(&state, user.id, true)?;
let mut rows = popular(&state, user.id)?;
rows.truncate(10);
Ok(Json(rows))
}
@@ -628,7 +626,7 @@ async fn get_directory(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<PopularRow>>, ApiError> {
let mut rows = popular(&state, user.id, false)?;
let mut rows = popular(&state, user.id)?;
rows.sort_by_key(sort_name);
Ok(Json(rows))
}
@@ -637,15 +635,14 @@ fn sort_name(p: &PopularRow) -> String {
p.title.clone().unwrap_or_else(|| p.id.clone()).to_lowercase()
}
/// Subscribes by id, because the lists never show a URL. Checked against the same lists, so
/// Subscribes by id, because the list never shows a URL. Checked against the same list, so
/// a guessed id cannot reach a private feed.
async fn subscribe_popular(
State(state): State<WebState>,
user: crate::db::User,
Path(id): Path<String>,
) -> Result<Json<serde_json::Value>, ApiError> {
let listed = |folders| popular(&state, user.id, folders).map(|rows| rows.iter().any(|p| p.id == id));
if !(listed(true)? || listed(false)?) {
if !popular(&state, user.id)?.iter().any(|p| p.id == id) {
return Err(ApiError::bad_request(format!("{id:?} is not in the directory")));
}
state.ctx.db.subscribe(user.id, &id)?;
@@ -1095,7 +1092,7 @@ async fn remove_feed(
{
state.ctx.db.unsubscribe(user.id, &child.id)?;
}
if state.ctx.db.subscriber_count(&id)? > 0 {
if state.ctx.db.subscriber_counts()?.contains_key(&id) {
return Ok(StatusCode::NO_CONTENT);
}