rays/ipx
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Block lists: words that hide items and keep them from downloading (#47)

Browse Source 
Each person has a list for every feed they read and one per feed. An item whose title or text
holds one of the words, matched as whole words so "ai" does not hide everything that "said"
anything, is hidden from them and, since the scanner now keeps each subscriber's filters
separate, is fetched only if someone else still wants it.

Whole-word matching is not something LIKE can do on both SQLite and Postgres, so the matches are
worked out in Rust into a `hidden` table whenever a list changes, someone subscribes, or a scan
brings in new items, and the queries only look that table up. Both new tables are tables rather
than columns because create_missing adds tables but never columns. Hidden counts as read for
the reaper and for "others still want this file", since whoever it is hidden from is as done
with it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-25 13:13:03 +00:00
parent 868dd673f3
commit bdda9b3d2e
8 changed files with 383 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/db.rs
View File

@@ -2,7 +2,7 @@
//! per-feed .ipxd plists, history.dat and qmcache.dat.
use anyhow::{Context, Result};
use crate::entity::{catalogue, enclosures, entries, feeds, sessions, settings, subscriptions, users};
use crate::entity::{blocklists, catalogue, enclosures, entries, feeds, hidden, sessions, settings, subscriptions, users};
use sea_orm::sea_query::{Expr, Func};
use sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set,
@@ -57,6 +57,8 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
schema.create_table_from_entity(sessions::Entity),
schema.create_table_from_entity(catalogue::Entity),
schema.create_table_from_entity(settings::Entity),
schema.create_table_from_entity(blocklists::Entity),
schema.create_table_from_entity(hidden::Entity),
] {
orm.execute(table.if_not_exists()).await.context("creating the schema")?;
}
@@ -156,6 +158,10 @@ pub struct Sub {
pub auto_download: Option<bool>,
pub allow_explicit: Option<bool>,
pub max_new_per_check: Option<i64>,
/// Words that keep an item from being downloaded for this person: their list for every feed
/// and their list for this one together. Filled by `subscribers` only; it is kept apart from
/// the settings above (`Db::set_blocklist`).
pub blocked: Vec<String>,
}
/// Someone who can sign in. `pass_hash` is None for an account that only ever arrives
@@ -200,6 +206,7 @@ impl From<subscriptions::Model> for Sub {
auto_download: s.auto_download,
allow_explicit: s.allow_explicit,
max_new_per_check: s.max_new_per_check,
blocked: vec![],
}
}
}
@@ -620,8 +627,11 @@ impl Db {
FROM enclosures e
LEFT JOIN (SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id) subs
ON subs.feed_id = e.feed_id
LEFT JOIN (SELECT feed_id, guid, count(*) AS n FROM entry_state
WHERE read GROUP BY feed_id, guid) readers
-- Someone an item is hidden from is as done with it as someone who read it.
LEFT JOIN (SELECT feed_id, guid, count(*) AS n
FROM (SELECT user_id, feed_id, guid FROM entry_state WHERE read
UNION SELECT user_id, feed_id, guid FROM hidden) done
GROUP BY feed_id, guid) readers
ON readers.feed_id = e.feed_id AND readers.guid = e.guid
WHERE e.path IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM entry_state s
@@ -683,13 +693,17 @@ impl Db {
.await?;
// Whatever went takes everyone's read state with it, rather than leaving rows
// pointing at an item that no longer exists.
self.exec(
"DELETE FROM entry_state WHERE NOT EXISTS (
SELECT 1 FROM entries e
WHERE e.feed_id = entry_state.feed_id AND e.guid = entry_state.guid)",
vec![],
)
.await?;
for table in ["entry_state", "hidden"] {
self.exec(
&format!(
"DELETE FROM {table} WHERE NOT EXISTS (
SELECT 1 FROM entries e
WHERE e.feed_id = {table}.feed_id AND e.guid = {table}.guid)"
),
vec![],
)
.await?;
}
Ok(n as usize)
}
}
@@ -701,7 +715,9 @@ impl Db {
"SELECT count(*) AS n FROM entries e
LEFT JOIN entry_state s
ON s.user_id = $2 AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = $1 AND NOT coalesce(s.read, false)",
WHERE e.feed_id = $1 AND NOT coalesce(s.read, false)
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = $2 AND h.feed_id = e.feed_id AND h.guid = e.guid)",
vec![feed_id.into(), user_id.into()],
)
.await?;
@@ -776,8 +792,16 @@ fn entries_from(a: &mut Args, user_id: i64, feed_id: Option<&str>, filter: Filte
format!(
"FROM entries e
LEFT JOIN entry_state s ON s.user_id = {user} AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE {scope} AND {} AND {search}",
filter.sql()
WHERE {scope} AND {} AND {search} AND {}",
filter.sql(),
not_hidden(&user)
)
}
/// Not hidden from this person by their block lists. `e` is entries.
fn not_hidden(user: &str) -> String {
format!(
"NOT EXISTS (SELECT 1 FROM hidden h WHERE h.user_id = {user} AND h.feed_id = e.feed_id AND h.guid = e.guid)"
)
}
@@ -1079,6 +1103,8 @@ impl Db {
("sessions", copy_table::<sessions::Entity>(&from.orm, &tx).await?),
("catalogue", copy_table::<catalogue::Entity>(&from.orm, &tx).await?),
("settings", copy_table::<settings::Entity>(&from.orm, &tx).await?),
("blocklists", copy_table::<blocklists::Entity>(&from.orm, &tx).await?),
("hidden", copy_table::<hidden::Entity>(&from.orm, &tx).await?),
];
if self.orm.get_database_backend() == sea_orm::DbBackend::Postgres {
// The copied ids came with the rows; the counters that hand out new ones start past
@@ -1166,9 +1192,12 @@ impl Db {
"SELECT coalesce(c.keywords, p.keywords) AS keywords,
coalesce(c.auto_download, p.auto_download) AS auto_download,
coalesce(c.allow_explicit, p.allow_explicit) AS allow_explicit,
coalesce(c.max_new_per_check, p.max_new_per_check) AS max_new_per_check
coalesce(c.max_new_per_check, p.max_new_per_check) AS max_new_per_check,
g.words AS global_blocked, b.words AS blocked
FROM subscriptions c
LEFT JOIN subscriptions p ON p.user_id = c.user_id AND p.feed_id = $2
LEFT JOIN blocklists g ON g.user_id = c.user_id AND g.feed_id = ''
LEFT JOIN blocklists b ON b.user_id = c.user_id AND b.feed_id = c.feed_id
WHERE c.feed_id = $1",
vec![feed_id.into(), group.map(str::to_owned).into()],
)
@@ -1181,6 +1210,10 @@ impl Db {
auto_download: r.try_get("", "auto_download")?,
allow_explicit: r.try_get("", "allow_explicit")?,
max_new_per_check: r.try_get("", "max_new_per_check")?,
blocked: [r.try_get("", "global_blocked")?, r.try_get("", "blocked")?]
.into_iter()
.flat_map(|w| keywords(w).unwrap_or_default())
.collect(),
})
})
.collect()
@@ -1217,11 +1250,13 @@ impl Db {
let r = self
.rows(
"SELECT sum(CASE WHEN st.flagged THEN 1 ELSE 0 END) AS starred,
sum(CASE WHEN st.read THEN 0 ELSE 1 END) AS unread
sum(CASE WHEN st.read OR h.guid IS NOT NULL THEN 0 ELSE 1 END) AS unread
FROM enclosures e
JOIN subscriptions s ON s.feed_id = e.feed_id AND s.user_id <> $2
LEFT JOIN entry_state st
ON st.user_id = s.user_id AND st.feed_id = e.feed_id AND st.guid = e.guid
LEFT JOIN hidden h
ON h.user_id = s.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid
WHERE e.id = $1",
vec![enclosure_id.into(), user_id.into()],
)
@@ -1234,11 +1269,99 @@ impl Db {
}
pub async fn subscribe(&self, user_id: i64, feed_id: &str) -> Result<()> {
self.exec(
"INSERT INTO subscriptions (user_id, feed_id) VALUES ($1, $2) ON CONFLICT DO NOTHING",
vec![user_id.into(), feed_id.into()],
)
.await?;
let new = self
.exec(
"INSERT INTO subscriptions (user_id, feed_id) VALUES ($1, $2) ON CONFLICT DO NOTHING",
vec![user_id.into(), feed_id.into()],
)
.await?;
if new > 0 {
// Their list for every feed applies to this one from the start.
self.rehide(feed_id).await?;
}
Ok(())
}
/// Someone's block list: for one feed, or with `feed_id` empty, for every feed.
pub async fn blocklist(&self, user_id: i64, feed_id: &str) -> Result<Vec<String>> {
Ok(blocklists::Entity::find_by_id((user_id, feed_id.to_owned()))
.one(&self.orm)
.await?
.and_then(|b| keywords(Some(b.words)))
.unwrap_or_default())
}
/// Replaces a block list and works out again what it hides: in that feed, or with
/// `feed_id` empty, in every feed the person reads.
pub async fn set_blocklist(&self, user_id: i64, feed_id: &str, words: &[String]) -> Result<()> {
if words.is_empty() {
blocklists::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?;
} else {
self.exec(
"INSERT INTO blocklists (user_id, feed_id, words) VALUES ($1, $2, $3)
ON CONFLICT (user_id, feed_id) DO UPDATE SET words = excluded.words",
vec![user_id.into(), feed_id.into(), serde_json::to_string(words)?.into()],
)
.await?;
}
if feed_id.is_empty() {
for sub in self.subscriptions_for(user_id).await? {
self.rehide(&sub.feed_id).await?;
}
} else {
self.rehide(feed_id).await?;
}
Ok(())
}
/// Works out again which of a feed's items each subscriber's block lists hide from them.
///
/// ponytail: every item of the feed for every subscriber, each time the feed is scanned with
/// something new or a list changes. Fine at hundreds of items a feed; look only at new items
/// on a scan if a feed with thousands makes scans slow.
pub async fn rehide(&self, feed_id: &str) -> Result<()> {
let lists: Vec<(i64, Vec<String>)> = self
.rows(
"SELECT s.user_id, g.words AS global, b.words AS feed
FROM subscriptions s
LEFT JOIN blocklists g ON g.user_id = s.user_id AND g.feed_id = ''
LEFT JOIN blocklists b ON b.user_id = s.user_id AND b.feed_id = s.feed_id
WHERE s.feed_id = $1",
vec![feed_id.into()],
)
.await?
.iter()
.map(|r| {
let words = [r.try_get("", "global")?, r.try_get("", "feed")?]
.into_iter()
.flat_map(|w: Option<String>| keywords(w).unwrap_or_default())
.collect();
Ok((r.try_get("", "user_id")?, words))
})
.collect::<Result<_>>()?;
hidden::Entity::delete_many().filter(hidden::Column::FeedId.eq(feed_id)).exec(&self.orm).await?;
let lists: Vec<_> = lists.into_iter().filter(|(_, w)| !w.is_empty()).collect();
if lists.is_empty() {
return Ok(());
}
let items = entries::Entity::find().filter(entries::Column::FeedId.eq(feed_id)).all(&self.orm).await?;
let rows: Vec<hidden::ActiveModel> = items
.iter()
.flat_map(|e| {
let hay = [e.title.as_deref().unwrap_or(""), e.description.as_deref().unwrap_or("")];
lists
.iter()
.filter(move |(_, words)| crate::download::blocked(words, &hay))
.map(|(user, _)| hidden::ActiveModel {
user_id: Set(*user),
feed_id: Set(feed_id.to_owned()),
guid: Set(e.guid.clone()),
})
})
.collect();
if !rows.is_empty() {
hidden::Entity::insert_many(rows).exec_without_returning(&self.orm).await?;
}
Ok(())
}
@@ -2050,6 +2173,42 @@ mod tests {
assert_eq!(listening().await, ["h", "e"]);
}
#[tokio::test]
async fn block_lists_hide_items_from_whoever_keeps_them() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'a',true), (2,'b',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'), (2,'f');
INSERT INTO entries (feed_id, guid, title, description, first_seen) VALUES
('f','x','Election night','',1),
('f','y','Baking bread','<p>No politics here, honest</p>',2),
('f','z','Gardening','',3);",
)
.await
.unwrap();
let shown = async |user| db.count_in(user, Some("f"), Filter::All, None).await.unwrap();
let words = |w: &[&str]| w.iter().map(|s| s.to_string()).collect::<Vec<_>>();
db.set_blocklist(1, "", &words(&["election"])).await.unwrap();
db.set_blocklist(1, "f", &words(&["politics"])).await.unwrap();
assert_eq!(shown(1).await, 1, "a title and a body each hide an item, both lists apply");
assert_eq!(db.unread_count(1, "f").await.unwrap(), 1, "hidden is not unread");
assert_eq!(shown(2).await, 3, "someone else's list hides nothing from you");
db.set_blocklist(1, "", &[]).await.unwrap();
assert_eq!(shown(1).await, 2, "emptying a list brings its items back");
assert!(db.blocklist(1, "").await.unwrap().is_empty());
// Someone subscribing starts with their list applied, and the scanner sees it.
db.set_blocklist(2, "", &words(&["gardening"])).await.unwrap();
db.exec_for_test("INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('g','w','Gardening',4)")
.await
.unwrap();
db.subscribe(2, "g").await.unwrap();
assert_eq!(db.count_in(2, Some("g"), Filter::All, None).await.unwrap(), 0);
assert_eq!(db.subscribers("g", None).await.unwrap()[0].blocked, ["gardening"]);
}
#[tokio::test]
async fn pending_takes_the_latest_episodes_first() {
// A cap of 3 must mean the three newest, not the three recorded first.