rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Signals: index the judge's query, six calls at once, one call per item on the server's key, tokens counted, a week's look-back an admin sets, edits that ask again only what was picked out (#274, #275, #276)

Browse Source 
The judge's query read all of entries (605 MB, 566 ms) on every wake; an
index on each feed's items by date, as the query asks, and the rechecks
in a half of their own. Calls ran one at a time, about 150 ms each, so a
new Signal's month of about 8,000 items took twenty minutes: six in flight
now. On the server's key everyone's Signals on an item share one call.
TypeSafe's token counts go to /metrics and each jev span.

An item's text is cut at 1,000 characters instead of 2,000: on 55 real
items of eight podcast feeds and six Signals, on the development key,
1,000 changed none of 330 verdicts (nor did the same request twice) and
saved 12% of the input tokens, 1,266 to 1,112 a call; 500 changed one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-17 14:13:25 +00:00
parent 026fbd1622
commit 2077ea5810
16 changed files with 293 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
crates/ipx/src/db.rs
View File

@@ -79,6 +79,10 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
// and visits the table only for the rows it shows. Sorting All Subscriptions from the
// table read all 13,700 of its rows, show notes and all, half a second to two cold (#188).
"CREATE INDEX IF NOT EXISTS entries_by_time ON entries (feed_id, guid, published, first_seen)",
// The Signals judge's question on every wake, what is recent in each subscribed feed: as
// the expression it is asked by. Without it, each wake read all of entries, 605 MB, 566 ms
// to find nothing (#274).
"CREATE INDEX IF NOT EXISTS entries_recent ON entries (feed_id, (coalesce(published, first_seen)))",
] {
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
@@ -113,6 +117,10 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
}
if orm.execute_unprepared("SELECT recheck FROM signal_verdicts LIMIT 0").await.is_err() {
let sql = "ALTER TABLE signal_verdicts ADD COLUMN recheck BOOLEAN NOT NULL DEFAULT false";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT icon FROM signals LIMIT 0").await.is_err() {
let sql = "ALTER TABLE signals ADD COLUMN icon TEXT";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
@@ -2589,8 +2597,6 @@ pub struct Unjudged {
pub signals: Vec<Ask>,
}
/// How far back a Signal looks when it is made: a month of items, not every back catalogue.
pub const SIGNAL_DAYS: i64 = 30;
impl Db {
pub async fn signals(&self, user_id: i64) -> Result<Vec<SignalRow>> {
@@ -2654,9 +2660,10 @@ impl Db {
Ok(m.id)
}
/// Changes a Signal. A new name, words or sentence puts every item to Jev again, since what it
/// said was about the old question (#259); a new icon does not. False for a Signal that is not
/// this person's.
/// Changes a Signal. A new name, words or sentence puts what it picked out to Jev again, since
/// that was the old question's answer; what it passed over stays passed over, and new items
/// are asked the new question as they come (#276). A new icon asks nothing. False for a Signal
/// that is not this person's.
pub async fn set_signal(&self, user_id: i64, id: i64, change: SignalChange<'_>) -> Result<bool> {
let SignalChange { name, words, description, icon } = change;
if self.signal(user_id, id).await?.is_none() {
@@ -2676,7 +2683,12 @@ impl Db {
m.icon = Set(i.map(str::to_owned));
}
if name.is_some() || words.is_some() || description.is_some() {
signal_verdicts::Entity::delete_many().filter(signal_verdicts::Column::SignalId.eq(id)).exec(&self.orm).await?;
signal_verdicts::Entity::update_many()
.col_expr(signal_verdicts::Column::Recheck, Expr::val(true).into())
.filter(signal_verdicts::Column::SignalId.eq(id))
.filter(signal_verdicts::Column::Hit.eq(true))
.exec(&self.orm)
.await?;
}
if name.is_some() || words.is_some() || description.is_some() || icon.is_some() {
m.update(&self.orm).await?;
@@ -2703,13 +2715,14 @@ impl Db {
.await
}
/// Up to `limit` items, newest first, that someone with a key has a Signal not yet asked
/// about: from the feeds they subscribe to, the last `SIGNAL_DAYS`, and not hidden from them.
/// `skip` are people whose key TypeSafe has refused. With `everyone`, the server's key is
/// used, and people without a key of their own are asked for too (#271).
pub async fn unjudged(&self, skip: &[i64], limit: i64, everyone: bool) -> Result<Vec<Unjudged>> {
/// Up to `limit` items, newest first, that someone with a key has a Signal to ask about: not
/// yet asked, from the last `days` (#276), or picked out before the Signal changed, however
/// old; from the feeds they subscribe to, and not hidden from them. `skip` are people whose
/// key TypeSafe has refused. With `everyone`, the server's key is used, and people without a
/// key of their own are asked for too (#271).
pub async fn unjudged(&self, skip: &[i64], limit: i64, everyone: bool, days: u64) -> Result<Vec<Unjudged>> {
let mut a = Args::default();
let since = a.p(now() - SIGNAL_DAYS * 86_400);
let since = a.p(now() - days as i64 * 86_400);
let skip = if skip.is_empty() {
String::new()
} else {
@@ -2717,23 +2730,32 @@ impl Db {
};
let limit = a.p(limit);
let keyed = if everyone { "true" } else { "u.typesafe_key IS NOT NULL" };
let sql = format!(
"SELECT g.id AS signal_id, g.name, g.keywords, g.description AS signal_description, g.user_id, u.typesafe_key, e.feed_id, e.guid, e.title,
e.description, f.title AS feed_title
FROM signals g
JOIN users u ON u.id = g.user_id
JOIN subscriptions s ON s.user_id = g.user_id
JOIN entries e ON e.feed_id = s.feed_id
LEFT JOIN feeds f ON f.id = e.feed_id
WHERE {keyed} {skip}
AND coalesce(e.published, e.first_seen) >= {since}
AND NOT EXISTS (SELECT 1 FROM signal_verdicts v
WHERE v.signal_id = g.id AND v.feed_id = e.feed_id AND v.guid = e.guid)
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = g.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid)
ORDER BY coalesce(e.published, e.first_seen) DESC, e.feed_id, e.guid, g.user_id
LIMIT {limit}"
// Two halves, not one OR: the recent half reads entries_recent, and the rechecks are a few
// rows of signal_verdicts. The same placeholders serve both.
let half = |items: &str| {
format!(
"SELECT g.id AS signal_id, g.name, g.keywords, g.description AS signal_description, g.user_id, u.typesafe_key,
e.feed_id, e.guid, e.title, e.description, f.title AS feed_title, coalesce(e.published, e.first_seen) AS at
FROM signals g
JOIN users u ON u.id = g.user_id
JOIN subscriptions s ON s.user_id = g.user_id
{items}
LEFT JOIN feeds f ON f.id = e.feed_id
WHERE {keyed} {skip}
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = g.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid)"
)
};
let recent = half(&format!(
"JOIN entries e ON e.feed_id = s.feed_id AND coalesce(e.published, e.first_seen) >= {since}
AND NOT EXISTS (SELECT 1 FROM signal_verdicts v
WHERE v.signal_id = g.id AND v.feed_id = e.feed_id AND v.guid = e.guid)"
));
let again = half(
"JOIN signal_verdicts v ON v.signal_id = g.id AND v.feed_id = s.feed_id AND v.recheck
JOIN entries e ON e.feed_id = v.feed_id AND e.guid = v.guid",
);
let sql = format!("SELECT * FROM ({recent} UNION ALL {again}) q ORDER BY at DESC, feed_id, guid, user_id LIMIT {limit}");
let mut out: Vec<Unjudged> = Vec::new();
for r in self.rows(&sql, a.0).await? {
let (user_id, feed_id, guid): (i64, String, String) =
@@ -2769,9 +2791,9 @@ impl Db {
// would refuse and SQLite would keep for nothing.
// shortcut: keywords changed mid-call keep a verdict on the old ones for that item.
self.exec(
"INSERT INTO signal_verdicts (signal_id, feed_id, guid, hit)
SELECT $1, $2, $3, $4 WHERE EXISTS (SELECT 1 FROM signals WHERE id = $1)
ON CONFLICT (signal_id, feed_id, guid) DO UPDATE SET hit = excluded.hit",
"INSERT INTO signal_verdicts (signal_id, feed_id, guid, hit, recheck)
SELECT $1, $2, $3, $4, false WHERE EXISTS (SELECT 1 FROM signals WHERE id = $1)
ON CONFLICT (signal_id, feed_id, guid) DO UPDATE SET hit = excluded.hit, recheck = false",
vec![(*signal).into(), feed_id.into(), guid.into(), (*hit).into()],
)
.await?;
@@ -3397,23 +3419,23 @@ mod tests {
let food = db.add_signal(1, "Food", &words(&["baking"]), Some("Bread and cakes")).await.unwrap();
db.add_signal(2, "Theirs", &words(&["x"]), None).await.unwrap();
assert!(db.unjudged(&[], 50, false).await.unwrap().is_empty(), "nobody has a key yet");
let everyone = db.unjudged(&[], 50, true).await.unwrap();
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "nobody has a key yet");
let everyone = db.unjudged(&[], 50, true, 30).await.unwrap();
let mut seen: Vec<_> = everyone.iter().map(|u| (u.user_id, u.guid.as_str(), u.key.is_none())).collect();
seen.sort();
assert_eq!(seen, [(1, "x", true), (1, "y", true), (2, "x", true)], "on the server's key, everyone's Signals, no key of their own needed");
db.set_typesafe_key(1, Some("k")).await.unwrap();
let queue = db.unjudged(&[], 50, false).await.unwrap();
let queue = db.unjudged(&[], 50, false, 30).await.unwrap();
let mut seen: Vec<_> = queue.iter().map(|u| (u.guid.as_str(), u.signals.len())).collect();
seen.sort();
assert_eq!(seen, [("x", 2), ("y", 2)], "each recent subscribed item once, with both Signals; old and unsubscribed left out");
let ask = queue[0].signals.iter().find(|a| a.id == food).unwrap();
assert_eq!((ask.name.as_str(), ask.description.as_deref()), ("Food", Some("Bread and cakes")), "Jev is asked about the name and the sentence");
assert!(db.unjudged(&[1], 50, false).await.unwrap().is_empty(), "a refused key is passed over");
assert!(db.unjudged(&[1], 50, false, 30).await.unwrap().is_empty(), "a refused key is passed over");
db.record_verdicts("f", "x", &[(f1, true), (food, false)]).await.unwrap();
db.record_verdicts("g", "y", &[(f1, false), (food, true)]).await.unwrap();
assert!(db.unjudged(&[], 50, false).await.unwrap().is_empty(), "nothing is asked twice");
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "nothing is asked twice");
let order = order_sql("published", "desc");
let page = db.signal_entries(1, f1, Filter::All, None, 0, 50, &order).await.unwrap();
@@ -3430,15 +3452,25 @@ mod tests {
assert!(!db.set_signal(2, f1, SignalChange { name: Some("mine"), ..Default::default() }).await.unwrap());
assert!(!db.remove_signal(2, f1).await.unwrap());
db.set_signal(1, f1, SignalChange { words: Some(&words(&["f1"])), ..Default::default() }).await.unwrap();
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 0, "new keywords are judged afresh");
db.record_verdicts("f", "x", &[(f1, true)]).await.unwrap();
// An edit asks again only what was picked out, however old, and leaves what was passed
// over (#276). Until answered, what was picked out stays in the list.
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { description: Some(Some("Grand prix racing")), ..Default::default() }).await.unwrap();
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 0, "so is a new sentence");
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().description.as_deref(), Some("Grand prix racing"));
let again: Vec<_> = db.unjudged(&[], 50, false, 30).await.unwrap().iter()
.map(|u| (u.guid.clone(), u.signals.iter().map(|a| a.id).collect::<Vec<_>>())).collect();
assert_eq!(again, [("x".to_string(), vec![f1]), ("old".to_string(), vec![f1])], "y was passed over, and stays so");
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2);
db.record_verdicts("f", "x", &[(f1, false)]).await.unwrap();
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "answered, not asked again");
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 1);
assert!(db.unjudged(&[], 50, false, 1).await.unwrap().is_empty());
db.record_verdicts("f", "x", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { icon: Some(Some("solid/flag-checkered")), ..Default::default() }).await.unwrap();
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 1, "a new icon asks nothing again");
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2, "x and old, kept");
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "a new icon asks nothing again");
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().icon.as_deref(), Some("solid/flag-checkered"));
assert!(db.remove_signal(1, food).await.unwrap());
db.record_verdicts("f", "x", &[(food, true)]).await.unwrap();