rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity
Files
f22c1993b1cda40944d98a13d6eec606ae2194ca
ipx/crates/ipx/src/db.rs
rays f22c1993b1 Import type names instead of writing their paths (#355) 
The style guide imports types and uses them unqualified. 222 places
wrote a path such as std::collections::HashMap or std::time::Duration
inline; each file now imports the type once, one use a module, and
names it bare. Done with a script, file by file, only where the name is
unambiguous there:
- one path for the name throughout the file, and not a name the file
  already binds otherwise (its own types, other imports, a function's
  local use);
- not a name generic enough to mislead bare (Error, Result, Value,
  Mutex, Client, Request, Response, Command, Event, Context, Map, Ok);
- only code a normal build compiles, so no import is unused outside
  tests or names a crate only tests depend on.

193 paths stay, those cases above. No behaviour change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 01:19:10 +00:00

4240 lines
197 KiB
Rust
Raw Blame History

//! The database, through SeaORM: SQLite today, Postgres to come (issue #18). Replaces the
//! per-feed .ipxd plists, history.dat and qmcache.dat.
use crate::entity::{
api_tokens, blocklists, catalogue, deadair_prefs, enclosures, entries, feeds, hidden, sessions, settings,
signal_verdicts, signals, subscriptions, users,
};
use anyhow::{Context, Result};
use sea_orm::sea_query::{Expr, Func};
use sea_orm::sqlx::sqlite::SqliteSynchronous;
use sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set,
Statement,
};
use sea_orm::{ConnectOptions, Database, DatabaseConnection};
use std::path::Path;
use std::time::SystemTime;
use tracing::Instrument;
/// A pool of connections, where there was one connection behind a global lock.
pub struct Db {
orm: DatabaseConnection,
/// A test's database file, removed when the test is done with it.
#[cfg(test)]
tmp: Option<std::path::PathBuf>,
}
#[cfg(test)]
impl Drop for Db {
fn drop(&mut self) {
if let Some(p) = &self.tmp {
for ext in ["", "-wal", "-shm"] {
let _ = std::fs::remove_file(format!("{}{ext}", p.display()));
}
}
}
}
/// Creates whatever tables and indexes the database is missing, from the entities in
/// `crate::entity`, on SQLite or Postgres alike. It only ever creates: an existing table is left
/// as it is. What an entity cannot say -- an index on an expression, or on two columns --
/// follows as plain SQL both databases accept.
///
/// Not SeaORM's schema sync, which is experimental and, despite its docs, drops a unique index
/// the entities do not describe: it dropped users_name_lower on every open, so every `ipx`
/// command took a write lock, and the healthcheck's `ipx status` timed out behind a busy daemon.
/// On Postgres it would have failed outright, dropping that index as a constraint.
///
/// ponytail: tables only, no columns. A column added to an existing table needs its own ALTER
/// here, as the old migrate() did for SQLite, or sea-orm-migration once there are several. A
/// database from before 0.7 took its last columns from that migrate(), so it upgrades through
/// a 0.7 release first.
async fn create_missing(orm: &DatabaseConnection) -> Result<()> {
use crate::entity::*;
use sea_orm::{ConnectionTrait, Schema};
let schema = Schema::new(orm.get_database_backend());
for mut table in [
schema.create_table_from_entity(feeds::Entity),
schema.create_table_from_entity(entries::Entity),
schema.create_table_from_entity(enclosures::Entity),
schema.create_table_from_entity(users::Entity),
schema.create_table_from_entity(subscriptions::Entity),
schema.create_table_from_entity(entry_state::Entity),
schema.create_table_from_entity(sessions::Entity),
schema.create_table_from_entity(catalogue::Entity),
schema.create_table_from_entity(settings::Entity),
schema.create_table_from_entity(blocklists::Entity),
schema.create_table_from_entity(hidden::Entity),
schema.create_table_from_entity(api_tokens::Entity),
schema.create_table_from_entity(deadair_plans::Entity),
schema.create_table_from_entity(deadair_prefs::Entity),
schema.create_table_from_entity(deadair_saved::Entity),
schema.create_table_from_entity(signals::Entity),
schema.create_table_from_entity(signal_verdicts::Entity),
] {
orm.execute(table.if_not_exists()).await.context("creating the schema")?;
}
for sql in [
"CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid)",
// The few rows with a file on disk, 15 of the server's many thousand: counting a person's
// files through enclosures_entry read every file row of their feeds, 58 ms a feed list
// (#165). Also what the healthcheck's count and the Downloaded filter look for.
"CREATE INDEX IF NOT EXISTS enclosures_on_disk ON enclosures (feed_id, guid) WHERE path IS NOT NULL",
"CREATE UNIQUE INDEX IF NOT EXISTS users_name_lower ON users (lower(name))",
// Every key and date an item list sorts by, so `entries_where` pages from the index alone
// and visits the table only for the rows it shows. Sorting All Subscriptions from the
// table read all 13,700 of its rows, show notes and all, half a second to two cold (#188).
"CREATE INDEX IF NOT EXISTS entries_by_time ON entries (feed_id, guid, published, first_seen)",
// The Signals judge's question on every wake, what is recent in each subscribed feed: as
// the expression it is asked by. Without it, each wake read all of entries, 605 MB, 566 ms
// to find nothing (#274).
"CREATE INDEX IF NOT EXISTS entries_recent ON entries (feed_id, (coalesce(published, first_seen)))",
// Unread lists read a feed's items from `subscriptions.unread_from` on (#347).
"CREATE INDEX IF NOT EXISTS entries_seen ON entries (feed_id, first_seen)",
] {
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
// A column added to a table that already exists, which CREATE TABLE IF NOT EXISTS leaves
// as it was. Looked for first, and added only when it is missing: this runs on every open,
// the healthcheck's `ipx status` included, and an ALTER takes a lock that waits behind a
// busy daemon, which is what made the healthcheck time out once before.
if orm.execute_unprepared("SELECT description FROM feeds LIMIT 0").await.is_err() {
// A feed is read whole only when it has changed since the last read, so each one's
// description would wait for its next item. Forgetting how each was last seen makes
// its next read, when it is due anyway, a whole one.
for sql in
["ALTER TABLE feeds ADD COLUMN description TEXT", "UPDATE feeds SET etag = NULL, last_modified = NULL"]
{
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
}
if orm.execute_unprepared("SELECT markdown FROM entries LIMIT 0").await.is_err() {
// Stored items get theirs as a scan finds them told differently, by the Markdown alone.
orm.execute_unprepared("ALTER TABLE entries ADD COLUMN markdown TEXT")
.await
.context("adding entries.markdown")?;
}
if orm.execute_unprepared("SELECT body_hash FROM feeds LIMIT 0").await.is_err() {
for sql in ["ALTER TABLE feeds ADD COLUMN body_hash TEXT", "ALTER TABLE feeds ADD COLUMN same_since BIGINT"] {
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
}
if orm.execute_unprepared("SELECT typesafe_key FROM users LIMIT 0").await.is_err() {
let sql = "ALTER TABLE users ADD COLUMN typesafe_key TEXT";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT kept FROM subscriptions LIMIT 0").await.is_err() {
let sql = "ALTER TABLE subscriptions ADD COLUMN kept BOOLEAN NOT NULL DEFAULT false";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT hide_from_all FROM subscriptions LIMIT 0").await.is_err() {
let sql = "ALTER TABLE subscriptions ADD COLUMN hide_from_all BOOLEAN NOT NULL DEFAULT false";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT hide_from_signals FROM subscriptions LIMIT 0").await.is_err() {
let sql = "ALTER TABLE subscriptions ADD COLUMN hide_from_signals BOOLEAN NOT NULL DEFAULT false";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT description FROM signals LIMIT 0").await.is_err() {
// Every verdict so far answered the old question, any one keyword on its own, which put
// a game's remaster in AI Development (#259): all of them are asked again.
for sql in ["ALTER TABLE signals ADD COLUMN description TEXT", "DELETE FROM signal_verdicts"] {
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
}
// An expression index has no statistics until its table is analysed, and without them
// Postgres guessed a third of entries were recent and read them all, entries_recent unused
// (#274). Analysed once when they are missing; autovacuum keeps them after.
// `/api/art` asks whether a feed or item names an image before serving it, and without an
// index that read all of entries, 571 ms an image, 50 at once on a page (#272). A hash index
// on Postgres, whose btree refuses a key over about 2.7 KB, longer than some image addresses.
orm.execute_unprepared(match orm.get_database_backend() {
sea_orm::DbBackend::Postgres => "CREATE INDEX IF NOT EXISTS entries_by_image ON entries USING hash (image)",
_ => "CREATE INDEX IF NOT EXISTS entries_by_image ON entries (image)",
})
.await
.context("indexing entries.image")?;
if orm.get_database_backend() == sea_orm::DbBackend::Postgres {
let has = Statement::from_string(
sea_orm::DbBackend::Postgres,
"SELECT 1 FROM pg_stats WHERE tablename = 'entries_recent'",
);
if orm.query_one_raw(has).await?.is_none() {
orm.execute_unprepared("ANALYZE entries").await.context("analysing entries")?;
}
}
if orm.execute_unprepared("SELECT recheck FROM signal_verdicts LIMIT 0").await.is_err() {
let sql = "ALTER TABLE signal_verdicts ADD COLUMN recheck BOOLEAN NOT NULL DEFAULT false";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT icon FROM signals LIMIT 0").await.is_err() {
let sql = "ALTER TABLE signals ADD COLUMN icon TEXT";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT unread_from FROM subscriptions LIMIT 0").await.is_err() {
let sql = "ALTER TABLE subscriptions ADD COLUMN unread_from BIGINT";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
if orm.execute_unprepared("SELECT pause_ms FROM deadair_prefs LIMIT 0").await.is_err() {
let sql = "ALTER TABLE deadair_prefs ADD COLUMN pause_ms BIGINT";
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
Ok(())
}
/// Where the database is: IPX_DATABASE_URL, a postgres:// URL, when it is set; otherwise the
/// SQLite file in the data directory, as it has always been.
pub fn location() -> String {
std::env::var("IPX_DATABASE_URL")
.ok()
.filter(|u| !u.trim().is_empty())
.unwrap_or_else(|| crate::config::data_dir().join("state.db").display().to_string())
}
/// The Postgres connection option that keeps notices from being sent at all; see `connect`.
const QUIET: &str = "options=-c%20client_min_messages%3Dwarning";
/// How many of one subscription's items, `sub`, are unread and not hidden: a query of its own
/// for each, so Postgres sees the feed and where its unread begins as constants and looks up the
/// few items since. Joined across every feed, it guessed a third of them and hashed all of the
/// person's read state, 22,949 rows for 44 unread (#352).
macro_rules! unread_in_sub {
() => {
"SELECT count(*) FROM entries e
LEFT JOIN entry_state s ON s.user_id = sub.user_id AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = sub.feed_id AND e.first_seen >= coalesce(sub.unread_from, 0)
AND NOT coalesce(s.read, false)
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = sub.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid)"
};
}
impl Db {
/// The connection pool as it stands, for /metrics (#194): connections open, those of them
/// idle, and the most it will open.
pub fn pool(&self) -> (u32, usize, u32) {
use sea_orm::DbBackend;
match self.orm.get_database_backend() {
DbBackend::Postgres => {
let p = self.orm.get_postgres_connection_pool();
(p.size(), p.num_idle(), p.options().get_max_connections())
}
_ => {
let p = self.orm.get_sqlite_connection_pool();
(p.size(), p.num_idle(), p.options().get_max_connections())
}
}
}
}
/// A URL fit for a log or an error: the password taken out.
fn redact(url: &str) -> String {
match (url.find("://"), url.rfind('@')) {
(Some(s), Some(at)) if at > s => match url[s + 3..at].find(':') {
Some(c) => format!("{}:***{}", &url[..s + 3 + c], &url[at..]),
None => url.to_owned(),
},
_ => url.to_owned(),
}
}
fn is_postgres(location: &str) -> bool {
location.starts_with("postgres://") || location.starts_with("postgresql://")
}
/// sqlx's defaults for SQLite are what ipx wants, foreign keys on and a five-second wait for a
/// lock, which is what rusqlite was set to, but for one.
async fn connect(location: &str, keep_open: bool) -> Result<DatabaseConnection> {
let mut opts = ConnectOptions::new(url_for(location));
opts.sqlx_logging(false);
// NORMAL, SQLite's own advice for WAL: synced at checkpoints, not after every commit. FULL,
// the default, waited for the disk after nearly every statement, and a feed's items are
// written a row at a time: 1.3s a feed on Tower's /tmp, where one synchronous write takes
// 52ms, and 39s to subscribe an admin to 1,500 feeds (#135). A crash of ipx loses nothing;
// only a power cut can lose the last transactions, and it never corrupts the file.
opts.map_sqlx_sqlite_opts(|o| o.synchronous(SqliteSynchronous::Normal));
// The same on Postgres, for ipx's own sessions: a commit waited for the WAL to reach the
// disk, 12.8ms on the databases project's server, and at about fifty commits a feed a scan
// took most of a second a feed (#140). A crash of the Postgres server can lose the last
// commits, about the last 0.6s, and never corrupts anything. Added to whatever options the
// URL sets, so it holds for the tests' search_path too.
opts.map_sqlx_postgres_opts(|o| o.options([("synchronous_commit", "off")]));
// Several connections, where SeaORM gives SQLite one unless told: every request, every scan
// write and every read went through it in turn, and twenty-five people browsing waited in
// line for it at 50 requests a second (#136). In WAL readers run beside the one writer. Each
// transaction here writes first, so it takes the write lock or waits out the busy timeout for
// it; SQLite refuses at once only a transaction that read a snapshot a write has since moved
// past. Postgres gets twenty, where sqlx's ten filled in bursts: a page load asks for dozens
// of covers at once, each with a lookup (#200). The server allows 100, shared with other apps.
opts.max_connections(if is_postgres(location) { 20 } else { 8 });
// The daemon keeps every connection open (#241): a Postgres connection takes 80-330 ms to
// open, its password exchange most of it, and a page load asked for more than were idle, so
// the feed list and item list waited on new ones, 300-900 ms where their queries took 5-30.
// Not a command such as `ipx status`, nor a test, which would hold twenty for one question.
if keep_open && is_postgres(location) {
opts.min_connections(20);
}
Database::connect(opts).await.with_context(|| format!("opening {}", redact(location)))
}
/// The URL `connect` hands sqlx for a location: a SQLite file, created if missing, or a Postgres
/// URL with notices turned off.
fn url_for(location: &str) -> String {
if !is_postgres(location) {
format!("sqlite://{location}?mode=rwc")
} else if location.contains("options=") {
location.to_owned() // someone chose their own; theirs stands
} else {
// Warnings and up only. Postgres answers every CREATE ... IF NOT EXISTS on an existing
// table with a notice, eleven on each open; sqlx logs each, no filter here wants them,
// and tracing-subscriber's per-layer filters then swallowed the next line ipx logged
// (0.3.23: "moved the feeds ... into the database" went missing that way).
let sep = if location.contains('?') { '&' } else { '?' };
format!("{location}{sep}{QUIET}")
}
}
/// One person's wants for one feed. `None` in a field means the feed's own setting stands.
#[derive(Debug, Clone, Default)]
pub struct Sub {
pub feed_id: String,
pub keywords: Option<Vec<String>>,
pub auto_download: Option<bool>,
pub allow_explicit: Option<bool>,
pub max_new_per_check: Option<i64>,
/// Words that keep an item from being downloaded for this person: their list for every feed
/// and their list for this one together. Filled by `subscribers` only; it is kept apart from
/// the settings above (`Db::set_blocklist`).
pub blocked: Vec<String>,
}
/// Someone who can sign in. `pass_hash` is None for an account that only ever arrives
/// through the proxy.
#[derive(Debug, Clone)]
pub struct User {
pub id: i64,
pub name: String,
pub pass_hash: Option<String>,
pub is_admin: bool,
/// When the account was made and when it last signed in, for whoever maintains the server.
pub created: Option<i64>,
pub last_login: Option<i64>,
}
impl From<enclosures::Model> for EncRow {
fn from(e: enclosures::Model) -> Self {
EncRow {
id: e.id,
deadair: None,
deadair_pending: false,
feed_id: e.feed_id,
guid: e.guid,
url: e.url,
mime: e.mime,
length: e.length,
path: e.path,
state: e.state,
last_error: e.last_error,
}
}
}
/// A subscription's keywords, stored as a JSON array.
fn keywords(json: Option<String>) -> Option<Vec<String>> {
json.and_then(|j| serde_json::from_str(&j).ok())
}
impl From<subscriptions::Model> for Sub {
fn from(s: subscriptions::Model) -> Self {
Sub {
feed_id: s.feed_id,
keywords: keywords(s.keywords),
auto_download: s.auto_download,
allow_explicit: s.allow_explicit,
max_new_per_check: s.max_new_per_check,
blocked: vec![],
}
}
}
impl From<users::Model> for User {
fn from(u: users::Model) -> Self {
User {
id: u.id,
name: u.name,
pass_hash: u.pass_hash,
is_admin: u.is_admin,
created: u.created,
last_login: u.last_login,
}
}
}
/// A feed derived from an OPML subscription rather than written into the config.
#[derive(Debug, Clone)]
pub struct Managed {
pub id: String,
pub url: String,
pub title: Option<String>,
pub group_id: String,
}
/// What `ipx list` shows next to each configured feed.
#[derive(Debug, Default)]
pub struct FeedSummary {
pub title: Option<String>,
pub image: Option<String>,
pub category: Option<String>,
/// Came from a subscribed OPML that no longer lists it, but it has downloads, so it
/// was kept rather than removed.
pub orphaned: bool,
pub last_checked: Option<i64>,
pub last_error: Option<String>,
/// When this run of failures began; see the `error_since` column.
pub error_since: Option<i64>,
/// Since when the feed has sent the same body (#242).
pub same_since: Option<i64>,
pub entries: i64,
pub downloaded: i64,
}
impl Db {
/// Opens the database at `location` (see `location()`): a postgres:// URL, or a SQLite file.
/// `keep_open` for the daemon, which keeps its whole pool connected.
pub async fn open(location: &str, keep_open: bool) -> Result<Self> {
if !is_postgres(location)
&& let Some(dir) = Path::new(location).parent()
{
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
}
let orm = connect(location, keep_open).await?;
if !is_postgres(location) {
// WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a
// setting of the file, kept once made, and making it takes a lock that cannot wait out
// a busy daemon, so it is made only when the file is not already in WAL.
let mode =
orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?;
if mode.and_then(|r| r.try_get_by_index::<String>(0).ok()).as_deref() != Some("wal") {
orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?;
}
}
create_missing(&orm).await?;
Ok(Self {
orm,
#[cfg(test)]
tmp: None,
})
}
/// A fresh, empty database for a test, its schema made from the entities as a real one's is.
/// A SQLite file of its own (two connections to one ":memory:" would be two databases); or,
/// with IPX_TEST_DATABASE_URL set, a Postgres schema of its own on that database, so the
/// same tests prove the SQL on both.
/// Runs a statement as it is, for a test outside this module setting up what it needs.
#[cfg(test)]
pub async fn sql(&self, sql: &str) -> Result<()> {
self.orm.execute_unprepared(sql).await?;
Ok(())
}
#[cfg(test)]
pub async fn memory() -> Result<Self> {
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let pid = std::process::id();
if let Some(url) = std::env::var("IPX_TEST_DATABASE_URL").ok().filter(|u| !u.is_empty()) {
let admin = connect(&url, false).await?;
// Schemas an earlier run left behind: anything of ours not made by this process.
let old = admin
.query_all_raw(Statement::from_string(
admin.get_database_backend(),
format!(
r"SELECT nspname FROM pg_namespace
WHERE nspname LIKE 'ipxt\_%' AND nspname NOT LIKE 'ipxt\_{pid}\_%'"
),
))
.await?;
for r in old {
let name: String = r.try_get_by_index(0)?;
admin.execute_unprepared(&format!("DROP SCHEMA IF EXISTS {name} CASCADE")).await?;
}
let schema = format!("ipxt_{pid}_{n}");
admin.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await?;
// Every connection in the pool starts in it, so each test sees only its own tables.
let sep = if url.contains('?') { '&' } else { '?' };
let orm = connect(
&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"),
false,
)
.await?;
create_missing(&orm).await?;
return Ok(Self { orm, tmp: None });
}
let path = std::env::temp_dir().join(format!("ipx-test-{pid}-{n}.db"));
let orm = connect(&path.display().to_string(), false).await?;
// As Db::open has it, so the tests' connections share the file as the daemon's do.
orm.execute_unprepared("PRAGMA journal_mode = WAL").await?;
create_missing(&orm).await?;
Ok(Self { orm, tmp: Some(path) })
}
/// The first column of every row, for a test to check what a query left behind.
#[cfg(test)]
pub async fn i64s_for_test(&self, sql: &str) -> Vec<i64> {
self.rows(sql, vec![]).await.unwrap().iter().map(|r| r.try_get_by_index(0).unwrap()).collect()
}
#[cfg(test)]
pub async fn strings_for_test(&self, sql: &str) -> Vec<String> {
self.rows(sql, vec![]).await.unwrap().iter().map(|r| r.try_get_by_index(0).unwrap()).collect()
}
#[cfg(test)]
pub async fn exec_for_test(&self, sql: &str) -> Result<()> {
self.orm.execute_unprepared(sql).await?;
Ok(())
}
/// What a feed says it is, for its page in the Directory.
pub async fn feed_description(&self, feed_id: &str) -> Result<Option<String>> {
Ok(feeds::Entity::find_by_id(feed_id.to_owned()).one(&self.orm).await?.and_then(|f| f.description))
}
#[tracing::instrument(skip_all)]
pub async fn feed_summary(&self, feed_id: &str) -> Result<FeedSummary> {
let mut sum = feeds::Entity::find_by_id(feed_id.to_owned())
.one(&self.orm)
.await?
.map(|f| FeedSummary {
title: f.title,
image: f.image,
last_checked: f.last_checked,
last_error: f.last_error,
orphaned: f.orphaned,
error_since: f.error_since,
category: f.category,
..Default::default()
})
.unwrap_or_default();
sum.entries =
entries::Entity::find().filter(entries::Column::FeedId.eq(feed_id)).count(&self.orm).await? as i64;
sum.downloaded = self.downloaded_count(feed_id).await?;
Ok(sum)
}
/// What the feed list shows of every feed, for one person, in five queries whatever the
/// number of feeds. Asked feed by feed (feed_summary, http_state, blocklist, unread_count)
/// it was six round trips a feed, about 950 for 160 feeds and 320 ms a page load (#94).
pub async fn feed_list(
&self,
user_id: i64,
// One feed only, for the row a live update sends (`web::feed_rows`).
only: Option<&str>,
) -> Result<std::collections::HashMap<String, FeedListing>> {
use std::collections::HashMap;
let one = || sea_orm::Value::from(only.map(str::to_owned));
// A step a query: the feed list's trace was 290 ms with nothing under it.
let counts = |of: &'static str, sql: &'static str, args: Vec<sea_orm::Value>| async move {
Instrument::instrument(self.rows(sql, args), step(of))
.await?
.iter()
.map(|r| Ok((r.try_get::<String>("", "feed_id")?, r.try_get::<i64>("", "n")?)))
.collect::<Result<HashMap<_, _>>>()
};
// Of this person's feeds only, as the unread count is: counting every item of every feed on
// the server, 1534 feeds for someone reading 146, was 184-316 ms, 70% of the feed list's
// time (#165). The Directory reads this for every feed's title and artwork, not a count.
// Five queries that need nothing of each other, run at once: one after another they were
// 59 ms of a 67 ms feed list, on a pool of idle connections (#346).
let entries = counts(
"entries",
"SELECT e.feed_id, count(*) AS n FROM entries e
JOIN subscriptions sub ON sub.user_id = $1 AND sub.feed_id = e.feed_id
WHERE (CAST($2 AS TEXT) IS NULL OR e.feed_id = $2) GROUP BY e.feed_id",
vec![user_id.into(), one()],
);
let downloaded = counts(
"downloaded",
"SELECT x.feed_id, count(*) AS n FROM enclosures x
JOIN subscriptions sub ON sub.user_id = $1 AND sub.feed_id = x.feed_id
WHERE x.path IS NOT NULL AND (CAST($2 AS TEXT) IS NULL OR x.feed_id = $2) GROUP BY x.feed_id",
vec![user_id.into(), one()],
);
let unread = counts(
"unread",
concat!(
"SELECT * FROM (SELECT sub.feed_id, (",
unread_in_sub!(),
") AS n FROM subscriptions sub
WHERE sub.user_id = $1 AND (CAST($2 AS TEXT) IS NULL OR sub.feed_id = $2)) x
WHERE n > 0"
),
vec![user_id.into(), one()],
);
let blocked = Instrument::instrument(
blocklists::Entity::find().filter(blocklists::Column::UserId.eq(user_id)).all(&self.orm),
step("blocked"),
);
let mut rows = feeds::Entity::find();
if let Some(id) = only {
rows = rows.filter(feeds::Column::Id.eq(id));
}
let rows = Instrument::instrument(rows.all(&self.orm), step("feeds"));
let (entries, downloaded, unread, blocked, rows) = tokio::try_join!(
entries,
downloaded,
unread,
async { blocked.await.map_err(anyhow::Error::from) },
async { rows.await.map_err(anyhow::Error::from) },
)?;
let mut blocked: HashMap<String, Vec<String>> =
blocked.into_iter().map(|b| (b.feed_id, keywords(Some(b.words)).unwrap_or_default())).collect();
Ok(rows
.into_iter()
.map(|f| {
let id = f.id.clone();
let listing = FeedListing {
summary: FeedSummary {
entries: entries.get(&id).copied().unwrap_or(0),
downloaded: downloaded.get(&id).copied().unwrap_or(0),
title: f.title,
image: f.image,
last_checked: f.last_checked,
last_error: f.last_error,
orphaned: f.orphaned,
error_since: f.error_since,
same_since: f.same_since,
category: f.category,
},
ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64),
blocked: blocked.remove(&id).unwrap_or_default(),
unread: unread.get(&id).copied().unwrap_or(0),
};
(id, listing)
})
.collect())
}
}
/// One feed as the feed list shows it; see `Db::feed_list`.
#[derive(Debug, Default)]
pub struct FeedListing {
pub summary: FeedSummary,
pub ttl_mins: Option<u64>,
pub blocked: Vec<String>,
pub unread: i64,
}
/// Everything `fetch` needs to decide whether to poll a feed, and how.
#[derive(Debug, Default)]
pub struct HttpState {
pub etag: Option<String>,
pub last_modified: Option<String>,
pub last_checked: Option<i64>,
pub ttl_mins: Option<u64>,
/// When the feed's current run of failures began, for backing off (`due_after`).
pub error_since: Option<i64>,
/// The last body's hash, and since when it has been that body (#242).
pub body_hash: Option<String>,
pub same_since: Option<i64>,
}
impl From<feeds::Model> for HttpState {
fn from(f: feeds::Model) -> Self {
HttpState {
etag: f.etag,
last_modified: f.last_modified,
last_checked: f.last_checked,
ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64),
error_since: f.error_since,
body_hash: f.body_hash,
same_since: f.same_since,
}
}
}
impl Db {
pub async fn http_state(&self, feed_id: &str) -> Result<HttpState> {
Ok(feeds::Entity::find_by_id(feed_id.to_owned()).one(&self.orm).await?.map(HttpState::from).unwrap_or_default())
}
/// Every feed's, in one query: a scan, and the daemon working out when the next is due, asked
/// feed by feed, 180 queries a minute (#114).
pub async fn http_states(&self) -> Result<std::collections::HashMap<String, HttpState>> {
Ok(feeds::Entity::find()
.all(&self.orm)
.await?
.into_iter()
.map(|f| (f.id.clone(), HttpState::from(f)))
.collect())
}
/// Upsert after a successful poll. Clears any previous error.
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip_all)]
pub async fn record_feed(
&self,
feed_id: &str,
url: &str,
title: Option<&str>,
etag: Option<&str>,
last_modified: Option<&str>,
ttl_mins: Option<u64>,
image: Option<&str>,
category: Option<&str>,
description: Option<&str>,
) -> Result<()> {
// category is taken as it comes, unlike title and image: a show that leaves a category
// should leave the Directory's chip too.
let o = |v: Option<&str>| sea_orm::Value::from(v.map(str::to_owned));
self.exec(
"INSERT INTO feeds (id, url, title, etag, last_modified, last_checked, ttl_mins, last_error, image, category, description)
VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9, $10)
ON CONFLICT (id) DO UPDATE SET
url = excluded.url,
title = coalesce(excluded.title, feeds.title),
etag = excluded.etag,
last_modified = excluded.last_modified,
last_checked = excluded.last_checked,
ttl_mins = excluded.ttl_mins,
image = coalesce(excluded.image, feeds.image),
category = excluded.category,
description = coalesce(excluded.description, feeds.description),
last_error = NULL,
error_since = NULL",
vec![
feed_id.into(),
url.into(),
o(title),
o(etag),
o(last_modified),
now().into(),
ttl_mins.map(|t| t as i64).into(),
o(image),
o(category),
o(description),
],
)
.await?;
Ok(())
}
/// 304, or any other poll that produced no new data: only the clock moves.
/// A body read whole that was not the last one: its hash, and that it changed now (#242).
pub async fn set_body(&self, feed_id: &str, hash: &str) -> Result<()> {
self.exec(
"UPDATE feeds SET body_hash = $2, same_since = $3 WHERE id = $1",
vec![feed_id.into(), hash.into(), now().into()],
)
.await?;
Ok(())
}
pub async fn touch_feed(&self, feed_id: &str, url: &str) -> Result<()> {
self.exec(
"INSERT INTO feeds (id, url, last_checked) VALUES ($1, $2, $3)
ON CONFLICT (id) DO UPDATE SET last_checked = excluded.last_checked,
last_error = NULL, error_since = NULL",
vec![feed_id.into(), url.into(), now().into()],
)
.await?;
Ok(())
}
/// Forgets the cached ETag/Last-Modified. Those validators belong to the old URL, so
/// keeping them across a URL change could produce a bogus 304 against the new one.
pub async fn clear_validators(&self, feed_id: &str) -> Result<()> {
self.exec(
"UPDATE feeds SET etag = NULL, last_modified = NULL, last_checked = NULL WHERE id = $1",
vec![feed_id.into()],
)
.await?;
Ok(())
}
pub async fn set_feed_error(&self, feed_id: &str, url: &str, msg: &str) -> Result<()> {
self.exec(
"INSERT INTO feeds (id, url, last_checked, last_error, error_since)
VALUES ($1, $2, $3, $4, $3)
ON CONFLICT (id) DO UPDATE SET
last_checked = excluded.last_checked,
last_error = excluded.last_error,
error_since = coalesce(feeds.error_since, excluded.error_since)",
vec![feed_id.into(), url.into(), now().into(), msg.into()],
)
.await?;
Ok(())
}
/// Returns true when this entry had not been seen before.
///
pub async fn record_entry(&self, feed_id: &str, e: &crate::feed::Entry) -> Result<bool> {
let inserted = self
.exec(
"INSERT INTO entries
(feed_id, guid, title, link, published, description, first_seen,
image, duration, episode, season, markdown)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT DO NOTHING",
vec![
feed_id.into(),
e.guid.clone().into(),
e.title.clone().into(),
e.link.clone().into(),
e.published.into(),
e.description.clone().into(),
now().into(),
e.image.clone().into(),
e.duration.into(),
e.episode.into(),
e.season.into(),
e.markdown.clone().into(),
],
)
.await?;
if inserted == 0 {
self.exec(
"UPDATE entries SET
title = coalesce($3, title),
description = coalesce($4, description),
image = coalesce($5, image),
duration = coalesce($6, duration),
episode = coalesce($7, episode),
season = coalesce($8, season),
markdown = coalesce($9, markdown)
WHERE feed_id = $1 AND guid = $2",
vec![
feed_id.into(),
e.guid.clone().into(),
e.title.clone().into(),
e.description.clone().into(),
e.image.clone().into(),
e.duration.into(),
e.episode.into(),
e.season.into(),
e.markdown.clone().into(),
],
)
.await?;
}
Ok(inserted == 1)
}
/// Returns true when this enclosure URL is new. False means we have downloaded it
/// before, or deliberately reaped it -- either way it is not fetched again.
pub async fn mark_downloaded(&self, url: &str, path: &Path, bytes: u64) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = 'done', path = $2, bytes_done = $3,
downloaded_at = $4, last_error = NULL WHERE url = $1",
vec![url.into(), path.to_string_lossy().into_owned().into(), (bytes as i64).into(), now().into()],
)
.await?;
Ok(())
}
/// The row stays -- a failed URL is still a URL we have seen. `state` says why it has
/// no file, and a retry is an explicit act rather than something a rescan does silently.
pub async fn mark_enclosure(&self, url: &str, state: &str, error: Option<&str>) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = $2, last_error = $3 WHERE url = $1",
vec![url.into(), state.into(), error.map(str::to_owned).into()],
)
.await?;
Ok(())
}
pub async fn record_enclosure(&self, feed_id: &str, guid: &str, enc: &crate::feed::Enclosure) -> Result<bool> {
let inserted = self
.exec(
"INSERT INTO enclosures (feed_id, guid, url, mime, length, state)
VALUES ($1, $2, $3, $4, $5, 'pending') ON CONFLICT DO NOTHING",
vec![feed_id.into(), guid.into(), enc.url.clone().into(), enc.mime.clone().into(), enc.length.into()],
)
.await?;
Ok(inserted == 1)
}
}
/// An enclosure waiting to be downloaded.
#[derive(Debug)]
pub struct Pending {
pub id: i64,
pub url: String,
pub mime: Option<String>,
}
impl Db {
/// The download queue is the table, not the parse result: what a scan could not finish is
/// picked up by the next. Only from the feed's `limit` newest items with a file, though: a
/// limit of 3 means the three latest episodes. Taking the newest three still pending, each
/// full read of a feed took the three before the ones already downloaded, working back
/// through every show's history: 4420 files queued across 12 shows on the default of 3
/// (#97). Unlimited, 0 in the settings, is the whole back catalogue, for an archive. An item
/// whose files were all skipped by a filter does not hold one of the places.
/// Puts a feed's waiting files in the queue or out of it: 'pending' for those among its
/// `limit` newest items, which the scan downloads on its own, and 'held' for the rest, listed
/// to download by hand. With all of them 'pending', the queue counted 4420 back-catalogue
/// files no scan would ever take (#97, #113). 0 holds them all, for a feed nobody downloads
/// automatically. A held file still holds its item's place among the newest.
#[tracing::instrument(skip_all)]
pub async fn hold_back(&self, feed_id: &str, limit: usize) -> Result<()> {
let newest = "SELECT n.guid FROM entries n
WHERE n.feed_id = $1
AND EXISTS (SELECT 1 FROM enclosures y
WHERE y.feed_id = n.feed_id AND y.guid = n.guid
AND y.state <> 'skipped')
ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC
LIMIT $2";
let limit: sea_orm::Value = (limit.min(i64::MAX as usize) as i64).into();
self.exec(
&format!("UPDATE enclosures SET state = 'held' WHERE feed_id = $1 AND state = 'pending' AND guid NOT IN ({newest})"),
vec![feed_id.into(), limit.clone()],
)
.await?;
// And back, when a limit is raised or a feed downloads again.
self.exec(
&format!(
"UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"
),
vec![feed_id.into(), limit],
)
.await?;
Ok(())
}
#[tracing::instrument(skip_all)]
pub async fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
self.rows(
"SELECT x.id, x.url, x.mime FROM enclosures x
JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid
WHERE x.feed_id = $1 AND x.state = 'pending'
AND e.guid IN (SELECT n.guid FROM entries n
WHERE n.feed_id = $1
AND EXISTS (SELECT 1 FROM enclosures y
WHERE y.feed_id = n.feed_id AND y.guid = n.guid
AND y.state <> 'skipped')
ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC
LIMIT $2)
ORDER BY coalesce(e.published, e.first_seen) DESC, x.id DESC
LIMIT $2",
// Unlimited is usize::MAX, which `as i64` makes -1: SQLite reads LIMIT -1 as no
// limit, Postgres refuses it, and the feed's downloads failed (#98).
vec![feed_id.into(), (limit.min(i64::MAX as usize) as i64).into()],
)
.await?
.iter()
.map(|r| Ok(Pending { id: r.try_get("", "id")?, url: r.try_get("", "url")?, mime: r.try_get("", "mime")? }))
.collect()
}
}
/// A downloaded file the reaper may consider.
#[derive(Debug, Clone, PartialEq)]
pub struct Candidate {
pub id: i64,
pub url: String,
pub path: String,
pub bytes: i64,
/// When it landed; the reaper works oldest-first.
pub age_key: i64,
pub read: bool,
}
impl Db {
/// Files on disk that may be deleted to get back under quota: starred by nobody,
/// with the ones everybody has finished going first, oldest first within each group.
///
/// One file serves every subscriber, so both tests are about all of them: **anyone**
/// starring it keeps it, and it only counts as read when **everyone** subscribed has
/// read it. A file whose feed nobody subscribes to has no one left to keep it, so it
/// sorts with the read ones. Only the marks of people who subscribe count: unsubscribing
/// leaves yours behind, and counted, a reader who had left made an episode someone had not
/// played look finished, and a pin of theirs kept a file for good (#156).
///
/// (The Python intended `read = 1 AND flagged = 0` but never achieved it -- a missing
/// plistlib import and an `EntreiesData` typo meant the filter always threw.)
pub async fn reap_candidates(&self) -> Result<Vec<Candidate>> {
// Yes/no as true and false, not 1 and 0: Postgres types a bare 1 as a 32-bit integer
// and will not hand it over as an i64.
self.rows(
"SELECT e.id, e.url, e.path, e.bytes_done, coalesce(e.downloaded_at, 0) AS age_key,
CASE WHEN coalesce(readers.n, 0) >= coalesce(subs.n, 0) THEN true ELSE false END AS read
FROM enclosures e
LEFT JOIN (SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id) subs
ON subs.feed_id = e.feed_id
-- Someone an item is hidden from is as done with it as someone who read it.
LEFT JOIN (SELECT done.feed_id, done.guid, count(*) AS n
FROM (SELECT user_id, feed_id, guid FROM entry_state WHERE read
UNION SELECT user_id, feed_id, guid FROM hidden) done
JOIN subscriptions u ON u.user_id = done.user_id AND u.feed_id = done.feed_id
GROUP BY done.feed_id, done.guid) readers
ON readers.feed_id = e.feed_id AND readers.guid = e.guid
WHERE e.path IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM entry_state s
JOIN subscriptions u ON u.user_id = s.user_id AND u.feed_id = s.feed_id
WHERE s.feed_id = e.feed_id AND s.guid = e.guid AND s.flagged)
ORDER BY 6 DESC, coalesce(e.downloaded_at, 0) ASC, e.id ASC",
vec![],
)
.await?
.iter()
.map(|r| {
Ok(Candidate {
id: r.try_get("", "id")?,
url: r.try_get("", "url")?,
path: r.try_get("", "path")?,
bytes: r.try_get("", "bytes_done")?,
age_key: r.try_get("", "age_key")?,
read: r.try_get("", "read")?,
})
})
.collect()
}
/// The row survives the file: that is what stops a reaped episode being re-downloaded.
pub async fn mark_reaped(&self, id: i64) -> Result<()> {
self.exec("UPDATE enclosures SET state = 'reaped', path = NULL WHERE id = $1", vec![id.into()]).await?;
Ok(())
}
/// Rows claiming a file that is no longer there (someone deleted it by hand).
pub async fn missing_files(&self) -> Result<Vec<(i64, String)>> {
Ok(enclosures::Entity::find()
.filter(enclosures::Column::Path.is_not_null())
.all(&self.orm)
.await?
.into_iter()
.filter_map(|e| Some((e.id, e.path?)))
.filter(|(_, p)| !Path::new(p).exists())
.collect())
}
/// Old entries that never had a file, or no longer have one. Enclosure rows stay --
/// they are the dedupe history.
/// Forgets feeds nobody subscribes to that were failing when the last subscriber left: not in
/// the catalogue, no file on disk. Unsubscribing leaves a feed's row and history behind, which
/// is right for one that worked; one that never did, like cnn-com added without its https://,
/// sat in the database with its error for good. Returns the ids forgotten.
pub async fn prune_abandoned_failures(&self) -> Result<Vec<String>> {
let gone: Vec<String> = self
.rows(
"DELETE FROM feeds
WHERE last_error IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = feeds.id)
AND NOT EXISTS (SELECT 1 FROM catalogue c WHERE c.id = feeds.id)
AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = feeds.id AND x.path IS NOT NULL)
RETURNING id",
vec![],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "id")?))
.collect::<Result<_>>()?;
for id in &gone {
self.forget_rows(id).await?;
}
Ok(gone)
}
/// Everything stored about a feed but its row: items, file rows, read state, block list.
async fn forget_rows(&self, id: &str) -> Result<()> {
for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.into()]).await?;
}
Ok(())
}
/// A feed's stored artwork still on http, items' and its own: hosts a feed no longer names
/// keep their old items' artwork on http otherwise (IGN's 2009 items, #110).
pub async fn http_images(&self, feed_id: &str) -> Result<Vec<String>> {
self.rows(
"SELECT DISTINCT image FROM entries WHERE feed_id = $1 AND image LIKE 'http://%'
UNION SELECT image FROM feeds WHERE id = $1 AND image LIKE 'http://%'",
vec![feed_id.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "image")?))
.collect()
}
/// Rewrites a feed's stored artwork on `host` from http to https, once the host is known to
/// serve it there (`feed::prefer_https`), for the items a scan does not write again.
pub async fn secure_images(&self, feed_id: &str, host: &str) -> Result<()> {
let like = format!("http://{host}/%");
for table in ["entries", "feeds"] {
let col = if table == "feeds" { "id" } else { "feed_id" };
self.exec(
&format!(
"UPDATE {table} SET image = 'https://' || substr(image, 8) WHERE {col} = $1 AND image LIKE $2"
),
vec![feed_id.into(), like.clone().into()],
)
.await?;
}
Ok(())
}
/// Forgets a feed and everything stored about it.
pub async fn forget_feed(&self, id: &str) -> Result<()> {
self.forget_rows(id).await?;
self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?;
Ok(())
}
/// Feeds nobody subscribes to that are dead, failing since before `dead_before`, or quiet,
/// their newest item from before `quiet_before`: what keeps the Directory to feeds worth
/// taking. One with a file on disk stays, and so does one an OPML subscription lists, which
/// comes back with the OPML anyway. A feed with no items yet is not quiet, only new.
pub async fn stale_unsubscribed(&self, dead_before: i64, quiet_before: i64) -> Result<Vec<String>> {
self.rows(
"SELECT f.id FROM feeds f
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id)
AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = f.id AND x.path IS NOT NULL)
AND f.group_id IS NULL AND NOT coalesce(f.managed, false)
AND ((f.error_since IS NOT NULL AND f.error_since < $1)
OR (SELECT max(coalesce(e.published, e.first_seen)) FROM entries e
WHERE e.feed_id = f.id) < $2)",
vec![dead_before.into(), quiet_before.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "id")?))
.collect()
}
pub async fn prune_entries(&self, older_than: i64) -> Result<usize> {
let n = self
.exec(
"DELETE FROM entries
WHERE coalesce(published, first_seen) < $1
AND NOT EXISTS (
SELECT 1 FROM enclosures e
WHERE e.feed_id = entries.feed_id AND e.guid = entries.guid
AND e.path IS NOT NULL)
-- Starred by anyone subscribed keeps it, the same rule the reaper follows.
AND NOT EXISTS (
SELECT 1 FROM entry_state s
JOIN subscriptions u ON u.user_id = s.user_id AND u.feed_id = s.feed_id
WHERE s.feed_id = entries.feed_id AND s.guid = entries.guid
AND s.flagged)",
vec![older_than.into()],
)
.await?;
// Whatever went takes everyone's read state with it, rather than leaving rows
// pointing at an item that no longer exists.
for table in ["entry_state", "hidden"] {
self.exec(
&format!(
"DELETE FROM {table} WHERE NOT EXISTS (
SELECT 1 FROM entries e
WHERE e.feed_id = {table}.feed_id AND e.guid = {table}.guid)"
),
vec![],
)
.await?;
}
Ok(n as usize)
}
}
impl Db {
pub async fn unread_count(&self, user_id: i64, feed_id: &str) -> Result<i64> {
let r = self
.rows(
"SELECT count(*) AS n FROM entries e
LEFT JOIN entry_state s
ON s.user_id = $2 AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = $1 AND NOT coalesce(s.read, false)
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = $2 AND h.feed_id = e.feed_id AND h.guid = e.guid)",
vec![feed_id.into(), user_id.into()],
)
.await?;
Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?)
}
/// (pending, downloaded) across all feeds, for the status command.
pub async fn counts(&self) -> Result<(i64, i64)> {
let pending =
enclosures::Entity::find().filter(enclosures::Column::State.eq("pending")).count(&self.orm).await?;
let downloaded =
enclosures::Entity::find().filter(enclosures::Column::Path.is_not_null()).count(&self.orm).await?;
Ok((pending as i64, downloaded as i64))
}
}
/// An entry plus its enclosures, for the web UI.
#[derive(Debug, serde::Serialize)]
pub struct EntryRow {
pub guid: String,
pub feed_id: String,
pub title: Option<String>,
pub link: Option<String>,
pub published: Option<i64>,
pub description: Option<String>,
/// The item's Markdown as written, when its feed sends it beside the HTML, and that Markdown
/// made HTML and cleaned: the page switches between the three (#285).
pub markdown: Option<String>,
pub markdown_html: Option<String>,
pub read: bool,
pub flagged: bool,
pub image: Option<String>,
pub duration: Option<i64>,
pub episode: Option<i64>,
pub season: Option<i64>,
pub position: i64,
pub enclosures: Vec<EncRow>,
}
/// A statement's parameters, gathered as its SQL is written: each `p` binds a value and gives
/// back its `$n`. Only what the SQL uses is bound -- Postgres refuses a parameter it cannot
/// place, where rusqlite needed every one mentioned, which is what `?1 IS NULL` was for.
#[derive(Default)]
struct Args(Vec<sea_orm::Value>);
impl Args {
fn p(&mut self, v: impl Into<sea_orm::Value>) -> String {
self.0.push(v.into());
format!("${}", self.0.len())
}
}
/// The FROM and WHERE that `entries_in` and `count_in` share: whose read state, which feeds
/// (one, or every feed the person subscribes to), the filter, and a case-insensitive search of
/// title and description.
fn entries_from(
a: &mut Args,
user_id: i64,
feeds: Option<&[&str]>,
filter: Filter,
search: Option<&str>,
guid: Option<&str>,
signal: Option<i64>,
) -> String {
let user = a.p(user_id);
let mut scope = match feeds {
// One feed as before, so its page keeps its plan: an OR with the shows' list, or an IN
// over a subquery, took clarkesworld's from 10 ms to 26 ms or a second (#283).
Some([f]) => format!("e.feed_id = {}", a.p(*f)),
Some([]) => "false".into(),
Some(fs) => format!("e.feed_id IN ({})", fs.iter().map(|f| a.p(*f)).collect::<Vec<_>>().join(", ")),
// A Signal and All Subscriptions leave out what this person hid from them (#288, #289).
// An episode under way still shows in Currently Listening, which is All Subscriptions'
// in-progress filter.
None if signal.is_some() => {
format!(
"e.feed_id IN (SELECT m.feed_id FROM subscriptions m WHERE m.user_id = {user} AND {})",
shown_in("hide_from_signals", "m")
)
}
None if filter != Filter::InProgress => {
format!(
"e.feed_id IN (SELECT m.feed_id FROM subscriptions m WHERE m.user_id = {user} AND {})",
shown_in("hide_from_all", "m")
)
}
None => format!("e.feed_id IN (SELECT feed_id FROM subscriptions WHERE user_id = {user})"),
};
if let Some(id) = signal {
scope = format!(
"{scope} AND EXISTS (SELECT 1 FROM signal_verdicts v WHERE v.signal_id = {} AND v.feed_id = e.feed_id AND v.guid = e.guid AND v.hit)",
a.p(id)
);
}
if let Some(g) = guid {
scope = format!("{scope} AND e.guid = {}", a.p(g));
}
let search = match search.map(|q| q.trim().to_lowercase()).filter(|q| !q.is_empty()) {
Some(q) => {
// % and _ as typed, not as wildcards: "100%" matched anything starting 100 (#296).
let q = q.replace('\\', "\\\\").replace('%', "\\%").replace('_', "\\_");
let like = a.p(format!("%{q}%"));
format!(
"(lower(coalesce(e.title, '')) LIKE {like} ESCAPE '\\' \
OR lower(coalesce(e.description, '')) LIKE {like} ESCAPE '\\')"
)
}
None => "true".into(),
};
// Unread starts from where this person's unread begins in each feed, so it reads the few
// items since, not every one: 11,727 read for 44 unread, 32 ms (#347). Every list is of
// subscribed feeds, so the join takes nothing away.
let since = if filter == Filter::Unread {
format!(
"JOIN subscriptions w ON w.user_id = {user} AND w.feed_id = e.feed_id
AND e.first_seen >= coalesce(w.unread_from, 0)"
)
} else {
String::new()
};
format!(
"FROM entries e
{since}
LEFT JOIN entry_state s ON s.user_id = {user} AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE {scope} AND {} AND {search} AND {}",
filter.sql(),
not_hidden(&user)
)
}
/// A subscription, `s`, not hidden from All Subscriptions or from Signals by `column`: neither on
/// the feed nor on the OPML or Patreon creator it is in (#288, #289).
fn shown_in(column: &str, s: &str) -> String {
format!(
"NOT {s}.{column} AND NOT EXISTS (SELECT 1 FROM feeds lf JOIN subscriptions lg
ON lg.user_id = {s}.user_id AND lg.feed_id = lf.group_id WHERE lf.id = {s}.feed_id AND lg.{column})"
)
}
/// Not hidden from this person by their block lists. `e` is entries.
fn not_hidden(user: &str) -> String {
format!(
"NOT EXISTS (SELECT 1 FROM hidden h WHERE h.user_id = {user} AND h.feed_id = e.feed_id AND h.guid = e.guid)"
)
}
/// The item table's ORDER BY. The column name picks one of these fixed expressions, so nothing
/// the caller sends reaches the query, and anything unrecognised is newest first. Ties fall back
/// to newest first too, so a page boundary is stable across "Load more".
///
/// ponytail: file type and size look at the item's first and largest file. The row shows the file
/// it summarises, which is almost always that one; sort by that one if they ever disagree.
///
/// Pinned items are not lifted above the rest: sorting by the pin column, or the Pinned tab, does
/// that when it is wanted, and lifting them always was undone (issues #35 and #36).
pub fn order_sql(col: &str, dir: &str) -> String {
let expr = match col {
"kept" => "coalesce(s.flagged, false)",
"title" => "lower(coalesce(e.title, ''))",
"feed" => "(SELECT lower(coalesce(f.title, f.id)) FROM feeds f WHERE f.id = e.feed_id)",
"type" => "(SELECT min(x.mime) FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid)",
"size" => "(SELECT max(x.length) FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid)",
_ => "coalesce(e.published, e.first_seen)",
};
// Where an item with no value goes, said outright: SQLite counts a NULL as the smallest value
// and Postgres as the largest, so "largest first" on Postgres opened with every item that has
// no file. NULLS FIRST going up and LAST going down keeps what SQLite did.
let dir = if dir == "asc" { "ASC NULLS FIRST" } else { "DESC NULLS LAST" };
// The guid breaks what ties remain: SQLite's rowid did, and Postgres has none.
format!("{expr} {dir}, coalesce(e.published, e.first_seen) DESC, e.guid DESC")
}
/// Which slice of a feed the UI is asking for.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Filter {
All,
Unread,
Downloaded,
Flagged,
/// Started (a saved playback position past the first few seconds) and short of the 90%
/// where `markPlayed` in the UI calls it finished. Not `read`: opening an item marks it
/// read, so filtering on that hid nearly every episode anyone had started. The length is the
/// one this person's player measured where there is one (`set_position`), else the feed's;
/// with neither, the episode counts as unfinished.
/// Currently Listening, below Popular, is this filter on every feed at once.
InProgress,
}
impl Filter {
pub fn parse(s: &str) -> Self {
match s {
"unread" => Self::Unread,
"downloaded" => Self::Downloaded,
"flagged" => Self::Flagged,
"in_progress" => Self::InProgress,
_ => Self::All,
}
}
/// The WHERE fragment for this filter. `e` is entries, and the EXISTS subquery is
/// correlated against it.
fn sql(self) -> &'static str {
match self {
// true and false, not 1 and 0: the columns are booleans on Postgres, and SQLite
// reads true and false as 1 and 0.
Self::All => "true",
// An item with no row of state is unread, so this one keeps its coalesce.
Self::Unread => "NOT coalesce(s.read, false)",
// Pinned and in progress need a row of state, said without coalesce so the database
// can tell, and starts from this person's few rows instead of every item: Currently
// Listening read all 574,000, 970 ms, where this takes 2 (#249).
Self::Flagged => "s.flagged",
Self::Downloaded => {
"EXISTS (SELECT 1 FROM enclosures x
WHERE x.feed_id = e.feed_id AND x.guid = e.guid AND x.path IS NOT NULL)"
}
Self::InProgress => {
"s.position > 5
AND (coalesce(s.duration, e.duration, 0) = 0
OR s.position * 10 < coalesce(s.duration, e.duration) * 9)"
}
}
}
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct EncRow {
pub id: i64,
/// Seconds DeadAir's copy of this file saves, when the person asking plays it from one
/// (`deadair::for_reader`). None from the database.
#[serde(skip_serializing_if = "Option::is_none")]
pub deadair: Option<i64>,
/// DeadAir is on for this person's show and the copy is still being made: the page says to
/// wait rather than leave them guessing (`deadair::for_reader`).
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub deadair_pending: bool,
pub feed_id: String,
pub guid: String,
pub url: String,
pub mime: Option<String>,
pub length: Option<i64>,
pub path: Option<String>,
pub state: String,
pub last_error: Option<String>,
}
impl Db {
/// One page of entries, each with its enclosures attached: one feed's, or every feed the
/// person subscribes to when `feed_id` is None (All Subscriptions). `search` matches title
/// and description, case-insensitively.
#[allow(clippy::too_many_arguments)]
pub async fn entries_in(
&self,
user_id: i64,
feed_id: Option<&str>,
filter: Filter,
search: Option<&str>,
offset: i64,
limit: i64,
order: &str,
) -> Result<Vec<EntryRow>> {
let feeds = feed_id.as_ref().map(std::slice::from_ref);
self.entries_where(user_id, feeds, filter, search, None, None, offset, limit, order).await
}
/// As `entries_in`, for several feeds together (an OPML's shows, #283), or every subscribed
/// feed when None.
#[allow(clippy::too_many_arguments)]
pub async fn entries_of(
&self,
user_id: i64,
feeds: Option<&[&str]>,
filter: Filter,
search: Option<&str>,
offset: i64,
limit: i64,
order: &str,
) -> Result<Vec<EntryRow>> {
self.entries_where(user_id, feeds, filter, search, None, None, offset, limit, order).await
}
/// The shows of an OPML (or a Patreon creator) this person subscribes to (#283).
pub async fn group_members(&self, user_id: i64, group: &str) -> Result<Vec<String>> {
self.rows(
"SELECT gs.feed_id FROM subscriptions gs JOIN feeds g ON g.id = gs.feed_id
WHERE gs.user_id = $1 AND g.group_id = $2",
vec![user_id.into(), group.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "feed_id")?))
.collect()
}
/// A Signal's page of entries, as `entries_in` gives All Subscriptions'. The caller has made
/// sure the Signal is this person's.
#[allow(clippy::too_many_arguments)]
pub async fn signal_entries(
&self,
user_id: i64,
signal: i64,
filter: Filter,
search: Option<&str>,
offset: i64,
limit: i64,
order: &str,
) -> Result<Vec<EntryRow>> {
self.entries_where(user_id, None, filter, search, None, Some(signal), offset, limit, order).await
}
pub async fn signal_count(&self, user_id: i64, signal: i64, filter: Filter, search: Option<&str>) -> Result<i64> {
let mut a = Args::default();
let sql =
format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, None, filter, search, None, Some(signal)));
let r = self.rows(&sql, a.0).await?;
Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?)
}
/// One entry as `entries_in` gives it: what the player asks for again when the clock it
/// plays in changes (DeadAir, #171).
pub async fn entry(&self, user_id: i64, feed_id: &str, guid: &str) -> Result<Option<EntryRow>> {
let order = order_sql("published", "desc");
Ok(self
.entries_where(user_id, Some(&[feed_id]), Filter::All, None, Some(guid), None, 0, 1, &order)
.await?
.pop())
}
#[allow(clippy::too_many_arguments)]
async fn entries_where(
&self,
user_id: i64,
feeds: Option<&[&str]>,
filter: Filter,
search: Option<&str>,
guid: Option<&str>,
signal: Option<i64>,
offset: i64,
limit: i64,
order: &str,
) -> Result<Vec<EntryRow>> {
let mut a = Args::default();
let from = entries_from(&mut a, user_id, feeds, filter, search, guid, signal);
// The page's keys first, then its rows: the sort then needs only `entries_by_time`, not
// every item's row, and the show notes are read for the page alone (#188).
let (limit, offset) = (a.p(limit), a.p(offset));
let user = a.p(user_id);
let sql = format!(
"SELECT e.guid, e.feed_id, e.title, e.link, e.published, e.description, e.markdown,
coalesce(s.read, false) AS read, coalesce(s.flagged, false) AS flagged, e.image,
coalesce(s.duration, e.duration) AS duration,
e.episode, e.season, coalesce(s.position, 0) AS position
FROM (SELECT e.feed_id, e.guid {from} ORDER BY {order} LIMIT {limit} OFFSET {offset}) p
JOIN entries e ON e.feed_id = p.feed_id AND e.guid = p.guid
LEFT JOIN entry_state s ON s.user_id = {user} AND s.feed_id = e.feed_id AND s.guid = e.guid
ORDER BY {order}"
);
let mut rows = tracing::Instrument::instrument(self.rows(&sql, a.0), step("page"))
.await?
.iter()
.map(|r| {
Ok(EntryRow {
guid: r.try_get("", "guid")?,
feed_id: r.try_get("", "feed_id")?,
title: r.try_get("", "title")?,
link: r.try_get("", "link")?,
published: r.try_get("", "published")?,
description: r.try_get("", "description")?,
markdown: r.try_get("", "markdown")?,
markdown_html: None,
read: r.try_get("", "read")?,
flagged: r.try_get("", "flagged")?,
image: r.try_get("", "image")?,
duration: r.try_get("", "duration")?,
episode: r.try_get("", "episode")?,
season: r.try_get("", "season")?,
position: r.try_get("", "position")?,
enclosures: vec![],
})
})
.collect::<Result<Vec<_>>>()?;
if rows.is_empty() {
return Ok(rows);
}
// Only the guids on this page, so a feed with thousands of entries stays cheap. A page
// can span feeds, so each file is matched to its row by feed as well as guid, below.
// The page's feeds too: the index is (feed_id, guid), and by guid alone Postgres read
// all of it, 587k rows and 250 ms for a page of 50 (#344).
let feeds: std::collections::BTreeSet<&str> = rows.iter().map(|r| r.feed_id.as_str()).collect();
let encs = tracing::Instrument::instrument(
enclosures::Entity::find()
.filter(enclosures::Column::FeedId.is_in(feeds.into_iter().map(str::to_owned)))
.filter(enclosures::Column::Guid.is_in(rows.iter().map(|r| r.guid.clone())))
.order_by_asc(enclosures::Column::Id)
.all(&self.orm),
step("enclosures"),
)
.await?;
for enc in encs {
if let Some(row) = rows.iter_mut().find(|r| r.guid == enc.guid && r.feed_id == enc.feed_id) {
row.enclosures.push(EncRow::from(enc));
}
}
Ok(rows)
}
/// How many entries `entries_in` would page through, so the UI knows whether there is more.
pub async fn count_in(
&self,
user_id: i64,
feed_id: Option<&str>,
filter: Filter,
search: Option<&str>,
) -> Result<i64> {
self.count_of(user_id, feed_id.as_ref().map(std::slice::from_ref), filter, search).await
}
/// As `count_in`, for several feeds together, or every subscribed feed when None.
pub async fn count_of(
&self,
user_id: i64,
feeds: Option<&[&str]>,
filter: Filter,
search: Option<&str>,
) -> Result<i64> {
let mut a = Args::default();
let sql = if filter == Filter::Unread && search.is_none() {
// Feed by feed, as the feed list counts (#352); the same feeds as `entries_from`.
let user = a.p(user_id);
let scope = match feeds {
Some([f]) => format!("sub.feed_id = {}", a.p(*f)),
Some([]) => "false".into(),
Some(fs) => format!("sub.feed_id IN ({})", fs.iter().map(|f| a.p(*f)).collect::<Vec<_>>().join(", ")),
None => shown_in("hide_from_all", "sub"),
};
format!(
"SELECT CAST(coalesce(sum(n), 0) AS BIGINT) AS n
FROM (SELECT ({}) AS n FROM subscriptions sub WHERE sub.user_id = {user} AND {scope}) x",
unread_in_sub!()
)
} else {
format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, feeds, filter, search, None, None))
};
let r = self.rows(&sql, a.0).await?;
Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?)
}
/// Where playback got to, so it resumes there next time -- for this listener only.
/// `duration` is the length their player measured, kept beside the position and preferred to
/// the feed's for time left and for when an episode counts as finished. A feed can be minutes
/// out: ReThinking's gave 41:23 for a 43:48 file, which said 0:08 left with 2:33 to play.
/// Not written to `entries`, where every scan puts the feed's figure back, and kept per
/// listener so one person's player never changes what anyone else sees.
pub async fn set_position(
&self,
user_id: i64,
feed_id: &str,
guid: &str,
secs: i64,
duration: Option<i64>,
) -> Result<()> {
// entry_state.duration, named in full: Postgres will not guess between it and excluded's.
self.exec(
"INSERT INTO entry_state (user_id, feed_id, guid, position, duration)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET position = excluded.position,
duration = coalesce(excluded.duration, entry_state.duration)",
vec![user_id.into(), feed_id.into(), guid.into(), secs.max(0).into(), duration.filter(|d| *d > 0).into()],
)
.await?;
Ok(())
}
// ---- the configuration: the catalogue and the server settings ----
/// The feeds and server settings, once the database holds them; None before, when they are
/// still config.toml's.
pub async fn stored_config(
&self,
) -> Result<Option<(crate::config::Stored, std::collections::BTreeMap<String, crate::config::Feed>)>> {
let Some(general) = settings::Entity::find_by_id("general".to_owned()).one(&self.orm).await? else {
return Ok(None);
};
let stored = serde_json::from_str(&general.value).context("reading the stored server settings")?;
let mut feeds = std::collections::BTreeMap::new();
for row in catalogue::Entity::find().all(&self.orm).await? {
let feed = serde_json::from_str(&row.spec).with_context(|| format!("reading feed {}", row.id))?;
feeds.insert(row.id, feed);
}
Ok(Some((stored, feeds)))
}
/// Takes the feeds and server settings from `cfg` (config.toml, as read) into a database that
/// has none. False when it already had them: another process got there first, and its copy
/// stands. One transaction, and the settings row goes in first, so two processes starting at
/// once cannot both import.
pub async fn import_config(&self, cfg: &crate::config::Config) -> Result<bool> {
use sea_orm::TransactionTrait;
let backend = self.orm.get_database_backend();
let tx = self.orm.begin().await?;
let claimed = tx
.execute_raw(Statement::from_sql_and_values(
backend,
"INSERT INTO settings (name, value) VALUES ('general', $1) ON CONFLICT DO NOTHING",
vec![serde_json::to_string(&crate::config::Stored::of(cfg))?.into()],
))
.await?
.rows_affected();
if claimed == 0 {
return Ok(false); // dropped, so rolled back
}
for (id, feed) in &cfg.feeds {
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }.insert(&tx).await?;
}
tx.commit().await?;
Ok(true)
}
/// Writes the feeds and server settings as they now stand: what config.toml's save did.
/// The whole catalogue at once, in one transaction, so a feed removed goes too.
pub async fn store_config(&self, cfg: &crate::config::Config) -> Result<()> {
use sea_orm::TransactionTrait;
let backend = self.orm.get_database_backend();
let tx = self.orm.begin().await?;
tx.execute_raw(Statement::from_sql_and_values(
backend,
"INSERT INTO settings (name, value) VALUES ('general', $1)
ON CONFLICT (name) DO UPDATE SET value = excluded.value",
vec![serde_json::to_string(&crate::config::Stored::of(cfg))?.into()],
))
.await?;
catalogue::Entity::delete_many().exec(&tx).await?;
for (id, feed) in &cfg.feeds {
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }.insert(&tx).await?;
}
tx.commit().await?;
Ok(())
}
// ---- hand-written SQL ----
//
// For what reads better as SQL than as a query builder: joins, sums, upserts. Written to
// run on SQLite and Postgres alike -- $1-style parameters, ON CONFLICT, CASE WHEN on a
// yes/no column rather than comparing it to 1 -- since both run it (issue #18).
fn stmt(&self, sql: &str, values: Vec<sea_orm::Value>) -> Statement {
Statement::from_sql_and_values(self.orm.get_database_backend(), sql, values)
}
/// Runs a statement; how many rows it changed.
async fn exec(&self, sql: &str, values: Vec<sea_orm::Value>) -> Result<u64> {
Ok(self.orm.execute_raw(self.stmt(sql, values)).await?.rows_affected())
}
async fn rows(&self, sql: &str, values: Vec<sea_orm::Value>) -> Result<Vec<sea_orm::QueryResult>> {
Ok(self.orm.query_all_raw(self.stmt(sql, values)).await?)
}
/// The first admin starts subscribed to the whole catalogue: whoever wrote config.toml meant
/// to read those feeds, and without this a fresh install signs in to an empty sidebar. Runs
/// only while nobody subscribes to anything, so an unsubscribe is never undone.
pub async fn adopt_catalogue(&self, user_id: i64, catalogue: &[String]) -> Result<usize> {
if subscriptions::Entity::find().count(&self.orm).await? > 0 {
return Ok(0);
}
for id in catalogue {
self.subscribe(user_id, id).await?;
}
// Feeds that exist only in the database (OPML children) count too. SQLite wants the
// WHERE to tell the SELECT from the ON CONFLICT; Postgres does not mind it.
self.exec(
"INSERT INTO subscriptions (user_id, feed_id) SELECT $1, id FROM feeds WHERE true
ON CONFLICT DO NOTHING",
vec![user_id.into()],
)
.await?;
Ok(catalogue.len())
}
// ---- subscriptions ----
/// What this person wants from a feed. Absent means they do not subscribe at all.
pub async fn subscription(&self, user_id: i64, feed_id: &str) -> Result<Option<Sub>> {
Ok(subscriptions::Entity::find_by_id((user_id, feed_id.to_owned())).one(&self.orm).await?.map(Sub::from))
}
/// Every feed this person subscribes to, with their settings.
pub async fn subscriptions_for(&self, user_id: i64) -> Result<Vec<Sub>> {
Ok(subscriptions::Entity::find()
.filter(subscriptions::Column::UserId.eq(user_id))
.all(&self.orm)
.await?
.into_iter()
.map(Sub::from)
.collect())
}
/// Everyone's settings for one feed. The scanner merges these into what it fetches
/// and downloads, since one file serves the lot.
/// In a group, whatever someone has not set on the feed itself comes from their
/// subscription to the group, as the group's settings dialog has always said it does.
#[tracing::instrument(skip_all)]
pub async fn subscribers(&self, feed_id: &str, group: Option<&str>) -> Result<Vec<Sub>> {
let rows = self
.rows(
"SELECT coalesce(c.keywords, p.keywords) AS keywords,
coalesce(c.auto_download, p.auto_download) AS auto_download,
coalesce(c.allow_explicit, p.allow_explicit) AS allow_explicit,
coalesce(c.max_new_per_check, p.max_new_per_check) AS max_new_per_check,
g.words AS global_blocked, b.words AS blocked
FROM subscriptions c
LEFT JOIN subscriptions p ON p.user_id = c.user_id AND p.feed_id = $2
LEFT JOIN blocklists g ON g.user_id = c.user_id AND g.feed_id = ''
LEFT JOIN blocklists b ON b.user_id = c.user_id AND b.feed_id = c.feed_id
WHERE c.feed_id = $1",
vec![feed_id.into(), group.map(str::to_owned).into()],
)
.await?;
rows.iter()
.map(|r| {
Ok(Sub {
feed_id: feed_id.to_owned(),
keywords: keywords(r.try_get("", "keywords")?),
auto_download: r.try_get("", "auto_download")?,
allow_explicit: r.try_get("", "allow_explicit")?,
max_new_per_check: r.try_get("", "max_new_per_check")?,
blocked: [r.try_get("", "global_blocked")?, r.try_get("", "blocked")?]
.into_iter()
.flat_map(|w| keywords(w).unwrap_or_default())
.collect(),
})
})
.collect()
}
/// Subscribers per feed, for the whole catalogue in one query -- the feed list would
/// otherwise ask once per feed.
pub async fn subscriber_counts(&self) -> Result<std::collections::HashMap<String, i64>> {
self.rows("SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id", vec![])
.await?
.iter()
.map(|r| Ok((r.try_get("", "feed_id")?, r.try_get("", "n")?)))
.collect()
}
/// Feeds with any audio or video enclosure: the Directory's Podcasts, with the rest Blogs.
/// Reaped files keep their rows, so a show whose files have all been purged still counts.
pub async fn media_feeds(&self) -> Result<std::collections::HashSet<String>> {
self.rows("SELECT DISTINCT feed_id FROM enclosures WHERE mime LIKE 'audio/%' OR mime LIKE 'video/%'", vec![])
.await?
.iter()
.map(|r| Ok(r.try_get("", "feed_id")?))
.collect()
}
/// Who else would miss this file: subscribers other than `user_id` who have starred
/// the item or have not read it yet. Deleting is deleting their copy too.
pub async fn others_wanting(&self, enclosure_id: i64, user_id: i64) -> Result<(i64, i64)> {
// CASE WHEN on the column itself, not `= 1`: a boolean on Postgres, 0 or 1 on SQLite,
// and NULL, for someone who never opened the item, falls to the ELSE either way.
let r = self
.rows(
"SELECT sum(CASE WHEN st.flagged THEN 1 ELSE 0 END) AS starred,
sum(CASE WHEN st.read OR h.guid IS NOT NULL THEN 0 ELSE 1 END) AS unread
FROM enclosures e
JOIN subscriptions s ON s.feed_id = e.feed_id AND s.user_id <> $2
LEFT JOIN entry_state st
ON st.user_id = s.user_id AND st.feed_id = e.feed_id AND st.guid = e.guid
LEFT JOIN hidden h
ON h.user_id = s.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid
WHERE e.id = $1",
vec![enclosure_id.into(), user_id.into()],
)
.await?;
let r = r.first().context("an aggregate always returns a row")?;
Ok((
r.try_get::<Option<i64>>("", "starred")?.unwrap_or(0),
r.try_get::<Option<i64>>("", "unread")?.unwrap_or(0),
))
}
pub async fn subscribe(&self, user_id: i64, feed_id: &str) -> Result<()> {
let new = self
.exec(
"INSERT INTO subscriptions (user_id, feed_id) VALUES ($1, $2) ON CONFLICT DO NOTHING",
vec![user_id.into(), feed_id.into()],
)
.await?;
if new > 0 {
// Their list for every feed applies to this one from the start.
self.rehide(feed_id).await?;
}
Ok(())
}
/// Someone's block list: for one feed, or with `feed_id` empty, for every feed.
pub async fn blocklist(&self, user_id: i64, feed_id: &str) -> Result<Vec<String>> {
Ok(blocklists::Entity::find_by_id((user_id, feed_id.to_owned()))
.one(&self.orm)
.await?
.and_then(|b| keywords(Some(b.words)))
.unwrap_or_default())
}
/// Replaces a block list and works out again what it hides: in that feed, or with
/// `feed_id` empty, in every feed the person reads.
pub async fn set_blocklist(&self, user_id: i64, feed_id: &str, words: &[String]) -> Result<()> {
if words.is_empty() {
blocklists::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?;
} else {
self.exec(
"INSERT INTO blocklists (user_id, feed_id, words) VALUES ($1, $2, $3)
ON CONFLICT (user_id, feed_id) DO UPDATE SET words = excluded.words",
vec![user_id.into(), feed_id.into(), serde_json::to_string(words)?.into()],
)
.await?;
}
if feed_id.is_empty() {
for sub in self.subscriptions_for(user_id).await? {
self.rehide(&sub.feed_id).await?;
}
} else {
self.rehide(feed_id).await?;
}
Ok(())
}
/// Works out again which of a feed's items each subscriber's block lists hide from them.
///
/// ponytail: every item of the feed for every subscriber, each time the feed is scanned with
/// something new or a list changes. Fine at hundreds of items a feed; look only at new items
/// on a scan if a feed with thousands makes scans slow.
#[tracing::instrument(skip_all)]
pub async fn rehide(&self, feed_id: &str) -> Result<()> {
let lists: Vec<(i64, Vec<String>)> = self
.rows(
"SELECT s.user_id, g.words AS global, b.words AS feed
FROM subscriptions s
LEFT JOIN blocklists g ON g.user_id = s.user_id AND g.feed_id = ''
LEFT JOIN blocklists b ON b.user_id = s.user_id AND b.feed_id = s.feed_id
WHERE s.feed_id = $1",
vec![feed_id.into()],
)
.await?
.iter()
.map(|r| {
let words = [r.try_get("", "global")?, r.try_get("", "feed")?]
.into_iter()
.flat_map(|w: Option<String>| keywords(w).unwrap_or_default())
.collect();
Ok((r.try_get("", "user_id")?, words))
})
.collect::<Result<_>>()?;
hidden::Entity::delete_many().filter(hidden::Column::FeedId.eq(feed_id)).exec(&self.orm).await?;
let lists: Vec<_> = lists.into_iter().filter(|(_, w)| !w.is_empty()).collect();
if lists.is_empty() {
return Ok(());
}
let items = entries::Entity::find().filter(entries::Column::FeedId.eq(feed_id)).all(&self.orm).await?;
let rows: Vec<hidden::ActiveModel> = items
.iter()
.flat_map(|e| {
let hay = [e.title.as_deref().unwrap_or(""), e.description.as_deref().unwrap_or("")];
lists.iter().filter(move |(_, words)| crate::download::blocked(words, &hay)).map(|(user, _)| {
hidden::ActiveModel {
user_id: Set(*user),
feed_id: Set(feed_id.to_owned()),
guid: Set(e.guid.clone()),
}
})
})
.collect();
if !rows.is_empty() {
hidden::Entity::insert_many(rows).exec_without_returning(&self.orm).await?;
}
Ok(())
}
/// The feeds this person pinned to the top of their list. Kept apart from `Sub`, which is
/// what the scanner merges into its policy, and which a pin has nothing to do with.
pub async fn pinned_feeds(&self, user_id: i64) -> Result<std::collections::HashSet<String>> {
Ok(subscriptions::Entity::find()
.filter(subscriptions::Column::UserId.eq(user_id))
.filter(subscriptions::Column::Pinned.eq(true))
.all(&self.orm)
.await?
.into_iter()
.map(|s| s.feed_id)
.collect())
}
/// The feeds from a list this person keeps should the list drop them (#280).
pub async fn kept_feeds(&self, user_id: i64) -> Result<std::collections::HashSet<String>> {
Ok(subscriptions::Entity::find()
.filter(subscriptions::Column::UserId.eq(user_id))
.filter(subscriptions::Column::Kept.eq(true))
.all(&self.orm)
.await?
.into_iter()
.map(|s| s.feed_id)
.collect())
}
/// False when they do not subscribe to it, as `set_pinned`.
pub async fn set_kept(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
let r = subscriptions::Entity::update_many()
.col_expr(subscriptions::Column::Kept, Expr::val(on))
.filter(subscriptions::Column::UserId.eq(user_id))
.filter(subscriptions::Column::FeedId.eq(feed_id))
.exec(&self.orm)
.await?;
Ok(r.rows_affected > 0)
}
/// The feeds and lists this person hid from All Subscriptions (#288) and from Signals (#289).
pub async fn hidden_from(
&self,
user_id: i64,
) -> Result<(std::collections::HashSet<String>, std::collections::HashSet<String>)> {
let (mut all, mut signals) = (std::collections::HashSet::new(), std::collections::HashSet::new());
let rows =
subscriptions::Entity::find().filter(subscriptions::Column::UserId.eq(user_id)).all(&self.orm).await?;
for s in rows {
if s.hide_from_all {
all.insert(s.feed_id.clone());
}
if s.hide_from_signals {
signals.insert(s.feed_id);
}
}
Ok((all, signals))
}
/// False when they do not subscribe to it, as `set_pinned`.
pub async fn set_hide_from_all(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
self.set_hidden_from(subscriptions::Column::HideFromAll, user_id, feed_id, on).await
}
/// As `set_hide_from_all`, for Signals (#289).
pub async fn set_hide_from_signals(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
self.set_hidden_from(subscriptions::Column::HideFromSignals, user_id, feed_id, on).await
}
async fn set_hidden_from(
&self,
column: subscriptions::Column,
user_id: i64,
feed_id: &str,
on: bool,
) -> Result<bool> {
let r = subscriptions::Entity::update_many()
.col_expr(column, Expr::val(on))
.filter(subscriptions::Column::UserId.eq(user_id))
.filter(subscriptions::Column::FeedId.eq(feed_id))
.exec(&self.orm)
.await?;
Ok(r.rows_affected > 0)
}
/// Who subscribes to a feed, and whether each keeps it (#280).
pub async fn subscribers_of(&self, feed_id: &str) -> Result<Vec<(i64, bool)>> {
Ok(subscriptions::Entity::find()
.filter(subscriptions::Column::FeedId.eq(feed_id))
.all(&self.orm)
.await?
.into_iter()
.map(|s| (s.user_id, s.kept))
.collect())
}
/// False when they do not subscribe to it, since there is then no row in their list to pin.
/// Whether a feed or an entry names `url` as its artwork: the only addresses /api/art will
/// fetch, so the web server cannot be pointed at anything else.
// ponytail: entries.image is unindexed, a scan per http:// image; index it if that shows up.
pub async fn names_image(&self, url: &str) -> Result<bool> {
Ok(feeds::Entity::find().filter(feeds::Column::Image.eq(url)).count(&self.orm).await? > 0
|| entries::Entity::find().filter(entries::Column::Image.eq(url)).count(&self.orm).await? > 0)
}
pub async fn set_pinned(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
let r = subscriptions::Entity::update_many()
.col_expr(subscriptions::Column::Pinned, Expr::val(on))
.filter(subscriptions::Column::UserId.eq(user_id))
.filter(subscriptions::Column::FeedId.eq(feed_id))
.exec(&self.orm)
.await?;
Ok(r.rows_affected > 0)
}
pub async fn unsubscribe(&self, user_id: i64, feed_id: &str) -> Result<()> {
subscriptions::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?;
Ok(())
}
/// Overwrites one person's settings for a feed. A None field means: follow the feed.
/// Names its columns, so the pin, which is not a setting, is left as it was.
pub async fn set_subscription(&self, user_id: i64, sub: &Sub) -> Result<()> {
let kw = sub.keywords.as_ref().map(serde_json::to_string).transpose()?;
self.exec(
"INSERT INTO subscriptions
(user_id, feed_id, keywords, auto_download, allow_explicit, max_new_per_check)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (user_id, feed_id) DO UPDATE SET
keywords = excluded.keywords,
auto_download = excluded.auto_download,
allow_explicit = excluded.allow_explicit,
max_new_per_check = excluded.max_new_per_check",
vec![
user_id.into(),
sub.feed_id.clone().into(),
kw.into(),
sub.auto_download.into(),
sub.allow_explicit.into(),
sub.max_new_per_check.into(),
],
)
.await?;
Ok(())
}
// ---- users and sessions ----
pub async fn create_user(&self, name: &str, pass_hash: Option<&str>, admin: bool) -> Result<i64> {
let m = users::ActiveModel {
name: Set(name.to_owned()),
pass_hash: Set(pass_hash.map(str::to_owned)),
is_admin: Set(admin),
created: Set(Some(now())),
..Default::default()
}
.insert(&self.orm)
.await?;
Ok(m.id)
}
/// Without regard to case, on either database: lower() both sides, which the unique index
/// on lower(name) serves. SQLite's COLLATE NOCASE on the column did this before, and
/// Postgres has no such thing.
pub async fn user_by_name(&self, name: &str) -> Result<Option<User>> {
use sea_orm::sea_query::ExprTrait;
Ok(users::Entity::find()
.filter(Expr::expr(Func::lower(Expr::col(users::Column::Name))).eq(Func::lower(name)))
.one(&self.orm)
.await?
.map(User::from))
}
pub async fn user_by_id(&self, id: i64) -> Result<Option<User>> {
Ok(users::Entity::find_by_id(id).one(&self.orm).await?.map(User::from))
}
pub async fn users(&self) -> Result<Vec<User>> {
Ok(users::Entity::find()
.order_by_asc(users::Column::Name)
.all(&self.orm)
.await?
.into_iter()
.map(User::from)
.collect())
}
/// Sets some of one person's columns, whatever else is in their row.
async fn update_user(&self, id: i64, m: users::ActiveModel) -> Result<()> {
users::Entity::update_many().set(m).filter(users::Column::Id.eq(id)).exec(&self.orm).await?;
Ok(())
}
/// Signs the account out everywhere, too: a password is changed because someone else may
/// know it, and a session they signed in with stayed good, every use putting off its expiry
/// (#334). API tokens stay; they are listed and removed in Settings.
pub async fn set_password(&self, id: i64, hash: &str) -> Result<()> {
self.update_user(id, users::ActiveModel { pass_hash: Set(Some(hash.to_owned())), ..Default::default() })
.await?;
sessions::Entity::delete_many().filter(sessions::Column::UserId.eq(id)).exec(&self.orm).await?;
Ok(())
}
pub async fn set_admin(&self, id: i64, admin: bool) -> Result<()> {
self.update_user(id, users::ActiveModel { is_admin: Set(admin), ..Default::default() }).await
}
/// The proxy signs people in by the name it vouches for, so an account made before the proxy
/// was set up has to take that name to be found by it. The name is unique, so a taken one is
/// refused here as well as by the caller.
pub async fn rename_user(&self, id: i64, name: &str) -> Result<()> {
self.update_user(id, users::ActiveModel { name: Set(name.to_owned()), ..Default::default() }).await
}
/// Records a sign-in, to the hour: the proxy vouches for every request, and writing each one
/// would buy nothing.
pub async fn signed_in(&self, id: i64) -> Result<()> {
// Here only: it is implemented for every type, and file-wide it shadows i64::max.
use sea_orm::sea_query::ExprTrait;
let now = now();
users::Entity::update_many()
.col_expr(users::Column::LastLogin, Expr::val(now))
.filter(users::Column::Id.eq(id))
.filter(Expr::expr(Func::coalesce([Expr::col(users::Column::LastLogin), Expr::val(0)])).lte(now - 3600))
.exec(&self.orm)
.await?;
Ok(())
}
/// Sessions go with the user: a deleted account must not leave a usable cookie behind.
/// The foreign key would take them anyway; this does not rely on it being switched on.
pub async fn delete_user(&self, id: i64) -> Result<()> {
sessions::Entity::delete_many().filter(sessions::Column::UserId.eq(id)).exec(&self.orm).await?;
api_tokens::Entity::delete_many().filter(api_tokens::Column::UserId.eq(id)).exec(&self.orm).await?;
users::Entity::delete_by_id(id).exec(&self.orm).await?;
Ok(())
}
pub async fn create_session(&self, user_id: i64, token: &str) -> Result<()> {
sessions::ActiveModel { token: Set(token.to_owned()), user_id: Set(user_id), seen: Set(now()) }
.insert(&self.orm)
.await?;
Ok(())
}
/// The theme this person chose when it was kept on the account, and light, dark or auto; None
/// for either if they never did. Only read now, to give a browser with no theme cookie its
/// first one (issue #69).
// ponytail: users.theme and theme_mode are never written any more; drop them once every
// browser in use has its own cookie.
pub async fn theme(&self, user_id: i64) -> Result<(Option<String>, Option<String>)> {
Ok(users::Entity::find_by_id(user_id)
.one(&self.orm)
.await?
.map(|u| (u.theme, u.theme_mode))
.unwrap_or_default())
}
/// The user behind a session cookie, if it is still live. Idle sessions expire after
/// `max_idle_secs`; touching `seen` is what keeps a session in daily use alive.
pub async fn session_user(&self, token: &str, max_idle_secs: i64) -> Result<Option<User>> {
// Here only: it is implemented for every type, and file-wide it shadows i64::max.
use sea_orm::sea_query::ExprTrait;
let cutoff = now() - max_idle_secs;
let found = sessions::Entity::find_by_id(token.to_owned())
.filter(sessions::Column::Seen.gte(cutoff))
.find_also_related(users::Entity)
.one(&self.orm)
.await?
.and_then(|(_, u)| u);
if found.is_some() {
sessions::Entity::update_many()
.col_expr(sessions::Column::Seen, Expr::val(now()))
.filter(sessions::Column::Token.eq(token))
.exec(&self.orm)
.await?;
} else {
// Either unknown or timed out; either way it is dead weight.
sessions::Entity::delete_many()
.filter(sessions::Column::Token.eq(token).or(sessions::Column::Seen.lt(cutoff)))
.exec(&self.orm)
.await?;
}
Ok(found.map(User::from))
}
/// Keeps a new API token's hash under its owner; the token itself is shown once and not kept.
pub async fn create_api_token(&self, user_id: i64, name: &str, hash: &str) -> Result<i64> {
let made = api_tokens::ActiveModel {
user_id: Set(user_id),
name: Set(name.to_owned()),
hash: Set(hash.to_owned()),
created: Set(now()),
last_used: Set(None),
..Default::default()
}
.insert(&self.orm)
.await?;
Ok(made.id)
}
/// One person's tokens, newest first: names and dates, as the tokens themselves are not kept.
pub async fn api_tokens(&self, user_id: i64) -> Result<Vec<api_tokens::Model>> {
Ok(api_tokens::Entity::find()
.filter(api_tokens::Column::UserId.eq(user_id))
.order_by_desc(api_tokens::Column::Id)
.all(&self.orm)
.await?)
}
/// Revokes one of this person's tokens; someone else's id does nothing. Whether it was theirs.
pub async fn delete_api_token(&self, user_id: i64, id: i64) -> Result<bool> {
let gone = api_tokens::Entity::delete_many()
.filter(api_tokens::Column::Id.eq(id))
.filter(api_tokens::Column::UserId.eq(user_id))
.exec(&self.orm)
.await?;
Ok(gone.rows_affected > 0)
}
/// The person an API token acts as, noting when it was used, as a session notes `seen`.
pub async fn api_token_user(&self, hash: &str) -> Result<Option<User>> {
let found = api_tokens::Entity::find()
.filter(api_tokens::Column::Hash.eq(hash))
.find_also_related(users::Entity)
.one(&self.orm)
.await?;
let Some((token, Some(user))) = found else { return Ok(None) };
api_tokens::Entity::update_many()
.col_expr(api_tokens::Column::LastUsed, Expr::val(now()))
.filter(api_tokens::Column::Id.eq(token.id))
.exec(&self.orm)
.await?;
Ok(Some(User::from(user)))
}
pub async fn delete_session(&self, token: &str) -> Result<()> {
sessions::Entity::delete_by_id(token.to_owned()).exec(&self.orm).await?;
Ok(())
}
/// Marks every entry of the given feeds read. Takes a list because an OPML subscription
/// holds no entries itself -- marking it read means the feeds inside it.
pub async fn mark_all_read(&self, user_id: i64, feed_ids: &[String]) -> Result<usize> {
let mut n = 0;
for id in feed_ids {
n += self
.exec(
"INSERT INTO entry_state (user_id, feed_id, guid, read)
SELECT $1, e.feed_id, e.guid, true FROM entries e
LEFT JOIN entry_state s
ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = $2 AND NOT coalesce(s.read, false)
ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET read = true",
vec![user_id.into(), id.clone().into()],
)
.await? as usize;
}
Ok(n)
}
/// The next N enclosures with no file, newest entry first -- what "download latest"
/// queues up.
pub async fn undownloaded(&self, feed_id: &str, limit: i64) -> Result<Vec<i64>> {
self.rows(
"SELECT x.id FROM enclosures x
JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid
WHERE x.feed_id = $1 AND x.path IS NULL AND x.state <> 'reaped'
ORDER BY coalesce(e.published, e.first_seen) DESC
LIMIT $2",
vec![feed_id.into(), limit.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "id")?))
.collect()
}
pub async fn enclosure(&self, id: i64) -> Result<Option<EncRow>> {
Ok(enclosures::Entity::find_by_id(id).one(&self.orm).await?.map(EncRow::from))
}
/// Read and kept, per person. The row is created on first touch.
pub async fn set_entry_flag(
&self,
user_id: i64,
feed_id: &str,
guid: &str,
field: EntryFlag,
on: bool,
) -> Result<()> {
let col = match field {
EntryFlag::Read => "read",
EntryFlag::Flagged => "flagged",
};
self.exec(
&format!(
"INSERT INTO entry_state (user_id, feed_id, guid, {col})
VALUES ($1, $2, $3, $4)
ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET {col} = excluded.{col}"
),
vec![user_id.into(), feed_id.into(), guid.into(), on.into()],
)
.await?;
// An item marked unread again may be older than where this person's unread was known to
// start, so that is forgotten until the next reap works it out (`advance_unread_from`).
if matches!(field, EntryFlag::Read) && !on {
self.exec(
"UPDATE subscriptions SET unread_from = NULL
WHERE user_id = $1 AND feed_id = $2 AND unread_from IS NOT NULL",
vec![user_id.into(), feed_id.into()],
)
.await?;
}
Ok(())
}
/// Moves each subscription's `unread_from` up to its oldest unread item, hidden ones included,
/// since a block list can be lifted. No more than an hour ago: an item a scan is storing now,
/// not yet visible to this statement, is first seen about now. Returns how many moved.
pub async fn advance_unread_from(&self) -> Result<u64> {
let cutoff = now() - 3600;
self.exec(
"UPDATE subscriptions SET unread_from = (
SELECT CASE WHEN min(e.first_seen) < $1 THEN min(e.first_seen) ELSE $1 END
FROM entries e
LEFT JOIN entry_state s
ON s.user_id = subscriptions.user_id AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = subscriptions.feed_id AND NOT coalesce(s.read, false))
WHERE true",
vec![cutoff.into()],
)
.await
}
/// How many files this feed has on disk. Decides whether a feed dropped from an OPML
/// can be removed or must be kept.
pub async fn downloaded_count(&self, feed_id: &str) -> Result<i64> {
Ok(enclosures::Entity::find()
.filter(enclosures::Column::FeedId.eq(feed_id))
.filter(enclosures::Column::Path.is_not_null())
.count(&self.orm)
.await? as i64)
}
/// Names a feed without touching its conditional-GET validators. An OPML subscription
/// takes its name from the document's own <head><title>.
pub async fn set_title(&self, feed_id: &str, title: &str) -> Result<()> {
self.exec(
"UPDATE feeds SET title = $2 WHERE id = $1 AND coalesce(title, '') <> $2",
vec![feed_id.into(), title.into()],
)
.await?;
Ok(())
}
/// Records a feed that came from an OPML. Its settings are the parent's; only what
/// identifies it is stored.
pub async fn upsert_managed(&self, id: &str, url: &str, title: &str, group_id: &str) -> Result<()> {
self.exec(
"INSERT INTO feeds (id, url, title, group_id, managed, orphaned)
VALUES ($1, $2, $3, $4, true, false)
ON CONFLICT (id) DO UPDATE SET
url = excluded.url,
title = coalesce(feeds.title, excluded.title),
group_id = excluded.group_id,
managed = true,
orphaned = false",
vec![id.into(), url.into(), title.into(), group_id.into()],
)
.await?;
Ok(())
}
/// Every feed derived from an OPML, whichever group.
pub async fn managed_feeds(&self) -> Result<Vec<Managed>> {
self.rows(
"SELECT id, url, title, group_id FROM feeds
WHERE managed AND group_id IS NOT NULL ORDER BY coalesce(title, id)",
vec![],
)
.await?
.iter()
.map(|r| {
Ok(Managed {
id: r.try_get("", "id")?,
url: r.try_get("", "url")?,
title: r.try_get("", "title")?,
group_id: r.try_get("", "group_id")?,
})
})
.collect()
}
/// Forgets a derived feed entirely. Only for one with nothing downloaded.
pub async fn drop_managed(&self, id: &str) -> Result<()> {
self.exec("DELETE FROM feeds WHERE id = $1 AND managed", vec![id.into()]).await?;
self.exec("DELETE FROM entries WHERE feed_id = $1", vec![id.into()]).await?;
self.exec("DELETE FROM enclosures WHERE feed_id = $1 AND path IS NULL", vec![id.into()]).await?;
Ok(())
}
/// Stops treating a feed as derived, because it now has its own config entry.
pub async fn unmanage(&self, id: &str) -> Result<()> {
self.exec("UPDATE feeds SET managed = false WHERE id = $1", vec![id.into()]).await?;
Ok(())
}
/// Empties a feed of its items, leaving its files alone.
pub async fn clear_entries(&self, feed_id: &str) -> Result<()> {
entries::Entity::delete_many().filter(entries::Column::FeedId.eq(feed_id)).exec(&self.orm).await?;
Ok(())
}
/// Every feed the database holds rows for, as (id, url), removed ones included.
pub async fn feed_urls(&self) -> Result<Vec<(String, String)>> {
Ok(feeds::Entity::find().all(&self.orm).await?.into_iter().map(|f| (f.id, f.url)).collect())
}
/// Hands a feed in a group the enclosures its parent holds, as (guid, url), with everyone's
/// read state for them. A Patreon creator read as one feed before it was split into shows
/// owns every show's files, and `enclosures.url` is unique, so without this each show would
/// list its items with nothing to play.
#[tracing::instrument(skip_all)]
pub async fn adopt(&self, parent: &str, child: &str, listed: &[(&str, &str)]) -> Result<()> {
use sea_orm::TransactionTrait;
let holds =
enclosures::Entity::find().filter(enclosures::Column::FeedId.eq(parent)).count(&self.orm).await? > 0;
if !holds {
return Ok(()); // An OPML, or a creator already shared out.
}
let backend = self.orm.get_database_backend();
let tx = self.orm.begin().await?;
for &(guid, url) in listed {
let moved = tx
.execute_raw(Statement::from_sql_and_values(
backend,
"UPDATE enclosures SET feed_id = $3, guid = $4 WHERE url = $1 AND feed_id = $2",
vec![url.into(), parent.into(), child.into(), guid.into()],
))
.await?
.rows_affected();
if moved == 1 {
// Patreon gives a post the same guid in every feed it appears in. Someone who
// already has a row for it under the child keeps that one; SQLite's UPDATE OR
// IGNORE did this, and Postgres has no such thing.
tx.execute_raw(Statement::from_sql_and_values(
backend,
"UPDATE entry_state SET feed_id = $2 WHERE feed_id = $1 AND guid = $3
AND NOT EXISTS (SELECT 1 FROM entry_state t
WHERE t.user_id = entry_state.user_id
AND t.feed_id = $2 AND t.guid = $3)",
vec![parent.into(), child.into(), guid.into()],
))
.await?;
}
}
tx.commit().await?;
Ok(())
}
/// A feed's enclosures skipped by one of its filters, by URL, with the reason: the verdicts a
/// change of settings can overturn. A torrent held back while torrents are off is not a
/// filter's call.
/// The guids of a feed's stored items and the URLs of its files: a scan inserts only what is
/// not among them. Inserting every item it read, stored or not, cost a round trip each, about
/// 10 ms: 13 s a scan for Clarkesworld's 1200 items, and 117 s of a full 315 s scan (#96).
#[tracing::instrument(skip_all)]
pub async fn stored_items(
&self,
feed_id: &str,
) -> Result<(std::collections::HashSet<String>, std::collections::HashSet<String>)> {
let col = |sql: &'static str, name: &'static str| async move {
self.rows(sql, vec![feed_id.into()])
.await?
.iter()
.map(|r| Ok(r.try_get::<String>("", name)?))
.collect::<Result<std::collections::HashSet<_>>>()
};
Ok((
col("SELECT guid FROM entries WHERE feed_id = $1", "guid").await?,
col("SELECT url FROM enclosures WHERE feed_id = $1", "url").await?,
))
}
/// Of these items, the stored ones the feed now tells differently: a title, text, artwork,
/// length or number that is not what is stored, a field the feed leaves out counting as
/// unchanged, as record_entry writes them. A scan inserted only items it had not stored
/// (#96), and with that alone a publisher's correction never arrived: Ain't It Cool News's
/// items kept a broken copy, each with the text of the one before it, after the feed was
/// fixed (#141). Only what the feed lists is read back, so a scan reads as much as it was
/// sent, not the archive.
#[tracing::instrument(skip_all)]
pub async fn changed_items(
&self,
feed_id: &str,
entries: &[crate::feed::Entry],
) -> Result<std::collections::HashSet<String>> {
if entries.is_empty() {
return Ok(Default::default());
}
let mut a = Args::default();
let feed = a.p(feed_id);
let guids: Vec<String> = entries.iter().map(|e| a.p(e.guid.clone())).collect();
let sql = format!(
"SELECT guid, title, description, image, duration, episode, season, markdown FROM entries
WHERE feed_id = {feed} AND guid IN ({})",
guids.join(", ")
);
type Stored =
(Option<String>, Option<String>, Option<String>, Option<i64>, Option<i64>, Option<i64>, Option<String>);
let mut stored = std::collections::HashMap::<String, Stored>::new();
for r in self.rows(&sql, a.0).await? {
stored.insert(
r.try_get("", "guid")?,
(
r.try_get("", "title")?,
r.try_get("", "description")?,
r.try_get("", "image")?,
r.try_get("", "duration")?,
r.try_get("", "episode")?,
r.try_get("", "season")?,
r.try_get("", "markdown")?,
),
);
}
fn differs<T: PartialEq>(new: &Option<T>, old: &Option<T>) -> bool {
new.is_some() && new != old
}
Ok(entries
.iter()
.filter(|e| {
stored.get(&e.guid).is_some_and(|s| {
differs(&e.title, &s.0)
|| differs(&e.description, &s.1)
|| differs(&e.image, &s.2)
|| differs(&e.duration, &s.3)
|| differs(&e.episode, &s.4)
|| differs(&e.season, &s.5)
|| differs(&e.markdown, &s.6)
})
})
.map(|e| e.guid.clone())
.collect())
}
#[tracing::instrument(skip_all)]
pub async fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> {
self.rows(
"SELECT url, last_error FROM enclosures
WHERE feed_id = $1 AND state = 'skipped' AND last_error IS NOT NULL
AND last_error <> 'torrents disabled'",
vec![feed_id.into()],
)
.await?
.iter()
.map(|r| Ok((r.try_get("", "url")?, r.try_get("", "last_error")?)))
.collect()
}
pub async fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> {
self.exec(
"INSERT INTO feeds (id, url, orphaned) VALUES ($1, '', $2)
ON CONFLICT (id) DO UPDATE SET orphaned = excluded.orphaned",
vec![feed_id.into(), on.into()],
)
.await?;
Ok(())
}
/// Nothing can be in flight the moment the daemon starts, so any row still marked
/// `downloading` is a leftover from a restart or a crash. Left alone it would sit
/// there forever: the pending queue skips it and nothing else ever revisits it.
pub async fn requeue_interrupted(&self) -> Result<usize> {
Ok(self
.exec("UPDATE enclosures SET state = 'pending' WHERE state = 'downloading' AND path IS NULL", vec![])
.await? as usize)
}
/// Puts an enclosure back in the queue so the next scan picks it up. This is how a
/// `skipped` verdict (from a filter that has since been changed) gets revisited.
pub async fn requeue(&self, id: i64) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = 'pending', last_error = NULL WHERE id = $1 AND path IS NULL",
vec![id.into()],
)
.await?;
Ok(())
}
}
#[derive(Debug, Clone, Copy)]
pub enum EntryFlag {
Read,
Flagged,
}
/// A span for one step of a request or a scan, and none outside one: the healthcheck and the page's
/// live updates run the same code, and every call there would be a trace of its own.
pub fn step(of: &'static str) -> tracing::Span {
if tracing::Span::current().is_none() { tracing::Span::none() } else { tracing::info_span!("step", of) }
}
/// A file on disk with no DeadAir copy made since it was downloaded (issue #171).
#[derive(Debug, Clone)]
pub struct DeadAirTodo {
pub id: i64,
pub feed_id: String,
pub guid: String,
pub path: String,
}
impl Db {
/// DeadAir settings as (person, feed, wanted, pause in ms), everyone's or one person's. An
/// empty feed is the switch beside the speed.
pub async fn deadair_prefs(&self, user_id: Option<i64>) -> Result<Vec<(i64, String, bool, Option<i64>)>> {
let mut a = Args::default();
let only = match user_id {
Some(u) => format!("WHERE user_id = {}", a.p(u)),
None => String::new(),
};
self.rows(&format!("SELECT user_id, feed_id, wanted, pause_ms FROM deadair_prefs {only}"), a.0)
.await?
.iter()
.map(|r| {
Ok((
r.try_get("", "user_id")?,
r.try_get("", "feed_id")?,
r.try_get("", "wanted")?,
r.try_get("", "pause_ms")?,
))
})
.collect()
}
/// Sets one DeadAir setting, or with None takes it away: for a feed, that is following the
/// switch (or the feed's group) again.
pub async fn set_deadair(&self, user_id: i64, feed_id: &str, wanted: Option<bool>) -> Result<()> {
match wanted {
None => {
deadair_prefs::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?;
}
Some(w) => {
self.exec(
"INSERT INTO deadair_prefs (user_id, feed_id, wanted) VALUES ($1, $2, $3)
ON CONFLICT (user_id, feed_id) DO UPDATE SET wanted = excluded.wanted",
vec![user_id.into(), feed_id.into(), w.into()],
)
.await?;
}
}
Ok(())
}
/// Sets the pause DeadAir leaves in a feed for someone (#182). Only beside a setting of the
/// switch: inserting one here would turn a feed's group's DeadAir off for it.
pub async fn set_deadair_pause(&self, user_id: i64, feed_id: &str, pause_ms: i64) -> Result<()> {
self.exec(
"UPDATE deadair_prefs SET pause_ms = $1 WHERE user_id = $2 AND feed_id = $3",
vec![pause_ms.into(), user_id.into(), feed_id.into()],
)
.await?;
Ok(())
}
/// Files on disk DeadAir has not looked at since they were downloaded, oldest first.
pub async fn deadair_todo(&self) -> Result<Vec<DeadAirTodo>> {
self.rows(
"SELECT e.id, e.feed_id, e.guid, e.path FROM enclosures e
LEFT JOIN deadair_plans d ON d.enclosure_id = e.id
WHERE e.path IS NOT NULL AND e.state = 'done'
AND (d.enclosure_id IS NULL OR d.made_at < coalesce(e.downloaded_at, 0))
ORDER BY e.id",
vec![],
)
.await?
.iter()
.map(|r| {
Ok(DeadAirTodo {
id: r.try_get("", "id")?,
feed_id: r.try_get("", "feed_id")?,
guid: r.try_get("", "guid")?,
path: r.try_get("", "path")?,
})
})
.collect()
}
/// Whether any of these people subscribes to the feed.
pub async fn subscribed_any(&self, users: &[i64], feed_id: &str) -> Result<bool> {
if users.is_empty() {
return Ok(false);
}
let mut a = Args::default();
let feed = a.p(feed_id);
let ids: Vec<String> = users.iter().map(|&u| a.p(u)).collect();
let sql = format!(
"SELECT 1 AS one FROM subscriptions WHERE feed_id = {feed} AND user_id IN ({}) LIMIT 1",
ids.join(", ")
);
Ok(!self.rows(&sql, a.0).await?.is_empty())
}
/// Who has heard an episode through: their place at 90% of its length or past it, where the
/// page calls it played. Not the read mark, which opening an item sets too: going by it,
/// DeadAir passed over every episode opened before it was played (#173).
pub async fn finished(&self, feed_id: &str, guid: &str) -> Result<Vec<i64>> {
self.rows(
"SELECT s.user_id FROM entry_state s
JOIN entries e ON e.feed_id = s.feed_id AND e.guid = s.guid
WHERE s.feed_id = $1 AND s.guid = $2 AND coalesce(s.duration, e.duration, 0) > 0
AND s.position * 10 >= coalesce(s.duration, e.duration) * 9",
vec![feed_id.into(), guid.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "user_id")?))
.collect()
}
/// Records what DeadAir found in an enclosure: its plans as JSON, `{}` for none.
pub async fn store_plans(&self, enclosure_id: i64, plans: &str) -> Result<()> {
self.exec(
"INSERT INTO deadair_plans (enclosure_id, plans, made_at) VALUES ($1, $2, $3)
ON CONFLICT (enclosure_id) DO UPDATE SET plans = excluded.plans, made_at = excluded.made_at",
vec![enclosure_id.into(), plans.into(), now().into()],
)
.await?;
Ok(())
}
/// DeadAir's plans for these enclosures, by pause, for each one it has looked at since it
/// was last downloaded and is still on disk. Empty for a file it does not apply to; missing
/// for one it has yet to look at.
pub async fn plans(
&self,
enclosure_ids: &[i64],
) -> Result<std::collections::HashMap<i64, std::collections::BTreeMap<i64, crate::deadair::Plan>>> {
if enclosure_ids.is_empty() {
return Ok(Default::default());
}
let mut a = Args::default();
let ids: Vec<String> = enclosure_ids.iter().map(|&i| a.p(i)).collect();
let sql = format!(
"SELECT d.enclosure_id, d.plans FROM deadair_plans d JOIN enclosures e ON e.id = d.enclosure_id
WHERE d.enclosure_id IN ({}) AND e.path IS NOT NULL AND d.made_at >= coalesce(e.downloaded_at, 0)",
ids.join(", ")
);
self.rows(&sql, a.0)
.await?
.iter()
.map(|r| {
let plans: String = r.try_get("", "plans")?;
Ok((r.try_get("", "enclosure_id")?, serde_json::from_str(&plans).context("reading DeadAir's plans")?))
})
.collect()
}
/// Counts an episode towards the time DeadAir has saved someone, once however often it is
/// played.
pub async fn count_deadair(&self, user_id: i64, feed_id: &str, guid: &str, saved_ms: i64) -> Result<()> {
self.exec(
"INSERT INTO deadair_saved (user_id, feed_id, guid, saved_ms, at) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (user_id, feed_id, guid) DO NOTHING",
vec![user_id.into(), feed_id.into(), guid.into(), saved_ms.into(), now().into()],
)
.await?;
Ok(())
}
/// An episode's length as someone's player measured it, else as its feed gives it.
pub async fn entry_duration(&self, user_id: i64, feed_id: &str, guid: &str) -> Result<Option<i64>> {
let rows = self
.rows(
"SELECT coalesce(s.duration, e.duration) AS duration FROM entries e
LEFT JOIN entry_state s ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid
WHERE e.feed_id = $2 AND e.guid = $3",
vec![user_id.into(), feed_id.into(), guid.into()],
)
.await?;
Ok(rows.first().map(|r| r.try_get::<Option<i64>>("", "duration")).transpose()?.flatten())
}
/// The time DeadAir has saved someone, in ms.
pub async fn deadair_total(&self, user_id: i64) -> Result<i64> {
let rows = self
.rows(
"SELECT CAST(coalesce(sum(saved_ms), 0) AS BIGINT) AS total FROM deadair_saved WHERE user_id = $1",
vec![user_id.into()],
)
.await?;
// Cast: Postgres sums bigints as NUMERIC, which does not read as an i64.
Ok(rows.first().map(|r| r.try_get::<i64>("", "total")).transpose()?.unwrap_or(0))
}
}
/// Unix seconds. Everything time-shaped in the DB is stored this way.
/// A Signal as its owner's sidebar lists it (#254).
#[derive(Debug, serde::Serialize)]
pub struct SignalRow {
pub id: i64,
pub name: String,
pub keywords: Vec<String>,
pub description: Option<String>,
pub icon: Option<String>,
pub items: i64,
pub unread: i64,
}
/// What a Signal is changed to: each field None to leave it as it is.
#[derive(Default)]
pub struct SignalChange<'a> {
pub name: Option<&'a str>,
pub words: Option<&'a [String]>,
pub description: Option<Option<&'a str>>,
pub icon: Option<Option<&'a str>>,
}
/// One Signal's question about an item: what it is called, its words, and its owner's sentence.
#[derive(Debug, Clone)]
pub struct Ask {
pub id: i64,
pub name: String,
pub keywords: Vec<String>,
pub description: Option<String>,
}
/// An item still to be put to Jev for someone, with each of their Signals that has not been
/// asked about it.
#[derive(Debug)]
pub struct Unjudged {
pub user_id: i64,
/// Their account's name, what /metrics bills by (#278).
pub user_name: String,
/// Their own key; None when the server's is used for everyone.
pub key: Option<String>,
pub feed_id: String,
pub guid: String,
pub feed_title: Option<String>,
pub title: Option<String>,
pub description: Option<String>,
pub signals: Vec<Ask>,
}
impl Db {
pub async fn signals(&self, user_id: i64) -> Result<Vec<SignalRow>> {
self.signal_rows(user_id, None).await
}
/// One of this person's Signals: None for anyone else's, as for one that does not exist.
pub async fn signal(&self, user_id: i64, id: i64) -> Result<Option<SignalRow>> {
Ok(self.signal_rows(user_id, Some(id)).await?.pop())
}
async fn signal_rows(&self, user_id: i64, id: Option<i64>) -> Result<Vec<SignalRow>> {
let mut a = Args::default();
let user = a.p(user_id);
let one = id.map(|i| format!("AND g.id = {}", a.p(i))).unwrap_or_default();
// The items it shows, and of those the unread: what its list counts, by the same rules. One
// pass for both; they were two copies of the same subquery (#348).
let sql = format!(
"SELECT g.id, g.name, g.keywords, g.description, g.icon,
coalesce(c.items, 0) AS items, coalesce(c.unread, 0) AS unread
FROM signals g
LEFT JOIN (SELECT v.signal_id, count(*) AS items,
sum(CASE WHEN NOT coalesce(st.read, false) THEN 1 ELSE 0 END) AS unread
FROM signal_verdicts v
JOIN signals vg ON vg.id = v.signal_id
JOIN entries e ON e.feed_id = v.feed_id AND e.guid = v.guid
JOIN subscriptions s ON s.user_id = vg.user_id AND s.feed_id = v.feed_id
LEFT JOIN entry_state st
ON st.user_id = vg.user_id AND st.feed_id = v.feed_id AND st.guid = v.guid
WHERE vg.user_id = {user} AND v.hit AND {} AND {}
GROUP BY v.signal_id) c ON c.signal_id = g.id
WHERE g.user_id = {user} {one} ORDER BY lower(g.name), g.id",
not_hidden(&user),
shown_in("hide_from_signals", "s")
);
self.rows(&sql, a.0)
.await?
.iter()
.map(|r| {
Ok(SignalRow {
id: r.try_get("", "id")?,
name: r.try_get("", "name")?,
keywords: keywords(r.try_get("", "keywords")?).unwrap_or_default(),
description: r.try_get("", "description")?,
icon: r.try_get("", "icon")?,
items: r.try_get("", "items")?,
unread: r.try_get("", "unread")?,
})
})
.collect()
}
pub async fn add_signal(
&self,
user_id: i64,
name: &str,
words: &[String],
description: Option<&str>,
) -> Result<i64> {
let m = signals::ActiveModel {
user_id: Set(user_id),
name: Set(name.to_owned()),
keywords: Set(serde_json::to_string(words)?),
description: Set(description.map(str::to_owned)),
created: Set(now()),
..Default::default()
}
.insert(&self.orm)
.await?;
Ok(m.id)
}
/// Changes a Signal. A new name, words or sentence puts what it picked out to Jev again, since
/// that was the old question's answer; what it passed over stays passed over, and new items
/// are asked the new question as they come (#276). A new icon asks nothing. False for a Signal
/// that is not this person's.
pub async fn set_signal(&self, user_id: i64, id: i64, change: SignalChange<'_>) -> Result<bool> {
let SignalChange { name, words, description, icon } = change;
if self.signal(user_id, id).await?.is_none() {
return Ok(false);
}
let mut m = signals::ActiveModel { id: Set(id), ..Default::default() };
if let Some(n) = name {
m.name = Set(n.to_owned());
}
if let Some(w) = words {
m.keywords = Set(serde_json::to_string(w)?);
}
if let Some(d) = description {
m.description = Set(d.map(str::to_owned));
}
if let Some(i) = icon {
m.icon = Set(i.map(str::to_owned));
}
if name.is_some() || words.is_some() || description.is_some() {
signal_verdicts::Entity::update_many()
.col_expr(signal_verdicts::Column::Recheck, Expr::val(true))
.filter(signal_verdicts::Column::SignalId.eq(id))
.filter(signal_verdicts::Column::Hit.eq(true))
.exec(&self.orm)
.await?;
}
if name.is_some() || words.is_some() || description.is_some() || icon.is_some() {
m.update(&self.orm).await?;
}
Ok(true)
}
pub async fn remove_signal(&self, user_id: i64, id: i64) -> Result<bool> {
if self.signal(user_id, id).await?.is_none() {
return Ok(false);
}
// Not left to the cascade: SQLite enforces foreign keys only when asked to.
signal_verdicts::Entity::delete_many().filter(signal_verdicts::Column::SignalId.eq(id)).exec(&self.orm).await?;
signals::Entity::delete_by_id(id).exec(&self.orm).await?;
Ok(true)
}
pub async fn typesafe_key_set(&self, user_id: i64) -> Result<bool> {
Ok(users::Entity::find_by_id(user_id).one(&self.orm).await?.is_some_and(|u| u.typesafe_key.is_some()))
}
pub async fn set_typesafe_key(&self, user_id: i64, key: Option<&str>) -> Result<()> {
self.update_user(
user_id,
users::ActiveModel { typesafe_key: Set(key.map(str::to_owned)), ..Default::default() },
)
.await
}
/// Up to `limit` items, newest first, that someone with a key has a Signal to ask about: not
/// yet asked, from the last `days` (#276), or picked out before the Signal changed, however
/// old; from the feeds they subscribe to, and not hidden from them. `skip` are people whose
/// key TypeSafe has refused. With `everyone`, the server's key is used, and people without a
/// key of their own are asked for too (#271).
pub async fn unjudged(&self, skip: &[i64], limit: i64, everyone: bool, days: u64) -> Result<Vec<Unjudged>> {
let mut a = Args::default();
let since = a.p(now() - days as i64 * 86_400);
let skip = if skip.is_empty() {
String::new()
} else {
format!("AND g.user_id NOT IN ({})", skip.iter().map(|u| a.p(*u)).collect::<Vec<_>>().join(", "))
};
let limit = a.p(limit);
let keyed = if everyone { "true" } else { "u.typesafe_key IS NOT NULL" };
// A feed kept out of Signals is not put to Jev at all, so it costs nothing (#289).
let in_signals = shown_in("hide_from_signals", "s");
// Two halves, not one OR: the recent half reads entries_recent, and the rechecks are a few
// rows of signal_verdicts. The same placeholders serve both.
let half = |items: &str| {
format!(
"SELECT g.id AS signal_id, g.name, g.keywords, g.description AS signal_description, g.user_id, u.name AS user_name, u.typesafe_key,
e.feed_id, e.guid, e.title, e.description, f.title AS feed_title, coalesce(e.published, e.first_seen) AS at
FROM signals g
JOIN users u ON u.id = g.user_id
JOIN subscriptions s ON s.user_id = g.user_id
{items}
LEFT JOIN feeds f ON f.id = e.feed_id
WHERE {keyed} {skip} AND {in_signals}
AND NOT EXISTS (SELECT 1 FROM hidden h
WHERE h.user_id = g.user_id AND h.feed_id = e.feed_id AND h.guid = e.guid)"
)
};
let recent = half(&format!(
"JOIN entries e ON e.feed_id = s.feed_id AND coalesce(e.published, e.first_seen) >= {since}
AND NOT EXISTS (SELECT 1 FROM signal_verdicts v
WHERE v.signal_id = g.id AND v.feed_id = e.feed_id AND v.guid = e.guid)"
));
let again = half(
"JOIN signal_verdicts v ON v.signal_id = g.id AND v.feed_id = s.feed_id AND v.recheck
JOIN entries e ON e.feed_id = v.feed_id AND e.guid = v.guid",
);
let sql = format!(
"SELECT * FROM ({recent} UNION ALL {again}) q ORDER BY at DESC, feed_id, guid, user_id LIMIT {limit}"
);
let mut out: Vec<Unjudged> = Vec::new();
for r in self.rows(&sql, a.0).await? {
let (user_id, feed_id, guid): (i64, String, String) =
(r.try_get("", "user_id")?, r.try_get("", "feed_id")?, r.try_get("", "guid")?);
let signal = Ask {
id: r.try_get("", "signal_id")?,
name: r.try_get("", "name")?,
keywords: keywords(r.try_get("", "keywords")?).unwrap_or_default(),
// Named apart from the item's own description, which is in the same row.
description: r.try_get("", "signal_description")?,
};
// Rows of one item for one person come together, by the ORDER BY.
match out.last_mut() {
Some(u) if (u.user_id, &u.feed_id, &u.guid) == (user_id, &feed_id, &guid) => u.signals.push(signal),
_ => out.push(Unjudged {
user_id,
user_name: r.try_get("", "user_name")?,
key: r.try_get("", "typesafe_key")?,
feed_id,
guid,
feed_title: r.try_get("", "feed_title")?,
title: r.try_get("", "title")?,
description: r.try_get("", "description")?,
signals: vec![signal],
}),
}
}
Ok(out)
}
pub async fn record_verdicts(&self, feed_id: &str, guid: &str, verdicts: &[(i64, bool)]) -> Result<()> {
for (signal, hit) in verdicts {
// Not for a Signal removed while Jev was being asked, which Postgres's foreign key
// would refuse and SQLite would keep for nothing.
// shortcut: keywords changed mid-call keep a verdict on the old ones for that item.
self.exec(
"INSERT INTO signal_verdicts (signal_id, feed_id, guid, hit, recheck)
SELECT $1, $2, $3, $4, false WHERE EXISTS (SELECT 1 FROM signals WHERE id = $1)
ON CONFLICT (signal_id, feed_id, guid) DO UPDATE SET hit = excluded.hit, recheck = false",
vec![(*signal).into(), feed_id.into(), guid.into(), (*hit).into()],
)
.await?;
}
Ok(())
}
}
pub fn now() -> i64 {
SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn unread_starts_where_it_was_worked_out_and_loses_nothing() {
let db = Db::memory().await.unwrap();
let me = db.create_user("me", None, false).await.unwrap();
db.exec_for_test(
"INSERT INTO feeds (id, url) VALUES ('f', 'http://x/f.xml');
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','old',100),('f','mid',200),('f','new',300);",
)
.await
.unwrap();
db.subscribe(me, "f").await.unwrap();
db.set_entry_flag(me, "f", "old", EntryFlag::Read, true).await.unwrap();
let unread = async || {
let order = order_sql("published", "desc");
let rows = db.entries_of(me, None, Filter::Unread, None, 0, 50, &order).await.unwrap();
let mut guids: Vec<String> = rows.into_iter().map(|r| r.guid).collect();
guids.sort();
let count = db.count_of(me, None, Filter::Unread, None).await.unwrap();
let listed = db.feed_list(me, None).await.unwrap()["f"].unread;
assert_eq!(count, guids.len() as i64);
assert_eq!(listed, count, "the feed list counts what the list shows");
guids
};
assert_eq!(unread().await, ["mid", "new"]);
db.advance_unread_from().await.unwrap();
let from = async || -> Option<i64> {
db.rows("SELECT unread_from FROM subscriptions", vec![]).await.unwrap()[0]
.try_get("", "unread_from")
.unwrap()
};
assert_eq!(from().await, Some(200), "the oldest unread");
assert_eq!(unread().await, ["mid", "new"], "nothing unread goes missing (#347)");
// An old item marked unread again comes back, though it is older than where unread began.
db.set_entry_flag(me, "f", "old", EntryFlag::Read, false).await.unwrap();
assert_eq!(unread().await, ["mid", "new", "old"]);
// With everything read, it starts no later than an hour ago.
db.mark_all_read(me, &["f".into()]).await.unwrap();
db.advance_unread_from().await.unwrap();
assert!(from().await.is_some_and(|t| t <= now() - 3600));
assert!(unread().await.is_empty());
}
#[tokio::test]
async fn a_new_password_signs_the_account_out_everywhere_but_keeps_its_tokens() {
let db = Db::memory().await.unwrap();
let (me, you) =
(db.create_user("me", None, false).await.unwrap(), db.create_user("you", None, false).await.unwrap());
db.create_session(me, "mine").await.unwrap();
db.create_session(you, "yours").await.unwrap();
db.create_api_token(me, "script", "hash").await.unwrap();
db.set_password(me, "new-hash").await.unwrap();
assert!(db.session_user("mine", 3600).await.unwrap().is_none(), "a stolen cookie stops working (#334)");
assert!(db.session_user("yours", 3600).await.unwrap().is_some(), "nobody else is signed out");
assert!(db.api_token_user("hash").await.unwrap().is_some());
}
#[tokio::test]
async fn deadair_looks_at_a_file_once_per_download_and_counts_an_episode_once() {
let db = Db::memory().await.unwrap();
let u = db.create_user("dee", None, false).await.unwrap();
db.set_deadair(u, "f", Some(true)).await.unwrap();
db.set_deadair(u, "g", Some(false)).await.unwrap();
db.set_deadair(u, "g", None).await.unwrap();
assert_eq!(db.deadair_prefs(Some(u)).await.unwrap(), vec![(u, "f".to_string(), true, None)]);
db.set_deadair_pause(u, "f", 500).await.unwrap();
// Only beside a setting of the switch: a row of its own would turn DeadAir off for "g".
db.set_deadair_pause(u, "g", 400).await.unwrap();
assert_eq!(db.deadair_prefs(Some(u)).await.unwrap(), vec![(u, "f".to_string(), true, Some(500))]);
for sql in [
"INSERT INTO enclosures (id, feed_id, guid, url, path, state, downloaded_at)
VALUES (1, 'f', 'g', 'http://x/a.mp3', '/x/a.mp3', 'done', 100)",
"INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (2, 'f', 'h', 'http://x/b.mp3', NULL, 'reaped')",
"INSERT INTO entries (feed_id, guid, first_seen, duration) VALUES ('f', 'g', 1, 600)",
] {
db.orm.execute_unprepared(sql).await.unwrap();
}
let todo = db.deadair_todo().await.unwrap();
assert_eq!(todo.iter().map(|t| t.id).collect::<Vec<_>>(), [1], "only a file on disk");
db.store_plans(1, r#"{"300": {"cuts": [[1000,3000]], "pieces": [[0,10,1.0]]}}"#).await.unwrap();
assert!(db.deadair_todo().await.unwrap().is_empty());
let p = db.plans(&[1, 2]).await.unwrap();
assert_eq!(
(p.len(), p[&1][&300].cuts.as_slice(), p[&1][&300].saved_ms()),
(1, &[crate::deadair::Cut(1000, 3000)][..], 2000)
);
// Downloaded again since: looked at again before it plays shortened.
db.orm
.execute_unprepared(&format!("UPDATE enclosures SET downloaded_at = {} WHERE id = 1", now() + 60))
.await
.unwrap();
assert_eq!(db.deadair_todo().await.unwrap().len(), 1);
assert!(db.plans(&[1]).await.unwrap().is_empty());
assert!(!db.subscribed_any(&[u], "f").await.unwrap());
db.subscribe(u, "f").await.unwrap();
assert!(db.subscribed_any(&[u], "f").await.unwrap());
// Opened, which marks it read, and not played: not finished (#173).
db.set_entry_flag(u, "f", "g", EntryFlag::Read, true).await.unwrap();
assert!(db.finished("f", "g").await.unwrap().is_empty());
assert_eq!(db.entry_duration(u, "f", "g").await.unwrap(), Some(600));
db.set_position(u, "f", "g", 10, Some(590)).await.unwrap();
assert_eq!(db.entry_duration(u, "f", "g").await.unwrap(), Some(590), "what the player measured");
assert!(db.finished("f", "g").await.unwrap().is_empty());
db.set_position(u, "f", "g", 540, Some(590)).await.unwrap();
assert_eq!(db.finished("f", "g").await.unwrap(), [u], "past 90%");
assert_eq!(db.deadair_total(u).await.unwrap(), 0);
db.count_deadair(u, "f", "g", 2000).await.unwrap();
db.count_deadair(u, "f", "g", 2000).await.unwrap();
assert_eq!(db.deadair_total(u).await.unwrap(), 2000, "a replay counts once");
}
#[tokio::test]
async fn a_read_does_not_wait_for_a_write_in_progress() {
use sea_orm::TransactionTrait;
let db = Db::memory().await.unwrap();
// A write held open, as a scan's or an import's is while it runs.
let tx = db.orm.begin().await.unwrap();
tx.execute_unprepared("INSERT INTO feeds (id, url) VALUES ('held', 'http://x/held')").await.unwrap();
// With one connection, which SQLite had (#136), this waited for the transaction to end.
let read = tokio::time::timeout(std::time::Duration::from_secs(2), db.feed_summary("other")).await;
assert!(read.is_ok(), "a read waited behind a write");
tx.commit().await.unwrap();
// And writers at once all get their turn, not a refusal.
let writes = (0..16).map(|i| {
let db = &db;
async move {
db.record_feed(&format!("f{i}"), "http://x/f", Some("F"), None, None, None, None, None, None).await
}
});
for r in futures_util::future::join_all(writes).await {
r.unwrap();
}
}
#[tokio::test]
async fn a_commit_does_not_wait_for_the_disk() {
let db = Db::memory().await.unwrap();
let backend = db.orm.get_database_backend();
if backend == sea_orm::DbBackend::Sqlite {
let row = db.orm.query_one_raw(Statement::from_string(backend, "PRAGMA synchronous")).await.unwrap();
assert_eq!(
row.unwrap().try_get_by_index::<i32>(0).unwrap(),
1,
"NORMAL, on every connection in the pool (#135)"
);
} else {
let row = db.orm.query_one_raw(Statement::from_string(backend, "SHOW synchronous_commit")).await.unwrap();
assert_eq!(
row.unwrap().try_get_by_index::<String>(0).unwrap(),
"off",
"with the test's own search_path as well (#140)"
);
}
}
#[tokio::test]
async fn a_database_from_before_descriptions_gains_the_column_and_reads_each_feed_whole() {
let db = Db::memory().await.unwrap();
let record = |etag: &'static str, about: Option<&'static str>| {
let db = &db;
async move {
db.record_feed("f", "https://x/f", Some("F"), Some(etag), Some("then"), None, None, None, about)
.await
.unwrap()
}
};
record("\"a\"", Some("About F")).await;
assert_eq!(db.feed_description("f").await.unwrap().as_deref(), Some("About F"));
record("\"a\"", None).await;
assert_eq!(db.feed_description("f").await.unwrap().as_deref(), Some("About F"), "kept when a read finds none");
// A database from before #130.
db.exec_for_test("ALTER TABLE feeds DROP COLUMN description").await.unwrap();
create_missing(&db.orm).await.unwrap();
assert_eq!(db.feed_description("f").await.unwrap(), None);
let st = db.http_state("f").await.unwrap();
assert!(
st.etag.is_none() && st.last_modified.is_none(),
"every feed is read whole next time, for its description"
);
// With the column there, opening the database again writes nothing.
record("\"b\"", Some("About F")).await;
create_missing(&db.orm).await.unwrap();
assert_eq!(db.http_state("f").await.unwrap().etag.as_deref(), Some("\"b\""));
}
#[tokio::test]
async fn a_database_from_before_body_hashes_gains_the_columns() {
let db = Db::memory().await.unwrap();
db.record_feed("f", "https://x/f", Some("F"), None, None, None, None, None, None).await.unwrap();
db.set_body("f", "abc").await.unwrap();
assert_eq!(db.http_state("f").await.unwrap().body_hash.as_deref(), Some("abc"));
// A database from before #242.
for sql in ["ALTER TABLE feeds DROP COLUMN body_hash", "ALTER TABLE feeds DROP COLUMN same_since"] {
db.exec_for_test(sql).await.unwrap();
}
create_missing(&db.orm).await.unwrap();
let st = db.http_state("f").await.unwrap();
assert_eq!((st.body_hash, st.same_since), (None, None), "every feed is read whole once more");
db.set_body("f", "def").await.unwrap();
assert_eq!(db.http_state("f").await.unwrap().body_hash.as_deref(), Some("def"));
}
#[tokio::test]
async fn every_sort_column_runs_and_orders_both_ways() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g');
INSERT INTO feeds (id, url, title) VALUES ('f','u','Zebra'),('g','v','Aardvark');
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','banana',100),('g','b','Apple',200),('f','c','cherry',300);
INSERT INTO enclosures (id, feed_id, guid, url, mime, length, state) VALUES
(1,'f','a','u1','audio/mpeg',300,'pending'),(2,'g','b','u2','image/png',10,'pending'),
(3,'f','c','u3','video/mp4',2000,'pending');",
)
.await
.unwrap();
let order = async |col: &str, dir: &str| -> Vec<String> {
db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir))
.await
.unwrap()
.into_iter()
.map(|e| e.guid)
.collect()
};
assert_eq!(order("title", "asc").await, ["b", "a", "c"], "Apple, banana, cherry: case folded");
assert_eq!(order("title", "desc").await, ["c", "a", "b"]);
assert_eq!(order("feed", "asc").await, ["b", "c", "a"], "Aardvark, then Zebra's newest first");
assert_eq!(order("type", "asc").await, ["a", "b", "c"], "audio, image, video");
assert_eq!(order("size", "desc").await, ["c", "a", "b"]);
// An item with no file has no size or type: last going down, first going up, on both
// databases (they disagree about NULL unless told).
db.exec_for_test("INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('f','n','no file',50);")
.await
.unwrap();
assert_eq!(order("size", "desc").await.last().map(String::as_str), Some("n"));
assert_eq!(order("type", "asc").await.first().map(String::as_str), Some("n"));
db.exec_for_test("DELETE FROM entries WHERE guid = 'n';").await.unwrap();
assert_eq!(order("published", "desc").await, ["c", "b", "a"]);
// A page from the middle: the keys are paged before the rows are read (#188).
let page: Vec<String> = db
.entries_in(1, None, Filter::All, None, 1, 1, &order_sql("published", "desc"))
.await
.unwrap()
.into_iter()
.map(|e| e.guid)
.collect();
assert_eq!(page, ["b"]);
db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).await.unwrap();
assert_eq!(order("kept", "desc").await[0], "a");
// A pin does not lift an item above the others in any other sort (#36).
assert_eq!(order("published", "desc").await, ["c", "b", "a"]);
// An unknown column or direction is newest first; the name itself never reaches the SQL.
assert_eq!(order("title; DROP TABLE entries", "sideways").await, ["c", "b", "a"]);
assert!(!order_sql("x'; --", "asc").contains("x'"));
}
#[tokio::test]
async fn the_feed_list_in_one_go_matches_asking_feed_by_feed() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g'),(2,'f'),(2,'h');
INSERT INTO feeds (id, url, title, ttl_mins) VALUES ('f','u','F',30),('g','v','G',null),('h','w','H',null);
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0),('f','c',0),('g','d',0),('h','e',0);
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'f','a',true),(2,'f','b',true);
INSERT INTO hidden (user_id, feed_id, guid) VALUES (1,'f','c');
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','a','u1','/tmp/a','done'),(2,'f','b','u2',null,'pending');
INSERT INTO blocklists (user_id, feed_id, words) VALUES (1,'g','[\"spoiler\"]');",
).await
.unwrap();
let list = db.feed_list(1, None).await.unwrap();
let g = db.feed_list(1, Some("g")).await.unwrap();
assert_eq!(g.keys().collect::<Vec<_>>(), ["g"]);
assert_eq!(
(g["g"].unread, g["g"].summary.entries, g["g"].blocked.clone()),
(1, 1, vec!["spoiler".to_string()])
);
for id in ["f", "g"] {
let one = &list[id];
let s = db.feed_summary(id).await.unwrap();
assert_eq!((one.summary.entries, one.summary.downloaded), (s.entries, s.downloaded), "{id}");
assert_eq!(one.unread, db.unread_count(1, id).await.unwrap(), "{id}");
assert_eq!(one.blocked, db.blocklist(1, id).await.unwrap(), "{id}");
assert_eq!(one.ttl_mins, db.http_state(id).await.unwrap().ttl_mins, "{id}");
}
assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
// Only sam reads h: still listed, for its title, but its items are not counted for ray (#165).
assert_eq!((list["h"].summary.title.as_deref(), list["h"].summary.entries), (Some("H"), 0));
}
#[tokio::test]
async fn a_hosts_artwork_moves_to_https_for_one_feed() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO feeds (id, url, image) VALUES ('f','u','http://a.example/logo.png'),('g','v','http://a.example/g.png');
INSERT INTO entries (feed_id, guid, first_seen, image) VALUES
('f','1',0,'http://a.example/1.jpg'),('f','2',0,'http://b.example/2.jpg'),('g','3',0,'http://a.example/3.jpg');",
).await
.unwrap();
let mut old = db.http_images("f").await.unwrap();
old.sort();
assert_eq!(old, ["http://a.example/1.jpg", "http://a.example/logo.png", "http://b.example/2.jpg"]);
db.secure_images("f", "a.example").await.unwrap();
assert_eq!(db.feed_summary("f").await.unwrap().image.as_deref(), Some("https://a.example/logo.png"));
assert!(db.names_image("https://a.example/1.jpg").await.unwrap());
assert!(db.names_image("http://b.example/2.jpg").await.unwrap()); // another host
assert!(db.names_image("http://a.example/3.jpg").await.unwrap()); // another feed
assert_eq!(db.feed_summary("g").await.unwrap().image.as_deref(), Some("http://a.example/g.png"));
}
#[tokio::test]
async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO feeds (id, url, error_since, managed, group_id) VALUES
('dead','a',100,false,null),('quiet','b',null,false,null),('news','c',null,false,null),
('new','d',null,false,null),('wanted','e',100,false,null),('kept','f',100,false,null),
('child','g',100,true,'opml'),('lately','h',900,false,null);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted');
INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');",
)
.await
.unwrap();
// Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100.
// news published lately, new has no items yet, wanted has a subscriber, kept a file,
// child comes from an OPML, lately began failing after the cutoff.
let mut stale = db.stale_unsubscribed(500, 500).await.unwrap();
stale.sort();
assert_eq!(stale, ["dead", "quiet"]);
db.forget_feed("dead").await.unwrap();
assert_eq!(db.feed_summary("dead").await.unwrap().entries, 0);
assert!(!db.feed_urls().await.unwrap().iter().any(|(id, _)| id == "dead"));
}
#[tokio::test]
async fn a_failing_feed_nobody_subscribes_to_is_forgotten() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO feeds (id, url, last_error) VALUES
('gone','cnn.com','relative URL'),('wanted','u','HTTP 503'),
('kept','v','HTTP 404'),('fine','w',null),('listed','x','HTTP 500');
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted');
INSERT INTO catalogue (id, spec) VALUES ('listed','{}');
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('gone','a',0),('kept','b',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'gone','a','u1',null,'pending'),(2,'kept','b','u2','/tmp/b','done');",
)
.await
.unwrap();
// Subscribed, holding a file, working, or in the catalogue: all stay.
assert_eq!(db.prune_abandoned_failures().await.unwrap(), vec!["gone".to_string()]);
assert_eq!(db.feed_summary("gone").await.unwrap().entries, 0);
assert!(db.enclosure(1).await.unwrap().is_none());
assert_eq!(db.feed_summary("kept").await.unwrap().entries, 1);
assert!(db.prune_abandoned_failures().await.unwrap().is_empty());
}
#[tokio::test]
async fn only_the_newest_files_stay_queued_and_the_rest_are_held() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400);
INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);",
)
.await
.unwrap();
let states = || async {
let mut v: Vec<(i64, String)> = db
.rows("SELECT id, state FROM enclosures ORDER BY id", vec![])
.await
.unwrap()
.iter()
.map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap()))
.collect();
v.sort();
v.into_iter().map(|(_, s)| s).collect::<Vec<_>>()
};
db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3
assert_eq!(states().await, ["held", "held", "done", "pending"]);
db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back
assert_eq!(states().await, ["held", "pending", "done", "pending"]);
db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held
assert_eq!(states().await, ["held", "held", "done", "held"]);
db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued
assert_eq!(states().await, ["pending", "pending", "done", "pending"]);
}
#[tokio::test]
async fn a_limit_takes_the_newest_episodes_not_the_back_catalogue() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400),('f','e5',0,500);
INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','done','/tmp/4'),
(5,'f','e5','u5','skipped',null);",
)
.await
.unwrap();
let ids = |p: Vec<Pending>| p.into_iter().map(|p| p.id).collect::<Vec<_>>();
// The newest three with a file worth having are e4, e3 and e2 (e5's was skipped): only
// e2 is waiting. e1 is back catalogue.
assert_eq!(ids(db.pending("f", 3).await.unwrap()), vec![2]);
// Once e2 is down, nothing: before #97 the next scan took e1.
db.exec_for_test("UPDATE enclosures SET state = 'done', path = '/tmp/2' WHERE id = 2").await.unwrap();
assert!(db.pending("f", 3).await.unwrap().is_empty());
// Unlimited is the archive: everything still waiting.
assert_eq!(ids(db.pending("f", usize::MAX).await.unwrap()), vec![1]);
}
#[tokio::test]
async fn an_unlimited_queue_takes_everything_waiting() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0);
INSERT INTO enclosures (id, feed_id, guid, url, state) VALUES (1,'f','a','u1','pending'),(2,'f','b','u2','pending');",
).await
.unwrap();
assert_eq!(db.pending("f", usize::MAX).await.unwrap().len(), 2);
}
#[tokio::test]
async fn a_scan_knows_a_feeds_stored_items_and_files() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0),('g','c',0);
INSERT INTO enclosures (id, feed_id, guid, url, state) VALUES (1,'f','a','u1','pending'),(2,'g','c','u2','pending');",
).await
.unwrap();
let (items, files) = db.stored_items("f").await.unwrap();
assert_eq!(items, ["a", "b"].map(String::from).into());
assert_eq!(files, ["u1"].map(String::from).into()); // u2 is g's: left to the insert to find
}
#[tokio::test]
async fn a_scan_finds_the_stored_items_the_feed_now_tells_differently() {
let db = Db::memory().await.unwrap();
let item = |guid: &str, title: &str, text: Option<&str>| crate::feed::Entry {
guid: guid.into(),
title: Some(title.into()),
description: text.map(str::to_owned),
..Default::default()
};
db.record_entry("f", &item("a", "A", Some("the review"))).await.unwrap();
db.record_entry("f", &item("b", "B", Some("the next review"))).await.unwrap();
db.record_entry("f", &item("c", "C", Some("as it was"))).await.unwrap();
let changed = db
.changed_items(
"f",
&[
item("a", "A", Some("the review, fixed")), // the text corrected
item("b", "B renamed", Some("the next review")), // retitled
item("c", "C", None), // says nothing of its text: what is stored stands
item("d", "D", Some("new")), // not stored: the insert's, not this
],
)
.await
.unwrap();
assert_eq!(changed, ["a", "b"].map(String::from).into());
assert!(db.changed_items("f", &[]).await.unwrap().is_empty());
// A Markdown copy the feed starts sending is a change, and is stored (#285).
let md = crate::feed::Entry { markdown: Some("*as it was*".into()), ..item("c", "C", Some("as it was")) };
assert_eq!(db.changed_items("f", std::slice::from_ref(&md)).await.unwrap(), ["c"].map(String::from).into());
assert!(!db.record_entry("f", &md).await.unwrap(), "rewritten, not new");
let order = order_sql("published", "desc");
let rows = db.entries_in(1, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap();
assert_eq!(rows.iter().find(|r| r.guid == "c").unwrap().markdown.as_deref(), Some("*as it was*"));
}
#[tokio::test]
async fn only_artwork_a_feed_names_is_fetched_for_the_page() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO feeds (id, url, image) VALUES ('f','u','http://cdn.example/show.jpg');
INSERT INTO entries (feed_id, guid, first_seen, image) VALUES ('f','a',0,'http://cdn.example/ep.jpg');",
)
.await
.unwrap();
assert!(db.names_image("http://cdn.example/show.jpg").await.unwrap());
assert!(db.names_image("http://cdn.example/ep.jpg").await.unwrap());
assert!(!db.names_image("http://192.168.1.1/admin").await.unwrap());
}
#[tokio::test]
async fn deleting_a_shared_file_asks_about_everyone_else() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false),(3,'kit',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'),(3,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','a','u1','/tmp/a','done');",
)
.await
.unwrap();
// Nobody has touched it: both others still have it unplayed.
assert_eq!(db.others_wanting(1, 1).await.unwrap(), (0, 2));
// Sam reads it, Kit stars it.
db.set_entry_flag(2, "f", "a", EntryFlag::Read, true).await.unwrap();
db.set_entry_flag(3, "f", "a", EntryFlag::Flagged, true).await.unwrap();
assert_eq!(db.others_wanting(1, 1).await.unwrap(), (1, 1), "one starred it, one has not played it");
// Asking as Kit, only Ray and Sam count -- and Kit's own star is not a reason to
// warn Kit.
db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).await.unwrap();
assert_eq!(db.others_wanting(1, 3).await.unwrap(), (0, 0));
}
#[tokio::test]
async fn read_state_belongs_to_one_person() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','One',100),('f','b','Two',200);",
)
.await
.unwrap();
assert_eq!(db.unread_count(1, "f").await.unwrap(), 2);
assert_eq!(db.unread_count(2, "f").await.unwrap(), 2);
db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).await.unwrap();
assert_eq!(db.unread_count(1, "f").await.unwrap(), 1, "ray read one of them");
assert_eq!(db.unread_count(2, "f").await.unwrap(), 2, "sam has read nothing");
// Starring and position are just as private.
db.set_entry_flag(1, "f", "b", EntryFlag::Flagged, true).await.unwrap();
db.set_position(2, "f", "b", 42, Some(600)).await.unwrap();
let order = order_sql("published", "desc");
let page = async |user| db.entries_in(user, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap();
let (ray, sam) = (page(1).await, page(2).await);
let ray_b = ray.iter().find(|e| e.guid == "b").unwrap();
let sam_b = sam.iter().find(|e| e.guid == "b").unwrap();
assert!(ray_b.flagged && ray_b.position == 0);
assert!(!sam_b.flagged && sam_b.position == 42);
// So is the length sam's player measured.
assert_ne!(ray_b.duration, Some(600));
assert_eq!(sam_b.duration, Some(600));
// Marking a whole feed read is likewise one person's business.
assert_eq!(db.mark_all_read(2, &["f".to_string()]).await.unwrap(), 2);
assert_eq!(db.unread_count(2, "f").await.unwrap(), 0);
assert_eq!(db.unread_count(1, "f").await.unwrap(), 1);
}
#[tokio::test]
async fn schema_is_idempotent_and_summary_handles_unknown_feeds() {
let db = Db::memory().await.unwrap();
// Creating what is missing again must not fail: open() does it on every start.
create_missing(&db.orm).await.unwrap();
let sum = db.feed_summary("never-seen").await.unwrap();
assert_eq!(sum.last_checked, None);
assert_eq!(sum.entries, 0);
assert_eq!(sum.downloaded, 0);
}
#[tokio::test]
async fn a_renamed_account_keeps_everything_but_its_name() {
let db = Db::memory().await.unwrap();
let ray = db.create_user("rays", None, true).await.unwrap();
db.create_user("sam", None, false).await.unwrap();
db.subscribe(ray, "f").await.unwrap();
db.rename_user(ray, "rays@sdf1.net").await.unwrap();
assert!(db.user_by_name("rays").await.unwrap().is_none());
let renamed = db.user_by_name("RAYS@sdf1.net").await.unwrap().unwrap();
assert_eq!((renamed.id, renamed.is_admin), (ray, true), "same account, still the admin");
assert_eq!(db.subscriptions_for(ray).await.unwrap().len(), 1, "and still subscribed");
assert!(db.rename_user(ray, "sam").await.is_err(), "a taken name is refused");
}
#[tokio::test]
async fn an_account_knows_when_it_was_made_and_last_signed_in() {
let db = Db::memory().await.unwrap();
let id = db.create_user("ray", None, true).await.unwrap();
let get = async || db.user_by_id(id).await.unwrap().unwrap();
assert!(get().await.created.is_some_and(|t| t > 0));
assert_eq!(get().await.last_login, None, "made, but never signed in");
db.signed_in(id).await.unwrap();
let first = get().await.last_login.unwrap();
// Within the hour, the proxy vouching again writes nothing; after it, it does.
db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 60)).await.unwrap();
db.signed_in(id).await.unwrap();
assert_eq!(get().await.last_login, Some(first - 60));
db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 7200)).await.unwrap();
db.signed_in(id).await.unwrap();
assert!(get().await.last_login.unwrap() >= first);
}
#[tokio::test]
async fn the_first_admin_starts_with_the_catalogue_and_only_once() {
// Cutting this along with the dead read columns left the browser suite's admin with an
// empty sidebar: it is how a fresh install's first account gets config.toml's feeds.
let db = Db::memory().await.unwrap();
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',true);").await.unwrap();
let subs = async || db.i64s_for_test("SELECT count(*) FROM subscriptions").await[0];
let catalogue = ["a".to_string(), "b".to_string()];
assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 2);
assert_eq!(subs().await, 2);
// Once anyone subscribes to anything it never runs again, so an unsubscribe sticks.
db.unsubscribe(1, "a").await.unwrap();
assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 0);
assert_eq!(subs().await, 1);
}
#[tokio::test]
async fn every_filter_works_with_and_without_a_search_term() {
// Regression: the search clause used to be omitted when no term was given, while
// ?2 was still bound -- rusqlite rejects a parameter the statement never mentions,
// so plain filtering failed with "Wrong number of parameters passed to query".
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, title, description, first_seen, duration) VALUES
('f','a','Alpha dive','notes one', 100,NULL),
('f','b','Beta', 'notes two', 200,NULL),
('f','c','Gamma dive','notes three',300,NULL),
('f','d','Delta', 'notes four', 400,NULL),
('f','e','Epsilon', 'notes five', 500,45),
('f','g','Gimel', 'notes six', 600,NULL),
('f','h','Heth', 'notes seven',700,900);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','b','u1','/tmp/b','done');
-- Read and starred belong to a person now, so say which one.
INSERT INTO users (id, name, is_admin) VALUES (7,'reader',true);
-- Every list is of feeds its reader subscribes to; Unread joins the subscription.
INSERT INTO subscriptions (user_id, feed_id) VALUES (7,'f');
INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES
(7,'f','b',true,false,0),
(7,'f','c',true,true,0),
-- Started, length unknown: this is Currently Listening.
(7,'f','d',false,false,42),
-- 42 of 45 seconds is past the 90% the player calls finished.
(7,'f','e',true,false,42),
-- Barely touched (opened, closed within seconds): not Currently Listening.
(7,'f','g',false,false,3),
-- Opened, and so read, but 42 of 900 seconds in: still Currently Listening.
-- Filtering on read hid exactly these (issue #14).
(7,'f','h',true,false,42);",
)
.await
.unwrap();
for f in [Filter::All, Filter::Unread, Filter::Downloaded, Filter::Flagged, Filter::InProgress] {
// Both paths must run without erroring, and agree with each other.
let order = order_sql("published", "desc");
let rows = db.entries_in(7, Some("f"), f, None, 0, 50, &order).await.unwrap();
let n = db.count_in(7, Some("f"), f, None).await.unwrap();
assert_eq!(rows.len() as i64, n, "{f:?} count disagrees with the page");
let rows = db.entries_in(7, Some("f"), f, Some("dive"), 0, 50, &order).await.unwrap();
let n = db.count_in(7, Some("f"), f, Some("dive")).await.unwrap();
assert_eq!(rows.len() as i64, n, "{f:?} with search disagrees");
}
assert_eq!(db.count_in(7, Some("f"), Filter::All, None).await.unwrap(), 7);
assert_eq!(db.count_in(7, Some("f"), Filter::Unread, None).await.unwrap(), 3);
assert_eq!(db.count_in(7, Some("f"), Filter::Downloaded, None).await.unwrap(), 1);
assert_eq!(db.count_in(7, Some("f"), Filter::Flagged, None).await.unwrap(), 1);
assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("dive")).await.unwrap(), 2);
for wild in ["%", "_", "\\"] {
assert_eq!(db.count_in(7, Some("f"), Filter::All, Some(wild)).await.unwrap(), 0, "{wild} is only itself");
}
assert_eq!(
db.count_in(7, Some("f"), Filter::All, Some("NOTES two")).await.unwrap(),
1,
"search is case-insensitive and covers the description"
);
// Currently Listening: started, not finished, and not just an accidental tap.
let order = order_sql("published", "desc");
let listening = async || {
let rows = db.entries_in(7, Some("f"), Filter::InProgress, None, 0, 50, &order).await.unwrap();
rows.into_iter().map(|e| e.guid).collect::<Vec<_>>()
};
assert_eq!(listening().await, ["h", "d"]);
// The player's measured length is the one that counts, in place of a missing one or over
// the feed's: d, 42 of a measured 45 seconds, is finished; e, which its feed calls 45
// seconds long, is 42 into a 9000-second file and is not. Times left use it too.
db.set_position(7, "f", "d", 42, Some(45)).await.unwrap();
db.set_position(7, "f", "e", 42, Some(9000)).await.unwrap();
assert_eq!(listening().await, ["h", "e"]);
let rows = db.entries_in(7, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap();
assert_eq!(rows.iter().find(|r| r.guid == "e").unwrap().duration, Some(9000));
// A save without one (before the player knows) keeps the length already measured.
db.set_position(7, "f", "e", 43, None).await.unwrap();
assert_eq!(listening().await, ["h", "e"]);
}
#[tokio::test]
async fn a_signal_shows_what_jev_said_fits_and_only_to_its_owner() {
let db = Db::memory().await.unwrap();
let t = now();
db.exec_for_test(&format!(
"INSERT INTO users (id, name, is_admin) VALUES (1,'a',true), (2,'b',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'), (1,'g'), (2,'f');
INSERT INTO entries (feed_id, guid, title, published, first_seen) VALUES
('f','x','Monaco qualifying',{t},{t}),
('g','y','Bread',{t},{t}),
('g','old','Silverstone 1998',100,100),
('h','z','Not subscribed',{t},{t});"
))
.await
.unwrap();
let words = |w: &[&str]| w.iter().map(|s| s.to_string()).collect::<Vec<_>>();
let f1 = db.add_signal(1, "F1", &words(&["formula 1"]), None).await.unwrap();
let food = db.add_signal(1, "Food", &words(&["baking"]), Some("Bread and cakes")).await.unwrap();
db.add_signal(2, "Theirs", &words(&["x"]), None).await.unwrap();
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "nobody has a key yet");
let everyone = db.unjudged(&[], 50, true, 30).await.unwrap();
let mut seen: Vec<_> = everyone.iter().map(|u| (u.user_id, u.guid.as_str(), u.key.is_none())).collect();
seen.sort();
assert_eq!(
seen,
[(1, "x", true), (1, "y", true), (2, "x", true)],
"on the server's key, everyone's Signals, no key of their own needed"
);
db.set_typesafe_key(1, Some("k")).await.unwrap();
let queue = db.unjudged(&[], 50, false, 30).await.unwrap();
let mut seen: Vec<_> = queue.iter().map(|u| (u.guid.as_str(), u.signals.len())).collect();
seen.sort();
assert_eq!(
seen,
[("x", 2), ("y", 2)],
"each recent subscribed item once, with both Signals; old and unsubscribed left out"
);
let ask = queue[0].signals.iter().find(|a| a.id == food).unwrap();
assert_eq!(
(ask.name.as_str(), ask.description.as_deref()),
("Food", Some("Bread and cakes")),
"Jev is asked about the name and the sentence"
);
assert!(db.unjudged(&[1], 50, false, 30).await.unwrap().is_empty(), "a refused key is passed over");
db.record_verdicts("f", "x", &[(f1, true), (food, false)]).await.unwrap();
db.record_verdicts("g", "y", &[(f1, false), (food, true)]).await.unwrap();
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "nothing is asked twice");
let order = order_sql("published", "desc");
let page = db.signal_entries(1, f1, Filter::All, None, 0, 50, &order).await.unwrap();
assert_eq!(page.iter().map(|e| e.guid.as_str()).collect::<Vec<_>>(), ["x"]);
assert_eq!(db.signal_count(1, food, Filter::All, None).await.unwrap(), 1);
assert_eq!(
db.signals(1).await.unwrap().iter().map(|s| (s.items, s.unread)).collect::<Vec<_>>(),
[(1, 1), (1, 1)]
);
db.exec_for_test(
"INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES (1,'f','x',true,false,0)",
)
.await
.unwrap();
let f = db.signal(1, f1).await.unwrap().unwrap();
assert_eq!((f.items, f.unread), (1, 0), "read in its feed is read in the Signal, and still one of its items");
assert!(db.signal(2, f1).await.unwrap().is_none(), "someone else's Signal is not found");
assert!(!db.set_signal(2, f1, SignalChange { name: Some("mine"), ..Default::default() }).await.unwrap());
assert!(!db.remove_signal(2, f1).await.unwrap());
// An edit asks again only what was picked out, however old, and leaves what was passed
// over (#276). Until answered, what was picked out stays in the list.
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { description: Some(Some("Grand prix racing")), ..Default::default() })
.await
.unwrap();
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().description.as_deref(), Some("Grand prix racing"));
let again: Vec<_> = db
.unjudged(&[], 50, false, 30)
.await
.unwrap()
.iter()
.map(|u| (u.guid.clone(), u.signals.iter().map(|a| a.id).collect::<Vec<_>>()))
.collect();
assert_eq!(
again,
[("x".to_string(), vec![f1]), ("old".to_string(), vec![f1])],
"y was passed over, and stays so"
);
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2);
db.record_verdicts("f", "x", &[(f1, false)]).await.unwrap();
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "answered, not asked again");
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 1);
assert!(db.unjudged(&[], 50, false, 1).await.unwrap().is_empty());
db.record_verdicts("f", "x", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { icon: Some(Some("solid/flag-checkered")), ..Default::default() })
.await
.unwrap();
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2, "x and old, kept");
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "a new icon asks nothing again");
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().icon.as_deref(), Some("solid/flag-checkered"));
assert!(db.remove_signal(1, food).await.unwrap());
db.record_verdicts("f", "x", &[(food, true)]).await.unwrap();
assert_eq!(db.signals(1).await.unwrap().len(), 1, "a verdict for a removed Signal is not kept");
}
#[tokio::test]
async fn block_lists_hide_items_from_whoever_keeps_them() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'a',true), (2,'b',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'), (2,'f');
INSERT INTO entries (feed_id, guid, title, description, first_seen) VALUES
('f','x','Election night','',1),
('f','y','Baking bread','<p>No politics here, honest</p>',2),
('f','z','Gardening','',3);",
)
.await
.unwrap();
let shown = async |user| db.count_in(user, Some("f"), Filter::All, None).await.unwrap();
let words = |w: &[&str]| w.iter().map(|s| s.to_string()).collect::<Vec<_>>();
db.set_blocklist(1, "", &words(&["election"])).await.unwrap();
db.set_blocklist(1, "f", &words(&["politics"])).await.unwrap();
assert_eq!(shown(1).await, 1, "a title and a body each hide an item, both lists apply");
assert_eq!(db.unread_count(1, "f").await.unwrap(), 1, "hidden is not unread");
assert_eq!(shown(2).await, 3, "someone else's list hides nothing from you");
db.set_blocklist(1, "", &[]).await.unwrap();
assert_eq!(shown(1).await, 2, "emptying a list brings its items back");
assert!(db.blocklist(1, "").await.unwrap().is_empty());
// Someone subscribing starts with their list applied, and the scanner sees it.
db.set_blocklist(2, "", &words(&["gardening"])).await.unwrap();
db.exec_for_test("INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('g','w','Gardening',4)")
.await
.unwrap();
db.subscribe(2, "g").await.unwrap();
assert_eq!(db.count_in(2, Some("g"), Filter::All, None).await.unwrap(), 0);
assert_eq!(db.subscribers("g", None).await.unwrap()[0].blocked, ["gardening"]);
}
#[tokio::test]
async fn an_opml_lists_its_shows_items_together_for_whoever_subscribes() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'a',true), (2,'b',false);
INSERT INTO feeds (id, url, group_id, managed) VALUES ('a','u1','o',true), ('b','u2','o',true), ('c','u3',NULL,false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'o'), (1,'a'), (1,'b'), (1,'c'), (2,'a');
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('a','1','A one',1), ('b','2','B two',2), ('c','3','C three',3);",
)
.await
.unwrap();
let mut shows = db.group_members(1, "o").await.unwrap();
shows.sort();
assert_eq!(shows, ["a", "b"], "the OPML's shows, not a feed outside it");
assert_eq!(db.group_members(2, "o").await.unwrap(), ["a"], "only the shows this person has");
let ids = ["a", "b", "o"];
let order = order_sql("published", "desc");
let rows = db.entries_of(1, Some(&ids), Filter::All, None, 0, 50, &order).await.unwrap();
let titles: Vec<_> = rows.iter().map(|r| r.title.clone().unwrap()).collect();
assert_eq!(titles, ["B two", "A one"], "both shows' items, newest first");
assert_eq!(db.count_of(1, Some(&ids), Filter::All, None).await.unwrap(), 2);
assert_eq!(db.count_of(1, Some(&[]), Filter::All, None).await.unwrap(), 0);
}
#[tokio::test]
async fn all_subscriptions_leaves_out_what_was_hidden_from_it() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'a',true), (2,'b',false);
INSERT INTO feeds (id, url, group_id, managed) VALUES ('a','u1','o',true), ('b','u2','o',true), ('c','u3',NULL,false), ('d','u4',NULL,false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'o'), (1,'a'), (1,'b'), (1,'c'), (1,'d'), (2,'a'), (2,'o');
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('a','1','A one',1), ('b','2','B two',2), ('c','3','C three',3), ('d','4','D four',4);
INSERT INTO entry_state (user_id, feed_id, guid, position) VALUES (1,'a','1',60);",
)
.await
.unwrap();
let order = order_sql("published", "desc");
let all = async |user, filter| {
let rows = db.entries_of(user, None, filter, None, 0, 50, &order).await.unwrap();
let n = db.count_of(user, None, filter, None).await.unwrap();
assert_eq!(rows.len() as i64, n, "the count is of the same list");
rows.into_iter().map(|r| r.title.unwrap()).collect::<Vec<_>>()
};
assert!(db.set_hide_from_all(1, "o", true).await.unwrap());
assert!(db.set_hide_from_all(1, "c", true).await.unwrap());
assert!(!db.set_hide_from_all(1, "x", true).await.unwrap(), "not subscribed");
assert_eq!(all(1, Filter::All).await, ["D four"], "the OPML's shows and the hidden feed left out");
assert_eq!(all(1, Filter::InProgress).await, ["A one"], "an episode under way still shows");
assert_eq!(all(2, Filter::All).await, ["A one"], "someone else's All Subscriptions is their own");
assert_eq!(db.count_of(1, Some(&["c"]), Filter::All, None).await.unwrap(), 1, "its own page still lists it");
db.set_hide_from_all(1, "o", false).await.unwrap();
assert_eq!(all(1, Filter::All).await, ["D four", "B two", "A one"]);
}
#[tokio::test]
async fn a_feed_kept_out_of_signals_is_neither_judged_nor_listed() {
let db = Db::memory().await.unwrap();
let now = now();
db.exec_for_test(&format!(
"INSERT INTO users (id, name, is_admin, typesafe_key) VALUES (1,'a',true,'k');
INSERT INTO feeds (id, url, group_id, managed) VALUES ('a','u1','o',true), ('c','u3',NULL,false), ('d','u4',NULL,false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'o'), (1,'a'), (1,'c'), (1,'d');
INSERT INTO signals (id, user_id, name, keywords, created) VALUES (9, 1, 'S', '[\"x\"]', {now});
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('a','1','A one',{now}), ('c','3','C three',{now}), ('d','4','D four',{now});
INSERT INTO signal_verdicts (signal_id, feed_id, guid, hit, recheck) VALUES (9,'a','1',true,false), (9,'c','3',true,false);"
))
.await
.unwrap();
let judged = async || {
let mut f: Vec<_> = db.unjudged(&[], 50, false, 30).await.unwrap().into_iter().map(|u| u.feed_id).collect();
f.sort();
f
};
let order = order_sql("published", "desc");
let listed = async || {
let rows = db.signal_entries(1, 9, Filter::All, None, 0, 50, &order).await.unwrap();
let n = db.signal_count(1, 9, Filter::All, None).await.unwrap();
assert_eq!(rows.len() as i64, n, "the count is of the same list");
assert_eq!(db.signal(1, 9).await.unwrap().unwrap().items, n, "and the Signal's own count");
rows.into_iter().map(|r| r.title.unwrap()).collect::<Vec<_>>()
};
assert_eq!(judged().await, ["d"]);
assert_eq!(listed().await, ["C three", "A one"]);
assert!(db.set_hide_from_signals(1, "o", true).await.unwrap());
assert!(db.set_hide_from_signals(1, "d", true).await.unwrap());
assert_eq!(judged().await, Vec::<String>::new(), "Jev is not asked about a hidden feed");
assert_eq!(listed().await, ["C three"], "the OPML's show left out with it");
assert_eq!(db.count_of(1, None, Filter::All, None).await.unwrap(), 3, "All Subscriptions keeps them");
}
#[tokio::test]
async fn pending_takes_the_latest_episodes_first() {
// A cap of 3 must mean the three newest, not the three recorded first.
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, published, first_seen) VALUES
('f','old',100,100), ('f','mid',200,200), ('f','new',300,300);
INSERT INTO enclosures (id, feed_id, guid, url, state) VALUES
(1,'f','old','u-old','pending'),
(2,'f','mid','u-mid','pending'),
(3,'f','new','u-new','pending');",
)
.await
.unwrap();
let got: Vec<String> = db.pending("f", 2).await.unwrap().into_iter().map(|p| p.url).collect();
assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later");
}
#[tokio::test]
async fn a_restart_requeues_interrupted_downloads() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
(1,'f','a','u1','downloading',NULL),
(2,'f','b','u2','pending',NULL),
(3,'f','c','u3','downloading','/tmp/already-here'),
(4,'f','d','u4','done','/tmp/x');",
)
.await
.unwrap();
assert_eq!(db.requeue_interrupted().await.unwrap(), 1, "only the in-flight, fileless one");
let state = async |id: i64| db.strings_for_test(&format!("SELECT state FROM enclosures WHERE id = {id}")).await;
assert_eq!(state(1).await, ["pending"]);
assert_eq!(state(3).await, ["downloading"], "it has a file; leave it alone");
assert_eq!(state(4).await, ["done"]);
}
#[tokio::test]
async fn a_show_takes_over_what_its_creator_held() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES
(1,'creator','a','u1','done','/x/a.mp3',NULL),
(2,'creator','b','u2','skipped',NULL,'explicit'),
(3,'creator','c','u3','skipped',NULL,'explicit'),
(4,'other','d','u4','skipped',NULL,'torrents disabled');
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',true);",
)
.await
.unwrap();
db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).await.unwrap();
let owner =
async |id: i64| db.strings_for_test(&format!("SELECT feed_id FROM enclosures WHERE id = {id}")).await;
assert_eq!(owner(1).await, ["show"], "a downloaded file moves with its item");
assert_eq!(owner(2).await, ["show"]);
assert_eq!(owner(3).await, ["creator"], "this show does not list it");
assert_eq!(owner(4).await, ["other"], "only the parent's are taken");
assert_eq!(
db.strings_for_test("SELECT feed_id FROM entry_state WHERE user_id = 1 AND guid = 'a'").await,
["show"],
"what you had read stays read"
);
// Only a filter's verdict can be overturned by a change of settings.
let skipped = db.skipped_by_filter("show").await.unwrap();
assert_eq!(skipped.get("u2").map(String::as_str), Some("explicit"));
assert_eq!(skipped.len(), 1);
assert!(db.skipped_by_filter("other").await.unwrap().is_empty(), "torrents disabled is not a filter");
}
#[tokio::test]
async fn a_feed_in_a_group_follows_your_settings_on_the_group() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES
(1,'group',true),(1,'show',NULL),(2,'group',true),(2,'show',false);",
)
.await
.unwrap();
let explicit = async |group| -> Vec<Option<bool>> {
let mut v: Vec<_> =
db.subscribers("show", group).await.unwrap().into_iter().map(|s| s.allow_explicit).collect();
v.sort();
v
};
assert_eq!(
explicit(Some("group")).await,
[Some(false), Some(true)],
"ray inherits; sam's own choice on the show wins"
);
assert_eq!(explicit(None).await, [None, Some(false)], "outside a group nothing is inherited");
}
#[tokio::test]
async fn error_since_marks_the_start_of_a_run_of_failures_and_clears_on_success() {
let db = Db::memory().await.unwrap();
db.set_feed_error("f", "http://x", "HTTP 404").await.unwrap();
// Backdate it, as if this feed had already been failing a while, so a second
// failure landing "now" is distinguishable from the first.
db.exec_for_test("UPDATE feeds SET error_since = error_since - 3600 WHERE id = 'f'").await.unwrap();
let first = db.feed_summary("f").await.unwrap().error_since.unwrap();
// macmanx: failed once, read fine an hour later. A second failure must not push
// error_since forward -- the UI decides "failing for a day" from the first one.
db.set_feed_error("f", "http://x", "HTTP 404").await.unwrap();
assert_eq!(db.feed_summary("f").await.unwrap().error_since, Some(first));
db.touch_feed("f", "http://x").await.unwrap();
let after = db.feed_summary("f").await.unwrap();
assert_eq!(after.last_error, None);
assert_eq!(after.error_since, None, "a clean check ends the run of failures");
}
#[tokio::test]
async fn a_pin_survives_saving_the_feeds_settings_and_needs_a_subscription() {
let db = Db::memory().await.unwrap();
let me = db.create_user("pat", None, false).await.unwrap();
assert!(!db.set_pinned(me, "f", true).await.unwrap(), "not subscribed: nothing to pin");
db.subscribe(me, "f").await.unwrap();
assert!(db.set_pinned(me, "f", true).await.unwrap());
// set_subscription writes the rest of the row; it must leave the pin alone.
db.set_subscription(me, &Sub { feed_id: "f".into(), auto_download: Some(false), ..Default::default() })
.await
.unwrap();
assert!(db.pinned_feeds(me).await.unwrap().contains("f"));
db.set_pinned(me, "f", false).await.unwrap();
assert!(db.pinned_feeds(me).await.unwrap().is_empty());
}
#[test]
fn postgres_is_asked_for_no_notices_and_a_password_never_reaches_a_log() {
assert_eq!(url_for("postgres://u:p@h/d"), format!("postgres://u:p@h/d?{QUIET}"));
assert_eq!(
url_for("postgres://u:p@h/d?sslmode=disable"),
format!("postgres://u:p@h/d?sslmode=disable&{QUIET}")
);
assert_eq!(
url_for("postgres://u:p@h/d?options=-c%20x%3Dy"),
"postgres://u:p@h/d?options=-c%20x%3Dy",
"theirs stands"
);
assert_eq!(url_for("/data/state.db"), "sqlite:///data/state.db?mode=rwc");
assert_eq!(redact("postgres://ipodderx:s3cret@h:5433/ipodderx"), "postgres://ipodderx:***@h:5433/ipodderx");
}
#[tokio::test]
async fn the_configuration_goes_in_once_and_comes_back_as_it_went() {
let db = Db::memory().await.unwrap();
assert!(db.stored_config().await.unwrap().is_none(), "nothing until it is taken in");
let mut cfg: crate::config::Config = toml::from_str(
r#"
[general]
schedule = "every 2h"
max_total_gb = 1.5
[feeds.a]
url = "http://x/a.xml"
keywords = ["one", "two"]
[feeds.b]
url = "http://x/b.xml"
password = "secret"
"#,
)
.unwrap();
assert!(db.import_config(&cfg).await.unwrap());
assert!(!db.import_config(&cfg).await.unwrap(), "a second import finds the first and stands back");
let (stored, feeds) = db.stored_config().await.unwrap().unwrap();
assert_eq!(stored, crate::config::Stored::of(&cfg));
assert_eq!(feeds.keys().collect::<Vec<_>>(), ["a", "b"]);
assert_eq!(feeds["a"].keywords, ["one", "two"]);
assert_eq!(feeds["b"].password.as_deref(), Some("secret"));
// Storing is the whole catalogue: a feed taken out goes, one put in arrives.
cfg.feeds.remove("a");
cfg.feeds.insert("c".into(), cfg.feeds["b"].clone());
cfg.general.schedule = "every 30m".into();
db.store_config(&cfg).await.unwrap();
let (stored, feeds) = db.stored_config().await.unwrap().unwrap();
assert_eq!(stored.schedule, "every 30m");
assert_eq!(feeds.keys().collect::<Vec<_>>(), ["b", "c"]);
}
#[tokio::test]
async fn enclosure_url_is_the_dedupe_key() {
let db = Db::memory().await.unwrap();
let insert =
"INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')";
db.exec_for_test(insert).await.unwrap();
assert!(db.exec_for_test(insert).await.is_err(), "duplicate url must be rejected");
}
}
Reference in New Issue View Git Blame Copy Permalink