rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SeaORM: feeds and scanning; rusqlite gone

Browse Source 
The last nineteen functions move to SeaORM: recording feeds, items and
enclosures, managed OPML feeds, folding WordPress's repeated files, and handing a
Patreon creator's files to its shows. Two SQLite-only forms go: GLOB becomes a
LIKE with the underscore escaped (broader, harmlessly: the fold still keys on
`_=` and digits), and UPDATE OR IGNORE becomes an UPDATE ... WHERE NOT EXISTS.
The two transactions are SeaORM transactions.

With nothing left on it, rusqlite goes, with the SQL schema and migrate(). The
entities are the schema: create_missing makes whatever tables and indexes a
database lacks, from them, with CREATE ... IF NOT EXISTS. Production's schema
already has every column migrate() added and none it dropped.

Not SeaORM's schema sync, used until now: despite its docs it drops a unique
index the entities do not describe, so it dropped users_name_lower on every open.
Every `ipx` command then took a write lock, and against a daemon busy writing,
`ipx status` -- the healthcheck -- failed 7 times in 15 where the old code
failed none. Now 15 in 15, as before. On Postgres it would not have started.

WAL is set only when a file is not already in it: setting it takes a lock that
cannot wait out a busy daemon.

Checked on copies of production: a forced scan of all 162 feeds against the real
feeds with no database errors; the feed list, filters, sorts, search and the
reaper's candidates against the old code on the same data, earlier in the
branch. The column comments from the SQL schema move to the entities.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-18 19:10:59 +00:00
parent a68bfb179b
commit 611716d8b7
8 changed files with 485 additions and 733 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- The database is reached through SeaORM, on the way to Postgres (issue #18); it is still the
same SQLite file, and nothing you see changes. A database from before 0.7 has to be opened by a
0.7 release first, which brings its tables up to date.
- A pinned item sits at the top of its list, above everything else in whatever order you sort
by, and moves there the moment you pin it. Sorting by the pin column itself still goes both
ways, and Currently Listening keeps its own order.

50
Cargo.lock generated
View File

@@ -1129,18 +1129,6 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -1848,7 +1836,6 @@ dependencies = [
"quick-xml 0.42.0",
"reqwest",
"rss",
"rusqlite",
"sea-orm",
"serde",
"serde_json",
@@ -3289,16 +3276,6 @@ dependencies = [
"libc",
]
[[package]]
name = "rsqlite-vfs"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
dependencies = [
"hashbrown 0.16.1",
"thiserror 2.0.20",
]
[[package]]
name = "rss"
version = "2.1.1"
@@ -3310,21 +3287,6 @@ dependencies = [
"quick-xml 0.41.0",
]
[[package]]
name = "rusqlite"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e"
dependencies = [
"bitflags 2.13.1",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
"sqlite-wasm-rs",
]
[[package]]
name = "rust_decimal"
version = "1.43.0"
@@ -3905,18 +3867,6 @@ dependencies = [
"lock_api",
]
[[package]]
name = "sqlite-wasm-rs"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
dependencies = [
"cc",
"js-sys",
"rsqlite-vfs",
"wasm-bindgen",
]
[[package]]
name = "sqlx"
version = "0.9.0"

3
Cargo.toml
View File

@@ -18,8 +18,7 @@ percent-encoding = "2.3.2"
quick-xml = { version = "0.42.0", features = ["escape-html"] }
reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "http2", "gzip", "stream", "json", "charset", "system-proxy"] }
rss = "2.1.1"
rusqlite = { version = "0.39", features = ["bundled"] }
sea-orm = { version = "~2.0.3", default-features = false, features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "schema-sync", "sqlite-use-returning-for-3_35"] }
sea-orm = { version = "2.0.3", default-features = false, features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "sqlite-use-returning-for-3_35"] }
serde = { version = "1.0.229", features = ["derive"] }
serde_json = "1.0.151"
tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "sync", "time", "signal"] }

964
src/db.rs
View File

File diff suppressed because it is too large Load Diff

28
src/entity.rs
View File

@@ -1,6 +1,6 @@
//! The database's tables as SeaORM entities: the one description of the schema, from which
//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see `db::sync`).
//! What each column means is in the comments on `db::SCHEMA`, the SQLite schema these match.
//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see
//! `db::create_missing`). Times are Unix seconds.
pub mod feeds {
use sea_orm::entity::prelude::*;
@@ -16,6 +16,7 @@ pub mod feeds {
pub title: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub image: Option<String>,
/// The channel's first <itunes:category>, for the Directory.
#[sea_orm(column_type = "Text", nullable)]
pub category: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
@@ -26,11 +27,19 @@ pub mod feeds {
pub ttl_mins: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub last_error: Option<String>,
/// When the current run of failures began; NULL while the feed is healthy. Kept through
/// repeated failures so the UI can tell a blip (macmanx: failed once, fine an hour
/// later) from a feed that has been down for a day.
pub error_since: Option<i64>,
/// Came from a subscribed OPML that no longer lists it, but has downloads, so kept.
#[sea_orm(default_value = false)]
pub orphaned: bool,
/// The OPML subscription this feed came from.
#[sea_orm(column_type = "Text", nullable)]
pub group_id: Option<String>,
/// Derived from an OPML and not written to config.toml. Writing 80-odd generated entries
/// into a hand-edited file made it unreadable; the OPML is the source of truth, so they
/// are re-derived instead. Customising one promotes it to config.
#[sea_orm(default_value = false)]
pub managed: bool,
}
@@ -84,7 +93,8 @@ pub mod enclosures {
pub feed_id: String,
#[sea_orm(column_type = "Text")]
pub guid: String,
/// The dedupe key: one file serves every subscriber.
/// The dedupe key, and the reason one file serves every subscriber. A reaped file keeps
/// its row with path NULL and state 'reaped', so a purged episode is never fetched again.
#[sea_orm(unique, column_type = "Text")]
pub url: String,
#[sea_orm(column_type = "Text", nullable)]
@@ -115,16 +125,20 @@ pub mod users {
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
/// Unique without regard to case: `db::sync` adds the index on lower(name), which
/// Unique without regard to case: `db::create_missing` adds the index on lower(name), which
/// works the same on both databases where SQLite's COLLATE NOCASE does not.
#[sea_orm(column_type = "Text")]
pub name: String,
/// NULL for someone who only ever arrives through the proxy: there is no password to
/// check, and leaving it empty is not the same as leaving it unset.
#[sea_orm(column_type = "Text", nullable)]
pub pass_hash: Option<String>,
#[sea_orm(default_value = false)]
pub is_admin: bool,
/// For whoever maintains the server. NULL where it is not known.
pub created: Option<i64>,
pub last_login: Option<i64>,
/// The theme chosen in Settings, and light, dark or auto. NULL until one is chosen.
#[sea_orm(column_type = "Text", nullable)]
pub theme: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
@@ -161,6 +175,8 @@ macro_rules! owned_by_user {
};
}
/// What one person wants from a feed. The feed, its items and its files are shared; this is the
/// part that is not. NULL in a column means: follow the feed's own setting.
pub mod subscriptions {
use sea_orm::entity::prelude::*;
@@ -177,6 +193,7 @@ pub mod subscriptions {
pub auto_download: Option<bool>,
pub allow_explicit: Option<bool>,
pub max_new_per_check: Option<i64>,
/// Pinned to the top of this person's feed list, a feed inside a folder included.
#[sea_orm(default_value = false)]
pub pinned: bool,
}
@@ -184,6 +201,8 @@ pub mod subscriptions {
owned_by_user!();
}
/// Read, kept and how far in. One row per person per item, created on first touch; an item
/// nobody has touched has no row at all, which is what unread means.
pub mod entry_state {
use sea_orm::entity::prelude::*;
@@ -202,6 +221,7 @@ pub mod entry_state {
pub flagged: bool,
#[sea_orm(default_value = 0)]
pub position: i64,
/// The length this person's player measured, beside the position it is measured against.
pub duration: Option<i64>,
}

134
src/main.rs
View File

@@ -220,7 +220,7 @@ async fn main() -> Result<()> {
});
match cli.command {
Command::List => list(&ctx, &config_path),
Command::List => list(&ctx, &config_path).await,
Command::Daemon { web } => daemon(ctx, config_path, web, events).await,
Command::Add { url, folder, keywords } => {
add(&ctx, &config_path, &url, folder, keywords).await
@@ -228,7 +228,7 @@ async fn main() -> Result<()> {
Command::Rm { feed } => rm(&ctx, &config_path, &feed).await,
Command::User { cmd } => user_cmd(&ctx, cmd).await,
Command::Import { file } => import(&ctx, &config_path, &file).await,
Command::Export { file } => export(&ctx, &file),
Command::Export { file } => export(&ctx, &file).await,
_ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await,
}
}
@@ -351,7 +351,7 @@ async fn run(ctx: &Arc<Ctx>, cmd: Cmd) -> Result<()> {
async fn status(ctx: &Ctx) -> Event {
match ctx.db.counts().await {
Ok((pending, downloaded)) => {
let feeds = subscriptions(ctx).map(|s| s.len()).unwrap_or(0);
let feeds = subscriptions(ctx).await.map(|s| s.len()).unwrap_or(0);
Event::Status { feeds, pending, downloaded }
}
Err(e) => Event::Error { msg: format!("{e:#}") },
@@ -401,7 +401,7 @@ async fn daemon(
// Before the parser knew WordPress's numbered player URLs, a file it listed twice was
// downloaded twice. The repeats fold into the first, and their spare copies are deleted.
match ctx.db.merge_repeated_enclosures(feed::same_file_key) {
match ctx.db.merge_repeated_enclosures(feed::same_file_key).await {
Ok((0, _)) => {}
Ok((n, spare)) => {
for path in &spare {
@@ -431,7 +431,7 @@ async fn daemon(
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60));
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
tracing::info!(
feeds = subscriptions(&ctx).map(|s| s.len()).unwrap_or(0),
feeds = subscriptions(&ctx).await.map(|s| s.len()).unwrap_or(0),
"daemon started"
);
@@ -575,7 +575,7 @@ async fn add(
let mut cfg = (*ctx.cfg()).clone();
let url = &feed::expand_input(url);
// Includes feeds derived from an OPML, or the same show could be added twice.
if let Some(existing) = subscriptions(ctx)?.iter().find(|s| feed::same_feed(&s.cfg.url, url)) {
if let Some(existing) = subscriptions(ctx).await?.iter().find(|s| feed::same_feed(&s.cfg.url, url)) {
anyhow::bail!("already subscribed as {:?}", existing.id);
}
let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
@@ -627,13 +627,13 @@ pub async fn add_one(
// Slugs must be unique across derived feeds too, or a new feed can collide with one
// an OPML already introduced.
let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx)?
let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx).await?
.into_iter()
.map(|s| (s.id, s.cfg))
.collect();
// A removed feed keeps its rows, so its id is only free again for the same feed: re-adding
// it gets its history back, and a different feed does not inherit someone else's.
for (id, other) in ctx.db.feed_urls()? {
for (id, other) in ctx.db.feed_urls().await? {
if !feed::same_feed(&other, url) {
taken.entry(id).or_insert_with(|| probe.clone());
}
@@ -656,7 +656,7 @@ async fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()>
if cfg.feeds.remove(feed).is_none() {
// Derived from an OPML: drop it here, though the subscription will list it again
// on the next read unless the OPML itself goes.
ctx.db.drop_managed(feed)?;
ctx.db.drop_managed(feed).await?;
println!("removed {feed}; it came from an OPML subscription and may return on the next read");
return Ok(());
}
@@ -703,7 +703,7 @@ pub async fn subscribe_opml(
let mut found = vec![];
collect_outlines(&doc.body.outlines, &mut found);
let known = subscriptions(ctx)?;
let known = subscriptions(ctx).await?;
let mut cfg = (*ctx.cfg()).clone();
let mut ids = vec![];
let mut grew = false;
@@ -770,7 +770,7 @@ pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, Strin
}
}
fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
let mut doc = opml::OPML::default();
doc.head = Some(opml::Head {
title: Some("ipx subscriptions".into()),
@@ -779,7 +779,7 @@ fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
for (id, feed) in &ctx.cfg().feeds {
let title = ctx
.db
.feed_summary(id)
.feed_summary(id).await
.ok()
.and_then(|s| s.title)
.unwrap_or_else(|| id.clone());
@@ -791,14 +791,14 @@ fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
Ok(())
}
fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> {
async fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> {
let cfg = ctx.cfg();
if cfg.feeds.is_empty() {
println!("No feeds configured in {}", config_path.display());
return Ok(());
}
for (id, feed) in &cfg.feeds {
let s = ctx.db.feed_summary(id)?;
let s = ctx.db.feed_summary(id).await?;
println!("{id} {}", s.title.as_deref().unwrap_or("-"));
println!(" url {}", feed.url);
println!(" last checked {}", ago(s.last_checked));
@@ -832,7 +832,7 @@ async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
let cfg = ctx.cfg();
let subs = subscriptions(ctx)?;
let subs = subscriptions(ctx).await?;
if let Some(id) = only
&& !subs.iter().any(|s| s.id == id)
{
@@ -843,7 +843,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
let mut fresh: Vec<String> = vec![];
for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) {
let (id, feed_cfg) = (&sub.id, &sub.cfg);
let state = ctx.db.http_state(id)?;
let state = ctx.db.http_state(id).await?;
if !force && let Some(last) = state.last_checked {
let due = last + due_after(&cfg, feed_cfg, state.ttl_mins) as i64;
@@ -890,20 +890,20 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
// One bad feed must not end the scan.
let msg = format!("{e:#}");
ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() });
ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?;
ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?;
}
}
}
// Feeds a subscribed OPML just introduced: scan them now, in this run.
if !fresh.is_empty() {
let subs = subscriptions(ctx)?;
let subs = subscriptions(ctx).await?;
for id in &fresh {
let Some(feed_cfg) = subs.iter().find(|s| &s.id == id).map(|s| &s.cfg) else {
continue;
};
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
let state = ctx.db.http_state(id)?;
let state = ctx.db.http_state(id).await?;
match scan_one(ctx, id, feed_cfg, &state).await {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
@@ -916,7 +916,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
Err(e) => {
let msg = format!("{e:#}");
ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() });
ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?;
ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?;
}
}
}
@@ -938,7 +938,7 @@ pub struct Sub {
///
/// A derived feed borrows its parent's settings wholesale. That is why it needs no config
/// entry -- there is nothing to store but its URL and where it came from.
pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
let cfg = ctx.cfg();
let mut out: Vec<Sub> = cfg
.feeds
@@ -946,7 +946,7 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
.map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false })
.collect();
for m in ctx.db.managed_feeds()? {
for m in ctx.db.managed_feeds().await? {
if cfg.feeds.contains_key(&m.id) {
continue; // promoted to config at some point; that entry wins
}
@@ -957,10 +957,10 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
// skip them here regardless so a row that slips through is never scanned.
continue;
}
let base = parent
.and_then(|p| p.folder.clone())
.or_else(|| ctx.db.feed_summary(&m.group_id).ok().and_then(|s| s.title))
.unwrap_or_else(|| m.group_id.clone());
let base = match parent.and_then(|p| p.folder.clone()) {
Some(folder) => folder,
None => ctx.db.feed_summary(&m.group_id).await.ok().and_then(|s| s.title).unwrap_or_else(|| m.group_id.clone()),
};
let title = m.title.clone().unwrap_or_else(|| m.id.clone());
out.push(Sub {
id: m.id.clone(),
@@ -994,15 +994,15 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
/// derived any more, so it is only unmanaged.
pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
let cfg = ctx.cfg();
for m in ctx.db.managed_feeds()?.into_iter().filter(|m| m.group_id == parent_id) {
for m in ctx.db.managed_feeds().await?.into_iter().filter(|m| m.group_id == parent_id) {
if cfg.feeds.contains_key(&m.id) {
// Scanned from its config entry and still read. Dropped as derived, its stored
// entries would go with it: davewiner's 11 were promoted without being unmanaged.
ctx.db.unmanage(&m.id)?;
ctx.db.unmanage(&m.id).await?;
} else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
ctx.db.set_orphaned(&m.id, true)?;
ctx.db.set_orphaned(&m.id, true).await?;
} else {
ctx.db.drop_managed(&m.id)?;
ctx.db.drop_managed(&m.id).await?;
}
}
Ok(())
@@ -1014,7 +1014,7 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
/// most of the ones stored.
async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
let cfg = ctx.cfg();
let before = ctx.db.managed_feeds()?;
let before = ctx.db.managed_feeds().await?;
let stranded: std::collections::BTreeSet<&str> = before
.iter()
.map(|m| m.group_id.as_str())
@@ -1023,7 +1023,7 @@ async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
for group in stranded {
retire_group(ctx, group).await?;
}
Ok(before.len() - ctx.db.managed_feeds()?.len())
Ok(before.len() - ctx.db.managed_feeds().await?.len())
}
/// Seconds to wait before re-checking a feed.
@@ -1068,19 +1068,19 @@ async fn scan_one(
if feed::is_patreon_creator(&feed_cfg.url) {
match feed::patreon_shows(&ctx.client, &feed_cfg.url).await {
Ok((name, shows)) if shows.len() > 1 => {
ctx.db.touch_feed(id, &feed_cfg.url)?;
ctx.db.touch_feed(id, &feed_cfg.url).await?;
if let Some(name) = name {
ctx.db.set_title(id, &name)?;
ctx.db.set_title(id, &name).await?;
}
// Read as one feed before it was split, it listed every show's items in one
// heap. The items go; its files and read state move to each show as the show
// lists them (`Db::adopt`), so no show comes up empty for want of a URL.
ctx.db.clear_entries(id)?;
ctx.db.clear_entries(id).await?;
return sync_group(ctx, id, feed_cfg, &shows).await;
}
Ok(_) => {} // One show: the creator's feed is that show.
// Already split: keep the shows it has rather than read the creator as one heap.
Err(e) if ctx.db.managed_feeds()?.iter().any(|m| m.group_id == id) => return Err(e),
Err(e) if ctx.db.managed_feeds().await?.iter().any(|m| m.group_id == id) => return Err(e),
Err(e) => tracing::warn!(
feed = id,
error = %format!("{e:#}"),
@@ -1101,29 +1101,29 @@ async fn scan_one(
// from backup, a manual edit, a cleanup that removed entries. Believe the database over
// the validator: drop it and ask again, or the feed stays empty until the publisher
// happens to change something.
if matches!(fetched, feed::Fetched::NotModified) && ctx.db.feed_summary(id)?.entries == 0 {
if matches!(fetched, feed::Fetched::NotModified) && ctx.db.feed_summary(id).await?.entries == 0 {
tracing::info!(feed = id, "not modified, but nothing stored; refetching without the validator");
ctx.db.clear_validators(id)?;
ctx.db.clear_validators(id).await?;
fetched = feed::fetch(&ctx.client, feed_cfg, None, None).await?;
}
let (bytes, etag, last_modified) = match fetched {
feed::Fetched::NotModified => {
ctx.db.touch_feed(id, &feed_cfg.url)?;
ctx.db.touch_feed(id, &feed_cfg.url).await?;
return Ok(Outcome::NotModified);
}
feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified),
};
if bytes.iter().all(u8::is_ascii_whitespace) {
ctx.db.touch_feed(id, &feed_cfg.url)?;
ctx.db.touch_feed(id, &feed_cfg.url).await?;
return Ok(Outcome::Empty);
}
// A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml"
// URL; sniffing the body also catches one served from a URL without that extension.
if feed::is_opml(&bytes) {
ctx.db.touch_feed(id, &feed_cfg.url)?;
ctx.db.touch_feed(id, &feed_cfg.url).await?;
return sync_opml(ctx, id, feed_cfg, &bytes).await;
}
@@ -1137,7 +1137,7 @@ async fn scan_one(
parsed.ttl_mins,
parsed.image.as_deref(),
parsed.category.as_deref(),
)?;
).await?;
let policy = policy_for(ctx, id, feed_cfg).await?;
if let Some(parent) = &feed_cfg.group {
@@ -1146,16 +1146,16 @@ async fn scan_one(
.iter()
.flat_map(|e| e.enclosures.iter().map(move |x| (e.guid.as_str(), x.url.as_str())))
.collect();
ctx.db.adopt(parent, id, &listed)?;
ctx.db.adopt(parent, id, &listed).await?;
}
// Verdicts are recorded in `state`, so the download queue below is just "everything still
// pending". A filter's verdict is looked at again on every scan, though: made once, at
// discovery, it outlived the setting behind it, and allowing explicit items afterwards
// changed nothing however often the feed was scanned.
let skipped = ctx.db.skipped_by_filter(id)?;
let skipped = ctx.db.skipped_by_filter(id).await?;
let mut scan = Scan::default();
for entry in &parsed.entries {
if ctx.db.record_entry(id, entry)? {
if ctx.db.record_entry(id, entry).await? {
scan.new_entries += 1;
}
for enc in &entry.enclosures {
@@ -1263,7 +1263,7 @@ async fn sync_opml(
) -> Result<Outcome> {
let listed = feed::parse_opml(bytes)?;
if let Some(title) = feed::opml_title(bytes) {
ctx.db.set_title(parent_id, &title)?;
ctx.db.set_title(parent_id, &title).await?;
}
sync_group(ctx, parent_id, parent, &listed).await
}
@@ -1282,13 +1282,13 @@ async fn sync_group(
listed: &[(String, String)],
) -> Result<Outcome> {
let cfg = ctx.cfg();
let existing = ctx.db.managed_feeds()?;
let existing = ctx.db.managed_feeds().await?;
let mut added = vec![];
for (title, url) in listed {
// Already known, whether derived or promoted into the config.
if let Some(m) = existing.iter().find(|m| &m.url == url) {
ctx.db.upsert_managed(&m.id, url, title, parent_id)?;
ctx.db.upsert_managed(&m.id, url, title, parent_id).await?;
continue;
}
// A Patreon show you added by hand may be spelled differently from the one listed.
@@ -1296,7 +1296,7 @@ async fn sync_group(
continue;
}
// A removed feed keeps its rows, so its id is only free again for the same feed.
let known = ctx.db.feed_urls()?;
let known = ctx.db.feed_urls().await?;
let taken: std::collections::BTreeMap<String, config::Feed> = cfg
.feeds
.keys()
@@ -1306,7 +1306,7 @@ async fn sync_group(
.map(|id| (id.clone(), parent.clone()))
.collect();
let id = config::unique_slug(title, &taken);
ctx.db.upsert_managed(&id, url, title, parent_id)?;
ctx.db.upsert_managed(&id, url, title, parent_id).await?;
added.push(id);
}
@@ -1314,7 +1314,7 @@ async fn sync_group(
// subscription means. Their own feeds are untouched.
for id in ctx
.db
.managed_feeds()?
.managed_feeds().await?
.iter()
.filter(|m| m.group_id == parent_id)
.map(|m| m.id.clone())
@@ -1336,11 +1336,11 @@ async fn sync_group(
}
if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
// Never orphan a downloaded file: keep the feed and say why in the UI.
ctx.db.set_orphaned(&m.id, true)?;
ctx.db.set_orphaned(&m.id, true).await?;
kept += 1;
tracing::info!(feed = %m.id, "dropped from the OPML but has downloads; keeping it");
} else {
ctx.db.drop_managed(&m.id)?;
ctx.db.drop_managed(&m.id).await?;
removed += 1;
tracing::info!(feed = %m.id, "dropped from the OPML with nothing downloaded; removed");
}
@@ -1546,7 +1546,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
}
// Must look through the derived feeds too: anything inside an OPML subscription has
// no config entry, so a config-only lookup called every one of them "unsubscribed".
let subs = subscriptions(ctx)?;
let subs = subscriptions(ctx).await?;
let feed_cfg = subs
.iter()
.find(|s| s.id == enc.feed_id)
@@ -1554,7 +1554,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
.ok_or_else(|| anyhow::anyhow!("enclosure {id} belongs to unsubscribed feed {:?}", enc.feed_id))?;
let feed_cfg = &feed_cfg;
let title = ctx.db.feed_summary(&enc.feed_id)?.title;
let title = ctx.db.feed_summary(&enc.feed_id).await?.title;
let folder = download::folder_for(&cfg, &enc.feed_id, feed_cfg, title.as_deref());
let dest_dir = cfg.general.download_dir.join(&folder);
@@ -1743,9 +1743,9 @@ mod tests {
// davewiner: the OPML subscription left config.toml, but its 922 derived rows
// stayed in the database and kept being scanned under the no-parent fallback.
let ctx = test_ctx(config::Config::default()).await;
ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").unwrap();
ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").await.unwrap();
assert!(
subscriptions(&ctx).unwrap().iter().all(|s| s.id != "child"),
subscriptions(&ctx).await.unwrap().iter().all(|s| s.id != "child"),
"a derived feed whose parent is gone from config must not be scanned"
);
}
@@ -1753,18 +1753,18 @@ mod tests {
#[tokio::test]
async fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() {
let ctx = test_ctx(config::Config::default()).await;
ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap();
ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap();
ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").await.unwrap();
ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").await.unwrap();
let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None };
ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap();
ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap();
retire_group(&ctx, "parent").await.unwrap();
let managed = ctx.db.managed_feeds().unwrap();
let managed = ctx.db.managed_feeds().await.unwrap();
assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten");
assert!(managed.iter().any(|m| m.id == "has-file"), "has a file on disk, so it is kept");
assert!(ctx.db.feed_summary("has-file").unwrap().orphaned, "and flagged as orphaned");
assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned");
}
#[tokio::test]
@@ -1775,15 +1775,15 @@ mod tests {
cfg.feeds.insert("promoted".into(), feed());
cfg.feeds.insert("live-opml".into(), feed());
let ctx = test_ctx(cfg).await;
ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").unwrap();
ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").unwrap();
ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").unwrap();
ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).unwrap();
ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").await.unwrap();
ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").await.unwrap();
ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").await.unwrap();
ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).await.unwrap();
assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged");
let managed: Vec<String> = ctx.db.managed_feeds().unwrap().into_iter().map(|m| m.id).collect();
let managed: Vec<String> = ctx.db.managed_feeds().await.unwrap().into_iter().map(|m| m.id).collect();
assert_eq!(managed, ["listed"], "a group still in config is left alone");
assert_eq!(ctx.db.feed_summary("promoted").unwrap().entries, 1, "its entries survive");
assert_eq!(ctx.db.feed_summary("promoted").await.unwrap().entries, 1, "its entries survive");
}
}

8
src/retention.rs
View File

@@ -66,7 +66,7 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
report.bytes_freed += remove(db, c, dry_run).await?;
}
if !dry_run {
report.entries_pruned = db.prune_entries(cutoff)?;
report.entries_pruned = db.prune_entries(cutoff).await?;
}
}
@@ -173,7 +173,7 @@ mod tests {
(2, 'f', 'half', 'u2', '/tmp/half', 10, 'done', 20),
(3, 'f', 'unread', 'u3', '/tmp/unread', 10, 'done', 30),
(4, 'f', 'read', 'u4', '/tmp/read', 10, 'done', 40);",
)
).await
.unwrap();
let got: Vec<i64> = db.reap_candidates().await.unwrap().iter().map(|c| c.id).collect();
@@ -198,9 +198,9 @@ mod tests {
('f', 'recent', 900);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1, 'f', 'has-file', 'u1', '/tmp/x', 'done');",
)
).await
.unwrap();
assert_eq!(db.prune_entries(500).unwrap(), 1, "only the old, fileless, unflagged one");
assert_eq!(db.prune_entries(500).await.unwrap(), 1, "only the old, fileless, unflagged one");
}
}

28
src/web.rs
View File

@@ -672,7 +672,7 @@ async fn feeds(
let cfg = state.ctx.cfg();
// Config entries plus the feeds derived from OPML subscriptions -- the catalogue.
// What comes back is only the part of it this person subscribes to.
let subs = crate::subscriptions(&state.ctx)?;
let subs = crate::subscriptions(&state.ctx).await?;
let mine: std::collections::HashMap<String, crate::db::Sub> = state
.ctx
.db
@@ -689,8 +689,8 @@ async fn feeds(
// the same fallback the scanner uses (`Db::subscribers`).
let up = feed.group.as_deref().and_then(|g| mine.get(g));
let Some(mine) = mine.get(id) else { continue };
let s = state.ctx.db.feed_summary(id)?;
let st = state.ctx.db.http_state(id)?;
let s = state.ctx.db.feed_summary(id).await?;
let st = state.ctx.db.http_state(id).await?;
out.push(FeedRow {
id: id.clone(),
url: feed.url.clone(),
@@ -806,7 +806,7 @@ async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> {
db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
let counts = db.subscriber_counts().await?;
let media = db.media_feeds().await?;
let catalogue = crate::subscriptions(&state.ctx)?;
let catalogue = crate::subscriptions(&state.ctx).await?;
let by_id: std::collections::HashMap<&str, &crate::config::Feed> =
catalogue.iter().map(|s| (s.id.as_str(), &s.cfg)).collect();
let is_folder: std::collections::HashSet<&str> =
@@ -823,7 +823,7 @@ async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> {
{
continue;
}
let sum = db.feed_summary(&s.id)?;
let sum = db.feed_summary(&s.id).await?;
let subscribed = mine.contains(&s.id);
out.push(PopularRow {
id: s.id.clone(),
@@ -1219,7 +1219,7 @@ async fn add_feed(
let url = crate::feed::expand_input(&body.url);
// Someone else may already have it. Then adding costs nothing: no second fetch, no
// second copy on disk, just another name against the same feed.
if let Some(existing) = crate::subscriptions(&state.ctx)?
if let Some(existing) = crate::subscriptions(&state.ctx).await?
.into_iter()
.find(|s| crate::feed::same_feed(&s.cfg.url, &url))
{
@@ -1341,13 +1341,13 @@ async fn patch_feed(
// Derived feeds have no config entry. Editing one is the moment it earns a real
// entry: promote it, so the config holds your decisions and nothing else.
if !cfg.feeds.contains_key(&id) {
let subs = crate::subscriptions(&state.ctx)?;
let subs = crate::subscriptions(&state.ctx).await?;
let found = subs
.iter()
.find(|s| s.id == id)
.ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
cfg.feeds.insert(id.clone(), found.cfg.clone());
state.ctx.db.unmanage(&id)?;
state.ctx.db.unmanage(&id).await?;
}
let checked = match &body.url {
@@ -1388,7 +1388,7 @@ async fn patch_feed(
if url_changed {
// Refreshing a rotated auth token is the common case; entries and download history
// are keyed by feed id, so they survive the change.
state.ctx.db.clear_validators(&id)?;
state.ctx.db.clear_validators(&id).await?;
}
Ok(StatusCode::NO_CONTENT)
}
@@ -1401,7 +1401,7 @@ async fn remove_feed(
// Unsubscribing is personal: it takes the feed off your list and leaves everyone
// else's alone.
state.ctx.db.unsubscribe(user.id, &id).await?;
for child in crate::subscriptions(&state.ctx)?
for child in crate::subscriptions(&state.ctx).await?
.iter()
.filter(|s| s.cfg.group.as_deref() == Some(id.as_str()))
{
@@ -1417,7 +1417,7 @@ async fn remove_feed(
if cfg.feeds.remove(&id).is_none() {
// A derived feed: forget it here, though the OPML will list it again on the next
// read unless you unsubscribe from the OPML itself.
state.ctx.db.drop_managed(&id)?;
state.ctx.db.drop_managed(&id).await?;
return Ok(StatusCode::NO_CONTENT);
}
cfg.save(&state.config_path)?;
@@ -1610,7 +1610,7 @@ async fn read_all(
// A subscription's own row has no entries, so marking it read means everything under it.
let mut ids = vec![id.clone()];
ids.extend(
crate::subscriptions(&state.ctx)?
crate::subscriptions(&state.ctx).await?
.into_iter()
.filter(|s| s.cfg.group.as_deref() == Some(id.as_str()))
.map(|s| s.id),
@@ -1676,7 +1676,7 @@ async fn export_opml(
}),
..Default::default()
};
for s in crate::subscriptions(&state.ctx)? {
for s in crate::subscriptions(&state.ctx).await? {
// A feed from an OPML subscription comes back with the OPML itself.
if s.managed || !mine.contains(&s.id) {
continue;
@@ -1684,7 +1684,7 @@ async fn export_opml(
let title = state
.ctx
.db
.feed_summary(&s.id)
.feed_summary(&s.id).await
.ok()
.and_then(|sum| sum.title)
.unwrap_or_else(|| s.id.clone());