rays
4f4fd4c3fe
main.rs held 3,200 lines: the command line, the shared context, the
daemon, scans, downloads, the catalogue and OPML, and their tests. The
style guide keeps a root module to declarations and a little glue, the
code in leaf modules named for what they hold:
cli.rs the commands, run and user_cmd, ago and duration
ctx.rs Ctx and CfgEdit, and Ctx::new, which main() built inline
daemon.rs the daemon's loop, web server, shutdown, status, reload
scan.rs due_at/due_after, scan_one, policy, downloads, torrents
catalogue.rs add/rm/leave/remove_user, subscriptions, retiring lists
lists.rs OPML import and syncing an OPML's or a creator's feeds
(not opml.rs, which would shadow the opml crate)
logging.rs logging::init and the JSON lines with their trace
test_util.rs the context, feeds and feed server tests share
first_password moves to auth.rs and reap to retention.rs. Each test went
with what it tests; all 151 still run. main.rs is the module list and a
main() that hands the command on, 113 lines.
Moved verbatim, then made pub(crate) where a sibling calls it, with
calls across modules written module::function as the guide asks.
Everything the old main.rs defined is defined once in the new files.
With Ctx no longer a pub type at the crate root, Db's pub methods stop
counting as public API, and the compiler sees what nothing outside tests
calls: entries_in, count_in, unread_count and feed_description are now
#[cfg(test)], and media_feeds, called by nothing, is gone.
ipx depends on opentelemetry_sdk directly now (it was already in the
lock through ipx-directory), for logging::init's signature.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
617 lines
28 KiB
Rust
617 lines
28 KiB
Rust
//! The feeds the server knows and who reads them: adding, removing, leaving, and what an
|
|
//! OPML or Patreon list leaves behind when it goes.
|
|
|
|
use crate::ctx::Ctx;
|
|
use crate::ipc::Event;
|
|
use crate::{cli, config, db, feed, lists};
|
|
use anyhow::{Context, Result};
|
|
use std::collections::BTreeMap;
|
|
use std::path::Path;
|
|
use url::Url;
|
|
|
|
/// Subscribes to one feed, naming it from its own title.
|
|
pub(crate) async fn add(
|
|
ctx: &Ctx,
|
|
url: &str,
|
|
folder: Option<String>,
|
|
keywords: Vec<String>,
|
|
category: Option<String>,
|
|
list: bool,
|
|
) -> Result<()> {
|
|
let url = feed::find_feed(&ctx.client, &feed::expand_input(url)).await?;
|
|
let (id, existed) = add_to_catalogue(ctx, &url, folder, keywords).await?;
|
|
// Listing a feed the catalogue already has, or giving it a category, is the point of asking
|
|
// again: it changes those, and nothing else. A feed an OPML lists has no entry to change.
|
|
if list || category.is_some() {
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
if let Some(f) = cfg.feeds.get_mut(&id) {
|
|
f.listed |= list;
|
|
if category.is_some() {
|
|
f.category = category;
|
|
}
|
|
ctx.store_cfg(cfg).await?;
|
|
if existed {
|
|
println!("{id} is already in the catalogue; updated its Directory listing");
|
|
return Ok(());
|
|
}
|
|
}
|
|
}
|
|
if existed {
|
|
anyhow::bail!("already subscribed as {id:?}");
|
|
}
|
|
println!("added {id}{}", if list { ", listed in the Directory" } else { "" });
|
|
Ok(())
|
|
}
|
|
|
|
/// Adds the feed at `url` to the catalogue unless it is there already, a feed an OPML lists
|
|
/// included, so one show is never added twice: its id, and whether it was there. For `ipx add`
|
|
/// and the page's Add feed alike, which were the same steps written twice (#161); each finds the
|
|
/// feed first, to say its own way why an address is not one.
|
|
pub async fn add_to_catalogue(
|
|
ctx: &Ctx,
|
|
url: &str,
|
|
folder: Option<String>,
|
|
keywords: Vec<String>,
|
|
) -> Result<(String, bool)> {
|
|
// Taken before the duplicate check, so two people adding the same feed at once add it once.
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
if let Some(existing) = subscriptions(ctx).await?.into_iter().find(|s| feed::same_feed(&s.cfg.url, url)) {
|
|
return Ok((existing.id, true));
|
|
}
|
|
let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
|
|
ctx.store_cfg(cfg).await?;
|
|
Ok((id, false))
|
|
}
|
|
|
|
/// Returns the new feed id. Adding checks the address is a feed first (`feed::find_feed`); this
|
|
/// still names one it cannot read from its URL rather than failing.
|
|
pub(crate) async fn add_one(
|
|
ctx: &Ctx,
|
|
cfg: &mut config::Config,
|
|
url: &str,
|
|
folder: Option<String>,
|
|
keywords: Vec<String>,
|
|
) -> Result<String> {
|
|
let probe = config::Feed {
|
|
url: url.to_owned(),
|
|
folder: folder.clone(),
|
|
group: None,
|
|
media_types: None,
|
|
schedule: None,
|
|
keywords: keywords.clone(),
|
|
allow_explicit: false,
|
|
auto_download: true,
|
|
max_new_per_check: None,
|
|
username: None,
|
|
password: None,
|
|
password_env: None,
|
|
category: None,
|
|
listed: false,
|
|
private: false,
|
|
};
|
|
|
|
let title = match feed::fetch(&ctx.feed_client, &probe.url, None, None, None).await.map(|(got, _)| got) {
|
|
// An OPML subscription is named from its own <head><title>, not by trying to
|
|
// parse it as a feed and falling back to the hostname.
|
|
Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
|
|
feed::opml_title(&bytes).unwrap_or_else(|| url_stem(url))
|
|
}
|
|
Ok(feed::Fetched::Body { bytes, .. }) => {
|
|
feed::parse(&bytes).ok().and_then(|f| f.title).unwrap_or_else(|| url_stem(url))
|
|
}
|
|
_ => {
|
|
tracing::warn!(url, "could not read the feed; naming it from its URL");
|
|
url_stem(url)
|
|
}
|
|
};
|
|
|
|
// Slugs must be unique across derived feeds too, or a new feed can collide with one
|
|
// an OPML already introduced.
|
|
let mut taken: BTreeMap<String, config::Feed> =
|
|
subscriptions(ctx).await?.into_iter().map(|s| (s.id, s.cfg)).collect();
|
|
// A removed feed keeps its rows, so its id is only free again for the same feed: re-adding
|
|
// it gets its history back, and a different feed does not inherit someone else's.
|
|
for (id, other) in ctx.db.feed_urls().await? {
|
|
if !feed::same_feed(&other, url) {
|
|
taken.entry(id).or_insert_with(|| probe.clone());
|
|
}
|
|
}
|
|
let id = config::unique_slug(&title, |s| taken.contains_key(s));
|
|
cfg.feeds.insert(id.clone(), probe);
|
|
Ok(id)
|
|
}
|
|
|
|
/// Host plus last path segment, for naming a feed we could not read.
|
|
pub(crate) fn url_stem(url: &str) -> String {
|
|
Url::parse(url).ok().and_then(|u| u.host_str().map(str::to_owned)).unwrap_or_else(|| url.to_owned())
|
|
}
|
|
|
|
pub(crate) async fn rm(ctx: &Ctx, feed: &str) -> Result<()> {
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
let Some(parent) = cfg.feeds.remove(feed) else {
|
|
// Derived from an OPML: drop it here, though the subscription will list it again
|
|
// on the next read unless the OPML itself goes.
|
|
ctx.db.drop_managed(feed).await?;
|
|
println!("removed {feed}; it came from an OPML subscription and may return on the next read");
|
|
return Ok(());
|
|
};
|
|
ctx.store_cfg(cfg).await?;
|
|
// State and files stay: re-adding the feed should not re-download its back catalogue.
|
|
println!("removed {feed}; downloads and history kept");
|
|
retire_group(ctx, feed, Some(&parent)).await?;
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) async fn import(ctx: &Ctx, file: &Path) -> Result<()> {
|
|
let text = std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?;
|
|
// The CLI speaks for the operator, as the shared web token does.
|
|
let admin = ctx
|
|
.db
|
|
.users()
|
|
.await?
|
|
.into_iter()
|
|
.find(|u| u.is_admin)
|
|
.ok_or_else(|| anyhow::anyhow!("no admin account to subscribe: ipx user add <name> --admin"))?;
|
|
let doc = feed::read_opml(&text).map_err(|e| anyhow::anyhow!("{} is not OPML: {e}", file.display()))?;
|
|
let (added, had) = lists::subscribe_opml(ctx, lists::opml_feeds(&doc)?, admin.id).await?;
|
|
println!("subscribed {} to {added} feed(s); {had} already there", admin.name);
|
|
Ok(())
|
|
}
|
|
|
|
/// The configuration ipx runs with: config.toml for where things are and who may sign in, the
|
|
/// database for the feeds and the server settings (issue #18). The first time a database holds
|
|
/// neither, it takes them from config.toml, which is then cut down to the rest, the original kept
|
|
/// beside it as config.toml.pre-database.
|
|
pub(crate) async fn assemble_config(db: &db::Db, mut cfg: config::Config, path: &Path) -> Result<config::Config> {
|
|
for _ in 0..2 {
|
|
if let Some((stored, feeds)) = db.stored_config().await? {
|
|
if config::Config::file_holds_stored(path) {
|
|
tracing::warn!(
|
|
"config.toml still lists feeds or server settings; they are ignored, since the \
|
|
database holds them now. Change them in the web UI, or with ipx add and rm."
|
|
);
|
|
}
|
|
stored.apply(&mut cfg);
|
|
cfg.feeds = feeds;
|
|
return Ok(cfg);
|
|
}
|
|
if db.import_config(&cfg).await? {
|
|
if path.exists() {
|
|
let original = path.with_extension("toml.pre-database");
|
|
if !original.exists() {
|
|
std::fs::copy(path, &original)
|
|
.with_context(|| format!("keeping the original as {}", original.display()))?;
|
|
}
|
|
cfg.save_bootstrap(path)?;
|
|
}
|
|
tracing::info!(
|
|
feeds = cfg.feeds.len(),
|
|
"moved the feeds and server settings from config.toml into the database"
|
|
);
|
|
return Ok(cfg);
|
|
}
|
|
// Another ipx imported between our look and our insert; the next pass reads its copy.
|
|
}
|
|
anyhow::bail!("the database says it holds the configuration and then that it does not")
|
|
}
|
|
|
|
pub(crate) async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
|
|
let mut doc = opml::OPML {
|
|
head: Some(opml::Head { title: Some("ipx subscriptions".into()), ..Default::default() }),
|
|
..Default::default()
|
|
};
|
|
for (id, feed) in &ctx.cfg().feeds {
|
|
let title = ctx.db.feed_summary(id).await.ok().and_then(|s| s.title).unwrap_or_else(|| id.clone());
|
|
doc.add_feed(&title, &feed.url);
|
|
}
|
|
let xml = doc.to_string().map_err(|e| anyhow::anyhow!("writing OPML: {e}"))?;
|
|
std::fs::write(file, xml).with_context(|| format!("writing {}", file.display()))?;
|
|
println!("exported {} feed(s) to {}", ctx.cfg().feeds.len(), file.display());
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) async fn list(ctx: &Ctx) -> Result<()> {
|
|
let cfg = ctx.cfg();
|
|
if cfg.feeds.is_empty() {
|
|
println!("No feeds configured. Add one with `ipx add <url>`, or in the web UI.");
|
|
return Ok(());
|
|
}
|
|
for (id, feed) in &cfg.feeds {
|
|
let s = ctx.db.feed_summary(id).await?;
|
|
// The id on a line of its own, labelled: first on the title's line, antirez.com's id
|
|
// "feed" read as a heading and 'ipx fetch antirez' was tried instead (issue #82).
|
|
println!("{}", s.title.as_deref().filter(|t| !t.is_empty()).unwrap_or(id));
|
|
println!(" id {id}");
|
|
println!(" url {}", feed.url);
|
|
println!(" last checked {}", cli::ago(s.last_checked));
|
|
println!(" entries {} ({} downloaded)", s.entries, s.downloaded);
|
|
if let Some(err) = &s.last_error {
|
|
println!(" last error {err}");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Removes feeds nobody subscribes to that are dead (failing for 30 days) or quiet (nothing new
|
|
/// in a year), from the catalogue and the database, so the Directory lists feeds worth taking.
|
|
/// A feed listed in it with no subscribers stays for as long as it works and publishes.
|
|
pub(crate) async fn clean_directory(ctx: &Ctx) -> Result<()> {
|
|
const DEAD: i64 = 30 * 86_400;
|
|
const QUIET: i64 = 365 * 86_400;
|
|
let now = db::now();
|
|
// Once a day, for feeds dead a month or quiet a year: before every scan, its query took 1 to
|
|
// 12 s every few minutes, and the scans and downloads waited behind it (#193).
|
|
use std::sync::atomic::Ordering::Relaxed;
|
|
if now - ctx.directory_cleaned.load(Relaxed) < 86_400 {
|
|
return Ok(());
|
|
}
|
|
ctx.directory_cleaned.store(now, Relaxed);
|
|
let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?;
|
|
if stale.is_empty() {
|
|
return Ok(());
|
|
}
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
// Every one removed, so not `any`, which would stop at the first.
|
|
let mut removed = std::collections::HashMap::new();
|
|
for id in &stale {
|
|
if let Some(f) = cfg.feeds.remove(id) {
|
|
removed.insert(id.clone(), f);
|
|
}
|
|
}
|
|
if !removed.is_empty() {
|
|
ctx.store_cfg(cfg).await?;
|
|
}
|
|
for id in &stale {
|
|
retire_group(ctx, id, removed.get(id)).await?;
|
|
ctx.db.forget_feed(id).await?;
|
|
tracing::info!(
|
|
feed = id,
|
|
"removed a feed nobody subscribes to that is dead or has published nothing in a year"
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// One feed to scan: either an entry you wrote in config.toml, or one derived from an
|
|
/// OPML subscription and held only in the database.
|
|
pub struct Sub {
|
|
pub id: String,
|
|
pub cfg: config::Feed,
|
|
/// True when it came from an OPML and has no config entry of its own.
|
|
pub managed: bool,
|
|
}
|
|
|
|
/// Everything to scan: your config entries, plus whatever the OPML subscriptions listed.
|
|
///
|
|
/// A derived feed borrows its parent's settings wholesale. That is why it needs no config
|
|
/// entry -- there is nothing to store but its URL and where it came from.
|
|
pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
|
|
let cfg = ctx.cfg();
|
|
let mut out: Vec<Sub> =
|
|
cfg.feeds.iter().map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false }).collect();
|
|
|
|
// A group's title, once a group: asked for each feed in it, three queries a feed, it was about
|
|
// 96 for The Glass Cannon Network's 32 shows on every feed list, live update and scan (#159).
|
|
let mut group_titles = std::collections::HashMap::<String, String>::new();
|
|
for m in ctx.db.managed_feeds().await? {
|
|
if cfg.feeds.contains_key(&m.id) {
|
|
continue; // promoted to config at some point; that entry wins
|
|
}
|
|
let parent = cfg.feeds.get(&m.group_id);
|
|
if parent.is_none() {
|
|
// The OPML or Patreon feed this was derived from is no longer in config --
|
|
// removing it should have retired these rows too (see `retire_group`), but
|
|
// skip them here regardless so a row that slips through is never scanned.
|
|
continue;
|
|
}
|
|
let base = match parent.and_then(|p| p.folder.clone()) {
|
|
Some(folder) => folder,
|
|
None => {
|
|
if !group_titles.contains_key(&m.group_id) {
|
|
let title = ctx.db.feed_summary(&m.group_id).await.ok().and_then(|s| s.title);
|
|
group_titles.insert(m.group_id.clone(), title.unwrap_or_else(|| m.group_id.clone()));
|
|
}
|
|
group_titles[&m.group_id].clone()
|
|
}
|
|
};
|
|
let title = m.title.clone().unwrap_or_else(|| m.id.clone());
|
|
let (username, password, password_env) = parent.map(|p| p.credentials_for(&m.url)).unwrap_or_default();
|
|
out.push(Sub {
|
|
id: m.id.clone(),
|
|
cfg: config::Feed {
|
|
url: m.url.clone(),
|
|
folder: Some(format!("{base}/{title}")),
|
|
group: Some(m.group_id.clone()),
|
|
media_types: parent.and_then(|p| p.media_types.clone()),
|
|
schedule: parent.and_then(|p| p.schedule.clone()),
|
|
keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(),
|
|
allow_explicit: parent.is_some_and(|p| p.allow_explicit),
|
|
auto_download: parent.is_none_or(|p| p.auto_download),
|
|
max_new_per_check: parent.and_then(|p| p.max_new_per_check),
|
|
username,
|
|
password,
|
|
password_env,
|
|
category: None,
|
|
listed: false,
|
|
private: false,
|
|
},
|
|
managed: true,
|
|
});
|
|
}
|
|
out.sort_by(|a, b| a.id.cmp(&b.id));
|
|
Ok(out)
|
|
}
|
|
|
|
/// Takes a feed off one person's list, and the feeds in it if it is a group. Everyone else's
|
|
/// lists are left alone.
|
|
pub async fn leave(ctx: &Ctx, user_id: i64, id: &str) -> Result<()> {
|
|
use tracing::Instrument;
|
|
ctx.db.unsubscribe(user_id, id).instrument(db::step("unsubscribe")).await?;
|
|
// What they kept stays on their list, a feed of its own once nobody reads the list (#280).
|
|
let kept = ctx.db.kept_feeds(user_id).instrument(db::step("kept")).await?;
|
|
let all = subscriptions(ctx).instrument(db::step("subscriptions")).await?;
|
|
let unkept = all.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id));
|
|
for child in unkept {
|
|
ctx.db.unsubscribe(user_id, &child.id).instrument(db::step("unsubscribe")).await?;
|
|
}
|
|
if ctx.db.subscriber_counts().instrument(db::step("subscriber_counts")).await?.contains_key(id) {
|
|
return Ok(());
|
|
}
|
|
|
|
// Nobody is left: the feed stops being scanned. Its files and history stay, so if
|
|
// someone subscribes again they do not pull the back catalogue a second time. A feed an
|
|
// admin listed stays in the Directory, for the next person.
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
if cfg.feeds.get(id).is_some_and(|f| f.listed) {
|
|
return Ok(());
|
|
}
|
|
let Some(parent) = cfg.feeds.remove(id) else {
|
|
// A derived feed: forget it here, though the OPML will list it again on the next
|
|
// read unless you unsubscribe from the OPML itself.
|
|
return ctx.db.drop_managed(id).await;
|
|
};
|
|
ctx.store_cfg(cfg).await?;
|
|
retire_group(ctx, id, Some(&parent)).await
|
|
}
|
|
|
|
/// Removes an account, leaving each of its feeds first as unsubscribing does. The foreign keys
|
|
/// took the subscriptions with the account, but not the feed: one only they read stayed in the
|
|
/// catalogue with nobody subscribed, downloading every new episode for no one (#157).
|
|
pub async fn remove_user(ctx: &Ctx, user_id: i64) -> Result<()> {
|
|
let catalogue = subscriptions(ctx).await?;
|
|
let mine: std::collections::HashSet<String> =
|
|
ctx.db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
|
|
for id in &mine {
|
|
// A feed in a group of theirs goes with the group.
|
|
let in_theirs = catalogue.iter().any(|s| &s.id == id && s.cfg.group.as_ref().is_some_and(|g| mine.contains(g)));
|
|
if !in_theirs {
|
|
leave(ctx, user_id, id).await?;
|
|
}
|
|
}
|
|
ctx.db.delete_user(user_id).await
|
|
}
|
|
|
|
/// Retires every feed derived from `parent_id`, now that nothing subscribes to the OPML or
|
|
/// Patreon feed that listed them: the same rule `sync_group` applies to one the list drops --
|
|
/// removed if nothing was downloaded, orphaned and kept otherwise. Called once the parent
|
|
/// itself is removed, since `subscriptions()` would otherwise keep scanning them under a
|
|
/// fallback policy meant for a feed with no parent at all. A feed promoted to config is not
|
|
/// derived any more, so it is only unmanaged. `parent` is the list's settings as they were before
|
|
/// it was removed, None when that is no longer known.
|
|
pub async fn retire_group(ctx: &Ctx, parent_id: &str, parent: Option<&config::Feed>) -> Result<()> {
|
|
let cfg = ctx.cfg();
|
|
let managed = ctx.db.managed_feeds().await?;
|
|
let derived = managed.into_iter().filter(|m| m.group_id == parent_id);
|
|
for m in derived {
|
|
if cfg.feeds.contains_key(&m.id) {
|
|
// Scanned from its config entry and still read. Dropped as derived, its stored
|
|
// entries would go with it: davewiner's 11 were promoted without being unmanaged.
|
|
ctx.db.unmanage(&m.id).await?;
|
|
} else if promote_kept(ctx, &m, parent).await? {
|
|
tracing::info!(feed = %m.id, "its list is gone but someone kept it; a feed of its own now");
|
|
} else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
|
|
ctx.db.set_orphaned(&m.id, true).await?;
|
|
} else {
|
|
ctx.db.drop_managed(&m.id).await?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Makes a feed from an OPML or Patreon list a feed of its own, for those who kept it (#280): a
|
|
/// catalogue entry in no group, which the list no longer adds or takes away. Whoever had it only
|
|
/// through the list, without keeping it, loses it, as they would have when the list dropped it.
|
|
/// False, changing nothing, when nobody kept it. `base` is the list's settings: most callers have
|
|
/// removed the list from config by now, so they pass them.
|
|
pub(crate) async fn promote_kept(ctx: &Ctx, m: &db::Managed, base: Option<&config::Feed>) -> Result<bool> {
|
|
let subs = ctx.db.subscribers_of(&m.id).await?;
|
|
if !subs.iter().any(|(_, kept)| *kept) {
|
|
return Ok(false);
|
|
}
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
// The list's own settings carry over; its folder and group do not.
|
|
let (username, password, password_env) = base.map(|p| p.credentials_for(&m.url)).unwrap_or_default();
|
|
cfg.feeds.entry(m.id.clone()).or_insert_with(|| config::Feed {
|
|
url: m.url.clone(),
|
|
folder: None,
|
|
group: None,
|
|
media_types: base.and_then(|p| p.media_types.clone()),
|
|
schedule: base.and_then(|p| p.schedule.clone()),
|
|
keywords: base.map(|p| p.keywords.clone()).unwrap_or_default(),
|
|
allow_explicit: base.is_some_and(|p| p.allow_explicit),
|
|
auto_download: base.is_none_or(|p| p.auto_download),
|
|
max_new_per_check: base.and_then(|p| p.max_new_per_check),
|
|
username,
|
|
password,
|
|
password_env,
|
|
category: None,
|
|
listed: false,
|
|
// Looked up after the list was removed, private was always false, and a private OPML's
|
|
// kept show went to the directory (#332). A list no longer known may have been private.
|
|
private: base.is_none_or(|p| p.private),
|
|
});
|
|
ctx.store_cfg(cfg).await?;
|
|
ctx.db.unmanage(&m.id).await?;
|
|
ctx.db.set_orphaned(&m.id, false).await?;
|
|
for (user, kept) in subs {
|
|
if !kept {
|
|
ctx.db.unsubscribe(user, &m.id).await?;
|
|
}
|
|
}
|
|
Ok(true)
|
|
}
|
|
|
|
/// Retires every group whose parent is gone from config, and returns how many derived rows that
|
|
/// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind:
|
|
/// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were
|
|
/// most of the ones stored.
|
|
pub(crate) async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
|
|
let cfg = ctx.cfg();
|
|
let before = ctx.db.managed_feeds().await?;
|
|
let stranded: std::collections::BTreeSet<&str> =
|
|
before.iter().map(|m| m.group_id.as_str()).filter(|g| !cfg.feeds.contains_key(*g)).collect();
|
|
for group in stranded {
|
|
retire_group(ctx, group, None).await?;
|
|
}
|
|
Ok(before.len() - ctx.db.managed_feeds().await?.len())
|
|
}
|
|
|
|
/// A feed that answered from a new address after permanent redirects moves there in the
|
|
/// catalogue, so it is read from there and no longer redirected every time. One an OPML lists
|
|
/// is left alone, since the OPML would only put the old address back; so is a move onto an
|
|
/// address another feed already has.
|
|
pub(crate) async fn follow_move(ctx: &Ctx, id: &str, to: &str) -> Result<()> {
|
|
// Said in the event below, not logged here: the event is the log line.
|
|
if subscriptions(ctx).await?.iter().any(|s| s.id != id && feed::same_feed(&s.cfg.url, to)) {
|
|
tracing::info!(feed = id, to, "the feed moved to an address another feed already has; leaving it");
|
|
return Ok(());
|
|
}
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
let Some(f) = cfg.feeds.get_mut(id) else { return Ok(()) };
|
|
let from = std::mem::replace(&mut f.url, to.to_owned());
|
|
ctx.store_cfg(cfg).await?;
|
|
ctx.out.emit(Event::FeedMoved { feed: id.to_owned(), from, to: to.to_owned() });
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::test_util::{feed, test_ctx};
|
|
|
|
#[tokio::test]
|
|
async fn the_first_start_moves_the_configuration_in_and_trims_the_file() {
|
|
let dir = std::env::temp_dir().join(format!("ipx-assemble-{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let path = dir.join("config.toml");
|
|
std::fs::write(
|
|
&path,
|
|
"[general]\nschedule = \"every 2h\"\n[web]\ntoken = \"t\"\n[feeds.show]\nurl = \"http://x/show.xml\"\n",
|
|
)
|
|
.unwrap();
|
|
let db = db::Db::memory().await.unwrap();
|
|
|
|
let first = assemble_config(&db, config::Config::load(&path).unwrap(), &path).await.unwrap();
|
|
assert_eq!(first.feeds.keys().collect::<Vec<_>>(), ["show"]);
|
|
assert_eq!(first.general.schedule, "every 2h");
|
|
assert!(dir.join("config.toml.pre-database").exists(), "the original is kept");
|
|
assert!(!config::Config::file_holds_stored(&path), "and the file no longer lists them");
|
|
|
|
// The next start reads them from the database, the trimmed file notwithstanding.
|
|
let next = assemble_config(&db, config::Config::load(&path).unwrap(), &path).await.unwrap();
|
|
assert_eq!(next.feeds.keys().collect::<Vec<_>>(), ["show"]);
|
|
assert_eq!(next.general.schedule, "every 2h");
|
|
assert_eq!(next.web.token, "t");
|
|
std::fs::remove_dir_all(&dir).unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dead_feeds_leave_the_directory_once_a_day_not_before_every_scan() {
|
|
let ctx = test_ctx(config::Config::default()).await;
|
|
let dead = |id: &str| {
|
|
format!("INSERT INTO feeds (id, url, error_since, managed) VALUES ('{id}', 'u-{id}', 100, false)")
|
|
};
|
|
let listed = async || ctx.db.feed_urls().await.unwrap().into_iter().map(|(id, _)| id).collect::<Vec<_>>();
|
|
ctx.db.sql(&dead("a")).await.unwrap();
|
|
clean_directory(&ctx).await.unwrap();
|
|
assert!(listed().await.is_empty(), "the first pass since starting looks");
|
|
ctx.db.sql(&dead("b")).await.unwrap();
|
|
clean_directory(&ctx).await.unwrap();
|
|
assert_eq!(listed().await, ["b"], "not again the same day (#193)");
|
|
ctx.directory_cleaned.store(db::now() - 86_401, std::sync::atomic::Ordering::Relaxed);
|
|
clean_directory(&ctx).await.unwrap();
|
|
assert!(listed().await.is_empty(), "a day on, it looks again");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_removed_account_takes_the_feeds_only_it_read_with_it() {
|
|
let ctx = test_ctx(config::Config::default()).await;
|
|
let mut cfg = ctx.edit_cfg().await;
|
|
for id in ["only-theirs", "shared", "listed"] {
|
|
cfg.feeds.insert(id.into(), config::Feed { listed: id == "listed", ..feed() });
|
|
}
|
|
ctx.store_cfg(cfg).await.unwrap();
|
|
let gone = ctx.db.create_user("gone", None, false).await.unwrap();
|
|
let stays = ctx.db.create_user("stays", None, true).await.unwrap();
|
|
for id in ["only-theirs", "shared", "listed"] {
|
|
ctx.db.subscribe(gone, id).await.unwrap();
|
|
}
|
|
ctx.db.subscribe(stays, "shared").await.unwrap();
|
|
|
|
remove_user(&ctx, gone).await.unwrap();
|
|
assert!(ctx.db.user_by_id(gone).await.unwrap().is_none());
|
|
// Nobody reads it now: out of the catalogue, not downloading for no one (#157).
|
|
assert_eq!(ctx.cfg().feeds.keys().collect::<Vec<_>>(), ["listed", "shared"]);
|
|
assert!(ctx.db.subscription(stays, "shared").await.unwrap().is_some());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() {
|
|
// davewiner: the OPML subscription left config.toml, but its 922 derived rows
|
|
// stayed in the database and kept being scanned under the no-parent fallback.
|
|
let ctx = test_ctx(config::Config::default()).await;
|
|
ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").await.unwrap();
|
|
assert!(
|
|
subscriptions(&ctx).await.unwrap().iter().all(|s| s.id != "child"),
|
|
"a derived feed whose parent is gone from config must not be scanned"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() {
|
|
let ctx = test_ctx(config::Config::default()).await;
|
|
ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").await.unwrap();
|
|
ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").await.unwrap();
|
|
let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None };
|
|
ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap();
|
|
ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap();
|
|
|
|
retire_group(&ctx, "parent", None).await.unwrap();
|
|
|
|
let managed = ctx.db.managed_feeds().await.unwrap();
|
|
assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten");
|
|
assert!(managed.iter().any(|m| m.id == "has-file"), "has a file on disk, so it is kept");
|
|
assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() {
|
|
// davewiner: the OPML left config before retire_group existed, and 11 of its feeds
|
|
// promoted to config since still said managed = 1.
|
|
let mut cfg = config::Config::default();
|
|
cfg.feeds.insert("promoted".into(), feed());
|
|
cfg.feeds.insert("live-opml".into(), feed());
|
|
let ctx = test_ctx(cfg).await;
|
|
ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").await.unwrap();
|
|
ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").await.unwrap();
|
|
ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").await.unwrap();
|
|
ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).await.unwrap();
|
|
|
|
assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged");
|
|
|
|
let managed: Vec<String> = ctx.db.managed_feeds().await.unwrap().into_iter().map(|m| m.id).collect();
|
|
assert_eq!(managed, ["listed"], "a group still in config is left alone");
|
|
assert_eq!(ctx.db.feed_summary("promoted").await.unwrap().entries, 1, "its entries survive");
|
|
}
|
|
}
|