rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

The Directory comes from the directory service (#205)

Browse Source 
ipx asks ipx-directory on its people's behalf, with the token an admin set
in the new Admin, Directory tab, so the page never talks to it and never
sees the token or a feed's address. /api/directory, /api/popular and a
feed's Directory page answer from it; subscribing takes the address from
the directory and goes through Add feed's steps. With no directory set
up, the Directory says so. The local directory code is gone; its privacy
checks are the snapshot builder's now, and their tests moved with them.

With sharing on, ipx sends its snapshot after every scan, whenever the
config changes (set_cfg) and every half hour: each public feed and how
many here subscribe, never an OPML or Patreon creator itself or anything
private. A feed can be marked private in its settings, for a paid feed
whose address does not give it away; anyone subscribed may mark it, only
an admin unmark it. The admin's category for a feed that names none goes
with it, which the directory uses only while the feed names none, so the
category an admin gives a blog still reaches the Directory.

The directory lists only feeds it has read as a feed: the first snapshot
after a start went before the first scan, when an OPML was not yet known
to be one, and the directory listed the OPML. It now fetches what a
snapshot brings in at once instead of within the minute.

The browser suite runs its own ipx-directory on SQLite; the Directory's
tests match feeds by title, since the directory names them by address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-12 04:31:32 +00:00
parent ed82166745
commit c47aa57909
19 changed files with 710 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/ipx/Cargo.toml
View File

@@ -5,6 +5,7 @@ edition.workspace = true
[dependencies]
ipx-feed.workspace = true
ipx-directory.workspace = true
ammonia.workspace = true
anyhow.workspace = true
argon2.workspace = true

43
crates/ipx/src/config.rs
View File

@@ -42,6 +42,14 @@ pub struct General {
/// How much artwork iPX keeps on disk, in MB, so the page loads it from iPX rather than from
/// every publisher; the least recently shown goes first. 0 keeps none.
pub art_cache_mb: u64,
/// The directory service the Directory shows (#205), as `https://ipx-dir.sdf1.net`. Empty
/// for none: then the Directory says so, and feeds are added by their address.
pub directory_url: String,
/// The token the directory gave this server. Never sent to the page: it says only whether
/// one is set.
pub directory_token: String,
/// Send the directory the public feeds people here subscribe to, and how many of them do.
pub directory_share: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
@@ -174,6 +182,10 @@ pub struct Feed {
/// in the catalogue when the last one leaves, where anyone else's feed is dropped.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub listed: bool,
/// Never sent to the directory (#205), whatever its address looks like: someone's own
/// access to a paid show that `feed::url_looks_private` does not recognise.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub private: bool,
}
fn yes() -> bool {
@@ -192,6 +204,9 @@ impl Default for General {
max_new_per_check: 3,
media_types: vec!["audio".into(), "video".into()],
art_cache_mb: 500,
directory_url: String::new(),
directory_token: String::new(),
directory_share: false,
}
}
}
@@ -310,6 +325,12 @@ pub struct Stored {
/// Settings saved before it existed have none: they get the default.
#[serde(default = "art_cache_mb")]
pub art_cache_mb: u64,
#[serde(default)]
pub directory_url: String,
#[serde(default)]
pub directory_token: String,
#[serde(default)]
pub directory_share: bool,
}
fn art_cache_mb() -> u64 {
@@ -317,8 +338,17 @@ fn art_cache_mb() -> u64 {
}
/// `[general]` keys that live in the database once it holds the configuration.
const STORED_KEYS: [&str; 6] =
["schedule", "max_total_gb", "max_age_days", "max_new_per_check", "media_types", "art_cache_mb"];
const STORED_KEYS: [&str; 9] = [
"schedule",
"max_total_gb",
"max_age_days",
"max_new_per_check",
"media_types",
"art_cache_mb",
"directory_url",
"directory_token",
"directory_share",
];
impl Stored {
pub fn of(cfg: &Config) -> Self {
@@ -330,6 +360,9 @@ impl Stored {
max_new_per_check: g.max_new_per_check,
media_types: g.media_types.clone(),
art_cache_mb: g.art_cache_mb,
directory_url: g.directory_url.clone(),
directory_token: g.directory_token.clone(),
directory_share: g.directory_share,
}
}
@@ -341,6 +374,9 @@ impl Stored {
g.max_new_per_check = self.max_new_per_check;
g.media_types = self.media_types;
g.art_cache_mb = self.art_cache_mb;
g.directory_url = self.directory_url;
g.directory_token = self.directory_token;
g.directory_share = self.directory_share;
}
}
@@ -617,7 +653,7 @@ mod tests {
taken.insert("the-daily".to_string(), Feed {
url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None, category: None, listed: false,
password: None, password_env: None, category: None, listed: false, private: false,
});
assert_eq!(unique_slug("The Daily", &taken), "the-daily-2");
}
@@ -639,6 +675,7 @@ mod tests {
password_env: None,
category: None,
listed: false,
private: false,
};
assert_eq!(f.password().as_deref(), Some("literal"));

201
crates/ipx/src/directory.rs Normal file
View File

@@ -0,0 +1,201 @@
//! The Directory, from the directory service (#205): ipx asks it on its people's behalf, so the
//! page never talks to it and never sees its token, and sends it the public feeds people here
//! subscribe to when an admin has turned sharing on.
use std::collections::{HashMap, HashSet};
use ipx_directory::{FeedPage, Listing, Me, SnapshotEntry, SnapshotResult};
use serde::de::DeserializeOwned;
use crate::{Ctx, Sub, config, feed};
/// How often the snapshot goes when nothing has woken the sender sooner.
const SHARE_EVERY: std::time::Duration = std::time::Duration::from_secs(30 * 60);
/// What went wrong asking the directory, in words for an admin.
#[derive(Debug)]
pub enum DirError {
/// No address, or no token, in the settings.
NotConfigured,
/// It answered, refusing: 401 is a token it does not know, 403 one without the access asked for.
Refused(u16, String),
/// It did not answer, or not with what was asked for.
Unreachable(String),
}
impl std::fmt::Display for DirError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DirError::NotConfigured => write!(f, "No directory is set up on this server. An admin sets one in Admin, Directory."),
DirError::Refused(401, _) => write!(f, "The directory does not accept this server's token. It may have been revoked."),
DirError::Refused(403, msg) => write!(f, "The directory refused: {msg}."),
DirError::Refused(404, _) => write!(f, "That is not in the directory."),
DirError::Refused(code, msg) => write!(f, "The directory answered {code}: {msg}"),
DirError::Unreachable(e) => write!(f, "The directory could not be reached: {e}"),
}
}
}
/// When the snapshot last went, and what came of it, for the admin page.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Shared {
pub at: i64,
#[serde(flatten)]
pub result: Option<SnapshotResult>,
pub error: Option<String>,
}
fn target(cfg: &config::Config) -> Option<(String, String)> {
let url = cfg.general.directory_url.trim().trim_end_matches('/');
let token = cfg.general.directory_token.trim();
(!url.is_empty() && !token.is_empty()).then(|| (url.to_owned(), token.to_owned()))
}
pub fn configured(cfg: &config::Config) -> bool {
target(cfg).is_some()
}
async fn call<T: DeserializeOwned>(ctx: &Ctx, method: reqwest::Method, path: &str, body: Option<&[SnapshotEntry]>) -> Result<T, DirError> {
let (base, token) = target(&ctx.cfg()).ok_or(DirError::NotConfigured)?;
let mut req = ctx.client.request(method, format!("{base}{path}")).bearer_auth(token).timeout(std::time::Duration::from_secs(20));
if let Some(b) = body {
req = req.json(b);
}
let resp = req.send().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
let status = resp.status();
if !status.is_success() {
let msg = resp.json::<serde_json::Value>().await.ok().and_then(|v| v["error"].as_str().map(str::to_owned)).unwrap_or_default();
return Err(DirError::Refused(status.as_u16(), msg));
}
resp.json().await.map_err(|e| DirError::Unreachable(format!("its answer did not read: {e:#}")))
}
pub async fn me(ctx: &Ctx) -> Result<Me, DirError> {
call(ctx, reqwest::Method::GET, "/v1/me", None).await
}
pub async fn feeds(ctx: &Ctx) -> Result<Vec<Listing>, DirError> {
call(ctx, reqwest::Method::GET, "/v1/feeds", None).await
}
pub async fn popular(ctx: &Ctx) -> Result<Vec<Listing>, DirError> {
call(ctx, reqwest::Method::GET, "/v1/popular?limit=10", None).await
}
pub async fn page(ctx: &Ctx, id: &str) -> Result<FeedPage, DirError> {
call(ctx, reqwest::Method::GET, &format!("/v1/feeds/{}", percent_encoding::utf8_percent_encode(id, percent_encoding::NON_ALPHANUMERIC)), None).await
}
/// Whether a feed is someone's own: marked private, given a username or password, or at an
/// address that carries a key (`feed::url_looks_private`).
pub fn looks_private(feed: &config::Feed) -> bool {
feed.private
|| feed.username.is_some()
|| feed.password.is_some()
|| feed.password_env.is_some()
|| feed::url_looks_private(&feed.url)
}
/// What the snapshot lists: every feed in the catalogue, with how many people here subscribe,
/// except an OPML or a Patreon creator itself, which is sent as the feeds in it, and anything
/// private, a feed inside a private OPML included. An OPML not read yet is not known to be one
/// and goes too; the directory lists only what it has read as a feed.
pub fn snapshot_entries(catalogue: &[Sub], counts: &HashMap<String, i64>) -> Vec<SnapshotEntry> {
let by_id: HashMap<&str, &config::Feed> = catalogue.iter().map(|s| (s.id.as_str(), &s.cfg)).collect();
let is_group: HashSet<&str> = catalogue.iter().filter_map(|s| s.cfg.group.as_deref()).collect();
catalogue
.iter()
.filter(|s| {
!is_group.contains(s.id.as_str())
&& !feed::is_patreon_creator(&s.cfg.url)
&& !looks_private(&s.cfg)
&& !s.cfg.group.as_deref().and_then(|g| by_id.get(g)).is_some_and(|f| looks_private(f))
})
.map(|s| SnapshotEntry {
url: s.cfg.url.clone(),
subscribers: counts.get(&s.id).copied().unwrap_or(0),
category: s.cfg.category.clone(),
})
.collect()
}
/// Sends the snapshot now.
pub async fn share(ctx: &Ctx) -> Result<SnapshotResult, DirError> {
let catalogue = crate::subscriptions(ctx).await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
let counts = ctx.db.subscriber_counts().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
let entries = snapshot_entries(&catalogue, &counts);
let r: SnapshotResult = call(ctx, reqwest::Method::POST, "/v1/snapshot", Some(&entries)).await?;
if r.refused_private > 0 {
// The directory caught what this side should have: worth knowing which check differs.
tracing::warn!(refused = r.refused_private, "the directory refused feeds as private that ipx sent");
}
Ok(r)
}
/// Sends the snapshot every half hour while sharing is on, and at once when the settings change.
pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
loop {
let cfg = ctx.cfg();
if cfg.general.directory_share && configured(&cfg) {
let got = share(&ctx).await;
match &got {
Ok(r) => tracing::info!(accepted = r.accepted, refused_private = r.refused_private, refused_blocked = r.refused_blocked, "shared with the directory"),
Err(e) => tracing::warn!(error = %e, "could not share with the directory"),
}
let (result, error) = match got {
Ok(r) => (Some(r), None),
Err(e) => (None, Some(e.to_string())),
};
*ctx.directory_shared.lock().unwrap() = Some(Shared { at: crate::db::now(), result, error });
}
tokio::select! {
_ = tokio::time::sleep(SHARE_EVERY) => {}
_ = ctx.directory_wake.notified() => {}
_ = stop.changed() => return,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sub(id: &str, url: &str, edit: impl FnOnce(&mut config::Feed)) -> Sub {
let mut cfg: config::Feed = serde_json::from_value(serde_json::json!({ "url": url })).unwrap();
edit(&mut cfg);
Sub { id: id.into(), cfg, managed: false }
}
#[test]
fn the_snapshot_holds_public_feeds_and_never_a_private_one() {
let catalogue = vec![
sub("public", "https://example.com/feed.xml", |_| {}),
sub("marked", "https://example.com/mine.xml", |f| f.private = true),
sub("password", "https://example.com/locked.xml", |f| f.username = Some("me".into())),
sub("paid", "https://feeds.supercast.com/feeds/abc", |_| {}),
sub("keyed", "https://example.com/feed?key=k", |_| {}),
// An OPML: sent as the feeds in it, never itself.
sub("list", "https://example.com/list.opml", |_| {}),
sub("in-list", "https://example.com/a.xml", |f| f.group = Some("list".into())),
// Inside a private OPML: as private as the OPML.
sub("secret-list", "https://example.com/l.opml?token=t", |_| {}),
sub("in-secret", "https://example.com/b.xml", |f| f.group = Some("secret-list".into())),
];
let counts = HashMap::from([("public".to_string(), 3), ("marked".to_string(), 1)]);
let got = snapshot_entries(&catalogue, &counts);
let urls: Vec<(&str, i64)> = got.iter().map(|e| (e.url.as_str(), e.subscribers)).collect();
assert_eq!(urls, [("https://example.com/feed.xml", 3), ("https://example.com/a.xml", 0)]);
let patreon = vec![sub("creator", "https://www.patreon.com/c/someone", |_| {})];
assert!(snapshot_entries(&patreon, &counts).is_empty(), "a Patreon creator goes as its shows");
}
#[test]
fn nothing_is_asked_without_an_address_and_a_token() {
let mut cfg = config::Config::default();
assert!(!configured(&cfg));
cfg.general.directory_url = "https://ipx-dir.example/".into();
assert!(!configured(&cfg));
cfg.general.directory_token = "t".into();
assert_eq!(target(&cfg), Some(("https://ipx-dir.example".into(), "t".into())));
}
}

2
crates/ipx/src/download.rs
View File

@@ -438,7 +438,7 @@ mod tests {
let mut f = crate::config::Feed {
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false,
max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false, private: false,
};
assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");

19
crates/ipx/src/main.rs
View File

@@ -4,6 +4,7 @@ mod auth;
mod config;
mod db;
mod deadair;
mod directory;
mod entity;
mod download;
// The crate root's import, so `crate::feed` reaches it from every module as before.
@@ -161,6 +162,10 @@ pub struct Ctx {
/// When the Directory was last cleared of dead feeds (`clean_directory`), 0 for not since
/// starting.
pub directory_cleaned: std::sync::atomic::AtomicI64,
/// Sends the snapshot to the directory now, as when its settings change (#205).
pub directory_wake: tokio::sync::Notify,
/// When the snapshot last went and what came of it, for the admin page.
pub directory_shared: std::sync::Mutex<Option<directory::Shared>>,
}
/// A copy of the config to change, holding off every other change until `store_cfg` keeps it.
@@ -230,6 +235,9 @@ impl Ctx {
fn set_cfg(&self, cfg: config::Config) {
*self.cfg.write().unwrap() = std::sync::Arc::new(cfg);
tracing::info!("config reloaded");
// A feed added, removed, made private or given a category, or the directory's own
// settings changed: the directory hears of it now, not at the next half hour (#205).
self.directory_wake.notify_one();
}
async fn torrents(&self) -> Result<&torrent::Torrents> {
@@ -355,6 +363,8 @@ async fn main() -> Result<()> {
cfg_write: Default::default(),
directory_cleaned: Default::default(),
deadair: Default::default(),
directory_wake: Default::default(),
directory_shared: Default::default(),
});
let result = match cli.command {
@@ -665,6 +675,7 @@ async fn daemon(
// Off the worker: the first pass over a library is hours of ffmpeg, and scans and downloads
// must not wait behind it.
tokio::spawn(deadair::run(ctx.clone(), rx_stop.clone()));
tokio::spawn(directory::run(ctx.clone(), rx_stop.clone()));
// Runs one job, abandoning it if a signal arrives. Returns false to end the loop.
async fn until_stopped(
@@ -907,6 +918,7 @@ async fn add_one(
password_env: None,
category: None,
listed: false,
private: false,
};
let title = match feed::fetch(&ctx.feed_client, &probe.url, None, None, None).await.map(|(got, _)| got) {
@@ -1034,6 +1046,7 @@ pub async fn subscribe_opml(
password_env: None,
category: None,
listed: false,
private: false,
},
);
grew = true;
@@ -1331,6 +1344,8 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
}
}
ctx.out.emit(Event::ScanDone { feeds: scanned });
// What people subscribe to may have changed, and new feeds have titles now: share it.
ctx.directory_wake.notify_one();
Ok(())
}
@@ -1397,6 +1412,7 @@ pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
password_env: parent.and_then(|p| p.password_env.clone()),
category: None,
listed: false,
private: false,
},
managed: true,
});
@@ -2426,6 +2442,7 @@ mod tests {
password_env: None,
category: None,
listed: false,
private: false,
}
}
@@ -2499,6 +2516,8 @@ mod tests {
retry: Default::default(),
cfg_write: Default::default(),
deadair: Default::default(),
directory_wake: Default::default(),
directory_shared: Default::default(),
directory_cleaned: Default::default(),
}
}

326
crates/ipx/src/web.rs
View File

@@ -59,6 +59,7 @@ pub fn router(state: WebState) -> Router {
.route("/api/fetch", post(fetch_now))
.route("/api/opml", get(export_opml).post(import_opml))
.route("/api/settings", get(get_settings).patch(patch_settings))
.route("/api/directory-status", get(get_directory_status))
.route("/api/status", get(status))
.route("/api/popular", get(get_popular))
.route("/api/directory", get(get_directory))
@@ -815,6 +816,10 @@ struct FeedRow {
subscribers: i64,
/// Pinned to the top of your list.
pinned: bool,
/// Marked private: kept out of the directory (#205).
private: bool,
/// Private whatever it is marked, for a login or an address with a key in it.
private_by_address: bool,
}
#[derive(Serialize)]
@@ -866,6 +871,8 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
// A feed not scanned yet has no row: blank, as feed_summary gave it.
let crate::db::FeedListing { summary: s, ttl_mins, blocked, unread } = listed.remove(id).unwrap_or_default();
out.push(FeedRow {
private: feed.private,
private_by_address: crate::directory::looks_private(&crate::config::Feed { private: false, ..feed.clone() }),
id: id.clone(),
url: feed.url.clone(),
title: s.title,
@@ -933,115 +940,85 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
Ok(out)
}
// ---- popular on this server ----
// ---- the directory (#205) ----
/// A feed that carries a credential is someone's paid or private subscription. Listing it
/// would let anyone signed in subscribe to it and read what they pay for. The address half is
/// `feed::url_looks_private`, which the directory shares.
fn looks_private(feed: &crate::config::Feed) -> bool {
feed.username.is_some()
|| feed.password.is_some()
|| feed.password_env.is_some()
|| crate::feed::url_looks_private(&feed.url)
}
/// Only an id, a title, artwork and a count: never a URL, which is where a key would be.
/// A feed as the Directory draws it: never a URL, so a key in someone's feed address never
/// reaches the page, even though the directory itself has them.
#[derive(Serialize)]
struct PopularRow {
/// The directory's id for it, which its page and Subscribe go by.
id: String,
title: Option<String>,
image: Option<String>,
/// Everyone on every server that shares with the directory.
subscribers: i64,
/// Yours already. Everyone counts, you included, so your own feeds are listed too.
/// Yours already.
subscribed: bool,
/// Apple's category, from the feed's own iTunes category or else the catalogue's, and the
/// subcategory within it where there is one: the Directory browses them in that order.
/// When it is yours, its id here, which opens it.
feed: Option<String>,
category: Option<String>,
subcategory: Option<String>,
/// Any audio or video enclosure. Unlike category, every feed has an answer, so the
/// Directory's Podcasts and Blogs between them hold everything.
podcast: bool,
}
/// Every feed that may be listed, with everyone counted, you included, most subscribers
/// first. Popular is the top of it, the directory is all of it, and it is all that
/// `subscribe_popular` will subscribe you to. An OPML or a Patreon creator is listed as the
/// feeds inside it and never itself: both lists are for finding a show.
/// The catalogue as others may see it. `everything` is the Directory: every feed, with or
/// without subscribers, so the ones an admin listed show before anyone takes them. Without it,
/// Popular: only what people subscribe to.
async fn popular(state: &WebState, user_id: i64, everything: bool) -> Result<Vec<PopularRow>> {
let db = &state.ctx.db;
// One pass for every feed's title, artwork and category, not three queries a feed.
let mut listed = db.feed_list(user_id, None).await?;
let mine: std::collections::HashSet<String> =
db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
let counts = db.subscriber_counts().await?;
let media = db.media_feeds().await?;
let catalogue = crate::subscriptions(&state.ctx).await?;
let ok = listable(&catalogue);
let mut out = vec![];
for s in &catalogue {
let n = counts.get(&s.id).copied().unwrap_or(0);
if (n == 0 && !everything) || !ok.contains(s.id.as_str()) {
continue;
}
let sum = listed.remove(&s.id).map(|l| l.summary).unwrap_or_default();
let subscribed = mine.contains(&s.id);
// The feed's own wins; an admin's is for the feeds, mostly blogs, that name none.
let path = sum.category.or_else(|| s.cfg.category.clone()).map(|c| crate::feed::category_path(&c));
out.push(PopularRow {
id: s.id.clone(),
title: sum.title,
image: sum.image,
subscribers: n,
subscribed,
category: path.as_ref().map(|p| p.0.clone()),
subcategory: path.and_then(|p| p.1),
podcast: media.contains(&s.id),
});
fn dir_error(e: crate::directory::DirError) -> ApiError {
use crate::directory::DirError;
let status = match &e {
DirError::NotConfigured => StatusCode::SERVICE_UNAVAILABLE,
DirError::Refused(404, _) => StatusCode::NOT_FOUND,
_ => StatusCode::BAD_GATEWAY,
};
if !matches!(e, DirError::NotConfigured | DirError::Refused(404, _)) {
tracing::warn!(error = %e, "asking the directory failed");
}
out.sort_by(|a, b| b.subscribers.cmp(&a.subscribers).then_with(|| sort_name(a).cmp(&sort_name(b))));
Ok(out)
ApiError { error: anyhow::Error::msg(e.to_string()), status }
}
/// The feeds the Directory may list: not an OPML or a Patreon creator itself, which is listed as
/// the feeds in it, and neither the feed nor the group it is in private. Asked of one feed, without
/// building the whole Directory, by a feed's Directory page and by subscribing from it: building
/// it was nearly all of the 0.57 s a feed's page took (#160).
fn listable(catalogue: &[crate::Sub]) -> std::collections::HashSet<&str> {
let by_id: std::collections::HashMap<&str, &crate::config::Feed> =
catalogue.iter().map(|s| (s.id.as_str(), &s.cfg)).collect();
let is_folder: std::collections::HashSet<&str> = catalogue.iter().filter_map(|s| s.cfg.group.as_deref()).collect();
catalogue
.iter()
// A feed inside an OPML that looks private is as private as the OPML.
.filter(|s| {
!is_folder.contains(s.id.as_str())
&& !looks_private(&s.cfg)
&& !s.cfg.group.as_deref().and_then(|g| by_id.get(g)).is_some_and(|f| looks_private(f))
/// The directory's listings as this person sees them: which are theirs already, matched by
/// address, since the directory and this server name a feed differently.
async fn rows_for(state: &WebState, user_id: i64, listed: Vec<ipx_directory::Listing>) -> Result<Vec<PopularRow>> {
let mine: std::collections::HashSet<String> =
state.ctx.db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
let here: std::collections::HashMap<String, String> = crate::subscriptions(&state.ctx)
.await?
.into_iter()
.filter(|s| mine.contains(&s.id))
.filter_map(|s| ipx_directory::store::normalise(&s.cfg.url).map(|u| (u, s.id)))
.collect();
Ok(listed
.into_iter()
.map(|l| {
let feed = ipx_directory::store::normalise(&l.url).and_then(|u| here.get(&u).cloned());
PopularRow {
id: l.id,
title: l.title,
image: l.image,
subscribers: l.subscribers,
subscribed: feed.is_some(),
feed,
category: l.category,
subcategory: l.subcategory,
podcast: l.podcast,
}
})
.map(|s| s.id.as_str())
.collect()
.collect())
}
async fn get_popular(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<PopularRow>>, ApiError> {
let mut rows = popular(&state, user.id, false).await?;
rows.truncate(10);
Ok(Json(rows))
let listed = crate::directory::popular(&state.ctx).await.map_err(dir_error)?;
Ok(Json(rows_for(&state, user.id, listed).await?))
}
/// Every feed that may be listed, A to Z, with the feeds inside an OPML in place of the OPML.
/// Every feed in the directory, A to Z.
async fn get_directory(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<PopularRow>>, ApiError> {
let mut rows = popular(&state, user.id, true).await?;
rows.sort_by_key(sort_name);
Ok(Json(rows))
let listed = crate::directory::feeds(&state.ctx).await.map_err(dir_error)?;
Ok(Json(rows_for(&state, user.id, listed).await?))
}
/// One of a listed feed's latest items, for its page in the Directory before anyone subscribes
@@ -1060,40 +1037,23 @@ struct ListedItem {
struct Listed {
/// What the feed says it is (#130), sanitized as its items are.
description: Option<String>,
/// Why its last check failed, when it did (#142), in the words a feed's own page uses.
/// Why its last check failed, when it did (#142), in the directory's words.
failing: Option<String>,
items: Vec<ListedItem>,
}
/// A feed's last error as its Directory page says it: explained once it has been failing a day, as
/// the feed's own page does, or else that the check failed; never the error as it stands, which can
/// name the feed's address, and the Directory names none. A day, so that a site down for an hour,
/// or a resolver for a minute, is not called gone (#143).
fn failure_words(err: &str, failing_for: i64) -> String {
Some(err)
.filter(|_| failing_for >= FLAG_AFTER_SECS)
.and_then(crate::feed::explain_failure)
.map_or_else(|| "Its last check failed.".to_owned(), |f| f.reason.to_owned())
}
/// A listed feed's description and latest twenty items. Only a feed the Directory lists, so a
/// guessed id reaches nothing private, as `subscribe_popular` checks.
/// A listed feed's description and latest items, from the directory, sanitized here: they are
/// the feed's own HTML, which the directory passes on as it was sent.
async fn get_listed(
State(state): State<WebState>,
user: crate::db::User,
_user: crate::db::User,
Path(id): Path<String>,
) -> Result<Json<Listed>, ApiError> {
if !listable(&crate::subscriptions(&state.ctx).await?).contains(id.as_str()) {
return Err(ApiError::not_found(format!("{id:?} is not in the directory")));
}
let order = crate::db::order_sql("published", "desc");
let rows = state.ctx.db.entries_in(user.id, Some(&id), crate::db::Filter::parse("all"), None, 0, 20, &order).await?;
let page = crate::directory::page(&state.ctx, &id).await.map_err(dir_error)?;
let mut sanitizer = feed_sanitizer();
let description = state.ctx.db.feed_description(&id).await?.map(|d| clean_description(&mut sanitizer, &d, None));
let summary = state.ctx.db.feed_summary(&id).await?;
let failing_for = summary.error_since.map_or(0, |since| crate::db::now() - since);
let failing = summary.last_error.as_deref().map(|e| failure_words(e, failing_for));
let items = rows
let description = page.description.as_deref().map(|d| clean_description(&mut sanitizer, d, None));
let items = page
.items
.into_iter()
.map(|e| ListedItem {
description: e.description.as_deref().map(|d| clean_description(&mut sanitizer, d, e.link.as_deref())),
@@ -1104,27 +1064,28 @@ async fn get_listed(
duration: e.duration,
})
.collect();
Ok(Json(Listed { description, failing, items }))
Ok(Json(Listed { description, failing: page.failing, items }))
}
fn sort_name(p: &PopularRow) -> String {
p.title.clone().unwrap_or_else(|| p.id.clone()).to_lowercase()
}
/// Subscribes by id, because the list never shows a URL. Checked against the same list, so
/// a guessed id cannot reach a private feed.
/// Subscribes by the directory's id, because the page never sees a URL: the address comes from
/// the directory, and goes through the same steps as Add feed.
async fn subscribe_popular(
State(state): State<WebState>,
user: crate::db::User,
Path(id): Path<String>,
) -> Result<Json<serde_json::Value>, ApiError> {
if !listable(&crate::subscriptions(&state.ctx).await?).contains(id.as_str()) {
return Err(ApiError::bad_request(format!("{id:?} is not in the directory")));
let page = crate::directory::page(&state.ctx, &id).await.map_err(|e| match e {
crate::directory::DirError::Refused(404, _) => ApiError::bad_request(format!("{id:?} is not in the directory")),
e => dir_error(e),
})?;
let (feed_id, existed) = crate::add_to_catalogue(&state.ctx, &page.listing.url, None, vec![]).await?;
let already = existed && state.ctx.db.subscription(user.id, &feed_id).await?.is_some();
state.ctx.db.subscribe(user.id, &feed_id).await?;
if !already {
explicit_on_add(&state, user.id, &feed_id, false).await?;
}
state.ctx.db.subscribe(user.id, &id).await?;
// A listed feed nobody took was checked once a day at most: read it now, not in an hour.
scan_soon(&state, Some(id.clone())).await;
Ok(Json(serde_json::json!({ "id": id })))
scan_soon(&state, Some(feed_id.clone())).await;
Ok(Json(serde_json::json!({ "id": feed_id })))
}
/// Validates a replacement feed URL: present, parseable, and not already subscribed under
@@ -1198,18 +1159,6 @@ impl IntoResponse for ApiError {
mod tests {
use super::*;
#[test]
fn a_failing_feed_says_why_in_the_directory_without_its_address() {
let day = FLAG_AFTER_SECS;
let nothing = "the feed's address answered with nothing at all";
assert_eq!(failure_words(nothing, day), "This address answers with nothing; the site may be gone.");
// Not before it has been failing a day, as on the feed's own page (#143).
assert_eq!(failure_words(nothing, 3600), "Its last check failed.");
// reqwest names the URL; the Directory names none.
let raw = "error sending request for url (http://daily-quests.com/comic/?feed=atom): connection closed";
assert_eq!(failure_words(raw, day), "Its last check failed.");
}
#[test]
fn a_relative_image_resolves_against_the_post() {
let mut b = feed_sanitizer();
@@ -1249,60 +1198,6 @@ mod tests {
assert!(page("modern", "dark\"").contains("<html lang=en>"));
}
#[test]
fn a_feed_with_a_credential_is_never_popular() {
let f = |url: &str| crate::config::Feed {
url: url.into(),
folder: None,
group: None,
media_types: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
category: None,
listed: false,
};
assert!(!looks_private(&f("https://feeds.twit.tv/twit.xml")));
assert!(!looks_private(&f("https://example.com/rss?format=mp3")));
// Patreon's shape: the key is a query parameter.
assert!(looks_private(&f("https://www.patreon.com/rss/x?auth=abc123&show=2073588")));
assert!(looks_private(&f("https://example.com/rss?api_key=abc")));
// Paid-feed services put the key in the path; the host gives them away.
assert!(looks_private(&f("https://feeds.supercast.com/feeds/abcdefghijklmnopqrstuvwx")));
assert!(looks_private(&f("https://someshow.supportingcast.fm/content/abc123.rss")));
// A long path segment alone is not a key: acast's public show ids look the same.
assert!(!looks_private(&f("https://feeds.acast.com/public/shows/0123456789abcdef01234567")));
assert!(looks_private(&f("https://ray:hunter2@example.com/rss")));
assert!(looks_private(&f("not a url")), "unparseable is not safe to list");
let mut basic = f("https://example.com/rss");
basic.username = Some("ray".into());
assert!(looks_private(&basic), "a feed with a login configured");
}
#[test]
fn the_directory_lists_feeds_not_their_groups_and_nothing_private() {
let sub = |id: &str, url: &str, group: Option<&str>| crate::Sub {
id: id.into(),
cfg: crate::config::Feed { group: group.map(Into::into), ..feed(url) },
managed: group.is_some(),
};
let catalogue = [
sub("plain", "https://example.com/rss", None),
sub("opml", "https://example.com/list.opml", None),
sub("kid", "https://kid.example/rss", Some("opml")),
sub("paid", "https://www.patreon.com/rss/x?auth=abc", None),
sub("paid-show", "https://www.patreon.com/rss/x?auth=abc&show=1", Some("paid")),
];
let mut got: Vec<&str> = listable(&catalogue).into_iter().collect();
got.sort();
assert_eq!(got, ["kid", "plain"], "the OPML as its feeds; a private feed and its shows never");
}
#[test]
fn a_requests_trace_is_the_pages_from_its_traceparent() {
use opentelemetry::trace::TraceContextExt;
@@ -1366,7 +1261,7 @@ mod tests {
crate::config::Feed {
url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None, category: None, listed: false,
password: None, password_env: None, category: None, listed: false, private: false,
}
}
@@ -1625,6 +1520,9 @@ struct FeedPatch {
deadair: Option<Option<bool>>,
/// The pause DeadAir leaves where it cuts, in ms: one of `deadair::PAUSES` (#182).
deadair_pause: Option<i64>,
/// Kept out of the directory (#205). Anyone subscribed may mark a feed private, since it is
/// most likely their own paid feed; only an admin may unmark it.
private: Option<bool>,
}
/// A block list as sent: trimmed, blanks and repeats dropped, and bounded, since each word is
@@ -1704,14 +1602,14 @@ async fn patch_feed(
// The rest describes the feed itself -- where its files land, its address, when it is
// polled -- and there is one of those however many people read it.
let feed_level =
body.url.is_some() || body.folder.is_some() || body.schedule.is_some() || body.category.is_some();
if !feed_level {
let admin_only =
body.url.is_some() || body.folder.is_some() || body.schedule.is_some() || body.category.is_some() || body.private == Some(false);
if !admin_only && body.private.is_none() {
return Ok(StatusCode::NO_CONTENT);
}
if !user.is_admin {
if admin_only && !user.is_admin {
return Err(ApiError::forbidden(
"the feed's address, folder, schedule and category are the same for everyone, so only an admin changes them",
"the feed's address, folder, schedule and category, and whether it may be shared, are the same for everyone, so only an admin changes them",
));
}
let mut cfg = state.ctx.edit_cfg().await;
@@ -1761,6 +1659,9 @@ async fn patch_feed(
if let Some(v) = body.category {
feed.category = v.map(|s| s.trim().to_owned()).filter(|s| !s.is_empty());
}
if let Some(v) = body.private {
feed.private = v;
}
state.ctx.store_cfg(cfg).await?;
if url_changed {
// Refreshing a rotated auth token is the common case; entries and download history
@@ -2315,6 +2216,10 @@ struct Settings {
max_total_gb: f64,
max_age_days: u64,
art_cache_mb: u64,
directory_url: String,
/// Whether a token is set: the token itself never leaves the server.
directory_token_set: bool,
directory_share: bool,
}
async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
@@ -2328,6 +2233,9 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
max_total_gb: cfg.general.max_total_gb,
max_age_days: cfg.general.max_age_days,
art_cache_mb: cfg.general.art_cache_mb,
directory_url: cfg.general.directory_url.clone(),
directory_token_set: !cfg.general.directory_token.trim().is_empty(),
directory_share: cfg.general.directory_share,
})
}
@@ -2339,6 +2247,11 @@ struct SettingsPatch {
max_total_gb: Option<f64>,
max_age_days: Option<u64>,
art_cache_mb: Option<u64>,
/// Empty for no directory.
directory_url: Option<String>,
/// Replaces the token; empty clears it.
directory_token: Option<String>,
directory_share: Option<bool>,
}
async fn patch_settings(
@@ -2378,10 +2291,43 @@ async fn patch_settings(
if let Some(v) = body.art_cache_mb {
cfg.general.art_cache_mb = v;
}
if let Some(v) = body.directory_url {
let v = v.trim().trim_end_matches('/').to_owned();
if !v.is_empty() && !url::Url::parse(&v).is_ok_and(|u| matches!(u.scheme(), "http" | "https")) {
return Err(ApiError::bad_request(format!("{v:?} is not a web address; try https://ipx-dir.example.net")));
}
cfg.general.directory_url = v;
}
if let Some(v) = body.directory_token {
cfg.general.directory_token = v.trim().to_owned();
}
if let Some(v) = body.directory_share {
cfg.general.directory_share = v;
}
state.ctx.store_cfg(cfg).await?;
Ok(StatusCode::NO_CONTENT)
}
/// What the admin page says about the directory: who it takes this server's token to be, or why
/// it would not say, and how the last snapshot went.
async fn get_directory_status(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<serde_json::Value>, ApiError> {
if !user.is_admin {
return Err(ApiError::forbidden("only an admin sees the directory's settings"));
}
let shared = state.ctx.directory_shared.lock().unwrap().clone();
let me = crate::directory::me(&state.ctx).await;
Ok(Json(serde_json::json!({
"configured": crate::directory::configured(&state.ctx.cfg()),
"name": me.as_ref().ok().map(|m| m.name.clone()),
"access": me.as_ref().ok().map(|m| m.access),
"error": me.err().map(|e| e.to_string()),
"shared": shared,
})))
}
#[derive(Deserialize)]
struct LogQuery {
/// Highest seq the client already has; 0 means "give me the tail".