Release 0.5.3: OPML orphan scan, feed error UI, small UI fixes
- Stop scanning an OPML/Patreon feed's derived rows once nobody subscribes to it; retire them (drop or orphan) the way sync_group already does when the list itself drops one. This is what let 922 defunct davewiner feeds keep scanning hourly after the OPML left config. - Repair feed XML with a bare `&`, and give a plain reason (moved web page with its new address when linked, or nothing yet for an empty body) instead of a raw parser error. - Show a failing feed's plain-English reason and next step (Unsubscribe / Use the new address) in the sidebar and on its own page, once it has been down a day. - Fix four small UI bugs: show-note links open in a new tab, video files play as video, an opened item no longer disappears from the Unread tab, and Subscribe/Unsubscribe get their own icons. - Fix Settings disappearing for non-admin accounts: it was hiding the whole modal instead of just the admin-only parts (Users, the editable schedule/quota, Save), which are the only parts the server actually refuses them. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DmQfE1eFPApnXWyPHBWqUA
This commit is contained in:
47
src/db.rs
47
src/db.rs
@@ -22,6 +22,10 @@ CREATE TABLE IF NOT EXISTS feeds (
|
||||
last_checked INTEGER,
|
||||
ttl_mins INTEGER,
|
||||
last_error TEXT,
|
||||
-- When the current run of failures began; NULL while the feed is healthy. Kept
|
||||
-- through repeated failures so the UI can tell a blip (macmanx: failed once, fine an
|
||||
-- hour later) from a feed that has been down for a day.
|
||||
error_since INTEGER,
|
||||
-- Came from a subscribed OPML that no longer lists it, but has downloads, so kept.
|
||||
orphaned INTEGER NOT NULL DEFAULT 0,
|
||||
-- The OPML subscription this feed came from.
|
||||
@@ -165,6 +169,7 @@ fn migrate(conn: &Connection) -> Result<()> {
|
||||
// and it came back the same day with `last_login` beside it.
|
||||
("users", "created", "INTEGER"),
|
||||
("users", "last_login", "INTEGER"),
|
||||
("feeds", "error_since", "INTEGER"),
|
||||
];
|
||||
let retired: &[(&str, &str)] = &[
|
||||
// Read state from before accounts, long since moved to entry_state. Two bugs came from
|
||||
@@ -208,6 +213,8 @@ pub struct FeedSummary {
|
||||
pub orphaned: bool,
|
||||
pub last_checked: Option<i64>,
|
||||
pub last_error: Option<String>,
|
||||
/// When this run of failures began; see the `error_since` column.
|
||||
pub error_since: Option<i64>,
|
||||
pub entries: i64,
|
||||
pub downloaded: i64,
|
||||
}
|
||||
@@ -249,7 +256,7 @@ impl Db {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut sum: FeedSummary = conn
|
||||
.query_row(
|
||||
"SELECT title, image, last_checked, last_error, coalesce(orphaned, 0)
|
||||
"SELECT title, image, last_checked, last_error, coalesce(orphaned, 0), error_since
|
||||
FROM feeds WHERE id = ?1",
|
||||
[feed_id],
|
||||
|r| {
|
||||
@@ -259,6 +266,7 @@ impl Db {
|
||||
last_checked: r.get(2)?,
|
||||
last_error: r.get(3)?,
|
||||
orphaned: r.get::<_, i64>(4)? != 0,
|
||||
error_since: r.get(5)?,
|
||||
..Default::default()
|
||||
})
|
||||
},
|
||||
@@ -333,7 +341,8 @@ impl Db {
|
||||
last_checked = excluded.last_checked,
|
||||
ttl_mins = excluded.ttl_mins,
|
||||
image = coalesce(excluded.image, feeds.image),
|
||||
last_error = NULL",
|
||||
last_error = NULL,
|
||||
error_since = NULL",
|
||||
rusqlite::params![feed_id, url, title, etag, last_modified, now(), ttl_mins.map(|t| t as i64), image],
|
||||
)?;
|
||||
Ok(())
|
||||
@@ -344,7 +353,8 @@ impl Db {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO feeds (id, url, last_checked) VALUES (?1, ?2, ?3)
|
||||
ON CONFLICT(id) DO UPDATE SET last_checked = excluded.last_checked, last_error = NULL",
|
||||
ON CONFLICT(id) DO UPDATE SET last_checked = excluded.last_checked,
|
||||
last_error = NULL, error_since = NULL",
|
||||
rusqlite::params![feed_id, url, now()],
|
||||
)?;
|
||||
Ok(())
|
||||
@@ -363,10 +373,15 @@ impl Db {
|
||||
|
||||
pub fn set_feed_error(&self, feed_id: &str, url: &str, msg: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let now = now();
|
||||
conn.execute(
|
||||
"INSERT INTO feeds (id, url, last_checked, last_error) VALUES (?1, ?2, ?3, ?4)
|
||||
ON CONFLICT(id) DO UPDATE SET last_checked = excluded.last_checked, last_error = excluded.last_error",
|
||||
rusqlite::params![feed_id, url, now(), msg],
|
||||
"INSERT INTO feeds (id, url, last_checked, last_error, error_since)
|
||||
VALUES (?1, ?2, ?3, ?4, ?3)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
last_checked = excluded.last_checked,
|
||||
last_error = excluded.last_error,
|
||||
error_since = coalesce(feeds.error_since, excluded.error_since)",
|
||||
rusqlite::params![feed_id, url, now, msg],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1766,6 +1781,26 @@ mod tests {
|
||||
assert_eq!(explicit(None), [None, Some(false)], "outside a group nothing is inherited");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_since_marks_the_start_of_a_run_of_failures_and_clears_on_success() {
|
||||
let db = Db::memory().unwrap();
|
||||
db.set_feed_error("f", "http://x", "HTTP 404").unwrap();
|
||||
// Backdate it, as if this feed had already been failing a while, so a second
|
||||
// failure landing "now" is distinguishable from the first.
|
||||
db.exec_for_test("UPDATE feeds SET error_since = error_since - 3600 WHERE id = 'f'").unwrap();
|
||||
let first = db.feed_summary("f").unwrap().error_since.unwrap();
|
||||
|
||||
// macmanx: failed once, read fine an hour later. A second failure must not push
|
||||
// error_since forward -- the UI decides "failing for a day" from the first one.
|
||||
db.set_feed_error("f", "http://x", "HTTP 404").unwrap();
|
||||
assert_eq!(db.feed_summary("f").unwrap().error_since, Some(first));
|
||||
|
||||
db.touch_feed("f", "http://x").unwrap();
|
||||
let after = db.feed_summary("f").unwrap();
|
||||
assert_eq!(after.last_error, None);
|
||||
assert_eq!(after.error_since, None, "a clean check ends the run of failures");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enclosure_url_is_the_dedupe_key() {
|
||||
let db = Db::memory().unwrap();
|
||||
|
||||
206
src/feed.rs
206
src/feed.rs
@@ -87,6 +87,50 @@ pub async fn fetch(
|
||||
Ok(Fetched::Body { bytes, etag, last_modified })
|
||||
}
|
||||
|
||||
/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
|
||||
/// it is, and whether there is a new address to switch to.
|
||||
pub struct Failure {
|
||||
pub reason: &'static str,
|
||||
pub new_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the
|
||||
/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling
|
||||
/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled --
|
||||
/// comes back `None`: transient by nature, or with nothing more useful to say than the raw
|
||||
/// text already shown once a feed is open.
|
||||
///
|
||||
/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP
|
||||
/// {status}")`, and the "got a web page" message above) plus the substrings a DNS failure
|
||||
/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing
|
||||
/// nothing extra, so a miss costs a clearer message, not a wrong one.
|
||||
pub fn explain_failure(msg: &str) -> Option<Failure> {
|
||||
if let Some(rest) = msg.strip_prefix("got a web page, not a feed") {
|
||||
let new_url = rest
|
||||
.strip_prefix("; it links ")
|
||||
.and_then(|r| r.strip_suffix(" as its feed"))
|
||||
.map(str::to_owned);
|
||||
return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
|
||||
}
|
||||
let low = msg.to_ascii_lowercase();
|
||||
if low.contains("http 404") {
|
||||
return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None });
|
||||
}
|
||||
if low.contains("http 401") || low.contains("http 403") {
|
||||
return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None });
|
||||
}
|
||||
if low.contains("http 402") {
|
||||
return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None });
|
||||
}
|
||||
if low.contains("dns error")
|
||||
|| low.contains("failed to lookup address")
|
||||
|| low.contains("no address associated")
|
||||
{
|
||||
return Some(Failure { reason: "This address no longer resolves; the site is gone.", new_url: None });
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True when a body is an OPML document rather than a feed.
|
||||
///
|
||||
/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
|
||||
@@ -220,11 +264,110 @@ pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
|
||||
Ok(ch) => Ok(from_rss(ch, bytes)),
|
||||
Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
|
||||
Ok(feed) => Ok(from_atom(feed)),
|
||||
Err(atom_err) => Err(anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})")),
|
||||
Err(atom_err) => {
|
||||
// Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
|
||||
// "&". Strict XML parsers refuse it; browsers don't. Retry once with
|
||||
// every offending "&" escaped rather than fail outright.
|
||||
let escaped = escape_bare_ampersands(bytes);
|
||||
if escaped != bytes {
|
||||
if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) {
|
||||
return Ok(from_rss(ch, &escaped));
|
||||
}
|
||||
if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) {
|
||||
return Ok(from_atom(feed));
|
||||
}
|
||||
}
|
||||
Err(match alternate_feed_link(bytes) {
|
||||
Some(href) if looks_like_html(bytes) => {
|
||||
anyhow!("got a web page, not a feed; it links {href} as its feed")
|
||||
}
|
||||
None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"),
|
||||
_ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"),
|
||||
})
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed
|
||||
/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a
|
||||
/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case.
|
||||
fn looks_like_html(bytes: &[u8]) -> bool {
|
||||
let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase();
|
||||
head.contains("<!doctype html") || head.contains("<html")
|
||||
}
|
||||
|
||||
/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml"
|
||||
/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now,
|
||||
/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos.
|
||||
fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
|
||||
let text = String::from_utf8_lossy(bytes);
|
||||
let lower = text.to_lowercase();
|
||||
let mut pos = 0;
|
||||
while let Some(rel) = lower[pos..].find("<link") {
|
||||
let start = pos + rel;
|
||||
let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
|
||||
pos = end + 1;
|
||||
let tag = &text[start..end];
|
||||
let tag_lower = &lower[start..end];
|
||||
let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'");
|
||||
let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml");
|
||||
if is_alternate && is_feed_type
|
||||
&& let Some(href) = tag_attr(tag, "href")
|
||||
{
|
||||
return Some(href);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The value of one attribute in an HTML/XML start tag, however it is quoted.
|
||||
fn tag_attr(tag: &str, name: &str) -> Option<String> {
|
||||
let key = format!("{name}=");
|
||||
let idx = tag.to_lowercase().find(&key)?;
|
||||
let after = &tag[idx + key.len()..];
|
||||
let quote = after.chars().next()?;
|
||||
if quote != '"' && quote != '\'' {
|
||||
return None;
|
||||
}
|
||||
let rest = &after[1..];
|
||||
let close = rest.find(quote)?;
|
||||
Some(rest[..close].trim().to_owned())
|
||||
}
|
||||
|
||||
/// Escapes every `&` that does not already start a recognized XML entity
|
||||
/// (`&`, `<`, `>`, `"`, `'`, or a numeric reference like `'`).
|
||||
fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> {
|
||||
fn is_entity_start(rest: &[u8]) -> bool {
|
||||
for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] {
|
||||
if rest.starts_with(named) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") {
|
||||
&rest[2..]
|
||||
} else if rest.starts_with(b"#") {
|
||||
&rest[1..]
|
||||
} else {
|
||||
return false;
|
||||
};
|
||||
let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count();
|
||||
len > 0 && digits.get(len) == Some(&b';')
|
||||
}
|
||||
|
||||
let mut out = Vec::with_capacity(bytes.len());
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) {
|
||||
out.extend_from_slice(b"&");
|
||||
} else {
|
||||
out.push(bytes[i]);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every `<enclosure>` of every `<item>`, in document order.
|
||||
///
|
||||
/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
|
||||
@@ -591,6 +734,46 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explain_failure_translates_the_errors_the_ui_should_flag() {
|
||||
assert_eq!(
|
||||
explain_failure("HTTP 404 Not Found").unwrap().reason,
|
||||
"The publisher took this feed down, or moved it."
|
||||
);
|
||||
assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests.");
|
||||
assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests.");
|
||||
assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan.");
|
||||
let dns = explain_failure("connecting: dns error: failed to lookup address information").unwrap();
|
||||
assert_eq!(dns.reason, "This address no longer resolves; the site is gone.");
|
||||
let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
|
||||
assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
|
||||
assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
|
||||
for transient in ["HTTP 500 Internal Server Error", "HTTP 429 Too Many Requests", "operation timed out"] {
|
||||
assert!(explain_failure(transient).is_none(), "{transient} must not be flagged");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_web_page_says_so_and_names_the_feed_it_links() {
|
||||
let html = br#"<!doctype html><html><head>
|
||||
<link rel="alternate" type="application/rss+xml" href="https://x.example/feed">
|
||||
</head><body>not a feed</body></html>"#;
|
||||
let err = parse(html).unwrap_err().to_string();
|
||||
assert_eq!(err, "got a web page, not a feed; it links https://x.example/feed as its feed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_web_page_with_no_feed_link_still_says_so() {
|
||||
let html = b"<!doctype html><html><body>moved</body></html>";
|
||||
assert_eq!(parse(html).unwrap_err().to_string(), "got a web page, not a feed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_that_is_not_html_gets_the_original_parser_errors() {
|
||||
let err = parse(b"not xml at all").unwrap_err().to_string();
|
||||
assert!(err.starts_with("not RSS ("), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_atom_enclosure_links() {
|
||||
let bytes = include_bytes!("../tests/data/atom.xml");
|
||||
@@ -613,6 +796,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bare_ampersand_in_a_link_is_repaired_and_parsed() {
|
||||
// kcpw.org: <link>https://kcpw.org/?post_type=post&p=125715</link> -- a bare "&"
|
||||
// that strict XML rejects but browsers accept.
|
||||
let xml = br#"<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel><title>X</title><link>https://x</link><description>d</description>
|
||||
<item><title>a</title><guid>g1</guid>
|
||||
<link>https://kcpw.org/?post_type=post&p=125715</link>
|
||||
<enclosure url="https://x/a.mp3?a=1&b=2" length="1" type="audio/mpeg"/></item>
|
||||
</channel></rss>"#;
|
||||
let feed = parse(xml).unwrap();
|
||||
assert_eq!(feed.entries[0].link.as_deref(), Some("https://kcpw.org/?post_type=post&p=125715"));
|
||||
assert_eq!(feed.entries[0].enclosures[0].url, "https://x/a.mp3?a=1&b=2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escape_bare_ampersands_leaves_real_entities_alone() {
|
||||
let out = escape_bare_ampersands(b"a&b <x> ' / c&d");
|
||||
assert_eq!(out, b"a&b <x> ' / c&d");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
|
||||
// Some feeds set a different itunes:title. The displayed title is always the RSS
|
||||
|
||||
77
src/main.rs
77
src/main.rs
@@ -637,6 +637,7 @@ fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> {
|
||||
cfg.save(config_path)?;
|
||||
// State and files stay: re-adding the feed should not re-download its back catalogue.
|
||||
println!("removed {feed}; downloads and history kept");
|
||||
retire_group(ctx, feed)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -842,6 +843,10 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
|
||||
feed: id.clone(),
|
||||
reason: "not modified".into(),
|
||||
}),
|
||||
Ok(Outcome::Empty) => ctx.out.emit(Event::FeedSkip {
|
||||
feed: id.clone(),
|
||||
reason: "nothing yet".into(),
|
||||
}),
|
||||
Ok(Outcome::Opml { added, removed, kept, total }) => {
|
||||
ctx.out.emit(Event::FeedSkip {
|
||||
feed: id.clone(),
|
||||
@@ -919,6 +924,12 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
|
||||
continue; // promoted to config at some point; that entry wins
|
||||
}
|
||||
let parent = cfg.feeds.get(&m.group_id);
|
||||
if parent.is_none() {
|
||||
// The OPML or Patreon feed this was derived from is no longer in config --
|
||||
// removing it should have retired these rows too (see `retire_group`), but
|
||||
// skip them here regardless so a row that slips through is never scanned.
|
||||
continue;
|
||||
}
|
||||
let base = parent
|
||||
.and_then(|p| p.folder.clone())
|
||||
.or_else(|| ctx.db.feed_summary(&m.group_id).ok().and_then(|s| s.title))
|
||||
@@ -947,6 +958,22 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Retires every feed derived from `parent_id`, now that nothing subscribes to the OPML or
|
||||
/// Patreon feed that listed them: the same rule `sync_group` applies to one the list drops --
|
||||
/// removed if nothing was downloaded, orphaned and kept otherwise. Called once the parent
|
||||
/// itself is removed, since `subscriptions()` would otherwise keep scanning them under a
|
||||
/// fallback policy meant for a feed with no parent at all.
|
||||
pub fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
|
||||
for m in ctx.db.managed_feeds()?.into_iter().filter(|m| m.group_id == parent_id) {
|
||||
if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 {
|
||||
ctx.db.set_orphaned(&m.id, true)?;
|
||||
} else {
|
||||
ctx.db.drop_managed(&m.id)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Seconds to wait before re-checking a feed.
|
||||
///
|
||||
/// A per-feed schedule is an explicit instruction and wins outright. Without one, the
|
||||
@@ -971,6 +998,9 @@ struct Scan {
|
||||
/// What a scan of one feed turned out to be.
|
||||
enum Outcome {
|
||||
NotModified,
|
||||
/// A response with nothing in it -- the British Antarctic Survey answers a 202 with an
|
||||
/// empty body when it has nothing new to publish. Not a parse failure; try again later.
|
||||
Empty,
|
||||
Feed(Scan),
|
||||
/// The URL is a list of feeds rather than a feed: an OPML, or a Patreon creator's shows.
|
||||
Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
|
||||
@@ -1033,6 +1063,11 @@ async fn scan_one(
|
||||
feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified),
|
||||
};
|
||||
|
||||
if bytes.iter().all(u8::is_ascii_whitespace) {
|
||||
ctx.db.touch_feed(id, &feed_cfg.url)?;
|
||||
return Ok(Outcome::Empty);
|
||||
}
|
||||
|
||||
// A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml"
|
||||
// URL; sniffing the body also catches one served from a URL without that extension.
|
||||
if feed::is_opml(&bytes) {
|
||||
@@ -1635,4 +1670,46 @@ mod tests {
|
||||
let p = merge_policy(&[sub(None, Some(false), None), sub(None, Some(true), None)], &feed(), 3);
|
||||
assert!(p.auto_download);
|
||||
}
|
||||
|
||||
fn test_ctx(cfg: config::Config) -> Ctx {
|
||||
Ctx {
|
||||
cfg: std::sync::RwLock::new(Arc::new(cfg)),
|
||||
db: db::Db::memory().unwrap(),
|
||||
client: reqwest::Client::new(),
|
||||
out: Emitter::terminal(),
|
||||
torrents: tokio::sync::OnceCell::new(),
|
||||
torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),
|
||||
config_path: PathBuf::new(),
|
||||
detach_torrents: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() {
|
||||
// davewiner: the OPML subscription left config.toml, but its 922 derived rows
|
||||
// stayed in the database and kept being scanned under the no-parent fallback.
|
||||
let ctx = test_ctx(config::Config::default());
|
||||
ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").unwrap();
|
||||
assert!(
|
||||
subscriptions(&ctx).unwrap().iter().all(|s| s.id != "child"),
|
||||
"a derived feed whose parent is gone from config must not be scanned"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() {
|
||||
let ctx = test_ctx(config::Config::default());
|
||||
ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap();
|
||||
ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap();
|
||||
let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None };
|
||||
ctx.db.record_enclosure("has-file", "g1", &enc).unwrap();
|
||||
ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).unwrap();
|
||||
|
||||
retire_group(&ctx, "parent").unwrap();
|
||||
|
||||
let managed = ctx.db.managed_feeds().unwrap();
|
||||
assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten");
|
||||
assert!(managed.iter().any(|m| m.id == "has-file"), "has a file on disk, so it is kept");
|
||||
assert!(ctx.db.feed_summary("has-file").unwrap().orphaned, "and flagged as orphaned");
|
||||
}
|
||||
}
|
||||
|
||||
25
src/web.rs
25
src/web.rs
@@ -488,6 +488,9 @@ struct FeedRow {
|
||||
last_checked: Option<i64>,
|
||||
next_check: Option<i64>,
|
||||
last_error: Option<String>,
|
||||
/// Set once `last_error` is a kind worth telling someone about and it has held for a
|
||||
/// day -- a feed that fails once and reads fine an hour later (macmanx) never gets here.
|
||||
failing: Option<FailingRow>,
|
||||
entries: i64,
|
||||
downloaded: i64,
|
||||
unread: i64,
|
||||
@@ -495,6 +498,15 @@ struct FeedRow {
|
||||
subscribers: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct FailingRow {
|
||||
reason: &'static str,
|
||||
new_url: Option<String>,
|
||||
}
|
||||
|
||||
/// A day, in seconds: how long an error has to hold before the UI mentions it.
|
||||
const FLAG_AFTER_SECS: i64 = 86_400;
|
||||
|
||||
async fn feeds(
|
||||
State(state): State<WebState>,
|
||||
user: crate::db::User,
|
||||
@@ -558,6 +570,12 @@ async fn feeds(
|
||||
next_check: s
|
||||
.last_checked
|
||||
.map(|t| t + crate::due_after(&cfg, feed, st.ttl_mins) as i64),
|
||||
failing: s
|
||||
.error_since
|
||||
.filter(|since| crate::db::now() - since >= FLAG_AFTER_SECS)
|
||||
.and_then(|_| s.last_error.as_deref())
|
||||
.and_then(crate::feed::explain_failure)
|
||||
.map(|f| FailingRow { reason: f.reason, new_url: f.new_url }),
|
||||
last_error: s.last_error,
|
||||
entries: s.entries,
|
||||
downloaded: s.downloaded,
|
||||
@@ -917,9 +935,13 @@ fn entry_page(
|
||||
let mut rows =
|
||||
db.entries_in(user_id, feed, filter, search, page.offset, page.limit.clamp(1, 200), &order)?;
|
||||
// Feed HTML is untrusted: it reaches the page only after ammonia has been through it.
|
||||
// Every link opens in a new tab -- ammonia's default rel="noopener noreferrer" already
|
||||
// keeps that safe -- so following one in show notes never navigates away from ipx.
|
||||
let mut sanitizer = ammonia::Builder::new();
|
||||
sanitizer.add_tag_attributes("a", &["target"]).set_tag_attribute_value("a", "target", "_blank");
|
||||
for row in &mut rows {
|
||||
if let Some(d) = &row.description {
|
||||
row.description = Some(ammonia::clean(d));
|
||||
row.description = Some(sanitizer.clean(d).to_string());
|
||||
}
|
||||
}
|
||||
let total = db.count_in(user_id, feed, filter, search)?;
|
||||
@@ -1147,6 +1169,7 @@ async fn remove_feed(
|
||||
}
|
||||
cfg.save(&state.config_path)?;
|
||||
state.ctx.reload_cfg(&state.config_path)?;
|
||||
crate::retire_group(&state.ctx, &id)?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user