Patreon creators split into their shows; filters follow settings
A Patreon token pasted into Add feed, or a creator link without &show=, becomes a folder of that creator's shows, found through Patreon's web API and kept in step like a subscribed OPML (sync_group, split out of sync_opml). A creator already read as one feed is split too: each show takes over the files and read state it held (Db::adopt). A creator with one show stays a plain feed. Filter verdicts are judged again every scan, so turning on Allow explicit brings skipped items back. Add feed has an explicit box. Feeds in a group follow your settings on the group, as its dialog said. A new feed no longer takes the id of a removed one at a different URL and shows its old items. See CHANGELOG.md [Unreleased] and docs/history.md. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wi22VSVrkAvqNj61eqHsm9
This commit is contained in:
134
src/db.rs
134
src/db.rs
@@ -920,14 +920,20 @@ impl Db {
|
||||
|
||||
/// Everyone's settings for one feed. The scanner merges these into what it fetches
|
||||
/// and downloads, since one file serves the lot.
|
||||
pub fn subscribers(&self, feed_id: &str) -> Result<Vec<Sub>> {
|
||||
/// In a group, whatever someone has not set on the feed itself comes from their
|
||||
/// subscription to the group, as the group's settings dialog has always said it does.
|
||||
pub fn subscribers(&self, feed_id: &str, group: Option<&str>) -> Result<Vec<Sub>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT keywords, auto_download, allow_explicit, max_new_per_check
|
||||
FROM subscriptions WHERE feed_id = ?1",
|
||||
"SELECT coalesce(c.keywords, p.keywords), coalesce(c.auto_download, p.auto_download),
|
||||
coalesce(c.allow_explicit, p.allow_explicit),
|
||||
coalesce(c.max_new_per_check, p.max_new_per_check)
|
||||
FROM subscriptions c
|
||||
LEFT JOIN subscriptions p ON p.user_id = c.user_id AND p.feed_id = ?2
|
||||
WHERE c.feed_id = ?1",
|
||||
)?;
|
||||
let out = stmt
|
||||
.query_map([feed_id], |r| {
|
||||
.query_map(params![feed_id, group], |r| {
|
||||
Ok(Sub {
|
||||
feed_id: feed_id.to_string(),
|
||||
keywords: r
|
||||
@@ -1325,6 +1331,71 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Empties a feed of its items, leaving its files alone.
|
||||
pub fn clear_entries(&self, feed_id: &str) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute("DELETE FROM entries WHERE feed_id = ?1", [feed_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every feed the database holds rows for, as (id, url), removed ones included.
|
||||
pub fn feed_urls(&self) -> Result<Vec<(String, String)>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare("SELECT id, coalesce(url, '') FROM feeds")?;
|
||||
let out = stmt
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Hands a feed in a group the enclosures its parent holds, as (guid, url), with everyone's
|
||||
/// read state for them. A Patreon creator read as one feed before it was split into shows
|
||||
/// owns every show's files, and `enclosures.url` is unique, so without this each show would
|
||||
/// list its items with nothing to play.
|
||||
pub fn adopt(&self, parent: &str, child: &str, listed: &[(&str, &str)]) -> Result<()> {
|
||||
let mut conn = self.conn.lock().unwrap();
|
||||
let holds: bool = conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM enclosures WHERE feed_id = ?1)",
|
||||
[parent],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if !holds {
|
||||
return Ok(()); // An OPML, or a creator already shared out.
|
||||
}
|
||||
let tx = conn.transaction()?;
|
||||
for &(guid, url) in listed {
|
||||
let moved = tx.execute(
|
||||
"UPDATE enclosures SET feed_id = ?3, guid = ?4 WHERE url = ?1 AND feed_id = ?2",
|
||||
params![url, parent, child, guid],
|
||||
)?;
|
||||
if moved == 1 {
|
||||
// Patreon gives a post the same guid in every feed it appears in.
|
||||
tx.execute(
|
||||
"UPDATE OR IGNORE entry_state SET feed_id = ?2 WHERE feed_id = ?1 AND guid = ?3",
|
||||
params![parent, child, guid],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A feed's enclosures skipped by one of its filters, by URL, with the reason: the verdicts a
|
||||
/// change of settings can overturn. A torrent held back while torrents are off is not a
|
||||
/// filter's call.
|
||||
pub fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT url, last_error FROM enclosures
|
||||
WHERE feed_id = ?1 AND state = 'skipped' AND last_error IS NOT NULL
|
||||
AND last_error != 'torrents disabled'",
|
||||
)?;
|
||||
let out = stmt
|
||||
.query_map([feed_id], |r| Ok((r.get(0)?, r.get(1)?)))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
@@ -1561,6 +1632,61 @@ mod tests {
|
||||
assert_eq!(state(4), "done");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_show_takes_over_what_its_creator_held() {
|
||||
let db = Db::memory().unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0);
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES
|
||||
(1,'creator','a','u1','done','/x/a.mp3',NULL),
|
||||
(2,'creator','b','u2','skipped',NULL,'explicit'),
|
||||
(3,'creator','c','u3','skipped',NULL,'explicit'),
|
||||
(4,'other','d','u4','skipped',NULL,'torrents disabled');
|
||||
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',1);",
|
||||
)
|
||||
.unwrap();
|
||||
db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).unwrap();
|
||||
{
|
||||
let conn = db.conn.lock().unwrap();
|
||||
let owner = |id: i64| -> String {
|
||||
conn.query_row("SELECT feed_id FROM enclosures WHERE id = ?1", [id], |r| r.get(0)).unwrap()
|
||||
};
|
||||
assert_eq!(owner(1), "show", "a downloaded file moves with its item");
|
||||
assert_eq!(owner(2), "show");
|
||||
assert_eq!(owner(3), "creator", "this show does not list it");
|
||||
assert_eq!(owner(4), "other", "only the parent's are taken");
|
||||
let read: String = conn
|
||||
.query_row("SELECT feed_id FROM entry_state WHERE user_id = 1 AND guid = 'a'", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(read, "show", "what you had read stays read");
|
||||
}
|
||||
|
||||
// Only a filter's verdict can be overturned by a change of settings.
|
||||
let skipped = db.skipped_by_filter("show").unwrap();
|
||||
assert_eq!(skipped.get("u2").map(String::as_str), Some("explicit"));
|
||||
assert_eq!(skipped.len(), 1);
|
||||
assert!(db.skipped_by_filter("other").unwrap().is_empty(), "torrents disabled is not a filter");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_feed_in_a_group_follows_your_settings_on_the_group() {
|
||||
let db = Db::memory().unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO users (id, name, is_admin, created) VALUES (1,'ray',1,0),(2,'sam',0,0);
|
||||
INSERT INTO subscriptions (user_id, feed_id, allow_explicit, created) VALUES
|
||||
(1,'group',1,0),(1,'show',NULL,0),(2,'group',1,0),(2,'show',0,0);",
|
||||
)
|
||||
.unwrap();
|
||||
let explicit = |group| -> Vec<Option<bool>> {
|
||||
let mut v: Vec<_> =
|
||||
db.subscribers("show", group).unwrap().into_iter().map(|s| s.allow_explicit).collect();
|
||||
v.sort();
|
||||
v
|
||||
};
|
||||
assert_eq!(explicit(Some("group")), [Some(false), Some(true)], "ray inherits; sam's own choice on the show wins");
|
||||
assert_eq!(explicit(None), [None, Some(false)], "outside a group nothing is inherited");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enclosure_url_is_the_dedupe_key() {
|
||||
let db = Db::memory().unwrap();
|
||||
|
||||
136
src/feed.rs
136
src/feed.rs
@@ -117,6 +117,103 @@ pub fn opml_title(bytes: &[u8]) -> Option<String> {
|
||||
.filter(|t| !t.is_empty())
|
||||
}
|
||||
|
||||
/// The token and show of a Patreon feed link, or None for any other URL.
|
||||
///
|
||||
/// Patreon gives each patron one token per creator. With no show it stands for the creator,
|
||||
/// whose feed carries every show at once.
|
||||
fn patreon_parts(url: &str) -> Option<(String, Option<String>)> {
|
||||
let u = url::Url::parse(url).ok()?;
|
||||
if !matches!(u.host_str()?, "patreon.com" | "www.patreon.com") || !u.path().starts_with("/rss") {
|
||||
return None;
|
||||
}
|
||||
let param = |name: &str| u.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v.into_owned());
|
||||
Some((param("auth")?, param("show")))
|
||||
}
|
||||
|
||||
/// A Patreon link naming a creator but no show.
|
||||
pub fn is_patreon_creator(url: &str) -> bool {
|
||||
matches!(patreon_parts(url), Some((_, None)))
|
||||
}
|
||||
|
||||
/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's
|
||||
/// feed, since the token alone says whose it is.
|
||||
pub fn expand_input(input: &str) -> String {
|
||||
let s = input.trim();
|
||||
let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
|
||||
if token { format!("https://www.patreon.com/rss?auth={s}") } else { s.to_owned() }
|
||||
}
|
||||
|
||||
/// Whether two URLs are the same feed. One Patreon show has several spellings -- by the
|
||||
/// creator's name, by number, or with no creator at all -- and the token and show are what
|
||||
/// identify it.
|
||||
pub fn same_feed(a: &str, b: &str) -> bool {
|
||||
a == b || patreon_parts(a).is_some_and(|p| Some(p) == patreon_parts(b))
|
||||
}
|
||||
|
||||
/// A Patreon creator's name and shows, each show as (title, feed URL).
|
||||
///
|
||||
/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes,
|
||||
/// finding shows stops and the show feeds already found keep working. The documented API
|
||||
/// needs an OAuth client per install and does not list shows.
|
||||
pub async fn patreon_shows(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
) -> Result<(Option<String>, Vec<(String, String)>)> {
|
||||
// The creator feed names its campaign by number in its self link, a few hundred bytes in.
|
||||
// The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up.
|
||||
let mut resp = client.get(url).send().await.context("connecting")?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status()));
|
||||
}
|
||||
let mut head = Vec::new();
|
||||
while patreon_campaign(&head).is_none() && head.len() < 64 * 1024 {
|
||||
let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break };
|
||||
head.extend_from_slice(&chunk);
|
||||
}
|
||||
let campaign = patreon_campaign(&head)
|
||||
.ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;
|
||||
|
||||
let api = format!(
|
||||
"https://www.patreon.com/api/campaigns/{campaign}\
|
||||
?include=shows&fields%5Bcampaign%5D=name&fields%5Bcollection%5D=title"
|
||||
);
|
||||
let resp = client.get(api).send().await.context("asking Patreon for the shows")?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(anyhow!("Patreon would not list the shows: HTTP {}", resp.status()));
|
||||
}
|
||||
let (name, shows) = parse_patreon_shows(&resp.bytes().await.context("reading the shows")?)?;
|
||||
Ok((name, shows.into_iter().map(|(id, title)| (title, format!("{url}&show={id}"))).collect()))
|
||||
}
|
||||
|
||||
/// The campaign number in the start of a Patreon feed.
|
||||
fn patreon_campaign(head: &[u8]) -> Option<String> {
|
||||
let text = String::from_utf8_lossy(head);
|
||||
text.match_indices("patreon.com/rss/").find_map(|(i, m)| {
|
||||
let id: String = text[i + m.len()..].chars().take_while(char::is_ascii_digit).collect();
|
||||
(!id.is_empty()).then_some(id)
|
||||
})
|
||||
}
|
||||
|
||||
/// A campaign's name and its shows as (id, title), from Patreon's JSON:API answer.
|
||||
fn parse_patreon_shows(json: &[u8]) -> Result<(Option<String>, Vec<(String, String)>)> {
|
||||
let v: serde_json::Value = serde_json::from_slice(json).context("Patreon's answer is not JSON")?;
|
||||
// Missing is not the same as none. Read as no shows, the creator feed would be scanned as
|
||||
// a plain feed, claim every show's files, and leave the shows empty once the list returned.
|
||||
let ids = v["data"]["relationships"]["shows"]["data"]
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("Patreon's answer does not list the shows"))?;
|
||||
let title = |id: &str| -> Option<String> {
|
||||
let show = v["included"].as_array()?.iter().find(|x| x["type"] == "collection" && x["id"] == id)?;
|
||||
show["attributes"]["title"].as_str().map(|t| t.trim().to_owned())
|
||||
};
|
||||
let shows = ids
|
||||
.iter()
|
||||
.filter_map(|s| s["id"].as_str())
|
||||
.map(|id| (id.to_owned(), title(id).unwrap_or_else(|| format!("Show {id}"))))
|
||||
.collect();
|
||||
Ok((v["data"]["attributes"]["name"].as_str().map(str::to_owned), shows))
|
||||
}
|
||||
|
||||
/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
|
||||
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
|
||||
match rss::Channel::read_from(bytes) {
|
||||
@@ -550,6 +647,45 @@ mod tests {
|
||||
assert!(!is_opml(include_bytes!("../tests/data/atom.xml")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_patreon_creator_is_a_list_of_its_shows() {
|
||||
let tok = "AbCdEfGhIjKlMnOpQrStUvWxYz012_-9";
|
||||
assert_eq!(expand_input(&format!(" {tok} ")), format!("https://www.patreon.com/rss?auth={tok}"));
|
||||
assert_eq!(expand_input("https://example.com/rss"), "https://example.com/rss");
|
||||
|
||||
assert!(is_patreon_creator(&format!("https://www.patreon.com/rss/glasscannon?auth={tok}")));
|
||||
assert!(is_patreon_creator(&format!("https://www.patreon.com/rss?auth={tok}")));
|
||||
assert!(!is_patreon_creator(&format!("https://www.patreon.com/rss/x?auth={tok}&show=1")), "one show is a feed");
|
||||
assert!(!is_patreon_creator(&format!("https://example.com/rss?auth={tok}")));
|
||||
|
||||
// The show you already have by name is the one a bare token would add by number.
|
||||
assert!(same_feed(
|
||||
&format!("https://www.patreon.com/rss/glasscannon?auth={tok}&show=2073588"),
|
||||
&format!("https://www.patreon.com/rss?auth={tok}&show=2073588"),
|
||||
));
|
||||
assert!(!same_feed(
|
||||
&format!("https://www.patreon.com/rss?auth={tok}&show=1"),
|
||||
&format!("https://www.patreon.com/rss?auth={tok}&show=2"),
|
||||
));
|
||||
|
||||
// The self link carries the campaign by number, whichever spelling was asked for.
|
||||
let head = br#"<rss><channel><link>https://www.patreon.com/glasscannon</link>
|
||||
<atom:link href="https://www.patreon.com/rss/369921?auth=t" rel="self"/>"#;
|
||||
assert_eq!(patreon_campaign(head).as_deref(), Some("369921"));
|
||||
assert_eq!(patreon_campaign(b"<rss><channel><title>T"), None);
|
||||
|
||||
let json = br#"{"data":{"id":"369921","type":"campaign","attributes":{"name":"The Glass Cannon Network"},
|
||||
"relationships":{"shows":{"data":[{"id":"2073588","type":"collection"},{"id":"2073636","type":"collection"}]}}},
|
||||
"included":[{"id":"2073588","type":"collection","attributes":{"title":"Get in the Trunk "}},
|
||||
{"id":"2073636","type":"collection","attributes":{"title":"Shadowdark"}}]}"#;
|
||||
let (name, shows) = parse_patreon_shows(json).unwrap();
|
||||
assert_eq!(name.as_deref(), Some("The Glass Cannon Network"));
|
||||
assert_eq!(shows, [("2073588".into(), "Get in the Trunk".into()), ("2073636".into(), "Shadowdark".into())]);
|
||||
|
||||
// An answer that stops naming the shows is an error, never "this creator has none".
|
||||
assert!(parse_patreon_shows(br#"{"data":{"attributes":{"name":"X"}}}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_item_may_carry_several_enclosures() {
|
||||
// The rss crate keeps only one per item -- the last -- so these come from the XML.
|
||||
|
||||
105
src/main.rs
105
src/main.rs
@@ -517,8 +517,9 @@ async fn add(
|
||||
keywords: Vec<String>,
|
||||
) -> Result<()> {
|
||||
let mut cfg = (*ctx.cfg()).clone();
|
||||
let url = &feed::expand_input(url);
|
||||
// Includes feeds derived from an OPML, or the same show could be added twice.
|
||||
if let Some(existing) = subscriptions(ctx)?.iter().find(|s| s.cfg.url == url) {
|
||||
if let Some(existing) = subscriptions(ctx)?.iter().find(|s| feed::same_feed(&s.cfg.url, url)) {
|
||||
anyhow::bail!("already subscribed as {:?}", existing.id);
|
||||
}
|
||||
let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
|
||||
@@ -569,10 +570,17 @@ pub async fn add_one(
|
||||
|
||||
// Slugs must be unique across derived feeds too, or a new feed can collide with one
|
||||
// an OPML already introduced.
|
||||
let taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx)?
|
||||
let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx)?
|
||||
.into_iter()
|
||||
.map(|s| (s.id, s.cfg))
|
||||
.collect();
|
||||
// A removed feed keeps its rows, so its id is only free again for the same feed: re-adding
|
||||
// it gets its history back, and a different feed does not inherit someone else's.
|
||||
for (id, other) in ctx.db.feed_urls()? {
|
||||
if !feed::same_feed(&other, url) {
|
||||
taken.entry(id).or_insert_with(|| probe.clone());
|
||||
}
|
||||
}
|
||||
let id = config::unique_slug(&title, &taken);
|
||||
cfg.feeds.insert(id.clone(), probe);
|
||||
Ok(id)
|
||||
@@ -807,7 +815,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
|
||||
ctx.out.emit(Event::FeedSkip {
|
||||
feed: id.clone(),
|
||||
reason: format!(
|
||||
"OPML: {total} feed(s) listed, {} added, {removed} unsubscribed, {kept} kept without a listing",
|
||||
"{total} feed(s) listed, {} added, {removed} unsubscribed, {kept} kept without a listing",
|
||||
added.len()
|
||||
),
|
||||
});
|
||||
@@ -963,7 +971,7 @@ struct Scan {
|
||||
enum Outcome {
|
||||
NotModified,
|
||||
Feed(Scan),
|
||||
/// The URL served an OPML document, so it is a subscription list rather than a feed.
|
||||
/// The URL is a list of feeds rather than a feed: an OPML, or a Patreon creator's shows.
|
||||
Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
|
||||
}
|
||||
|
||||
@@ -973,6 +981,31 @@ async fn scan_one(
|
||||
feed_cfg: &config::Feed,
|
||||
state: &db::HttpState,
|
||||
) -> Result<Outcome> {
|
||||
// A Patreon creator with more than one show is a list of feeds, like an OPML.
|
||||
if feed::is_patreon_creator(&feed_cfg.url) {
|
||||
match feed::patreon_shows(&ctx.client, &feed_cfg.url).await {
|
||||
Ok((name, shows)) if shows.len() > 1 => {
|
||||
ctx.db.touch_feed(id, &feed_cfg.url)?;
|
||||
if let Some(name) = name {
|
||||
ctx.db.set_title(id, &name)?;
|
||||
}
|
||||
// Read as one feed before it was split, it listed every show's items in one
|
||||
// heap. The items go; its files and read state move to each show as the show
|
||||
// lists them (`Db::adopt`), so no show comes up empty for want of a URL.
|
||||
ctx.db.clear_entries(id)?;
|
||||
return sync_group(ctx, id, feed_cfg, &shows).await;
|
||||
}
|
||||
Ok(_) => {} // One show: the creator's feed is that show.
|
||||
// Already split: keep the shows it has rather than read the creator as one heap.
|
||||
Err(e) if ctx.db.managed_feeds()?.iter().any(|m| m.group_id == id) => return Err(e),
|
||||
Err(e) => tracing::warn!(
|
||||
feed = id,
|
||||
error = %format!("{e:#}"),
|
||||
"could not list the Patreon shows; reading it as one feed"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
let mut fetched = feed::fetch(
|
||||
&ctx.client,
|
||||
feed_cfg,
|
||||
@@ -1018,19 +1051,38 @@ async fn scan_one(
|
||||
)?;
|
||||
|
||||
let policy = policy_for(ctx, id, feed_cfg)?;
|
||||
if let Some(parent) = &feed_cfg.group {
|
||||
let listed: Vec<(&str, &str)> = parsed
|
||||
.entries
|
||||
.iter()
|
||||
.flat_map(|e| e.enclosures.iter().map(move |x| (e.guid.as_str(), x.url.as_str())))
|
||||
.collect();
|
||||
ctx.db.adopt(parent, id, &listed)?;
|
||||
}
|
||||
// Verdicts are recorded in `state`, so the download queue below is just "everything still
|
||||
// pending". A filter's verdict is looked at again on every scan, though: made once, at
|
||||
// discovery, it outlived the setting behind it, and allowing explicit items afterwards
|
||||
// changed nothing however often the feed was scanned.
|
||||
let skipped = ctx.db.skipped_by_filter(id)?;
|
||||
let mut scan = Scan::default();
|
||||
for entry in &parsed.entries {
|
||||
if ctx.db.record_entry(id, entry)? {
|
||||
scan.new_entries += 1;
|
||||
}
|
||||
for enc in &entry.enclosures {
|
||||
if !ctx.db.record_enclosure(id, &entry.guid, enc)? {
|
||||
continue; // Seen before: downloaded, skipped or deliberately reaped.
|
||||
}
|
||||
// Filters run once, at discovery, and are recorded in `state`. The download
|
||||
// queue below is then just "everything still pending".
|
||||
if let Some(reason) = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc) {
|
||||
ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
|
||||
let was = if ctx.db.record_enclosure(id, &entry.guid, enc)? {
|
||||
None
|
||||
} else if let Some(reason) = skipped.get(&enc.url) {
|
||||
Some(reason.as_str())
|
||||
} else {
|
||||
continue; // Settled: queued, downloaded, reaped, or another feed's file.
|
||||
};
|
||||
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
|
||||
if now != was {
|
||||
match now {
|
||||
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?,
|
||||
None => ctx.db.mark_enclosure(&enc.url, "pending", None)?,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1114,12 +1166,6 @@ async fn scan_one(
|
||||
Ok(Outcome::Feed(scan))
|
||||
}
|
||||
|
||||
/// Brings the feed list in step with a subscribed OPML.
|
||||
///
|
||||
/// New entries are added under the OPML's group and folder. An entry that has gone from
|
||||
/// the OPML is unsubscribed *only if nothing was ever downloaded for it* -- otherwise it
|
||||
/// is kept and flagged, because dropping it would orphan files on disk with nothing in
|
||||
/// the UI to explain them.
|
||||
async fn sync_opml(
|
||||
ctx: &Arc<Ctx>,
|
||||
parent_id: &str,
|
||||
@@ -1130,25 +1176,44 @@ async fn sync_opml(
|
||||
if let Some(title) = feed::opml_title(bytes) {
|
||||
ctx.db.set_title(parent_id, &title)?;
|
||||
}
|
||||
sync_group(ctx, parent_id, parent, &listed).await
|
||||
}
|
||||
|
||||
/// Brings the feed list in step with a list of feeds: a subscribed OPML, or a Patreon
|
||||
/// creator's shows.
|
||||
///
|
||||
/// New entries are added under the list's group and folder. An entry that has gone from
|
||||
/// the list is unsubscribed *only if nothing was ever downloaded for it* -- otherwise it
|
||||
/// is kept and flagged, because dropping it would orphan files on disk with nothing in
|
||||
/// the UI to explain them.
|
||||
async fn sync_group(
|
||||
ctx: &Arc<Ctx>,
|
||||
parent_id: &str,
|
||||
parent: &config::Feed,
|
||||
listed: &[(String, String)],
|
||||
) -> Result<Outcome> {
|
||||
let cfg = ctx.cfg();
|
||||
let existing = ctx.db.managed_feeds()?;
|
||||
let mut added = vec![];
|
||||
|
||||
for (title, url) in &listed {
|
||||
for (title, url) in listed {
|
||||
// Already known, whether derived or promoted into the config.
|
||||
if let Some(m) = existing.iter().find(|m| &m.url == url) {
|
||||
ctx.db.upsert_managed(&m.id, url, title, parent_id)?;
|
||||
continue;
|
||||
}
|
||||
if cfg.feeds.values().any(|f| &f.url == url) {
|
||||
// A Patreon show you added by hand may be spelled differently from the one listed.
|
||||
if cfg.feeds.values().any(|f| feed::same_feed(&f.url, url)) {
|
||||
continue;
|
||||
}
|
||||
// A removed feed keeps its rows, so its id is only free again for the same feed.
|
||||
let known = ctx.db.feed_urls()?;
|
||||
let taken: std::collections::BTreeMap<String, config::Feed> = cfg
|
||||
.feeds
|
||||
.keys()
|
||||
.chain(existing.iter().map(|m| &m.id))
|
||||
.chain(added.iter())
|
||||
.chain(known.iter().filter(|(_, u)| !feed::same_feed(u, url)).map(|(id, _)| id))
|
||||
.map(|id| (id.clone(), parent.clone()))
|
||||
.collect();
|
||||
let id = config::unique_slug(title, &taken);
|
||||
@@ -1255,7 +1320,7 @@ pub struct Policy {
|
||||
|
||||
fn policy_for(ctx: &Ctx, id: &str, feed_cfg: &config::Feed) -> Result<Policy> {
|
||||
let global = ctx.cfg().general.max_new_per_check;
|
||||
Ok(merge_policy(&ctx.db.subscribers(id)?, feed_cfg, global))
|
||||
Ok(merge_policy(&ctx.db.subscribers(id, feed_cfg.group.as_deref())?, feed_cfg, global))
|
||||
}
|
||||
|
||||
fn merge_policy(subs: &[db::Sub], feed_cfg: &config::Feed, global: usize) -> Policy {
|
||||
|
||||
42
src/web.rs
42
src/web.rs
@@ -495,6 +495,9 @@ async fn feeds(
|
||||
let mut out = Vec::with_capacity(mine.len());
|
||||
for sub in &subs {
|
||||
let (id, feed) = (&sub.id, &sub.cfg);
|
||||
// In a group, what you have not set on the feed comes from your settings on the group,
|
||||
// the same fallback the scanner uses (`Db::subscribers`).
|
||||
let up = feed.group.as_deref().and_then(|g| mine.get(g));
|
||||
let Some(mine) = mine.get(id) else { continue };
|
||||
let s = state.ctx.db.feed_summary(id)?;
|
||||
let st = state.ctx.db.http_state(id)?;
|
||||
@@ -504,11 +507,22 @@ async fn feeds(
|
||||
title: s.title,
|
||||
image: s.image,
|
||||
folder: feed.folder.clone(),
|
||||
keywords: mine.keywords.clone().unwrap_or_else(|| feed.keywords.clone()),
|
||||
allow_explicit: mine.allow_explicit.unwrap_or(feed.allow_explicit),
|
||||
auto_download: mine.auto_download.unwrap_or(feed.auto_download),
|
||||
keywords: mine
|
||||
.keywords
|
||||
.clone()
|
||||
.or_else(|| up.and_then(|u| u.keywords.clone()))
|
||||
.unwrap_or_else(|| feed.keywords.clone()),
|
||||
allow_explicit: mine
|
||||
.allow_explicit
|
||||
.or(up.and_then(|u| u.allow_explicit))
|
||||
.unwrap_or(feed.allow_explicit),
|
||||
auto_download: mine
|
||||
.auto_download
|
||||
.or(up.and_then(|u| u.auto_download))
|
||||
.unwrap_or(feed.auto_download),
|
||||
max_new_per_check: mine
|
||||
.max_new_per_check
|
||||
.or(up.and_then(|u| u.max_new_per_check))
|
||||
.map(|n| n as usize)
|
||||
.or(feed.max_new_per_check),
|
||||
group: feed.group.clone(),
|
||||
@@ -893,6 +907,19 @@ struct NewFeed {
|
||||
folder: Option<String>,
|
||||
#[serde(default)]
|
||||
keywords: Vec<String>,
|
||||
#[serde(default)]
|
||||
allow_explicit: bool,
|
||||
}
|
||||
|
||||
/// The Add feed dialog's explicit box. Like everything on a feed's own dialog it is yours, so it
|
||||
/// goes on your subscription, and before the first scan, which would otherwise skip every
|
||||
/// explicit item.
|
||||
fn explicit_on_add(state: &WebState, user_id: i64, feed_id: &str, allow: bool) -> Result<(), ApiError> {
|
||||
if allow {
|
||||
let sub = crate::db::Sub { feed_id: feed_id.to_owned(), allow_explicit: Some(true), ..Default::default() };
|
||||
state.ctx.db.set_subscription(user_id, &sub)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn add_feed(
|
||||
@@ -901,23 +928,28 @@ async fn add_feed(
|
||||
Json(body): Json<NewFeed>,
|
||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||
let mut cfg = (*state.ctx.cfg()).clone();
|
||||
let url = crate::feed::expand_input(&body.url);
|
||||
// Someone else may already have it. Then adding costs nothing: no second fetch, no
|
||||
// second copy on disk, just another name against the same feed.
|
||||
if let Some(existing) = crate::subscriptions(&state.ctx)?
|
||||
.into_iter()
|
||||
.find(|s| s.cfg.url == body.url)
|
||||
.find(|s| crate::feed::same_feed(&s.cfg.url, &url))
|
||||
{
|
||||
let already = state.ctx.db.subscription(user.id, &existing.id)?.is_some();
|
||||
state.ctx.db.subscribe(user.id, &existing.id)?;
|
||||
if !already {
|
||||
explicit_on_add(&state, user.id, &existing.id, body.allow_explicit)?;
|
||||
}
|
||||
scan_soon(&state, Some(existing.id.clone())).await;
|
||||
return Ok(Json(
|
||||
serde_json::json!({ "id": existing.id, "existing": already }),
|
||||
));
|
||||
}
|
||||
let id = crate::add_one(&state.ctx, &mut cfg, &body.url, body.folder, body.keywords).await?;
|
||||
let id = crate::add_one(&state.ctx, &mut cfg, &url, body.folder, body.keywords).await?;
|
||||
cfg.save(&state.config_path)?;
|
||||
state.ctx.reload_cfg(&state.config_path)?;
|
||||
state.ctx.db.subscribe(user.id, &id)?;
|
||||
explicit_on_add(&state, user.id, &id, body.allow_explicit)?;
|
||||
scan_soon(&state, Some(id.clone())).await;
|
||||
Ok(Json(serde_json::json!({ "id": id, "existing": false })))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user