From f1f605e180f93ea4d117544763229a94c126bcee Mon Sep 17 00:00:00 2001 From: rays Date: Sat, 19 Sep 2026 14:29:50 +0000 Subject: [PATCH] Keep a Substack subtitle above the post Substack puts a post's subtitle in and leaves it out of content:encoded, and body() took content:encoded alone, so the subtitle was lost (a known gap in CLAUDE.md). A description is now shown above the body, as

, when it is short plain text the body does not already contain. Podcast feeds that repeat their notes in both, whole or cut short with an ellipsis, are unchanged; the comparison is by words, since a tag taken out of the body leaves stray spaces around punctuation. Checked against Experimental History's feed (subtitles appear) and The Daily's (notes in both fields, shown once). Entries are inserted with ON CONFLICT DO NOTHING, so only posts first seen from now on get it. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 5 ++++ CLAUDE.md | 2 -- src/feed.rs | 77 ++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6edce1d..b165810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The daemon no longer prints the web token when it starts, so it stays out of `docker logs`. It says where the token is kept instead: `[web] token` in config.toml. +### Fixed + +- A Substack post shows its subtitle above the post, as Substack does. Only posts that arrive from + now on have it. + ## [0.8.2] - 2026-09-19 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index a22b627..0a20056 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -185,8 +185,6 @@ Deliberate simplifications get a `ponytail:` comment naming the ceiling and the * Cloudflare's `Cf-Access-Jwt-Assertion` is not verified — ipx trusts the hop plus `trusted_proxies` (documented in [docs/sso.md](docs/sso.md)). -* A feed's `` subtitle is dropped whenever `content:encoded` exists, which loses - Substack-style subtitles. # Command output diff --git a/src/feed.rs b/src/feed.rs index 61a1be1..e416af4 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -698,10 +698,51 @@ fn title_text(s: Option<&str>) -> Option { /// began halfway through a tag and the page showed the rest of the tag as text. The same item's /// `description` was whole. With no description to fall back on, a damaged body beats none. fn body(content: Option<&str>, description: Option<&str>) -> Option { - non_empty(content) - .filter(|c| !starts_mid_tag(c)) - .or_else(|| non_empty(description)) - .or_else(|| non_empty(content)) + match non_empty(content).filter(|c| !starts_mid_tag(c)) { + Some(c) => Some(match subtitle(&c, description) { + Some(s) => format!("

{}

{c}", quick_xml::escape::escape(s.as_str())), + None => c, + }), + None => non_empty(description).or_else(|| non_empty(content)), + } +} + +/// A description that is a subtitle rather than a second copy of the notes: Substack puts the +/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it. +/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a +/// description found in the body is not shown twice. +/// +/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its +/// notes passes too and shows above them, which reads fine; a real subtitle field would need an +/// `entries` column. +fn subtitle(body: &str, description: Option<&str>) -> Option { + let d = title_text(description)?; + if d.contains('<') || d.chars().count() > 300 { + return None; + } + // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut + // description ends in "…" or "[...]". + let words = |s: &str| { + s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::>().join(" ").to_lowercase() + }; + let want = words(&d); + let text = title_text(Some(&text_of(body))).unwrap_or_default(); + (!want.is_empty() && !words(&text).contains(&want)).then_some(d) +} + +/// HTML with its tags taken out, each replaced by a space so words either side stay apart. +fn text_of(html: &str) -> String { + let mut out = String::with_capacity(html.len()); + let mut in_tag = false; + for c in html.chars() { + match c { + '<' => in_tag = true, + '>' if in_tag => { in_tag = false; out.push(' '); } + _ if !in_tag => out.push(c), + _ => {} + } + } + out } /// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose @@ -841,12 +882,36 @@ mod tests { let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant">

What if

"#; let whole = r#"

What if

"#; assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole)); - assert_eq!(body(Some("

Notes

"), Some("Summary")).as_deref(), Some("

Notes

"), "a whole body wins"); - assert_eq!(body(Some("Plain notes, no tags."), Some("Summary")).as_deref(), Some("Plain notes, no tags.")); + assert_eq!(body(Some("

Notes

"), Some("

Notes

")).as_deref(), Some("

Notes

"), "a whole body wins"); + assert_eq!(body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(), Some("Plain notes, no tags.")); assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none"); assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary")); } + #[test] + fn a_subtitle_missing_from_the_body_is_kept_above_it() { + // Substack: the subtitle is the description, and content:encoded does not repeat it. + assert_eq!( + body(Some("

The post.

"), Some("Why the tag & I fell out")).as_deref(), + Some("

The post.

"), + "a description with markup in it is notes, not a subtitle", + ); + assert_eq!( + body(Some("

The post.

"), Some("Why Q&A threads go wrong")).as_deref(), + Some("

Why Q&A threads go wrong

The post.

"), + ); + // A podcast repeating its notes, whole, cut short, or differently spaced: shown once. + let notes = "

This week we talk about tape, drums and a very long list.

"; + for d in ["This week we talk about tape, drums and a very long list.", + "This week we talk about tape, drums…", + "This week we talk about\ntape [...]"] { + assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body"); + } + let long = "word ".repeat(80); + assert_eq!(body(Some("

The post.

"), Some(&long)).as_deref(), Some("

The post.

"), + "a long description is notes, not a subtitle"); + } + #[test] fn feed_level_explicit_overrides_entries() { let xml = br#"