From 3a8d564eb78b2470b1952830beeef70616eb883e Mon Sep 17 00:00:00 2001 From: rays Date: Sun, 18 Oct 2026 18:05:05 +0000 Subject: [PATCH] Items written in Markdown show formatted (#284) A description with Markdown and no HTML tag is rendered to HTML with pulldown-cmark, and an item with no text but RSS's (https://source.scripting.com/, as Scripting News sends beside its HTML) uses that. HTML is left alone, and plain text without Markdown in it stays as it was, so a line like "1984. ..." is not made a list. ammonia cleans the result as any feed's HTML. Stored items keep what they have: the scanner does not rewrite an item it has. Reference: http://scripting.com/2026/10/06/005559.html Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 19 ++++++++++ Cargo.toml | 1 + crates/directory/CHANGELOG.md | 4 +++ crates/feed/Cargo.toml | 1 + crates/feed/src/lib.rs | 68 +++++++++++++++++++++++++++++++++-- crates/ipx/CHANGELOG.md | 1 + 6 files changed, 91 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cc4f87..5e3d4c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1931,6 +1931,7 @@ dependencies = [ "atom_syndication", "chrono", "opml", + "pulldown-cmark", "quick-xml 0.42.0", "reqwest", "rss", @@ -3128,6 +3129,24 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags 2.13.1", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + +[[package]] +name = "pulldown-cmark-escape" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" + [[package]] name = "quanta" version = "0.12.6" diff --git a/Cargo.toml b/Cargo.toml index 6dac8e4..7f89f16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ opentelemetry = { version = "0.33", default-features = false, features = ["trace opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "gzip-http"] } opentelemetry_sdk = { version = "0.33", default-features = false, features = ["trace"] } opml = "1.1.6" +pulldown-cmark = { version = "0.13.4", default-features = false, features = ["html"] } mimalloc = "0.1.52" percent-encoding = "2.3.2" quick-xml = { version = "0.42.0", features = ["escape-html"] } diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index 2e95d14..df98b84 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- A feed's items written in Markdown show formatted on its page, from a Markdown description or RSS's `` (#284). + ### Security - ammonia, which cleans publishers' HTML, updated to 4.2.4 for three advisories, among them XSS through SVG animation tags; rustls to 0.23.46 for five and h2 to 0.4.20 for one (#281). diff --git a/crates/feed/Cargo.toml b/crates/feed/Cargo.toml index 5f656c0..883307e 100644 --- a/crates/feed/Cargo.toml +++ b/crates/feed/Cargo.toml @@ -9,6 +9,7 @@ anyhow.workspace = true atom_syndication.workspace = true chrono.workspace = true opml.workspace = true +pulldown-cmark.workspace = true quick-xml.workspace = true reqwest.workspace = true rss.workspace = true diff --git a/crates/feed/src/lib.rs b/crates/feed/src/lib.rs index 395ad9b..1c552c1 100644 --- a/crates/feed/src/lib.rs +++ b/crates/feed/src/lib.rs @@ -767,8 +767,9 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed { title: title_text(item.title()), link: non_empty(item.link()), published: item.pub_date().and_then(parse_date), - // Content wins over description, as __getEntries preferred entry.content. - description: body(item.content(), item.description()), + // Content wins over description, as __getEntries preferred entry.content. The + // source namespace's Markdown copy only when there is neither (#284). + description: notes(item.content(), item.description(), source_markdown(item)), categories: item .categories() .iter() @@ -837,7 +838,7 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { title: title_text(Some(e.title().as_str())), link: alt.map(str::to_owned), published: e.published().or(Some(e.updated())).map(|d| d.timestamp()), - description: body(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str())), + description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None), categories: e.categories().iter().map(|c| c.term().to_owned()).collect(), explicit: false, image: None, @@ -936,6 +937,42 @@ fn title_text(s: Option<&str>) -> Option { non_empty(Some(&out)) } +/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in +/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML. +fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option { + match body(content, description) { + Some(b) if is_markdown(&b) => Some(markdown_html(&b)), + Some(b) => Some(b), + None => non_empty(markdown).map(|m| markdown_html(&m)), + } +} + +/// ``, from https://source.scripting.com/: the item's text in Markdown. +/// +/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding +/// it to another prefix is not read, until one turns up. +fn source_markdown(item: &rss::Item) -> Option<&str> { + item.extensions().get("source")?.get("markdown")?.first()?.value() +} + +/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold. +/// Plain text without any is left as it was, since rendering it would change nothing but +/// could still read a line starting "1984." as a list. +fn is_markdown(s: &str) -> bool { + let tag = s + .match_indices('<') + .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!')); + let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m)); + !tag && (s.contains("](") || s.contains("**") || s.lines().any(block)) +} + +fn markdown_html(md: &str) -> String { + use pulldown_cmark::{Options, Parser, html}; + let mut out = String::new(); + html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH)); + out +} + /// An item's show notes: its full body when that is whole, else its description. /// /// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name @@ -1361,6 +1398,31 @@ mod tests { assert_eq!(parse(atom).unwrap().description.as_deref(), Some("Or here.")); } + #[test] + fn markdown_notes_are_made_html_and_html_is_left_alone() { + let feed = |items: &str| { + let xml = format!( + r#"R{items}"# + ); + parse(xml.as_bytes()).unwrap().entries.into_iter().map(|e| e.description).collect::>() + }; + let got = feed( + r#"a# Hi + +Some **bold** and [a link](https://x.example/). + bOnly *Markdown* here. + c<p>HTML **as sent**</p>ignored + d1984. Plain text, left as it is."#, + ); + assert_eq!( + got[0].as_deref(), + Some("

Hi

\n

Some bold and a link.

\n") + ); + assert_eq!(got[1].as_deref(), Some("

Only Markdown here.

\n"), "the Markdown copy, alone"); + assert_eq!(got[2].as_deref(), Some("

HTML **as sent**

"), "HTML wins, untouched"); + assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is.")); + } + #[test] fn a_file_wordpress_lists_twice_is_one_enclosure() { let xml = br#"R diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index f60de0c..9eec4d8 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- An item written in Markdown shows formatted, not as Markdown's raw characters: a description in Markdown, or, with no other text, RSS's ``. HTML a feed sends is left as it is. For items new from now on (#284). - A show from an OPML subscription or a Patreon creator can be kept: `kept` on `PATCH /api/feeds/{id}` and in the feed list. When the OPML drops a kept show, or you leave the OPML, it stays on your list as a feed of its own (#280). - `/metrics` counts Signals: items judged and fitting, calls to Jev by how they ended, the time they took and the tokens TypeSafe counted, time since the judge's last pass, and keys refused; a failed call to Jev shows as a failed trace (#270, #275). - An admin setting for how many days back a new Signal reads (#276).