From d5ffe5d0420e68e3ff7c4dbcaac682880b2d38a4 Mon Sep 17 00:00:00 2001 From: rays Date: Sun, 18 Oct 2026 18:39:16 +0000 Subject: [PATCH] Switch an item between its HTML and its Markdown (#285) A feed that sends beside its HTML (Scripting News) has its Markdown kept in entries.markdown, added to existing databases by an ALTER. A stored item gets it when a scan finds it told differently, the Markdown counting as a change, the way corrections arrive (#141). The API sends it as written and made HTML, cleaned by ammonia like the description. The item's page has a choice between its meta line and its buttons: HTML, Markdown or Markdown source, remembered in the browser. Co-Authored-By: Claude Opus 5.5 --- crates/feed/src/lib.rs | 18 +++++++++++++++- crates/ipx/CHANGELOG.md | 1 + crates/ipx/src/db.rs | 38 ++++++++++++++++++++++++++++------ crates/ipx/src/entity.rs | 3 +++ crates/ipx/src/web.rs | 19 +++++++++++------ site/api.mjs | 2 ++ tests/ui/app.spec.js | 29 ++++++++++++++++++++++++++ tests/ui/fixtures/markdown.xml | 8 +++++++ web/CHANGELOG.md | 1 + web/app.css | 2 ++ web/src/items.ts | 21 ++++++++++++++++++- web/src/types.ts | 3 +++ 12 files changed, 131 insertions(+), 14 deletions(-) create mode 100644 tests/ui/fixtures/markdown.xml diff --git a/crates/feed/src/lib.rs b/crates/feed/src/lib.rs index 1c552c1..5b7da37 100644 --- a/crates/feed/src/lib.rs +++ b/crates/feed/src/lib.rs @@ -27,6 +27,9 @@ pub struct Entry { pub link: Option, pub published: Option, pub description: Option, + /// The item's Markdown, when the feed sends it beside HTML of its own (``): + /// the page can show either (#285). With no HTML it is the description instead (#284). + pub markdown: Option, pub categories: Vec, pub explicit: bool, /// Episode artwork; falls back to the feed's in the UI. @@ -770,6 +773,8 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed { // Content wins over description, as __getEntries preferred entry.content. The // source namespace's Markdown copy only when there is neither (#284). description: notes(item.content(), item.description(), source_markdown(item)), + markdown: non_empty(source_markdown(item)) + .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()), categories: item .categories() .iter() @@ -839,6 +844,7 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { link: alt.map(str::to_owned), published: e.published().or(Some(e.updated())).map(|d| d.timestamp()), description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None), + markdown: None, categories: e.categories().iter().map(|c| c.term().to_owned()).collect(), explicit: false, image: None, @@ -966,7 +972,8 @@ fn is_markdown(s: &str) -> bool { !tag && (s.contains("](") || s.contains("**") || s.lines().any(block)) } -fn markdown_html(md: &str) -> String { +/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML. +pub fn markdown_html(md: &str) -> String { use pulldown_cmark::{Options, Parser, html}; let mut out = String::new(); html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH)); @@ -1423,6 +1430,15 @@ Some **bold** and [a link](https://x.example/). assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is.")); } + #[test] + fn markdown_sent_beside_html_is_kept_for_the_page_to_switch_to() { + let xml = br#"R + both<p>Hi</p>*Hi* + alone*Hi*"#; + let got: Vec<_> = parse(xml).unwrap().entries.into_iter().map(|e| e.markdown).collect(); + assert_eq!(got, [Some("*Hi*".to_owned()), None], "only beside HTML; alone, it is the description (#284)"); + } + #[test] fn a_file_wordpress_lists_twice_is_one_enclosure() { let xml = br#"R diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index e67d0c2..785370d 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- An item whose feed sends Markdown beside its HTML, as Scripting News does with ``, keeps it: `markdown` and `markdown_html` in the API's items (#285). - An item written in Markdown shows formatted, not as Markdown's raw characters: a description in Markdown, or, with no other text, RSS's ``. HTML a feed sends is left as it is (#284). - A show from an OPML subscription or a Patreon creator can be kept: `kept` on `PATCH /api/feeds/{id}` and in the feed list. When the OPML drops a kept show, or you leave the OPML, it stays on your list as a feed of its own (#280). - `/metrics` counts Signals: items judged and fitting, calls to Jev by how they ended, the time they took and the tokens TypeSafe counted, time since the judge's last pass, and keys refused; a failed call to Jev shows as a failed trace (#270, #275). diff --git a/crates/ipx/src/db.rs b/crates/ipx/src/db.rs index dea0b43..86e3ca1 100644 --- a/crates/ipx/src/db.rs +++ b/crates/ipx/src/db.rs @@ -103,6 +103,12 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> { orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?; } } + if orm.execute_unprepared("SELECT markdown FROM entries LIMIT 0").await.is_err() { + // Stored items get theirs as a scan finds them told differently, by the Markdown alone. + orm.execute_unprepared("ALTER TABLE entries ADD COLUMN markdown TEXT") + .await + .context("adding entries.markdown")?; + } if orm.execute_unprepared("SELECT body_hash FROM feeds LIMIT 0").await.is_err() { for sql in ["ALTER TABLE feeds ADD COLUMN body_hash TEXT", "ALTER TABLE feeds ADD COLUMN same_since BIGINT"] { orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?; @@ -734,8 +740,8 @@ impl Db { .exec( "INSERT INTO entries (feed_id, guid, title, link, published, description, first_seen, - image, duration, episode, season) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + image, duration, episode, season, markdown) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT DO NOTHING", vec![ feed_id.into(), @@ -749,6 +755,7 @@ impl Db { e.duration.into(), e.episode.into(), e.season.into(), + e.markdown.clone().into(), ], ) .await?; @@ -760,7 +767,8 @@ impl Db { image = coalesce($5, image), duration = coalesce($6, duration), episode = coalesce($7, episode), - season = coalesce($8, season) + season = coalesce($8, season), + markdown = coalesce($9, markdown) WHERE feed_id = $1 AND guid = $2", vec![ feed_id.into(), @@ -771,6 +779,7 @@ impl Db { e.duration.into(), e.episode.into(), e.season.into(), + e.markdown.clone().into(), ], ) .await?; @@ -1131,6 +1140,10 @@ pub struct EntryRow { pub link: Option, pub published: Option, pub description: Option, + /// The item's Markdown as written, when its feed sends it beside the HTML, and that Markdown + /// made HTML and cleaned: the page switches between the three (#285). + pub markdown: Option, + pub markdown_html: Option, pub read: bool, pub flagged: bool, pub image: Option, @@ -1409,7 +1422,7 @@ impl Db { let (limit, offset) = (a.p(limit), a.p(offset)); let user = a.p(user_id); let sql = format!( - "SELECT e.guid, e.feed_id, e.title, e.link, e.published, e.description, + "SELECT e.guid, e.feed_id, e.title, e.link, e.published, e.description, e.markdown, coalesce(s.read, false) AS read, coalesce(s.flagged, false) AS flagged, e.image, coalesce(s.duration, e.duration) AS duration, e.episode, e.season, coalesce(s.position, 0) AS position @@ -1430,6 +1443,8 @@ impl Db { link: r.try_get("", "link")?, published: r.try_get("", "published")?, description: r.try_get("", "description")?, + markdown: r.try_get("", "markdown")?, + markdown_html: None, read: r.try_get("", "read")?, flagged: r.try_get("", "flagged")?, image: r.try_get("", "image")?, @@ -2338,11 +2353,12 @@ impl Db { let feed = a.p(feed_id); let guids: Vec = entries.iter().map(|e| a.p(e.guid.clone())).collect(); let sql = format!( - "SELECT guid, title, description, image, duration, episode, season FROM entries + "SELECT guid, title, description, image, duration, episode, season, markdown FROM entries WHERE feed_id = {feed} AND guid IN ({})", guids.join(", ") ); - type Stored = (Option, Option, Option, Option, Option, Option); + type Stored = + (Option, Option, Option, Option, Option, Option, Option); let mut stored = std::collections::HashMap::::new(); for r in self.rows(&sql, a.0).await? { stored.insert( @@ -2354,6 +2370,7 @@ impl Db { r.try_get("", "duration")?, r.try_get("", "episode")?, r.try_get("", "season")?, + r.try_get("", "markdown")?, ), ); } @@ -2370,6 +2387,7 @@ impl Db { || differs(&e.duration, &s.3) || differs(&e.episode, &s.4) || differs(&e.season, &s.5) + || differs(&e.markdown, &s.6) }) }) .map(|e| e.guid.clone()) @@ -3315,6 +3333,14 @@ mod tests { .unwrap(); assert_eq!(changed, ["a", "b"].map(String::from).into()); assert!(db.changed_items("f", &[]).await.unwrap().is_empty()); + + // A Markdown copy the feed starts sending is a change, and is stored (#285). + let md = crate::feed::Entry { markdown: Some("*as it was*".into()), ..item("c", "C", Some("as it was")) }; + assert_eq!(db.changed_items("f", std::slice::from_ref(&md)).await.unwrap(), ["c"].map(String::from).into()); + assert!(!db.record_entry("f", &md).await.unwrap(), "rewritten, not new"); + let order = order_sql("published", "desc"); + let rows = db.entries_in(1, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap(); + assert_eq!(rows.iter().find(|r| r.guid == "c").unwrap().markdown.as_deref(), Some("*as it was*")); } #[tokio::test] diff --git a/crates/ipx/src/entity.rs b/crates/ipx/src/entity.rs index 04b06a3..bc9b6c5 100644 --- a/crates/ipx/src/entity.rs +++ b/crates/ipx/src/entity.rs @@ -78,6 +78,9 @@ pub mod entries { #[sea_orm(column_type = "Text", nullable)] pub description: Option, pub first_seen: i64, + /// The item's Markdown, when its feed sends it beside the HTML (#285). + #[sea_orm(column_type = "Text", nullable)] + pub markdown: Option, #[sea_orm(column_type = "Text", nullable)] pub image: Option, pub duration: Option, diff --git a/crates/ipx/src/web.rs b/crates/ipx/src/web.rs index 9e2f0ee..d831501 100644 --- a/crates/ipx/src/web.rs +++ b/crates/ipx/src/web.rs @@ -1517,9 +1517,7 @@ async fn entry_page( step("sanitize").in_scope(|| { let mut sanitizer = feed_sanitizer(); for row in &mut rows { - if let Some(d) = &row.description { - row.description = Some(clean_description(&mut sanitizer, d, row.link.as_deref())); - } + clean_row(&mut sanitizer, row); } }); crate::deadair::for_reader(&state.ctx, user_id, &mut rows).instrument(step("deadair")).await?; @@ -1543,12 +1541,21 @@ async fn one_entry( let mut rows = [row]; crate::deadair::for_reader(&state.ctx, user.id, &mut rows).await?; let [mut row] = rows; - if let Some(d) = &row.description { - row.description = Some(clean_description(&mut feed_sanitizer(), d, row.link.as_deref())); - } + clean_row(&mut feed_sanitizer(), &mut row); Ok(Json(row)) } +/// An item's text as the page may show it: the feed's HTML cleaned, and its Markdown, where it +/// sent one, made HTML and cleaned the same way (#285). +fn clean_row(sanitizer: &mut ammonia::Builder<'static>, row: &mut crate::db::EntryRow) { + let link = row.link.clone(); + if let Some(d) = &row.description { + row.description = Some(clean_description(sanitizer, d, link.as_deref())); + } + row.markdown_html = + row.markdown.as_deref().map(|m| clean_description(sanitizer, &ipx_feed::markdown_html(m), link.as_deref())); +} + /// Feed HTML is untrusted: it reaches the page only after ammonia has been through it. fn feed_sanitizer() -> ammonia::Builder<'static> { let mut b = ammonia::Builder::new(); diff --git a/site/api.mjs b/site/api.mjs index d71766c..440ee30 100644 --- a/site/api.mjs +++ b/site/api.mjs @@ -519,6 +519,8 @@ export const ipx = { ['guid', 'String', ''], ['feed_id', 'String', ''], ['title', 'Option', ''], ['link', 'Option', ''], ['published', 'Option', ''], ['description', 'Option', 'Show notes, HTML, already sanitized.'], + ['markdown', 'Option', 'The item’s Markdown as written, when its feed sends Markdown beside its HTML (RSS’s source:markdown).'], + ['markdown_html', 'Option', 'That Markdown made HTML, sanitized as description is.'], ['read', 'bool', ''], ['flagged', 'bool', 'Kept: its file is never cleared to save space.'], ['image', 'Option', ''], ['duration', 'Option', 'Seconds.'], ['episode', 'Option', ''], ['season', 'Option', ''], diff --git a/tests/ui/app.spec.js b/tests/ui/app.spec.js index 38a840f..217947c 100644 --- a/tests/ui/app.spec.js +++ b/tests/ui/app.spec.js @@ -521,6 +521,35 @@ test('a show inside an OPML can be kept, and only it', async ({ page }) => { await expect(page.locator('#content [data-a="keep"]')).toHaveCount(0); }); +test('an item sent as HTML and Markdown switches between them', async ({ page }) => { + const api = (p, init) => page.evaluate(([p, init]) => fetch(p, { headers: { 'Content-Type': 'application/json' }, ...init }) + .then((r) => r.json().catch(() => null)), [p, init]); + const added = await api('/api/feeds', { method: 'POST', body: JSON.stringify({ url: 'http://127.0.0.1:8792/markdown.xml' }) }); + const id = added.id || added.feed?.id || 'markdown-blog'; + await page.reload(); + await page.locator('#feedlist .feed', { hasText: 'Markdown Blog' }).click({ timeout: 20_000 }); + await page.locator('#content .tabs button[data-f="all"]').click(); + await page.locator('.ep', { hasText: 'Both Ways' }).click(); + + // The publisher's HTML first; the choice is between the title and the item's buttons (#285). + const view = page.locator('#detail .dmeta .tview'); + await expect(view).toHaveValue('html'); + await expect(page.locator('#detail .dbody b')).toHaveText('HTML'); + await view.selectOption('markdown'); + await expect(page.locator('#detail .dbody strong')).toHaveText('Markdown'); + await view.selectOption('source'); + await expect(page.locator('#detail .dbody pre.mdsrc')).toHaveText('The **Markdown** copy.'); + // Kept for the next item; set back so nothing after this sees it. + expect(await page.evaluate(() => localStorage.getItem('ipx.textview'))).toBe('source'); + await view.selectOption('html'); + + // An item with no Markdown has no choice to make. + await page.locator('#feedlist .feed', { hasText: 'Test Show' }).first().click(); + await page.locator('.ep').first().click(); + await expect(page.locator('#detail .tview')).toHaveCount(0); + await api('/api/feeds/' + encodeURIComponent(id), { method: 'DELETE' }); +}); + test('marking an OPML subscription read covers the feeds inside it', async ({ page }) => { await page.evaluate(() => fetch('/api/fetch', { diff --git a/tests/ui/fixtures/markdown.xml b/tests/ui/fixtures/markdown.xml new file mode 100644 index 0000000..d6c31d0 --- /dev/null +++ b/tests/ui/fixtures/markdown.xml @@ -0,0 +1,8 @@ + + +Markdown Bloghttp://127.0.0.1:8792/HTML and Markdown both, as Scripting News sends. +Both Waysmd-1 + Mon, 01 Sep 2026 10:00:00 +0000 + <p>The <b>HTML</b> copy.</p> + The **Markdown** copy. + diff --git a/web/CHANGELOG.md b/web/CHANGELOG.md index 93828fd..06e3d6a 100644 --- a/web/CHANGELOG.md +++ b/web/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- An item sent as both HTML and Markdown, as Scripting News's are, has a choice beside its buttons to show it as the publisher's HTML, as the Markdown, or as the Markdown source, kept from one item to the next (#285). - A show inside an OPML subscription has a Keep button, a plus, on its page, so it stays on your list as a show of its own if the OPML drops it or you unsubscribe from the OPML (#280). - Admin, Server: a setting to give everyone Signals on the server's TypeSafe key (#271). - Admin, Server: how many days back a new Signal reads (#276). diff --git a/web/app.css b/web/app.css index e311ecb..c462111 100644 --- a/web/app.css +++ b/web/app.css @@ -848,6 +848,8 @@ body.scan-this .fhead [data-a=scan] .i,body.scan-any .fhead [data-a=scanall] .i, /* The reader's buttons go as one: as items of the line itself, a long feed name took them onto the next line one at a time (#146). */ .dacts{display:flex;gap:inherit;flex:none} +.dmeta .tview{width:auto;padding:3px 6px;font-size:12px} +.dbody .mdsrc{white-space:pre-wrap;font:13px/1.6 ui-monospace,SFMono-Regular,Menlo,monospace} #detail .encbox{margin:0 0 10px} .dbody{font-size:14.5px;line-height:1.65;overflow-wrap:anywhere;color:var(--fg)} .dbody img{max-width:100%;height:auto;border-radius:6px} diff --git a/web/src/items.ts b/web/src/items.ts index a077fcc..7cb5774 100644 --- a/web/src/items.ts +++ b/web/src/items.ts @@ -376,6 +376,12 @@ function showDetail(e: Entry|null){ box.innerHTML=detailHtml(e); if(files) files.innerHTML=narrow?'':`
Files
${encs}`; $('#dback').onclick=()=>showDetail(null); + const view=box.querySelector('.tview'); + if(view) view.onchange=()=>{ + textView=view.value; + try{ localStorage.setItem('ipx.textview',textView); }catch{} + box.querySelector('.dbody')!.innerHTML=textOf(e); + }; for(const root of [box,files]) if(root) $$('button[data-a]',root).forEach(b=> b.onclick=()=>epAction(b.dataset.a!,e,null,b.dataset.enc?Number(b.dataset.enc):undefined)); syncPlayButtons(); @@ -401,6 +407,8 @@ function detailHtml(e: Entry){ [f&&esc(f.title||f.id), num, dateOf(e.published), e.duration&&clock(e.duration), e.enclosures.some(x=>x.deadair_pending)&&DEAD_PENDING] .filter(Boolean).map(s=>`${s}`).join('')} + ${e.markdown?``:''} `:''} ${narrow?encs:''} -
${(e.description&&e.description.trim())||'No show notes.'}
`; +
${textOf(e)}
`; +} + +/// How an item whose feed sends Markdown beside its HTML is shown, kept from one item to the +/// next (#285): the publisher's HTML, the Markdown made HTML, or the Markdown as written. +const TEXT_VIEWS=[['html','HTML'],['markdown','Markdown'],['source','Markdown source']]; +let textView=(()=>{ try{ return localStorage.getItem('ipx.textview'); }catch{ return null; } })()||'html'; +function textOf(e: Entry){ + // Both cleaned server-side with ammonia, as the description is. + if(e.markdown&&textView==='markdown'&&e.markdown_html) return e.markdown_html; + if(e.markdown&&textView==='source') return `
${esc(e.markdown)}
`; + return (e.description&&e.description.trim())||'No show notes.'; } /// One enclosure: a play button when the file is here, otherwise what it is and a way to get it. diff --git a/web/src/types.ts b/web/src/types.ts index ac03155..7b0f043 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -49,6 +49,9 @@ interface Entry { link: string | null; published: number | null; description: string | null; + /** The item's Markdown as written, when its feed sends it beside the HTML, and made HTML (#285). */ + markdown: string | null; + markdown_html: string | null; read: boolean; flagged: boolean; image: string | null;