diff --git a/CHANGELOG.md b/CHANGELOG.md index cf61848..9ee3c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The long form, with what was wrong before and how it was found, is in - Directory shows each feed as its cover art in a grid, title and subscriber count underneath, instead of a list. Popular and the Add a feed dialog keep their rows. +- The pages are set in Inter, served by ipx itself. Classic keeps Lucida Grande. - Keeping an item is now pinning it: a thumbtack in place of the flag, and Pin, Pinned and Unpin in place of Keep, Kept and Stop keeping. A pinned item is still never deleted. - Currently Listening is its own place in the feed list, below Popular, instead of a section at @@ -33,6 +34,11 @@ The long form, with what was wrong before and how it was found, is in ### Fixed +- A WordPress post that embeds the file it encloses no longer lists, and downloads, that file + twice. Items that already had it twice are folded into one at startup, and the spare copy + deleted. +- The pinned column's heading lines up with the pins under it, and every heading sits a pixel + further right, over its column. - The feeds left behind by an OPML subscription removed before ipx retired them are cleared at startup: forgotten if nothing was downloaded, kept as orphaned if something was. Feeds from it that have since been given their own settings stay as they are, with their items. Removing an diff --git a/src/db.rs b/src/db.rs index b7974db..22c9882 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1345,6 +1345,58 @@ impl Db { Ok(()) } + /// Folds enclosures of one item that `key` says are the same file into the first of them, for + /// WordPress's numbered player URLs (`feed::same_file_key`). The first is the one the parser + /// keeps, so it keeps its row, taking a repeat's file if it has none of its own; the repeats' + /// rows go. Returns how many went and the copies left spare, for the caller to delete. + pub fn merge_repeated_enclosures(&self, key: impl Fn(&str) -> String) -> Result<(usize, Vec)> { + use std::collections::hash_map::Entry; + let mut conn = self.conn.lock().unwrap(); + let tx = conn.transaction()?; + let rows: Vec<(i64, String, String, String, Option)> = { + let mut stmt = tx.prepare( + "SELECT id, feed_id, guid, url, path FROM enclosures + WHERE (feed_id, guid) IN + (SELECT feed_id, guid FROM enclosures WHERE url GLOB '*[?&]_=[0-9]*') + ORDER BY id", + )?; + stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))? + .collect::>()? + }; + // The first row of each file, and whether it has the file on disk yet. + let mut first: std::collections::HashMap<(String, String, String), (i64, bool)> = Default::default(); + let (mut gone, mut spare) = (0, vec![]); + for (id, feed, guid, url, path) in rows { + match first.entry((feed, guid, key(&url))) { + Entry::Vacant(v) => { + v.insert((id, path.is_some())); + } + Entry::Occupied(mut o) => { + let (keep, has) = o.get_mut(); + if let Some(p) = path { + if *has { + spare.push(p); + } else { + // The only copy is the repeat's: Rands' episode 97 was downloaded + // under its ?_=2 URL alone. + tx.execute( + "UPDATE enclosures SET (path, state, bytes_done, downloaded_at) = + (SELECT path, state, bytes_done, downloaded_at FROM enclosures WHERE id = ?2) + WHERE id = ?1", + params![*keep, id], + )?; + *has = true; + } + } + tx.execute("DELETE FROM enclosures WHERE id = ?1", [id])?; + gone += 1; + } + } + } + tx.commit()?; + Ok((gone, spare)) + } + /// Stops treating a feed as derived, because it now has its own config entry. pub fn unmanage(&self, id: &str) -> Result<()> { let conn = self.conn.lock().unwrap(); @@ -1469,6 +1521,39 @@ pub fn now() -> i64 { mod tests { use super::*; + #[test] + fn a_file_wordpress_listed_twice_is_folded_into_one() { + let db = Db::memory().unwrap(); + db.exec_for_test( + "INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES + (1,'f','a','https://x/a.mp3','/d/a-2.mp3','done'), + (2,'f','a','https://x/a.mp3?_=2','/d/a.mp3','done'), + (3,'f','b','https://x/b.mp3',NULL,'reaped'), + (4,'f','b','https://x/b.mp3?_=2','/d/b.mp3','done'), + (5,'f','c','https://x/c.mp3?_=1','/d/c.mp3','done'), + (6,'f','d','https://x/d1.mp3?_=1','/d/d1.mp3','done'), + (7,'f','d','https://x/d2.mp3?_=2','/d/d2.mp3','done');", + ) + .unwrap(); + let key = crate::feed::same_file_key; + assert_eq!(db.merge_repeated_enclosures(key).unwrap(), (2, vec!["/d/a.mp3".to_string()])); + { + let conn = db.conn.lock().unwrap(); + let ids: Vec = conn + .prepare("SELECT id FROM enclosures ORDER BY id") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ids, [1, 3, 5, 6, 7], "a lone ?_=1 and two different files stay"); + let (path, state): (String, String) = + conn.query_row("SELECT path, state FROM enclosures WHERE id = 3", [], |r| Ok((r.get(0)?, r.get(1)?))).unwrap(); + assert_eq!((path.as_str(), state.as_str()), ("/d/b.mp3", "done"), "the only copy moves, not deleted"); + } + assert_eq!(db.merge_repeated_enclosures(key).unwrap(), (0, vec![]), "and only once"); + } + #[test] fn adding_category_drops_validators_once_and_keeps_the_schedule() { let conn = Connection::open_in_memory().unwrap(); diff --git a/src/feed.rs b/src/feed.rs index cd8c754..61a1be1 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -480,7 +480,7 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed { .filter_map(|(idx, item)| { // Straight from the XML, so an item with several keeps all of them. Falls // back to the parsed one if the scan and the parser disagree on item count. - let enclosures: Vec = per_item.get(idx).cloned().unwrap_or_else(|| { + let mut enclosures: Vec = per_item.get(idx).cloned().unwrap_or_else(|| { item.enclosure() .into_iter() .map(|e| Enclosure { @@ -491,6 +491,7 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed { .filter(|e| !e.url.is_empty()) .collect() }); + drop_player_repeats(&mut enclosures); let guid = pick_guid( item.guid().map(|g| g.value()), @@ -555,7 +556,7 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { .iter() .filter_map(|e| { // Atom carries enclosures as . - let enclosures: Vec = e + let mut enclosures: Vec = e .links() .iter() .filter(|l| l.rel() == "enclosure") @@ -566,6 +567,7 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed { }) .filter(|e| !e.url.is_empty()) .collect(); + drop_player_repeats(&mut enclosures); let alt = e .links() @@ -631,6 +633,31 @@ fn non_empty(s: Option<&str>) -> Option { s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned) } +/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post +/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion +/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays. +fn drop_player_repeats(encs: &mut Vec) { + let mut seen = std::collections::HashSet::new(); + encs.retain(|e| seen.insert(same_file_key(&e.url))); +} + +/// An enclosure URL without WordPress's player number, for telling repeats of one file apart +/// from different files. +pub fn same_file_key(url: &str) -> String { + let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() }; + let kept: Vec<(String, String)> = u + .query_pairs() + .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit()))) + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect(); + if kept.is_empty() { + u.set_query(None); + } else { + u.query_pairs_mut().clear().extend_pairs(kept); + } + u.to_string() +} + /// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through /// the XML parser with its HTML entities intact: The Verge's "Meta’s" reached the page as /// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is @@ -777,6 +804,21 @@ mod tests { ); } + #[test] + fn a_file_wordpress_lists_twice_is_one_enclosure() { + let xml = br#"R + The Promotion Paradoxp + + + + "#; + let urls: Vec = + parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect(); + assert_eq!(urls, ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"], "the repeat goes, a different file stays"); + assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k")); + assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number"); + } + #[test] fn titles_are_read_as_text_not_html() { // The Verge: an Atom title of type="html", its entity inside CDATA. diff --git a/src/main.rs b/src/main.rs index fda4451..787a4a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -398,6 +398,21 @@ async fn daemon( Err(e) => tracing::warn!(error = ?e, "could not retire feeds whose OPML is no longer in config"), } + // Before the parser knew WordPress's numbered player URLs, a file it listed twice was + // downloaded twice. The repeats fold into the first, and their spare copies are deleted. + match ctx.db.merge_repeated_enclosures(feed::same_file_key) { + Ok((0, _)) => {} + Ok((n, spare)) => { + for path in &spare { + if let Err(e) = std::fs::remove_file(path) { + tracing::warn!(path, error = %e, "could not delete a spare copy"); + } + } + tracing::info!(enclosures = n, files = spare.len(), "folded files WordPress listed twice"); + } + Err(e) => tracing::warn!(error = ?e, "could not fold files WordPress listed twice"), + } + let (tx_cmd, mut rx_cmd) = mpsc::channel::(64); let web = start_web(&ctx, &config_path, web_addr, &tx_cmd, &events).await?; diff --git a/src/web.rs b/src/web.rs index b9b047e..7f83807 100644 --- a/src/web.rs +++ b/src/web.rs @@ -65,6 +65,7 @@ pub fn router(state: WebState) -> Router { .route("/login", get(login_page)) .route("/api/login", post(login)) .route("/icon.png", get(icon)) + .route("/inter.woff2", get(inter)) .layer(middleware::from_fn(access_log)) .with_state(state) } @@ -452,6 +453,16 @@ async fn icon() -> impl IntoResponse { ) } +/// Inter, the pages' typeface, served from the binary as the icon is, so neither page loads +/// anything from anyone else. Outside the auth layer for the sign-in page. Its licence, the SIL +/// Open Font License, is web/Inter-LICENSE.txt. +async fn inter() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "font/woff2"), (header::CACHE_CONTROL, "max-age=604800")], + include_bytes!("../web/InterVariable.woff2").as_slice(), + ) +} + fn constant_time_eq(a: &str, b: &str) -> bool { let (a, b) = (a.as_bytes(), b.as_bytes()); if a.len() != b.len() { diff --git a/tests/ui/app.spec.js b/tests/ui/app.spec.js index 22d9f0a..d72f2c1 100644 --- a/tests/ui/app.spec.js +++ b/tests/ui/app.spec.js @@ -966,3 +966,20 @@ test('an admin can give a blog its Directory category', async ({ page }) => { for (const id of ['picture-blog', 'test-show']) await patch(id, null); expect((await listed('picture-blog')).category).toBeNull(); }); + +test('the pinned heading sits over its pins, and the page is set in Inter', async ({ page }) => { + await page.locator('.feed', { hasText: 'Test Show' }).click(); + await expect(page.locator('#eps .ep').first()).toBeVisible({ timeout: 20_000 }); + const boxes = { + headCell: await page.locator('.ephead [data-sort="kept"]').boundingBox(), + headIcon: await page.locator('.ephead [data-sort="kept"] svg').first().boundingBox(), + rowCell: await page.locator('#eps .ep .fl').first().boundingBox(), + rowIcon: await page.locator('#eps .ep .fl svg').first().boundingBox(), + }; + const mid = b => b.x + b.width / 2; + expect(Math.abs(mid(boxes.headIcon) - mid(boxes.rowIcon)), JSON.stringify(boxes)).toBeLessThan(1); + + // From ipx itself, not a font service. + expect((await page.request.get('/inter.woff2')).headers()['content-type']).toBe('font/woff2'); + expect(await page.evaluate(() => document.fonts.ready.then(() => document.fonts.check('14px Inter')))).toBe(true); +}); diff --git a/web/Inter-LICENSE.txt b/web/Inter-LICENSE.txt new file mode 100644 index 0000000..9b2ca37 --- /dev/null +++ b/web/Inter-LICENSE.txt @@ -0,0 +1,92 @@ +Copyright (c) 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION AND CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/web/InterVariable.woff2 b/web/InterVariable.woff2 new file mode 100644 index 0000000..5a8d3e7 Binary files /dev/null and b/web/InterVariable.woff2 differ diff --git a/web/index.html b/web/index.html index f537150..a0a90cf 100644 --- a/web/index.html +++ b/web/index.html @@ -7,6 +7,10 @@ iPodderX