diff --git a/CLAUDE.md b/CLAUDE.md index 81d8d1b..da7915b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,11 +155,17 @@ name (`onclick="closeModal()"`) and the browser tests reach others through `page After any edit to it: ```sh -npx tsc -p . +npx tsc -p . && npx tsc -p tsconfig.sw.json node tests/page-smoke.js node --test tests/page-unit.js ``` +The service worker that keeps iPX readable offline (#389) is TypeScript too, `web/sw/sw.ts`, +a script of its own built to `/sw.js`, and checked by `tsconfig.sw.json` against a worker's +globals, which cannot share a program with the page's. It goes to the network first for +everything, so online nothing is an old copy. The browser tests block it +(`serviceWorkers: 'block'`), since it answers before `page.route` can; the offline tests allow it. + The first type-checks `web/src` in TypeScript's `strict` mode: every parameter and variable has a type, what the API sends is typed in `web/src/types.ts`, which mirrors the Rust structs and changes with them, and a value that may be null has to be checked before use. `$` returns the element diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index 53dce85..0553d1b 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `/api/art` also serves the pictures in the text of items iPX has sent, through its artwork cache, so the page can keep them to read offline (#389). - The log has a line for each swipe in the page's reader, saying what came of it and where the reader stood, and a warning when a drag would not scroll the reader, with what lay over it (#384). - `/metrics` says how long statements take and, for hand-written SQL, how long they waited for a connection (#367); counts feed failures by kind, new items, downloads saved and failed, and the silence DeadAir found (#368); and holds the page's own timings, a click to its list, play to sound, a stall (#369). - Traces for what had none: each command the worker runs, from the CLI, the socket or the page, with its job inside (#373); the daemon's start, from opening the database to serving (#374); a list's sync and an OPML upload, step by step (#372); and the tap or key in the page that set a trace's requests off, as its root span, from the tap to its last answer (#376). diff --git a/crates/ipx/src/art.rs b/crates/ipx/src/art.rs index 01e1c11..bc90ef2 100644 --- a/crates/ipx/src/art.rs +++ b/crates/ipx/src/art.rs @@ -1,14 +1,60 @@ //! Artwork kept on disk, so the page loads it from iPX: fast after the first time, nothing asked //! of each publisher's server for every visit, and still there when a publisher is not. -use std::collections::HashMap; use std::collections::hash_map::DefaultHasher; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fs::File; use std::path::{Path, PathBuf}; use std::sync::{LazyLock, Mutex}; use std::time::SystemTime; use std::time::{Duration, Instant}; +/// Pictures in item text ipx has sent, which `/api/art` may then fetch and keep as it does +/// artwork, so the page can save them to read offline (#389): only an address some feed's text +/// named, as artwork is only one some feed names. The oldest are forgotten past the limit, and +/// all of them at a restart, until a list sends them again. +static IN_TEXT: LazyLock, VecDeque)>> = LazyLock::new(Mutex::default); +const IN_TEXT_MAX: usize = 20_000; + +/// Remembers the pictures in a piece of cleaned item text. +pub fn name_in_text(html: &str) { + let srcs = img_srcs(html); + if srcs.is_empty() { + return; + } + let mut known = IN_TEXT.lock().unwrap_or_else(|e| e.into_inner()); + for src in srcs { + if known.0.insert(src.clone()) { + known.1.push_back(src); + } + } + while known.1.len() > IN_TEXT_MAX { + if let Some(old) = known.1.pop_front() { + known.0.remove(&old); + } + } +} + +pub fn named_in_text(url: &str) -> bool { + IN_TEXT.lock().unwrap_or_else(|e| e.into_inner()).0.contains(url) +} + +/// The web addresses of the ``s in HTML as ammonia writes it: attributes in double +/// quotes, `&`, `"` and no-break spaces escaped, and no `srcset`, which it does not let through. +fn img_srcs(html: &str) -> Vec { + let mut out = Vec::new(); + for (at, _) in html.match_indices("').map_or(html.len(), |end| at + end)]; + let Some(start) = tag.find(" src=\"").map(|i| i + 6) else { continue }; + let Some(len) = tag[start..].find('"') else { continue }; + let src = tag[start..start + len].replace(""", "\"").replace(" ", "\u{a0}").replace("&", "&"); + if src.starts_with("https://") || src.starts_with("http://") { + out.push(src); + } + } + out +} + pub fn dir() -> PathBuf { crate::config::data_dir().join("art") } @@ -107,6 +153,15 @@ pub fn trim(dir: &Path, limit: u64) -> usize { #[cfg(test)] mod tests { + #[test] + fn the_pictures_in_cleaned_text_are_named_and_nothing_else() { + let html = r#"

a

xno srcb"#; + assert_eq!(super::img_srcs(html), ["https://img.example/a.jpg?w=1&h=2"]); + super::name_in_text(html); + assert!(super::named_in_text("https://img.example/a.jpg?w=1&h=2")); + assert!(!super::named_in_text("https://img.example/b.jpg")); + } + use super::*; #[test] diff --git a/crates/ipx/src/web.rs b/crates/ipx/src/web.rs index 12a430f..aa84d86 100644 --- a/crates/ipx/src/web.rs +++ b/crates/ipx/src/web.rs @@ -1697,7 +1697,9 @@ fn clean_description(sanitizer: &mut ammonia::Builder, html: &str, link: Option< Some(b) => ammonia::UrlRelative::RewriteWithBase(b), None => ammonia::UrlRelative::PassThrough, }); - sanitizer.clean(html).to_string() + let clean = sanitizer.clean(html).to_string(); + crate::art::name_in_text(&clean); + clean } #[derive(Deserialize)] @@ -2299,7 +2301,8 @@ struct ArtQuery { u: String, } -/// Artwork, from iPX: the page asks here for every image a feed or item names, and the first +/// Artwork, from iPX: the page asks here for every image a feed or item names (and, to keep for +/// offline, the pictures in item text, #389), and the first /// time it is fetched from the publisher and kept on disk (`art`, up to `art_cache_mb`), so /// later it is fast, the publisher is not asked on every visit, and it outlives the publisher's /// server. Only an address some feed or item names as its artwork, and only an image up to @@ -2322,6 +2325,7 @@ async fn art(State(state): State, Query(q): Query) -> Respon // Only what a feed here or the directory names: anything else would make ipx fetch any // address it is given. The Directory shows podcasts nobody here subscribes to (#273). if !crate::directory::listed_art(&q.u) + && !crate::art::named_in_text(&q.u) && !state.ctx.db.names_image(&q.u).instrument(db::step("names_image")).await.unwrap_or(false) { return (StatusCode::NOT_FOUND, "no feed names that artwork").into_response(); diff --git a/package.json b/package.json index f7b7a8a..c580514 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "description": "Builds the ipx web pages from web/src (web/build.mjs, run by build.rs) and tests them. The Rust tests cover the server.", "scripts": { "build": "node web/build.mjs", - "typecheck": "tsc -p .", + "typecheck": "tsc -p . && tsc -p tsconfig.sw.json", "smoke": "node tests/page-smoke.js", "test": "playwright test", "test:headed": "playwright test --headed", diff --git a/playwright.config.js b/playwright.config.js index 1549170..fe339da 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -18,6 +18,9 @@ module.exports = defineConfig({ baseURL: 'http://127.0.0.1:8791', trace: 'retain-on-failure', screenshot: 'only-on-failure', + // The offline service worker (#389) answers requests before page.route sees them, which the + // tests that fake an answer rely on. The offline tests allow it for themselves. + serviceWorkers: 'block', }, webServer: [ { diff --git a/tests/page-unit.js b/tests/page-unit.js index 0d7dc8c..3ba86d8 100644 --- a/tests/page-unit.js +++ b/tests/page-unit.js @@ -291,3 +291,12 @@ test('a reader is stuck only when a drag up or down had room to move it and did assert.equal(run('stuckDrag(-20, 0, 0, 0, 1000)'), false); // too short to mean it assert.equal(run('stuckDrag(-100, 150, 0, 0, 1000)'), false); // a swipe, not a scroll }); + +test('the pictures saved for offline are those in unread items\' text, as the browser will ask for them (#389)', () => { + const pics = run(`textPictures([ + {read:false, description:'

x

anone', markdown_html:''}, + {read:false, description:'', markdown_html:null}, + {read:true, description:'', markdown_html:null}, + ])`); + assert.deepEqual([...pics], ['https://a.example/1.jpg?w=1&h=2']); +}); diff --git a/tests/ui/app.spec.js b/tests/ui/app.spec.js index ff994ff..6e1667f 100644 --- a/tests/ui/app.spec.js +++ b/tests/ui/app.spec.js @@ -2485,3 +2485,61 @@ test('an error toast is announced and stays until it is dismissed', async ({ pag await t.getByRole('button', { name: 'Dismiss' }).click(); await expect(t).toHaveCount(0); }); + +// Last: it adds a feed, which the feed counts of earlier tests do not expect. +test.describe('offline (#389)', () => { + test.use({ serviceWorkers: 'allow' }); + const outbox = page => page.evaluate(async () => (await (await caches.open('ipx-outbox')).keys()).length); + + test('iPX opens offline with what it read and its pictures, and sends a read made offline once back', async ({ page, context }) => { + const api = (p, init) => page.evaluate(([p, init]) => fetch(p, { headers: { 'Content-Type': 'application/json' }, ...init }) + .then((r) => r.json().catch(() => null)), [p, init]); + const added = await api('/api/feeds', { method: 'POST', body: JSON.stringify({ url: 'http://127.0.0.1:8792/offline.xml' }) }); + const id = added.id || added.feed?.id || 'offline-blog'; + await page.evaluate(() => navigator.serviceWorker.ready); + await page.reload(); + await expect.poll(() => page.evaluate(() => !!navigator.serviceWorker.controller)).toBe(true); + await page.locator('#feedlist .feed', { hasText: 'Offline Blog' }).click({ timeout: 20_000 }); + await page.locator('#content .tabs button[data-f="unread"]').click(); + await expect(page.locator('.ep', { hasText: 'Read on a Plane' })).toBeVisible(); + // The unread item's picture, kept through ipx: no feed names it as artwork, only its text. + await expect.poll(() => page.evaluate(async () => (await (await caches.open('ipx-pics')).keys()).length)).toBe(1); + + await context.setOffline(true); + await page.reload(); + // The page, the feed list and the list kept, with the feed and tab open before. + await page.locator('.ep', { hasText: 'Read on a Plane' }).click({ timeout: 20_000 }); + await expect(page.locator('#detail .dbody')).toContainText('Read on a plane, with no connection.'); + await expect.poll(() => page.locator('#detail .dbody img').evaluate(i => i.complete ? i.naturalWidth : 0)).toBeGreaterThan(0); + // Opening it read it, which waits to be sent. + await expect.poll(() => outbox(page)).toBe(1); + + await context.setOffline(false); + await expect.poll(() => outbox(page), { timeout: 15_000 }).toBe(0); + const list = await api(`/api/feeds/${encodeURIComponent(id)}/entries?offset=0&limit=50&filter=all`); + expect(list.entries.find(e => e.guid === 'offline-1').read).toBe(true); + }); + + test('online, a picture in an item\'s text still comes from its publisher, through the worker', async ({ page }) => { + await page.evaluate(() => navigator.serviceWorker.ready); + await page.reload(); + await page.locator('#feedlist .feed', { hasText: 'Offline Blog' }).click({ timeout: 20_000 }); + await page.locator('#content .tabs button[data-f="all"]').click(); + await page.locator('.ep', { hasText: 'Read on a Plane' }).click(); + const img = page.locator('#detail .dbody img'); + await expect.poll(() => img.evaluate(i => i.complete ? i.naturalWidth : 0)).toBeGreaterThan(0); + // Past the worker, not round it: what it passes on online is what it would keep. + expect(await page.evaluate(() => performance.getEntriesByName('http://127.0.0.1:8792/pic.png')[0]?.workerStart > 0)).toBe(true); + }); + + test('signing out leaves nothing kept for offline on the device', async ({ page }) => { + await page.evaluate(() => navigator.serviceWorker.ready); + await page.reload(); + await expect.poll(() => page.evaluate(async () => (await caches.keys()).filter(k => k.startsWith('ipx-')).length)).toBeGreaterThan(0); + await page.locator('#signout').evaluate(b => b.click()); + await page.waitForURL(u => !u.pathname.endsWith('/') || u.pathname === '/login' || u.pathname.includes('signed-out')); + // What was read, its pictures and unsent writes. The sign-in page's own script and styles are + // kept again as it loads: the same files for everyone, nothing of the account's. + expect(await page.evaluate(async () => (await caches.keys()).filter(k => k.startsWith('ipx-') && k !== 'ipx-page'))).toEqual([]); + }); +}); diff --git a/tests/ui/fixtures/offline.xml b/tests/ui/fixtures/offline.xml new file mode 100644 index 0000000..eeed292 --- /dev/null +++ b/tests/ui/fixtures/offline.xml @@ -0,0 +1,7 @@ + + +Offline Bloghttp://127.0.0.1:8792/A post with a picture in its text, to read offline (#389). +Read on a Planeoffline-1 + Mon, 01 Sep 2026 10:00:00 +0000 + <p>Read on a plane, with no connection.</p><img src="http://127.0.0.1:8792/pic.png" alt="a picture"> + diff --git a/tests/ui/fixtures/pic.png b/tests/ui/fixtures/pic.png new file mode 100644 index 0000000..8904d14 Binary files /dev/null and b/tests/ui/fixtures/pic.png differ diff --git a/tests/ui/fixtures/serve.js b/tests/ui/fixtures/serve.js index 6cf14ea..1fe5256 100644 --- a/tests/ui/fixtures/serve.js +++ b/tests/ui/fixtures/serve.js @@ -50,6 +50,8 @@ http.createServer((req, res) => { : file.endsWith('.opml') ? 'text/x-opml' // Artwork as an image, or /api/art refuses it as not one. : file.endsWith('.jpg') ? 'image/jpeg' + // A picture a browser can draw: the JPEGs are a few bytes that only say they are one. + : file.endsWith('.png') ? 'image/png' : file.endsWith('.html') ? 'text/html' : 'application/xml'; res.writeHead(200, { 'content-type': type, 'content-length': body.length }); res.end(body); diff --git a/tsconfig.sw.json b/tsconfig.sw.json new file mode 100644 index 0000000..2489b51 --- /dev/null +++ b/tsconfig.sw.json @@ -0,0 +1,14 @@ +{ + // Type-checks the service worker (web/sw), apart from the page: a worker's globals are not a + // page's, and the two libraries cannot be in one program (#389). Nothing is emitted: web/build.mjs + // does that with swc, as for the page. + "compilerOptions": { + "target": "es2022", + "lib": ["es2022", "webworker"], + "noEmit": true, + "strict": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true + }, + "include": ["web/sw/*.ts"] +} diff --git a/web/CHANGELOG.md b/web/CHANGELOG.md index d98e129..0664262 100644 --- a/web/CHANGELOG.md +++ b/web/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- iPX opens and reads without a connection: the lists it has fetched (All Subscriptions, Unread and the feeds it preloads or you opened) are kept on the device with the pictures in their unread items, up to 200 MB of them, and items read or flagged and places in an episode while offline are sent when the connection is back. Signing out clears it all (#389). - An open page, or the home-screen app, takes up a new version of iPX by itself once it reconnects after a deploy, back on the item you had open; while something is playing, a dialog is open or a search is typed, it offers a reload instead (#388). ### Fixed diff --git a/web/Caddyfile b/web/Caddyfile index 0e15a28..57d4fe4 100644 --- a/web/Caddyfile +++ b/web/Caddyfile @@ -141,6 +141,8 @@ http://{$WEB_BIND::8080} { # keeps them a year; the icons a day, under names that do not change. @hashed path /app.js /admin.js /login.js /app.css /hls.js /boot.js /icons/index.json header @hashed Cache-Control "public, max-age=31536000, immutable" + # The service worker (#389): the browser looks for a new one at each visit, and must find it. + header /sw.js Cache-Control no-cache # The same PNG as /favicon.png, for a browser that asks for /favicon.ico on its own. header /favicon.ico Content-Type image/png @icons path *.png *.svg *.ico *.woff2 diff --git a/web/build.mjs b/web/build.mjs index 27bc27a..c57f25e 100644 --- a/web/build.mjs +++ b/web/build.mjs @@ -24,7 +24,7 @@ const WEB_VERSION = JSON.parse(fs.readFileSync(path.join(here, '..', 'package.js // The files are one script, concatenated in this order, not modules: they share one top-level // scope, as the single inline script did, and code that runs at load needs what came before it. const PAGES = { - 'index.html': { script: 'app.js', src: ['util', 'theme', 'feeds', 'feedpage', 'items', 'player', 'dialogs', 'signals', 'gestures', 'events', 'native'] }, + 'index.html': { script: 'app.js', src: ['util', 'theme', 'feeds', 'feedpage', 'items', 'player', 'dialogs', 'signals', 'gestures', 'offline', 'events', 'native'] }, 'admin.html': { script: 'admin.js', src: ['util', 'theme', 'admin'] }, 'login.html': { script: 'login.js', src: ['login'] }, }; @@ -73,6 +73,17 @@ export function buildStyle({ minify = true } = {}) { return out; } +/// The offline service worker (#389), web/sw/sw.ts made JavaScript, at the top of the site. +export function buildWorker({ minify = true } = {}) { + const src = fs.readFileSync(path.join(here, 'sw', 'sw.ts'), 'utf8'); + return swc.transformSync(src, { + filename: 'sw.ts', + jsc: { parser: { syntax: 'typescript' }, target: 'es2022', minify: minify ? { compress: true, mangle: true } : undefined }, + isModule: false, + minify, + }).code; +} + /// The page and its script, built: { html, js, script }, where script is the file's name. export function buildPage(name, { minify = true } = {}) { const { script, src: files } = PAGES[name]; @@ -116,6 +127,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { const out = process.argv[2] || path.join(here, 'dist'); fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, STYLE), buildStyle()); + fs.writeFileSync(path.join(out, 'sw.js'), buildWorker()); fs.writeFileSync(path.join(out, 'hls.js'), HLS); // Served as they are, under the names the pages and the stylesheet use. /favicon.ico is the // PNG, for a browser that asks for that on its own; iOS asks for the precomposed touch icon diff --git a/web/src/dialogs.ts b/web/src/dialogs.ts index 6c9b0e9..ffbca05 100644 --- a/web/src/dialogs.ts +++ b/web/src/dialogs.ts @@ -846,9 +846,11 @@ $('#scanAll').onclick=scanAll; $('#prefs').onclick=prefsModal; // Someone the proxy signed in is signed out by the proxy: ipx's own sign-out cannot stick while // the proxy still vouches for them. /api/me says where, when that is the case. -$('#signout').onclick=async()=>{ await api('/api/logout',{method:'POST'}); location.href=S.me?.sign_out||'/login'; }; +// What was kept for offline goes too: the next person on this device is not to read it (#389). +$('#signout').onclick=async()=>{ await api('/api/logout',{method:'POST'}); await clearOffline(); location.href=S.me?.sign_out||'/login'; }; api('/api/me').then(u=>{ S.me=u; + offlineOwner(u.name); $('#who').textContent=u.name+(u.admin?' · admin':''); // ipx's version and the page's, each released on its own. // Also at the foot of Settings: the tooltip alone was out of reach of a keyboard and of a phone (#309). diff --git a/web/src/events.ts b/web/src/events.ts index bf2801f..747eae8 100644 --- a/web/src/events.ts +++ b/web/src/events.ts @@ -13,7 +13,7 @@ let dropped=false; function connect(){ sse=new EventSource('/api/events'); // Back after a drop: whatever was sent meanwhile went to nobody. - sse.onopen=()=>{ if(dropped){ dropped=false; catchUp(); checkVersion(); } }; + sse.onopen=()=>{ if(dropped){ dropped=false; catchUp(); checkVersion(); flushOutbox(); } }; const soon=(fn: ()=>void,ms=500)=>{ let t: number; return ()=>{ clearTimeout(t); t=setTimeout(fn,ms); }; }; // Caught: a network gone when one fires (a laptop asleep) was an unhandled rejection (#219). const refreshEntries=soon(()=>{ if(S.feed) loadEntries(false,true).catch(()=>{}); }); diff --git a/web/src/items.ts b/web/src/items.ts index a0323f5..4f0207d 100644 --- a/web/src/items.ts +++ b/web/src/items.ts @@ -25,6 +25,7 @@ function listUrl(feed: string, filter=S.filter, offset=0){ } function keepList(key: string, r: ListPage){ lists.delete(key); lists.set(key,r); + if(r) savePictures(r.entries); if(lists.size>LISTS_KEPT) lists.delete(lists.keys().next().value!); } /// Fetches a list's first page ahead of a click: when the page has loaded, and when the pointer @@ -112,6 +113,7 @@ function showList(r: ListPage, append: boolean, asked: number){ if(w && !(w.done!==undefined&&w.done!S.entries.some(x=>entryKey(x)===entryKey(e)))) : r.entries; // A background scan finishing refreshes the list from the server, which -- on the Unread // tab -- would drop the item you have open the moment reading it took it off the filter. diff --git a/web/src/offline.ts b/web/src/offline.ts new file mode 100644 index 0000000..1cefb42 --- /dev/null +++ b/web/src/offline.ts @@ -0,0 +1,44 @@ +/* ---------------- offline (#389) ---------------- */ +// The service worker (web/sw.js) keeps the page, what it read from the API and the pictures in +// unread items' text, and holds reads, flags and places made offline until the connection is +// back. Only in a secure context: over plain HTTP on a LAN address there is none, and iPX is as +// it was. +const worker=('serviceWorker' in navigator) ? navigator.serviceWorker : null; +worker?.register('/sw.js').catch(()=>{}); +const toWorker=(msg: object)=>worker?.controller?.postMessage(msg); +/// What waited for the connection goes now: on its return, and when the live stream comes back. +function flushOutbox(){ toWorker({flush:true}); } +window.addEventListener('online',flushOutbox); +window.addEventListener('offline',()=>toast('You are offline. What iPX has saved still opens, and what you read is sent when you are back.')); + +/// The pictures in the text of the unread items among `entries`, as their HTML names them. +function textPictures(entries: Entry[]): string[]{ + const out=new Set(); + for(const e of entries){ + if(e.read) continue; + for(const html of [e.description,e.markdown_html]){ + for(const m of (html||'').matchAll(/]*?\ssrc="([^"]+)"/g)){ + const src=m[1]!.replace(/"/g,'"').replace(/ /g,' ').replace(/&/g,'&'); + if(/^https?:\/\//.test(src)) out.add(src); + } + } + } + return [...out]; +} +/// Asks the worker to keep, through ipx, the pictures of a list's unread items for offline. +function savePictures(entries: Entry[]){ + const p=textPictures(entries); + if(p.length) toWorker({pictures:p}); +} + +/// Everything kept for offline, gone: at sign-out, and when another account signs in here. +async function clearOffline(){ + try{ for(const k of await caches.keys()) if(k.startsWith('ipx-')) await caches.delete(k); }catch{} +} +function offlineOwner(name: string){ + try{ + const was=localStorage.getItem('ipx.owner'); + if(was!==null&&was!==name) clearOffline(); + localStorage.setItem('ipx.owner',name); + }catch{} +} diff --git a/web/sw/sw.ts b/web/sw/sw.ts new file mode 100644 index 0000000..2ca4204 --- /dev/null +++ b/web/sw/sw.ts @@ -0,0 +1,195 @@ +// iPX offline (#389): the page and what it last read from the API, kept so iPX opens and reads +// with no connection; pictures in items' text, through ipx's /api/art, to see them too; and the +// reads, flags and places in an episode made offline, kept and sent when the connection is back. +// +// Online, everything goes to the network first and only its answers are kept, so nothing seen +// online is ever an old copy, and a deploy's new page arrives as it would without this (#388). +// Offline, the kept copy answers. Cleared on sign-out and on a change of account (the page). +// +// Not part of app.js: a service worker is a script of its own, at the top of the site so it sees +// every request, and typed against the worker's globals, not the page's (tsconfig.sw.json). Its +// scope is the origin's; it never sees another site's. + +// The worker's own global: the WebWorker library declares `self` for workers of every kind. +const sw = self as unknown as ServiceWorkerGlobalScope; +/// A write kept for later, as the outbox holds it. +type Saved = {url: string, body: string, type: string}; + +const PAGE = 'ipx-page', API = 'ipx-api', PICS = 'ipx-pics', OUTBOX = 'ipx-outbox'; +// shortcut: a byte budget over pictures counted when kept, oldest out first; nothing tracks +// which were looked at since. Raise it, or prune by use, if 200 MB proves too little or too much. +const PICS_BUDGET = 200 << 20; +// The writes made offline that are worth keeping for later: an item read or flagged, and a place +// in an episode. Anything else (adding a feed, settings) fails offline as it always has. +const QUEUED = /^\/api\/entries\/[^/]+\/[^/]+\/(flags|position)$/; + +sw.addEventListener('install', () => { void sw.skipWaiting(); }); +sw.addEventListener('activate', e => e.waitUntil(sw.clients.claim())); + +sw.addEventListener('fetch', e => { + const req = e.request, url = new URL(req.url); + if (url.origin !== location.origin) { + // A picture in an item's text, asked for from its publisher: the publisher while online, the + // copy ipx kept for it while not. + if (req.method === 'GET' && req.destination === 'image') e.respondWith(picture(req)); + return; + } + // The live stream, audio and video pass by: a stream cannot be kept, a file is too big to. + if (url.pathname === '/api/events' || url.pathname.startsWith('/media/') || url.pathname === '/sw.js') return; + if (req.method !== 'GET') { + if (req.method === 'POST' && QUEUED.test(url.pathname)) e.respondWith(write(req)); + return; + } + if (req.mode === 'navigate') e.respondWith(page(req)); + // Named by a hash of their contents: what is kept is what the network would send. + else if (url.searchParams.has('v') && !url.pathname.startsWith('/api/')) e.respondWith(hashed(req)); + else e.respondWith(networkFirst(req, url.pathname.startsWith('/api/') ? API : PAGE)); +}); + +sw.addEventListener('message', e => { + const m = e.data as {flush?: boolean, pictures?: string[]} | null; + if (m?.flush) e.waitUntil(flush()); + if (Array.isArray(m?.pictures)) e.waitUntil(keepPictures(m.pictures)); +}); + +/// The page itself, under one name whatever its query: a redirect to sign in, or to Cloudflare +/// Access, is not kept, and the browser follows it as it would. +async function page(req: Request): Promise { + try { + const r = await fetch(req); + if (r.ok && r.type === 'basic' && new URL(req.url).pathname === '/') (await caches.open(PAGE)).put('/', r.clone()); + return r; + } catch { + return (await caches.match('/', { cacheName: PAGE })) || Response.error(); + } +} + +async function hashed(req: Request): Promise { + const kept = await caches.match(req, { cacheName: PAGE }); + if (kept) return kept; + const r = await fetch(req); + if (r.ok) { + const c = await caches.open(PAGE), path = new URL(req.url).pathname; + // The version before this one is no use once this one is here. + for (const k of await c.keys()) if (new URL(k.url).pathname === path) await c.delete(k); + await c.put(req, r.clone()); + } + return r; +} + +async function networkFirst(req: Request, name: string): Promise { + try { + const r = await fetch(req); + if (r.ok && r.type === 'basic') (await caches.open(name)).put(req, r.clone()); + // Back online: whatever waited to be sent goes now. + if (name === API) void flush(); + return r; + } catch { + const kept = await caches.match(req, { cacheName: name }); + if (!kept) return Response.error(); + return name === API ? withQueuedReads(kept) : kept; + } +} + +/// A list kept before reads made offline still has those items unread: they are marked as the +/// outbox has them, so a page opened again offline shows what was read. +async function withQueuedReads(kept: Response): Promise { + const reads = new Map(); + for (const w of await queued()) { + const m = new URL(w.url, location.origin).pathname.match(/^\/api\/entries\/([^/]+)\/([^/]+)\/flags$/); + if (!m) continue; + const read: unknown = (JSON.parse(w.body || '{}') as {read?: unknown}).read; + if (typeof read === 'boolean') reads.set(decodeURIComponent(m[1]!) + '\n' + decodeURIComponent(m[2]!), read); + } + if (!reads.size || !(kept.headers.get('Content-Type') || '').includes('json')) return kept; + // Only the fields read here: a page of items, or one item, as the API sends them. + type Item = {feed_id: string, guid: string, read: boolean}; + const body = await kept.clone().json().catch((): null => null) as {entries?: Item[]} & Partial | null; + const mark = (e: Partial) => { const r = reads.get(e.feed_id + '\n' + e.guid); if (r !== undefined) e.read = r; }; + if (Array.isArray(body?.entries)) body.entries.forEach(mark); + else if (body?.guid) mark(body); + else return kept; + // Its own headers only: the kept one's length and encoding are not this body's. + return new Response(JSON.stringify(body), { status: kept.status, headers: { 'Content-Type': 'application/json' } }); +} + +// ---- writes made offline ------------------------------------------------------------------- + +let seq = 0; +async function write(req: Request): Promise { + const saved: Saved = { url: req.url, body: await req.clone().text(), type: req.headers.get('Content-Type') || 'application/json' }; + // One already waiting goes first: sent now, this one would land before it and be undone by it. + if ((await queued()).length === 0) { + try { return await fetch(req); } catch { /* offline: kept below */ } + } + await (await caches.open(OUTBOX)).put(`/__outbox/${Date.now()}-${seq++}`, new Response(JSON.stringify(saved))); + void flush(); + // What the server answers for a write the page does not need back: it carries on as if sent. + return new Response(null, { status: 204, headers: { 'X-Ipx-Queued': '1' } }); +} + +async function queued(): Promise<(Saved & {key: Request})[]> { + const c = await caches.open(OUTBOX); + const all = await Promise.all((await c.keys()).map(async k => { + const r = await c.match(k); + return r ? { key: k, ...(await r.json() as Saved) } : null; + })); + return all.filter(w => w !== null); +} + +let flushing: Promise | null = null; +/// Sends the writes in the order they were made; the first that cannot reach the server stops +/// it for the next try. One the server answered at all is done, whatever it said: a refusal now +/// would be a refusal again. +function flush(): Promise { + flushing ||= (async () => { + const c = await caches.open(OUTBOX); + for (const w of await queued()) { + try { + await fetch(w.url, { method: 'POST', body: w.body, headers: { 'Content-Type': w.type }, credentials: 'same-origin' }); + } catch { break; } + await c.delete(w.key); + } + })().finally(() => { flushing = null; }); + return flushing; +} + +// ---- pictures ---------------------------------------------------------------------------------- + +const artFor = (src: string) => `/api/art?u=${encodeURIComponent(src)}`; + +async function picture(req: Request): Promise { + try { + return await fetch(req); + } catch { + return (await caches.match(artFor(req.url), { cacheName: PICS })) || Response.error(); + } +} + +/// Fetches through ipx, three at a time, the pictures not kept yet, then trims to the budget. +async function keepPictures(srcs: string[]) { + const c = await caches.open(PICS); + const todo: string[] = []; + // Kept under the address the browser will ask for, which it normalises (a space is %20); asked + // of ipx as the text names it, which is what ipx knows it by. + const key = (s: string) => { try { return artFor(new URL(s).href); } catch { return null; } }; + for (const s of srcs) { const k = key(s); if (k && !(await c.match(k))) todo.push(s); } + if (!todo.length) return; + const one = async () => { + for (let s; (s = todo.shift()) !== undefined;) { + const k = key(s); + if (!k) continue; + try { + const r = await fetch(artFor(s)); + if (!r.ok) continue; + const blob = await r.blob(); + await c.put(k, new Response(blob, { headers: { 'Content-Type': r.headers.get('Content-Type') || '', 'X-Size': String(blob.size) } })); + } catch { return; } // offline: the rest wait for the next list + } + }; + await Promise.all([one(), one(), one()]); + const keys = await c.keys(); + const sizes = await Promise.all(keys.map(async k => Number((await c.match(k))?.headers.get('X-Size')) || 0)); + let total = sizes.reduce((a, b) => a + b, 0); + for (let i = 0; i < keys.length && total > PICS_BUDGET; i++) { await c.delete(keys[i]!); total -= sizes[i]!; } +}