iPX opens and reads without a connection (#389)

With no network the home-screen app showed nothing: no service worker,
and the lists the page had fetched, full text included, lived only in
its memory.

A service worker (web/sw/sw.ts, TypeScript checked against a worker's
globals by tsconfig.sw.json, built by swc to /sw.js) goes to the network
first for the page and every API read and keeps the answers, so online
nothing is an old copy and #388's version check works as before;
offline the kept copy answers. Scripts named by hash come from what is
kept. Reads, flags and places in an episode that cannot reach the
server wait in an outbox, sent in order when the connection is back
(and new ones queue behind them meanwhile, so none lands out of turn);
a kept list served offline shows the reads waiting there.

Pictures: a worker cannot keep another site's pictures sensibly, since
browsers count each opaque answer as megabytes. ipx now remembers the
picture addresses in item text it sends (bounded, in memory) and
/api/art fetches those as it does artwork, through the guarded client;
the worker keeps them, for unread items, under a 200 MB budget, and
serves them offline in place of the publisher's. Online, pictures still
come from their publisher.

Sign-out, or another account signing in on the device, clears what was
kept. The browser suite blocks the worker except in the offline tests,
since it answers before page.route can.

The fixture JPEGs are placeholders a browser cannot draw; the offline
test uses a real PNG.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

web/CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- iPX opens and reads without a connection: the lists it has fetched (All Subscriptions, Unread and the feeds it preloads or you opened) are kept on the device with the pictures in their unread items, up to 200 MB of them, and items read or flagged and places in an episode while offline are sent when the connection is back. Signing out clears it all (#389).
 - An open page, or the home-screen app, takes up a new version of iPX by itself once it reconnects after a deploy, back on the item you had open; while something is playing, a dialog is open or a search is typed, it offers a reload instead (#388).
 
 ### Fixed

web/Caddyfile

@@ -141,6 +141,8 @@ http://{$WEB_BIND::8080} {
 	# keeps them a year; the icons a day, under names that do not change.
 	@hashed path /app.js /admin.js /login.js /app.css /hls.js /boot.js /icons/index.json
 	header @hashed Cache-Control "public, max-age=31536000, immutable"
+	# The service worker (#389): the browser looks for a new one at each visit, and must find it.
+	header /sw.js Cache-Control no-cache
 	# The same PNG as /favicon.png, for a browser that asks for /favicon.ico on its own.
 	header /favicon.ico Content-Type image/png
 	@icons path *.png *.svg *.ico *.woff2

web/build.mjs

@@ -24,7 +24,7 @@ const WEB_VERSION = JSON.parse(fs.readFileSync(path.join(here, '..', 'package.js
 // The files are one script, concatenated in this order, not modules: they share one top-level
 // scope, as the single inline script did, and code that runs at load needs what came before it.
 const PAGES = {
-  'index.html': { script: 'app.js', src: ['util', 'theme', 'feeds', 'feedpage', 'items', 'player', 'dialogs', 'signals', 'gestures', 'events', 'native'] },
+  'index.html': { script: 'app.js', src: ['util', 'theme', 'feeds', 'feedpage', 'items', 'player', 'dialogs', 'signals', 'gestures', 'offline', 'events', 'native'] },
   'admin.html': { script: 'admin.js', src: ['util', 'theme', 'admin'] },
   'login.html': { script: 'login.js', src: ['login'] },
 };
@@ -73,6 +73,17 @@ export function buildStyle({ minify = true } = {}) {
   return out;
 }
 
+/// The offline service worker (#389), web/sw/sw.ts made JavaScript, at the top of the site.
+export function buildWorker({ minify = true } = {}) {
+  const src = fs.readFileSync(path.join(here, 'sw', 'sw.ts'), 'utf8');
+  return swc.transformSync(src, {
+    filename: 'sw.ts',
+    jsc: { parser: { syntax: 'typescript' }, target: 'es2022', minify: minify ? { compress: true, mangle: true } : undefined },
+    isModule: false,
+    minify,
+  }).code;
+}
+
 /// The page and its script, built: { html, js, script }, where script is the file's name.
 export function buildPage(name, { minify = true } = {}) {
   const { script, src: files } = PAGES[name];
@@ -116,6 +127,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
   const out = process.argv[2] || path.join(here, 'dist');
   fs.mkdirSync(out, { recursive: true });
   fs.writeFileSync(path.join(out, STYLE), buildStyle());
+  fs.writeFileSync(path.join(out, 'sw.js'), buildWorker());
   fs.writeFileSync(path.join(out, 'hls.js'), HLS);
   // Served as they are, under the names the pages and the stylesheet use. /favicon.ico is the
   // PNG, for a browser that asks for that on its own; iOS asks for the precomposed touch icon

web/src/dialogs.ts

@@ -846,9 +846,11 @@ $<HTMLButtonElement>('#scanAll').onclick=scanAll;
 $<HTMLButtonElement>('#prefs').onclick=prefsModal;
 // Someone the proxy signed in is signed out by the proxy: ipx's own sign-out cannot stick while
 // the proxy still vouches for them. /api/me says where, when that is the case.
-$<HTMLButtonElement>('#signout').onclick=async()=>{ await api('/api/logout',{method:'POST'}); location.href=S.me?.sign_out||'/login'; };
+// What was kept for offline goes too: the next person on this device is not to read it (#389).
+$<HTMLButtonElement>('#signout').onclick=async()=>{ await api('/api/logout',{method:'POST'}); await clearOffline(); location.href=S.me?.sign_out||'/login'; };
 api('/api/me').then(u=>{
   S.me=u;
+  offlineOwner(u.name);
   $('#who').textContent=u.name+(u.admin?' · admin':'');
   // ipx's version and the page's, each released on its own.
   // Also at the foot of Settings: the tooltip alone was out of reach of a keyboard and of a phone (#309).

web/src/events.ts

@@ -13,7 +13,7 @@ let dropped=false;
 function connect(){
   sse=new EventSource('/api/events');
   // Back after a drop: whatever was sent meanwhile went to nobody.
-  sse.onopen=()=>{ if(dropped){ dropped=false; catchUp(); checkVersion(); } };
+  sse.onopen=()=>{ if(dropped){ dropped=false; catchUp(); checkVersion(); flushOutbox(); } };
   const soon=(fn: ()=>void,ms=500)=>{ let t: number; return ()=>{ clearTimeout(t); t=setTimeout(fn,ms); }; };
   // Caught: a network gone when one fires (a laptop asleep) was an unhandled rejection (#219).
   const refreshEntries=soon(()=>{ if(S.feed) loadEntries(false,true).catch(()=>{}); });

web/src/items.ts

@@ -25,6 +25,7 @@ function listUrl(feed: string, filter=S.filter, offset=0){
 }
 function keepList(key: string, r: ListPage){
   lists.delete(key); lists.set(key,r);
+  if(r) savePictures(r.entries);
   if(lists.size>LISTS_KEPT) lists.delete(lists.keys().next().value!);
 }
 /// Fetches a list's first page ahead of a click: when the page has loaded, and when the pointer
@@ -112,6 +113,7 @@ function showList(r: ListPage, append: boolean, asked: number){
     if(w && !(w.done!==undefined&&w.done<asked)) e.read=w.read;
   }
   // An item new since the last page shifts the next one down by it, repeating one already here.
+  if(append) savePictures(r.entries);
   let entries = append ? S.entries.concat(r.entries.filter(e=>!S.entries.some(x=>entryKey(x)===entryKey(e)))) : r.entries;
   // A background scan finishing refreshes the list from the server, which -- on the Unread
   // tab -- would drop the item you have open the moment reading it took it off the filter.

web/src/offline.ts

@@ -0,0 +1,44 @@
+/* ---------------- offline (#389) ---------------- */
+// The service worker (web/sw.js) keeps the page, what it read from the API and the pictures in
+// unread items' text, and holds reads, flags and places made offline until the connection is
+// back. Only in a secure context: over plain HTTP on a LAN address there is none, and iPX is as
+// it was.
+const worker=('serviceWorker' in navigator) ? navigator.serviceWorker : null;
+worker?.register('/sw.js').catch(()=>{});
+const toWorker=(msg: object)=>worker?.controller?.postMessage(msg);
+/// What waited for the connection goes now: on its return, and when the live stream comes back.
+function flushOutbox(){ toWorker({flush:true}); }
+window.addEventListener('online',flushOutbox);
+window.addEventListener('offline',()=>toast('You are offline. What iPX has saved still opens, and what you read is sent when you are back.'));
+
+/// The pictures in the text of the unread items among `entries`, as their HTML names them.
+function textPictures(entries: Entry[]): string[]{
+  const out=new Set<string>();
+  for(const e of entries){
+    if(e.read) continue;
+    for(const html of [e.description,e.markdown_html]){
+      for(const m of (html||'').matchAll(/<img\b[^>]*?\ssrc="([^"]+)"/g)){
+        const src=m[1]!.replace(/&quot;/g,'"').replace(/&nbsp;/g,' ').replace(/&amp;/g,'&');
+        if(/^https?:\/\//.test(src)) out.add(src);
+      }
+    }
+  }
+  return [...out];
+}
+/// Asks the worker to keep, through ipx, the pictures of a list's unread items for offline.
+function savePictures(entries: Entry[]){
+  const p=textPictures(entries);
+  if(p.length) toWorker({pictures:p});
+}
+
+/// Everything kept for offline, gone: at sign-out, and when another account signs in here.
+async function clearOffline(){
+  try{ for(const k of await caches.keys()) if(k.startsWith('ipx-')) await caches.delete(k); }catch{}
+}
+function offlineOwner(name: string){
+  try{
+    const was=localStorage.getItem('ipx.owner');
+    if(was!==null&&was!==name) clearOffline();
+    localStorage.setItem('ipx.owner',name);
+  }catch{}
+}

web/sw/sw.ts

@@ -0,0 +1,195 @@
+// iPX offline (#389): the page and what it last read from the API, kept so iPX opens and reads
+// with no connection; pictures in items' text, through ipx's /api/art, to see them too; and the
+// reads, flags and places in an episode made offline, kept and sent when the connection is back.
+//
+// Online, everything goes to the network first and only its answers are kept, so nothing seen
+// online is ever an old copy, and a deploy's new page arrives as it would without this (#388).
+// Offline, the kept copy answers. Cleared on sign-out and on a change of account (the page).
+//
+// Not part of app.js: a service worker is a script of its own, at the top of the site so it sees
+// every request, and typed against the worker's globals, not the page's (tsconfig.sw.json). Its
+// scope is the origin's; it never sees another site's.
+
+// The worker's own global: the WebWorker library declares `self` for workers of every kind.
+const sw = self as unknown as ServiceWorkerGlobalScope;
+/// A write kept for later, as the outbox holds it.
+type Saved = {url: string, body: string, type: string};
+
+const PAGE = 'ipx-page', API = 'ipx-api', PICS = 'ipx-pics', OUTBOX = 'ipx-outbox';
+// shortcut: a byte budget over pictures counted when kept, oldest out first; nothing tracks
+// which were looked at since. Raise it, or prune by use, if 200 MB proves too little or too much.
+const PICS_BUDGET = 200 << 20;
+// The writes made offline that are worth keeping for later: an item read or flagged, and a place
+// in an episode. Anything else (adding a feed, settings) fails offline as it always has.
+const QUEUED = /^\/api\/entries\/[^/]+\/[^/]+\/(flags|position)$/;
+
+sw.addEventListener('install', () => { void sw.skipWaiting(); });
+sw.addEventListener('activate', e => e.waitUntil(sw.clients.claim()));
+
+sw.addEventListener('fetch', e => {
+  const req = e.request, url = new URL(req.url);
+  if (url.origin !== location.origin) {
+    // A picture in an item's text, asked for from its publisher: the publisher while online, the
+    // copy ipx kept for it while not.
+    if (req.method === 'GET' && req.destination === 'image') e.respondWith(picture(req));
+    return;
+  }
+  // The live stream, audio and video pass by: a stream cannot be kept, a file is too big to.
+  if (url.pathname === '/api/events' || url.pathname.startsWith('/media/') || url.pathname === '/sw.js') return;
+  if (req.method !== 'GET') {
+    if (req.method === 'POST' && QUEUED.test(url.pathname)) e.respondWith(write(req));
+    return;
+  }
+  if (req.mode === 'navigate') e.respondWith(page(req));
+  // Named by a hash of their contents: what is kept is what the network would send.
+  else if (url.searchParams.has('v') && !url.pathname.startsWith('/api/')) e.respondWith(hashed(req));
+  else e.respondWith(networkFirst(req, url.pathname.startsWith('/api/') ? API : PAGE));
+});
+
+sw.addEventListener('message', e => {
+  const m = e.data as {flush?: boolean, pictures?: string[]} | null;
+  if (m?.flush) e.waitUntil(flush());
+  if (Array.isArray(m?.pictures)) e.waitUntil(keepPictures(m.pictures));
+});
+
+/// The page itself, under one name whatever its query: a redirect to sign in, or to Cloudflare
+/// Access, is not kept, and the browser follows it as it would.
+async function page(req: Request): Promise<Response> {
+  try {
+    const r = await fetch(req);
+    if (r.ok && r.type === 'basic' && new URL(req.url).pathname === '/') (await caches.open(PAGE)).put('/', r.clone());
+    return r;
+  } catch {
+    return (await caches.match('/', { cacheName: PAGE })) || Response.error();
+  }
+}
+
+async function hashed(req: Request): Promise<Response> {
+  const kept = await caches.match(req, { cacheName: PAGE });
+  if (kept) return kept;
+  const r = await fetch(req);
+  if (r.ok) {
+    const c = await caches.open(PAGE), path = new URL(req.url).pathname;
+    // The version before this one is no use once this one is here.
+    for (const k of await c.keys()) if (new URL(k.url).pathname === path) await c.delete(k);
+    await c.put(req, r.clone());
+  }
+  return r;
+}
+
+async function networkFirst(req: Request, name: string): Promise<Response> {
+  try {
+    const r = await fetch(req);
+    if (r.ok && r.type === 'basic') (await caches.open(name)).put(req, r.clone());
+    // Back online: whatever waited to be sent goes now.
+    if (name === API) void flush();
+    return r;
+  } catch {
+    const kept = await caches.match(req, { cacheName: name });
+    if (!kept) return Response.error();
+    return name === API ? withQueuedReads(kept) : kept;
+  }
+}
+
+/// A list kept before reads made offline still has those items unread: they are marked as the
+/// outbox has them, so a page opened again offline shows what was read.
+async function withQueuedReads(kept: Response): Promise<Response> {
+  const reads = new Map<string, boolean>();
+  for (const w of await queued()) {
+    const m = new URL(w.url, location.origin).pathname.match(/^\/api\/entries\/([^/]+)\/([^/]+)\/flags$/);
+    if (!m) continue;
+    const read: unknown = (JSON.parse(w.body || '{}') as {read?: unknown}).read;
+    if (typeof read === 'boolean') reads.set(decodeURIComponent(m[1]!) + '\n' + decodeURIComponent(m[2]!), read);
+  }
+  if (!reads.size || !(kept.headers.get('Content-Type') || '').includes('json')) return kept;
+  // Only the fields read here: a page of items, or one item, as the API sends them.
+  type Item = {feed_id: string, guid: string, read: boolean};
+  const body = await kept.clone().json().catch((): null => null) as {entries?: Item[]} & Partial<Item> | null;
+  const mark = (e: Partial<Item>) => { const r = reads.get(e.feed_id + '\n' + e.guid); if (r !== undefined) e.read = r; };
+  if (Array.isArray(body?.entries)) body.entries.forEach(mark);
+  else if (body?.guid) mark(body);
+  else return kept;
+  // Its own headers only: the kept one's length and encoding are not this body's.
+  return new Response(JSON.stringify(body), { status: kept.status, headers: { 'Content-Type': 'application/json' } });
+}
+
+// ---- writes made offline -------------------------------------------------------------------
+
+let seq = 0;
+async function write(req: Request): Promise<Response> {
+  const saved: Saved = { url: req.url, body: await req.clone().text(), type: req.headers.get('Content-Type') || 'application/json' };
+  // One already waiting goes first: sent now, this one would land before it and be undone by it.
+  if ((await queued()).length === 0) {
+    try { return await fetch(req); } catch { /* offline: kept below */ }
+  }
+  await (await caches.open(OUTBOX)).put(`/__outbox/${Date.now()}-${seq++}`, new Response(JSON.stringify(saved)));
+  void flush();
+  // What the server answers for a write the page does not need back: it carries on as if sent.
+  return new Response(null, { status: 204, headers: { 'X-Ipx-Queued': '1' } });
+}
+
+async function queued(): Promise<(Saved & {key: Request})[]> {
+  const c = await caches.open(OUTBOX);
+  const all = await Promise.all((await c.keys()).map(async k => {
+    const r = await c.match(k);
+    return r ? { key: k, ...(await r.json() as Saved) } : null;
+  }));
+  return all.filter(w => w !== null);
+}
+
+let flushing: Promise<void> | null = null;
+/// Sends the writes in the order they were made; the first that cannot reach the server stops
+/// it for the next try. One the server answered at all is done, whatever it said: a refusal now
+/// would be a refusal again.
+function flush(): Promise<void> {
+  flushing ||= (async () => {
+    const c = await caches.open(OUTBOX);
+    for (const w of await queued()) {
+      try {
+        await fetch(w.url, { method: 'POST', body: w.body, headers: { 'Content-Type': w.type }, credentials: 'same-origin' });
+      } catch { break; }
+      await c.delete(w.key);
+    }
+  })().finally(() => { flushing = null; });
+  return flushing;
+}
+
+// ---- pictures ----------------------------------------------------------------------------------
+
+const artFor = (src: string) => `/api/art?u=${encodeURIComponent(src)}`;
+
+async function picture(req: Request): Promise<Response> {
+  try {
+    return await fetch(req);
+  } catch {
+    return (await caches.match(artFor(req.url), { cacheName: PICS })) || Response.error();
+  }
+}
+
+/// Fetches through ipx, three at a time, the pictures not kept yet, then trims to the budget.
+async function keepPictures(srcs: string[]) {
+  const c = await caches.open(PICS);
+  const todo: string[] = [];
+  // Kept under the address the browser will ask for, which it normalises (a space is %20); asked
+  // of ipx as the text names it, which is what ipx knows it by.
+  const key = (s: string) => { try { return artFor(new URL(s).href); } catch { return null; } };
+  for (const s of srcs) { const k = key(s); if (k && !(await c.match(k))) todo.push(s); }
+  if (!todo.length) return;
+  const one = async () => {
+    for (let s; (s = todo.shift()) !== undefined;) {
+      const k = key(s);
+      if (!k) continue;
+      try {
+        const r = await fetch(artFor(s));
+        if (!r.ok) continue;
+        const blob = await r.blob();
+        await c.put(k, new Response(blob, { headers: { 'Content-Type': r.headers.get('Content-Type') || '', 'X-Size': String(blob.size) } }));
+      } catch { return; }   // offline: the rest wait for the next list
+    }
+  };
+  await Promise.all([one(), one(), one()]);
+  const keys = await c.keys();
+  const sizes = await Promise.all(keys.map(async k => Number((await c.match(k))?.headers.get('X-Size')) || 0));
+  let total = sizes.reduce((a, b) => a + b, 0);
+  for (let i = 0; i < keys.length && total > PICS_BUDGET; i++) { await c.delete(keys[i]!); total -= sizes[i]!; }
+}