Give a feed with no category of its own one of Apple's (#117)
Most blogs and news sites name no <itunes:category>, so they sat under no chip in the Directory unless an admin set one by hand. When TYPESAFE_KEY is set, a feed's first read, or a forced refresh, asks TypeSafe's Jev to pick one of Apple's categories or subcategories from the feed's title and up to 15 item titles, for a catalogue feed with neither its own category nor an admin's. The answer is stored as the catalogue's category, so a feed is not asked twice, and an admin can change it. Only on a first read or a forced refresh to keep the calls down: feeds already subscribed are categorised by one `ipx fetch --force`. Tried on eight subscribed feeds on 2026-10-04: XDA, Daring Fireball, Ars Technica and The Verge as Tech News, The Old New Thing as Technology, John D. Cook as Mathematics, BoardGameGeek as Games, Pluralistic as News Commentary (0.55). About 2,000 input tokens a feed at $0.042 a million. The likeliest answer is kept however unsure. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- A feed that names no category of its own, as most blogs do not, is given one of Apple's podcast categories from its title and latest items when `TYPESAFE_KEY` is set, so it shows under a Directory chip. Asked when the feed is added, or on a forced refresh for feeds already there; an admin can change it in the feed's settings.
|
||||||
- Artwork is kept on disk once shown and loads from iPX, not from each publisher's server: fast after the first time, and still there when the publisher's server is not. The admin page sets how much is kept (500 MB by default).
|
- Artwork is kept on disk once shown and loads from iPX, not from each publisher's server: fast after the first time, and still there when the publisher's server is not. The admin page sets how much is kept (500 MB by default).
|
||||||
- `ipx add --list --category News <url>` puts a feed in the Directory for anyone to subscribe to,
|
- `ipx add --list --category News <url>` puts a feed in the Directory for anyone to subscribe to,
|
||||||
and it stays there when its last subscriber leaves. Run for a feed already in the catalogue,
|
and it stays there when its last subscriber leaves. Run for a feed already in the catalogue,
|
||||||
|
|||||||
80
src/feed.rs
80
src/feed.rs
@@ -1008,6 +1008,77 @@ fn parse_date(s: &str) -> Option<i64> {
|
|||||||
.map(|d| d.timestamp())
|
.map(|d| d.timestamp())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Apple's podcast categories, each with its subcategories
|
||||||
|
/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own
|
||||||
|
/// `<itunes:category>` comes from, and a fixed one, so the Directory's chips stay few.
|
||||||
|
const CATEGORIES: &[(&str, &[&str])] = &[
|
||||||
|
("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]),
|
||||||
|
("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]),
|
||||||
|
("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]),
|
||||||
|
("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]),
|
||||||
|
("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]),
|
||||||
|
("Government", &[]),
|
||||||
|
("History", &[]),
|
||||||
|
("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]),
|
||||||
|
("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]),
|
||||||
|
("Leisure", &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"]),
|
||||||
|
("Music", &["Music Commentary", "Music History", "Music Interviews"]),
|
||||||
|
("News", &["Business News", "Daily News", "Entertainment News", "News Commentary", "Politics", "Sports News", "Tech News"]),
|
||||||
|
("Religion & Spirituality", &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"]),
|
||||||
|
("Science", &["Astronomy", "Chemistry", "Earth Sciences", "Life Sciences", "Mathematics", "Natural Sciences", "Nature", "Physics", "Social Sciences"]),
|
||||||
|
("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]),
|
||||||
|
("Sports", &["Baseball", "Basketball", "Cricket", "Fantasy Sports", "American Football", "Golf", "Hockey", "Rugby", "Running", "Football (Soccer)", "Swimming", "Tennis", "Volleyball", "Wilderness", "Wrestling"]),
|
||||||
|
("Technology", &[]),
|
||||||
|
("True Crime", &[]),
|
||||||
|
("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]),
|
||||||
|
];
|
||||||
|
|
||||||
|
/// A Directory category for a feed that names none of its own, which most blogs do not (#117):
|
||||||
|
/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and
|
||||||
|
/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest
|
||||||
|
/// is taken however unsure, since an admin can change it and asking again would cost a call a scan.
|
||||||
|
pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result<String> {
|
||||||
|
let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect();
|
||||||
|
// A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous.
|
||||||
|
let mut criteria = serde_json::Map::new();
|
||||||
|
for (category, subs) in CATEGORIES {
|
||||||
|
criteria.insert(category.to_string(), serde_json::Value::Null);
|
||||||
|
for sub in *subs {
|
||||||
|
criteria.insert(sub.to_string(), format!("Within {category}").into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = serde_json::json!({
|
||||||
|
"model": "jev-latest",
|
||||||
|
"state": { "title": feed.title, "recent_items": items },
|
||||||
|
"questions": { "category": {
|
||||||
|
"type": "choice",
|
||||||
|
"instructions": "Which category best describes what this feed publishes?",
|
||||||
|
"criteria": criteria,
|
||||||
|
}},
|
||||||
|
});
|
||||||
|
let answer: serde_json::Value = client
|
||||||
|
.post("https://api.typesafe.ai/v1/systemone")
|
||||||
|
.bearer_auth(key)
|
||||||
|
.json(&body)
|
||||||
|
.timeout(FEED_TIMEOUT)
|
||||||
|
.send()
|
||||||
|
.await?
|
||||||
|
.error_for_status()?
|
||||||
|
.json()
|
||||||
|
.await?;
|
||||||
|
picked(&answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn picked(answer: &serde_json::Value) -> Result<String> {
|
||||||
|
let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default();
|
||||||
|
CATEGORIES
|
||||||
|
.iter()
|
||||||
|
.flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter()))
|
||||||
|
.find(|c| **c == choice)
|
||||||
|
.map(|c| c.to_string())
|
||||||
|
.ok_or_else(|| anyhow!("no category in the answer: {answer}"))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
/// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
|
/// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
|
||||||
@@ -1491,5 +1562,14 @@ mod tests {
|
|||||||
assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
|
assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
|
||||||
assert_eq!(pick_guid(None, None, None, None), None);
|
assert_eq!(pick_guid(None, None, None, None), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn picked_takes_only_an_apple_category() {
|
||||||
|
let answer = |c: &str| serde_json::json!({"answers": {"category": {"type": "choice", "choice": c}}});
|
||||||
|
assert_eq!(super::picked(&answer("Technology")).unwrap(), "Technology");
|
||||||
|
assert_eq!(super::picked(&answer("Tech News")).unwrap(), "Tech News", "a subcategory");
|
||||||
|
assert!(super::picked(&answer("Blogs")).is_err(), "not one of Apple's");
|
||||||
|
assert!(super::picked(&serde_json::json!({"error": "rate limited"})).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
22
src/main.rs
22
src/main.rs
@@ -1526,6 +1526,28 @@ async fn scan_one(
|
|||||||
parsed.image.as_deref(),
|
parsed.image.as_deref(),
|
||||||
parsed.category.as_deref(),
|
parsed.category.as_deref(),
|
||||||
).await?;
|
).await?;
|
||||||
|
// Asked on a feed's first read, nothing of it stored yet, or when someone forces a refresh:
|
||||||
|
// each call costs, and a feed already here without a category can wait for a forced one.
|
||||||
|
// The answer becomes the catalogue's category, so a feed is not asked twice.
|
||||||
|
// A feed an OPML lists is not in the catalogue and has nowhere to keep one.
|
||||||
|
if parsed.category.is_none()
|
||||||
|
&& (force || stored.entries == 0)
|
||||||
|
&& !parsed.entries.is_empty()
|
||||||
|
&& ctx.cfg().feeds.get(id).is_some_and(|f| f.category.is_none())
|
||||||
|
&& let Ok(key) = std::env::var("TYPESAFE_KEY")
|
||||||
|
{
|
||||||
|
match feed::classify(&ctx.client, &key, &parsed).await {
|
||||||
|
Ok(category) => {
|
||||||
|
let mut cfg = (*ctx.cfg()).clone();
|
||||||
|
if let Some(f) = cfg.feeds.get_mut(id).filter(|f| f.category.is_none()) {
|
||||||
|
tracing::info!(feed = id, category = %category, "categorised");
|
||||||
|
f.category = Some(category);
|
||||||
|
ctx.store_cfg(cfg).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => tracing::warn!(feed = id, "could not categorise: {e:#}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(parent) = &feed_cfg.group {
|
if let Some(parent) = &feed_cfg.group {
|
||||||
let listed: Vec<(&str, &str)> = parsed
|
let listed: Vec<(&str, &str)> = parsed
|
||||||
|
|||||||
Reference in New Issue
Block a user