rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Postgres: pick the database by URL, copy-db, tests on both

Browse Source 
- IPX_DATABASE_URL (postgres://...) picks the database; unset, it is the SQLite
  file as before. Passwords are taken out of anything logged.
- `ipx copy-db <state.db>` copies every table into the empty database the URL
  names, in one transaction, and moves the id counters past the copied ids. A
  copy of production went across in 14s with every count and column
  fingerprint identical.
- With IPX_TEST_DATABASE_URL set, each test gets a Postgres schema of its own;
  all 79 pass on both databases. Fixtures write booleans as true/false.
- Sorts say where an item with no value goes (NULLS FIRST going up, LAST going
  down): SQLite counts NULL as smallest, Postgres as largest, so "largest first"
  on Postgres led with every item that has no file. Tested on both.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-18 20:28:50 +00:00
parent bd8f6ab855
commit bc53e0f730
3 changed files with 193 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
src/db.rs
View File

@@ -67,14 +67,64 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
Ok(())
}
/// One table's rows, a page at a time in primary-key order, from one database to another.
async fn copy_table<E>(from: &sea_orm::DatabaseConnection, to: &impl ConnectionTrait) -> Result<u64>
where
E: EntityTrait,
E::Model: sea_orm::IntoActiveModel<E::ActiveModel> + Send + Sync,
E::ActiveModel: ActiveModelTrait<Entity = E> + Send,
{
use sea_orm::{IntoActiveModel, Iterable, PrimaryKeyToColumn};
let mut query = E::find();
for key in E::PrimaryKey::iter() {
query = query.order_by_asc(key.into_column());
}
let mut pages = query.paginate(from, 1000);
let mut n = 0;
while let Some(rows) = pages.fetch_and_next().await? {
n += rows.len() as u64;
// reset_all: every column written, the primary key included, not just the changed ones.
E::insert_many(rows.into_iter().map(|m| m.into_active_model().reset_all()))
.exec_without_returning(to)
.await?;
}
Ok(n)
}
/// Where the database is: IPX_DATABASE_URL, a postgres:// URL, when it is set; otherwise the
/// SQLite file in the data directory, as it has always been.
pub fn location() -> String {
std::env::var("IPX_DATABASE_URL")
.ok()
.filter(|u| !u.trim().is_empty())
.unwrap_or_else(|| crate::config::data_dir().join("state.db").display().to_string())
}
/// A URL fit for a log or an error: the password taken out.
fn redact(url: &str) -> String {
match (url.find("://"), url.rfind('@')) {
(Some(s), Some(at)) if at > s => match url[s + 3..at].find(':') {
Some(c) => format!("{}:***{}", &url[..s + 3 + c], &url[at..]),
None => url.to_owned(),
},
_ => url.to_owned(),
}
}
fn is_postgres(location: &str) -> bool {
location.starts_with("postgres://") || location.starts_with("postgresql://")
}
/// sqlx's defaults for SQLite are what ipx wants: foreign keys on, and a five-second wait for a
/// lock, which is what rusqlite was set to.
async fn connect(path: &Path) -> Result<sea_orm::DatabaseConnection> {
let mut opts = sea_orm::ConnectOptions::new(format!("sqlite://{}?mode=rwc", path.display()));
async fn connect(location: &str) -> Result<sea_orm::DatabaseConnection> {
let url =
if is_postgres(location) { location.to_owned() } else { format!("sqlite://{location}?mode=rwc") };
let mut opts = sea_orm::ConnectOptions::new(url);
opts.sqlx_logging(false);
sea_orm::Database::connect(opts)
.await
.with_context(|| format!("opening {}", path.display()))
.with_context(|| format!("opening {}", redact(location)))
}
@@ -175,18 +225,23 @@ pub struct FeedSummary {
}
impl Db {
pub async fn open(path: &Path) -> Result<Self> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)
.with_context(|| format!("creating {}", dir.display()))?;
/// Opens the database at `location` (see `location()`): a postgres:// URL, or a SQLite file.
pub async fn open(location: &str) -> Result<Self> {
if !is_postgres(location)
&& let Some(dir) = Path::new(location).parent()
{
std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?;
}
let orm = connect(path).await?;
// WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a setting of
// the file, kept once made, and making it takes a lock that cannot wait out a busy
// daemon, so it is made only when the file is not already in WAL.
let mode = orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?;
if mode.and_then(|r| r.try_get_by_index::<String>(0).ok()).as_deref() != Some("wal") {
orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?;
let orm = connect(location).await?;
if !is_postgres(location) {
// WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a
// setting of the file, kept once made, and making it takes a lock that cannot wait out
// a busy daemon, so it is made only when the file is not already in WAL.
let mode =
orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?;
if mode.and_then(|r| r.try_get_by_index::<String>(0).ok()).as_deref() != Some("wal") {
orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?;
}
}
create_missing(&orm).await?;
Ok(Self {
@@ -196,17 +251,41 @@ impl Db {
})
}
/// A fresh database for a test, in a file of its own: two connections to one ":memory:"
/// would be two databases. Its schema comes from the entities, as a real one's does.
/// A fresh, empty database for a test, its schema made from the entities as a real one's is.
/// A SQLite file of its own (two connections to one ":memory:" would be two databases); or,
/// with IPX_TEST_DATABASE_URL set, a Postgres schema of its own on that database, so the
/// same tests prove the SQL on both.
#[cfg(test)]
pub async fn memory() -> Result<Self> {
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
let path = std::env::temp_dir().join(format!(
"ipx-test-{}-{}.db",
std::process::id(),
N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
let orm = connect(&path).await?;
let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let pid = std::process::id();
if let Some(url) = std::env::var("IPX_TEST_DATABASE_URL").ok().filter(|u| !u.is_empty()) {
let admin = connect(&url).await?;
// Schemas an earlier run left behind: anything of ours not made by this process.
let old = admin
.query_all_raw(Statement::from_string(
admin.get_database_backend(),
format!(
r"SELECT nspname FROM pg_namespace
WHERE nspname LIKE 'ipxt\_%' AND nspname NOT LIKE 'ipxt\_{pid}\_%'"
),
))
.await?;
for r in old {
let name: String = r.try_get_by_index(0)?;
admin.execute_unprepared(&format!("DROP SCHEMA IF EXISTS {name} CASCADE")).await?;
}
let schema = format!("ipxt_{pid}_{n}");
admin.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await?;
// Every connection in the pool starts in it, so each test sees only its own tables.
let sep = if url.contains('?') { '&' } else { '?' };
let orm = connect(&format!("{url}{sep}options=-c%20search_path%3D{schema}")).await?;
create_missing(&orm).await?;
return Ok(Self { orm, tmp: None });
}
let path = std::env::temp_dir().join(format!("ipx-test-{pid}-{n}.db"));
let orm = connect(&path.display().to_string()).await?;
create_missing(&orm).await?;
Ok(Self { orm, tmp: Some(path) })
}
@@ -698,7 +777,10 @@ pub fn order_sql(col: &str, dir: &str, pinned_first: bool) -> String {
"size" => "(SELECT max(x.length) FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid)",
_ => "coalesce(e.published, e.first_seen)",
};
let dir = if dir == "asc" { "ASC" } else { "DESC" };
// Where an item with no value goes, said outright: SQLite counts a NULL as the smallest value
// and Postgres as the largest, so "largest first" on Postgres opened with every item that has
// no file. NULLS FIRST going up and LAST going down keeps what SQLite did.
let dir = if dir == "asc" { "ASC NULLS FIRST" } else { "DESC NULLS LAST" };
let pins = if pinned_first && col != "kept" { "coalesce(s.flagged, false) DESC, " } else { "" };
// The guid breaks what ties remain: SQLite's rowid did, and Postgres has none.
format!("{pins}{expr} {dir}, coalesce(e.published, e.first_seen) DESC, e.guid DESC")
@@ -882,6 +964,44 @@ impl Db {
Ok(())
}
// ---- moving to another database ----
/// Copies every row of `from` into this database, which must be empty: the move from the
/// SQLite file to Postgres (issue #18). One transaction, so a copy that fails part-way leaves
/// nothing behind and can simply be run again. Returns each table's row count.
pub async fn copy_from(&self, from: &Db) -> Result<Vec<(&'static str, u64)>> {
use crate::entity::*;
use sea_orm::TransactionTrait;
let held = users::Entity::find().count(&self.orm).await?
+ feeds::Entity::find().count(&self.orm).await?
+ entries::Entity::find().count(&self.orm).await?;
anyhow::ensure!(held == 0, "the database being copied into already holds rows; it has to be empty");
let tx = self.orm.begin().await?;
// Users first: the tables that belong to a person refer to them.
let counts = vec![
("users", copy_table::<users::Entity>(&from.orm, &tx).await?),
("feeds", copy_table::<feeds::Entity>(&from.orm, &tx).await?),
("entries", copy_table::<entries::Entity>(&from.orm, &tx).await?),
("enclosures", copy_table::<enclosures::Entity>(&from.orm, &tx).await?),
("subscriptions", copy_table::<subscriptions::Entity>(&from.orm, &tx).await?),
("entry_state", copy_table::<entry_state::Entity>(&from.orm, &tx).await?),
("sessions", copy_table::<sessions::Entity>(&from.orm, &tx).await?),
];
if self.orm.get_database_backend() == sea_orm::DbBackend::Postgres {
// The copied ids came with the rows; the counters that hand out new ones start past
// them, or the next account or file would collide with one copied.
for table in ["users", "enclosures"] {
tx.execute_unprepared(&format!(
"SELECT setval(pg_get_serial_sequence('{table}', 'id'), \
coalesce((SELECT max(id) FROM {table}), 0) + 1, false)"
))
.await?;
}
}
tx.commit().await?;
Ok(counts)
}
// ---- hand-written SQL ----
//
// For what reads better as SQL than as a query builder: joins, sums, upserts. Written to
@@ -1598,7 +1718,7 @@ mod tests {
async fn every_sort_column_runs_and_orders_both_ways() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g');
INSERT INTO feeds (id, url, title) VALUES ('f','u','Zebra'),('g','v','Aardvark');
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
@@ -1620,6 +1740,14 @@ mod tests {
assert_eq!(order("feed", "asc").await, ["b", "c", "a"], "Aardvark, then Zebra's newest first");
assert_eq!(order("type", "asc").await, ["a", "b", "c"], "audio, image, video");
assert_eq!(order("size", "desc").await, ["c", "a", "b"]);
// An item with no file has no size or type: last going down, first going up, on both
// databases (they disagree about NULL unless told).
db.exec_for_test("INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('f','n','no file',50);")
.await
.unwrap();
assert_eq!(order("size", "desc").await.last().map(String::as_str), Some("n"));
assert_eq!(order("type", "asc").await.first().map(String::as_str), Some("n"));
db.exec_for_test("DELETE FROM entries WHERE guid = 'n';").await.unwrap();
assert_eq!(order("published", "desc").await, ["c", "b", "a"]);
db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).await.unwrap();
assert_eq!(order("kept", "desc").await[0], "a");
@@ -1643,7 +1771,7 @@ mod tests {
async fn deleting_a_shared_file_asks_about_everyone_else() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0),(3,'kit',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false),(3,'kit',false);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'),(3,'f');
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
@@ -1669,7 +1797,7 @@ mod tests {
async fn read_state_belongs_to_one_person() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','One',100),('f','b','Two',200);",
).await
@@ -1751,7 +1879,7 @@ mod tests {
// Cutting this along with the dead read columns left the browser suite's admin with an
// empty sidebar: it is how a fresh install's first account gets config.toml's feeds.
let db = Db::memory().await.unwrap();
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);").await
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',true);").await
.unwrap();
let subs = async || db.i64s_for_test("SELECT count(*) FROM subscriptions").await[0];
let catalogue = ["a".to_string(), "b".to_string()];
@@ -1781,19 +1909,19 @@ mod tests {
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','b','u1','/tmp/b','done');
-- Read and starred belong to a person now, so say which one.
INSERT INTO users (id, name, is_admin) VALUES (7,'reader',1);
INSERT INTO users (id, name, is_admin) VALUES (7,'reader',true);
INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES
(7,'f','b',1,0,0),
(7,'f','c',1,1,0),
(7,'f','b',true,false,0),
(7,'f','c',true,true,0),
-- Started, length unknown: this is Currently Listening.
(7,'f','d',0,0,42),
(7,'f','d',false,false,42),
-- 42 of 45 seconds is past the 90% the player calls finished.
(7,'f','e',1,0,42),
(7,'f','e',true,false,42),
-- Barely touched (opened, closed within seconds): not Currently Listening.
(7,'f','g',0,0,3),
(7,'f','g',false,false,3),
-- Opened, and so read, but 42 of 900 seconds in: still Currently Listening.
-- Filtering on read hid exactly these (issue #14).
(7,'f','h',1,0,42);",
(7,'f','h',true,false,42);",
).await
.unwrap();
@@ -1877,13 +2005,13 @@ mod tests {
async fn a_show_takes_over_what_its_creator_held() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES
(1,'creator','a','u1','done','/x/a.mp3',NULL),
(2,'creator','b','u2','skipped',NULL,'explicit'),
(3,'creator','c','u3','skipped',NULL,'explicit'),
(4,'other','d','u4','skipped',NULL,'torrents disabled');
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',1);",
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',true);",
).await
.unwrap();
db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).await.unwrap();
@@ -1909,9 +2037,9 @@ mod tests {
async fn a_feed_in_a_group_follows_your_settings_on_the_group() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0);
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES
(1,'group',1),(1,'show',NULL),(2,'group',1),(2,'show',0);",
(1,'group',true),(1,'show',NULL),(2,'group',true),(2,'show',false);",
).await
.unwrap();
let explicit = async |group| -> Vec<Option<bool>> {