diff --git a/Cargo.lock b/Cargo.lock index d3feee2..426bf58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,17 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.3", + "once_cell", + "version_check 0.9.4", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -662,6 +673,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fast_chemail" version = "0.9.6" @@ -897,7 +920,10 @@ dependencies = [ "log 0.4.14", "multipart", "pathdiff", + "r2d2", + "r2d2_sqlite", "rand 0.6.5", + "rusqlite", "sequoia-openpgp", "serde", "serde_derive", @@ -958,6 +984,18 @@ name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -1243,6 +1281,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "libsqlite3-sys" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cafc7c74096c336d9d27145f7ebd4f4b6f95ba16aa5a282387267e6925cb58" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.5" @@ -1860,6 +1908,27 @@ dependencies = [ "proc-macro2 1.0.36", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log 0.4.14", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ca3c9468a76fc2ad724c486a59682fc362efeac7b18d1c012958bc19f34800" +dependencies = [ + "r2d2", + "rusqlite", +] + [[package]] name = "rand" version = "0.4.6" @@ -2220,6 +2289,21 @@ dependencies = [ "rocket", ] +[[package]] +name = "rusqlite" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba4d3462c8b2e4d7f4fcfcf2b296dc6b65404fbbc7b63daa37fd485c149daf7" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -2277,6 +2361,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.1.0" diff --git a/database/Cargo.toml b/database/Cargo.toml index 91eebad..903d3b7 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -23,6 +23,9 @@ fs2 = "0.4" walkdir = "2.2" chrono = "0.4" zbase32 = "0.1.2" +r2d2 = "0.8" +r2d2_sqlite = "0.19" +rusqlite = "0.26" [lib] name = "hagrid_database" diff --git a/database/src/lib.rs b/database/src/lib.rs index 0ce7548..2fde527 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -25,6 +25,7 @@ extern crate hex; extern crate walkdir; extern crate chrono; extern crate zbase32; +extern crate r2d2_sqlite; extern crate sequoia_openpgp as openpgp; use openpgp::{ @@ -41,7 +42,8 @@ pub mod wkd; pub mod sync; mod fs; -pub use self::fs::Filesystem as KeyDatabase; +mod sqlite; +pub use self::sqlite::Sqlite as KeyDatabase; mod stateful_tokens; pub use stateful_tokens::StatefulTokens; diff --git a/database/src/sqlite.rs b/database/src/sqlite.rs new file mode 100644 index 0000000..cddd640 --- /dev/null +++ b/database/src/sqlite.rs @@ -0,0 +1,475 @@ +use std::collections::HashMap; +use std::convert::TryFrom; +use std::fs::{ + create_dir_all, read_link, remove_file, rename, set_permissions, File, + OpenOptions, Permissions, +}; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use pathdiff::diff_paths; +use std::time::SystemTime; +use tempfile; +use url::form_urlencoded; + +use sync::FlockMutexGuard; +use types::{Email, Fingerprint, KeyID}; +use Result; +use {Database, Query}; + +use wkd; + +use openpgp::Cert; +use openpgp_utils::POLICY; + +use r2d2_sqlite::rusqlite::params; +use r2d2_sqlite::SqliteConnectionManager; + +pub struct Sqlite { + pool: r2d2::Pool, + + keys_dir_log: PathBuf, + dry_run: bool, +} + +impl Sqlite { + pub fn new_file(base_dir: impl Into) -> Result { + let base_dir: PathBuf = base_dir.into(); + + let db_file = base_dir.join("keys.sqlite"); + let manager = SqliteConnectionManager::file(db_file); + + Self::new_internal(base_dir, manager) + } + + pub fn new_memory(base_dir: impl Into) -> Result { + let base_dir: PathBuf = base_dir.into(); + + let manager = SqliteConnectionManager::memory(); + + Self::new_internal(base_dir, manager) + } + + fn new_internal( + base_dir: PathBuf, + manager: SqliteConnectionManager, + ) -> Result { + let keys_dir_log = base_dir.join("log"); + let dry_run = false; + + let pool = r2d2::Pool::new(manager)?; + pool.get()?.execute( + "CREATE TABLE IF NOT EXISTS certs ( + fingerprint TEXT NOT NULL PRIMARY KEY, + full BLOB NOT NULL, + published BLOB + )", + [], + )?; + + Ok(Self { pool, keys_dir_log, dry_run }) + } + + fn read_from_path( + &self, + path: &Path, + allow_internal: bool, + ) -> Option { + todo!() + } + + fn read_from_path_bytes( + &self, + path: &Path, + allow_internal: bool, + ) -> Option> { + todo!() + } + + /// Returns the Fingerprint the given path is pointing to. + pub fn path_to_fingerprint(path: &Path) -> Option { + todo!() + } + + /// Returns the KeyID the given path is pointing to. + fn path_to_keyid(path: &Path) -> Option { + todo!() + } + + /// Returns the Email the given path is pointing to. + fn path_to_email(path: &Path) -> Option { + todo!() + } + + /// Returns the backing primary key fingerprint for any key path. + pub fn path_to_primary(path: &Path) -> Option { + todo!() + } + + fn link_email_vks(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + todo!() + } + + fn link_email_wkd(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + todo!() + } + + fn unlink_email_vks(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + todo!() + } + + fn unlink_email_wkd(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + todo!() + } + + fn open_logfile(&self, file_name: &str) -> Result { + let file_path = self.keys_dir_log.join(file_name); + Ok(OpenOptions::new().create(true).append(true).open(file_path)?) + } + + fn perform_checks( + &self, + checks_dir: &Path, + tpks: &mut HashMap, + check: impl Fn(&Path, &Cert, &Fingerprint) -> Result<()>, + ) -> Result<()> { + // XXX: stub + Ok(()) + } +} + +impl Database for Sqlite { + type MutexGuard = FlockMutexGuard; + type TempCert = String; + + fn lock(&self) -> Result { + todo!() + } + + fn write_to_temp(&self, content: &[u8]) -> Result { + todo!() + } + + fn write_log_append( + &self, + filename: &str, + fpr_primary: &Fingerprint, + ) -> Result<()> { + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let fingerprint_line = + format!("{:010} {}\n", timestamp, fpr_primary.to_string()); + + self.open_logfile(filename)?.write_all(fingerprint_line.as_bytes())?; + + Ok(()) + } + + fn move_tmp_to_full( + &self, + file: Self::TempCert, + fpr: &Fingerprint, + ) -> Result<()> { + Ok(()) + } + + fn move_tmp_to_published( + &self, + file: Self::TempCert, + fpr: &Fingerprint, + ) -> Result<()> { + Ok(()) + } + + fn move_tmp_to_published_wkd( + &self, + file: Option, + fpr: &Fingerprint, + ) -> Result<()> { + Ok(()) + } + + fn write_to_quarantine( + &self, + fpr: &Fingerprint, + content: &[u8], + ) -> Result<()> { + Ok(()) + } + + fn check_link_fpr( + &self, + fpr: &Fingerprint, + fpr_target: &Fingerprint, + ) -> Result> { + Ok(None) + } + + fn lookup_primary_fingerprint(&self, term: &Query) -> Option { + None + } + + fn link_email(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + Ok(()) + } + + fn unlink_email(&self, email: &Email, fpr: &Fingerprint) -> Result<()> { + Ok(()) + } + + fn link_fpr( + &self, + from: &Fingerprint, + primary_fpr: &Fingerprint, + ) -> Result<()> { + Ok(()) + } + + fn unlink_fpr( + &self, + from: &Fingerprint, + primary_fpr: &Fingerprint, + ) -> Result<()> { + Ok(()) + } + + // XXX: slow + fn by_fpr_full(&self, fpr: &Fingerprint) -> Option { + None + } + + // XXX: slow + fn by_primary_fpr(&self, fpr: &Fingerprint) -> Option { + None + } + + // XXX: slow + fn by_fpr(&self, fpr: &Fingerprint) -> Option { + None + } + + // XXX: slow + fn by_email(&self, email: &Email) -> Option { + None + } + + // XXX: slow + fn by_email_wkd(&self, email: &Email) -> Option> { + None + } + + // XXX: slow + fn by_kid(&self, kid: &KeyID) -> Option { + None + } + + /// Checks the database for consistency. + /// + /// Note that this operation may take a long time, and is + /// generally only useful for testing. + fn check_consistency(&self) -> Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openpgp::cert::CertBuilder; + use tempfile::TempDir; + use test; + + fn open_db() -> (TempDir, Sqlite, PathBuf) { + let tmpdir = TempDir::new().unwrap(); + + let db = Sqlite::new_memory(tmpdir.path()).unwrap(); + let log_path = db.keys_dir_log.join(db.get_current_log_filename()); + + (tmpdir, db, log_path) + } + + #[test] + fn new() { + let (_tmp_dir, db, _log_path) = open_db(); + let k1 = CertBuilder::new() + .add_userid("a@invalid.example.org") + .generate() + .unwrap() + .0; + let k2 = CertBuilder::new() + .add_userid("b@invalid.example.org") + .generate() + .unwrap() + .0; + let k3 = CertBuilder::new() + .add_userid("c@invalid.example.org") + .generate() + .unwrap() + .0; + + assert!(db.merge(k1).unwrap().into_tpk_status().email_status.len() > 0); + assert!( + db.merge(k2.clone()).unwrap().into_tpk_status().email_status.len() + > 0 + ); + assert!( + !db.merge(k2).unwrap().into_tpk_status().email_status.len() > 0 + ); + assert!( + db.merge(k3.clone()).unwrap().into_tpk_status().email_status.len() + > 0 + ); + assert!( + !db.merge(k3.clone()).unwrap().into_tpk_status().email_status.len() + > 0 + ); + assert!( + !db.merge(k3).unwrap().into_tpk_status().email_status.len() > 0 + ); + } + + #[test] + fn uid_verification() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_uid_verification(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn uid_deletion() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_uid_deletion(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn subkey_lookup() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_subkey_lookup(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn kid_lookup() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_kid_lookup(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn upload_revoked_tpk() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_upload_revoked_tpk(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn uid_revocation() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_uid_revocation(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn regenerate() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_regenerate(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn key_reupload() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_reupload(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn uid_replacement() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_uid_replacement(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn uid_unlinking() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_unlink_uid(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn same_email_1() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_same_email_1(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn same_email_2() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_same_email_2(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn same_email_3() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_same_email_3(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn same_email_4() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_same_email_4(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn no_selfsig() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_no_selfsig(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn bad_uids() { + let (_tmp_dir, mut db, log_path) = open_db(); + test::test_bad_uids(&mut db, &log_path); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn reverse_fingerprint_to_path() { + let tmpdir = TempDir::new().unwrap(); + let db = Sqlite::new_memory(tmpdir.path()).unwrap(); + + let fp: Fingerprint = + "CBCD8F030588653EEDD7E2659B7DD433F254904A".parse().unwrap(); + + // XXX: fixme + //assert_eq!(Sqlite::path_to_fingerprint(&db.link_by_fingerprint(&fp)), + // Some(fp.clone())); + db.check_consistency().expect("inconsistent database"); + } + + #[test] + fn attested_key_signatures() -> Result<()> { + let (_tmp_dir, mut db, log_path) = open_db(); + test::attested_key_signatures(&mut db, &log_path)?; + db.check_consistency()?; + Ok(()) + } + + #[test] + fn nonexportable_sigs() -> Result<()> { + let (_tmp_dir, mut db, log_path) = open_db(); + test::nonexportable_sigs(&mut db, &log_path)?; + db.check_consistency()?; + Ok(()) + } +} diff --git a/shell.nix b/shell.nix index 63f17b3..23a2409 100644 --- a/shell.nix +++ b/shell.nix @@ -17,6 +17,7 @@ pkgs.mkShell { ]; buildInputs = with pkgs; [ + sqlite openssl clang diff --git a/src/delete.rs b/src/delete.rs index 3da0e7d..a715900 100644 --- a/src/delete.rs +++ b/src/delete.rs @@ -50,7 +50,7 @@ fn main() { fn real_main() -> Result<()> { let opt = Opt::from_args(); - let db = KeyDatabase::new_from_base(opt.base.canonicalize()?)?; + let db = KeyDatabase::new_file(opt.base.canonicalize()?)?; delete(&db, &opt.query.parse()?, opt.all_bindings, opt.all) } diff --git a/src/web/mod.rs b/src/web/mod.rs index 28b283d..9985daa 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -444,11 +444,9 @@ fn configure_prometheus(config: &Config) -> Option { fn configure_db_service(config: &Config) -> Result { let keys_internal_dir: PathBuf = config.get_str("keys_internal_dir")?.into(); - let keys_external_dir: PathBuf = config.get_str("keys_external_dir")?.into(); - let tmp_dir: PathBuf = config.get_str("tmp_dir")?.into(); - let fs_db = KeyDatabase::new(keys_internal_dir, keys_external_dir, tmp_dir)?; - Ok(fs_db) + let sqlite_db = KeyDatabase::new_file(keys_internal_dir)?; + Ok(sqlite_db) } fn configure_hagrid_state(config: &Config) -> Result {