sqlite: Initial work.

2022-01-04 15:13:39 +01:00 · 2022-01-04 15:13:39 +01:00 · c2593de5b7
parent 33224d1855
commit c2593de5b7
7 changed files with 578 additions and 6 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -71,6 +71,17 @@ dependencies = [
 "opaque-debug 0.3.0",
 ]

+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.3",
+ "once_cell",
+ "version_check 0.9.4",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.18"
@ -662,6 +673,18 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"

+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
 [[package]]
 name = "fast_chemail"
 version = "0.9.6"
@ -897,7 +920,10 @@ dependencies = [
 "log 0.4.14",
 "multipart",
 "pathdiff",
+ "r2d2",
+ "r2d2_sqlite",
 "rand 0.6.5",
+ "rusqlite",
 "sequoia-openpgp",
 "serde",
 "serde_derive",
@ -958,6 +984,18 @@ name = "hashbrown"
 version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
+dependencies = [
+ "hashbrown",
+]

 [[package]]
 name = "heck"
@ -1243,6 +1281,16 @@ dependencies = [
 "winapi 0.3.9",
 ]

+[[package]]
+name = "libsqlite3-sys"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2cafc7c74096c336d9d27145f7ebd4f4b6f95ba16aa5a282387267e6925cb58"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.5"
@ -1860,6 +1908,27 @@ dependencies = [
 "proc-macro2 1.0.36",
 ]

+[[package]]
+name = "r2d2"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
+dependencies = [
+ "log 0.4.14",
+ "parking_lot",
+ "scheduled-thread-pool",
+]
+
+[[package]]
+name = "r2d2_sqlite"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54ca3c9468a76fc2ad724c486a59682fc362efeac7b18d1c012958bc19f34800"
+dependencies = [
+ "r2d2",
+ "rusqlite",
+]
+
 [[package]]
 name = "rand"
 version = "0.4.6"
@ -2220,6 +2289,21 @@ dependencies = [
 "rocket",
 ]

+[[package]]
+name = "rusqlite"
+version = "0.26.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ba4d3462c8b2e4d7f4fcfcf2b296dc6b65404fbbc7b63daa37fd485c149daf7"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "memchr",
+ "smallvec",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.21"
@ -2277,6 +2361,15 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
+dependencies = [
+ "parking_lot",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
--- a/database/Cargo.toml
+++ b/database/Cargo.toml
@ -23,6 +23,9 @@ fs2 = "0.4"
 walkdir = "2.2"
 chrono = "0.4"
 zbase32 = "0.1.2"
+r2d2 = "0.8"
+r2d2_sqlite = "0.19"
+rusqlite = "0.26"

 [lib]
 name = "hagrid_database"
--- a/database/src/lib.rs
+++ b/database/src/lib.rs
@ -25,6 +25,7 @@ extern crate hex;
 extern crate walkdir;
 extern crate chrono;
 extern crate zbase32;
+extern crate r2d2_sqlite;

 extern crate sequoia_openpgp as openpgp;
 use openpgp::{
@ -41,7 +42,8 @@ pub mod wkd;
 pub mod sync;

 mod fs;
-pub use self::fs::Filesystem as KeyDatabase;
+mod sqlite;
+pub use self::sqlite::Sqlite as KeyDatabase;

 mod stateful_tokens;
 pub use stateful_tokens::StatefulTokens;
--- a/database/src/sqlite.rs
+++ b/database/src/sqlite.rs
@ -0,0 +1,475 @@
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::fs::{
+    create_dir_all, read_link, remove_file, rename, set_permissions, File,
+    OpenOptions, Permissions,
+};
+use std::io::Write;
+use std::os::unix::fs::PermissionsExt;
+use std::path::{Path, PathBuf};
+
+use pathdiff::diff_paths;
+use std::time::SystemTime;
+use tempfile;
+use url::form_urlencoded;
+
+use sync::FlockMutexGuard;
+use types::{Email, Fingerprint, KeyID};
+use Result;
+use {Database, Query};
+
+use wkd;
+
+use openpgp::Cert;
+use openpgp_utils::POLICY;
+
+use r2d2_sqlite::rusqlite::params;
+use r2d2_sqlite::SqliteConnectionManager;
+
+pub struct Sqlite {
+    pool: r2d2::Pool<SqliteConnectionManager>,
+
+    keys_dir_log: PathBuf,
+    dry_run: bool,
+}
+
+impl Sqlite {
+    pub fn new_file(base_dir: impl Into<PathBuf>) -> Result<Self> {
+        let base_dir: PathBuf = base_dir.into();
+
+        let db_file = base_dir.join("keys.sqlite");
+        let manager = SqliteConnectionManager::file(db_file);
+
+        Self::new_internal(base_dir, manager)
+    }
+
+    pub fn new_memory(base_dir: impl Into<PathBuf>) -> Result<Self> {
+        let base_dir: PathBuf = base_dir.into();
+
+        let manager = SqliteConnectionManager::memory();
+
+        Self::new_internal(base_dir, manager)
+    }
+
+    fn new_internal(
+        base_dir: PathBuf,
+        manager: SqliteConnectionManager,
+    ) -> Result<Self> {
+        let keys_dir_log = base_dir.join("log");
+        let dry_run = false;
+
+        let pool = r2d2::Pool::new(manager)?;
+        pool.get()?.execute(
+            "CREATE TABLE IF NOT EXISTS certs (
+                fingerprint     TEXT NOT NULL PRIMARY KEY,
+                full            BLOB NOT NULL,
+                published       BLOB
+            )",
+            [],
+        )?;
+
+        Ok(Self { pool, keys_dir_log, dry_run })
+    }
+
+    fn read_from_path(
+        &self,
+        path: &Path,
+        allow_internal: bool,
+    ) -> Option<String> {
+        todo!()
+    }
+
+    fn read_from_path_bytes(
+        &self,
+        path: &Path,
+        allow_internal: bool,
+    ) -> Option<Vec<u8>> {
+        todo!()
+    }
+
+    /// Returns the Fingerprint the given path is pointing to.
+    pub fn path_to_fingerprint(path: &Path) -> Option<Fingerprint> {
+        todo!()
+    }
+
+    /// Returns the KeyID the given path is pointing to.
+    fn path_to_keyid(path: &Path) -> Option<KeyID> {
+        todo!()
+    }
+
+    /// Returns the Email the given path is pointing to.
+    fn path_to_email(path: &Path) -> Option<Email> {
+        todo!()
+    }
+
+    /// Returns the backing primary key fingerprint for any key path.
+    pub fn path_to_primary(path: &Path) -> Option<Fingerprint> {
+        todo!()
+    }
+
+    fn link_email_vks(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        todo!()
+    }
+
+    fn link_email_wkd(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        todo!()
+    }
+
+    fn unlink_email_vks(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        todo!()
+    }
+
+    fn unlink_email_wkd(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        todo!()
+    }
+
+    fn open_logfile(&self, file_name: &str) -> Result<File> {
+        let file_path = self.keys_dir_log.join(file_name);
+        Ok(OpenOptions::new().create(true).append(true).open(file_path)?)
+    }
+
+    fn perform_checks(
+        &self,
+        checks_dir: &Path,
+        tpks: &mut HashMap<Fingerprint, Cert>,
+        check: impl Fn(&Path, &Cert, &Fingerprint) -> Result<()>,
+    ) -> Result<()> {
+        // XXX: stub
+        Ok(())
+    }
+}
+
+impl Database for Sqlite {
+    type MutexGuard = FlockMutexGuard;
+    type TempCert = String;
+
+    fn lock(&self) -> Result<Self::MutexGuard> {
+        todo!()
+    }
+
+    fn write_to_temp(&self, content: &[u8]) -> Result<Self::TempCert> {
+        todo!()
+    }
+
+    fn write_log_append(
+        &self,
+        filename: &str,
+        fpr_primary: &Fingerprint,
+    ) -> Result<()> {
+        let timestamp = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let fingerprint_line =
+            format!("{:010} {}\n", timestamp, fpr_primary.to_string());
+
+        self.open_logfile(filename)?.write_all(fingerprint_line.as_bytes())?;
+
+        Ok(())
+    }
+
+    fn move_tmp_to_full(
+        &self,
+        file: Self::TempCert,
+        fpr: &Fingerprint,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn move_tmp_to_published(
+        &self,
+        file: Self::TempCert,
+        fpr: &Fingerprint,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn move_tmp_to_published_wkd(
+        &self,
+        file: Option<Self::TempCert>,
+        fpr: &Fingerprint,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn write_to_quarantine(
+        &self,
+        fpr: &Fingerprint,
+        content: &[u8],
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn check_link_fpr(
+        &self,
+        fpr: &Fingerprint,
+        fpr_target: &Fingerprint,
+    ) -> Result<Option<Fingerprint>> {
+        Ok(None)
+    }
+
+    fn lookup_primary_fingerprint(&self, term: &Query) -> Option<Fingerprint> {
+        None
+    }
+
+    fn link_email(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        Ok(())
+    }
+
+    fn unlink_email(&self, email: &Email, fpr: &Fingerprint) -> Result<()> {
+        Ok(())
+    }
+
+    fn link_fpr(
+        &self,
+        from: &Fingerprint,
+        primary_fpr: &Fingerprint,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn unlink_fpr(
+        &self,
+        from: &Fingerprint,
+        primary_fpr: &Fingerprint,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    // XXX: slow
+    fn by_fpr_full(&self, fpr: &Fingerprint) -> Option<String> {
+        None
+    }
+
+    // XXX: slow
+    fn by_primary_fpr(&self, fpr: &Fingerprint) -> Option<String> {
+        None
+    }
+
+    // XXX: slow
+    fn by_fpr(&self, fpr: &Fingerprint) -> Option<String> {
+        None
+    }
+
+    // XXX: slow
+    fn by_email(&self, email: &Email) -> Option<String> {
+        None
+    }
+
+    // XXX: slow
+    fn by_email_wkd(&self, email: &Email) -> Option<Vec<u8>> {
+        None
+    }
+
+    // XXX: slow
+    fn by_kid(&self, kid: &KeyID) -> Option<String> {
+        None
+    }
+
+    /// Checks the database for consistency.
+    ///
+    /// Note that this operation may take a long time, and is
+    /// generally only useful for testing.
+    fn check_consistency(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use openpgp::cert::CertBuilder;
+    use tempfile::TempDir;
+    use test;
+
+    fn open_db() -> (TempDir, Sqlite, PathBuf) {
+        let tmpdir = TempDir::new().unwrap();
+
+        let db = Sqlite::new_memory(tmpdir.path()).unwrap();
+        let log_path = db.keys_dir_log.join(db.get_current_log_filename());
+
+        (tmpdir, db, log_path)
+    }
+
+    #[test]
+    fn new() {
+        let (_tmp_dir, db, _log_path) = open_db();
+        let k1 = CertBuilder::new()
+            .add_userid("a@invalid.example.org")
+            .generate()
+            .unwrap()
+            .0;
+        let k2 = CertBuilder::new()
+            .add_userid("b@invalid.example.org")
+            .generate()
+            .unwrap()
+            .0;
+        let k3 = CertBuilder::new()
+            .add_userid("c@invalid.example.org")
+            .generate()
+            .unwrap()
+            .0;
+
+        assert!(db.merge(k1).unwrap().into_tpk_status().email_status.len() > 0);
+        assert!(
+            db.merge(k2.clone()).unwrap().into_tpk_status().email_status.len()
+                > 0
+        );
+        assert!(
+            !db.merge(k2).unwrap().into_tpk_status().email_status.len() > 0
+        );
+        assert!(
+            db.merge(k3.clone()).unwrap().into_tpk_status().email_status.len()
+                > 0
+        );
+        assert!(
+            !db.merge(k3.clone()).unwrap().into_tpk_status().email_status.len()
+                > 0
+        );
+        assert!(
+            !db.merge(k3).unwrap().into_tpk_status().email_status.len() > 0
+        );
+    }
+
+    #[test]
+    fn uid_verification() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_uid_verification(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn uid_deletion() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_uid_deletion(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn subkey_lookup() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_subkey_lookup(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn kid_lookup() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_kid_lookup(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn upload_revoked_tpk() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_upload_revoked_tpk(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn uid_revocation() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_uid_revocation(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn regenerate() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_regenerate(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn key_reupload() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_reupload(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn uid_replacement() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_uid_replacement(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn uid_unlinking() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_unlink_uid(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn same_email_1() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_same_email_1(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn same_email_2() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_same_email_2(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn same_email_3() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_same_email_3(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn same_email_4() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_same_email_4(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn no_selfsig() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_no_selfsig(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn bad_uids() {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::test_bad_uids(&mut db, &log_path);
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn reverse_fingerprint_to_path() {
+        let tmpdir = TempDir::new().unwrap();
+        let db = Sqlite::new_memory(tmpdir.path()).unwrap();
+
+        let fp: Fingerprint =
+            "CBCD8F030588653EEDD7E2659B7DD433F254904A".parse().unwrap();
+
+        // XXX: fixme
+        //assert_eq!(Sqlite::path_to_fingerprint(&db.link_by_fingerprint(&fp)),
+        //           Some(fp.clone()));
+        db.check_consistency().expect("inconsistent database");
+    }
+
+    #[test]
+    fn attested_key_signatures() -> Result<()> {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::attested_key_signatures(&mut db, &log_path)?;
+        db.check_consistency()?;
+        Ok(())
+    }
+
+    #[test]
+    fn nonexportable_sigs() -> Result<()> {
+        let (_tmp_dir, mut db, log_path) = open_db();
+        test::nonexportable_sigs(&mut db, &log_path)?;
+        db.check_consistency()?;
+        Ok(())
+    }
+}
--- a/shell.nix
+++ b/shell.nix
@ -17,6 +17,7 @@ pkgs.mkShell {
  ];

  buildInputs = with pkgs; [
+    sqlite
    openssl

    clang
--- a/src/delete.rs
+++ b/src/delete.rs
@ -50,7 +50,7 @@ fn main() {

 fn real_main() -> Result<()> {
    let opt = Opt::from_args();
-    let db = KeyDatabase::new_from_base(opt.base.canonicalize()?)?;
+    let db = KeyDatabase::new_file(opt.base.canonicalize()?)?;
    delete(&db, &opt.query.parse()?, opt.all_bindings, opt.all)
 }

--- a/src/web/mod.rs
+++ b/src/web/mod.rs
@ -444,11 +444,9 @@ fn configure_prometheus(config: &Config) -> Option<PrometheusMetrics> {

 fn configure_db_service(config: &Config) -> Result<KeyDatabase> {
    let keys_internal_dir: PathBuf = config.get_str("keys_internal_dir")?.into();
-    let keys_external_dir: PathBuf = config.get_str("keys_external_dir")?.into();
-    let tmp_dir: PathBuf = config.get_str("tmp_dir")?.into();

-    let fs_db = KeyDatabase::new(keys_internal_dir, keys_external_dir, tmp_dir)?;
-    Ok(fs_db)
+    let sqlite_db = KeyDatabase::new_file(keys_internal_dir)?;
+    Ok(sqlite_db)
 }

 fn configure_hagrid_state(config: &Config) -> Result<HagridState> {