From 7b896a0bb779cbec1570e9f40cfb56809d80740d Mon Sep 17 00:00:00 2001 From: Emily C Date: Wed, 14 Dec 2022 12:49:29 -0500 Subject: [PATCH 01/33] Ledger enclave support for router and store (#2896) * Pulling changes in from milliec/ledger-router-dev * Run cargo fmt * Making requested changes and clarifications to Fog ledger router comments. * Additional comment fixes * Fix CI lint * PR feedback nits Co-authored-by: Andrew Wygle --- fog/ledger/enclave/trusted/Cargo.lock | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index 83f4b81014..5c505c7647 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -884,6 +884,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "mc-common" +version = "2.1.0-pre0" +dependencies = [ + "displaydoc", + "hex_fmt", + "mc-account-keys", + "mc-attest-verifier-types", + "mc-common", + "mc-consensus-scp-types", + "mc-crypto-digestible", + "mc-crypto-digestible-signature", + "mc-crypto-keys", + "mc-crypto-ring-signature", + "mc-transaction-core", + "mc-transaction-types", + "mc-util-from-random", + "mc-util-repr-bytes", + "prost", + "serde", + "zeroize", +] + [[package]] name = "mc-common" version = "4.0.0" From 1f4bbc65399c59d139ecc863e9b070eb782f28fe Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Fri, 2 Dec 2022 16:10:57 -0800 Subject: [PATCH 02/33] Fog Ledger Router Admin service --- fog/ledger/server/Cargo.toml | 1 + fog/ledger/server/src/lib.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 8120e936cb..5441835c00 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -61,6 +61,7 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" +itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 0602257026..ffafda47f2 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -13,6 +13,8 @@ mod router_server; mod router_service; mod server; mod untrusted_tx_out_service; +mod key_image_router_service; +mod router_admin_service; use mc_util_metrics::ServiceMetrics; From 371457d45c059608fb0de771b3825b08efff0c5e Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 6 Dec 2022 09:54:26 -0800 Subject: [PATCH 03/33] Sort itertools properly in Cargo.toml --- fog/ledger/server/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 5441835c00..8120e936cb 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -61,7 +61,6 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" -itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } From 146fe193cc2b59c1e5ddef716b8b58c1a7e9eee5 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Fri, 2 Dec 2022 16:11:40 -0800 Subject: [PATCH 04/33] Key Image Router Server + Binary --- fog/ledger/server/src/bin/key_image_router.rs | 71 +++++++++++ fog/ledger/server/src/config.rs | 4 +- .../server/src/key_image_router_server.rs | 117 ++++++++++++++++++ 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 fog/ledger/server/src/bin/key_image_router.rs create mode 100644 fog/ledger/server/src/key_image_router_server.rs diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/key_image_router.rs new file mode 100644 index 0000000000..3409f7d6dd --- /dev/null +++ b/fog/ledger/server/src/bin/key_image_router.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::{ + collections::HashMap, + env, + str::FromStr, + sync::{Arc, RwLock}, +}; + +use clap::Parser; +use grpcio::ChannelBuilder; +use mc_common::logger::log; +use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; +use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; +use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; +use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; + +use mc_util_grpc::ConnectionUriGrpcioChannel; +use mc_util_uri::UriScheme; + +fn main() { + mc_common::setup_panic_handler(); + let config = LedgerRouterConfig::parse(); + let (logger, _global_logger_guard) = + mc_common::logger::create_app_logger(mc_common::logger::o!()); + + let enclave_path = env::current_exe() + .expect("Could not get the path of our executable") + .with_file_name(ENCLAVE_FILE); + log::info!( + logger, + "enclave path {}, responder ID {}", + enclave_path.to_str().unwrap(), + &config.client_responder_id + ); + let enclave = LedgerSgxEnclave::new( + enclave_path, + &config.client_responder_id, + config.omap_capacity, + logger.clone(), + ); + + let mut ledger_store_grpc_clients = HashMap::new(); + let grpc_env = Arc::new( + grpcio::EnvBuilder::new() + .name_prefix("Main-RPC".to_string()) + .build(), + ); + for i in 0..50 { + let shard_uri_string = format!( + "{}://node{}.test.mobilecoin.com:3225", + KeyImageStoreScheme::SCHEME_INSECURE, + i + ); + let shard_uri = KeyImageStoreUri::from_str(&shard_uri_string).unwrap(); + let ledger_store_grpc_client = KeyImageStoreApiClient::new( + ChannelBuilder::default_channel_builder(grpc_env.clone()) + .connect_to_uri(&shard_uri, &logger), + ); + ledger_store_grpc_clients.insert(shard_uri, Arc::new(ledger_store_grpc_client)); + } + let ledger_store_grpc_clients = Arc::new(RwLock::new(ledger_store_grpc_clients)); + + let mut router_server = + KeyImageRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); + router_server.start(); + + loop { + std::thread::sleep(std::time::Duration::from_millis(1000)); + } +} diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index e886817650..2257b37c43 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -96,8 +96,8 @@ pub struct LedgerRouterConfig { #[clap(long, env = "MC_ADMIN_LISTEN_URI")] pub admin_listen_uri: AdminUri, - /// Number of query attempts with no forward progress before reporting an - /// error. + /// Number of query attempts with no forward progress + /// before reporting an error. #[clap(long, default_value = "3")] pub query_retries: usize, diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs new file mode 100644 index 0000000000..bce08d6dd8 --- /dev/null +++ b/fog/ledger/server/src/key_image_router_server.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use futures::executor::block_on; +use mc_common::logger::{log, Logger}; +use mc_fog_api::ledger_grpc; +use mc_fog_ledger_enclave::LedgerEnclaveProxy; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; +use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; + +use crate::{ + config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, + router_admin_service::LedgerRouterAdminService, +}; + +pub struct KeyImageRouterServer { + router_server: grpcio::Server, + admin_server: grpcio::Server, + logger: Logger, +} + +impl KeyImageRouterServer { + pub fn new( + config: LedgerRouterConfig, + enclave: E, + shards: Arc>>>, + logger: Logger, + ) -> KeyImageRouterServer + where + E: LedgerEnclaveProxy, + { + let readiness_indicator = ReadinessIndicator::default(); + + let env = Arc::new( + grpcio::EnvBuilder::new() + .name_prefix("key-image-router-server".to_string()) + .build(), + ); + + // Health check service - will be used in both cases + let health_service = + mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) + .into_service(); + + // Build our router server. + // Init ledger router service. + let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( + enclave, + shards.clone(), + logger.clone(), + )); + log::debug!(logger, "Constructed Key Image Router GRPC Service"); + + // Init ledger router admin service. + let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( + LedgerRouterAdminService::new(shards, logger.clone()), + ); + log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); + + // Package service into grpc server + log::info!( + logger, + "Starting Key Image Router server on {}", + config.client_listen_uri.addr(), + ); + + let router_server_builder = grpcio::ServerBuilder::new(env.clone()) + .register_service(ledger_router_service) + .register_service(health_service) + .bind_using_uri(&config.client_listen_uri, logger.clone()); + let admin_server_builder = grpcio::ServerBuilder::new(env) + .register_service(ledger_router_admin_service) + .bind_using_uri(&config.admin_listen_uri, logger.clone()); + + let router_server = router_server_builder.build().unwrap(); + let admin_server = admin_server_builder.build().unwrap(); + + Self { + router_server, + admin_server, + logger, + } + } + + /// Starts the server + pub fn start(&mut self) { + self.router_server.start(); + for (host, port) in self.router_server.bind_addrs() { + log::info!(self.logger, "Router API listening on {}:{}", host, port); + } + self.admin_server.start(); + for (host, port) in self.admin_server.bind_addrs() { + log::info!( + self.logger, + "Router Admin API listening on {}:{}", + host, + port + ); + } + } + + /// Stops the server + pub fn stop(&mut self) { + block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); + block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); + } +} + +impl Drop for KeyImageRouterServer { + fn drop(&mut self) { + self.stop(); + } +} From 05936e2681e126093d6aae80fd422df177c88034 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 6 Dec 2022 09:59:18 -0800 Subject: [PATCH 05/33] Update router config for parameterized retries --- fog/ledger/server/src/key_image_router_server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs index bce08d6dd8..f17184bbdf 100644 --- a/fog/ledger/server/src/key_image_router_server.rs +++ b/fog/ledger/server/src/key_image_router_server.rs @@ -51,6 +51,7 @@ impl KeyImageRouterServer { let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( enclave, shards.clone(), + config.query_retries, logger.clone(), )); log::debug!(logger, "Constructed Key Image Router GRPC Service"); From 5b7da2170352b5eb0e1684a37652b5090cc8463b Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 11 Jan 2023 21:20:06 -0800 Subject: [PATCH 06/33] Changes due to rebase --- fog/ledger/server/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index ffafda47f2..d19ac0f5e1 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -5,6 +5,8 @@ mod config; mod counters; mod db_fetcher; mod error; +mod key_image_router_server; +mod key_image_router_service; mod key_image_service; mod merkle_proof_service; mod router_admin_service; @@ -13,8 +15,6 @@ mod router_server; mod router_service; mod server; mod untrusted_tx_out_service; -mod key_image_router_service; -mod router_admin_service; use mc_util_metrics::ServiceMetrics; From a5bf8150ffc7bd606c550ca5eea5a9e4f4a1fc3c Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 11 Jan 2023 22:14:00 -0800 Subject: [PATCH 07/33] normalize naming - ledger router, key image store --- .../{key_image_router.rs => ledger_router.rs} | 4 +- .../server/src/key_image_router_server.rs | 118 ------------------ fog/ledger/server/src/lib.rs | 4 +- fog/ledger/server/src/router_server.rs | 41 +++--- 4 files changed, 25 insertions(+), 142 deletions(-) rename fog/ledger/server/src/bin/{key_image_router.rs => ledger_router.rs} (93%) delete mode 100644 fog/ledger/server/src/key_image_router_server.rs diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/ledger_router.rs similarity index 93% rename from fog/ledger/server/src/bin/key_image_router.rs rename to fog/ledger/server/src/bin/ledger_router.rs index 3409f7d6dd..fe610f897e 100644 --- a/fog/ledger/server/src/bin/key_image_router.rs +++ b/fog/ledger/server/src/bin/ledger_router.rs @@ -12,7 +12,7 @@ use grpcio::ChannelBuilder; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; -use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; +use mc_fog_ledger_server::{LedgerRouterServer, LedgerRouterConfig}; use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; use mc_util_grpc::ConnectionUriGrpcioChannel; @@ -62,7 +62,7 @@ fn main() { let ledger_store_grpc_clients = Arc::new(RwLock::new(ledger_store_grpc_clients)); let mut router_server = - KeyImageRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); + LedgerRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); router_server.start(); loop { diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs deleted file mode 100644 index f17184bbdf..0000000000 --- a/fog/ledger/server/src/key_image_router_server.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::executor::block_on; -use mc_common::logger::{log, Logger}; -use mc_fog_api::ledger_grpc; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; -use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; - -use crate::{ - config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, - router_admin_service::LedgerRouterAdminService, -}; - -pub struct KeyImageRouterServer { - router_server: grpcio::Server, - admin_server: grpcio::Server, - logger: Logger, -} - -impl KeyImageRouterServer { - pub fn new( - config: LedgerRouterConfig, - enclave: E, - shards: Arc>>>, - logger: Logger, - ) -> KeyImageRouterServer - where - E: LedgerEnclaveProxy, - { - let readiness_indicator = ReadinessIndicator::default(); - - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("key-image-router-server".to_string()) - .build(), - ); - - // Health check service - will be used in both cases - let health_service = - mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) - .into_service(); - - // Build our router server. - // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( - enclave, - shards.clone(), - config.query_retries, - logger.clone(), - )); - log::debug!(logger, "Constructed Key Image Router GRPC Service"); - - // Init ledger router admin service. - let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( - LedgerRouterAdminService::new(shards, logger.clone()), - ); - log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); - - // Package service into grpc server - log::info!( - logger, - "Starting Key Image Router server on {}", - config.client_listen_uri.addr(), - ); - - let router_server_builder = grpcio::ServerBuilder::new(env.clone()) - .register_service(ledger_router_service) - .register_service(health_service) - .bind_using_uri(&config.client_listen_uri, logger.clone()); - let admin_server_builder = grpcio::ServerBuilder::new(env) - .register_service(ledger_router_admin_service) - .bind_using_uri(&config.admin_listen_uri, logger.clone()); - - let router_server = router_server_builder.build().unwrap(); - let admin_server = admin_server_builder.build().unwrap(); - - Self { - router_server, - admin_server, - logger, - } - } - - /// Starts the server - pub fn start(&mut self) { - self.router_server.start(); - for (host, port) in self.router_server.bind_addrs() { - log::info!(self.logger, "Router API listening on {}:{}", host, port); - } - self.admin_server.start(); - for (host, port) in self.admin_server.bind_addrs() { - log::info!( - self.logger, - "Router Admin API listening on {}:{}", - host, - port - ); - } - } - - /// Stops the server - pub fn stop(&mut self) { - block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); - block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); - } -} - -impl Drop for KeyImageRouterServer { - fn drop(&mut self) { - self.stop(); - } -} diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index d19ac0f5e1..93e90356c7 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -5,8 +5,8 @@ mod config; mod counters; mod db_fetcher; mod error; -mod key_image_router_server; -mod key_image_router_service; +mod router_server; +mod router_service; mod key_image_service; mod merkle_proof_service; mod router_admin_service; diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index a7ff4a8bce..2ad2ec2e4e 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -40,11 +40,11 @@ impl LedgerRouterServer { let env = Arc::new( grpcio::EnvBuilder::new() - .name_prefix("ledger-router-server".to_string()) + .name_prefix("key-image-router-server".to_string()) .build(), ); - // Health check service - will be used in both router + admin interface + // Health check service - will be used in both cases let health_service = mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) .into_service(); @@ -68,19 +68,20 @@ impl LedgerRouterServer { // Package service into grpc server log::info!( logger, - "Starting Ledger Router server on {}", + "Starting Key Image Router server on {}", config.client_listen_uri.addr(), ); - let router_server = grpcio::ServerBuilder::new(env.clone()) + let router_server_builder = grpcio::ServerBuilder::new(env.clone()) .register_service(ledger_router_service) .register_service(health_service) - .build_using_uri(&config.client_listen_uri, logger.clone()) - .expect("Could not build Ledger Router Server"); - let admin_server = grpcio::ServerBuilder::new(env) + .bind_using_uri(&config.client_listen_uri, logger.clone()); + let admin_server_builder = grpcio::ServerBuilder::new(env) .register_service(ledger_router_admin_service) - .build_using_uri(&config.admin_listen_uri, logger.clone()) - .expect("Could not build Ledger Router Admin Server"); + .bind_using_uri(&config.admin_listen_uri, logger.clone()); + + let router_server = router_server_builder.build().unwrap(); + let admin_server = admin_server_builder.build().unwrap(); Self { router_server, @@ -94,18 +95,18 @@ impl LedgerRouterServer { /// Starts the server pub fn start(&mut self) { self.router_server.start(); - log::info!( - self.logger, - "Router API listening on {}", - self.client_listen_uri.addr() - ); - + for (host, port) in self.router_server.bind_addrs() { + log::info!(self.logger, "Router API listening on {}:{}", host, port); + } self.admin_server.start(); - log::info!( - self.logger, - "Router Admin API listening on {}", - self.admin_listen_uri.addr() - ); + for (host, port) in self.admin_server.bind_addrs() { + log::info!( + self.logger, + "Router Admin API listening on {}:{}", + host, + port + ); + } } /// Stops the server From 1826691f085c9d15f57a87315d0c0f929cccf37b Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 12 Jan 2023 10:41:16 -0800 Subject: [PATCH 08/33] Linting fixes --- fog/ledger/server/src/bin/ledger_router.rs | 2 +- fog/ledger/server/src/lib.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/fog/ledger/server/src/bin/ledger_router.rs b/fog/ledger/server/src/bin/ledger_router.rs index fe610f897e..c077d6446e 100644 --- a/fog/ledger/server/src/bin/ledger_router.rs +++ b/fog/ledger/server/src/bin/ledger_router.rs @@ -12,7 +12,7 @@ use grpcio::ChannelBuilder; use mc_common::logger::log; use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; -use mc_fog_ledger_server::{LedgerRouterServer, LedgerRouterConfig}; +use mc_fog_ledger_server::{LedgerRouterConfig, LedgerRouterServer}; use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; use mc_util_grpc::ConnectionUriGrpcioChannel; diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 93e90356c7..0602257026 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -5,8 +5,6 @@ mod config; mod counters; mod db_fetcher; mod error; -mod router_server; -mod router_service; mod key_image_service; mod merkle_proof_service; mod router_admin_service; From 7e86d37ff52956f20dd7959a051706935691ded3 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 16 Jan 2023 19:45:04 -0800 Subject: [PATCH 09/33] Accept code review suggestions --- fog/ledger/server/src/bin/ledger_router.rs | 71 ---------------------- fog/ledger/server/src/bin/router.rs | 28 ++------- fog/ledger/server/src/router_server.rs | 6 +- 3 files changed, 9 insertions(+), 96 deletions(-) delete mode 100644 fog/ledger/server/src/bin/ledger_router.rs diff --git a/fog/ledger/server/src/bin/ledger_router.rs b/fog/ledger/server/src/bin/ledger_router.rs deleted file mode 100644 index c077d6446e..0000000000 --- a/fog/ledger/server/src/bin/ledger_router.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - env, - str::FromStr, - sync::{Arc, RwLock}, -}; - -use clap::Parser; -use grpcio::ChannelBuilder; -use mc_common::logger::log; -use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; -use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; -use mc_fog_ledger_server::{LedgerRouterConfig, LedgerRouterServer}; -use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; - -use mc_util_grpc::ConnectionUriGrpcioChannel; -use mc_util_uri::UriScheme; - -fn main() { - mc_common::setup_panic_handler(); - let config = LedgerRouterConfig::parse(); - let (logger, _global_logger_guard) = - mc_common::logger::create_app_logger(mc_common::logger::o!()); - - let enclave_path = env::current_exe() - .expect("Could not get the path of our executable") - .with_file_name(ENCLAVE_FILE); - log::info!( - logger, - "enclave path {}, responder ID {}", - enclave_path.to_str().unwrap(), - &config.client_responder_id - ); - let enclave = LedgerSgxEnclave::new( - enclave_path, - &config.client_responder_id, - config.omap_capacity, - logger.clone(), - ); - - let mut ledger_store_grpc_clients = HashMap::new(); - let grpc_env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("Main-RPC".to_string()) - .build(), - ); - for i in 0..50 { - let shard_uri_string = format!( - "{}://node{}.test.mobilecoin.com:3225", - KeyImageStoreScheme::SCHEME_INSECURE, - i - ); - let shard_uri = KeyImageStoreUri::from_str(&shard_uri_string).unwrap(); - let ledger_store_grpc_client = KeyImageStoreApiClient::new( - ChannelBuilder::default_channel_builder(grpc_env.clone()) - .connect_to_uri(&shard_uri, &logger), - ); - ledger_store_grpc_clients.insert(shard_uri, Arc::new(ledger_store_grpc_client)); - } - let ledger_store_grpc_clients = Arc::new(RwLock::new(ledger_store_grpc_clients)); - - let mut router_server = - LedgerRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); - router_server.start(); - - loop { - std::thread::sleep(std::time::Duration::from_millis(1000)); - } -} diff --git a/fog/ledger/server/src/bin/router.rs b/fog/ledger/server/src/bin/router.rs index f366005ea9..173f5a2aa4 100644 --- a/fog/ledger/server/src/bin/router.rs +++ b/fog/ledger/server/src/bin/router.rs @@ -27,28 +27,12 @@ fn main() { let enclave_path = env::current_exe() .expect("Could not get the path of our executable") .with_file_name(ENCLAVE_FILE); - - if let Some(enclave_path_str) = enclave_path.to_str() { - log::info!( - logger, - "enclave path {}, responder ID {}", - enclave_path_str, - &config.client_responder_id - ); - } else { - log::info!( - logger, - "enclave path {:?}, responder ID {}", - enclave_path, - &config.client_responder_id - ); - log::warn!( - logger, - "enclave path {:?} is not valid Unicode!", - enclave_path - ); - } - + log::info!( + logger, + "enclave path {}, responder ID {}", + enclave_path.to_str().unwrap(), + &config.client_responder_id + ); let enclave = LedgerSgxEnclave::new( enclave_path, &config.client_responder_id, diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index 2ad2ec2e4e..6d61119b13 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -40,11 +40,11 @@ impl LedgerRouterServer { let env = Arc::new( grpcio::EnvBuilder::new() - .name_prefix("key-image-router-server".to_string()) + .name_prefix("ledger-router-server".to_string()) .build(), ); - // Health check service - will be used in both cases + // Health check service - will be used in both router + admin interface let health_service = mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) .into_service(); @@ -68,7 +68,7 @@ impl LedgerRouterServer { // Package service into grpc server log::info!( logger, - "Starting Key Image Router server on {}", + "Starting Ledger Router server on {}", config.client_listen_uri.addr(), ); From d387b2d64aa712f1c4e57da6f3d69484cc6e15c5 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 16 Jan 2023 20:21:21 -0800 Subject: [PATCH 10/33] Updates for GRPCIO 0.12 --- fog/ledger/enclave/trusted/Cargo.lock | 23 ----------------- fog/ledger/server/src/router_server.rs | 35 +++++++++++++------------- 2 files changed, 17 insertions(+), 41 deletions(-) diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index 5c505c7647..83f4b81014 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -884,29 +884,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "mc-common" -version = "2.1.0-pre0" -dependencies = [ - "displaydoc", - "hex_fmt", - "mc-account-keys", - "mc-attest-verifier-types", - "mc-common", - "mc-consensus-scp-types", - "mc-crypto-digestible", - "mc-crypto-digestible-signature", - "mc-crypto-keys", - "mc-crypto-ring-signature", - "mc-transaction-core", - "mc-transaction-types", - "mc-util-from-random", - "mc-util-repr-bytes", - "prost", - "serde", - "zeroize", -] - [[package]] name = "mc-common" version = "4.0.0" diff --git a/fog/ledger/server/src/router_server.rs b/fog/ledger/server/src/router_server.rs index 6d61119b13..a7ff4a8bce 100644 --- a/fog/ledger/server/src/router_server.rs +++ b/fog/ledger/server/src/router_server.rs @@ -72,16 +72,15 @@ impl LedgerRouterServer { config.client_listen_uri.addr(), ); - let router_server_builder = grpcio::ServerBuilder::new(env.clone()) + let router_server = grpcio::ServerBuilder::new(env.clone()) .register_service(ledger_router_service) .register_service(health_service) - .bind_using_uri(&config.client_listen_uri, logger.clone()); - let admin_server_builder = grpcio::ServerBuilder::new(env) + .build_using_uri(&config.client_listen_uri, logger.clone()) + .expect("Could not build Ledger Router Server"); + let admin_server = grpcio::ServerBuilder::new(env) .register_service(ledger_router_admin_service) - .bind_using_uri(&config.admin_listen_uri, logger.clone()); - - let router_server = router_server_builder.build().unwrap(); - let admin_server = admin_server_builder.build().unwrap(); + .build_using_uri(&config.admin_listen_uri, logger.clone()) + .expect("Could not build Ledger Router Admin Server"); Self { router_server, @@ -95,18 +94,18 @@ impl LedgerRouterServer { /// Starts the server pub fn start(&mut self) { self.router_server.start(); - for (host, port) in self.router_server.bind_addrs() { - log::info!(self.logger, "Router API listening on {}:{}", host, port); - } + log::info!( + self.logger, + "Router API listening on {}", + self.client_listen_uri.addr() + ); + self.admin_server.start(); - for (host, port) in self.admin_server.bind_addrs() { - log::info!( - self.logger, - "Router Admin API listening on {}:{}", - host, - port - ); - } + log::info!( + self.logger, + "Router Admin API listening on {}", + self.admin_listen_uri.addr() + ); } /// Stops the server From 4c198654be3b0050713d51ca588b7ec6b7750025 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 16 Jan 2023 20:40:05 -0800 Subject: [PATCH 11/33] Remove some unwraps in ledger_router binary --- fog/ledger/server/src/bin/router.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/fog/ledger/server/src/bin/router.rs b/fog/ledger/server/src/bin/router.rs index 173f5a2aa4..f366005ea9 100644 --- a/fog/ledger/server/src/bin/router.rs +++ b/fog/ledger/server/src/bin/router.rs @@ -27,12 +27,28 @@ fn main() { let enclave_path = env::current_exe() .expect("Could not get the path of our executable") .with_file_name(ENCLAVE_FILE); - log::info!( - logger, - "enclave path {}, responder ID {}", - enclave_path.to_str().unwrap(), - &config.client_responder_id - ); + + if let Some(enclave_path_str) = enclave_path.to_str() { + log::info!( + logger, + "enclave path {}, responder ID {}", + enclave_path_str, + &config.client_responder_id + ); + } else { + log::info!( + logger, + "enclave path {:?}, responder ID {}", + enclave_path, + &config.client_responder_id + ); + log::warn!( + logger, + "enclave path {:?} is not valid Unicode!", + enclave_path + ); + } + let enclave = LedgerSgxEnclave::new( enclave_path, &config.client_responder_id, From 101c94af54e1e3ff52ebac44fc5d003fb380b4ec Mon Sep 17 00:00:00 2001 From: Millie C Date: Tue, 22 Nov 2022 20:28:25 -0500 Subject: [PATCH 12/33] Pulling changes in from milliec/ledger-router-dev --- Cargo.lock | 5 +++++ fog/ledger/enclave/api/src/lib.rs | 2 +- fog/ledger/enclave/impl/Cargo.toml | 1 + fog/ledger/enclave/trusted/Cargo.lock | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c3dd9d4a3f..28786e415e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3925,6 +3925,7 @@ dependencies = [ "mc-oblivious-ram", "mc-oblivious-traits", "mc-sgx-compat", + "mc-sgx-debug", "mc-sgx-report-cache-api", "mc-transaction-core", "mc-util-serial", @@ -5207,6 +5208,10 @@ dependencies = [ "mc-sgx-css", ] +[[package]] +name = "mc-sgx-debug" +version = "2.1.0-pre0" + [[package]] name = "mc-sgx-debug-edl" version = "4.0.0" diff --git a/fog/ledger/enclave/api/src/lib.rs b/fog/ledger/enclave/api/src/lib.rs index dc4fabb4d4..cf3d0f72bc 100644 --- a/fog/ledger/enclave/api/src/lib.rs +++ b/fog/ledger/enclave/api/src/lib.rs @@ -35,7 +35,7 @@ pub type Result = StdResult; /// gather data that will be encrypted for the client in `outputs_for_client`. /// /// Key image check is now in ORAM, replacing untrusted -/// which was doing the check directly. +/// which was doing the check directly.Sha #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct OutputContext { /// The global txout indices being requested diff --git a/fog/ledger/enclave/impl/Cargo.toml b/fog/ledger/enclave/impl/Cargo.toml index fdbfdcc3b3..0105fe39f3 100644 --- a/fog/ledger/enclave/impl/Cargo.toml +++ b/fog/ledger/enclave/impl/Cargo.toml @@ -18,6 +18,7 @@ mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave", default-featu mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } mc-crypto-rand = { path = "../../../../crypto/rand" } mc-sgx-compat = { path = "../../../../sgx/compat", default-features = false } +mc-sgx-debug = { path = "../../../../sgx/debug" } mc-sgx-report-cache-api = { path = "../../../../sgx/report-cache/api" } mc-transaction-core = { path = "../../../../transaction/core" } mc-util-serial = { path = "../../../../util/serial" } diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index 83f4b81014..ce6b129cd2 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -1228,6 +1228,7 @@ dependencies = [ "mc-oblivious-ram", "mc-oblivious-traits", "mc-sgx-compat", + "mc-sgx-debug", "mc-sgx-report-cache-api", "mc-transaction-core", "mc-util-serial", From c0f758c707b3d679de0e78b28f6dd794d9d03a3d Mon Sep 17 00:00:00 2001 From: Millie C Date: Thu, 1 Dec 2022 15:31:41 -0500 Subject: [PATCH 13/33] Making requested changes and clarifications to Fog ledger router comments. --- fog/ledger/enclave/api/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fog/ledger/enclave/api/src/lib.rs b/fog/ledger/enclave/api/src/lib.rs index cf3d0f72bc..dc4fabb4d4 100644 --- a/fog/ledger/enclave/api/src/lib.rs +++ b/fog/ledger/enclave/api/src/lib.rs @@ -35,7 +35,7 @@ pub type Result = StdResult; /// gather data that will be encrypted for the client in `outputs_for_client`. /// /// Key image check is now in ORAM, replacing untrusted -/// which was doing the check directly.Sha +/// which was doing the check directly. #[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct OutputContext { /// The global txout indices being requested From cadb1f8984a4644075d3c81ebeed7a9a0480a04f Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 12 Dec 2022 14:12:59 -0800 Subject: [PATCH 14/33] PR feedback nits --- Cargo.lock | 5 ----- fog/ledger/enclave/impl/Cargo.toml | 1 - fog/ledger/enclave/trusted/Cargo.lock | 1 - 3 files changed, 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28786e415e..c3dd9d4a3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3925,7 +3925,6 @@ dependencies = [ "mc-oblivious-ram", "mc-oblivious-traits", "mc-sgx-compat", - "mc-sgx-debug", "mc-sgx-report-cache-api", "mc-transaction-core", "mc-util-serial", @@ -5208,10 +5207,6 @@ dependencies = [ "mc-sgx-css", ] -[[package]] -name = "mc-sgx-debug" -version = "2.1.0-pre0" - [[package]] name = "mc-sgx-debug-edl" version = "4.0.0" diff --git a/fog/ledger/enclave/impl/Cargo.toml b/fog/ledger/enclave/impl/Cargo.toml index 0105fe39f3..fdbfdcc3b3 100644 --- a/fog/ledger/enclave/impl/Cargo.toml +++ b/fog/ledger/enclave/impl/Cargo.toml @@ -18,7 +18,6 @@ mc-crypto-ake-enclave = { path = "../../../../crypto/ake/enclave", default-featu mc-crypto-keys = { path = "../../../../crypto/keys", default-features = false } mc-crypto-rand = { path = "../../../../crypto/rand" } mc-sgx-compat = { path = "../../../../sgx/compat", default-features = false } -mc-sgx-debug = { path = "../../../../sgx/debug" } mc-sgx-report-cache-api = { path = "../../../../sgx/report-cache/api" } mc-transaction-core = { path = "../../../../transaction/core" } mc-util-serial = { path = "../../../../util/serial" } diff --git a/fog/ledger/enclave/trusted/Cargo.lock b/fog/ledger/enclave/trusted/Cargo.lock index ce6b129cd2..83f4b81014 100644 --- a/fog/ledger/enclave/trusted/Cargo.lock +++ b/fog/ledger/enclave/trusted/Cargo.lock @@ -1228,7 +1228,6 @@ dependencies = [ "mc-oblivious-ram", "mc-oblivious-traits", "mc-sgx-compat", - "mc-sgx-debug", "mc-sgx-report-cache-api", "mc-transaction-core", "mc-util-serial", From abe74db2a9b74d049fd51ee4820578fd182a2710 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 22 Nov 2022 20:39:21 -0800 Subject: [PATCH 15/33] Key Image Router Service --- .../server/src/key_image_router_service.rs | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 fog/ledger/server/src/key_image_router_service.rs diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs new file mode 100644 index 0000000000..dad10a08fc --- /dev/null +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -0,0 +1,83 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use futures::{FutureExt, TryFutureExt}; +use grpcio::{DuplexSink, RequestStream, RpcContext}; +use mc_common::logger::{log, Logger}; +use mc_fog_api::{ + ledger::{LedgerRequest, LedgerResponse}, + ledger_grpc::{self, LedgerApi}, +}; +use mc_fog_ledger_enclave::LedgerEnclaveProxy; +use mc_fog_uri::KeyImageStoreUri; +use mc_util_grpc::rpc_logger; +use mc_util_metrics::SVC_COUNTERS; + +use crate::router_handlers; + +#[derive(Clone)] +pub struct KeyImageRouterService +where + E: LedgerEnclaveProxy, +{ + enclave: E, + shards: Arc>>>, + logger: Logger, +} + +impl KeyImageRouterService { + /// Creates a new LedgerRouterService that can be used by a gRPC server to + /// fulfill gRPC requests. + #[allow(dead_code)] // FIXME + pub fn new( + enclave: E, + shards: Arc>>>, + logger: Logger, + ) -> Self { + Self { + enclave, + shards, + logger, + } + } +} + +impl LedgerApi for KeyImageRouterService +where + E: LedgerEnclaveProxy, +{ + fn request( + &mut self, + ctx: RpcContext, + requests: RequestStream, + responses: DuplexSink, + ) { + log::info!(self.logger, "Request received in request fn"); + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + log::warn!( + self.logger, + "Streaming GRPC Ledger API only partially implemented." + ); + let logger = logger.clone(); + + let shards = self.shards.read().expect("RwLock poisoned"); + let future = router_handlers::handle_requests( + shards.values().cloned().collect(), + self.enclave.clone(), + requests, + responses, + logger.clone(), + ) + .map_err(move |err: grpcio::Error| log::error!(&logger, "failed to reply: {}", err)) + // TODO: Do more with the error than just push it to the log. + .map(|_| ()); + + ctx.spawn(future) + }); + } +} From 7f9b62043469dfa5870f51e8cd7eaef244a268fa Mon Sep 17 00:00:00 2001 From: NotGyro Date: Wed, 23 Nov 2022 19:26:06 -0500 Subject: [PATCH 16/33] Update fog/ledger/server/src/key_image_router_service.rs Co-authored-by: Nick Santana --- fog/ledger/server/src/key_image_router_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index dad10a08fc..fb8ebc8439 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -73,7 +73,7 @@ where responses, logger.clone(), ) - .map_err(move |err: grpcio::Error| log::error!(&logger, "failed to reply: {}", err)) + .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) // TODO: Do more with the error than just push it to the log. .map(|_| ()); From 698e3bd3a60a5ab66039f2eace6be6a61fa20680 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 1 Dec 2022 20:04:04 -0800 Subject: [PATCH 17/33] Address PR feedback around logging and comments. --- fog/ledger/server/src/key_image_router_service.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index fb8ebc8439..ae1d3280c2 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -56,7 +56,6 @@ where requests: RequestStream, responses: DuplexSink, ) { - log::info!(self.logger, "Request received in request fn"); let _timer = SVC_COUNTERS.req(&ctx); mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { log::warn!( From 15f223d66ae9a43fc329d5a9a0a51ed52f0209ea Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Fri, 2 Dec 2022 13:38:04 -0800 Subject: [PATCH 18/33] Parameterize allowed number of retries for query loop --- fog/ledger/server/src/key_image_router_service.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index ae1d3280c2..707674547a 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -26,6 +26,7 @@ where { enclave: E, shards: Arc>>>, + query_retries: usize, logger: Logger, } @@ -36,11 +37,13 @@ impl KeyImageRouterService { pub fn new( enclave: E, shards: Arc>>>, + query_retries: usize, logger: Logger, ) -> Self { Self { enclave, shards, + query_retries, logger, } } @@ -70,6 +73,7 @@ where self.enclave.clone(), requests, responses, + self.query_retries, logger.clone(), ) .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) From d03f4467452a2aef71199cfa389372ff404bd18f Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Fri, 2 Dec 2022 16:10:57 -0800 Subject: [PATCH 19/33] Fog Ledger Router Admin service --- fog/ledger/server/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 8120e936cb..5441835c00 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -61,6 +61,7 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" +itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } From 88320174c8bf8eb01914ae5463d1cd25a0e487c8 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 6 Dec 2022 09:54:26 -0800 Subject: [PATCH 20/33] Sort itertools properly in Cargo.toml --- fog/ledger/server/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 5441835c00..8120e936cb 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -61,7 +61,6 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" -itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } From 8cf5fd4146b7ee4e5d105b780c04a542d68b936e Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Fri, 2 Dec 2022 16:11:40 -0800 Subject: [PATCH 21/33] Key Image Router Server + Binary --- fog/ledger/server/src/bin/key_image_router.rs | 71 +++++++++++ .../server/src/key_image_router_server.rs | 117 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 fog/ledger/server/src/bin/key_image_router.rs create mode 100644 fog/ledger/server/src/key_image_router_server.rs diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/key_image_router.rs new file mode 100644 index 0000000000..3409f7d6dd --- /dev/null +++ b/fog/ledger/server/src/bin/key_image_router.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::{ + collections::HashMap, + env, + str::FromStr, + sync::{Arc, RwLock}, +}; + +use clap::Parser; +use grpcio::ChannelBuilder; +use mc_common::logger::log; +use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; +use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; +use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; +use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; + +use mc_util_grpc::ConnectionUriGrpcioChannel; +use mc_util_uri::UriScheme; + +fn main() { + mc_common::setup_panic_handler(); + let config = LedgerRouterConfig::parse(); + let (logger, _global_logger_guard) = + mc_common::logger::create_app_logger(mc_common::logger::o!()); + + let enclave_path = env::current_exe() + .expect("Could not get the path of our executable") + .with_file_name(ENCLAVE_FILE); + log::info!( + logger, + "enclave path {}, responder ID {}", + enclave_path.to_str().unwrap(), + &config.client_responder_id + ); + let enclave = LedgerSgxEnclave::new( + enclave_path, + &config.client_responder_id, + config.omap_capacity, + logger.clone(), + ); + + let mut ledger_store_grpc_clients = HashMap::new(); + let grpc_env = Arc::new( + grpcio::EnvBuilder::new() + .name_prefix("Main-RPC".to_string()) + .build(), + ); + for i in 0..50 { + let shard_uri_string = format!( + "{}://node{}.test.mobilecoin.com:3225", + KeyImageStoreScheme::SCHEME_INSECURE, + i + ); + let shard_uri = KeyImageStoreUri::from_str(&shard_uri_string).unwrap(); + let ledger_store_grpc_client = KeyImageStoreApiClient::new( + ChannelBuilder::default_channel_builder(grpc_env.clone()) + .connect_to_uri(&shard_uri, &logger), + ); + ledger_store_grpc_clients.insert(shard_uri, Arc::new(ledger_store_grpc_client)); + } + let ledger_store_grpc_clients = Arc::new(RwLock::new(ledger_store_grpc_clients)); + + let mut router_server = + KeyImageRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); + router_server.start(); + + loop { + std::thread::sleep(std::time::Duration::from_millis(1000)); + } +} diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs new file mode 100644 index 0000000000..bce08d6dd8 --- /dev/null +++ b/fog/ledger/server/src/key_image_router_server.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use futures::executor::block_on; +use mc_common::logger::{log, Logger}; +use mc_fog_api::ledger_grpc; +use mc_fog_ledger_enclave::LedgerEnclaveProxy; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; +use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; + +use crate::{ + config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, + router_admin_service::LedgerRouterAdminService, +}; + +pub struct KeyImageRouterServer { + router_server: grpcio::Server, + admin_server: grpcio::Server, + logger: Logger, +} + +impl KeyImageRouterServer { + pub fn new( + config: LedgerRouterConfig, + enclave: E, + shards: Arc>>>, + logger: Logger, + ) -> KeyImageRouterServer + where + E: LedgerEnclaveProxy, + { + let readiness_indicator = ReadinessIndicator::default(); + + let env = Arc::new( + grpcio::EnvBuilder::new() + .name_prefix("key-image-router-server".to_string()) + .build(), + ); + + // Health check service - will be used in both cases + let health_service = + mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) + .into_service(); + + // Build our router server. + // Init ledger router service. + let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( + enclave, + shards.clone(), + logger.clone(), + )); + log::debug!(logger, "Constructed Key Image Router GRPC Service"); + + // Init ledger router admin service. + let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( + LedgerRouterAdminService::new(shards, logger.clone()), + ); + log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); + + // Package service into grpc server + log::info!( + logger, + "Starting Key Image Router server on {}", + config.client_listen_uri.addr(), + ); + + let router_server_builder = grpcio::ServerBuilder::new(env.clone()) + .register_service(ledger_router_service) + .register_service(health_service) + .bind_using_uri(&config.client_listen_uri, logger.clone()); + let admin_server_builder = grpcio::ServerBuilder::new(env) + .register_service(ledger_router_admin_service) + .bind_using_uri(&config.admin_listen_uri, logger.clone()); + + let router_server = router_server_builder.build().unwrap(); + let admin_server = admin_server_builder.build().unwrap(); + + Self { + router_server, + admin_server, + logger, + } + } + + /// Starts the server + pub fn start(&mut self) { + self.router_server.start(); + for (host, port) in self.router_server.bind_addrs() { + log::info!(self.logger, "Router API listening on {}:{}", host, port); + } + self.admin_server.start(); + for (host, port) in self.admin_server.bind_addrs() { + log::info!( + self.logger, + "Router Admin API listening on {}:{}", + host, + port + ); + } + } + + /// Stops the server + pub fn stop(&mut self) { + block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); + block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); + } +} + +impl Drop for KeyImageRouterServer { + fn drop(&mut self) { + self.stop(); + } +} From 5d47c3ef8eb969a7c0bcff5d87d2f93ad330b12d Mon Sep 17 00:00:00 2001 From: Millie C Date: Tue, 13 Dec 2022 20:20:38 -0500 Subject: [PATCH 22/33] Key image store changes pulled in from milliec/ledger-router-dev --- Cargo.lock | 9 + fog/ledger/server/Cargo.toml | 14 + fog/ledger/server/src/bin/key_image_store.rs | 72 +++++ fog/ledger/server/src/config.rs | 80 ++++- fog/ledger/server/src/key_image_service.rs | 197 ++++++++++-- .../server/src/key_image_store_server.rs | 152 +++++++++ fog/ledger/server/src/lib.rs | 6 +- fog/ledger/server/src/server.rs | 5 +- fog/ledger/server/tests/store_tests.rs | 304 ++++++++++++++++++ 9 files changed, 811 insertions(+), 28 deletions(-) create mode 100644 fog/ledger/server/src/bin/key_image_store.rs create mode 100644 fog/ledger/server/src/key_image_store_server.rs create mode 100644 fog/ledger/server/tests/store_tests.rs diff --git a/Cargo.lock b/Cargo.lock index c3dd9d4a3f..136d3a4813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3949,6 +3949,7 @@ dependencies = [ name = "mc-fog-ledger-server" version = "4.0.0" dependencies = [ + "aes-gcm 0.9.4", "clap 4.0.32", "displaydoc", "futures", @@ -3957,6 +3958,7 @@ dependencies = [ "lazy_static", "mc-account-keys", "mc-api", + "mc-attest-ake", "mc-attest-api", "mc-attest-core", "mc-attest-enclave-api", @@ -3966,6 +3968,8 @@ dependencies = [ "mc-blockchain-types", "mc-common", "mc-crypto-keys", + "mc-crypto-rand", + "mc-crypto-x509-test-vectors", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-ledger-connection", @@ -3985,6 +3989,7 @@ dependencies = [ "mc-util-encodings", "mc-util-from-random", "mc-util-grpc", + "mc-util-keyfile", "mc-util-metrics", "mc-util-parse", "mc-util-serial", @@ -3993,12 +3998,16 @@ dependencies = [ "mc-util-uri", "mc-watcher", "mc-watcher-api", + "pem", + "portpicker", "rand 0.8.5", "retry", "serde", "serde_json", + "sha2 0.10.6", "tempdir", "url", + "x509-signature", ] [[package]] diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 8120e936cb..af99c77e7e 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -17,6 +17,10 @@ path = "src/bin/main.rs" name = "ledger_router" path = "src/bin/router.rs" +[[bin]] +name = "ledger_store" +path = "src/bin/key_image_store.rs" + [dependencies] mc-attest-api = { path = "../../../attest/api" } mc-attest-core = { path = "../../../attest/core" } @@ -61,6 +65,7 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" +itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } @@ -70,9 +75,13 @@ mc-util-build-sgx = { path = "../../../util/build/sgx" } # mobilecoin mc-account-keys = { path = "../../../account-keys" } mc-api = { path = "../../../api" } +mc-attest-ake = { path = "../../../attest/ake" } mc-blockchain-test-utils = { path = "../../../blockchain/test-utils" } mc-common = { path = "../../../common", features = ["loggers"] } mc-crypto-keys = { path = "../../../crypto/keys" } +mc-crypto-rand = { path = "../../../crypto/rand" } +mc-crypto-x509-test-vectors = { path = "../../../crypto/x509/test-vectors" } +mc-util-keyfile = { path = "../../../util/keyfile" } mc-util-test-helper = { path = "../../../util/test-helper" } mc-util-uri = { path = "../../../util/uri" } @@ -83,4 +92,9 @@ mc-fog-ledger-test-infra = { path = "../test_infra" } mc-fog-test-infra = { path = "../../test_infra" } # third-party +aes-gcm = "0.9.4" +pem = "1.1" +sha2 = "0.10" tempdir = "0.3" +x509-signature = "0.5" +portpicker = "0.1.1" diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs new file mode 100644 index 0000000000..2929e566cb --- /dev/null +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -0,0 +1,72 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use clap::Parser; +use grpcio::{RpcStatus, RpcStatusCode}; +use mc_common::{logger::log, time::SystemTimeProvider}; +use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; +use mc_fog_ledger_server::{KeyImageStoreServer, LedgerStoreConfig}; +use mc_ledger_db::LedgerDB; +use mc_util_grpc::AdminServer; +use mc_watcher::watcher_db::WatcherDB; + +use std::{env, sync::Arc}; + +fn main() { + mc_common::setup_panic_handler(); + let config = LedgerStoreConfig::parse(); + let (logger, _global_logger_guard) = + mc_common::logger::create_app_logger(mc_common::logger::o!()); + + let enclave_path = env::current_exe() + .expect("Could not get the path of our executable") + .with_file_name(ENCLAVE_FILE); + log::info!( + logger, + "enclave path {}, responder ID {}", + enclave_path.to_str().unwrap(), + &config.client_responder_id + ); + let enclave = LedgerSgxEnclave::new( + enclave_path, + &config.client_responder_id, + config.omap_capacity, + logger.clone(), + ); + + //Get our ledger connection started. + let db = LedgerDB::open(&config.ledger_db).expect("Could not read ledger DB"); + let watcher = + WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB"); + + let mut store_server = KeyImageStoreServer::new_from_config( + config.clone(), + enclave, + db, + watcher, + SystemTimeProvider::default(), + logger.clone(), + ); + store_server.start(); + + //Initialize the admin api + let config2 = config.clone(); + let get_config_json = Arc::new(move || { + serde_json::to_string(&config2) + .map_err(|err| RpcStatus::with_message(RpcStatusCode::INTERNAL, format!("{:?}", err))) + }); + let _admin_server = config.admin_listen_uri.as_ref().map(|admin_listen_uri| { + AdminServer::start( + None, + admin_listen_uri, + "Fog Ledger".to_owned(), + config.client_responder_id.to_string(), + Some(get_config_json), + logger, + ) + .expect("Failed starting admin server") + }); + + loop { + std::thread::sleep(std::time::Duration::from_millis(1000)); + } +} diff --git a/fog/ledger/server/src/config.rs b/fog/ledger/server/src/config.rs index 2257b37c43..954796ee24 100644 --- a/fog/ledger/server/src/config.rs +++ b/fog/ledger/server/src/config.rs @@ -7,7 +7,7 @@ use clap::Parser; use mc_attest_core::ProviderId; use mc_common::ResponderId; -use mc_fog_uri::FogLedgerUri; +use mc_fog_uri::{FogLedgerUri, KeyImageStoreUri}; use mc_util_parse::parse_duration_in_seconds; use mc_util_uri::AdminUri; use serde::Serialize; @@ -115,3 +115,81 @@ pub struct LedgerRouterConfig { #[clap(long, default_value = "1048576", env = "MC_OMAP_CAPACITY")] pub omap_capacity: u64, } + +/// Configuration parameters for the Fog Ledger Store service. +#[derive(Clone, Parser, Serialize)] +#[clap(version)] +pub struct LedgerStoreConfig { + /// The chain id of the network we are a part of + #[clap(long, env = "MC_CHAIN_ID")] + pub chain_id: String, + + /// The ID with which to respond to client attestation requests. + /// + /// This ID needs to match the host:port clients use in their URI when + /// referencing this node. + #[clap(long, env = "MC_CLIENT_RESPONDER_ID")] + pub client_responder_id: ResponderId, + + /// gRPC listening URI for client requests. + #[clap(long, env = "MC_CLIENT_LISTEN_URI")] + pub client_listen_uri: KeyImageStoreUri, + + /// Path to ledger db (lmdb) + #[clap(long, value_parser(clap::value_parser!(PathBuf)), env = "MC_LEDGER_DB")] + pub ledger_db: PathBuf, + + /// Path to watcher db (lmdb) - includes block timestamps + #[clap(long, value_parser(clap::value_parser!(PathBuf)), env = "MC_WATCHER_DB")] + pub watcher_db: PathBuf, + + /// IAS Api Key. + #[clap(long, env = "MC_IAS_API_KEY")] + pub ias_api_key: String, + + /// IAS Service Provider ID. + #[clap(long, env = "MC_IAS_SPID")] + pub ias_spid: ProviderId, + + /// Optional admin listening URI. + #[clap(long, env = "MC_ADMIN_LISTEN_URI")] + pub admin_listen_uri: Option, + + /// Enables authenticating client requests using Authorization tokens using + /// the provided hex-encoded 32 bytes shared secret. + #[clap(long, value_parser = mc_util_parse::parse_hex::<[u8; 32]>, env = "MC_CLIENT_AUTH_TOKEN_SECRET")] + pub client_auth_token_secret: Option<[u8; 32]>, + + /// Maximal client authentication token lifetime, in seconds (only relevant + /// when --client-auth-token-secret is used. Defaults to 86400 - 24 + /// hours). + #[clap(long, default_value = "86400", value_parser = parse_duration_in_seconds, env = "MC_CLIENT_AUTH_TOKEN_MAX_LIFETIME")] + pub client_auth_token_max_lifetime: Duration, + + /// The capacity to build the OMAP (ORAM hash table) with. + /// About 75% of this capacity can be used. + /// The hash table will overflow when there are more Keyimages than this, + /// and the server will have to be restarted with a larger number. + /// + /// Note: At time of writing, the hash table will be allocated to use all + /// available SGX EPC memory, and then beyond that it will be allocated on + /// the heap in the untrusted side. Once the needed capacity exceeds RAM, + /// you will either get killed by OOM killer, or it will start being swapped + /// to disk by linux kernel. + #[clap(long, default_value = "1048576", env = "MC_OMAP_CAPACITY")] + pub omap_capacity: u64, +} + +/// Uri for any node in the key image store system. +/// Old-style single-node servers and routers are both referred to with +/// a KeyImageClientListenUri::ClientFacing(FogLedgerUri), whereas ledger +/// store shard Uris will be KeyImageClientListenUri::Store(KeyImageStoreUri). +#[derive(Clone, Serialize)] +pub enum KeyImageClientListenUri { + /// URI used by the KeyImageStoreServer when fulfilling direct client + /// requests. + ClientFacing(FogLedgerUri), + /// URI used by the KeyImageStoreServer when fulfilling Fog Ledger Router + /// requests. + Store(KeyImageStoreUri), +} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index 6959ae1348..fd8ad438a8 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -7,9 +7,15 @@ use mc_attest_api::{ }; use mc_blockchain_types::MAX_BLOCK_VERSION; use mc_common::logger::{log, Logger}; -use mc_fog_api::ledger_grpc::FogKeyImageApi; +use mc_fog_api::{ + ledger::{ + MultiKeyImageStoreRequest, MultiKeyImageStoreResponse, MultiKeyImageStoreResponseStatus, + }, + ledger_grpc::{FogKeyImageApi, KeyImageStoreApi}, +}; use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_ledger_enclave_api::{Error as EnclaveError, UntrustedKeyImageQueryResponse}; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; use mc_ledger_db::Ledger; use mc_util_grpc::{ check_request_chain_id, rpc_internal_error, rpc_invalid_arg_error, rpc_logger, @@ -20,6 +26,8 @@ use std::sync::{Arc, Mutex}; #[derive(Clone)] pub struct KeyImageService { + /// The ClientListenUri for this Fog Ledger Service. + client_listen_uri: KeyImageClientListenUri, chain_id: String, ledger: L, watcher: WatcherDB, @@ -32,6 +40,7 @@ pub struct KeyImageService { impl KeyImageService { pub fn new( + client_listen_uri: KeyImageClientListenUri, chain_id: String, ledger: L, watcher: WatcherDB, @@ -41,6 +50,7 @@ impl KeyImageService { logger: Logger, ) -> Self { Self { + client_listen_uri, chain_id, ledger, watcher, @@ -63,18 +73,39 @@ impl KeyImageService { self.db_poll_shared_state.clone() } - /// Unwrap and forward to enclave - // self.enclave.check_key_images should take both an AttestMessage and an - // UntrustedKeyImageQueryResponse object that contains any data that is - // needed that isn't in the ORAM. This might be like "num_blocks" and similar - // stuff. self.enclave.check_key_images should return an AttestMessage that - // we send back to the user. - fn check_key_images_auth( + pub fn auth_impl( &mut self, - request: attest::Message, - ) -> Result { - log::trace!(self.logger, "Getting encrypted request"); + mut req: AuthMessage, + logger: &Logger, + ) -> Result { + // TODO: Use the prost message directly, once available + match self.enclave.client_accept(req.take_data().into()) { + Ok((response, _)) => { + let mut result = attest::AuthMessage::new(); + result.set_data(response.into()); + Ok(result) + } + Err(client_error) => { + // There's no requirement on the remote party to trigger this, so it's debug. + log::debug!( + logger, + "KeyImageStoreApi::client_accept failed: {}", + client_error + ); + let rpc_permissions_error = rpc_permissions_error( + "client_auth", + format!("Permission denied: {}", client_error), + logger, + ); + Err(rpc_permissions_error) + } + } + } + /// Generate an UntrustedKeyImageQueryResponse + /// for use in [KeyImageService::check_key_images_auth()] + /// and [KeyImageService::check_key_image_store_auth()] + fn prepare_untrusted_query(&mut self) -> UntrustedKeyImageQueryResponse { let ( highest_processed_block_count, last_known_block_cumulative_txo_count, @@ -88,12 +119,27 @@ impl KeyImageService { ) }; - let untrusted_query_response = UntrustedKeyImageQueryResponse { + UntrustedKeyImageQueryResponse { highest_processed_block_count, last_known_block_cumulative_txo_count, latest_block_version, max_block_version: latest_block_version.max(*MAX_BLOCK_VERSION), - }; + } + } + + /// Unwrap and forward to enclave + // self.enclave.check_key_images should take both an AttestMessage and an + // UntrustedKeyImageQueryResponse object that contains any data that is + // needed that isn't in the ORAM. This might be like "num_blocks" and similar + // stuff. self.enclave.check_key_images should return an AttestMessage that + // we send back to the user. + fn check_key_images_auth( + &mut self, + request: attest::Message, + ) -> Result { + log::trace!(self.logger, "Getting encrypted request"); + + let untrusted_query_response = self.prepare_untrusted_query(); let result_blob = self .enclave @@ -105,6 +151,28 @@ impl KeyImageService { Ok(resp) } + /// Unwrap and forward to enclave + // self.enclave.check_key_images should take both a NonceMessage and an + // UntrustedKeyImageQueryResponse object that contains any data that is + // needed that isn't in the ORAM. This might be like "num_blocks" and similar + // stuff. self.enclave.check_key_images should return an AttestMessage that + // we send back to the user. + fn check_key_image_store_auth( + &mut self, + request: attest::NonceMessage, + ) -> Result { + log::trace!(self.logger, "Getting encrypted request"); + + let untrusted_query_response = self.prepare_untrusted_query(); + + let response = self + .enclave + .check_key_image_store(request.into(), untrusted_query_response) + .map_err(|e| self.enclave_err_to_rpc_status("enclave request", e))?; + + Ok(response.into()) + } + // Helper function that is common fn enclave_err_to_rpc_status(&self, context: &str, src: EnclaveError) -> RpcStatus { // Treat prost-decode error as an invalid arg, @@ -118,6 +186,38 @@ impl KeyImageService { other => rpc_internal_error(context, format!("{}", &other), &self.logger), } } + + /// Handle MultiKeyImageStoreRequest contents sent by a router to this + /// store. + fn process_queries( + &mut self, + fog_ledger_store_uri: KeyImageStoreUri, + queries: Vec, + ) -> MultiKeyImageStoreResponse { + let mut response = MultiKeyImageStoreResponse::new(); + // The router needs our own URI, in case auth fails / hasn't been started yet. + response.set_store_uri(fog_ledger_store_uri.url().to_string()); + + for query in queries.into_iter() { + // Only one of the query messages in the multi-store query is intended for this + // store. It's a bit of a broadcast model - all queries are sent to + // all stores, and then the stores evaluate which message is meant + // for them. + if let Ok(attested_message) = self.check_key_image_store_auth(query) { + response.set_query_response(attested_message); + response.set_status(MultiKeyImageStoreResponseStatus::SUCCESS); + //Note that set_fog_ledger_store_uri has been taken care of above. + + return response; + } + } + + // TODO: different response code for "none found matching" from "authentication + // error," potentially? + + response.set_status(MultiKeyImageStoreResponseStatus::AUTHENTICATION_ERROR); + response + } } impl FogKeyImageApi for KeyImageService { @@ -147,9 +247,8 @@ impl FogKeyImageApi for KeyImageServic return send_result(ctx, sink, err.into(), logger); } - // TODO: Use the prost message directly, once available - match self.enclave.client_accept(request.into()) { - Ok((response, _session_id)) => { + match self.auth_impl(request.into(), &logger) { + Ok(response) => { send_result(ctx, sink, Ok(response.into()), logger); } Err(client_error) => { @@ -161,18 +260,68 @@ impl FogKeyImageApi for KeyImageServic client_error ); // TODO: increment failed inbound peering counter. - send_result( - ctx, - sink, - Err(rpc_permissions_error( - "client_auth", - "Permission denied", - logger, - )), + send_result(ctx, sink, Err(client_error), logger); + } + } + }); + } +} + +impl KeyImageStoreApi for KeyImageService { + fn auth( + &mut self, + ctx: grpcio::RpcContext, + req: AuthMessage, + sink: grpcio::UnarySink, + ) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + if let Err(err) = self.authenticator.authenticate_rpc(&ctx) { + return send_result(ctx, sink, err.into(), logger); + } + + match self.auth_impl(req.into(), &logger) { + Ok(response) => { + send_result(ctx, sink, Ok(response.into()), logger); + } + Err(client_error) => { + // This is debug because there's no requirement on the remote party to trigger + // it. + log::info!( logger, + "LedgerEnclave::client_accept failed: {}", + client_error ); + // TODO: increment failed inbound peering counter. + send_result(ctx, sink, Err(client_error), logger); } } }); } + + fn multi_key_image_store_query( + &mut self, + ctx: grpcio::RpcContext, + req: MultiKeyImageStoreRequest, + sink: grpcio::UnarySink, + ) { + let _timer = SVC_COUNTERS.req(&ctx); + mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { + if let Err(err) = self.authenticator.authenticate_rpc(&ctx) { + return send_result(ctx, sink, err.into(), logger); + } + if let KeyImageClientListenUri::Store(store_uri) = self.client_listen_uri.clone() { + let response = self.process_queries(store_uri, req.queries.into_vec()); + send_result(ctx, sink, Ok(response), logger) + } else { + let rpc_permissions_error = rpc_permissions_error( + "multi_key_image_store_query", + "Permission denied: the multi_key_image_store_query is not accessible to clients" + .to_string(), + logger, + ); + send_result(ctx, sink, Err(rpc_permissions_error), logger) + } + }); + } } diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs new file mode 100644 index 0000000000..6106fe3235 --- /dev/null +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -0,0 +1,152 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +use std::sync::{Arc, Mutex}; + +use futures::executor::block_on; +use mc_common::{ + logger::{log, Logger}, + time::TimeProvider, +}; +use mc_fog_api::ledger_grpc; +use mc_fog_ledger_enclave::LedgerEnclaveProxy; +use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme}; +use mc_ledger_db::LedgerDB; +use mc_util_grpc::{ + AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, + TokenAuthenticator, +}; +use mc_util_uri::Uri; +use mc_watcher::watcher_db::WatcherDB; + +use crate::{ + config::LedgerStoreConfig, server::DbPollSharedState, KeyImageClientListenUri, KeyImageService, +}; + +pub struct KeyImageStoreServer { + server: grpcio::Server, + logger: Logger, +} + +impl KeyImageStoreServer { + /// Creates a new key image store server instance + pub fn new_from_config( + config: LedgerStoreConfig, + enclave: E, + ledger: LedgerDB, + watcher: WatcherDB, + time_provider: impl TimeProvider + 'static, + logger: Logger, + ) -> KeyImageStoreServer + where + E: LedgerEnclaveProxy, + { + let client_authenticator: Arc = + if let Some(shared_secret) = config.client_auth_token_secret.as_ref() { + Arc::new(TokenAuthenticator::new( + *shared_secret, + config.client_auth_token_max_lifetime, + time_provider, + )) + } else { + Arc::new(AnonymousAuthenticator::default()) + }; + + Self::new( + config.chain_id, + client_authenticator, + config.client_listen_uri, + enclave, + ledger, + watcher, + logger, + ) + } + + pub fn new( + chain_id: String, + client_authenticator: Arc, + client_listen_uri: Uri, + enclave: E, + ledger: LedgerDB, + watcher: WatcherDB, + logger: Logger, + ) -> KeyImageStoreServer + where + E: LedgerEnclaveProxy, + { + let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); + + let key_image_service = KeyImageService::new( + KeyImageClientListenUri::Store(client_listen_uri.clone()), + chain_id, + ledger.clone(), + watcher.clone(), + enclave, + shared_state, + client_authenticator.clone(), + logger.clone(), + ); + Self::new_from_service(key_image_service, client_listen_uri.clone(), logger.clone()) + } + + pub fn new_from_service( + key_image_service: KeyImageService, + client_listen_uri: Uri, + logger: Logger, + ) -> KeyImageStoreServer + where + E: LedgerEnclaveProxy, + { + let readiness_indicator = ReadinessIndicator::default(); + + let env = Arc::new( + grpcio::EnvBuilder::new() + .name_prefix("key-image-store-server".to_string()) + .build(), + ); + + // Health check service + let health_service = + mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) + .into_service(); + + // Build our store server. + // Init ledger store service. + let ledger_store_service = ledger_grpc::create_key_image_store_api(key_image_service); + log::debug!(logger, "Constructed Key Image Store GRPC Service"); + + // Package service into grpc server + log::info!( + logger, + "Starting Key Image Store server on {}", + client_listen_uri.addr(), + ); + + let server_builder = grpcio::ServerBuilder::new(env) + .register_service(ledger_store_service) + .register_service(health_service) + .bind_using_uri(&client_listen_uri, logger.clone()); + let server = server_builder.build().unwrap(); + + Self { server, logger } + } + + /// Starts the server + pub fn start(&mut self) { + self.server.start(); + for (host, port) in self.server.bind_addrs() { + log::info!(self.logger, "API listening on {}:{}", host, port); + } + } + + /// Stops the server + pub fn stop(&mut self) { + block_on(self.server.shutdown()).expect("Could not stop grpc server"); + } +} + +impl Drop for KeyImageStoreServer { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 0602257026..0a8256abc3 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -17,7 +17,9 @@ mod untrusted_tx_out_service; use mc_util_metrics::ServiceMetrics; pub use block_service::BlockService; -pub use config::{LedgerRouterConfig, LedgerServerConfig}; +pub use config::{ + KeyImageClientListenUri, LedgerRouterConfig, LedgerServerConfig, LedgerStoreConfig, +}; pub use key_image_service::KeyImageService; pub use merkle_proof_service::MerkleProofService; pub use server::LedgerServer; @@ -27,3 +29,5 @@ lazy_static::lazy_static! { pub static ref SVC_COUNTERS: ServiceMetrics = ServiceMetrics::new_and_registered("fog_ledger"); } pub use router_server::LedgerRouterServer; +pub use key_image_store_server::KeyImageStoreServer; +pub use server::DbPollSharedState; diff --git a/fog/ledger/server/src/server.rs b/fog/ledger/server/src/server.rs index f6159af053..8428857fec 100644 --- a/fog/ledger/server/src/server.rs +++ b/fog/ledger/server/src/server.rs @@ -1,8 +1,8 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation use crate::{ - config::LedgerServerConfig, counters, db_fetcher::DbFetcher, BlockService, KeyImageService, - MerkleProofService, UntrustedTxOutService, + config::LedgerServerConfig, counters, db_fetcher::DbFetcher, BlockService, + KeyImageClientListenUri, KeyImageService, MerkleProofService, UntrustedTxOutService, }; use displaydoc::Display; use futures::executor::block_on; @@ -103,6 +103,7 @@ impl LedgerServer KeyImageStoreUri { + // If a load-balancer were set up in the middle here + // this might need to be changed to + // {KeyImageStoreScheme::SCHEME_INSECURE}://localhost:1234/?responder-id={test_name} + let name = format!("{}://localhost:{}", KeyImageStoreScheme::SCHEME_INSECURE, port); + KeyImageStoreUri::from_str(&name).unwrap() +} + +pub struct TestingContext { + pub enclave: LedgerSgxEnclave, + pub ledger: LedgerDB, + pub responder_id: ResponderId, + pub rng: R, + pub store_config: LedgerStoreConfig, + pub tempdir: TempDir, + pub tx_source_url: Url, + pub watcher: WatcherDB, + pub watcher_path: TempDir, +} + +impl TestingContext { + pub fn new(test_name: &'static str, + logger: Logger, + port: u16, + omap_capacity: u64, + rng: R) + -> Self { + // Set up our directories. + let test_dir_name = format!("fog_ledger_test_{}", test_name); + let tempdir = TempDir::new(&test_dir_name).expect("Could not produce test_ledger tempdir"); + let test_path = PathBuf::from(tempdir.path()); + let user_keys_path = test_path.join(PathBuf::from("keys/")); + if !user_keys_path.exists() { + std::fs::create_dir(&user_keys_path).unwrap(); + } + + let test_uri = uri_for_test(port); + // This ID needs to match the host:port clients use in their URI when referencing the host node. + let responder_id = test_uri.responder_id().unwrap(); + + let enclave_path = std::env::current_exe() + .expect("Could not get the path of our executable") + // The test ends up in target/debug/deps/ + // rather than just target/debug/. So, + // we need the parent directory. + .parent() + .unwrap() + .with_file_name(ENCLAVE_FILE); + + let enclave = + LedgerSgxEnclave::new(enclave_path, &responder_id, omap_capacity, logger.clone()); + + // Make LedgerDB + let ledger_path = test_path.join(PathBuf::from("fog_ledger")); + let ledger = recreate_ledger_db(ledger_path.as_path()); + + // Set up wallet db. + let test_url_name = format!("http://{}.wallet.test.test", test_name); + let url = Url::parse(&test_url_name).unwrap(); + + let db_tmp = TempDir::new("wallet_db").expect("Could not make tempdir for wallet db"); + WatcherDB::create(db_tmp.path()).unwrap(); + let watcher = WatcherDB::open_rw(db_tmp.path(), &[url.clone()], logger).unwrap(); + + let config = LedgerStoreConfig { + chain_id: test_name.to_string(), + client_responder_id: responder_id.clone(), + client_listen_uri: test_uri.clone(), + ledger_db: ledger_path, + watcher_db: PathBuf::from(db_tmp.path()), + ias_api_key: Default::default(), + ias_spid: Default::default(), + admin_listen_uri: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + omap_capacity, + }; + + Self { + enclave, + ledger, + responder_id, + rng, + tempdir, + tx_source_url: url, + store_config: config, + watcher, + watcher_path: db_tmp, + } + } +} + +lazy_static::lazy_static! { + pub static ref TEST_OP_COUNTERS: OpMetrics = OpMetrics::new_and_registered("consensus_service"); +} + +lazy_static::lazy_static! { + pub static ref TEST_ENCLAVE_REPORT_TIMESTAMP: IntGauge = TEST_OP_COUNTERS.gauge("enclave_report_timestamp"); +} + +#[test_with_logger] +pub fn direct_key_image_store_check(logger: Logger) { + const TEST_NAME: &'static str = "direct_key_image_store_check"; + const PORT_START: u16 = 3223; + const OMAP_CAPACITY: u64 = 768; + + let port = PORT_START; + + let rng = RngType::from_entropy(); + let TestingContext { + enclave, + ledger, + responder_id, + mut rng, + tempdir: _tempdir, + tx_source_url: _tx_source_url, + watcher, + store_config, + watcher_path: _watcher_path } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); + + let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); + + let client_listen_uri = store_config.client_listen_uri.clone(); + let store_service = KeyImageService::new( + KeyImageClientListenUri::Store(client_listen_uri.clone()), + store_config.chain_id.clone(), + ledger, + watcher, + enclave.clone(), //LedgerSgxEnclave is an Arc internally + shared_state.clone(), + Arc::new(AnonymousAuthenticator::default()), + logger.clone(), + ); + + let mut store_server = KeyImageStoreServer::new_from_service( + store_service, + client_listen_uri.clone(), + logger.clone(), + ); + store_server.start(); + + // Set up IAS verficiation + // This will be a SimClient in testing contexts. + let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); + let mut report_cache_thread = Some(ReportCacheThread::start( + enclave.clone(), + ias_client.clone(), + store_config.ias_spid, + &TEST_ENCLAVE_REPORT_TIMESTAMP, + logger.clone(), + ).unwrap()).unwrap(); + + // Make GRPC client for sending requests. + + // Get the enclave to generate an auth request. + let client_auth_request = enclave + .ledger_store_init(responder_id.clone()) + .unwrap(); + // Submit auth request and wait for the response. + let (auth_response, router_to_store_session ) = enclave.frontend_accept(client_auth_request).unwrap(); + // Finish the enclave's handshake with itself. + enclave.ledger_store_connect(responder_id.clone(), auth_response.into()).unwrap(); + println!("router_to_store_session is: {:?}", &router_to_store_session); + + // Generate a dummy key image we're going to check against. + let mut test_key_image_bytes: [u8; 32] = [0u8; 32]; + rng.fill(&mut test_key_image_bytes); + let test_key_image = KeyImageData { + key_image: test_key_image_bytes.into(), + block_index: 1, + timestamp: 255, + }; + enclave + .add_key_image_data(vec![test_key_image.clone()]) + .unwrap(); + + // Set up the client's end of the encrypted connection. + let initiator = Start::new(responder_id.to_string()); + + let init_input = ClientInitiate::::default(); + let (initiator, auth_request_output) = initiator.try_next(&mut rng, init_input).unwrap(); + + // Authenticate our "client" with the server. + let auth_message = attest::AuthMessage::from(auth_request_output); + let (client_auth_response, client_session) = enclave.client_accept(auth_message.into()).unwrap(); + println!("Initial client_session is {:?}", &client_session); + // We will need to double-convert, ClientAuthResponse -> AuthMessage -> AuthResponseOutput + let auth_message = attest::AuthMessage::from(client_auth_response); + // Initiator accepts responder's message. + let auth_response_event = AuthResponseInput::new(auth_message.into(), Verifier::default()); + // Should be a valid noise connection at this point. + let (mut noise_connection, _verification_report) = + initiator.try_next(&mut rng, auth_response_event).unwrap(); + + //Construct our request. + let key_images_request = CheckKeyImagesRequest { + queries: vec![KeyImageQuery{ + key_image: test_key_image.key_image.clone(), + start_block: 1 + }], + }; + // Protobuf-encoded plaintext. + let message_encoded = mc_util_serial::encode(&key_images_request); + let ciphertext = noise_connection.encrypt(&[], &message_encoded).unwrap(); + let msg: EnclaveMessage = EnclaveMessage { + aad: vec![], + channel_id: client_session, + data: ciphertext, + }; + + // Decrypt and seal + let sealed_query = enclave.decrypt_and_seal_query(msg).unwrap(); + println!("Client session on sealed_query is {:?}", &sealed_query.channel_id); + let mut multi_query = enclave.create_multi_key_image_store_query_data(sealed_query.clone()).unwrap(); + + let query: EnclaveMessage = multi_query.pop().unwrap(); + println!("Nonce session on message is {:?}", query.channel_id); + + // Get an untrusted query + let ( + highest_processed_block_count, + last_known_block_cumulative_txo_count, + latest_block_version, + ) = { + let shared_state = shared_state.lock().expect("mutex poisoned"); + ( + shared_state.highest_processed_block_count, + shared_state.last_known_block_cumulative_txo_count, + shared_state.latest_block_version, + ) + }; + + let untrusted_kiqr = UntrustedKeyImageQueryResponse { + highest_processed_block_count, + last_known_block_cumulative_txo_count, + latest_block_version, + max_block_version: latest_block_version.max(*MAX_BLOCK_VERSION), + }; + + let result = enclave.check_key_image_store(query, untrusted_kiqr.clone()).unwrap(); + + let responses_btree: BTreeMap> = BTreeMap::from([(responder_id, result)]); + + let client_response = enclave.collate_shard_query_responses(sealed_query, responses_btree).unwrap(); + + let plaintext_bytes = noise_connection.decrypt(&client_response.aad, &client_response.data).unwrap(); + + let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes).unwrap(); + assert_eq!(done_response.results.len(), 1); + + let test_results: Vec<(KeyImage, u32)> = done_response.results.into_iter() + .map(|result| { + (result.key_image.clone(), result.key_image_result_code) + }).collect(); + + // The key image result code for a spent key image is 1. + assert!( + test_results.contains( + &(test_key_image.key_image, 1) + ) + ); + + report_cache_thread.stop().unwrap(); +} From c9bd1204911d9878156b6021183aa7547683fd36 Mon Sep 17 00:00:00 2001 From: Millie C Date: Thu, 15 Dec 2022 15:56:30 -0500 Subject: [PATCH 23/33] Cargo fmt --- fog/ledger/server/tests/store_tests.rs | 162 +++++++++++++++---------- 1 file changed, 95 insertions(+), 67 deletions(-) diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index dff85d3c49..eb8e8996cd 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -1,7 +1,8 @@ use std::{ + collections::BTreeMap, path::PathBuf, str::FromStr, - sync::{Arc, Mutex}, collections::BTreeMap, + sync::{Arc, Mutex}, }; use mc_attest_ake::{AuthResponseInput, ClientInitiate, Start, Transition}; @@ -16,21 +17,23 @@ use mc_common::{ }; use mc_crypto_keys::X25519; use mc_crypto_rand::{CryptoRng, RngCore}; -use mc_fog_ledger_enclave::{KeyImageData, LedgerEnclave, LedgerSgxEnclave, ENCLAVE_FILE, CheckKeyImagesResponse}; +use mc_fog_ledger_enclave::{ + CheckKeyImagesResponse, KeyImageData, LedgerEnclave, LedgerSgxEnclave, ENCLAVE_FILE, +}; use mc_fog_ledger_enclave_api::UntrustedKeyImageQueryResponse; use mc_fog_ledger_server::{ DbPollSharedState, KeyImageClientListenUri, KeyImageService, KeyImageStoreServer, LedgerStoreConfig, }; use mc_fog_types::ledger::{CheckKeyImagesRequest, KeyImageQuery}; -use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri, ConnectionUri}; -use mc_ledger_db::{LedgerDB, test_utils::recreate_ledger_db}; +use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme, KeyImageStoreUri}; +use mc_ledger_db::{test_utils::recreate_ledger_db, LedgerDB}; use mc_sgx_report_cache_untrusted::ReportCacheThread; use mc_transaction_core::ring_signature::KeyImage; -use mc_util_grpc::{AnonymousAuthenticator}; +use mc_util_grpc::AnonymousAuthenticator; use mc_util_metrics::{IntGauge, OpMetrics}; -use mc_util_test_helper::{SeedableRng, RngType, Rng}; -use mc_util_uri::{UriScheme}; +use mc_util_test_helper::{Rng, RngType, SeedableRng}; +use mc_util_uri::UriScheme; use mc_watcher::watcher_db::WatcherDB; use aes_gcm::Aes256Gcm; @@ -38,11 +41,16 @@ use sha2::Sha512; use tempdir::TempDir; use url::Url; -pub fn uri_for_test(port: u16) -> KeyImageStoreUri { - // If a load-balancer were set up in the middle here - // this might need to be changed to - // {KeyImageStoreScheme::SCHEME_INSECURE}://localhost:1234/?responder-id={test_name} - let name = format!("{}://localhost:{}", KeyImageStoreScheme::SCHEME_INSECURE, port); +pub fn uri_for_test(port: u16) -> KeyImageStoreUri { + // If a load-balancer were set up in the middle here + // this might need to be changed to + // {KeyImageStoreScheme::SCHEME_INSECURE}://localhost:1234/? + // responder-id={test_name} + let name = format!( + "{}://localhost:{}", + KeyImageStoreScheme::SCHEME_INSECURE, + port + ); KeyImageStoreUri::from_str(&name).unwrap() } @@ -50,7 +58,7 @@ pub struct TestingContext { pub enclave: LedgerSgxEnclave, pub ledger: LedgerDB, pub responder_id: ResponderId, - pub rng: R, + pub rng: R, pub store_config: LedgerStoreConfig, pub tempdir: TempDir, pub tx_source_url: Url, @@ -59,13 +67,14 @@ pub struct TestingContext { } impl TestingContext { - pub fn new(test_name: &'static str, - logger: Logger, - port: u16, - omap_capacity: u64, - rng: R) - -> Self { - // Set up our directories. + pub fn new( + test_name: &'static str, + logger: Logger, + port: u16, + omap_capacity: u64, + rng: R, + ) -> Self { + // Set up our directories. let test_dir_name = format!("fog_ledger_test_{}", test_name); let tempdir = TempDir::new(&test_dir_name).expect("Could not produce test_ledger tempdir"); let test_path = PathBuf::from(tempdir.path()); @@ -75,7 +84,8 @@ impl TestingContext { } let test_uri = uri_for_test(port); - // This ID needs to match the host:port clients use in their URI when referencing the host node. + // This ID needs to match the host:port clients use in their URI when + // referencing the host node. let responder_id = test_uri.responder_id().unwrap(); let enclave_path = std::env::current_exe() @@ -141,28 +151,29 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { const TEST_NAME: &'static str = "direct_key_image_store_check"; - const PORT_START: u16 = 3223; - const OMAP_CAPACITY: u64 = 768; + const PORT_START: u16 = 3223; + const OMAP_CAPACITY: u64 = 768; let port = PORT_START; - let rng = RngType::from_entropy(); - let TestingContext { + let rng = RngType::from_entropy(); + let TestingContext { enclave, ledger, responder_id, mut rng, tempdir: _tempdir, tx_source_url: _tx_source_url, - watcher, - store_config, - watcher_path: _watcher_path } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); - + watcher, + store_config, + watcher_path: _watcher_path, + } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); + let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); - let client_listen_uri = store_config.client_listen_uri.clone(); + let client_listen_uri = store_config.client_listen_uri.clone(); let store_service = KeyImageService::new( - KeyImageClientListenUri::Store(client_listen_uri.clone()), + KeyImageClientListenUri::Store(client_listen_uri.clone()), store_config.chain_id.clone(), ledger, watcher, @@ -181,25 +192,31 @@ pub fn direct_key_image_store_check(logger: Logger) { // Set up IAS verficiation // This will be a SimClient in testing contexts. - let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let mut report_cache_thread = Some(ReportCacheThread::start( - enclave.clone(), - ias_client.clone(), - store_config.ias_spid, - &TEST_ENCLAVE_REPORT_TIMESTAMP, - logger.clone(), - ).unwrap()).unwrap(); + let ias_client = + AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); + let mut report_cache_thread = Some( + ReportCacheThread::start( + enclave.clone(), + ias_client.clone(), + store_config.ias_spid, + &TEST_ENCLAVE_REPORT_TIMESTAMP, + logger.clone(), + ) + .unwrap(), + ) + .unwrap(); // Make GRPC client for sending requests. // Get the enclave to generate an auth request. - let client_auth_request = enclave - .ledger_store_init(responder_id.clone()) - .unwrap(); + let client_auth_request = enclave.ledger_store_init(responder_id.clone()).unwrap(); // Submit auth request and wait for the response. - let (auth_response, router_to_store_session ) = enclave.frontend_accept(client_auth_request).unwrap(); + let (auth_response, router_to_store_session) = + enclave.frontend_accept(client_auth_request).unwrap(); // Finish the enclave's handshake with itself. - enclave.ledger_store_connect(responder_id.clone(), auth_response.into()).unwrap(); + enclave + .ledger_store_connect(responder_id.clone(), auth_response.into()) + .unwrap(); println!("router_to_store_session is: {:?}", &router_to_store_session); // Generate a dummy key image we're going to check against. @@ -222,9 +239,11 @@ pub fn direct_key_image_store_check(logger: Logger) { // Authenticate our "client" with the server. let auth_message = attest::AuthMessage::from(auth_request_output); - let (client_auth_response, client_session) = enclave.client_accept(auth_message.into()).unwrap(); - println!("Initial client_session is {:?}", &client_session); - // We will need to double-convert, ClientAuthResponse -> AuthMessage -> AuthResponseOutput + let (client_auth_response, client_session) = + enclave.client_accept(auth_message.into()).unwrap(); + println!("Initial client_session is {:?}", &client_session); + // We will need to double-convert, ClientAuthResponse -> AuthMessage -> + // AuthResponseOutput let auth_message = attest::AuthMessage::from(client_auth_response); // Initiator accepts responder's message. let auth_response_event = AuthResponseInput::new(auth_message.into(), Verifier::default()); @@ -234,9 +253,9 @@ pub fn direct_key_image_store_check(logger: Logger) { //Construct our request. let key_images_request = CheckKeyImagesRequest { - queries: vec![KeyImageQuery{ - key_image: test_key_image.key_image.clone(), - start_block: 1 + queries: vec![KeyImageQuery { + key_image: test_key_image.key_image.clone(), + start_block: 1, }], }; // Protobuf-encoded plaintext. @@ -250,11 +269,16 @@ pub fn direct_key_image_store_check(logger: Logger) { // Decrypt and seal let sealed_query = enclave.decrypt_and_seal_query(msg).unwrap(); - println!("Client session on sealed_query is {:?}", &sealed_query.channel_id); - let mut multi_query = enclave.create_multi_key_image_store_query_data(sealed_query.clone()).unwrap(); + println!( + "Client session on sealed_query is {:?}", + &sealed_query.channel_id + ); + let mut multi_query = enclave + .create_multi_key_image_store_query_data(sealed_query.clone()) + .unwrap(); - let query: EnclaveMessage = multi_query.pop().unwrap(); - println!("Nonce session on message is {:?}", query.channel_id); + let query: EnclaveMessage = multi_query.pop().unwrap(); + println!("Nonce session on message is {:?}", query.channel_id); // Get an untrusted query let ( @@ -277,28 +301,32 @@ pub fn direct_key_image_store_check(logger: Logger) { max_block_version: latest_block_version.max(*MAX_BLOCK_VERSION), }; - let result = enclave.check_key_image_store(query, untrusted_kiqr.clone()).unwrap(); + let result = enclave + .check_key_image_store(query, untrusted_kiqr.clone()) + .unwrap(); - let responses_btree: BTreeMap> = BTreeMap::from([(responder_id, result)]); + let responses_btree: BTreeMap> = + BTreeMap::from([(responder_id, result)]); - let client_response = enclave.collate_shard_query_responses(sealed_query, responses_btree).unwrap(); + let client_response = enclave + .collate_shard_query_responses(sealed_query, responses_btree) + .unwrap(); - let plaintext_bytes = noise_connection.decrypt(&client_response.aad, &client_response.data).unwrap(); + let plaintext_bytes = noise_connection + .decrypt(&client_response.aad, &client_response.data) + .unwrap(); let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes).unwrap(); assert_eq!(done_response.results.len(), 1); - let test_results: Vec<(KeyImage, u32)> = done_response.results.into_iter() - .map(|result| { - (result.key_image.clone(), result.key_image_result_code) - }).collect(); + let test_results: Vec<(KeyImage, u32)> = done_response + .results + .into_iter() + .map(|result| (result.key_image.clone(), result.key_image_result_code)) + .collect(); // The key image result code for a spent key image is 1. - assert!( - test_results.contains( - &(test_key_image.key_image, 1) - ) - ); + assert!(test_results.contains(&(test_key_image.key_image, 1))); report_cache_thread.stop().unwrap(); } From 0e0b8498c9f9b819af2fab82e4b2ad59594f8c52 Mon Sep 17 00:00:00 2001 From: Millie C Date: Fri, 16 Dec 2022 19:56:25 -0500 Subject: [PATCH 24/33] Run clippy --- fog/ledger/server/src/key_image_service.rs | 8 ++++---- .../server/src/key_image_store_server.rs | 6 +++--- fog/ledger/server/tests/store_tests.rs | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index fd8ad438a8..1fa5700617 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -247,9 +247,9 @@ impl FogKeyImageApi for KeyImageServic return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(request.into(), &logger) { + match self.auth_impl(request, logger) { Ok(response) => { - send_result(ctx, sink, Ok(response.into()), logger); + send_result(ctx, sink, Ok(response), logger); } Err(client_error) => { // This is debug because there's no requirement on the remote party to trigger @@ -280,9 +280,9 @@ impl KeyImageStoreApi for KeyImageServ return send_result(ctx, sink, err.into(), logger); } - match self.auth_impl(req.into(), &logger) { + match self.auth_impl(req, logger) { Ok(response) => { - send_result(ctx, sink, Ok(response.into()), logger); + send_result(ctx, sink, Ok(response), logger); } Err(client_error) => { // This is debug because there's no requirement on the remote party to trigger diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs index 6106fe3235..17b32ded4a 100644 --- a/fog/ledger/server/src/key_image_store_server.rs +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -79,14 +79,14 @@ impl KeyImageStoreServer { let key_image_service = KeyImageService::new( KeyImageClientListenUri::Store(client_listen_uri.clone()), chain_id, - ledger.clone(), - watcher.clone(), + ledger, + watcher, enclave, shared_state, client_authenticator.clone(), logger.clone(), ); - Self::new_from_service(key_image_service, client_listen_uri.clone(), logger.clone()) + Self::new_from_service(key_image_service, client_listen_uri, logger.clone()) } pub fn new_from_service( diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index eb8e8996cd..c0f01246de 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -115,7 +115,7 @@ impl TestingContext { let config = LedgerStoreConfig { chain_id: test_name.to_string(), client_responder_id: responder_id.clone(), - client_listen_uri: test_uri.clone(), + client_listen_uri: test_uri, ledger_db: ledger_path, watcher_db: PathBuf::from(db_tmp.path()), ias_api_key: Default::default(), @@ -150,7 +150,7 @@ lazy_static::lazy_static! { #[test_with_logger] pub fn direct_key_image_store_check(logger: Logger) { - const TEST_NAME: &'static str = "direct_key_image_store_check"; + const TEST_NAME: &str = "direct_key_image_store_check"; const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; @@ -185,7 +185,7 @@ pub fn direct_key_image_store_check(logger: Logger) { let mut store_server = KeyImageStoreServer::new_from_service( store_service, - client_listen_uri.clone(), + client_listen_uri, logger.clone(), ); store_server.start(); @@ -197,7 +197,7 @@ pub fn direct_key_image_store_check(logger: Logger) { let mut report_cache_thread = Some( ReportCacheThread::start( enclave.clone(), - ias_client.clone(), + ias_client, store_config.ias_spid, &TEST_ENCLAVE_REPORT_TIMESTAMP, logger.clone(), @@ -215,7 +215,7 @@ pub fn direct_key_image_store_check(logger: Logger) { enclave.frontend_accept(client_auth_request).unwrap(); // Finish the enclave's handshake with itself. enclave - .ledger_store_connect(responder_id.clone(), auth_response.into()) + .ledger_store_connect(responder_id.clone(), auth_response) .unwrap(); println!("router_to_store_session is: {:?}", &router_to_store_session); @@ -228,7 +228,7 @@ pub fn direct_key_image_store_check(logger: Logger) { timestamp: 255, }; enclave - .add_key_image_data(vec![test_key_image.clone()]) + .add_key_image_data(vec![test_key_image]) .unwrap(); // Set up the client's end of the encrypted connection. @@ -254,7 +254,7 @@ pub fn direct_key_image_store_check(logger: Logger) { //Construct our request. let key_images_request = CheckKeyImagesRequest { queries: vec![KeyImageQuery { - key_image: test_key_image.key_image.clone(), + key_image: test_key_image.key_image, start_block: 1, }], }; @@ -302,7 +302,7 @@ pub fn direct_key_image_store_check(logger: Logger) { }; let result = enclave - .check_key_image_store(query, untrusted_kiqr.clone()) + .check_key_image_store(query, untrusted_kiqr) .unwrap(); let responses_btree: BTreeMap> = @@ -322,7 +322,7 @@ pub fn direct_key_image_store_check(logger: Logger) { let test_results: Vec<(KeyImage, u32)> = done_response .results .into_iter() - .map(|result| (result.key_image.clone(), result.key_image_result_code)) + .map(|result| (result.key_image, result.key_image_result_code)) .collect(); // The key image result code for a spent key image is 1. From 9e71767257178fd55037d42ed8402ae108308d66 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 6 Dec 2022 09:54:26 -0800 Subject: [PATCH 25/33] Sort itertools properly in Cargo.toml --- fog/ledger/server/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index af99c77e7e..337efee54a 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -65,7 +65,6 @@ retry = "2.0" serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = "1.0" url = "2.3" -itertools = "0.10" [build-dependencies] mc-util-build-script = { path = "../../../util/build/script" } From af883186c0d7ac38f9f6341b486cd28daf16c533 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Mon, 16 Jan 2023 21:51:01 -0800 Subject: [PATCH 26/33] Rebase and update to match current fog ledger router branch --- Cargo.lock | 2 +- fog/ledger/server/Cargo.toml | 6 +- fog/ledger/server/src/bin/key_image_router.rs | 71 ------------------- fog/ledger/server/src/key_image_service.rs | 2 +- .../server/src/key_image_store_server.rs | 28 +++++--- fog/ledger/server/src/lib.rs | 3 +- fog/ledger/server/tests/store_tests.rs | 18 ++--- 7 files changed, 30 insertions(+), 100 deletions(-) delete mode 100644 fog/ledger/server/src/bin/key_image_router.rs diff --git a/Cargo.lock b/Cargo.lock index 136d3a4813..af7e8c1417 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3949,7 +3949,7 @@ dependencies = [ name = "mc-fog-ledger-server" version = "4.0.0" dependencies = [ - "aes-gcm 0.9.4", + "aes-gcm", "clap 4.0.32", "displaydoc", "futures", diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 337efee54a..08af5672e4 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -18,7 +18,7 @@ name = "ledger_router" path = "src/bin/router.rs" [[bin]] -name = "ledger_store" +name = "key_image_store" path = "src/bin/key_image_store.rs" [dependencies] @@ -91,9 +91,9 @@ mc-fog-ledger-test-infra = { path = "../test_infra" } mc-fog-test-infra = { path = "../../test_infra" } # third-party -aes-gcm = "0.9.4" +aes-gcm = "0.10.1" pem = "1.1" +portpicker = "0.1.1" sha2 = "0.10" tempdir = "0.3" x509-signature = "0.5" -portpicker = "0.1.1" diff --git a/fog/ledger/server/src/bin/key_image_router.rs b/fog/ledger/server/src/bin/key_image_router.rs deleted file mode 100644 index 3409f7d6dd..0000000000 --- a/fog/ledger/server/src/bin/key_image_router.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - env, - str::FromStr, - sync::{Arc, RwLock}, -}; - -use clap::Parser; -use grpcio::ChannelBuilder; -use mc_common::logger::log; -use mc_fog_api::ledger_grpc::KeyImageStoreApiClient; -use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE}; -use mc_fog_ledger_server::{KeyImageRouterServer, LedgerRouterConfig}; -use mc_fog_uri::{KeyImageStoreScheme, KeyImageStoreUri}; - -use mc_util_grpc::ConnectionUriGrpcioChannel; -use mc_util_uri::UriScheme; - -fn main() { - mc_common::setup_panic_handler(); - let config = LedgerRouterConfig::parse(); - let (logger, _global_logger_guard) = - mc_common::logger::create_app_logger(mc_common::logger::o!()); - - let enclave_path = env::current_exe() - .expect("Could not get the path of our executable") - .with_file_name(ENCLAVE_FILE); - log::info!( - logger, - "enclave path {}, responder ID {}", - enclave_path.to_str().unwrap(), - &config.client_responder_id - ); - let enclave = LedgerSgxEnclave::new( - enclave_path, - &config.client_responder_id, - config.omap_capacity, - logger.clone(), - ); - - let mut ledger_store_grpc_clients = HashMap::new(); - let grpc_env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("Main-RPC".to_string()) - .build(), - ); - for i in 0..50 { - let shard_uri_string = format!( - "{}://node{}.test.mobilecoin.com:3225", - KeyImageStoreScheme::SCHEME_INSECURE, - i - ); - let shard_uri = KeyImageStoreUri::from_str(&shard_uri_string).unwrap(); - let ledger_store_grpc_client = KeyImageStoreApiClient::new( - ChannelBuilder::default_channel_builder(grpc_env.clone()) - .connect_to_uri(&shard_uri, &logger), - ); - ledger_store_grpc_clients.insert(shard_uri, Arc::new(ledger_store_grpc_client)); - } - let ledger_store_grpc_clients = Arc::new(RwLock::new(ledger_store_grpc_clients)); - - let mut router_server = - KeyImageRouterServer::new(config, enclave, ledger_store_grpc_clients, logger); - router_server.start(); - - loop { - std::thread::sleep(std::time::Duration::from_millis(1000)); - } -} diff --git a/fog/ledger/server/src/key_image_service.rs b/fog/ledger/server/src/key_image_service.rs index 1fa5700617..a23deaa957 100644 --- a/fog/ledger/server/src/key_image_service.rs +++ b/fog/ledger/server/src/key_image_service.rs @@ -1,5 +1,5 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use crate::{server::DbPollSharedState, SVC_COUNTERS}; +use crate::{server::DbPollSharedState, KeyImageClientListenUri, SVC_COUNTERS}; use grpcio::{RpcContext, RpcStatus, UnarySink}; use mc_attest_api::{ attest, diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs index 17b32ded4a..1716e438eb 100644 --- a/fog/ledger/server/src/key_image_store_server.rs +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -9,13 +9,12 @@ use mc_common::{ }; use mc_fog_api::ledger_grpc; use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme}; +use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; use mc_ledger_db::LedgerDB; use mc_util_grpc::{ AnonymousAuthenticator, Authenticator, ConnectionUriGrpcioServer, ReadinessIndicator, TokenAuthenticator, }; -use mc_util_uri::Uri; use mc_watcher::watcher_db::WatcherDB; use crate::{ @@ -24,6 +23,7 @@ use crate::{ pub struct KeyImageStoreServer { server: grpcio::Server, + client_listen_uri: KeyImageStoreUri, logger: Logger, } @@ -65,7 +65,7 @@ impl KeyImageStoreServer { pub fn new( chain_id: String, client_authenticator: Arc, - client_listen_uri: Uri, + client_listen_uri: KeyImageStoreUri, enclave: E, ledger: LedgerDB, watcher: WatcherDB, @@ -91,7 +91,7 @@ impl KeyImageStoreServer { pub fn new_from_service( key_image_service: KeyImageService, - client_listen_uri: Uri, + client_listen_uri: KeyImageStoreUri, logger: Logger, ) -> KeyImageStoreServer where @@ -122,21 +122,27 @@ impl KeyImageStoreServer { client_listen_uri.addr(), ); - let server_builder = grpcio::ServerBuilder::new(env) + let server = grpcio::ServerBuilder::new(env) .register_service(ledger_store_service) .register_service(health_service) - .bind_using_uri(&client_listen_uri, logger.clone()); - let server = server_builder.build().unwrap(); + .build_using_uri(&client_listen_uri, logger.clone()) + .unwrap(); - Self { server, logger } + Self { + server, + client_listen_uri, + logger, + } } /// Starts the server pub fn start(&mut self) { self.server.start(); - for (host, port) in self.server.bind_addrs() { - log::info!(self.logger, "API listening on {}:{}", host, port); - } + log::info!( + self.logger, + "API listening on {}", + self.client_listen_uri.addr() + ); } /// Stops the server diff --git a/fog/ledger/server/src/lib.rs b/fog/ledger/server/src/lib.rs index 0a8256abc3..ce271b03a2 100644 --- a/fog/ledger/server/src/lib.rs +++ b/fog/ledger/server/src/lib.rs @@ -6,6 +6,7 @@ mod counters; mod db_fetcher; mod error; mod key_image_service; +mod key_image_store_server; mod merkle_proof_service; mod router_admin_service; mod router_handlers; @@ -28,6 +29,6 @@ pub use untrusted_tx_out_service::UntrustedTxOutService; lazy_static::lazy_static! { pub static ref SVC_COUNTERS: ServiceMetrics = ServiceMetrics::new_and_registered("fog_ledger"); } -pub use router_server::LedgerRouterServer; pub use key_image_store_server::KeyImageStoreServer; +pub use router_server::LedgerRouterServer; pub use server::DbPollSharedState; diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index c0f01246de..76f45f6537 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -5,6 +5,7 @@ use std::{ sync::{Arc, Mutex}, }; +use itertools::Itertools; use mc_attest_ake::{AuthResponseInput, ClientInitiate, Start, Transition}; use mc_attest_api::attest; use mc_attest_enclave_api::{ClientSession, EnclaveMessage, NonceSession}; @@ -29,7 +30,6 @@ use mc_fog_types::ledger::{CheckKeyImagesRequest, KeyImageQuery}; use mc_fog_uri::{ConnectionUri, KeyImageStoreScheme, KeyImageStoreUri}; use mc_ledger_db::{test_utils::recreate_ledger_db, LedgerDB}; use mc_sgx_report_cache_untrusted::ReportCacheThread; -use mc_transaction_core::ring_signature::KeyImage; use mc_util_grpc::AnonymousAuthenticator; use mc_util_metrics::{IntGauge, OpMetrics}; use mc_util_test_helper::{Rng, RngType, SeedableRng}; @@ -183,11 +183,8 @@ pub fn direct_key_image_store_check(logger: Logger) { logger.clone(), ); - let mut store_server = KeyImageStoreServer::new_from_service( - store_service, - client_listen_uri, - logger.clone(), - ); + let mut store_server = + KeyImageStoreServer::new_from_service(store_service, client_listen_uri, logger.clone()); store_server.start(); // Set up IAS verficiation @@ -227,9 +224,7 @@ pub fn direct_key_image_store_check(logger: Logger) { block_index: 1, timestamp: 255, }; - enclave - .add_key_image_data(vec![test_key_image]) - .unwrap(); + enclave.add_key_image_data(vec![test_key_image]).unwrap(); // Set up the client's end of the encrypted connection. let initiator = Start::new(responder_id.to_string()); @@ -319,11 +314,10 @@ pub fn direct_key_image_store_check(logger: Logger) { let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes).unwrap(); assert_eq!(done_response.results.len(), 1); - let test_results: Vec<(KeyImage, u32)> = done_response + let mut test_results = done_response .results .into_iter() - .map(|result| (result.key_image, result.key_image_result_code)) - .collect(); + .map(|result| (result.key_image, result.key_image_result_code)); // The key image result code for a spent key image is 1. assert!(test_results.contains(&(test_key_image.key_image, 1))); From 038ae67341a536ccc0ffc25f797b73eb21fcf82f Mon Sep 17 00:00:00 2001 From: Emily C Date: Wed, 1 Feb 2023 20:19:12 -0500 Subject: [PATCH 27/33] Remove unused dependencies in fog-ledger-server Co-authored-by: Nick Santana --- fog/ledger/server/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 08af5672e4..5110119576 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -79,8 +79,6 @@ mc-blockchain-test-utils = { path = "../../../blockchain/test-utils" } mc-common = { path = "../../../common", features = ["loggers"] } mc-crypto-keys = { path = "../../../crypto/keys" } mc-crypto-rand = { path = "../../../crypto/rand" } -mc-crypto-x509-test-vectors = { path = "../../../crypto/x509/test-vectors" } -mc-util-keyfile = { path = "../../../util/keyfile" } mc-util-test-helper = { path = "../../../util/test-helper" } mc-util-uri = { path = "../../../util/uri" } From 70a5311b824913dc6c32dd891bc8fde62c743302 Mon Sep 17 00:00:00 2001 From: Emily C Date: Wed, 1 Feb 2023 20:19:49 -0500 Subject: [PATCH 28/33] Apply suggestions - Remove unused deps Co-authored-by: Nick Santana --- fog/ledger/server/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index 5110119576..56d53e5629 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -90,8 +90,5 @@ mc-fog-test-infra = { path = "../../test_infra" } # third-party aes-gcm = "0.10.1" -pem = "1.1" -portpicker = "0.1.1" sha2 = "0.10" tempdir = "0.3" -x509-signature = "0.5" From d104c9a535699e8749d02f1bd1a6d82404493bbf Mon Sep 17 00:00:00 2001 From: Emily C Date: Wed, 1 Feb 2023 20:24:26 -0500 Subject: [PATCH 29/33] Apply suggestions from comments Applying suggestions from @nick-mobilecoin's review Co-authored-by: Nick Santana --- fog/ledger/server/src/bin/key_image_store.rs | 6 +-- .../server/src/key_image_router_service.rs | 13 +++--- .../server/src/key_image_store_server.rs | 2 +- fog/ledger/server/tests/store_tests.rs | 44 ++++++++----------- 4 files changed, 27 insertions(+), 38 deletions(-) diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs index 2929e566cb..34346eb227 100644 --- a/fog/ledger/server/src/bin/key_image_store.rs +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -12,10 +12,10 @@ use mc_watcher::watcher_db::WatcherDB; use std::{env, sync::Arc}; fn main() { - mc_common::setup_panic_handler(); - let config = LedgerStoreConfig::parse(); let (logger, _global_logger_guard) = mc_common::logger::create_app_logger(mc_common::logger::o!()); + mc_common::setup_panic_handler(); + let config = LedgerStoreConfig::parse(); let enclave_path = env::current_exe() .expect("Could not get the path of our executable") @@ -23,7 +23,7 @@ fn main() { log::info!( logger, "enclave path {}, responder ID {}", - enclave_path.to_str().unwrap(), + enclave_path.to_str().expect("enclave path is not valid UTF-8"), &config.client_responder_id ); let enclave = LedgerSgxEnclave::new( diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs index 707674547a..1465e91a11 100644 --- a/fog/ledger/server/src/key_image_router_service.rs +++ b/fog/ledger/server/src/key_image_router_service.rs @@ -1,10 +1,6 @@ // Copyright (c) 2018-2022 The MobileCoin Foundation -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - +use crate::{router_handlers, SVC_COUNTERS}; use futures::{FutureExt, TryFutureExt}; use grpcio::{DuplexSink, RequestStream, RpcContext}; use mc_common::logger::{log, Logger}; @@ -15,9 +11,10 @@ use mc_fog_api::{ use mc_fog_ledger_enclave::LedgerEnclaveProxy; use mc_fog_uri::KeyImageStoreUri; use mc_util_grpc::rpc_logger; -use mc_util_metrics::SVC_COUNTERS; - -use crate::router_handlers; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; #[derive(Clone)] pub struct KeyImageRouterService diff --git a/fog/ledger/server/src/key_image_store_server.rs b/fog/ledger/server/src/key_image_store_server.rs index 1716e438eb..5c8e24c5df 100644 --- a/fog/ledger/server/src/key_image_store_server.rs +++ b/fog/ledger/server/src/key_image_store_server.rs @@ -86,7 +86,7 @@ impl KeyImageStoreServer { client_authenticator.clone(), logger.clone(), ); - Self::new_from_service(key_image_service, client_listen_uri, logger.clone()) + Self::new_from_service(key_image_service, client_listen_uri, logger) } pub fn new_from_service( diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store_tests.rs index 76f45f6537..eec8593590 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store_tests.rs @@ -41,7 +41,7 @@ use sha2::Sha512; use tempdir::TempDir; use url::Url; -pub fn uri_for_test(port: u16) -> KeyImageStoreUri { +fn uri_for_test(port: u16) -> KeyImageStoreUri { // If a load-balancer were set up in the middle here // this might need to be changed to // {KeyImageStoreScheme::SCHEME_INSECURE}://localhost:1234/? @@ -54,7 +54,7 @@ pub fn uri_for_test(port: u16) -> KeyImageStoreUri { KeyImageStoreUri::from_str(&name).unwrap() } -pub struct TestingContext { +pub struct TestingContext { pub enclave: LedgerSgxEnclave, pub ledger: LedgerDB, pub responder_id: ResponderId, @@ -68,7 +68,7 @@ pub struct TestingContext { impl TestingContext { pub fn new( - test_name: &'static str, + test_name: AsRef logger: Logger, port: u16, omap_capacity: u64, @@ -78,15 +78,13 @@ impl TestingContext { let test_dir_name = format!("fog_ledger_test_{}", test_name); let tempdir = TempDir::new(&test_dir_name).expect("Could not produce test_ledger tempdir"); let test_path = PathBuf::from(tempdir.path()); - let user_keys_path = test_path.join(PathBuf::from("keys/")); - if !user_keys_path.exists() { - std::fs::create_dir(&user_keys_path).unwrap(); - } + let user_keys_path = test_path.join("keys"); + std::fs::create_dir_all(&user_keys_path).expect("Failed creatting user keys directory"); let test_uri = uri_for_test(port); // This ID needs to match the host:port clients use in their URI when // referencing the host node. - let responder_id = test_uri.responder_id().unwrap(); + let responder_id = test_uri.responder_id().expect("Test URI is invalid"); let enclave_path = std::env::current_exe() .expect("Could not get the path of our executable") @@ -101,7 +99,7 @@ impl TestingContext { LedgerSgxEnclave::new(enclave_path, &responder_id, omap_capacity, logger.clone()); // Make LedgerDB - let ledger_path = test_path.join(PathBuf::from("fog_ledger")); + let ledger_path = test_path.join("fog_ledger"); let ledger = recreate_ledger_db(ledger_path.as_path()); // Set up wallet db. @@ -154,8 +152,6 @@ pub fn direct_key_image_store_check(logger: Logger) { const PORT_START: u16 = 3223; const OMAP_CAPACITY: u64 = 768; - let port = PORT_START; - let rng = RngType::from_entropy(); let TestingContext { enclave, @@ -191,15 +187,12 @@ pub fn direct_key_image_store_check(logger: Logger) { // This will be a SimClient in testing contexts. let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let mut report_cache_thread = Some( - ReportCacheThread::start( - enclave.clone(), - ias_client, - store_config.ias_spid, - &TEST_ENCLAVE_REPORT_TIMESTAMP, - logger.clone(), - ) - .unwrap(), + let mut report_cache_thread = ReportCacheThread::start( + enclave.clone(), + ias_client, + store_config.ias_spid, + &TEST_ENCLAVE_REPORT_TIMESTAMP, + logger.clone(), ) .unwrap(); @@ -214,7 +207,6 @@ pub fn direct_key_image_store_check(logger: Logger) { enclave .ledger_store_connect(responder_id.clone(), auth_response) .unwrap(); - println!("router_to_store_session is: {:?}", &router_to_store_session); // Generate a dummy key image we're going to check against. let mut test_key_image_bytes: [u8; 32] = [0u8; 32]; @@ -272,7 +264,7 @@ pub fn direct_key_image_store_check(logger: Logger) { .create_multi_key_image_store_query_data(sealed_query.clone()) .unwrap(); - let query: EnclaveMessage = multi_query.pop().unwrap(); + let query = multi_query.pop().expect("Query should have had one message"); println!("Nonce session on message is {:?}", query.channel_id); // Get an untrusted query @@ -314,13 +306,13 @@ pub fn direct_key_image_store_check(logger: Logger) { let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes).unwrap(); assert_eq!(done_response.results.len(), 1); - let mut test_results = done_response + let test_results = done_response .results .into_iter() - .map(|result| (result.key_image, result.key_image_result_code)); + .map(|result| (result.key_image, result.key_image_result_code)) + .collect::>(); // The key image result code for a spent key image is 1. - assert!(test_results.contains(&(test_key_image.key_image, 1))); + assert_eq!(test_results, &[(test_key_image.key_image, 1)]); - report_cache_thread.stop().unwrap(); } From d6b83a429d69bfe6008e1de5f224fc807be7bb01 Mon Sep 17 00:00:00 2001 From: Millie C Date: Wed, 1 Feb 2023 20:45:16 -0500 Subject: [PATCH 30/33] Resolving some code quality issues in direct_key_image_store_check() --- Cargo.lock | 5 -- .../server/tests/{store_tests.rs => store.rs} | 72 ++++++++++--------- 2 files changed, 38 insertions(+), 39 deletions(-) rename fog/ledger/server/tests/{store_tests.rs => store.rs} (83%) diff --git a/Cargo.lock b/Cargo.lock index af7e8c1417..1557964ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3969,7 +3969,6 @@ dependencies = [ "mc-common", "mc-crypto-keys", "mc-crypto-rand", - "mc-crypto-x509-test-vectors", "mc-fog-api", "mc-fog-enclave-connection", "mc-fog-ledger-connection", @@ -3989,7 +3988,6 @@ dependencies = [ "mc-util-encodings", "mc-util-from-random", "mc-util-grpc", - "mc-util-keyfile", "mc-util-metrics", "mc-util-parse", "mc-util-serial", @@ -3998,8 +3996,6 @@ dependencies = [ "mc-util-uri", "mc-watcher", "mc-watcher-api", - "pem", - "portpicker", "rand 0.8.5", "retry", "serde", @@ -4007,7 +4003,6 @@ dependencies = [ "sha2 0.10.6", "tempdir", "url", - "x509-signature", ] [[package]] diff --git a/fog/ledger/server/tests/store_tests.rs b/fog/ledger/server/tests/store.rs similarity index 83% rename from fog/ledger/server/tests/store_tests.rs rename to fog/ledger/server/tests/store.rs index eec8593590..e8d22767fc 100644 --- a/fog/ledger/server/tests/store_tests.rs +++ b/fog/ledger/server/tests/store.rs @@ -5,7 +5,6 @@ use std::{ sync::{Arc, Mutex}, }; -use itertools::Itertools; use mc_attest_ake::{AuthResponseInput, ClientInitiate, Start, Transition}; use mc_attest_api::attest; use mc_attest_enclave_api::{ClientSession, EnclaveMessage, NonceSession}; @@ -51,7 +50,7 @@ fn uri_for_test(port: u16) -> KeyImageStoreUri { KeyImageStoreScheme::SCHEME_INSECURE, port ); - KeyImageStoreUri::from_str(&name).unwrap() + KeyImageStoreUri::from_str(&name).expect("Could not create a URI for a key-image store test using localhost.") } pub struct TestingContext { @@ -68,14 +67,14 @@ pub struct TestingContext { impl TestingContext { pub fn new( - test_name: AsRef + test_name: impl AsRef, logger: Logger, port: u16, omap_capacity: u64, rng: R, ) -> Self { // Set up our directories. - let test_dir_name = format!("fog_ledger_test_{}", test_name); + let test_dir_name = format!("fog_ledger_test_{}", test_name.as_ref()); let tempdir = TempDir::new(&test_dir_name).expect("Could not produce test_ledger tempdir"); let test_path = PathBuf::from(tempdir.path()); let user_keys_path = test_path.join("keys"); @@ -92,7 +91,7 @@ impl TestingContext { // rather than just target/debug/. So, // we need the parent directory. .parent() - .unwrap() + .expect("Failed to get parent of enclave path.") .with_file_name(ENCLAVE_FILE); let enclave = @@ -103,15 +102,16 @@ impl TestingContext { let ledger = recreate_ledger_db(ledger_path.as_path()); // Set up wallet db. - let test_url_name = format!("http://{}.wallet.test.test", test_name); - let url = Url::parse(&test_url_name).unwrap(); + let test_url_name = format!("http://{}.wallet.test.test", test_name.as_ref()); + let url = Url::parse(&test_url_name).expect("Failed to parse test url as a Url struct."); let db_tmp = TempDir::new("wallet_db").expect("Could not make tempdir for wallet db"); - WatcherDB::create(db_tmp.path()).unwrap(); - let watcher = WatcherDB::open_rw(db_tmp.path(), &[url.clone()], logger).unwrap(); + WatcherDB::create(db_tmp.path()).expect("Could not create WatcherDB."); + let watcher = WatcherDB::open_rw(db_tmp.path(), &[url.clone()], logger) + .expect("Failed to open WatcherDB."); let config = LedgerStoreConfig { - chain_id: test_name.to_string(), + chain_id: test_name.as_ref().to_string(), client_responder_id: responder_id.clone(), client_listen_uri: test_uri, ledger_db: ledger_path, @@ -163,7 +163,7 @@ pub fn direct_key_image_store_check(logger: Logger) { watcher, store_config, watcher_path: _watcher_path, - } = TestingContext::new(TEST_NAME, logger.clone(), port, OMAP_CAPACITY, rng); + } = TestingContext::new(TEST_NAME, logger.clone(), PORT_START, OMAP_CAPACITY, rng); let shared_state = Arc::new(Mutex::new(DbPollSharedState::default())); @@ -187,26 +187,26 @@ pub fn direct_key_image_store_check(logger: Logger) { // This will be a SimClient in testing contexts. let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let mut report_cache_thread = ReportCacheThread::start( + let _report_cache_thread = ReportCacheThread::start( enclave.clone(), ias_client, store_config.ias_spid, &TEST_ENCLAVE_REPORT_TIMESTAMP, logger.clone(), - ) - .unwrap(); + ).expect("Failed to start IAS client."); // Make GRPC client for sending requests. // Get the enclave to generate an auth request. - let client_auth_request = enclave.ledger_store_init(responder_id.clone()).unwrap(); + let client_auth_request = enclave.ledger_store_init(responder_id.clone()) + .expect("Could not initialize ledger store on the enclave."); // Submit auth request and wait for the response. - let (auth_response, router_to_store_session) = - enclave.frontend_accept(client_auth_request).unwrap(); + let (auth_response, _router_to_store_session) = + enclave.frontend_accept(client_auth_request).expect("frontend_accept() failed."); // Finish the enclave's handshake with itself. enclave .ledger_store_connect(responder_id.clone(), auth_response) - .unwrap(); + .expect("Failed to complete the connection to a fog ledger store."); // Generate a dummy key image we're going to check against. let mut test_key_image_bytes: [u8; 32] = [0u8; 32]; @@ -216,19 +216,22 @@ pub fn direct_key_image_store_check(logger: Logger) { block_index: 1, timestamp: 255, }; - enclave.add_key_image_data(vec![test_key_image]).unwrap(); + enclave.add_key_image_data(vec![test_key_image]) + .expect("Error adding key image data to the enclave."); // Set up the client's end of the encrypted connection. let initiator = Start::new(responder_id.to_string()); let init_input = ClientInitiate::::default(); - let (initiator, auth_request_output) = initiator.try_next(&mut rng, init_input).unwrap(); + let (initiator, auth_request_output) = initiator.try_next(&mut rng, init_input) + .expect("Could not encrypt auth message."); // Authenticate our "client" with the server. let auth_message = attest::AuthMessage::from(auth_request_output); let (client_auth_response, client_session) = - enclave.client_accept(auth_message.into()).unwrap(); - println!("Initial client_session is {:?}", &client_session); + enclave.client_accept(auth_message.into()) + .expect("Unable to connect a dummy \"client\" connection to the enclave."); + // We will need to double-convert, ClientAuthResponse -> AuthMessage -> // AuthResponseOutput let auth_message = attest::AuthMessage::from(client_auth_response); @@ -236,7 +239,8 @@ pub fn direct_key_image_store_check(logger: Logger) { let auth_response_event = AuthResponseInput::new(auth_message.into(), Verifier::default()); // Should be a valid noise connection at this point. let (mut noise_connection, _verification_report) = - initiator.try_next(&mut rng, auth_response_event).unwrap(); + initiator.try_next(&mut rng, auth_response_event) + .expect("Could not get a noise connection and verification report from the initiator."); //Construct our request. let key_images_request = CheckKeyImagesRequest { @@ -247,7 +251,8 @@ pub fn direct_key_image_store_check(logger: Logger) { }; // Protobuf-encoded plaintext. let message_encoded = mc_util_serial::encode(&key_images_request); - let ciphertext = noise_connection.encrypt(&[], &message_encoded).unwrap(); + let ciphertext = noise_connection.encrypt(&[], &message_encoded) + .expect("Failed to encrypt request from the client to the router."); let msg: EnclaveMessage = EnclaveMessage { aad: vec![], channel_id: client_session, @@ -255,14 +260,12 @@ pub fn direct_key_image_store_check(logger: Logger) { }; // Decrypt and seal - let sealed_query = enclave.decrypt_and_seal_query(msg).unwrap(); - println!( - "Client session on sealed_query is {:?}", - &sealed_query.channel_id - ); + let sealed_query = enclave.decrypt_and_seal_query(msg) + .expect("Unable to decrypt and seal client message."); + let mut multi_query = enclave .create_multi_key_image_store_query_data(sealed_query.clone()) - .unwrap(); + .expect("Could not create multi key image store query data."); let query = multi_query.pop().expect("Query should have had one message"); println!("Nonce session on message is {:?}", query.channel_id); @@ -290,20 +293,21 @@ pub fn direct_key_image_store_check(logger: Logger) { let result = enclave .check_key_image_store(query, untrusted_kiqr) - .unwrap(); + .expect("Checking key image store enclave failed."); let responses_btree: BTreeMap> = BTreeMap::from([(responder_id, result)]); let client_response = enclave .collate_shard_query_responses(sealed_query, responses_btree) - .unwrap(); + .expect("Error in collate_shard_query_responses()."); let plaintext_bytes = noise_connection .decrypt(&client_response.aad, &client_response.data) - .unwrap(); + .expect("Could not decrypt response to client."); - let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes).unwrap(); + let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes) + .expect("Failed to decode CheckKeyImagesResponse."); assert_eq!(done_response.results.len(), 1); let test_results = done_response From 7f8f80ea39b0d489ff312a4852faadecca499878 Mon Sep 17 00:00:00 2001 From: Millie C Date: Sun, 5 Feb 2023 23:46:54 -0500 Subject: [PATCH 31/33] Cargo fmt --- fog/ledger/server/src/bin/key_image_store.rs | 4 +- fog/ledger/server/tests/store.rs | 49 ++++++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs index 34346eb227..4dc30d2c6f 100644 --- a/fog/ledger/server/src/bin/key_image_store.rs +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -23,7 +23,9 @@ fn main() { log::info!( logger, "enclave path {}, responder ID {}", - enclave_path.to_str().expect("enclave path is not valid UTF-8"), + enclave_path + .to_str() + .expect("enclave path is not valid UTF-8"), &config.client_responder_id ); let enclave = LedgerSgxEnclave::new( diff --git a/fog/ledger/server/tests/store.rs b/fog/ledger/server/tests/store.rs index e8d22767fc..30b1abed35 100644 --- a/fog/ledger/server/tests/store.rs +++ b/fog/ledger/server/tests/store.rs @@ -50,7 +50,8 @@ fn uri_for_test(port: u16) -> KeyImageStoreUri { KeyImageStoreScheme::SCHEME_INSECURE, port ); - KeyImageStoreUri::from_str(&name).expect("Could not create a URI for a key-image store test using localhost.") + KeyImageStoreUri::from_str(&name) + .expect("Could not create a URI for a key-image store test using localhost.") } pub struct TestingContext { @@ -187,22 +188,25 @@ pub fn direct_key_image_store_check(logger: Logger) { // This will be a SimClient in testing contexts. let ias_client = AttestClient::new(&store_config.ias_api_key).expect("Could not create IAS client"); - let _report_cache_thread = ReportCacheThread::start( + let _report_cache_thread = ReportCacheThread::start( enclave.clone(), ias_client, store_config.ias_spid, &TEST_ENCLAVE_REPORT_TIMESTAMP, logger.clone(), - ).expect("Failed to start IAS client."); + ) + .expect("Failed to start IAS client."); // Make GRPC client for sending requests. // Get the enclave to generate an auth request. - let client_auth_request = enclave.ledger_store_init(responder_id.clone()) + let client_auth_request = enclave + .ledger_store_init(responder_id.clone()) .expect("Could not initialize ledger store on the enclave."); // Submit auth request and wait for the response. - let (auth_response, _router_to_store_session) = - enclave.frontend_accept(client_auth_request).expect("frontend_accept() failed."); + let (auth_response, _router_to_store_session) = enclave + .frontend_accept(client_auth_request) + .expect("frontend_accept() failed."); // Finish the enclave's handshake with itself. enclave .ledger_store_connect(responder_id.clone(), auth_response) @@ -216,30 +220,32 @@ pub fn direct_key_image_store_check(logger: Logger) { block_index: 1, timestamp: 255, }; - enclave.add_key_image_data(vec![test_key_image]) + enclave + .add_key_image_data(vec![test_key_image]) .expect("Error adding key image data to the enclave."); // Set up the client's end of the encrypted connection. let initiator = Start::new(responder_id.to_string()); let init_input = ClientInitiate::::default(); - let (initiator, auth_request_output) = initiator.try_next(&mut rng, init_input) + let (initiator, auth_request_output) = initiator + .try_next(&mut rng, init_input) .expect("Could not encrypt auth message."); // Authenticate our "client" with the server. let auth_message = attest::AuthMessage::from(auth_request_output); - let (client_auth_response, client_session) = - enclave.client_accept(auth_message.into()) + let (client_auth_response, client_session) = enclave + .client_accept(auth_message.into()) .expect("Unable to connect a dummy \"client\" connection to the enclave."); - + // We will need to double-convert, ClientAuthResponse -> AuthMessage -> // AuthResponseOutput let auth_message = attest::AuthMessage::from(client_auth_response); // Initiator accepts responder's message. let auth_response_event = AuthResponseInput::new(auth_message.into(), Verifier::default()); // Should be a valid noise connection at this point. - let (mut noise_connection, _verification_report) = - initiator.try_next(&mut rng, auth_response_event) + let (mut noise_connection, _verification_report) = initiator + .try_next(&mut rng, auth_response_event) .expect("Could not get a noise connection and verification report from the initiator."); //Construct our request. @@ -251,7 +257,8 @@ pub fn direct_key_image_store_check(logger: Logger) { }; // Protobuf-encoded plaintext. let message_encoded = mc_util_serial::encode(&key_images_request); - let ciphertext = noise_connection.encrypt(&[], &message_encoded) + let ciphertext = noise_connection + .encrypt(&[], &message_encoded) .expect("Failed to encrypt request from the client to the router."); let msg: EnclaveMessage = EnclaveMessage { aad: vec![], @@ -260,14 +267,17 @@ pub fn direct_key_image_store_check(logger: Logger) { }; // Decrypt and seal - let sealed_query = enclave.decrypt_and_seal_query(msg) + let sealed_query = enclave + .decrypt_and_seal_query(msg) .expect("Unable to decrypt and seal client message."); - + let mut multi_query = enclave .create_multi_key_image_store_query_data(sealed_query.clone()) .expect("Could not create multi key image store query data."); - let query = multi_query.pop().expect("Query should have had one message"); + let query = multi_query + .pop() + .expect("Query should have had one message"); println!("Nonce session on message is {:?}", query.channel_id); // Get an untrusted query @@ -306,8 +316,8 @@ pub fn direct_key_image_store_check(logger: Logger) { .decrypt(&client_response.aad, &client_response.data) .expect("Could not decrypt response to client."); - let done_response: CheckKeyImagesResponse = mc_util_serial::decode(&plaintext_bytes) - .expect("Failed to decode CheckKeyImagesResponse."); + let done_response: CheckKeyImagesResponse = + mc_util_serial::decode(&plaintext_bytes).expect("Failed to decode CheckKeyImagesResponse."); assert_eq!(done_response.results.len(), 1); let test_results = done_response @@ -318,5 +328,4 @@ pub fn direct_key_image_store_check(logger: Logger) { // The key image result code for a spent key image is 1. assert_eq!(test_results, &[(test_key_image.key_image, 1)]); - } From 45fa023910c88a0e41c8c4d4dc3ca66b6780f38c Mon Sep 17 00:00:00 2001 From: Millie C Date: Sun, 5 Feb 2023 23:49:42 -0500 Subject: [PATCH 32/33] key image server and key image service moved to router server and router service --- .../server/src/key_image_router_server.rs | 117 ------------------ .../server/src/key_image_router_service.rs | 83 ------------- 2 files changed, 200 deletions(-) delete mode 100644 fog/ledger/server/src/key_image_router_server.rs delete mode 100644 fog/ledger/server/src/key_image_router_service.rs diff --git a/fog/ledger/server/src/key_image_router_server.rs b/fog/ledger/server/src/key_image_router_server.rs deleted file mode 100644 index bce08d6dd8..0000000000 --- a/fog/ledger/server/src/key_image_router_server.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use futures::executor::block_on; -use mc_common::logger::{log, Logger}; -use mc_fog_api::ledger_grpc; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::{ConnectionUri, KeyImageStoreUri}; -use mc_util_grpc::{ConnectionUriGrpcioServer, ReadinessIndicator}; - -use crate::{ - config::LedgerRouterConfig, key_image_router_service::KeyImageRouterService, - router_admin_service::LedgerRouterAdminService, -}; - -pub struct KeyImageRouterServer { - router_server: grpcio::Server, - admin_server: grpcio::Server, - logger: Logger, -} - -impl KeyImageRouterServer { - pub fn new( - config: LedgerRouterConfig, - enclave: E, - shards: Arc>>>, - logger: Logger, - ) -> KeyImageRouterServer - where - E: LedgerEnclaveProxy, - { - let readiness_indicator = ReadinessIndicator::default(); - - let env = Arc::new( - grpcio::EnvBuilder::new() - .name_prefix("key-image-router-server".to_string()) - .build(), - ); - - // Health check service - will be used in both cases - let health_service = - mc_util_grpc::HealthService::new(Some(readiness_indicator.into()), logger.clone()) - .into_service(); - - // Build our router server. - // Init ledger router service. - let ledger_router_service = ledger_grpc::create_ledger_api(KeyImageRouterService::new( - enclave, - shards.clone(), - logger.clone(), - )); - log::debug!(logger, "Constructed Key Image Router GRPC Service"); - - // Init ledger router admin service. - let ledger_router_admin_service = ledger_grpc::create_ledger_router_admin_api( - LedgerRouterAdminService::new(shards, logger.clone()), - ); - log::debug!(logger, "Constructed Key Image Router Admin GRPC Service"); - - // Package service into grpc server - log::info!( - logger, - "Starting Key Image Router server on {}", - config.client_listen_uri.addr(), - ); - - let router_server_builder = grpcio::ServerBuilder::new(env.clone()) - .register_service(ledger_router_service) - .register_service(health_service) - .bind_using_uri(&config.client_listen_uri, logger.clone()); - let admin_server_builder = grpcio::ServerBuilder::new(env) - .register_service(ledger_router_admin_service) - .bind_using_uri(&config.admin_listen_uri, logger.clone()); - - let router_server = router_server_builder.build().unwrap(); - let admin_server = admin_server_builder.build().unwrap(); - - Self { - router_server, - admin_server, - logger, - } - } - - /// Starts the server - pub fn start(&mut self) { - self.router_server.start(); - for (host, port) in self.router_server.bind_addrs() { - log::info!(self.logger, "Router API listening on {}:{}", host, port); - } - self.admin_server.start(); - for (host, port) in self.admin_server.bind_addrs() { - log::info!( - self.logger, - "Router Admin API listening on {}:{}", - host, - port - ); - } - } - - /// Stops the server - pub fn stop(&mut self) { - block_on(self.router_server.shutdown()).expect("Could not stop router grpc server"); - block_on(self.admin_server.shutdown()).expect("Could not stop router admin grpc server"); - } -} - -impl Drop for KeyImageRouterServer { - fn drop(&mut self) { - self.stop(); - } -} diff --git a/fog/ledger/server/src/key_image_router_service.rs b/fog/ledger/server/src/key_image_router_service.rs deleted file mode 100644 index 1465e91a11..0000000000 --- a/fog/ledger/server/src/key_image_router_service.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2018-2022 The MobileCoin Foundation - -use crate::{router_handlers, SVC_COUNTERS}; -use futures::{FutureExt, TryFutureExt}; -use grpcio::{DuplexSink, RequestStream, RpcContext}; -use mc_common::logger::{log, Logger}; -use mc_fog_api::{ - ledger::{LedgerRequest, LedgerResponse}, - ledger_grpc::{self, LedgerApi}, -}; -use mc_fog_ledger_enclave::LedgerEnclaveProxy; -use mc_fog_uri::KeyImageStoreUri; -use mc_util_grpc::rpc_logger; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -#[derive(Clone)] -pub struct KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, -} - -impl KeyImageRouterService { - /// Creates a new LedgerRouterService that can be used by a gRPC server to - /// fulfill gRPC requests. - #[allow(dead_code)] // FIXME - pub fn new( - enclave: E, - shards: Arc>>>, - query_retries: usize, - logger: Logger, - ) -> Self { - Self { - enclave, - shards, - query_retries, - logger, - } - } -} - -impl LedgerApi for KeyImageRouterService -where - E: LedgerEnclaveProxy, -{ - fn request( - &mut self, - ctx: RpcContext, - requests: RequestStream, - responses: DuplexSink, - ) { - let _timer = SVC_COUNTERS.req(&ctx); - mc_common::logger::scoped_global_logger(&rpc_logger(&ctx, &self.logger), |logger| { - log::warn!( - self.logger, - "Streaming GRPC Ledger API only partially implemented." - ); - let logger = logger.clone(); - - let shards = self.shards.read().expect("RwLock poisoned"); - let future = router_handlers::handle_requests( - shards.values().cloned().collect(), - self.enclave.clone(), - requests, - responses, - self.query_retries, - logger.clone(), - ) - .map_err(move |err| log::error!(&logger, "failed to reply: {}", err)) - // TODO: Do more with the error than just push it to the log. - .map(|_| ()); - - ctx.spawn(future) - }); - } -} From 073a05bbddf5e38d7ddbede6d4421efaa050b4cd Mon Sep 17 00:00:00 2001 From: Emily C Date: Tue, 7 Feb 2023 13:26:08 -0500 Subject: [PATCH 33/33] Apply suggestions from code review Co-authored-by: Sam Dealy <33067698+samdealy@users.noreply.github.com> Co-authored-by: Nick Santana --- fog/ledger/server/src/bin/key_image_store.rs | 2 +- fog/ledger/server/tests/store.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fog/ledger/server/src/bin/key_image_store.rs b/fog/ledger/server/src/bin/key_image_store.rs index 4dc30d2c6f..e476b79c41 100644 --- a/fog/ledger/server/src/bin/key_image_store.rs +++ b/fog/ledger/server/src/bin/key_image_store.rs @@ -54,7 +54,7 @@ fn main() { let config2 = config.clone(); let get_config_json = Arc::new(move || { serde_json::to_string(&config2) - .map_err(|err| RpcStatus::with_message(RpcStatusCode::INTERNAL, format!("{:?}", err))) + .map_err(|err| RpcStatus::with_message(RpcStatusCode::INTERNAL, format!("{err:?}"))) }); let _admin_server = config.admin_listen_uri.as_ref().map(|admin_listen_uri| { AdminServer::start( diff --git a/fog/ledger/server/tests/store.rs b/fog/ledger/server/tests/store.rs index 30b1abed35..43398c6524 100644 --- a/fog/ledger/server/tests/store.rs +++ b/fog/ledger/server/tests/store.rs @@ -79,7 +79,7 @@ impl TestingContext { let tempdir = TempDir::new(&test_dir_name).expect("Could not produce test_ledger tempdir"); let test_path = PathBuf::from(tempdir.path()); let user_keys_path = test_path.join("keys"); - std::fs::create_dir_all(&user_keys_path).expect("Failed creatting user keys directory"); + std::fs::create_dir_all(&user_keys_path).expect("Failed creating user keys directory"); let test_uri = uri_for_test(port); // This ID needs to match the host:port clients use in their URI when