From 40fd951d0d5a41253eb08ca2bbb0ab3b76bd4831 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 28 Feb 2023 10:28:10 -0800 Subject: [PATCH 1/8] Router integration test --- Cargo.lock | 6 + fog/ledger/server/Cargo.toml | 1 + fog/ledger/server/tests/router_integration.rs | 572 ++++++++++++++++++ fog/ledger/test_infra/Cargo.toml | 7 + fog/ledger/test_infra/src/lib.rs | 73 +++ 5 files changed, 659 insertions(+) create mode 100644 fog/ledger/server/tests/router_integration.rs diff --git a/Cargo.lock b/Cargo.lock index 38092b5c61..65516add99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3919,6 +3919,7 @@ dependencies = [ "serde_json", "sha2 0.10.6", "tempdir", + "tokio", "url", "x509-signature", ] @@ -3927,6 +3928,8 @@ dependencies = [ name = "mc-fog-ledger-test-infra" version = "4.0.2" dependencies = [ + "http", + "hyper", "mc-attest-core", "mc-attest-enclave-api", "mc-blockchain-types", @@ -3938,6 +3941,8 @@ dependencies = [ "mc-ledger-db", "mc-sgx-report-cache-api", "mc-transaction-core", + "rand 0.8.5", + "tokio", ] [[package]] @@ -8238,6 +8243,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot 0.11.2", "pin-project-lite", "signal-hook-registry", "tokio-macros", diff --git a/fog/ledger/server/Cargo.toml b/fog/ledger/server/Cargo.toml index aa360a381f..cdf8764ec3 100644 --- a/fog/ledger/server/Cargo.toml +++ b/fog/ledger/server/Cargo.toml @@ -98,4 +98,5 @@ pem = "1.1" portpicker = "0.1.1" sha2 = "0.10" tempdir = "0.3" +tokio = { version = "1.0", features = ["full"] } x509-signature = "0.5" diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs new file mode 100644 index 0000000000..fad07c3ec3 --- /dev/null +++ b/fog/ledger/server/tests/router_integration.rs @@ -0,0 +1,572 @@ +use mc_account_keys::{AccountKey, PublicAddress}; +use mc_api::watcher::TimestampResultCode; +use mc_attest_net::{Client as AttestClient, RaClient}; +use mc_attest_verifier::{MrSignerVerifier, Verifier, DEBUG_ENCLAVE}; +use mc_blockchain_types::BlockVersion; +use mc_common::{ + logger, + logger::{log, Logger}, + time::SystemTimeProvider, +}; +use mc_crypto_rand::{CryptoRng, RngCore}; +use mc_fog_ledger_connection::{KeyImageResultExtension, LedgerGrpcClient}; +use mc_fog_ledger_enclave::LedgerSgxEnclave; +use mc_fog_ledger_server::{ + sharding_strategy::EpochShardingStrategy, KeyImageStoreServer, LedgerRouterConfig, + LedgerRouterServer, LedgerStoreConfig, ShardingStrategy, +}; +use mc_fog_ledger_test_infra::ShardProxyServer; +use mc_fog_test_infra::get_enclave_path; +use mc_fog_types::common::BlockRange; +use mc_fog_uri::{FogLedgerUri, KeyImageStoreUri}; +use mc_ledger_db::{test_utils::recreate_ledger_db, LedgerDB}; +use mc_transaction_core::{ring_signature::KeyImage, tokens::Mob, Amount, Token}; +use mc_util_test_helper::{RngType, SeedableRng}; +use mc_util_uri::{AdminUri, ConnectionUri}; +use mc_watcher::watcher_db::WatcherDB; +use rand::thread_rng; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr, SocketAddr}, + path::PathBuf, + str::FromStr, + sync::Arc, +}; +use url::Url; + +const TEST_URL: &str = "http://www.my_url1.com"; +const CHAIN_ID: &str = "local"; + +fn setup_watcher_db(path: PathBuf, logger: Logger) -> WatcherDB { + let url = Url::parse(TEST_URL).unwrap(); + + // create does not open + WatcherDB::create(&path).unwrap(); + WatcherDB::open_rw(&path, &[url], logger).unwrap() +} + +fn create_store_config( + store_uri: &KeyImageStoreUri, + block_range: BlockRange, + omap_capacity: u64, +) -> LedgerStoreConfig { + LedgerStoreConfig { + chain_id: CHAIN_ID.to_string(), + client_responder_id: store_uri + .responder_id() + .expect("Couldn't get responder ID for store"), + client_listen_uri: store_uri.clone(), + ledger_db: Default::default(), + watcher_db: Default::default(), + ias_api_key: Default::default(), + ias_spid: Default::default(), + admin_listen_uri: None, + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + omap_capacity, + sharding_strategy: ShardingStrategy::Epoch(EpochShardingStrategy::new(block_range)), + } +} + +fn temp_dir(name: &str) -> Result { + let path: PathBuf = [std::env::temp_dir(), std::path::PathBuf::from(name)] + .iter() + .collect(); + std::fs::create_dir_all(&path) + .unwrap_or_else(|_| panic!("Couldn't create temporary path {:?}", &path)); + Ok(path) +} + +fn add_block_to_ledger( + ledger_db: &mut LedgerDB, + recipients: &[PublicAddress], + key_images: &[KeyImage], + rng: &mut (impl CryptoRng + RngCore), + watcher: &mut WatcherDB, +) -> u64 { + let amount = Amount::new(10, Mob::ID); + let block_data = mc_ledger_db::test_utils::add_block_to_ledger( + ledger_db, + BlockVersion::MAX, + recipients, + amount, + key_images, + rng, + ) + .expect("failed to add block"); + let block_index = block_data.block().index; + + let signature = block_data.signature().expect("missing signature"); + for src_url in watcher.get_config_urls().unwrap().iter() { + watcher + .add_block_signature( + src_url, + block_index, + signature.clone(), + format!("00/{}", block_index), + ) + .expect("Could not add block signature"); + } + + block_index + 1 +} + +fn populate_ledger(blocks_config: &BlockConfig, ledger: &mut LedgerDB, watcher: &mut WatcherDB) { + let mut rng = thread_rng(); + + let alice = AccountKey::random_with_fog(&mut rng); + let recipients = vec![alice.default_subaddress()]; + // Origin block cannot have key images + add_block_to_ledger(ledger, &recipients, &[], &mut rng, watcher); + + for block in blocks_config { + let recipients: Vec<_> = block.keys().cloned().collect(); + let key_images: Vec<_> = block.values().flat_map(|x| x.clone()).collect(); + + add_block_to_ledger(ledger, &recipients, &key_images, &mut rng, watcher); + } +} + +fn create_store( + test_config: &StoreConfig, + blocks_config: &BlockConfig, + block_range: BlockRange, + _grpc_env: Arc, + logger: Logger, +) -> KeyImageStoreServer { + let uri = KeyImageStoreUri::from_str(&format!( + "insecure-key-image-store://{}", + test_config.address + )) + .unwrap(); + let block_range = test_config + .block_range + .as_ref() + .unwrap_or(&block_range) + .clone(); + let config = create_store_config(&uri, block_range.clone(), test_config.omap_capacity); + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + config.omap_capacity, + logger.clone(), + ); + + let ledger_db_tmp = + temp_dir(&test_config.address.to_string()).expect("Could not get test_ledger tempdir"); + let ledger_db_path = ledger_db_tmp.as_path(); + let mut ledger = recreate_ledger_db(ledger_db_path); + + let watcher_db_tmp = + temp_dir(&test_config.address.to_string()).expect("Could not make tempdir for watcher db"); + let watcher_db_path = watcher_db_tmp.as_path(); + let mut watcher = setup_watcher_db(watcher_db_path.to_path_buf(), logger.clone()); + + populate_ledger(blocks_config, &mut ledger, &mut watcher); + + let ra_client = AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + let mut store = KeyImageStoreServer::new_from_config( + config, + enclave, + ra_client, + ledger, + watcher, + EpochShardingStrategy::new(block_range), + SystemTimeProvider::default(), + logger.clone(), + ); + store.start(); + + store +} + +fn create_shard( + config: &ShardConfig, + _grpc_env: Arc, + _logger: Logger, +) -> ShardProxyServer { + ShardProxyServer::new( + &config.address, + config + .stores + .iter() + .map(|x| x.address.to_string()) + .collect(), + ) +} + +fn create_router( + test_config: &TestEnvironmentConfig, + blocks_config: &BlockConfig, + _grpc_env: Arc, + logger: Logger, +) -> LedgerRouterServer { + let uri = FogLedgerUri::from_str(&format!( + "insecure-fog-ledger://{}", + test_config.router_address + )) + .unwrap(); + let admin_uri = AdminUri::from_str(&format!( + "insecure-mca://{}", + test_config.router_admin_address + )) + .unwrap(); + + let ledger_db_tmp = temp_dir(&test_config.router_address.to_string()) + .expect("Could not get test_ledger tempdir"); + let ledger_db_path = ledger_db_tmp.as_path(); + let mut ledger = recreate_ledger_db(ledger_db_path); + + let watcher_db_tmp = temp_dir(&test_config.router_address.to_string()) + .expect("Could not make tempdir for watcher db"); + let watcher_db_path = watcher_db_tmp.as_path(); + let mut watcher = setup_watcher_db(watcher_db_path.to_path_buf(), logger.clone()); + + populate_ledger(blocks_config, &mut ledger, &mut watcher); + + let config = LedgerRouterConfig { + chain_id: "local".to_string(), + ledger_db: ledger_db_path.to_path_buf(), + watcher_db: watcher_db_path.to_path_buf(), + shard_uris: test_config + .shards + .iter() + .map(|x| { + KeyImageStoreUri::from_str(&format!("insecure-key-image-store://{}", x.address)) + .unwrap() + }) + .collect(), + client_responder_id: uri + .responder_id() + .expect("Couldn't get responder ID for router"), + client_listen_uri: uri, + admin_listen_uri: admin_uri, + ias_spid: Default::default(), + ias_api_key: Default::default(), + client_auth_token_secret: None, + client_auth_token_max_lifetime: Default::default(), + query_retries: 3, + omap_capacity: test_config.omap_capacity, + }; + + let enclave = LedgerSgxEnclave::new( + get_enclave_path(mc_fog_ledger_enclave::ENCLAVE_FILE), + &config.client_responder_id, + config.omap_capacity, + logger.clone(), + ); + + let ra_client = AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); + + let mut router = + LedgerRouterServer::new_from_config(config, enclave, ra_client, ledger, watcher, logger); + router.start(); + router +} + +fn create_router_client( + config: &TestEnvironmentConfig, + grpc_env: Arc, + logger: Logger, +) -> LedgerGrpcClient { + let uri = FogLedgerUri::from_str(&format!("insecure-fog-ledger://{}", config.router_address)) + .unwrap(); + + let mut mr_signer_verifier = + MrSignerVerifier::from(mc_fog_ledger_enclave_measurement::sigstruct()); + mr_signer_verifier + .allow_hardening_advisories(mc_fog_ledger_enclave_measurement::HARDENING_ADVISORIES); + let mut verifier = Verifier::default(); + verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); + + LedgerGrpcClient::new(uri, verifier, grpc_env, logger.clone()) +} + +fn create_env( + config: TestEnvironmentConfig, + blocks_config: BlockConfig, + grpc_env: Arc, + logger: Logger, +) -> TestEnvironment { + let mut shards = vec![]; + let mut stores = vec![]; + for shard in config.shards.iter() { + for store in shard.stores.iter() { + stores.push(create_store( + store, + &blocks_config, + shard.block_range.clone(), + grpc_env.clone(), + logger.clone(), + )); + } + + shards.push(create_shard(shard, grpc_env.clone(), logger.clone())); + } + + let router = create_router(&config, &blocks_config, grpc_env.clone(), logger.clone()); + let router_client = create_router_client(&config, grpc_env, logger.clone()); + + TestEnvironment { + stores, + shards, + router, + router_client, + } +} + +struct TestEnvironment { + stores: Vec>, + shards: Vec, + router: LedgerRouterServer, + router_client: LedgerGrpcClient, +} + +#[derive(Serialize, Deserialize)] +struct TestEnvironmentConfig { + router_address: SocketAddr, + router_admin_address: SocketAddr, + shards: Vec, + omap_capacity: u64, +} + +#[derive(Serialize, Deserialize)] +struct ShardConfig { + address: SocketAddr, + block_range: BlockRange, + stores: Vec, +} + +#[derive(Serialize, Deserialize)] +struct StoreConfig { + address: SocketAddr, + block_range: Option, + omap_capacity: u64, +} + +type BlockConfig = Vec>>; + +fn free_sockaddr() -> SocketAddr { + let port = portpicker::pick_unused_port().unwrap(); + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port) +} + +#[tokio::test] +async fn smoke_test() { + let logger = logger::create_test_logger("smoke_test".to_string()); + log::info!(logger, "test"); + // Three shards, three stores each, correct config, each stores three blocks, + // each has three users with three keys each + let mut rng = RngType::from_seed([0u8; 32]); + let mut shards_config = vec![]; + for i in 0..3 { + let mut stores_config = vec![]; + for _ in 0..3 { + let store = StoreConfig { + address: free_sockaddr(), + block_range: None, + omap_capacity: 1000, + }; + stores_config.push(store); + } + let shard = ShardConfig { + address: free_sockaddr(), + block_range: BlockRange::new((i * 3) + 1, ((i + 1) * 3) + 1), + stores: stores_config, + }; + shards_config.push(shard); + } + let config = TestEnvironmentConfig { + router_address: free_sockaddr(), + router_admin_address: free_sockaddr(), + shards: shards_config, + omap_capacity: 1000, + }; + + let mut blocks_config = vec![]; + let mut key_index = 0; + for _ in 1..10 { + let mut block = HashMap::new(); + for _ in 0..3 { + let account = AccountKey::random_with_fog(&mut rng); + let mut keys = vec![]; + for _ in 0..3 { + keys.push(KeyImage::from(key_index as u64)); + key_index += 1; + } + block.insert(account.default_subaddress(), keys); + } + blocks_config.push(block); + } + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let mut test_environment = create_env(config, blocks_config, grpc_env, logger.clone()); + + // Check that we can get all the key images from each shard + for i in 0..key_index { + let key = KeyImage::from(i as u64); + let response = test_environment + .router_client + .check_key_images(&[key]) + .await + .expect("check_key_images failed"); + assert_eq!(response.results[0].key_image, key); + assert_eq!(response.results[0].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + } + + // Grab them all at once + let keys: Vec<_> = (0..key_index).map(|i| KeyImage::from(i as u64)).collect(); + let response = test_environment + .router_client + .check_key_images(&keys) + .await + .expect("check_key_images failed"); + for i in 0..key_index { + let key = KeyImage::from(i as u64); + assert_eq!(response.results[i as usize].key_image, key); + assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[i as usize].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + } + + // Check that an unspent key image is unspent + let key = KeyImage::from(126u64); + let response = test_environment + .router_client + .check_key_images(&[key]) + .await + .expect("check_key_images failed"); + assert_eq!(response.results[0].key_image, key); + assert_eq!(response.results[0].status(), Ok(None)); // Not spent + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + + drop(test_environment.router_client); + test_environment.router.stop(); + for shard in test_environment.shards { + let _ = shard.stop().await; + } + for mut store in test_environment.stores { + store.stop(); + } +} + +#[tokio::test] +async fn overlapping_stores() { + let logger = logger::create_test_logger("overlapping_stores".to_string()); + log::info!(logger, "test"); + // Three shards, three stores each, correct config, each stores three blocks, + // each has three users with three keys each - but the blocks overlap (so + // total of 5 blocks) + let mut rng = RngType::from_seed([0u8; 32]); + let mut shards_config = vec![]; + for i in 0..3 { + let mut stores_config = vec![]; + for _ in 0..3 { + let store = StoreConfig { + address: free_sockaddr(), + block_range: None, + omap_capacity: 1000, + }; + stores_config.push(store); + } + let shard = ShardConfig { + address: free_sockaddr(), + block_range: BlockRange::new(i + 1, i + 4), + stores: stores_config, + }; + shards_config.push(shard); + } + let config = TestEnvironmentConfig { + router_address: free_sockaddr(), + router_admin_address: free_sockaddr(), + shards: shards_config, + omap_capacity: 1000, + }; + + let mut blocks_config = vec![]; + let mut key_index = 0; + for _ in 1..5 { + let mut block = HashMap::new(); + for _ in 0..3 { + let account = AccountKey::random_with_fog(&mut rng); + let mut keys = vec![]; + for _ in 0..3 { + keys.push(KeyImage::from(key_index as u64)); + key_index += 1; + } + block.insert(account.default_subaddress(), keys); + } + blocks_config.push(block); + } + + let grpc_env = Arc::new(grpcio::EnvBuilder::new().build()); + + let mut test_environment = create_env(config, blocks_config, grpc_env, logger.clone()); + + // Check that we can get all the key images from each shard + for i in 0..key_index { + let key = KeyImage::from(i as u64); + let response = test_environment + .router_client + .check_key_images(&[key]) + .await + .expect("check_key_images failed"); + assert_eq!(response.results[0].key_image, key); + assert_eq!(response.results[0].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + } + + // Grab them all at once + let keys: Vec<_> = (0..key_index).map(|i| KeyImage::from(i as u64)).collect(); + let response = test_environment + .router_client + .check_key_images(&keys) + .await + .expect("check_key_images failed"); + for i in 0..key_index { + let key = KeyImage::from(i as u64); + assert_eq!(response.results[i as usize].key_image, key); + assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[i as usize].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + } + + // Check that an unspent key image is unspent + let key = KeyImage::from(126u64); + let response = test_environment + .router_client + .check_key_images(&[key]) + .await + .expect("check_key_images failed"); + assert_eq!(response.results[0].key_image, key); + assert_eq!(response.results[0].status(), Ok(None)); // Not spent + assert_eq!( + response.results[0].timestamp_result_code, + TimestampResultCode::TimestampFound as u32 + ); + + drop(test_environment.router_client); + test_environment.router.stop(); + for shard in test_environment.shards { + let _ = shard.stop().await; + } + for mut store in test_environment.stores { + store.stop(); + } +} diff --git a/fog/ledger/test_infra/Cargo.toml b/fog/ledger/test_infra/Cargo.toml index c38839b78f..0f967a676a 100644 --- a/fog/ledger/test_infra/Cargo.toml +++ b/fog/ledger/test_infra/Cargo.toml @@ -22,3 +22,10 @@ mc-transaction-core = { path = "../../../transaction/core" } mc-fog-ledger-enclave = { path = "../enclave" } mc-fog-ledger-enclave-api = { path = "../enclave/api" } mc-fog-types = { path = "../../types" } + +# third party +http = "0.2" +hyper = { version = "0.14", features = ["full"] } +rand = "0.8" +tokio = { version = "1", features = ["full"] } + diff --git a/fog/ledger/test_infra/src/lib.rs b/fog/ledger/test_infra/src/lib.rs index 155ac24801..4ea8be09db 100644 --- a/fog/ledger/test_infra/src/lib.rs +++ b/fog/ledger/test_infra/src/lib.rs @@ -2,6 +2,12 @@ //! Functionality for mocking and testing components in the ledger server +use http::Uri; +use hyper::{ + client::HttpConnector, + service::{make_service_fn, service_fn}, + Body, Client, Request, Response, Server, +}; use mc_attest_core::{IasNonce, Quote, QuoteNonce, Report, TargetInfo, VerificationReport}; use mc_attest_enclave_api::{ ClientAuthRequest, ClientAuthResponse, ClientSession, EnclaveMessage, NonceAuthRequest, @@ -24,6 +30,9 @@ use mc_transaction_core::{ tx::{TxOut, TxOutMembershipElement, TxOutMembershipProof}, TokenId, }; +use rand::seq::SliceRandom; +use std::{net::SocketAddr, sync::Arc}; +use tokio::{sync::oneshot, task::JoinHandle}; #[derive(Default, Clone)] pub struct MockEnclave {} @@ -286,3 +295,67 @@ impl Ledger for MockLedger { unimplemented!() } } + +pub struct ShardProxyServer { + server_handle: JoinHandle>, + stop_channel: oneshot::Sender<()>, +} + +impl ShardProxyServer { + async fn route( + request: Request, + client: Arc>, + endpoints: Arc>, + ) -> Result, hyper::Error> { + let endpoint = { + let mut rng = rand::thread_rng(); + endpoints.choose(&mut rng).unwrap() + }; + let (mut parts, body) = request.into_parts(); + + let mut uri_parts = parts.uri.clone().into_parts(); + uri_parts.authority = Some(endpoint.parse().unwrap()); + uri_parts.scheme = Some("http".parse().unwrap()); + parts.uri = Uri::from_parts(uri_parts).unwrap(); + + let request = Request::from_parts(parts, body); + let response = client.request(request).await; + response + } + + async fn shutdown(channel: oneshot::Receiver<()>) { + channel.await.unwrap_or(()); + } + + pub fn new(address: &SocketAddr, endpoints: Vec) -> Self { + let client = Arc::new(Client::builder().http2_only(true).build_http()); + let endpoints = Arc::new(endpoints); + let (tx, rx) = oneshot::channel::<()>(); + + let make_service = make_service_fn(move |_| { + let client = client.clone(); + let endpoints = endpoints.clone(); + + async move { + Ok::<_, hyper::Error>(service_fn(move |req| { + Self::route(req, client.clone(), endpoints.clone()) + })) + } + }); + + let server = Server::bind(address).serve(make_service); + let graceful = server.with_graceful_shutdown(Self::shutdown(rx)); + + let server_handle = tokio::spawn(async move { graceful.await }); + + Self { + server_handle, + stop_channel: tx, + } + } + + pub async fn stop(self) -> Result<(), hyper::Error> { + self.stop_channel.send(()).unwrap(); + self.server_handle.await.unwrap_or(Ok(())) + } +} From 2e9e0205a64ff4ab7f8e1d96b9488b504d8dbbdf Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Wed, 1 Mar 2023 21:46:24 -0800 Subject: [PATCH 2/8] Clippy updates after rebase --- fog/ledger/server/tests/router_integration.rs | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index fad07c3ec3..5e390bc86f 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -104,7 +104,7 @@ fn add_block_to_ledger( src_url, block_index, signature.clone(), - format!("00/{}", block_index), + format!("00/{block_index}"), ) .expect("Could not add block signature"); } @@ -174,7 +174,7 @@ fn create_store( watcher, EpochShardingStrategy::new(block_range), SystemTimeProvider::default(), - logger.clone(), + logger, ); store.start(); @@ -259,8 +259,7 @@ fn create_router( let ra_client = AttestClient::new(&config.ias_api_key).expect("Could not create IAS client"); - let mut router = - LedgerRouterServer::new_from_config(config, enclave, ra_client, ledger, watcher, logger); + let mut router = LedgerRouterServer::new(config, enclave, ra_client, ledger, watcher, logger); router.start(); router } @@ -280,7 +279,7 @@ fn create_router_client( let mut verifier = Verifier::default(); verifier.mr_signer(mr_signer_verifier).debug(DEBUG_ENCLAVE); - LedgerGrpcClient::new(uri, verifier, grpc_env, logger.clone()) + LedgerGrpcClient::new(uri, verifier, grpc_env, logger) } fn create_env( @@ -306,7 +305,7 @@ fn create_env( } let router = create_router(&config, &blocks_config, grpc_env.clone(), logger.clone()); - let router_client = create_router_client(&config, grpc_env, logger.clone()); + let router_client = create_router_client(&config, grpc_env, logger); TestEnvironment { stores, @@ -392,7 +391,7 @@ async fn smoke_test() { let account = AccountKey::random_with_fog(&mut rng); let mut keys = vec![]; for _ in 0..3 { - keys.push(KeyImage::from(key_index as u64)); + keys.push(KeyImage::from(key_index)); key_index += 1; } block.insert(account.default_subaddress(), keys); @@ -406,7 +405,7 @@ async fn smoke_test() { // Check that we can get all the key images from each shard for i in 0..key_index { - let key = KeyImage::from(i as u64); + let key = KeyImage::from(i); let response = test_environment .router_client .check_key_images(&[key]) @@ -421,14 +420,14 @@ async fn smoke_test() { } // Grab them all at once - let keys: Vec<_> = (0..key_index).map(|i| KeyImage::from(i as u64)).collect(); + let keys: Vec<_> = (0..key_index).map(KeyImage::from).collect(); let response = test_environment .router_client .check_key_images(&keys) .await .expect("check_key_images failed"); for i in 0..key_index { - let key = KeyImage::from(i as u64); + let key = KeyImage::from(i); assert_eq!(response.results[i as usize].key_image, key); assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); assert_eq!( @@ -502,7 +501,7 @@ async fn overlapping_stores() { let account = AccountKey::random_with_fog(&mut rng); let mut keys = vec![]; for _ in 0..3 { - keys.push(KeyImage::from(key_index as u64)); + keys.push(KeyImage::from(key_index)); key_index += 1; } block.insert(account.default_subaddress(), keys); @@ -516,7 +515,7 @@ async fn overlapping_stores() { // Check that we can get all the key images from each shard for i in 0..key_index { - let key = KeyImage::from(i as u64); + let key = KeyImage::from(i); let response = test_environment .router_client .check_key_images(&[key]) @@ -531,14 +530,14 @@ async fn overlapping_stores() { } // Grab them all at once - let keys: Vec<_> = (0..key_index).map(|i| KeyImage::from(i as u64)).collect(); + let keys: Vec<_> = (0..key_index).map(KeyImage::from).collect(); let response = test_environment .router_client .check_key_images(&keys) .await .expect("check_key_images failed"); for i in 0..key_index { - let key = KeyImage::from(i as u64); + let key = KeyImage::from(i); assert_eq!(response.results[i as usize].key_image, key); assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); assert_eq!( From 830ef6c1307f625e05427ae989d3156286d8a743 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 14 Mar 2023 01:57:46 -0700 Subject: [PATCH 3/8] Implement Drop for TestEnvironment --- fog/ledger/server/tests/router_integration.rs | 45 +++++++++---------- fog/ledger/test_infra/src/lib.rs | 18 +++++--- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index 5e390bc86f..d39055d1c9 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -310,16 +310,31 @@ fn create_env( TestEnvironment { stores, shards, - router, + _router: router, router_client, } } struct TestEnvironment { - stores: Vec>, - shards: Vec, - router: LedgerRouterServer, router_client: LedgerGrpcClient, + _router: LedgerRouterServer, + shards: Vec, + stores: Vec>, +} + +impl Drop for TestEnvironment { + fn drop(&mut self) { + for shard in &mut self.shards { + tokio::task::block_in_place(move || { + tokio::runtime::Handle::current().block_on(async move { + shard.stop().await; + }) + }); + } + for store in &mut self.stores { + store.stop(); + } + } } #[derive(Serialize, Deserialize)] @@ -351,7 +366,7 @@ fn free_sockaddr() -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn smoke_test() { let logger = logger::create_test_logger("smoke_test".to_string()); log::info!(logger, "test"); @@ -449,18 +464,9 @@ async fn smoke_test() { response.results[0].timestamp_result_code, TimestampResultCode::TimestampFound as u32 ); - - drop(test_environment.router_client); - test_environment.router.stop(); - for shard in test_environment.shards { - let _ = shard.stop().await; - } - for mut store in test_environment.stores { - store.stop(); - } } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn overlapping_stores() { let logger = logger::create_test_logger("overlapping_stores".to_string()); log::info!(logger, "test"); @@ -559,13 +565,4 @@ async fn overlapping_stores() { response.results[0].timestamp_result_code, TimestampResultCode::TimestampFound as u32 ); - - drop(test_environment.router_client); - test_environment.router.stop(); - for shard in test_environment.shards { - let _ = shard.stop().await; - } - for mut store in test_environment.stores { - store.stop(); - } } diff --git a/fog/ledger/test_infra/src/lib.rs b/fog/ledger/test_infra/src/lib.rs index 4ea8be09db..c4e5d59fde 100644 --- a/fog/ledger/test_infra/src/lib.rs +++ b/fog/ledger/test_infra/src/lib.rs @@ -297,8 +297,8 @@ impl Ledger for MockLedger { } pub struct ShardProxyServer { - server_handle: JoinHandle>, - stop_channel: oneshot::Sender<()>, + server_handle: Option>>, + stop_channel: Option>, } impl ShardProxyServer { @@ -349,13 +349,17 @@ impl ShardProxyServer { let server_handle = tokio::spawn(async move { graceful.await }); Self { - server_handle, - stop_channel: tx, + server_handle: Some(server_handle), + stop_channel: Some(tx), } } - pub async fn stop(self) -> Result<(), hyper::Error> { - self.stop_channel.send(()).unwrap(); - self.server_handle.await.unwrap_or(Ok(())) + pub async fn stop(&mut self) { + if let Some(stop_channel) = self.stop_channel.take() { + let _ = stop_channel.send(()); + } + if let Some(server_handle) = self.server_handle.take() { + let _ = server_handle.await; + } } } From 755415d7935f580054899df33fdbb4b6ac6bd7db Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 14 Mar 2023 02:18:23 -0700 Subject: [PATCH 4/8] Eliminate temp dirs after test --- fog/ledger/server/tests/router_integration.rs | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index d39055d1c9..fc19a602ca 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -29,10 +29,11 @@ use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, net::{IpAddr, Ipv4Addr, SocketAddr}, - path::PathBuf, + path::{Path, PathBuf}, str::FromStr, sync::Arc, }; +use tempdir::TempDir; use url::Url; const TEST_URL: &str = "http://www.my_url1.com"; @@ -69,15 +70,6 @@ fn create_store_config( } } -fn temp_dir(name: &str) -> Result { - let path: PathBuf = [std::env::temp_dir(), std::path::PathBuf::from(name)] - .iter() - .collect(); - std::fs::create_dir_all(&path) - .unwrap_or_else(|_| panic!("Couldn't create temporary path {:?}", &path)); - Ok(path) -} - fn add_block_to_ledger( ledger_db: &mut LedgerDB, recipients: &[PublicAddress], @@ -132,6 +124,8 @@ fn create_store( test_config: &StoreConfig, blocks_config: &BlockConfig, block_range: BlockRange, + watcher_db_path: &Path, + ledger_db_path: &Path, _grpc_env: Arc, logger: Logger, ) -> KeyImageStoreServer { @@ -153,14 +147,7 @@ fn create_store( logger.clone(), ); - let ledger_db_tmp = - temp_dir(&test_config.address.to_string()).expect("Could not get test_ledger tempdir"); - let ledger_db_path = ledger_db_tmp.as_path(); let mut ledger = recreate_ledger_db(ledger_db_path); - - let watcher_db_tmp = - temp_dir(&test_config.address.to_string()).expect("Could not make tempdir for watcher db"); - let watcher_db_path = watcher_db_tmp.as_path(); let mut watcher = setup_watcher_db(watcher_db_path.to_path_buf(), logger.clone()); populate_ledger(blocks_config, &mut ledger, &mut watcher); @@ -199,6 +186,8 @@ fn create_shard( fn create_router( test_config: &TestEnvironmentConfig, blocks_config: &BlockConfig, + watcher_db_path: &Path, + ledger_db_path: &Path, _grpc_env: Arc, logger: Logger, ) -> LedgerRouterServer { @@ -213,14 +202,7 @@ fn create_router( )) .unwrap(); - let ledger_db_tmp = temp_dir(&test_config.router_address.to_string()) - .expect("Could not get test_ledger tempdir"); - let ledger_db_path = ledger_db_tmp.as_path(); let mut ledger = recreate_ledger_db(ledger_db_path); - - let watcher_db_tmp = temp_dir(&test_config.router_address.to_string()) - .expect("Could not make tempdir for watcher db"); - let watcher_db_path = watcher_db_tmp.as_path(); let mut watcher = setup_watcher_db(watcher_db_path.to_path_buf(), logger.clone()); populate_ledger(blocks_config, &mut ledger, &mut watcher); @@ -290,21 +272,44 @@ fn create_env( ) -> TestEnvironment { let mut shards = vec![]; let mut stores = vec![]; + let mut tempdirs = vec![]; for shard in config.shards.iter() { for store in shard.stores.iter() { + let watcher_db_dir = + TempDir::new("watcher_db").expect("Couldn't create temporary path for watcher DB"); + let ledger_db_dir = + TempDir::new("ledger_db").expect("Couldn't create temporary path for ledger DB"); stores.push(create_store( store, &blocks_config, shard.block_range.clone(), + watcher_db_dir.path(), + ledger_db_dir.path(), grpc_env.clone(), logger.clone(), )); + tempdirs.push(watcher_db_dir); + tempdirs.push(ledger_db_dir); } shards.push(create_shard(shard, grpc_env.clone(), logger.clone())); } - let router = create_router(&config, &blocks_config, grpc_env.clone(), logger.clone()); + let watcher_db_dir = + TempDir::new("watcher_db").expect("Couldn't create temporary path for watcher DB"); + let ledger_db_dir = + TempDir::new("ledger_db").expect("Couldn't create temporary path for ledger DB"); + let router = create_router( + &config, + &blocks_config, + watcher_db_dir.path(), + ledger_db_dir.path(), + grpc_env.clone(), + logger.clone(), + ); + tempdirs.push(watcher_db_dir); + tempdirs.push(ledger_db_dir); + let router_client = create_router_client(&config, grpc_env, logger); TestEnvironment { @@ -312,6 +317,7 @@ fn create_env( shards, _router: router, router_client, + _tempdirs: tempdirs, } } @@ -320,6 +326,7 @@ struct TestEnvironment { _router: LedgerRouterServer, shards: Vec, stores: Vec>, + _tempdirs: Vec, } impl Drop for TestEnvironment { From 048042e540981e5ba8c00a0e2d71d066ab1932b8 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Tue, 14 Mar 2023 02:19:41 -0700 Subject: [PATCH 5/8] Remove unused GRPC arguments --- fog/ledger/server/tests/router_integration.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index fc19a602ca..802352adc2 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -126,7 +126,6 @@ fn create_store( block_range: BlockRange, watcher_db_path: &Path, ledger_db_path: &Path, - _grpc_env: Arc, logger: Logger, ) -> KeyImageStoreServer { let uri = KeyImageStoreUri::from_str(&format!( @@ -168,11 +167,7 @@ fn create_store( store } -fn create_shard( - config: &ShardConfig, - _grpc_env: Arc, - _logger: Logger, -) -> ShardProxyServer { +fn create_shard(config: &ShardConfig, _logger: Logger) -> ShardProxyServer { ShardProxyServer::new( &config.address, config @@ -188,7 +183,6 @@ fn create_router( blocks_config: &BlockConfig, watcher_db_path: &Path, ledger_db_path: &Path, - _grpc_env: Arc, logger: Logger, ) -> LedgerRouterServer { let uri = FogLedgerUri::from_str(&format!( @@ -285,14 +279,13 @@ fn create_env( shard.block_range.clone(), watcher_db_dir.path(), ledger_db_dir.path(), - grpc_env.clone(), logger.clone(), )); tempdirs.push(watcher_db_dir); tempdirs.push(ledger_db_dir); } - shards.push(create_shard(shard, grpc_env.clone(), logger.clone())); + shards.push(create_shard(shard, logger.clone())); } let watcher_db_dir = @@ -304,7 +297,6 @@ fn create_env( &blocks_config, watcher_db_dir.path(), ledger_db_dir.path(), - grpc_env.clone(), logger.clone(), ); tempdirs.push(watcher_db_dir); From a677b59111c37bd24543d29b34e2424d6a9d80e3 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 16 Mar 2023 01:29:58 -0700 Subject: [PATCH 6/8] Load environment config from JSON file. Might delete later. --- fog/ledger/server/tests/data/normal.json | 79 +++++++++++++++++++ fog/ledger/server/tests/data/overlap.json | 79 +++++++++++++++++++ fog/ledger/server/tests/router_integration.rs | 65 +++------------ 3 files changed, 167 insertions(+), 56 deletions(-) create mode 100644 fog/ledger/server/tests/data/normal.json create mode 100644 fog/ledger/server/tests/data/overlap.json diff --git a/fog/ledger/server/tests/data/normal.json b/fog/ledger/server/tests/data/normal.json new file mode 100644 index 0000000000..3c74644b66 --- /dev/null +++ b/fog/ledger/server/tests/data/normal.json @@ -0,0 +1,79 @@ +{ + "router_address": "127.0.0.1:21095", + "router_admin_address": "127.0.0.1:16761", + "shards": [ + { + "address": "127.0.0.1:24242", + "block_range": { + "start_block": 1, + "end_block": 4 + }, + "stores": [ + { + "address": "127.0.0.1:20488", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:21157", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:18213", + "block_range": null, + "omap_capacity": 1000 + } + ] + }, + { + "address": "127.0.0.1:19430", + "block_range": { + "start_block": 4, + "end_block": 7 + }, + "stores": [ + { + "address": "127.0.0.1:16542", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:21010", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:22270", + "block_range": null, + "omap_capacity": 1000 + } + ] + }, + { + "address": "127.0.0.1:23515", + "block_range": { + "start_block": 7, + "end_block": 10 + }, + "stores": [ + { + "address": "127.0.0.1:17984", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:22947", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:15143", + "block_range": null, + "omap_capacity": 1000 + } + ] + } + ], + "omap_capacity": 1000 +} \ No newline at end of file diff --git a/fog/ledger/server/tests/data/overlap.json b/fog/ledger/server/tests/data/overlap.json new file mode 100644 index 0000000000..53da31d257 --- /dev/null +++ b/fog/ledger/server/tests/data/overlap.json @@ -0,0 +1,79 @@ +{ + "router_address": "127.0.0.1:19336", + "router_admin_address": "127.0.0.1:21616", + "shards": [ + { + "address": "127.0.0.1:22977", + "block_range": { + "start_block": 1, + "end_block": 4 + }, + "stores": [ + { + "address": "127.0.0.1:21902", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:19791", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:17895", + "block_range": null, + "omap_capacity": 1000 + } + ] + }, + { + "address": "127.0.0.1:21165", + "block_range": { + "start_block": 2, + "end_block": 5 + }, + "stores": [ + { + "address": "127.0.0.1:20836", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:20965", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:18061", + "block_range": null, + "omap_capacity": 1000 + } + ] + }, + { + "address": "127.0.0.1:17297", + "block_range": { + "start_block": 3, + "end_block": 6 + }, + "stores": [ + { + "address": "127.0.0.1:22423", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:22547", + "block_range": null, + "omap_capacity": 1000 + }, + { + "address": "127.0.0.1:23316", + "block_range": null, + "omap_capacity": 1000 + } + ] + } + ], + "omap_capacity": 1000 +} \ No newline at end of file diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index 802352adc2..3f2c3d6562 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -28,7 +28,7 @@ use rand::thread_rng; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::SocketAddr, path::{Path, PathBuf}, str::FromStr, sync::Arc, @@ -360,43 +360,17 @@ struct StoreConfig { type BlockConfig = Vec>>; -fn free_sockaddr() -> SocketAddr { - let port = portpicker::pick_unused_port().unwrap(); - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port) -} - #[tokio::test(flavor = "multi_thread")] async fn smoke_test() { let logger = logger::create_test_logger("smoke_test".to_string()); log::info!(logger, "test"); // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each - let mut rng = RngType::from_seed([0u8; 32]); - let mut shards_config = vec![]; - for i in 0..3 { - let mut stores_config = vec![]; - for _ in 0..3 { - let store = StoreConfig { - address: free_sockaddr(), - block_range: None, - omap_capacity: 1000, - }; - stores_config.push(store); - } - let shard = ShardConfig { - address: free_sockaddr(), - block_range: BlockRange::new((i * 3) + 1, ((i + 1) * 3) + 1), - stores: stores_config, - }; - shards_config.push(shard); - } - let config = TestEnvironmentConfig { - router_address: free_sockaddr(), - router_admin_address: free_sockaddr(), - shards: shards_config, - omap_capacity: 1000, - }; + let env_config_str = include_str!("data/normal.json"); + let config: TestEnvironmentConfig = + serde_json::from_str(env_config_str).expect("Couldn't load config"); + let mut rng = RngType::from_seed([0u8; 32]); let mut blocks_config = vec![]; let mut key_index = 0; for _ in 1..10 { @@ -472,32 +446,11 @@ async fn overlapping_stores() { // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each - but the blocks overlap (so // total of 5 blocks) - let mut rng = RngType::from_seed([0u8; 32]); - let mut shards_config = vec![]; - for i in 0..3 { - let mut stores_config = vec![]; - for _ in 0..3 { - let store = StoreConfig { - address: free_sockaddr(), - block_range: None, - omap_capacity: 1000, - }; - stores_config.push(store); - } - let shard = ShardConfig { - address: free_sockaddr(), - block_range: BlockRange::new(i + 1, i + 4), - stores: stores_config, - }; - shards_config.push(shard); - } - let config = TestEnvironmentConfig { - router_address: free_sockaddr(), - router_admin_address: free_sockaddr(), - shards: shards_config, - omap_capacity: 1000, - }; + let env_config_str = include_str!("data/overlap.json"); + let config: TestEnvironmentConfig = + serde_json::from_str(env_config_str).expect("Couldn't load config"); + let mut rng = RngType::from_seed([0u8; 32]); let mut blocks_config = vec![]; let mut key_index = 0; for _ in 1..5 { From 3c3f9544d74217ab23fb876a73d4801ed3e61270 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 16 Mar 2023 01:31:04 -0700 Subject: [PATCH 7/8] Revert "Load environment config from JSON file. Might delete later." This reverts commit a677b59111c37bd24543d29b34e2424d6a9d80e3. --- fog/ledger/server/tests/data/normal.json | 79 ------------------- fog/ledger/server/tests/data/overlap.json | 79 ------------------- fog/ledger/server/tests/router_integration.rs | 65 ++++++++++++--- 3 files changed, 56 insertions(+), 167 deletions(-) delete mode 100644 fog/ledger/server/tests/data/normal.json delete mode 100644 fog/ledger/server/tests/data/overlap.json diff --git a/fog/ledger/server/tests/data/normal.json b/fog/ledger/server/tests/data/normal.json deleted file mode 100644 index 3c74644b66..0000000000 --- a/fog/ledger/server/tests/data/normal.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "router_address": "127.0.0.1:21095", - "router_admin_address": "127.0.0.1:16761", - "shards": [ - { - "address": "127.0.0.1:24242", - "block_range": { - "start_block": 1, - "end_block": 4 - }, - "stores": [ - { - "address": "127.0.0.1:20488", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:21157", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:18213", - "block_range": null, - "omap_capacity": 1000 - } - ] - }, - { - "address": "127.0.0.1:19430", - "block_range": { - "start_block": 4, - "end_block": 7 - }, - "stores": [ - { - "address": "127.0.0.1:16542", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:21010", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:22270", - "block_range": null, - "omap_capacity": 1000 - } - ] - }, - { - "address": "127.0.0.1:23515", - "block_range": { - "start_block": 7, - "end_block": 10 - }, - "stores": [ - { - "address": "127.0.0.1:17984", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:22947", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:15143", - "block_range": null, - "omap_capacity": 1000 - } - ] - } - ], - "omap_capacity": 1000 -} \ No newline at end of file diff --git a/fog/ledger/server/tests/data/overlap.json b/fog/ledger/server/tests/data/overlap.json deleted file mode 100644 index 53da31d257..0000000000 --- a/fog/ledger/server/tests/data/overlap.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "router_address": "127.0.0.1:19336", - "router_admin_address": "127.0.0.1:21616", - "shards": [ - { - "address": "127.0.0.1:22977", - "block_range": { - "start_block": 1, - "end_block": 4 - }, - "stores": [ - { - "address": "127.0.0.1:21902", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:19791", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:17895", - "block_range": null, - "omap_capacity": 1000 - } - ] - }, - { - "address": "127.0.0.1:21165", - "block_range": { - "start_block": 2, - "end_block": 5 - }, - "stores": [ - { - "address": "127.0.0.1:20836", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:20965", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:18061", - "block_range": null, - "omap_capacity": 1000 - } - ] - }, - { - "address": "127.0.0.1:17297", - "block_range": { - "start_block": 3, - "end_block": 6 - }, - "stores": [ - { - "address": "127.0.0.1:22423", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:22547", - "block_range": null, - "omap_capacity": 1000 - }, - { - "address": "127.0.0.1:23316", - "block_range": null, - "omap_capacity": 1000 - } - ] - } - ], - "omap_capacity": 1000 -} \ No newline at end of file diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index 3f2c3d6562..802352adc2 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -28,7 +28,7 @@ use rand::thread_rng; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, - net::SocketAddr, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, str::FromStr, sync::Arc, @@ -360,17 +360,43 @@ struct StoreConfig { type BlockConfig = Vec>>; +fn free_sockaddr() -> SocketAddr { + let port = portpicker::pick_unused_port().unwrap(); + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port) +} + #[tokio::test(flavor = "multi_thread")] async fn smoke_test() { let logger = logger::create_test_logger("smoke_test".to_string()); log::info!(logger, "test"); // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each - let env_config_str = include_str!("data/normal.json"); - let config: TestEnvironmentConfig = - serde_json::from_str(env_config_str).expect("Couldn't load config"); - let mut rng = RngType::from_seed([0u8; 32]); + let mut shards_config = vec![]; + for i in 0..3 { + let mut stores_config = vec![]; + for _ in 0..3 { + let store = StoreConfig { + address: free_sockaddr(), + block_range: None, + omap_capacity: 1000, + }; + stores_config.push(store); + } + let shard = ShardConfig { + address: free_sockaddr(), + block_range: BlockRange::new((i * 3) + 1, ((i + 1) * 3) + 1), + stores: stores_config, + }; + shards_config.push(shard); + } + let config = TestEnvironmentConfig { + router_address: free_sockaddr(), + router_admin_address: free_sockaddr(), + shards: shards_config, + omap_capacity: 1000, + }; + let mut blocks_config = vec![]; let mut key_index = 0; for _ in 1..10 { @@ -446,11 +472,32 @@ async fn overlapping_stores() { // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each - but the blocks overlap (so // total of 5 blocks) - let env_config_str = include_str!("data/overlap.json"); - let config: TestEnvironmentConfig = - serde_json::from_str(env_config_str).expect("Couldn't load config"); - let mut rng = RngType::from_seed([0u8; 32]); + let mut shards_config = vec![]; + for i in 0..3 { + let mut stores_config = vec![]; + for _ in 0..3 { + let store = StoreConfig { + address: free_sockaddr(), + block_range: None, + omap_capacity: 1000, + }; + stores_config.push(store); + } + let shard = ShardConfig { + address: free_sockaddr(), + block_range: BlockRange::new(i + 1, i + 4), + stores: stores_config, + }; + shards_config.push(shard); + } + let config = TestEnvironmentConfig { + router_address: free_sockaddr(), + router_admin_address: free_sockaddr(), + shards: shards_config, + omap_capacity: 1000, + }; + let mut blocks_config = vec![]; let mut key_index = 0; for _ in 1..5 { From a9e6378878e8d9d676e53f68ffa97e6049272d33 Mon Sep 17 00:00:00 2001 From: Andrew Wygle Date: Thu, 16 Mar 2023 01:53:06 -0700 Subject: [PATCH 8/8] Remove magic numbers --- fog/ledger/server/tests/router_integration.rs | 65 ++++++++++++++----- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/fog/ledger/server/tests/router_integration.rs b/fog/ledger/server/tests/router_integration.rs index 802352adc2..fdcef3fcec 100644 --- a/fog/ledger/server/tests/router_integration.rs +++ b/fog/ledger/server/tests/router_integration.rs @@ -371,11 +371,14 @@ async fn smoke_test() { log::info!(logger, "test"); // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each + let num_shards = 3; + let stores_per_shard = 3; + let blocks_per_shard = 3; let mut rng = RngType::from_seed([0u8; 32]); let mut shards_config = vec![]; - for i in 0..3 { + for i in 0..num_shards { let mut stores_config = vec![]; - for _ in 0..3 { + for _ in 0..stores_per_shard { let store = StoreConfig { address: free_sockaddr(), block_range: None, @@ -385,7 +388,8 @@ async fn smoke_test() { } let shard = ShardConfig { address: free_sockaddr(), - block_range: BlockRange::new((i * 3) + 1, ((i + 1) * 3) + 1), + // the 1-block offset is because block 0 cannot contain key images + block_range: BlockRange::new_from_length((i * blocks_per_shard) + 1, blocks_per_shard), stores: stores_config, }; shards_config.push(shard); @@ -399,12 +403,15 @@ async fn smoke_test() { let mut blocks_config = vec![]; let mut key_index = 0; - for _ in 1..10 { + let num_blocks = blocks_per_shard * num_shards; + let users_per_block = 3; + let keys_per_user = 3; + for _ in 0..num_blocks { let mut block = HashMap::new(); - for _ in 0..3 { + for _ in 0..users_per_block { let account = AccountKey::random_with_fog(&mut rng); let mut keys = vec![]; - for _ in 0..3 { + for _ in 0..keys_per_user { keys.push(KeyImage::from(key_index)); key_index += 1; } @@ -418,6 +425,7 @@ async fn smoke_test() { let mut test_environment = create_env(config, blocks_config, grpc_env, logger.clone()); // Check that we can get all the key images from each shard + let keys_per_block = users_per_block * keys_per_user; for i in 0..key_index { let key = KeyImage::from(i); let response = test_environment @@ -425,8 +433,12 @@ async fn smoke_test() { .check_key_images(&[key]) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].key_image, key); - assert_eq!(response.results[0].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[0].status(), + Ok(Some((i / keys_per_block) + 1)) + ); assert_eq!( response.results[0].timestamp_result_code, TimestampResultCode::TimestampFound as u32 @@ -440,10 +452,14 @@ async fn smoke_test() { .check_key_images(&keys) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), key_index as usize); for i in 0..key_index { let key = KeyImage::from(i); assert_eq!(response.results[i as usize].key_image, key); - assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[i as usize].status(), + Ok(Some((i / keys_per_block) + 1)) + ); assert_eq!( response.results[i as usize].timestamp_result_code, TimestampResultCode::TimestampFound as u32 @@ -457,6 +473,7 @@ async fn smoke_test() { .check_key_images(&[key]) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].key_image, key); assert_eq!(response.results[0].status(), Ok(None)); // Not spent assert_eq!( @@ -472,11 +489,14 @@ async fn overlapping_stores() { // Three shards, three stores each, correct config, each stores three blocks, // each has three users with three keys each - but the blocks overlap (so // total of 5 blocks) + let num_shards = 3; + let stores_per_shard = 3; + let blocks_per_shard = 3; let mut rng = RngType::from_seed([0u8; 32]); let mut shards_config = vec![]; - for i in 0..3 { + for i in 0..num_shards { let mut stores_config = vec![]; - for _ in 0..3 { + for _ in 0..stores_per_shard { let store = StoreConfig { address: free_sockaddr(), block_range: None, @@ -486,7 +506,7 @@ async fn overlapping_stores() { } let shard = ShardConfig { address: free_sockaddr(), - block_range: BlockRange::new(i + 1, i + 4), + block_range: BlockRange::new_from_length(i + 1, blocks_per_shard), stores: stores_config, }; shards_config.push(shard); @@ -500,12 +520,15 @@ async fn overlapping_stores() { let mut blocks_config = vec![]; let mut key_index = 0; - for _ in 1..5 { + let num_blocks = 5; + let users_per_block = 3; + let keys_per_user = 3; + for _ in 0..num_blocks { let mut block = HashMap::new(); - for _ in 0..3 { + for _ in 0..users_per_block { let account = AccountKey::random_with_fog(&mut rng); let mut keys = vec![]; - for _ in 0..3 { + for _ in 0..keys_per_user { keys.push(KeyImage::from(key_index)); key_index += 1; } @@ -519,6 +542,7 @@ async fn overlapping_stores() { let mut test_environment = create_env(config, blocks_config, grpc_env, logger.clone()); // Check that we can get all the key images from each shard + let keys_per_block = users_per_block * keys_per_user; for i in 0..key_index { let key = KeyImage::from(i); let response = test_environment @@ -526,8 +550,12 @@ async fn overlapping_stores() { .check_key_images(&[key]) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].key_image, key); - assert_eq!(response.results[0].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[0].status(), + Ok(Some((i / keys_per_block) + 1)) + ); assert_eq!( response.results[0].timestamp_result_code, TimestampResultCode::TimestampFound as u32 @@ -541,10 +569,14 @@ async fn overlapping_stores() { .check_key_images(&keys) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), key_index as usize); for i in 0..key_index { let key = KeyImage::from(i); assert_eq!(response.results[i as usize].key_image, key); - assert_eq!(response.results[i as usize].status(), Ok(Some((i / 9) + 1))); + assert_eq!( + response.results[i as usize].status(), + Ok(Some((i / keys_per_block) + 1)) + ); assert_eq!( response.results[i as usize].timestamp_result_code, TimestampResultCode::TimestampFound as u32 @@ -558,6 +590,7 @@ async fn overlapping_stores() { .check_key_images(&[key]) .await .expect("check_key_images failed"); + assert_eq!(response.results.len(), 1); assert_eq!(response.results[0].key_image, key); assert_eq!(response.results[0].status(), Ok(None)); // Not spent assert_eq!(